From b8af2b9d69dfe0ca38b4fa37d53e175234e31c9c Mon Sep 17 00:00:00 2001 From: sagar-salvi-apptware <159135491+sagar-salvi-apptware@users.noreply.github.com> Date: Wed, 3 Jul 2024 18:05:37 +0530 Subject: [PATCH] fix(ingestion/glue): ensure date formatting works on all platforms for aws glue (#10836) --- .../src/datahub/ingestion/source/aws/glue.py | 2 +- .../unit/glue/glue_delta_mces_golden.json | 2 +- .../glue_malformed_delta_mces_golden.json | 2 +- .../tests/unit/glue/glue_mces_golden.json | 110 ++++++++--------- .../glue_mces_platform_instance_golden.json | 114 +++++++++--------- 5 files changed, 115 insertions(+), 115 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py b/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py index 05aa90dd76f6b5..68e4370107b230 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py +++ b/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py @@ -901,7 +901,7 @@ def gen_database_containers( parameters["LocationUri"] = database["LocationUri"] if database.get("CreateTime") is not None: create_time: datetime.datetime = database["CreateTime"] - parameters["CreateTime"] = create_time.strftime("%B %-d, %Y at %H:%M:%S") + parameters["CreateTime"] = create_time.strftime("%B %d, %Y at %H:%M:%S") yield from gen_containers( container_key=database_container_key, name=database["Name"], diff --git a/metadata-ingestion/tests/unit/glue/glue_delta_mces_golden.json b/metadata-ingestion/tests/unit/glue/glue_delta_mces_golden.json index 992ec338dea300..01ecdbb4fd331e 100644 --- a/metadata-ingestion/tests/unit/glue/glue_delta_mces_golden.json +++ b/metadata-ingestion/tests/unit/glue/glue_delta_mces_golden.json @@ -11,7 +11,7 @@ "instance": "delta_platform_instance", "env": "PROD", "database": "delta-database", - "CreateTime": "June 9, 2021 at 14:14:19" + "CreateTime": "June 09, 2021 at 14:14:19" }, "name": "delta-database", "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/delta-database" diff --git a/metadata-ingestion/tests/unit/glue/glue_malformed_delta_mces_golden.json b/metadata-ingestion/tests/unit/glue/glue_malformed_delta_mces_golden.json index b8e3445eea5094..e71ede4f370ffb 100644 --- a/metadata-ingestion/tests/unit/glue/glue_malformed_delta_mces_golden.json +++ b/metadata-ingestion/tests/unit/glue/glue_malformed_delta_mces_golden.json @@ -11,7 +11,7 @@ "instance": "delta_platform_instance", "env": "PROD", "database": "delta-database", - "CreateTime": "June 9, 2021 at 14:14:19" + "CreateTime": "June 09, 2021 at 14:14:19" }, "name": "delta-database", "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/delta-database" diff --git a/metadata-ingestion/tests/unit/glue/glue_mces_golden.json b/metadata-ingestion/tests/unit/glue/glue_mces_golden.json index b72124f23d7498..c688eb9a2d1150 100644 --- a/metadata-ingestion/tests/unit/glue/glue_mces_golden.json +++ b/metadata-ingestion/tests/unit/glue/glue_mces_golden.json @@ -13,7 +13,7 @@ "param1": "value1", "param2": "value2", "LocationUri": "s3://test-bucket/test-prefix", - "CreateTime": "June 9, 2021 at 14:14:19" + "CreateTime": "June 09, 2021 at 14:14:19" }, "name": "flights-database", "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/flights-database" @@ -55,6 +55,59 @@ } } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "glue", + "env": "PROD", + "database": "test-database", + "CreateTime": "June 01, 2021 at 14:55:02" + }, + "name": "test-database", + "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/test-database" + } + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:glue" + } + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Database" + ] + } + } +}, { "entityType": "container", "entityUrn": "urn:li:container:110bc08849d1c1bde5fc345dab5c3ae7", @@ -66,7 +119,7 @@ "platform": "glue", "env": "PROD", "database": "empty-database", - "CreateTime": "June 1, 2021 at 14:55:13" + "CreateTime": "June 01, 2021 at 14:55:13" }, "name": "empty-database", "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/empty-database" @@ -336,59 +389,6 @@ } } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7", - "changeType": "UPSERT", - "aspectName": "containerProperties", - "aspect": { - "json": { - "customProperties": { - "platform": "glue", - "env": "PROD", - "database": "test-database", - "CreateTime": "June 1, 2021 at 14:55:02" - }, - "name": "test-database", - "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/test-database" - } - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "json": { - "platform": "urn:li:dataPlatform:glue" - } - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "Database" - ] - } - } -}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { diff --git a/metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json b/metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json index c66ba75548f14a..de82d26c571622 100644 --- a/metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json +++ b/metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json @@ -14,7 +14,7 @@ "param1": "value1", "param2": "value2", "LocationUri": "s3://test-bucket/test-prefix", - "CreateTime": "June 9, 2021 at 14:14:19" + "CreateTime": "June 09, 2021 at 14:14:19" }, "name": "flights-database", "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/flights-database" @@ -57,6 +57,61 @@ } } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "glue", + "instance": "some_instance_name", + "env": "PROD", + "database": "test-database", + "CreateTime": "June 01, 2021 at 14:55:02" + }, + "name": "test-database", + "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/test-database" + } + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:glue", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:glue,some_instance_name)" + } + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Database" + ] + } + } +}, { "entityType": "container", "entityUrn": "urn:li:container:ac4381240e82d55400c22e4392e744a4", @@ -69,7 +124,7 @@ "instance": "some_instance_name", "env": "PROD", "database": "empty-database", - "CreateTime": "June 1, 2021 at 14:55:13" + "CreateTime": "June 01, 2021 at 14:55:13" }, "name": "empty-database", "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/empty-database" @@ -341,61 +396,6 @@ } } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca", - "changeType": "UPSERT", - "aspectName": "containerProperties", - "aspect": { - "json": { - "customProperties": { - "platform": "glue", - "instance": "some_instance_name", - "env": "PROD", - "database": "test-database", - "CreateTime": "June 1, 2021 at 14:55:02" - }, - "name": "test-database", - "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/test-database" - } - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "json": { - "platform": "urn:li:dataPlatform:glue", - "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:glue,some_instance_name)" - } - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "Database" - ] - } - } -}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {