From 5bee25fa209b215dbf5dace5f7eeb2d68c4c9761 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Tue, 5 Mar 2024 05:27:41 -0800 Subject: [PATCH] feat(ingest): add Looks subtype + stop reemitting browsePathV2 (#9978) --- .../src/datahub/ingestion/api/source.py | 3 +- .../datahub/ingestion/api/source_helpers.py | 15 --- .../ingestion/source/common/subtypes.py | 3 + .../ingestion/source/looker/looker_source.py | 8 +- .../integration/looker/expected_output.json | 93 ------------------- .../golden_test_independent_look_ingest.json | 84 ++++++++++++----- 6 files changed, 73 insertions(+), 133 deletions(-) delete mode 100644 metadata-ingestion/tests/integration/looker/expected_output.json diff --git a/metadata-ingestion/src/datahub/ingestion/api/source.py b/metadata-ingestion/src/datahub/ingestion/api/source.py index 0679c884ba0ede..906a431666e17f 100644 --- a/metadata-ingestion/src/datahub/ingestion/api/source.py +++ b/metadata-ingestion/src/datahub/ingestion/api/source.py @@ -34,7 +34,6 @@ auto_materialize_referenced_tags, auto_status_aspect, auto_workunit_reporter, - re_emit_browse_path_v2, ) from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent @@ -297,7 +296,7 @@ def _get_browse_path_processor(self, dry_run: bool) -> MetadataWorkUnitProcessor drop_dirs=[s for s in browse_path_drop_dirs if s is not None], dry_run=dry_run, ) - return lambda stream: re_emit_browse_path_v2(browse_path_processor(stream)) + return lambda stream: browse_path_processor(stream) class TestableSource(Source): diff --git a/metadata-ingestion/src/datahub/ingestion/api/source_helpers.py b/metadata-ingestion/src/datahub/ingestion/api/source_helpers.py index 84051a5887966f..80eb283424d69f 100644 --- a/metadata-ingestion/src/datahub/ingestion/api/source_helpers.py +++ b/metadata-ingestion/src/datahub/ingestion/api/source_helpers.py @@ -197,21 +197,6 @@ def auto_lowercase_urns( yield wu -def re_emit_browse_path_v2( - stream: Iterable[MetadataWorkUnit], -) -> Iterable[MetadataWorkUnit]: - """Re-emit browse paths v2 aspects, to avoid race condition where server overwrites with default.""" - browse_path_v2_workunits = [] - - for wu in stream: - yield wu - if wu.is_primary_source and wu.get_aspect_of_type(BrowsePathsV2Class): - browse_path_v2_workunits.append(wu) - - for wu in browse_path_v2_workunits: - yield wu - - def auto_browse_path_v2( stream: Iterable[MetadataWorkUnit], *, diff --git a/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py b/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py index 1def0dd02097b4..466cc64519f24c 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py +++ b/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py @@ -48,3 +48,6 @@ class BIContainerSubTypes(str, Enum): class BIAssetSubTypes(str, Enum): # Generic SubTypes REPORT = "Report" + + # Looker + LOOKER_LOOK = "Look" diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py index a66675b8be0b08..e42e7b82bf8aba 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py @@ -47,6 +47,7 @@ ) from datahub.ingestion.api.source_helpers import auto_workunit from datahub.ingestion.api.workunit import MetadataWorkUnit +from datahub.ingestion.source.common.subtypes import BIAssetSubTypes from datahub.ingestion.source.looker import looker_usage from datahub.ingestion.source.looker.looker_common import ( InputFieldElement, @@ -89,6 +90,7 @@ OwnerClass, OwnershipClass, OwnershipTypeClass, + SubTypesClass, ) from datahub.utilities.advanced_thread_executor import BackpressureAwareExecutor @@ -624,7 +626,11 @@ def _make_chart_metadata_events( chart_mce = MetadataChangeEvent(proposedSnapshot=chart_snapshot) proposals: List[Union[MetadataChangeEvent, MetadataChangeProposalWrapper]] = [ - chart_mce + chart_mce, + MetadataChangeProposalWrapper( + entityUrn=chart_urn, + aspect=SubTypesClass(typeNames=[BIAssetSubTypes.LOOKER_LOOK]), + ), ] # If extracting embeds is enabled, produce an MCP for embed URL. diff --git a/metadata-ingestion/tests/integration/looker/expected_output.json b/metadata-ingestion/tests/integration/looker/expected_output.json deleted file mode 100644 index 96393ce4b531a3..00000000000000 --- a/metadata-ingestion/tests/integration/looker/expected_output.json +++ /dev/null @@ -1,93 +0,0 @@ -[ -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(looker,dashboard_elements.2)", - "aspects": [ - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": {}, - "externalUrl": null, - "title": "", - "description": "Some text", - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null - }, - "chartUrl": "https://looker.company.com/x/", - "inputs": [], - "type": null, - "access": null, - "lastRefreshed": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "looker-test", - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { - "urn": "urn:li:dashboard:(looker,dashboards.1)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { - "customProperties": {}, - "externalUrl": null, - "title": "foo", - "description": "lorem ipsum", - "charts": [ - "urn:li:chart:(looker,dashboard_elements.2)" - ], - "datasets": [], - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null - }, - "dashboardUrl": "https://looker.company.com/dashboards/1", - "access": null, - "lastRefreshed": null - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "looker-test", - "properties": null - } -} -] diff --git a/metadata-ingestion/tests/integration/looker/golden_test_independent_look_ingest.json b/metadata-ingestion/tests/integration/looker/golden_test_independent_look_ingest.json index 9521c9af4bbdcc..e1102466059743 100644 --- a/metadata-ingestion/tests/integration/looker/golden_test_independent_look_ingest.json +++ b/metadata-ingestion/tests/integration/looker/golden_test_independent_look_ingest.json @@ -34,7 +34,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -49,7 +50,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -109,7 +111,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -169,7 +172,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -212,7 +216,26 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(looker,dashboard_elements.looks_1)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Look" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -227,7 +250,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -242,7 +266,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -338,7 +363,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -355,7 +381,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -370,7 +397,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -392,7 +420,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -488,7 +517,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -505,7 +535,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -520,7 +551,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -542,7 +574,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -561,7 +594,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -580,7 +614,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -599,7 +634,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -614,7 +650,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -629,7 +666,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -644,7 +682,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } }, { @@ -659,7 +698,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "looker-test" + "runId": "looker-test", + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file