feat(ingest): add Looks subtype + stop reemitting browsePathV2 (#9978)

datahub-project · Mar 5, 2024 · 5bee25f · 5bee25f
1 parent a85db60
commit 5bee25f
Show file tree

Hide file tree

Showing 6 changed files with 73 additions and 133 deletions.
diff --git a/metadata-ingestion/src/datahub/ingestion/api/source.py b/metadata-ingestion/src/datahub/ingestion/api/source.py
@@ -34,7 +34,6 @@
     auto_materialize_referenced_tags,
     auto_status_aspect,
     auto_workunit_reporter,
-    re_emit_browse_path_v2,
 )
 from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent
@@ -297,7 +296,7 @@ def _get_browse_path_processor(self, dry_run: bool) -> MetadataWorkUnitProcessor
             drop_dirs=[s for s in browse_path_drop_dirs if s is not None],
             dry_run=dry_run,
         )
-        return lambda stream: re_emit_browse_path_v2(browse_path_processor(stream))
+        return lambda stream: browse_path_processor(stream)
 
 
 class TestableSource(Source):

diff --git a/metadata-ingestion/src/datahub/ingestion/api/source_helpers.py b/metadata-ingestion/src/datahub/ingestion/api/source_helpers.py
@@ -197,21 +197,6 @@ def auto_lowercase_urns(
             yield wu
 
 
-def re_emit_browse_path_v2(
-    stream: Iterable[MetadataWorkUnit],
-) -> Iterable[MetadataWorkUnit]:
-    """Re-emit browse paths v2 aspects, to avoid race condition where server overwrites with default."""
-    browse_path_v2_workunits = []
-
-    for wu in stream:
-        yield wu
-        if wu.is_primary_source and wu.get_aspect_of_type(BrowsePathsV2Class):
-            browse_path_v2_workunits.append(wu)
-
-    for wu in browse_path_v2_workunits:
-        yield wu
-
-
 def auto_browse_path_v2(
     stream: Iterable[MetadataWorkUnit],
     *,

diff --git a/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py b/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py
@@ -48,3 +48,6 @@ class BIContainerSubTypes(str, Enum):
 class BIAssetSubTypes(str, Enum):
     # Generic SubTypes
     REPORT = "Report"
+
+    # Looker
+    LOOKER_LOOK = "Look"
diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py
@@ -47,6 +47,7 @@
 )
 from datahub.ingestion.api.source_helpers import auto_workunit
 from datahub.ingestion.api.workunit import MetadataWorkUnit
+from datahub.ingestion.source.common.subtypes import BIAssetSubTypes
 from datahub.ingestion.source.looker import looker_usage
 from datahub.ingestion.source.looker.looker_common import (
     InputFieldElement,
@@ -89,6 +90,7 @@
     OwnerClass,
     OwnershipClass,
     OwnershipTypeClass,
+    SubTypesClass,
 )
 from datahub.utilities.advanced_thread_executor import BackpressureAwareExecutor
 
@@ -624,7 +626,11 @@ def _make_chart_metadata_events(
         chart_mce = MetadataChangeEvent(proposedSnapshot=chart_snapshot)
 
         proposals: List[Union[MetadataChangeEvent, MetadataChangeProposalWrapper]] = [
-            chart_mce
+            chart_mce,
+            MetadataChangeProposalWrapper(
+                entityUrn=chart_urn,
+                aspect=SubTypesClass(typeNames=[BIAssetSubTypes.LOOKER_LOOK]),
+            ),
         ]
 
         # If extracting embeds is enabled, produce an MCP for embed URL.

diff --git a/metadata-ingestion/tests/integration/looker/expected_output.json b/metadata-ingestion/tests/integration/looker/expected_output.json