feat(ingest/looker): update browse paths to align with looker UI

docs/how/updating-datahub.md

@@ -25,7 +25,8 @@ This file documents any backwards-incompatible changes in DataHub and assists pe
 - #10026 - The dbt `use_compiled_code` option has been removed, because we now support capturing both source and compiled dbt SQL. This can be configured using `include_compiled_code`, which will be default enabled in 0.13.1.
 - #10055 - Assertion entities generated by dbt are now associated with the dbt dataset entity, and not the entity in the data warehouse.
 - #10090 - For Redshift ingestion, `use_lineage_v2` is now enabled by default.
-
+- #10147 - For looker ingestion, the browse paths for looker Dashboard, Chart, View, Explore have been updated to align with Looker UI. This does not affect URNs or lineage but primarily affects (improves) browsing experience.
+- 
 ### Potential Downtime
 
 ### Deprecations

metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py

@@ -1199,6 +1199,7 @@ class LookerDashboardElement:
     type: Optional[str] = None
     description: Optional[str] = None
     input_fields: Optional[List[InputFieldElement]] = None
+    folder_path: Optional[str] = None  # for independent looks.
 
     def url(self, base_url: str) -> str:
         # A dashboard element can use a look or just a raw query against an explore

metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py

@@ -452,6 +452,12 @@ def _get_looker_dashboard_element(  # noqa: C901
                 else:
                     slug = ""
 
+                look_folder_path = None
+                if element.look.folder is not None:
+                    look_folder_path = self._get_folder_path(
+                        element.look.folder, self.looker_api
+                    )
+
                 return LookerDashboardElement(
                     id=element.id,
                     title=title,
@@ -464,6 +470,7 @@ def _get_looker_dashboard_element(  # noqa: C901
                         for exp in explores
                     ],
                     input_fields=input_fields,
+                    folder_path=look_folder_path,
                 )
 
         # Failing the above two approaches, pick out details from result_maker
@@ -524,10 +531,12 @@ def _get_looker_dashboard_element(  # noqa: C901
                 type=element.type,
                 description=element.subtitle_text,
                 look_id=element.look_id,
-                query_slug=element.result_maker.query.slug
-                if element.result_maker.query is not None
-                and element.result_maker.query.slug is not None
-                else "",
+                query_slug=(
+                    element.result_maker.query.slug
+                    if element.result_maker.query is not None
+                    and element.result_maker.query.slug is not None
+                    else ""
+                ),
                 upstream_explores=[
                     LookerExplore(model_name=model, name=exp) for exp in explores
                 ],
@@ -603,11 +612,15 @@ def _make_chart_metadata_events(
             chartUrl=dashboard_element.url(self.source_config.external_base_url or ""),
             inputs=dashboard_element.get_view_urns(self.source_config),
             customProperties={
-                "upstream_fields": ",".join(
-                    sorted(set(field.name for field in dashboard_element.input_fields))
+                "upstream_fields": (
+                    ",".join(
+                        sorted(
+                            set(field.name for field in dashboard_element.input_fields)
+                        )
+                    )
+                    if dashboard_element.input_fields
+                    else ""
                 )
-                if dashboard_element.input_fields
-                else ""
             },
         )
         chart_snapshot.aspects.append(chart_info)
@@ -617,6 +630,13 @@ def _make_chart_metadata_events(
                 paths=[f"/Folders/{dashboard.folder_path}/{dashboard.title}"]
             )
             chart_snapshot.aspects.append(browse_path)
+        elif (
+            dashboard is None and dashboard_element.folder_path is not None
+        ):  # independent look
+            browse_path = BrowsePathsClass(
+                paths=[f"/Folders/{dashboard_element.folder_path}"]
+            )
+            chart_snapshot.aspects.append(browse_path)
 
         if dashboard is not None:
             ownership = self.get_ownership(dashboard)
@@ -1084,10 +1104,12 @@ def process_dashboard(
         looker_dashboard = self._get_looker_dashboard(dashboard_object, self.looker_api)
         mces = self._make_dashboard_and_chart_mces(looker_dashboard)
         workunits = [
-            MetadataWorkUnit(id=f"looker-{mce.proposedSnapshot.urn}", mce=mce)
-            if isinstance(mce, MetadataChangeEvent)
-            else MetadataWorkUnit(
-                id=f"looker-{mce.aspectName}-{mce.entityUrn}", mcp=mce
+            (
+                MetadataWorkUnit(id=f"looker-{mce.proposedSnapshot.urn}", mce=mce)
+                if isinstance(mce, MetadataChangeEvent)
+                else MetadataWorkUnit(
+                    id=f"looker-{mce.aspectName}-{mce.entityUrn}", mcp=mce
+                )
             )
             for mce in mces
         ]
@@ -1177,12 +1199,7 @@ def extract_independent_looks(self) -> Iterable[MetadataWorkUnit]:
         self.reporter.report_stage_start("extract_independent_looks")
 
         logger.debug("Extracting looks not part of Dashboard")
-        look_fields: List[str] = [
-            "id",
-            "title",
-            "description",
-            "query_id",
-        ]
+        look_fields: List[str] = ["id", "title", "description", "query_id", "folder"]
         query_fields: List[str] = [
             "id",
             "view",
@@ -1227,8 +1244,8 @@ def extract_independent_looks(self) -> Iterable[MetadataWorkUnit]:
                     subtitle_text=look.description,
                     look_id=look.id,
                     dashboard_id=None,  # As this is independent look
-                    look=LookWithQuery(query=query),
-                )
+                    look=LookWithQuery(query=query, folder=look.folder),
+                ),
             )
 
             if dashboard_element is not None:

metadata-ingestion/tests/integration/looker/golden_test_independent_look_ingest.json

@@ -337,6 +337,13 @@
                             }
                         ]
                     }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/Folders/Shared"
+                        ]
+                    }
                 }
             ]
         }
@@ -397,6 +404,29 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "chart",
+    "entityUrn": "urn:li:chart:(looker,dashboard_elements.looks_1)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Folders"
+                },
+                {
+                    "id": "Shared"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "looker-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "proposedSnapshot": {
         "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {

metadata-ingestion/tests/integration/looker/test_looker.py

@@ -310,6 +310,7 @@ def setup_mock_look(mocked_client):
             title="Outer Look",
             description="I am not part of any Dashboard",
             query_id="1",
+            folder=FolderBase(name="Shared", id="shared-folder-id"),
         )
     ]
 

metadata-ingestion/tests/integration/lookml/test_lookml.py

@@ -578,7 +578,7 @@ def test_lookml_git_info(pytestconfig, tmp_path, mock_time):
                     "parse_table_names_from_sql": True,
                     "project_name": "lkml_samples",
                     "model_pattern": {"deny": ["data2"]},
-                    "github_info": {"repo": "datahub/looker-demo", "branch": "master"},
+                    "git_info": {"repo": "datahub/looker-demo", "branch": "master"},
                     "emit_reachable_views_only": False,
                     "process_refinements": False,
                 },
@@ -693,7 +693,7 @@ def test_hive_platform_drops_ids(pytestconfig, tmp_path, mock_time):
                     "parse_table_names_from_sql": True,
                     "project_name": "lkml_samples",
                     "model_pattern": {"deny": ["data2"]},
-                    "github_info": {"repo": "datahub/looker-demo", "branch": "master"},
+                    "git_info": {"repo": "datahub/looker-demo", "branch": "master"},
                     "emit_reachable_views_only": False,
                     "process_refinements": False,
                 },
@@ -791,7 +791,7 @@ def test_lookml_base_folder():
 
     LookMLSourceConfig.parse_obj(
         {
-            "github_info": {
+            "git_info": {
                 "repo": "acryldata/long-tail-companions-looker",
                 "deploy_key": "this-is-fake",
             },