From f77867132ffac70fa427a7f3409516c32b38e182 Mon Sep 17 00:00:00 2001 From: J Feldman Date: Thu, 15 Feb 2024 11:33:48 -0500 Subject: [PATCH 1/7] Fix (ingest): Update looker source for Looker query API breaking change --- .../src/datahub/ingestion/source/looker/looker_source.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py index ab9887c9005710..46c1177a995981 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py @@ -1199,7 +1199,9 @@ def extract_independent_looks(self) -> Iterable[MetadataWorkUnit]: logger.info(f"query_id is None for look {look.title}({look.id})") continue - query: Query = self.looker_api.get_query(look.query_id, query_fields) + query: Query = self.looker_api.look(look.id, fields="query").query + filtered_query_dict = {key: getattr(query, key) for key in query_fields if hasattr(query, key)} + query = Query(**filtered_query_dict) dashboard_element: Optional[ LookerDashboardElement From 2883ed0f91503d64dbc99f9b6ca1572503e1a29c Mon Sep 17 00:00:00 2001 From: J Feldman Date: Thu, 15 Feb 2024 12:25:37 -0500 Subject: [PATCH 2/7] Add comment --- .../datahub/ingestion/source/looker/looker_source.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py index 46c1177a995981..af96b2f9e225cb 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py @@ -1200,8 +1200,14 @@ def extract_independent_looks(self) -> Iterable[MetadataWorkUnit]: continue query: Query = self.looker_api.look(look.id, fields="query").query - filtered_query_dict = {key: getattr(query, key) for key in query_fields if hasattr(query, key)} - query = Query(**filtered_query_dict) + # Only include fields that are in the query_fields list + query = Query( + **{ + key: getattr(query, key) + for key in query_fields + if hasattr(query, key) + } + ) dashboard_element: Optional[ LookerDashboardElement From 402227fb036344cca578c32c9884dc1b5c4ccd15 Mon Sep 17 00:00:00 2001 From: J Feldman Date: Thu, 15 Feb 2024 12:39:58 -0500 Subject: [PATCH 3/7] Add get_look method to LookerAPI class --- .../ingestion/source/looker/looker_lib_wrapper.py | 9 +++++++++ .../src/datahub/ingestion/source/looker/looker_source.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_lib_wrapper.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_lib_wrapper.py index fe50cc1c649aca..706262942bc5ae 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_lib_wrapper.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_lib_wrapper.py @@ -64,6 +64,7 @@ class LookerAPIStats(BaseModel): all_looks_calls: int = 0 all_models_calls: int = 0 get_query_calls: int = 0 + get_look_calls: int = 0 search_looks_calls: int = 0 search_dashboards_calls: int = 0 @@ -255,6 +256,14 @@ def get_query(self, query_id: str, fields: Union[str, List[str]]) -> Query: transport_options=self.transport_options, ) + def get_look(self, look_id: str, fields: Union[str, List[str]]) -> Look: + self.client_stats.get_look_calls += 1 + return self.client.look( + look_id=look_id, + fields=self.__fields_mapper(fields), + transport_options=self.transport_options, + ) + def search_dashboards( self, fields: Union[str, List[str]], deleted: str ) -> Sequence[Dashboard]: diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py index af96b2f9e225cb..a692ce9fa12fb5 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py @@ -1199,7 +1199,7 @@ def extract_independent_looks(self) -> Iterable[MetadataWorkUnit]: logger.info(f"query_id is None for look {look.title}({look.id})") continue - query: Query = self.looker_api.look(look.id, fields="query").query + query: Query = self.looker_api.get_look(look.id, fields="query").query # Only include fields that are in the query_fields list query = Query( **{ From bae9f62c3d8c2717ae0ec3ff940945c975990279 Mon Sep 17 00:00:00 2001 From: J Feldman Date: Thu, 15 Feb 2024 14:20:03 -0500 Subject: [PATCH 4/7] Fix lint --- .../src/datahub/ingestion/source/looker/looker_lib_wrapper.py | 3 ++- .../src/datahub/ingestion/source/looker/looker_source.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_lib_wrapper.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_lib_wrapper.py index 706262942bc5ae..dc2436c2b52989 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_lib_wrapper.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_lib_wrapper.py @@ -15,6 +15,7 @@ DBConnection, Folder, Look, + LookWithQuery, LookmlModel, LookmlModelExplore, Query, @@ -256,7 +257,7 @@ def get_query(self, query_id: str, fields: Union[str, List[str]]) -> Query: transport_options=self.transport_options, ) - def get_look(self, look_id: str, fields: Union[str, List[str]]) -> Look: + def get_look(self, look_id: str, fields: Union[str, List[str]]) -> LookWithQuery: self.client_stats.get_look_calls += 1 return self.client.look( look_id=look_id, diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py index a692ce9fa12fb5..f136760c973a98 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py @@ -1199,7 +1199,7 @@ def extract_independent_looks(self) -> Iterable[MetadataWorkUnit]: logger.info(f"query_id is None for look {look.title}({look.id})") continue - query: Query = self.looker_api.get_look(look.id, fields="query").query + query: Query = self.looker_api.get_look(look.id, fields=["query"]).query # Only include fields that are in the query_fields list query = Query( **{ From 735bb13aface0b88c45e7146cf61e4a9640c4d25 Mon Sep 17 00:00:00 2001 From: J Feldman Date: Thu, 15 Feb 2024 14:48:51 -0500 Subject: [PATCH 5/7] Lint --- .../src/datahub/ingestion/source/looker/looker_lib_wrapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_lib_wrapper.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_lib_wrapper.py index dc2436c2b52989..364d6b136d5040 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_lib_wrapper.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_lib_wrapper.py @@ -15,9 +15,9 @@ DBConnection, Folder, Look, - LookWithQuery, LookmlModel, LookmlModelExplore, + LookWithQuery, Query, User, WriteQuery, From 724448b96f53c08106e395b898bcc8d2fccb9834 Mon Sep 17 00:00:00 2001 From: J Feldman Date: Thu, 15 Feb 2024 15:26:29 -0500 Subject: [PATCH 6/7] Fix mypy --- .../ingestion/source/looker/looker_source.py | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py index f136760c973a98..a66675b8be0b08 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py @@ -1199,15 +1199,18 @@ def extract_independent_looks(self) -> Iterable[MetadataWorkUnit]: logger.info(f"query_id is None for look {look.title}({look.id})") continue - query: Query = self.looker_api.get_look(look.id, fields=["query"]).query - # Only include fields that are in the query_fields list - query = Query( - **{ - key: getattr(query, key) - for key in query_fields - if hasattr(query, key) - } - ) + if look.id is not None: + query: Optional[Query] = self.looker_api.get_look( + look.id, fields=["query"] + ).query + # Only include fields that are in the query_fields list + query = Query( + **{ + key: getattr(query, key) + for key in query_fields + if hasattr(query, key) + } + ) dashboard_element: Optional[ LookerDashboardElement From a1b829c1dc2fa461b840e2ea8c4835e52dc91767 Mon Sep 17 00:00:00 2001 From: J Feldman Date: Thu, 15 Feb 2024 17:35:09 -0500 Subject: [PATCH 7/7] Fix integration test --- .../tests/integration/looker/test_looker.py | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/metadata-ingestion/tests/integration/looker/test_looker.py b/metadata-ingestion/tests/integration/looker/test_looker.py index ee6610cf75679c..3e614dbdbc8605 100644 --- a/metadata-ingestion/tests/integration/looker/test_looker.py +++ b/metadata-ingestion/tests/integration/looker/test_looker.py @@ -311,15 +311,17 @@ def setup_mock_look(mocked_client): ) ] - mocked_client.query.return_value = Query( - id="1", - view="sales_explore", - model="sales_model", - fields=[ - "sales.profit", - ], - dynamic_fields=None, - filters=None, + mocked_client.look.return_value = LookWithQuery( + query=Query( + id="1", + view="sales_explore", + model="sales_model", + fields=[ + "sales.profit", + ], + dynamic_fields=None, + filters=None, + ) )