From c5f5e23c9fa8fccfcf6cae18078951c721cf2428 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Tue, 3 Jan 2023 11:01:54 +0000 Subject: [PATCH 1/2] remove redundant variable --- spectacles/validators/data_test.py | 72 ++++++++++++++++-------------- 1 file changed, 38 insertions(+), 34 deletions(-) diff --git a/spectacles/validators/data_test.py b/spectacles/validators/data_test.py index c5d8c95b..23f6233c 100644 --- a/spectacles/validators/data_test.py +++ b/spectacles/validators/data_test.py @@ -93,40 +93,44 @@ async def get_tests(self, project: Project) -> List[DataTest]: async def validate(self, tests: List[DataTest]) -> List[DataTestError]: data_test_errors: List[DataTestError] = [] - - async def run_test(test: DataTest) -> None: - results = await self.client.run_lookml_test( - test.project_name, model=test.explore.model_name, test=test.name - ) - test.explore.queried = True - result = results[0] # For a single test, list with length 1 - - if result["success"]: - test.passed = True - test.explore.successes.append( - { - "model": test.explore.model_name, - "explore": test.explore.name, - "metadata": { - "test_name": result["test_name"], - "lookml_url": test.lookml_url, - "explore_url": test.explore_url, - }, - } + query_slot = asyncio.Semaphore( + 15 + ) # This is the per-user query limit in Looker for most instances + + async def run_test(test: DataTest, query_slot: asyncio.Semaphore) -> None: + async with query_slot: + results = await self.client.run_lookml_test( + test.project_name, model=test.explore.model_name, test=test.name ) - else: - test.passed = False - for error in result["errors"]: - error = DataTestError( - model=error["model_id"], - explore=error["explore"], - message=error["message"], - test_name=result["test_name"], - lookml_url=test.lookml_url, - explore_url=test.explore_url, + test.explore.queried = True + result = results[0] # For a single test, list with length 1 + + if result["success"]: + test.passed = True + test.explore.successes.append( + { + "model": test.explore.model_name, + "explore": test.explore.name, + "metadata": { + "test_name": result["test_name"], + "lookml_url": test.lookml_url, + "explore_url": test.explore_url, + }, + } ) - data_test_errors.append(error) - test.explore.errors.append(error) - - await asyncio.gather(*(run_test(test) for test in tests)) + else: + test.passed = False + for error in result["errors"]: + error = DataTestError( + model=error["model_id"], + explore=error["explore"], + message=error["message"], + test_name=result["test_name"], + lookml_url=test.lookml_url, + explore_url=test.explore_url, + ) + data_test_errors.append(error) + test.explore.errors.append(error) + + await asyncio.gather(*(run_test(test, query_slot) for test in tests)) return data_test_errors From 5e338a1bd41c01715f3693de74c4637e00af8b65 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Tue, 3 Jan 2023 16:45:30 +0000 Subject: [PATCH 2/2] move query slot limit to CONSTANT --- spectacles/validators/data_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/spectacles/validators/data_test.py b/spectacles/validators/data_test.py index 23f6233c..afe646d3 100644 --- a/spectacles/validators/data_test.py +++ b/spectacles/validators/data_test.py @@ -5,6 +5,8 @@ from spectacles.lookml import Explore, Project from spectacles.exceptions import SpectaclesException, DataTestError +QUERY_SLOT_LIMIT = 15 # This is the per-user query limit in Looker for most instances + @dataclass class DataTest: @@ -93,9 +95,7 @@ async def get_tests(self, project: Project) -> List[DataTest]: async def validate(self, tests: List[DataTest]) -> List[DataTestError]: data_test_errors: List[DataTestError] = [] - query_slot = asyncio.Semaphore( - 15 - ) # This is the per-user query limit in Looker for most instances + query_slot = asyncio.Semaphore(QUERY_SLOT_LIMIT) async def run_test(test: DataTest, query_slot: asyncio.Semaphore) -> None: async with query_slot: