From 509ff69671c9759ec05e586e07167cd134885b95 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Mon, 8 Apr 2024 15:57:24 -0700 Subject: [PATCH] feat(ingest/redshift): filter out system queries from usage --- .../src/datahub/ingestion/source/redshift/query.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/query.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/query.py index c67da13ced88e..1bc82556ce4bc 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/redshift/query.py +++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/query.py @@ -678,6 +678,11 @@ def usage_query(start_time: str, end_time: str, database: str) -> str: AND ss.starttime < '{end_time}' AND sti.database = '{database}' AND sq.aborted = 0 + AND NOT ( + sq.querytxt LIKE 'small table validation: %' + OR sq.querytxt LIKE 'Small table conversion: %' + OR sq.querytxt LIKE 'padb_fetch_sample: %' + ) ORDER BY ss.endtime DESC; """.strip()