feat(ingest): bump acryl-sqlglot dep (#10554)

datahub-project · May 21, 2024 · 2b6c78b · 2b6c78b
1 parent 416b461
commit 2b6c78b
Show file tree

Hide file tree

Showing 6 changed files with 44 additions and 10 deletions.
diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py
@@ -99,7 +99,7 @@
 sqlglot_lib = {
     # Using an Acryl fork of sqlglot.
     # https://github.com/tobymao/sqlglot/compare/main...hsheth2:sqlglot:hsheth?expand=1
-    "acryl-sqlglot[rs]==23.11.2.dev2",
+    "acryl-sqlglot[rs]==23.17.1.dev10",
 }
 
 classification_lib = {

diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_expand_select_star_basic.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_expand_select_star_basic.json
@@ -34,7 +34,7 @@
                         "com.linkedin.pegasus2avro.schema.NumberType": {}
                     }
                 },
-                "native_column_type": "DECIMAL"
+                "native_column_type": "DECIMAL(38, 0)"
             },
             "upstreams": [
                 {
@@ -52,7 +52,7 @@
                         "com.linkedin.pegasus2avro.schema.NumberType": {}
                     }
                 },
-                "native_column_type": "DECIMAL"
+                "native_column_type": "DECIMAL(38, 0)"
             },
             "upstreams": [
                 {
@@ -160,7 +160,7 @@
                         "com.linkedin.pegasus2avro.schema.NumberType": {}
                     }
                 },
-                "native_column_type": "DECIMAL"
+                "native_column_type": "DECIMAL(38, 0)"
             },
             "upstreams": [
                 {

diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_postgres_select_subquery.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_postgres_select_subquery.json
@@ -50,10 +50,10 @@
                 "column": "c",
                 "column_type": {
                     "type": {
-                        "com.linkedin.pegasus2avro.schema.ArrayType": {}
+                        "com.linkedin.pegasus2avro.schema.NumberType": {}
                     }
                 },
-                "native_column_type": "INT[]"
+                "native_column_type": "INT"
             },
             "upstreams": [
                 {

diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_postgres_update_subselect.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_postgres_update_subselect.json
@@ -0,0 +1,36 @@
+{
+    "query_type": "UPDATE",
+    "query_type_props": {},
+    "query_fingerprint": "c4b756bd9eceebd04abaa0c273b12898f88fc2a9d1f4c7ae03faa7c4e096fa21",
+    "in_tables": [
+        "urn:li:dataset:(urn:li:dataPlatform:postgres,my_db.my_schema.accounts,PROD)",
+        "urn:li:dataset:(urn:li:dataPlatform:postgres,my_db.my_schema.employees,PROD)"
+    ],
+    "out_tables": [
+        "urn:li:dataset:(urn:li:dataPlatform:postgres,my_db.my_schema.accounts,PROD)"
+    ],
+    "column_lineage": [
+        {
+            "downstream": {
+                "table": "urn:li:dataset:(urn:li:dataPlatform:postgres,my_db.my_schema.accounts,PROD)",
+                "column": "sales_person_name",
+                "column_type": {
+                    "type": {
+                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                    }
+                },
+                "native_column_type": "VARCHAR"
+            },
+            "upstreams": [
+                {
+                    "table": "urn:li:dataset:(urn:li:dataPlatform:postgres,my_db.my_schema.employees,PROD)",
+                    "column": "name"
+                }
+            ]
+        }
+    ],
+    "debug_info": {
+        "confidence": 0.9,
+        "generalized_statement": "UPDATE accounts SET sales_person_name = (SELECT name FROM employees WHERE employees.id = accounts.sales_person_id)"
+    }
+}
diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_redshift_system_automove.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_redshift_system_automove.json
@@ -3,7 +3,7 @@
     "query_type_props": {
         "kind": "TABLE"
     },
-    "query_fingerprint": "93a463cd97acfcf1bf0a57a087596771f72d65e013f856a9d8c17c81b6439a2f",
+    "query_fingerprint": "f9ecadf44a23ac60349f68bdbcb65fcab8b30c6fbeb972ae42fa13c551ea7458",
     "in_tables": [
         "urn:li:dataset:(urn:li:dataPlatform:redshift,my_db.public.permanent_1,PROD)"
     ],
@@ -40,6 +40,6 @@
     ],
     "debug_info": {
         "confidence": 0.2,
-        "generalized_statement": "CREATE TABLE \"pg_automv\".\"mv_tbl__auto_mv_12708107__0_recomputed\" AS (SELECT COUNT(CAST(? AS INTEGER)) AS \"aggvar_3\", COUNT(CAST(? AS INTEGER)) AS \"num_rec\" FROM \"public\".\"permanent_1\" AS \"permanent_1\" WHERE ((CAST(\"permanent_1\".\"insertxid\" AS SMALLINT) <= ?) AND (CAST(\"permanent_1\".\"deletexid\" AS SMALLINT) > ?)) OR (CAST(FALSE AS BOOLEAN) AND (CAST(\"permanent_1\".\"insertxid\" AS SMALLINT) = ?) AND (CAST(\"permanent_1\".\"deletexid\" AS SMALLINT) <> ?)))"
+        "generalized_statement": "CREATE TABLE \"pg_automv\".\"mv_tbl__auto_mv_12708107__0_recomputed\" BACKUP YES DISTSTYLE KEY DISTKEY(\"2\") AS (SELECT COUNT(CAST(? AS INTEGER)) AS \"aggvar_3\", COUNT(CAST(? AS INTEGER)) AS \"num_rec\" FROM \"public\".\"permanent_1\" AS \"permanent_1\" WHERE ((CAST(\"permanent_1\".\"insertxid\" AS BIGINT) <= ?) AND (CAST(\"permanent_1\".\"deletexid\" AS BIGINT) > ?)) OR (CAST(FALSE AS BOOLEAN) AND (CAST(\"permanent_1\".\"insertxid\" AS BIGINT) = ?) AND (CAST(\"permanent_1\".\"deletexid\" AS BIGINT) <> ?)))"
     }
 }
diff --git a/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py b/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py
@@ -956,7 +956,6 @@ def test_postgres_select_subquery():
     )
 
 
-@pytest.mark.skip(reason="We can't parse column-list syntax with sub-selects yet")
 def test_postgres_update_subselect():
     assert_sql_result(
         """
@@ -1078,7 +1077,6 @@ def test_redshift_union_view():
     )
 
 
-@pytest.mark.skip(reason="sqlglot doesn't recognize the BACKUP directive right now")
 def test_redshift_system_automove() -> None:
     # Came across this in the Redshift query log, but it seems to be a system-generated query.
     assert_sql_result(