added quoting for snowflake

kgmcquate · Jan 3, 2024 · 89d0789 · 89d0789
1 parent dc5ac28
commit 89d0789
Show file tree

Hide file tree

Showing 5 changed files with 51 additions and 31 deletions.
diff --git a/.github/workflows/dbt_test.yml b/.github/workflows/dbt_test.yml
@@ -19,6 +19,9 @@ jobs:
       matrix:
         python-version: ["3.9", "3.10", "3.11"]
 
+    env:
+      DBT_TARGET: duckdb
+
     steps:
     - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
@@ -27,19 +30,19 @@ jobs:
         python-version: ${{ matrix.python-version }}
 
     - name: pip install
-      run: pip install dbt-core duckdb dbt-duckdb
+      run: pip install dbt-core duckdb dbt-${DBT_TARGET}
 
     - name: Install DBT deps
-      run: dbt deps --target duckdb
+      run: dbt deps --target ${DBT_TARGET}
 
     - name: load test data
-      run: dbt seed --target duckdb
+      run: dbt seed --target ${DBT_TARGET}
 
     - name: Create seed tests
       run: bash tests/create_seeds_tests_yml.sh
 
     - name: dbt test
-      run: dbt test --target duckdb
+      run: dbt test --target ${DBT_TARGET}
 
   test-postgres:
     name: Test on Postgres
@@ -53,20 +56,23 @@ jobs:
         env:
           POSTGRES_PASSWORD: postgres
 
+    env:
+      DBT_TARGET: postgres
+
     steps:
     - uses: actions/checkout@v4
 
     - name: pip install
-      run: pip install dbt-core dbt-postgres
+      run: pip install dbt-core dbt-${DBT_TARGET}
 
     - name: Install DBT deps
-      run: dbt deps --target postgres
+      run: dbt deps --target ${DBT_TARGET}
 
     - name: load test data
-      run: dbt seed --target postgres
+      run: dbt seed --target ${DBT_TARGET}
 
     - name: dbt test
-      run: dbt test --target postgres
+      run: dbt test --target ${DBT_TARGET}
 
   test-snowflake:
     name: Test on Snowflake
@@ -76,18 +82,22 @@ jobs:
 
     env:
       SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
+      DBT_TARGET: snowflake
 
     steps:
     - uses: actions/checkout@v4
 
     - name: pip install
-      run: pip install dbt-core dbt-snowflake
+      run: pip install dbt-core dbt-${DBT_TARGET}
 
     - name: Install DBT deps
-      run: dbt deps --target snowflake
+      run: dbt deps --target ${DBT_TARGET}
 
     - name: load test data
-      run: dbt seed --target snowflake
+      run: dbt seed --target ${DBT_TARGET}
+
+    - name: Create seed tests
+      run: bash tests/create_seeds_tests_yml.sh
 
     - name: dbt test
-      run: dbt test --target snowflake
+      run: dbt test --target ${DBT_TARGET}
diff --git a/integration_tests/tests/create_seeds_tests_yml.sh b/integration_tests/tests/create_seeds_tests_yml.sh
@@ -1,7 +1,7 @@
-dbt compile -q \
-    --inline "{{ testgen.get_test_suggestions(ref('users'), resource_type='seeds') }}" \
+dbt compile --target ${DBT_TARGET} -q \
+    --inline "{{ testgen.get_test_suggestions(ref('users'), resource_type='seeds', column_config={'quote': true} ) }}" \
     > seeds/users_test_suggestions.yml
 
-dbt compile -q \
+dbt compile --target ${DBT_TARGET} -q \
     --inline "{{ testgen.get_test_suggestions(ref('colnames_with_spaces'), resource_type='seeds', column_config={'quote': true}) }}"  \
     > seeds/colnames_with_spaces_test_suggestions.yml
diff --git a/macros/test_generation/get_accepted_values_test_suggestions.sql b/macros/test_generation/get_accepted_values_test_suggestions.sql
@@ -6,6 +6,7 @@
     {{ return("array_agg(" ~ adapter.quote(colname) ~ "::VARCHAR)") }}
 {% endmacro %}
 
+
 {% macro get_accepted_values_test_suggestions(
         table_relation,
         sample = false,
@@ -64,12 +65,13 @@
 
     {% set count_distinct_exprs = [] %}
     {% for column in columns %}
+        {# Use capitals for colnames because of snowflake #}
         {% do count_distinct_exprs.append(
             "
-            select " ~ loop.index ~ " AS ordering, 
-                '" ~ column.column ~ "' AS colname, 
-                count(1) as cardinality, " ~ 
-                testgen.array_agg(column.column) ~ " AS unique_values 
+            select " ~ loop.index ~ " AS ORDERING, 
+                '" ~ column.column ~ "' AS COLNAME, 
+                count(1) as CARDINALITY, " ~ 
+                testgen.array_agg(column.column) ~ " AS UNIQUE_VALUES
             from (
                 select " ~ adapter.quote(column.column) ~ "
                 from " ~ table_relation ~ "
@@ -83,20 +85,26 @@
         SELECT * FROM (
             {{ count_distinct_exprs | join("\nUNION ALL\n") }}
         ) t2
-        WHERE cardinality <= {{ max_cardinality|string }}
-        ORDER BY ordering ASC
+        WHERE CARDINALITY <= {{ max_cardinality|string }}
+        ORDER BY ORDERING ASC
     {% endset %}
 
+    {# {{ print(count_distinct_sql) }} #}
+
     {% set cardinality_results = testgen.query_as_list(count_distinct_sql) %}
 
+    {# {{ print(cardinality_results) }} #}
+
     {% set column_tests = [] %}
     {% for cardinality_result in cardinality_results %}
 
+        {# {{ print(cardinality_result.COLNAME) }} #}
+
         {% set col_config = {
-                "name": cardinality_result.colname,
+                "name": cardinality_result.COLNAME,
                 "description": "Accepted values test generated by dbt-testgen",
                 "tests": [
-                    {"accepted_values": {"values": fromjson(cardinality_result.unique_values)|sort}}
+                    {"accepted_values": {"values": fromjson(cardinality_result.UNIQUE_VALUES)|sort}}
                 ]
             }
         %}
@@ -109,6 +117,8 @@
 
     {% set new_dbt_config = {resource_type: [{"name": table_relation.identifier, "columns": column_tests}]} %}
 
+    {{ print(new_dbt_config) }}
+
     {% set merged_dbt_config = testgen.merge_dbt_configs(dbt_config, new_dbt_config) %}
 
     {% do return(merged_dbt_config) %}

diff --git a/macros/test_generation/get_range_test_suggestions.sql b/macros/test_generation/get_range_test_suggestions.sql
@@ -48,9 +48,9 @@
     {% set min_max_exprs = [] %}
     {% for column in number_cols %}
         {% do min_max_exprs.append(
-            "SELECT '" ~ column.column ~ "' AS colname, " ~ 
-                "MIN(" ~ adapter.quote(column.column) ~ ") as col_min, " ~ 
-                "MAX(" ~ adapter.quote(column.column) ~ ") as col_max " ~ 
+            "SELECT '" ~ column.column ~ "' AS COLNAME, " ~ 
+                "MIN(" ~ adapter.quote(column.column) ~ ") as COL_MIN, " ~ 
+                "MAX(" ~ adapter.quote(column.column) ~ ") as COL_MAX " ~ 
             "FROM " ~ table_relation
         ) %}
     {% endfor %}
@@ -67,13 +67,13 @@
     {% set column_tests = [] %}
     {% for result in results %}
         {% set col_config = {
-                "name": result.colname,
+                "name": result.COLNAME,
                 "description": "Numeric range test generated by dbt-testgen",
                 "tests": [
                     {
                         "dbt_utils.accepted_range": {
-                            "min_value": result.col_min,
-                            "max_value": result.col_max
+                            "min_value": result.COL_MIN,
+                            "max_value": result.COL_MAX
                         }
                     }
                 ]

diff --git a/macros/test_generation/get_uniqueness_test_suggestions.sql b/macros/test_generation/get_uniqueness_test_suggestions.sql
@@ -83,7 +83,7 @@
             {% do column_combo_quoted.append(adapter.quote(col))%}
         {% endfor %}
         {% do count_distinct_exprs.append(
-            "SELECT " ~ loop.index ~ " AS ordering, count(1) AS cardinality 
+            "SELECT " ~ loop.index ~ " AS ORDERING, count(1) AS CARDINALITY
             from (
                 SELECT 1 FROM " ~ table_relation ~ " 
                 GROUP BY " ~ column_combo_quoted|join(", ") ~ "
@@ -106,7 +106,7 @@
 
     {% set unique_keys = [] %}
     {% for cardinality_result in cardinality_results %}
-        {% if cardinality_result[1].cardinality == table_count %}
+        {% if cardinality_result[1].CARDINALITY == table_count %}
             {% do unique_keys.append(cardinality_result[0]) %}
         {% endif %}
     {% endfor %}