Merge pull request #4 from kgmcquate/develop

Added more tests, test generation functionality, and docs

.github/workflows/dbt_test.yml

@@ -15,13 +15,16 @@ jobs:
   test-duckdb:
     name: Test on DuckDB
     runs-on: ubuntu-latest
-    #environment: production
-    container:
-      image: python:3.11
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.10", "3.11"]
 
     steps:
-    - name: Checkout
-      uses: actions/checkout@v3
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
 
     - name: pip install
       run: pip install dbt-core duckdb dbt-duckdb
@@ -32,29 +35,30 @@ jobs:
     - name: load test data
       run: dbt seed --target duckdb
 
+    - name: Create seed tests
+      run: bash tests/create_seeds_tests_yml.sh
+
     - name: dbt test
       run: dbt test --target duckdb
 
   test-postgres:
     name: Test on Postgres
     runs-on: ubuntu-latest
-    #environment: production
     container:
       image: python:3.11
 
-    # Service containers to run with `container-job`
     services:
-      # Label used to access the service container
       postgres:
-        # Docker Hub image
         image: postgres
-        # Provide the password for postgres
         env:
           POSTGRES_PASSWORD: postgres
 
     steps:
-    - name: Checkout
-      uses: actions/checkout@v3
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
 
     - name: pip install
       run: pip install dbt-core duckdb dbt-postgres
@@ -67,6 +71,3 @@ jobs:
 
     - name: dbt test
       run: dbt test --target postgres
-
-
-

README.md

@@ -4,13 +4,33 @@ dbt-testgen autogenerates dbt test yaml based on real data.
 
 Inspired by [dbt-codegen]() and [deequ Constraint Suggestion](https://github.com/awslabs/deequ/blob/master/src/main/scala/com/amazon/deequ/examples/constraint_suggestion_example.md)
 
+## Usage
+The DBT config YAML is generated by a Jinja macro, `get_test_suggestions`, which you can run like this:
+```powershell
+dbt compile -q --inline "{{ testgen.get_test_suggestions(ref('mymodel')) }}"
+```
+Output:
+```yaml
+models:
+- name: mymodel
+  columns:
+  - name: user_id
+    tests:
+    - unique
+    - not_null
+    - dbt_utils.accepted_range:
+        min_value: 1
+        max_value: 30
+```
+
 ## Test types
-- uniqueness
-- not_null
-- string length
-- range
-- accepted_values 
-- freshness
+dbt-testgen can generate these types of tests:
+- [uniqueness](#uniqueness)
+- [not_null](#not-null)
+- [string length](#string-length)
+- [range](#range)
+- [accepted_values](#accepted-values)
+- [freshness](#freshness)
 
 ### Uniqueness
 

integration_tests/dbt_project.yml

@@ -4,8 +4,6 @@ config-version: 2
 
 profile: 'integration_tests'
 
-model-paths: ["models"]
-analysis-paths: ["analyses"]
 test-paths: ["tests"]
 seed-paths: ["seeds"]
 macro-paths: ["macros"]
@@ -17,4 +15,5 @@ clean-targets:
 
 seeds:
   +schema: integration_test_data
-  +quote_columns: true
+  +quote_columns: true
+

integration_tests/packages.yml

@@ -1,2 +1,4 @@
 packages:
-    - local: ../
+    - local: ../
+    - package: dbt-labs/dbt_utils
+      version: 1.1.1

integration_tests/tests/create_seeds_tests_yml.sh

@@ -0,0 +1,7 @@
+dbt compile -q \
+    --inline "{{ testgen.get_test_suggestions(ref('users'), resource_type='seeds') }}" \
+    > seeds/users_test_suggestions.yml
+
+dbt compile -q \
+    --inline "{{ testgen.get_test_suggestions(ref('colnames_with_spaces'), resource_type='seeds', column_config={'quote': true}) }}"  \
+    > seeds/colnames_with_spaces_test_suggestions.yml

integration_tests/tests/generate_accepted_values_tests/accepted_values_colnames_with_spaces.sql

@@ -1,10 +1,10 @@
 
 
-{% set actual_yaml = toyaml(fromjson(tojson(
+{% set actual_yaml = testgen.to_yaml(
         testgen.get_accepted_values_test_suggestions(
             ref('colnames_with_spaces')
         )
-    )))
+    )
 %}
 
 {% set expected_yaml %}

integration_tests/tests/generate_accepted_values_tests/accepted_values_users.sql

@@ -1,10 +1,10 @@
 
 
-{% set actual_yaml = toyaml(fromjson(tojson(
+{% set actual_yaml = testgen.to_yaml(
         testgen.get_accepted_values_test_suggestions(
             ref('users')
         )
-    )))
+    )
 %}
 
 {% set expected_yaml %}

integration_tests/tests/generate_range_tests/range_colnames_with_spaces.sql

@@ -1,10 +1,10 @@
 
 
-{% set actual_yaml = toyaml(fromjson(tojson(
+{% set actual_yaml = testgen.to_yaml(
         testgen.get_range_test_suggestions(
             ref('colnames_with_spaces')
         )
-    )))
+    )
 %}
 
 {% set expected_yaml %}

integration_tests/tests/generate_range_tests/range_users.sql

@@ -1,10 +1,10 @@
 
 
-{% set actual_yaml = toyaml(fromjson(tojson(
+{% set actual_yaml = testgen.to_yaml(
         testgen.get_range_test_suggestions(
             ref('users')
         )
-    )))
+    )
 %}
 
 {% set expected_yaml %}

integration_tests/tests/generate_uniqueness_tests/uniqueness_colnames_with_spaces.sql

@@ -0,0 +1,36 @@
+
+
+{% set actual_yaml = testgen.to_yaml(
+        testgen.get_uniqueness_test_suggestions(
+            ref('colnames_with_spaces'),
+            composite_key_length = 1,
+            column_config={'quote': true}
+        )
+    )
+%}
+
+{% set expected_yaml %}
+models:
+- name: colnames_with_spaces
+  columns:
+  - name: First Name
+    description: Uniqueness test generated by dbt-testgen
+    tests:
+    - unique
+    - not_null
+    quote: true
+  - name: Age (Years)
+    description: Uniqueness test generated by dbt-testgen
+    tests:
+    - unique
+    - not_null
+    quote: true
+  - name: Current City
+    description: Uniqueness test generated by dbt-testgen
+    tests:
+    - unique
+    - not_null
+    quote: true
+{% endset %}
+
+{{ assert_equal (actual_yaml | trim, expected_yaml | trim) }}

integration_tests/tests/generate_uniqueness_tests/uniqueness_users.sql

@@ -1,11 +1,11 @@
 
 
-{% set actual_yaml = toyaml(fromjson(tojson(
+{% set actual_yaml = testgen.to_yaml(
         testgen.get_uniqueness_test_suggestions(
             ref('users'),
-            compound_key_length = 1
+            composite_key_length = 1
         )
-    )))
+    )
 %}
 
 {% set expected_yaml %}

integration_tests/tests/merge_dbt_configs/merge_dbt_configs_1.sql

@@ -44,6 +44,7 @@ models:
 - name: users
   columns:
   - name: user_id
+    description: Uniqueness test generated by dbt-testgen
     tests:
     - unique:
         tags:
@@ -52,6 +53,7 @@ models:
         tags:
         - uniqueness
   - name: email
+    description: Uniqueness test generated by dbt-testgen
     tests:
     - unique:
         tags:

integration_tests/tests/merge_dbt_configs/merge_dbt_configs_2.sql

@@ -42,6 +42,7 @@ models:
 - name: users
   columns:
   - name: user_id
+    description: Uniqueness test generated by dbt-testgen
     tests:
     - unique:
         tags:

integration_tests/tests/test_test_selection/test_selection_users.sql

@@ -1,38 +1,44 @@
 
 
-{% set actual_yaml = toyaml(fromjson(tojson(
+{% set actual_yaml = testgen.to_yaml(
         testgen.get_test_suggestions(
-            ref('users')
+            ref('users'),
+            return_object=true
         )
-    )))
+    )
 %}
 
 {% set expected_yaml %}
 models:
 - name: users
   columns:
   - name: user_id
+    description: Numeric range test generated by dbt-testgen
     tests:
     - unique
     - not_null
     - dbt_utils.accepted_range:
         min_value: 1
         max_value: 30
   - name: username
+    description: Uniqueness test generated by dbt-testgen
     tests:
     - unique
     - not_null
   - name: email
+    description: Uniqueness test generated by dbt-testgen
     tests:
     - unique
     - not_null
   - name: user_status
+    description: Accepted values test generated by dbt-testgen
     tests:
     - accepted_values:
         values:
         - active
         - inactive
   - name: age
+    description: Numeric range test generated by dbt-testgen
     tests:
     - dbt_utils.accepted_range:
         min_value: 22

macros/helpers/to_yaml.sql

@@ -0,0 +1,3 @@
+{% macro to_yaml(the_obj) %}
+{{ return(toyaml(fromjson(tojson(the_obj)))) }}
+{% endmacro %}

macros/test_selection/get_test_suggestions.sql → macros/test_aggregation/get_test_suggestions.sql

@@ -3,12 +3,15 @@
         table_relation,
         sample = false,
         limit = None,
-        is_source = false,
+        resource_type = "models",
+        column_config = {},
         exclude_types = [],
         exclude_cols = [],
         tags = [],
         tests = ["uniqueness", "accepted_values", "range"],
-        dbt_config = None
+        composite_key_length = 1,
+        dbt_config = None,
+        return_object = false
     ) %}
     {# Run macro for the specific target DB #}
     {% if execute %}
@@ -17,10 +20,12 @@
                 table_relation=table_relation,
                 sample=sample,
                 limit=limit,
-                is_source=is_source,
+                resource_type=resource_type,
+                column_config=column_config,
                 exclude_types=exclude_types,
                 exclude_cols=exclude_cols,
                 tags=tags,
+                composite_key_length=composite_key_length,
                 dbt_config=dbt_config
             ) %}
         {% endif %}
@@ -30,7 +35,8 @@
                 table_relation=table_relation,
                 sample=sample,
                 limit=limit,
-                is_source=is_source,
+                resource_type=resource_type,
+                column_config=column_config,
                 exclude_types=exclude_types,
                 exclude_cols=exclude_cols,
                 tags=tags,
@@ -43,15 +49,19 @@
                 table_relation=table_relation,
                 sample=sample,
                 limit=limit,
-                is_source=is_source,
+                resource_type=resource_type,
+                column_config=column_config,
                 exclude_types=exclude_types,
                 exclude_cols=exclude_cols,
                 tags=tags,
                 dbt_config=dbt_config
             ) %}
         {% endif %}
 
-        {{ return(dbt_config) }}
-
+        {% if return_object %}
+            {{ return(dbt_config) }}
+        {% else %}
+            {{ print(testgen.to_yaml(dbt_config)) }}
+        {% endif %}
     {% endif %}
 {%- endmacro %}