From 05bc22b6757eb55d6f14caca9c2c859131402dbe Mon Sep 17 00:00:00 2001 From: Joel Labes Date: Mon, 7 Sep 2020 22:57:35 +1200 Subject: [PATCH 1/5] Create accepted_range test --- macros/schema_tests/accepted_range.sql | 32 ++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 macros/schema_tests/accepted_range.sql diff --git a/macros/schema_tests/accepted_range.sql b/macros/schema_tests/accepted_range.sql new file mode 100644 index 00000000..f364ffb1 --- /dev/null +++ b/macros/schema_tests/accepted_range.sql @@ -0,0 +1,32 @@ +{% macro test_accepted_range(model, min_value = none, max_value = none, inclusive = true, where = "true") %} + +{%- set column_name = kwargs.get('column_name', kwargs.get('field')) -%} + +with meet_condition as( + select {{ column_name }} + from {{ model }} + where {{ where }} +), + +validation_errors as ( + select * + from meet_condition + where + -- never true, defaults to an empty result set. Exists to ensure any combo of the `or` clauses below succeeds + 1 = 2 + + {%- if min_value is not none %} + -- records with a value >= min_value are permitted. The `not` flips this to find records that don't meet the rule. + or not {{ column_name }} >{{"=" if inclusive}} {{min_value}} + {%- endif %} + + {%- if max_value is not none %} + -- records with a value <= max_value are permitted. The `not` flips this to find records that don't meet the rule. + or not {{ column_name }} <{{"=" if inclusive}} {{max_value}} + {%- endif %} +) + +select count(*) +from validation_errors + +{% endmacro %} From 638b08a10f00aeac6768e969c8c01c7c53513fc5 Mon Sep 17 00:00:00 2001 From: Joel Labes Date: Mon, 7 Sep 2020 22:57:48 +1200 Subject: [PATCH 2/5] Add integration tests --- .../schema_tests/data_test_accepted_range.csv | 3 +++ .../models/schema_tests/schema.yml | 17 +++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 integration_tests/data/schema_tests/data_test_accepted_range.csv diff --git a/integration_tests/data/schema_tests/data_test_accepted_range.csv b/integration_tests/data/schema_tests/data_test_accepted_range.csv new file mode 100644 index 00000000..38b90a50 --- /dev/null +++ b/integration_tests/data/schema_tests/data_test_accepted_range.csv @@ -0,0 +1,3 @@ +id +-1 +11 \ No newline at end of file diff --git a/integration_tests/models/schema_tests/schema.yml b/integration_tests/models/schema_tests/schema.yml index 55de5a68..bd6999b4 100644 --- a/integration_tests/models/schema_tests/schema.yml +++ b/integration_tests/models/schema_tests/schema.yml @@ -101,3 +101,20 @@ models: combination_of_columns: - month - product + + - name: data_test_accepted_range + tests: + - dbt_utils.accepted_range: + min_value: -1 + max_value: 11 + inclusive: true + + - dbt_utils.accepted_range: + min_value: -2 + max_value: 11.1 + inclusive: false + + - dbt_utils.accepted_range: + min_value: 0 + inclusive: true + where: "id <> -1" \ No newline at end of file From 21c7c08f392ad6be11590f81f97539b8ab619234 Mon Sep 17 00:00:00 2001 From: Joel Labes Date: Mon, 7 Sep 2020 22:58:10 +1200 Subject: [PATCH 3/5] Add accepted_range documentation --- README.md | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/README.md b/README.md index e52bc984..ce453b11 100644 --- a/README.md +++ b/README.md @@ -418,6 +418,45 @@ An optional `quote_columns` parameter (`default=false`) can also be used if a co quote_columns: true ``` + +#### accepted_range ([source](macros/schema_tests/accepted_range.sql)) +This test checks that a column's values fall inside an expected range. Any combination of `min_value` and `max_value` is allowed, and the range can be inclusive or exclusive. Provide a `where` argument to filter to specific records only. + +In addition to comparisons to a scalar value, you can also compare to another column's values. Any data type that supports the `>` or `<` operators can be compared, so you could also run tests like checking that all order dates are in the past. + +Usage: +```yaml +version: 2 + +models: + - name: model_name + columns: + - name: user_id + tests: + - dbt_utils.accepted_range: + min_value: 0 + inclusive: false + + - name: account_created_at + tests: + - dbt_utils.accepted_range: + max_value: "getdate()" + #inclusive is true by default + + - name: num_returned_orders + tests: + - dbt_utils.accepted_range + min_value: 0 + max_value: "num_orders" + + - name: num_web_sessions + tests: + - dbt_utils.accepted_range + min_value: 0 + inclusive: false + where: "num_orders > 0" +``` + --- ### SQL helpers #### get_query_results_as_dict ([source](macros/sql/get_query_results_as_dict.sql)) From dbffa2ab55761f483936e4da68c1b04d172f0ea0 Mon Sep 17 00:00:00 2001 From: Joel Labes Date: Wed, 9 Sep 2020 09:34:43 +1200 Subject: [PATCH 4/5] Fixing invalid yaml in readme --- README.md | 18 +++++++++--------- macros/schema_tests/accepted_range.sql | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index ce453b11..c2af35c7 100644 --- a/README.md +++ b/README.md @@ -440,21 +440,21 @@ models: - name: account_created_at tests: - dbt_utils.accepted_range: - max_value: "getdate()" - #inclusive is true by default + max_value: "getdate()" + #inclusive is true by default - name: num_returned_orders tests: - - dbt_utils.accepted_range - min_value: 0 - max_value: "num_orders" + - dbt_utils.accepted_range: + min_value: 0 + max_value: "num_orders" - name: num_web_sessions tests: - - dbt_utils.accepted_range - min_value: 0 - inclusive: false - where: "num_orders > 0" + - dbt_utils.accepted_range: + min_value: 0 + inclusive: false + where: "num_orders > 0" ``` --- diff --git a/macros/schema_tests/accepted_range.sql b/macros/schema_tests/accepted_range.sql index f364ffb1..4c599f8a 100644 --- a/macros/schema_tests/accepted_range.sql +++ b/macros/schema_tests/accepted_range.sql @@ -17,12 +17,12 @@ validation_errors as ( {%- if min_value is not none %} -- records with a value >= min_value are permitted. The `not` flips this to find records that don't meet the rule. - or not {{ column_name }} >{{"=" if inclusive}} {{min_value}} + or not {{ column_name }} > {{- "=" if inclusive }} {{ min_value }} {%- endif %} {%- if max_value is not none %} -- records with a value <= max_value are permitted. The `not` flips this to find records that don't meet the rule. - or not {{ column_name }} <{{"=" if inclusive}} {{max_value}} + or not {{ column_name }} < {{- "=" if inclusive }} {{ max_value }} {%- endif %} ) From 2978e2768df6775571a5f0894c93083ce5ded309 Mon Sep 17 00:00:00 2001 From: Joel Labes Date: Tue, 15 Sep 2020 09:28:23 +1200 Subject: [PATCH 5/5] Add missing columns: entry Co-authored-by: Claire Carroll --- .../models/schema_tests/schema.yml | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/integration_tests/models/schema_tests/schema.yml b/integration_tests/models/schema_tests/schema.yml index bd6999b4..0026b957 100644 --- a/integration_tests/models/schema_tests/schema.yml +++ b/integration_tests/models/schema_tests/schema.yml @@ -103,18 +103,20 @@ models: - product - name: data_test_accepted_range - tests: - - dbt_utils.accepted_range: - min_value: -1 - max_value: 11 - inclusive: true - - - dbt_utils.accepted_range: - min_value: -2 - max_value: 11.1 - inclusive: false - - - dbt_utils.accepted_range: - min_value: 0 - inclusive: true - where: "id <> -1" \ No newline at end of file + columns: + - name: id + tests: + - dbt_utils.accepted_range: + min_value: -1 + max_value: 11 + inclusive: true + + - dbt_utils.accepted_range: + min_value: -2 + max_value: 11.1 + inclusive: false + + - dbt_utils.accepted_range: + min_value: 0 + inclusive: true + where: "id <> -1"