diff --git a/CHANGELOG.md b/CHANGELOG.md index 8648e52b..e5d73fb8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ ## Features +* Switch usage of `adapter_macro` to `adapter.dispatch`, and define `dbt_utils_dispatch_list`, +enabling users of community-supported database plugins to add or override macro implementations +specific to their database (#267) +* Use `add_ephemeral_prefix` instead of hard-coding a string literal, to support +database adapters that use different prefixes (#267) + ## Quality of life # dbt-utils v0.5.1 diff --git a/README.md b/README.md index 30607d4e..0300d1cb 100644 --- a/README.md +++ b/README.md @@ -777,6 +777,42 @@ We welcome contributions to this repo! To contribute a new feature or a fix, ple ---- +### Dispatch macros + +**Note:** This is primarily relevant to users and maintainers of community-supported +database plugins. If you use Postgres, Redshift, Snowflake, or Bigquery, this likely +does not apply to you. + +dbt v0.18.0 introduces `adapter.dispatch()`, a reliable way to define different implementations of the same macro +across different databases. + +All dispatched macros in `dbt_utils` have an override setting: a `var` named +`dbt_utils_dispatch_list` that accepts a list of package names. If you set this +variable in your project, when dbt searches for implementations of a dispatched +`dbt_utils` macro, it will search through your listed packages _before_ using +the implementations defined in `dbt_utils`. + +Set the variable: +```yml +vars: + dbt_utils_dispatch_list: + - first_package_to_search # likely the name of your root project + - second_package_to_search # likely an "add-on" package, such as spark_utils + # dbt_utils is always the last place searched +``` + +When running on Spark, if dbt needs to dispatch `dbt_utils.datediff`, it will search for the following in order: +``` +first_package_to_search.spark__datediff +first_package_to_search.default__datediff +second_package_to_search.spark__datediff +second_package_to_search.default__datediff +dbt_utils.spark__datediff +dbt_utils.default__datediff +``` + +---- + ### Getting started with dbt - [What is dbt]? diff --git a/dbt_project.yml b/dbt_project.yml index 03a57409..e9a847d3 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,7 +1,7 @@ name: 'dbt_utils' version: '0.1.0' -require-dbt-version: [">=0.17.0", "<0.18.0"] +require-dbt-version: [">=0.18.0", "<0.19.0"] config-version: 2 target-path: "target" diff --git a/integration_tests/models/datetime/test_date_spine.sql b/integration_tests/models/datetime/test_date_spine.sql index 4d0d4004..93cd07f1 100644 --- a/integration_tests/models/datetime/test_date_spine.sql +++ b/integration_tests/models/datetime/test_date_spine.sql @@ -9,8 +9,7 @@ with date_spine as ( {% if target.type == 'postgres' %} - {{ log("WARNING: Not testing - datediff macro is unsupported on Postgres", info=True) }} - select * from {{ ref('data_date_spine') }} + {{ dbt_utils.date_spine("day", "'2018-01-01'::date", "'2018-01-10'::date") }} {% elif target.type == 'bigquery' %} select cast(date_day as date) as date_day diff --git a/integration_tests/models/sql/test_get_relations_by_pattern.sql b/integration_tests/models/sql/test_get_relations_by_pattern.sql index 5a51dd05..b10d8eef 100644 --- a/integration_tests/models/sql/test_get_relations_by_pattern.sql +++ b/integration_tests/models/sql/test_get_relations_by_pattern.sql @@ -1,4 +1,4 @@ {{ config(materialized = 'table') }} -{% set relations = dbt_utils.get_relations_by_pattern(target.schema, 'data_events_') %} +{% set relations = dbt_utils.get_relations_by_pattern(target.schema, 'data_events_%') %} {{ dbt_utils.union_relations(relations) }} \ No newline at end of file diff --git a/macros/cross_db_utils/_get_utils_namespaces.sql b/macros/cross_db_utils/_get_utils_namespaces.sql new file mode 100644 index 00000000..e841381c --- /dev/null +++ b/macros/cross_db_utils/_get_utils_namespaces.sql @@ -0,0 +1,4 @@ +{% macro _get_utils_namespaces() %} + {% set override_namespaces = var('dbt_utils_dispatch_list', []) %} + {% do return(override_namespaces + ['dbt_utils']) %} +{% endmacro %} diff --git a/macros/cross_db_utils/_is_ephemeral.sql b/macros/cross_db_utils/_is_ephemeral.sql index e13be407..34c70db9 100644 --- a/macros/cross_db_utils/_is_ephemeral.sql +++ b/macros/cross_db_utils/_is_ephemeral.sql @@ -1,7 +1,8 @@ {% macro _is_ephemeral(obj, macro) %} {%- if obj.is_cte -%} - {% if obj.name.startswith('__dbt__CTE__') %} - {% set model_name = obj.name[12:] %} + {% set ephemeral_prefix = api.Relation.add_ephemeral_prefix('') %} + {% if obj.name.startswith(ephemeral_prefix) %} + {% set model_name = obj.name[(ephemeral_prefix|length):] %} {% else %} {% set model_name = obj.name %} {%- endif -%} diff --git a/macros/cross_db_utils/concat.sql b/macros/cross_db_utils/concat.sql index 88512611..58435016 100644 --- a/macros/cross_db_utils/concat.sql +++ b/macros/cross_db_utils/concat.sql @@ -1,23 +1,23 @@ + + {% macro concat(fields) -%} - {{ adapter_macro('dbt_utils.concat', fields) }} + {{ adapter.dispatch('concat', packages = dbt_utils._get_utils_namespaces())(fields) }} {%- endmacro %} - {% macro default__concat(fields) -%} concat({{ fields|join(', ') }}) {%- endmacro %} - {% macro alternative_concat(fields) %} {{ fields|join(' || ') }} {% endmacro %} {% macro redshift__concat(fields) %} - {{dbt_utils.alternative_concat(fields)}} + {{ dbt_utils.alternative_concat(fields) }} {% endmacro %} {% macro snowflake__concat(fields) %} - {{dbt_utils.alternative_concat(fields)}} + {{ dbt_utils.alternative_concat(fields) }} {% endmacro %} diff --git a/macros/cross_db_utils/current_timestamp.sql b/macros/cross_db_utils/current_timestamp.sql index d838dce9..090e246d 100644 --- a/macros/cross_db_utils/current_timestamp.sql +++ b/macros/cross_db_utils/current_timestamp.sql @@ -1,5 +1,5 @@ {% macro current_timestamp() -%} - {{ adapter_macro('dbt_utils.current_timestamp') }} + {{ adapter.dispatch('current_timestamp', packages = dbt_utils._get_utils_namespaces())() }} {%- endmacro %} {% macro default__current_timestamp() %} @@ -17,7 +17,7 @@ {% macro current_timestamp_in_utc() -%} - {{ adapter_macro('dbt_utils.current_timestamp_in_utc') }} + {{ adapter.dispatch('current_timestamp_in_utc', packages = dbt_utils._get_utils_namespaces())() }} {%- endmacro %} {% macro default__current_timestamp_in_utc() %} diff --git a/macros/cross_db_utils/datatypes.sql b/macros/cross_db_utils/datatypes.sql index 24632ddd..1f949281 100644 --- a/macros/cross_db_utils/datatypes.sql +++ b/macros/cross_db_utils/datatypes.sql @@ -1,7 +1,7 @@ {# string ------------------------------------------------- #} {%- macro type_string() -%} - {{ adapter_macro('dbt_utils.type_string') }} + {{ adapter.dispatch('type_string', packages = dbt_utils._get_utils_namespaces())() }} {%- endmacro -%} {% macro default__type_string() %} @@ -25,7 +25,7 @@ {# timestamp ------------------------------------------------- #} {%- macro type_timestamp() -%} - {{ adapter_macro('dbt_utils.type_timestamp') }} + {{ adapter.dispatch('type_timestamp', packages = dbt_utils._get_utils_namespaces())() }} {%- endmacro -%} {% macro default__type_timestamp() %} @@ -40,7 +40,7 @@ {# float ------------------------------------------------- #} {%- macro type_float() -%} - {{ adapter_macro('dbt_utils.type_float') }} + {{ adapter.dispatch('type_float', packages = dbt_utils._get_utils_namespaces())() }} {%- endmacro -%} {% macro default__type_float() %} @@ -54,7 +54,7 @@ {# numeric ------------------------------------------------ #} {%- macro type_numeric() -%} - {{ adapter_macro('dbt_utils.type_numeric') }} + {{ adapter.dispatch('type_numeric', packages = dbt_utils._get_utils_namespaces())() }} {%- endmacro -%} {% macro default__type_numeric() %} @@ -69,7 +69,7 @@ {# bigint ------------------------------------------------- #} {%- macro type_bigint() -%} - {{ adapter_macro('dbt_utils.type_bigint') }} + {{ adapter.dispatch('type_bigint', packages = dbt_utils._get_utils_namespaces())() }} {%- endmacro -%} {% macro default__type_bigint() %} @@ -83,7 +83,7 @@ {# int ------------------------------------------------- #} {%- macro type_int() -%} - {{ adapter_macro('dbt_utils.type_int') }} + {{ adapter.dispatch('type_int', packages = dbt_utils._get_utils_namespaces())() }} {%- endmacro -%} {% macro default__type_int() %} diff --git a/macros/cross_db_utils/date_trunc.sql b/macros/cross_db_utils/date_trunc.sql index d6073739..08653134 100644 --- a/macros/cross_db_utils/date_trunc.sql +++ b/macros/cross_db_utils/date_trunc.sql @@ -1,5 +1,5 @@ {% macro date_trunc(datepart, date) -%} - {{ adapter_macro('dbt_utils.date_trunc', datepart, date) }} + {{ adapter.dispatch('date_trunc', packages = dbt_utils._get_utils_namespaces()) (datepart, date) }} {%- endmacro %} {% macro default__date_trunc(datepart, date) %} diff --git a/macros/cross_db_utils/dateadd.sql b/macros/cross_db_utils/dateadd.sql index 92977df8..e4a7eb43 100644 --- a/macros/cross_db_utils/dateadd.sql +++ b/macros/cross_db_utils/dateadd.sql @@ -1,5 +1,5 @@ {% macro dateadd(datepart, interval, from_date_or_timestamp) %} - {{ adapter_macro('dbt_utils.dateadd', datepart, interval, from_date_or_timestamp) }} + {{ adapter.dispatch('dateadd', packages = dbt_utils._get_utils_namespaces())(datepart, interval, from_date_or_timestamp) }} {% endmacro %} @@ -23,7 +23,6 @@ {% endmacro %} - {% macro postgres__dateadd(datepart, interval, from_date_or_timestamp) %} {{ from_date_or_timestamp }} + ((interval '1 {{ datepart }}') * ({{ interval }})) diff --git a/macros/cross_db_utils/datediff.sql b/macros/cross_db_utils/datediff.sql index c0c1e7d4..a4cb4d6a 100644 --- a/macros/cross_db_utils/datediff.sql +++ b/macros/cross_db_utils/datediff.sql @@ -1,5 +1,5 @@ {% macro datediff(first_date, second_date, datepart) %} - {{ adapter_macro('dbt_utils.datediff', first_date, second_date, datepart) }} + {{ adapter.dispatch('datediff', packages = dbt_utils._get_utils_namespaces())(first_date, second_date, datepart) }} {% endmacro %} @@ -24,7 +24,6 @@ {% endmacro %} - {% macro postgres__datediff(first_date, second_date, datepart) %} {% if datepart == 'year' %} diff --git a/macros/cross_db_utils/except.sql b/macros/cross_db_utils/except.sql index 95a022f2..c9e8206d 100644 --- a/macros/cross_db_utils/except.sql +++ b/macros/cross_db_utils/except.sql @@ -1,5 +1,5 @@ {% macro except() %} - {{ adapter_macro('dbt_utils.except') }} + {{ adapter.dispatch('except', packages = dbt_utils._get_utils_namespaces())() }} {% endmacro %} diff --git a/macros/cross_db_utils/hash.sql b/macros/cross_db_utils/hash.sql index b7d2c235..8244b7f8 100644 --- a/macros/cross_db_utils/hash.sql +++ b/macros/cross_db_utils/hash.sql @@ -1,5 +1,5 @@ {% macro hash(field) -%} - {{ adapter_macro('dbt_utils.hash', field) }} + {{ adapter.dispatch('hash', packages = dbt_utils._get_utils_namespaces()) (field) }} {%- endmacro %} diff --git a/macros/cross_db_utils/identifier.sql b/macros/cross_db_utils/identifier.sql index ede09f66..1a379261 100644 --- a/macros/cross_db_utils/identifier.sql +++ b/macros/cross_db_utils/identifier.sql @@ -4,7 +4,7 @@ Use `adapter.quote` instead. The {}.{} model triggered this warning. \ '.format(model.package_name, model.name) -%} {%- do exceptions.warn(error_message) -%} - {{ adapter_macro('dbt_utils.identifier', value) }} + {{ adapter.dispatch('identifier', packages = dbt_utils._get_utils_namespaces()) (value) }} {% endmacro %} {% macro default__identifier(value) -%} diff --git a/macros/cross_db_utils/intersect.sql b/macros/cross_db_utils/intersect.sql index c2853bd0..252479a0 100644 --- a/macros/cross_db_utils/intersect.sql +++ b/macros/cross_db_utils/intersect.sql @@ -1,5 +1,5 @@ {% macro intersect() %} - {{ adapter_macro('dbt_utils.intersect') }} + {{ adapter.dispatch('intersect', packages = dbt_utils._get_utils_namespaces())() }} {% endmacro %} diff --git a/macros/cross_db_utils/last_day.sql b/macros/cross_db_utils/last_day.sql index 0efe6257..1a58b51b 100644 --- a/macros/cross_db_utils/last_day.sql +++ b/macros/cross_db_utils/last_day.sql @@ -4,7 +4,7 @@ testing is required to validate that it will work on other dateparts. */ {% macro last_day(date, datepart) %} - {{ adapter_macro('dbt_utils.last_day', date, datepart) }} + {{ adapter.dispatch('last_day', packages = dbt_utils._get_utils_namespaces()) (date, datepart) }} {% endmacro %} diff --git a/macros/cross_db_utils/length.sql b/macros/cross_db_utils/length.sql index a2bfc8f0..a2595d2e 100644 --- a/macros/cross_db_utils/length.sql +++ b/macros/cross_db_utils/length.sql @@ -1,5 +1,5 @@ {% macro length(expression) -%} - {{ adapter_macro('dbt_utils.length', expression) }} + {{ adapter.dispatch('length', packages = dbt_utils._get_utils_namespaces()) (expression) }} {% endmacro %} diff --git a/macros/cross_db_utils/literal.sql b/macros/cross_db_utils/literal.sql index 2d29b36f..b486fc2f 100644 --- a/macros/cross_db_utils/literal.sql +++ b/macros/cross_db_utils/literal.sql @@ -1,6 +1,6 @@ {%- macro string_literal(value) -%} - {{ adapter_macro('dbt_utils.string_literal', value) }} + {{ adapter.dispatch('string_literal', packages = dbt_utils._get_utils_namespaces()) (value) }} {%- endmacro -%} {% macro default__string_literal(value) -%} diff --git a/macros/cross_db_utils/position.sql b/macros/cross_db_utils/position.sql index df4a58bb..a5b70c3a 100644 --- a/macros/cross_db_utils/position.sql +++ b/macros/cross_db_utils/position.sql @@ -1,5 +1,5 @@ {% macro position(substring_text, string_text) -%} - {{ adapter_macro('dbt_utils.position', substring_text, string_text) }} + {{ adapter.dispatch('position', packages = dbt_utils._get_utils_namespaces()) (substring_text, string_text) }} {% endmacro %} diff --git a/macros/cross_db_utils/replace.sql b/macros/cross_db_utils/replace.sql index 00b2837d..f1dc363d 100644 --- a/macros/cross_db_utils/replace.sql +++ b/macros/cross_db_utils/replace.sql @@ -1,5 +1,5 @@ {% macro replace(field, old_chars, new_chars) -%} - {{ adapter_macro('dbt_utils.replace', field, old_chars, new_chars) }} + {{ adapter.dispatch('replace', packages = dbt_utils._get_utils_namespaces()) (field, old_chars, new_chars) }} {% endmacro %} diff --git a/macros/cross_db_utils/right.sql b/macros/cross_db_utils/right.sql index 5a2d5226..992b848b 100644 --- a/macros/cross_db_utils/right.sql +++ b/macros/cross_db_utils/right.sql @@ -1,5 +1,5 @@ {% macro right(string_text, length_expression) -%} - {{ adapter_macro('dbt_utils.right', string_text, length_expression) }} + {{ adapter.dispatch('right', packages = dbt_utils._get_utils_namespaces()) (string_text, length_expression) }} {% endmacro %} {% macro default__right(string_text, length_expression) %} diff --git a/macros/cross_db_utils/safe_cast.sql b/macros/cross_db_utils/safe_cast.sql index 592a0d39..1c3ef532 100644 --- a/macros/cross_db_utils/safe_cast.sql +++ b/macros/cross_db_utils/safe_cast.sql @@ -1,5 +1,5 @@ {% macro safe_cast(field, type) %} - {{ adapter_macro('dbt_utils.safe_cast', field, type) }} + {{ adapter.dispatch('safe_cast', packages = dbt_utils._get_utils_namespaces()) (field, type) }} {% endmacro %} diff --git a/macros/cross_db_utils/split_part.sql b/macros/cross_db_utils/split_part.sql index 6434cf91..0f1b2f22 100644 --- a/macros/cross_db_utils/split_part.sql +++ b/macros/cross_db_utils/split_part.sql @@ -1,5 +1,5 @@ {% macro split_part(string_text, delimiter_text, part_number) %} - {{ adapter_macro('dbt_utils.split_part', string_text, delimiter_text, part_number) }} + {{ adapter.dispatch('split_part', packages = dbt_utils._get_utils_namespaces()) (string_text, delimiter_text, part_number) }} {% endmacro %} diff --git a/macros/cross_db_utils/width_bucket.sql b/macros/cross_db_utils/width_bucket.sql index 39e0b0b6..8e714410 100644 --- a/macros/cross_db_utils/width_bucket.sql +++ b/macros/cross_db_utils/width_bucket.sql @@ -1,5 +1,5 @@ {% macro width_bucket(expr, min_value, max_value, num_buckets) %} - {{ adapter_macro('dbt_utils.width_bucket', expr, min_value, max_value, num_buckets) }} + {{ adapter.dispatch('width_bucket', packages = dbt_utils._get_utils_namespaces()) (expr, min_value, max_value, num_buckets) }} {% endmacro %} diff --git a/macros/sql/get_relations_by_pattern.sql b/macros/sql/get_relations_by_pattern.sql index 78a766ad..e09a6aba 100644 --- a/macros/sql/get_relations_by_pattern.sql +++ b/macros/sql/get_relations_by_pattern.sql @@ -2,7 +2,7 @@ {%- call statement('get_tables', fetch_result=True) %} - {{ dbt_utils.get_tables_by_pattern(schema_pattern, table_pattern, exclude, database) }} + {{ dbt_utils.get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude, database) }} {%- endcall -%} @@ -20,4 +20,16 @@ {{ return([]) }} {%- endif -%} -{% endmacro %} \ No newline at end of file +{% endmacro %} + +{% macro get_tables_by_pattern(schema_pattern, table_pattern, exclude='', database=target.database) %} + {%- set error_message = ' + Warning: the `get_tables_by_pattern` macro is no longer supported and will be deprecated in a future release of dbt-utils. \ + Use the `get_relations_by_prefix` macro instead. \ + The {}.{} model triggered this warning. \ + '.format(model.package_name, model.name) -%} + {%- do exceptions.warn(error_message) -%} + + {{ return(dbt_utils.get_relations_by_pattern(schema_pattern, table_pattern, exclude='', database=target.database)) }} + +{% endmacro %} diff --git a/macros/sql/get_tables_by_pattern.sql b/macros/sql/get_tables_by_pattern.sql deleted file mode 100644 index ad388ca4..00000000 --- a/macros/sql/get_tables_by_pattern.sql +++ /dev/null @@ -1,12 +0,0 @@ -{% macro get_tables_by_pattern(schema_pattern, table_pattern, exclude='', database=target.database) %} - - select distinct - table_schema as "table_schema", table_name as "table_name" - from {{database}}.information_schema.tables - where table_schema ilike '{{ schema_pattern }}' - and table_name ilike '{{ table_pattern }}' - and table_name not ilike '{{ exclude }}' - -{% endmacro %} - - diff --git a/macros/sql/get_tables_by_pattern_sql.sql b/macros/sql/get_tables_by_pattern_sql.sql new file mode 100644 index 00000000..df47aa23 --- /dev/null +++ b/macros/sql/get_tables_by_pattern_sql.sql @@ -0,0 +1,28 @@ +{% macro get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude='', database=target.database) %} + {{ adapter.dispatch('get_tables_by_pattern_sql', packages = dbt_utils._get_utils_namespaces()) + (schema_pattern, table_pattern, exclude='', database=target.database) }} +{% endmacro %} + +{% macro default__get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude='', database=target.database) %} + + select distinct + table_schema as "table_schema", table_name as "table_name" + from {{database}}.information_schema.tables + where table_schema ilike '{{ schema_pattern }}' + and table_name ilike '{{ table_pattern }}' + and table_name not ilike '{{ exclude }}' + +{% endmacro %} + + +{% macro bigquery__get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude='', database=target.database) %} + + select distinct + table_schema, table_name + + from {{adapter.quote(database)}}.{{schema}}.INFORMATION_SCHEMA.TABLES + where table_schema = '{{schema_pattern}}' + and lower(table_name) like lower ('{{table_pattern}}') + and lower(table_name) not like lower ('{{exclude}}') + +{% endmacro %} diff --git a/macros/sql/get_tables_by_prefix_sql.sql b/macros/sql/get_tables_by_prefix_sql.sql index 2f70bf95..46e52221 100644 --- a/macros/sql/get_tables_by_prefix_sql.sql +++ b/macros/sql/get_tables_by_prefix_sql.sql @@ -1,27 +1,10 @@ {% macro get_tables_by_prefix_sql(schema, prefix, exclude='', database=target.database) %} - {{ adapter_macro('dbt_utils.get_tables_by_prefix_sql', schema, prefix, exclude, database) }} -{% endmacro %} - -{% macro default__get_tables_by_prefix_sql(schema, prefix, exclude='', database=target.database) %} - - select distinct - table_schema as "table_schema", table_name as "table_name" - from {{database}}.information_schema.tables - where table_schema ilike '{{ schema }}' - and table_name ilike '{{ prefix }}%' - and table_name not ilike '{{ exclude }}' - -{% endmacro %} - - -{% macro bigquery__get_tables_by_prefix_sql(schema, prefix, exclude='', database=target.database) %} - select distinct - table_schema, table_name - - from {{adapter.quote(database)}}.{{schema}}.INFORMATION_SCHEMA.TABLES - where table_schema = '{{schema}}' - and lower(table_name) like lower ('{{prefix}}%') - and lower(table_name) not like lower ('{{exclude}}') - + {{ dbt_utils.get_tables_by_pattern_sql( + schema_pattern = schema, + table_pattern = prefix ~ '%', + exclude = exclude, + database = database + ) }} + {% endmacro %} diff --git a/run_test.sh b/run_test.sh index 737e571a..6bede664 100755 --- a/run_test.sh +++ b/run_test.sh @@ -6,7 +6,7 @@ if [[ ! -f $VENV ]]; then . $VENV pip install --upgrade pip setuptools - pip install "dbt>=0.17.0,<0.18.0" + pip install "dbt==0.18.0rc1" fi . $VENV