diff --git a/CHANGELOG.md b/CHANGELOG.md index f72afbd4..805f9ee5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ## Fixes - Enable a negative part_number for `split_part()` ([#557](https://github.com/dbt-labs/dbt-utils/issues/557), [#559](https://github.com/dbt-labs/dbt-utils/pull/559)) +- Make `exclude` case insensitive for `union_relations()` ([#578](https://github.com/dbt-labs/dbt-utils/issues/557), [#587](https://github.com/dbt-labs/dbt-utils/issues/587)) ## Quality of life - Documentation about listagg macro ([#544](https://github.com/dbt-labs/dbt-utils/issues/544), [#560](https://github.com/dbt-labs/dbt-utils/pull/560)) @@ -21,6 +22,7 @@ - [@clausherther](https://github.com/clausherther) (#555) - [@epapineau](https://github.com/epapineau) (#583) - [@b-per](https://github.com/b-per) (#559) +- [@dbeatty10](https://github.com/dbeatty10), [@jeremyyeo](https://github.com/jeremyyeo) (#587) # dbt-utils v0.8.4 ## Fixes diff --git a/integration_tests/data/sql/data_union_exclude_expected.csv b/integration_tests/data/sql/data_union_exclude_expected.csv new file mode 100644 index 00000000..3283e5c6 --- /dev/null +++ b/integration_tests/data/sql/data_union_exclude_expected.csv @@ -0,0 +1,6 @@ +id,favorite_color,favorite_number +1,,pi +2,,e +3,,4 +1,"green",7 +2,"pink",13 diff --git a/integration_tests/models/sql/schema.yml b/integration_tests/models/sql/schema.yml index e5927fbf..df10ccac 100644 --- a/integration_tests/models/sql/schema.yml +++ b/integration_tests/models/sql/schema.yml @@ -163,6 +163,16 @@ models: tests: - dbt_utils.not_constant + - name: test_union_exclude_lowercase + tests: + - dbt_utils.equality: + compare_model: ref('data_union_exclude_expected') + + - name: test_union_exclude_uppercase + tests: + - dbt_utils.equality: + compare_model: ref('data_union_exclude_expected') + - name: test_get_relations_by_pattern tests: - dbt_utils.equality: diff --git a/integration_tests/models/sql/test_union_exclude_base_lowercase.sql b/integration_tests/models/sql/test_union_exclude_base_lowercase.sql new file mode 100644 index 00000000..dc005a8e --- /dev/null +++ b/integration_tests/models/sql/test_union_exclude_base_lowercase.sql @@ -0,0 +1,8 @@ + +{{ dbt_utils.union_relations( + relations=[ + ref('data_union_table_1'), + ref('data_union_table_2'), + ], + exclude=['name'] +) }} diff --git a/integration_tests/models/sql/test_union_exclude_base_uppercase.sql b/integration_tests/models/sql/test_union_exclude_base_uppercase.sql new file mode 100644 index 00000000..af3dc3fa --- /dev/null +++ b/integration_tests/models/sql/test_union_exclude_base_uppercase.sql @@ -0,0 +1,8 @@ + +{{ dbt_utils.union_relations( + relations=[ + ref('data_union_table_1'), + ref('data_union_table_2'), + ], + exclude=['NAME'] +) }} diff --git a/integration_tests/models/sql/test_union_exclude_lowercase.sql b/integration_tests/models/sql/test_union_exclude_lowercase.sql new file mode 100644 index 00000000..95efc06a --- /dev/null +++ b/integration_tests/models/sql/test_union_exclude_lowercase.sql @@ -0,0 +1,4 @@ +select + {{ dbt_utils.star(ref("test_union_exclude_base_lowercase"), except=["_dbt_source_relation"]) }} + +from {{ ref("test_union_exclude_base_lowercase") }} diff --git a/integration_tests/models/sql/test_union_exclude_uppercase.sql b/integration_tests/models/sql/test_union_exclude_uppercase.sql new file mode 100644 index 00000000..3e0ab4f9 --- /dev/null +++ b/integration_tests/models/sql/test_union_exclude_uppercase.sql @@ -0,0 +1,4 @@ +select + {{ dbt_utils.star(ref("test_union_exclude_base_uppercase"), except=["_DBT_SOURCE_RELATION"]) }} + +from {{ ref("test_union_exclude_base_uppercase") }} diff --git a/macros/sql/union.sql b/macros/sql/union.sql index 0d4ebccb..74bbd988 100644 --- a/macros/sql/union.sql +++ b/macros/sql/union.sql @@ -17,6 +17,20 @@ {%- set relation_columns = {} -%} {%- set column_superset = {} -%} + {%- set all_excludes = [] -%} + {%- set all_includes = [] -%} + + {%- if exclude -%} + {%- for exc in exclude -%} + {%- do all_excludes.append(exc | lower) -%} + {%- endfor -%} + {%- endif -%} + + {%- if include -%} + {%- for inc in include -%} + {%- do all_includes.append(inc | lower) -%} + {%- endfor -%} + {%- endif -%} {%- for relation in relations -%} @@ -28,10 +42,10 @@ {%- for col in cols -%} {#- If an exclude list was provided and the column is in the list, do nothing -#} - {%- if exclude and col.column in exclude -%} + {%- if exclude and col.column | lower in all_excludes -%} {#- If an include list was provided and the column is not in the list, do nothing -#} - {%- elif include and col.column not in include -%} + {%- elif include and col.column | lower not in all_includes -%} {#- Otherwise add the column to the column superset -#} {%- else -%}