Support for BigQuery adapter

.github/workflows/ci_test_package.yml

@@ -162,4 +162,3 @@ jobs:
           title: "SQLFluff Lint"
           input: "./annotations.json"
 
-

.gitignore

@@ -9,3 +9,5 @@ logs/
 Pipfile
 Pipfile.lock
 env.sh
+
+.python_version

README.md

@@ -56,6 +56,7 @@ models:
     +schema: your_destination_schema # optional, default is your target database
     staging:
       +schema: your_destination_schema # optional, default is your target schema
+  ...
 ```
 
 Note that the model materializations are defined in this package's `dbt_project.yml`, so do not set them in your project.
@@ -145,6 +146,7 @@ pipx install tox
 ```
 tox -e integration_snowflake # For the Snowflake tests
 tox -e integration_databricks # For the Databricks tests
+tox -e integration_bigquery # For the BigQuery tests
 ```
 
 The Spark tests require installing the [ODBC driver](https://www.databricks.com/spark/odbc-drivers-download). On a Mac, DBT_ENV_SPARK_DRIVER_PATH should be set to `/Library/simba/spark/lib/libsparkodbc_sbu.dylib`. Spark tests have not yet been added to the integration tests.

dbt_project.yml

@@ -4,6 +4,11 @@ config-version: 2
 require-dbt-version: ">=1.0.0"
 profile: "dbt_artifacts"
 
+clean-targets: # folders to be removed by `dbt clean`
+  - "target"
+  - "dbt_packages"
+  - "logs"
+
 models:
   dbt_artifacts:
     +materialized: view

macros/create_exposures_table_if_not_exists.sql

@@ -51,6 +51,23 @@
     )
 {%- endmacro %}
 
+{% macro bigquery__get_create_exposures_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
+    create table {{database_name}}.{{schema_name}}.{{table_name}} (
+        command_invocation_id STRING,
+        node_id STRING,
+        run_started_at TIMESTAMP,
+        name STRING,
+        type STRING,
+        owner JSON,
+        maturity STRING,
+        path STRING,
+        description STRING,
+        url STRING,
+        package_name STRING,
+        depends_on_nodes ARRAY<STRING>
+    )
+{%- endmacro %}
+
 {% macro default__get_create_exposures_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
     create table {{database_name}}.{{schema_name}}.{{table_name}} (
         command_invocation_id STRING,

macros/create_invocations_table_if_not_exists.sql

@@ -61,6 +61,28 @@
     )
 {%- endmacro %}
 
+{% macro bigquery__get_create_invocations_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
+    create table {{database_name}}.{{schema_name}}.{{table_name}} (
+        command_invocation_id STRING,
+        dbt_version STRING,
+        project_name STRING,
+        run_started_at TIMESTAMP,
+        dbt_command STRING,
+        full_refresh_flag BOOLEAN,
+        target_profile_name STRING,
+        target_name STRING,
+        target_schema STRING,
+        target_threads INTEGER,
+        dbt_cloud_project_id STRING,
+        dbt_cloud_job_id STRING,
+        dbt_cloud_run_id STRING,
+        dbt_cloud_run_reason_category STRING,
+        dbt_cloud_run_reason STRING,
+        env_vars JSON,
+        dbt_vars JSON
+    )
+{%- endmacro %}
+
 {% macro default__get_create_invocations_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
     create table {{database_name}}.{{schema_name}}.{{table_name}} (
         command_invocation_id STRING,

macros/create_model_executions_table_if_not_exists.sql

@@ -53,6 +53,25 @@
     )
 {%- endmacro %}
 
+{% macro bigquery__get_create_model_executions_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
+    create table {{schema_name}}.{{table_name}} (
+        command_invocation_id STRING,
+        node_id STRING,
+        run_started_at TIMESTAMP,
+        was_full_refresh BOOLEAN,
+        thread_id STRING,
+        status STRING,
+        compile_started_at TIMESTAMP,
+        query_completed_at TIMESTAMP,
+        total_node_runtime FLOAT64,
+        rows_affected INTEGER,
+        bytes_processed INTEGER,
+        materialization STRING,
+        schema STRING,
+        name STRING
+    )
+{%- endmacro %}
+
 {% macro default__get_create_model_executions_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
     create table {{database_name}}.{{schema_name}}.{{table_name}} (
         command_invocation_id STRING,

macros/create_models_table_if_not_exists.sql

@@ -49,6 +49,22 @@
     )
 {%- endmacro %}
 
+{% macro bigquery__get_create_models_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
+    create table {{database_name}}.{{schema_name}}.{{table_name}} (
+        command_invocation_id STRING,
+        node_id STRING,
+        run_started_at TIMESTAMP,
+        database STRING,
+        schema STRING,
+        name STRING,
+        depends_on_nodes ARRAY<STRING>,
+        package_name STRING,
+        path STRING,
+        checksum STRING,
+        materialization STRING
+    )
+{%- endmacro %}
+
 {% macro default__get_create_models_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
     create table {{database_name}}.{{schema_name}}.{{table_name}} (
         command_invocation_id STRING,

macros/create_seed_executions_table_if_not_exists.sql

@@ -53,6 +53,24 @@
     )
 {%- endmacro %}
 
+{% macro bigquery__get_create_seed_executions_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
+    create table {{database_name}}.{{schema_name}}.{{table_name}} (
+        command_invocation_id STRING,
+        node_id STRING,
+        run_started_at TIMESTAMP,
+        was_full_refresh BOOLEAN,
+        thread_id STRING,
+        status STRING,
+        compile_started_at TIMESTAMP,
+        query_completed_at TIMESTAMP,
+        total_node_runtime FLOAT64,
+        rows_affected INTEGER,
+        materialization STRING,
+        schema STRING,
+        name STRING
+    )
+{%- endmacro %}
+
 {% macro default__get_create_seed_executions_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
     create table {{database_name}}.{{schema_name}}.{{table_name}} (
         command_invocation_id STRING,

macros/create_seeds_table_if_not_exists.sql

@@ -45,6 +45,20 @@
     )
 {%- endmacro %}
 
+{% macro bigquery__get_create_seeds_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
+    create table {{database_name}}.{{schema_name}}.{{table_name}} (
+        command_invocation_id STRING,
+        node_id STRING,
+        run_started_at TIMESTAMP,
+        database STRING,
+        schema STRING,
+        name STRING,
+        package_name STRING,
+        path STRING,
+        checksum STRING
+    )
+{%- endmacro %}
+
 {% macro default__get_create_seeds_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
     create table {{database_name}}.{{schema_name}}.{{table_name}} (
         command_invocation_id STRING,

macros/create_snapshot_executions_table_if_not_exists.sql

@@ -53,6 +53,24 @@
     )
 {%- endmacro %}
 
+{% macro bigquery__get_create_snapshot_executions_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
+    create or replace table {{schema_name}}.{{table_name}} (
+        command_invocation_id STRING,
+        node_id STRING,
+        run_started_at TIMESTAMP,
+        was_full_refresh BOOLEAN,
+        thread_id STRING,
+        status STRING,
+        compile_started_at TIMESTAMP,
+        query_completed_at TIMESTAMP,
+        total_node_runtime FLOAT64,
+        rows_affected INTEGER,
+        materialization STRING,
+        schema STRING,
+        name STRING
+    )
+{%- endmacro %}
+
 {% macro default__get_create_snapshot_executions_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
     create table {{database_name}}.{{schema_name}}.{{table_name}} (
         command_invocation_id STRING,

macros/create_snapshots_table_if_not_exists.sql

@@ -49,6 +49,22 @@
     )
 {%- endmacro %}
 
+{% macro bigquery__get_create_snapshots_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
+    create table {{database_name}}.{{schema_name}}.{{table_name}} (
+        command_invocation_id STRING,
+        node_id STRING,
+        run_started_at TIMESTAMP,
+        database STRING,
+        schema STRING,
+        name STRING,
+        depends_on_nodes ARRAY<STRING>,
+        package_name STRING,
+        path STRING,
+        checksum STRING,
+        strategy STRING
+    )
+{%- endmacro %}
+
 {% macro default__get_create_snapshots_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
     create table {{database_name}}.{{schema_name}}.{{table_name}} (
         command_invocation_id STRING,

macros/create_sources_table_if_not_exists.sql

@@ -49,6 +49,22 @@
     )
 {%- endmacro %}
 
+{% macro bigquery__get_create_sources_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
+    create table {{database_name}}.{{schema_name}}.{{table_name}} (
+        command_invocation_id STRING,
+        node_id STRING,
+        run_started_at TIMESTAMP,
+        database STRING,
+        schema STRING,
+        source_name STRING,
+        loader STRING,
+        name STRING,
+        identifier STRING,
+        loaded_at_field STRING,
+        freshness JSON
+    )
+{%- endmacro %}
+
 {% macro default__get_create_sources_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
     create table {{database_name}}.{{schema_name}}.{{table_name}} (
         command_invocation_id STRING,

macros/create_test_executions_table_if_not_exists.sql

@@ -49,6 +49,22 @@
     )
 {%- endmacro %}
 
+{% macro bigquery__get_create_test_executions_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
+    create table {{database_name}}.{{schema_name}}.{{table_name}} (
+        command_invocation_id STRING,
+        node_id STRING,
+        run_started_at TIMESTAMP,
+        was_full_refresh BOOLEAN,
+        thread_id STRING,
+        status STRING,
+        compile_started_at TIMESTAMP,
+        query_completed_at TIMESTAMP,
+        total_node_runtime FLOAT64,
+        rows_affected INTEGER,
+        failures INTEGER
+    )
+{%- endmacro %}
+
 {% macro default__get_create_test_executions_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
     create table {{database_name}}.{{schema_name}}.{{table_name}} (
         command_invocation_id STRING,

macros/create_tests_table_if_not_exists.sql

@@ -43,6 +43,19 @@
     )
 {%- endmacro %}
 
+{% macro bigquery__get_create_tests_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
+    create table {{database_name}}.{{schema_name}}.{{table_name}} (
+        command_invocation_id STRING,
+        node_id STRING,
+        run_started_at TIMESTAMP,
+        name STRING,
+        depends_on_nodes ARRAY<STRING>,
+        package_name STRING,
+        test_path STRING,
+        tags ARRAY<STRING>
+    )
+{%- endmacro %}
+
 {% macro default__get_create_tests_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
     create table {{database_name}}.{{schema_name}}.{{table_name}} (
         command_invocation_id STRING,

macros/insert_into_metadata_table.sql

@@ -1,5 +1,7 @@
 {% macro insert_into_metadata_table(database_name, schema_name, table_name, content) -%}
-    {{ return(adapter.dispatch('insert_into_metadata_table', 'dbt_artifacts')(database_name, schema_name, table_name, content)) }}
+    {% if content != "" %}
+        {{ return(adapter.dispatch('insert_into_metadata_table', 'dbt_artifacts')(database_name, schema_name, table_name, content)) }}
+    {% endif %}
 {%- endmacro %}
 
 {% macro spark__insert_into_metadata_table(database_name, schema_name, table_name, content) -%}
@@ -19,3 +21,16 @@
 
     {% do run_query(insert_into_table_query) %}
 {%- endmacro %}
+
+{% macro bigquery__insert_into_metadata_table(database_name, schema_name, table_name, content) -%}
+
+        {% set insert_into_table_query %}
+        insert into {{database_name}}.{{ schema_name }}.{{ table_name }}
+        VALUES
+        {{ content }}
+        {% endset %}
+
+        {% do run_query(insert_into_table_query) %}
+
+{%- endmacro %}
+

macros/parse_json.sql

@@ -9,3 +9,8 @@
 {% macro snowflake__parse_json(field) -%}
     parse_json({{ field }})
 {%- endmacro %}
+
+{% macro bigquery__parse_json(field) -%}
+    parse_json({{ field }})
+{%- endmacro %}
+

macros/upload_exposures.sql

@@ -1,9 +1,12 @@
 {% macro upload_exposures(graph) -%}
-    {% set src_dbt_exposures = source('dbt_artifacts', 'exposures') %}
     {% set exposures = [] %}
     {% for node in graph.exposures.values() %}
         {% do exposures.append(node) %}
     {% endfor %}
+    {{ return(adapter.dispatch('get_exposures_dml_sql', 'dbt_artifacts')(exposures)) }}
+{%- endmacro %}
+
+{% macro default__get_exposures_dml_sql(exposures) -%}
 
     {% if exposures != [] %}
         {% set exposure_values %}
@@ -39,13 +42,35 @@
             {%- if not loop.last %},{%- endif %}
         {%- endfor %}
         {% endset %}
-
-        {{ dbt_artifacts.insert_into_metadata_table(
-            database_name=src_dbt_exposures.database,
-            schema_name=src_dbt_exposures.schema,
-            table_name=src_dbt_exposures.identifier,
-            content=exposure_values
-            )
-        }}
+        {{ exposure_values }}
+    {% else %}
+        {{ return("") }}
     {% endif %}
 {% endmacro -%}
+
+{% macro bigquery__get_exposures_dml_sql(exposures) -%}
+    {% if exposures != [] %}
+        {% set exposure_values %}
+            {% for exposure in exposures -%}
+                (
+                    '{{ invocation_id }}', {# command_invocation_id #}
+                    '{{ exposure.unique_id | replace("'","\\'") }}', {# node_id #}
+                    '{{ run_started_at }}', {# run_started_at #}
+                    '{{ exposure.name | replace("'","\\'") }}', {# name #}
+                    '{{ exposure.type }}', {# type #}
+                    parse_json('{{ tojson(exposure.owner) | replace("'","\\'") }}'), {# owner #}
+                    '{{ exposure.maturity }}', {# maturity #}
+                    '{{ exposure.original_file_path | replace('\\', '\\\\') }}', {# path #}
+                    '{{ exposure.description | replace("'","\\'") }}', {# description #}
+                    '{{ exposure.url }}', {# url #}
+                    '{{ exposure.package_name }}', {# package_name #}
+                    {{ tojson(exposure.depends_on.nodes) }} {# depends_on_nodes #}
+                )
+                {%- if not loop.last %},{%- endif %}
+            {%- endfor %}
+        {% endset %}
+        {{ exposure_values }}
+    {% else %}
+        {{ return("") }}
+    {% endif %}
+{%- endmacro %}