Support for BigQuery adapter

.github/workflows/ci_test_package.yml

@@ -162,4 +162,3 @@ jobs:
           title: "SQLFluff Lint"
           input: "./annotations.json"
 
-

.gitignore

@@ -9,3 +9,5 @@ logs/
 Pipfile
 Pipfile.lock
 env.sh
+
+.python_version

README.md

@@ -36,7 +36,10 @@ packages:
 3. Add an on-run-end hook to your `dbt_project.yml`: `on-run-end: "{{ dbt_artifacts.upload_results(results) }}"`
 (We recommend adding a conditional here so that the upload only occurs in your production environment, such as `on-run-end: "{% if target.name == 'prod' %}{{ dbt_artifacts.upload_results(results) }}{% endif %}"`)
 
-4. Create the tables dbt_artifacts uploads to with `dbt run-operation create_dbt_artifacts_tables`
+4. In case you don't want the artifacts to be uploaded for certain dbt commands (for instance 'dbt compile'), add the following conditional:
+`on-run-end: "{% if not var('dbt_artifacts_upload_results', False) %}{{ dbt_artifacts.upload_results(results) }}{% endif %}"`
+If you then run the following command by providing a variable to the command, the 'on-run-end' will not execute:
+`dbt compile --vars 'artifact: disable'`
 
 5. Run your project!
 
@@ -56,6 +59,7 @@ models:
     +schema: your_destination_schema # optional, default is your target database
     staging:
       +schema: your_destination_schema # optional, default is your target schema
+  ...
 ```
 
 Note that the model materializations are defined in this package's `dbt_project.yml`, so do not set them in your project.
@@ -145,6 +149,7 @@ pipx install tox
 ```
 tox -e integration_snowflake # For the Snowflake tests
 tox -e integration_databricks # For the Databricks tests
+tox -e integration_bigquery # For the BigQuery tests
 ```
 
 The Spark tests require installing the [ODBC driver](https://www.databricks.com/spark/odbc-drivers-download). On a Mac, DBT_ENV_SPARK_DRIVER_PATH should be set to `/Library/simba/spark/lib/libsparkodbc_sbu.dylib`. Spark tests have not yet been added to the integration tests.

dbt_project.yml

@@ -4,6 +4,11 @@ config-version: 2
 require-dbt-version: ">=1.0.0"
 profile: "dbt_artifacts"
 
+clean-targets: # folders to be removed by `dbt clean`
+  - "target"
+  - "dbt_packages"
+  - "logs"
+
 models:
   dbt_artifacts:
     +materialized: view

macros/create_exposures_table_if_not_exists.sql

@@ -51,6 +51,23 @@
     )
 {%- endmacro %}
 
+{% macro bigquery__get_create_exposures_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
+    create table {{database_name}}.{{schema_name}}.{{table_name}} (
+        command_invocation_id STRING,
+        node_id STRING,
+        run_started_at TIMESTAMP,
+        name STRING,
+        type STRING,
+        owner JSON,
+        maturity STRING,
+        path STRING,
+        description STRING,
+        url STRING,
+        package_name STRING,
+        depends_on_nodes ARRAY<STRING>
+    )
+{%- endmacro %}
+
 {% macro default__get_create_exposures_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
     create table {{database_name}}.{{schema_name}}.{{table_name}} (
         command_invocation_id STRING,

macros/create_invocations_table_if_not_exists.sql

@@ -61,6 +61,28 @@
     )
 {%- endmacro %}
 
+{% macro bigquery__get_create_invocations_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
+    create table {{database_name}}.{{schema_name}}.{{table_name}} (
+        command_invocation_id STRING,
+        dbt_version STRING,
+        project_name STRING,
+        run_started_at TIMESTAMP,
+        dbt_command STRING,
+        full_refresh_flag BOOLEAN,
+        target_profile_name STRING,
+        target_name STRING,
+        target_schema STRING,
+        target_threads INTEGER,
+        dbt_cloud_project_id STRING,
+        dbt_cloud_job_id STRING,
+        dbt_cloud_run_id STRING,
+        dbt_cloud_run_reason_category STRING,
+        dbt_cloud_run_reason STRING,
+        env_vars JSON,
+        dbt_vars JSON
+    )
+{%- endmacro %}
+
 {% macro default__get_create_invocations_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
     create table {{database_name}}.{{schema_name}}.{{table_name}} (
         command_invocation_id STRING,

macros/create_model_executions_table_if_not_exists.sql

@@ -53,6 +53,25 @@
     )
 {%- endmacro %}
 
+{% macro bigquery__get_create_model_executions_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
+    create table {{schema_name}}.{{table_name}} (
+        command_invocation_id STRING,
+        node_id STRING,
+        run_started_at TIMESTAMP,
+        was_full_refresh BOOLEAN,
+        thread_id STRING,
+        status STRING,
+        compile_started_at TIMESTAMP,
+        query_completed_at TIMESTAMP,
+        total_node_runtime FLOAT64,
+        rows_affected INTEGER,
+        bytes_processed INTEGER,
+        materialization STRING,
+        schema STRING,
+        name STRING
+    )
+{%- endmacro %}
+
 {% macro default__get_create_model_executions_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
     create table {{database_name}}.{{schema_name}}.{{table_name}} (
         command_invocation_id STRING,

macros/create_models_table_if_not_exists.sql

@@ -49,6 +49,22 @@
     )
 {%- endmacro %}
 
+{% macro bigquery__get_create_models_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
+    create table {{database_name}}.{{schema_name}}.{{table_name}} (
+        command_invocation_id STRING,
+        node_id STRING,
+        run_started_at TIMESTAMP,
+        database STRING,
+        schema STRING,
+        name STRING,
+        depends_on_nodes ARRAY<STRING>,
+        package_name STRING,
+        path STRING,
+        checksum STRING,
+        materialization STRING
+    )
+{%- endmacro %}
+
 {% macro default__get_create_models_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
     create table {{database_name}}.{{schema_name}}.{{table_name}} (
         command_invocation_id STRING,

macros/create_seed_executions_table_if_not_exists.sql

@@ -53,6 +53,24 @@
     )
 {%- endmacro %}
 
+{% macro bigquery__get_create_seed_executions_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
+    create table {{database_name}}.{{schema_name}}.{{table_name}} (
+        command_invocation_id STRING,
+        node_id STRING,
+        run_started_at TIMESTAMP,
+        was_full_refresh BOOLEAN,
+        thread_id STRING,
+        status STRING,
+        compile_started_at TIMESTAMP,
+        query_completed_at TIMESTAMP,
+        total_node_runtime FLOAT64,
+        rows_affected INTEGER,
+        materialization STRING,
+        schema STRING,
+        name STRING
+    )
+{%- endmacro %}
+
 {% macro default__get_create_seed_executions_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
     create table {{database_name}}.{{schema_name}}.{{table_name}} (
         command_invocation_id STRING,

macros/create_seeds_table_if_not_exists.sql

@@ -45,6 +45,20 @@
     )
 {%- endmacro %}
 
+{% macro bigquery__get_create_seeds_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
+    create table {{database_name}}.{{schema_name}}.{{table_name}} (
+        command_invocation_id STRING,
+        node_id STRING,
+        run_started_at TIMESTAMP,
+        database STRING,
+        schema STRING,
+        name STRING,
+        package_name STRING,
+        path STRING,
+        checksum STRING
+    )
+{%- endmacro %}
+
 {% macro default__get_create_seeds_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
     create table {{database_name}}.{{schema_name}}.{{table_name}} (
         command_invocation_id STRING,

macros/create_snapshot_executions_table_if_not_exists.sql

@@ -53,6 +53,24 @@
     )
 {%- endmacro %}
 
+{% macro bigquery__get_create_snapshot_executions_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
+    create or replace table {{schema_name}}.{{table_name}} (
+        command_invocation_id STRING,
+        node_id STRING,
+        run_started_at TIMESTAMP,
+        was_full_refresh BOOLEAN,
+        thread_id STRING,
+        status STRING,
+        compile_started_at TIMESTAMP,
+        query_completed_at TIMESTAMP,
+        total_node_runtime FLOAT64,
+        rows_affected INTEGER,
+        materialization STRING,
+        schema STRING,
+        name STRING
+    )
+{%- endmacro %}
+
 {% macro default__get_create_snapshot_executions_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
     create table {{database_name}}.{{schema_name}}.{{table_name}} (
         command_invocation_id STRING,

macros/create_snapshots_table_if_not_exists.sql

@@ -49,6 +49,22 @@
     )
 {%- endmacro %}
 
+{% macro bigquery__get_create_snapshots_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
+    create table {{database_name}}.{{schema_name}}.{{table_name}} (
+        command_invocation_id STRING,
+        node_id STRING,
+        run_started_at TIMESTAMP,
+        database STRING,
+        schema STRING,
+        name STRING,
+        depends_on_nodes ARRAY<STRING>,
+        package_name STRING,
+        path STRING,
+        checksum STRING,
+        strategy STRING
+    )
+{%- endmacro %}
+
 {% macro default__get_create_snapshots_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
     create table {{database_name}}.{{schema_name}}.{{table_name}} (
         command_invocation_id STRING,

macros/create_sources_table_if_not_exists.sql

@@ -49,6 +49,22 @@
     )
 {%- endmacro %}
 
+{% macro bigquery__get_create_sources_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
+    create table {{database_name}}.{{schema_name}}.{{table_name}} (
+        command_invocation_id STRING,
+        node_id STRING,
+        run_started_at TIMESTAMP,
+        database STRING,
+        schema STRING,
+        source_name STRING,
+        loader STRING,
+        name STRING,
+        identifier STRING,
+        loaded_at_field STRING,
+        freshness JSON
+    )
+{%- endmacro %}
+
 {% macro default__get_create_sources_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
     create table {{database_name}}.{{schema_name}}.{{table_name}} (
         command_invocation_id STRING,

macros/create_test_executions_table_if_not_exists.sql

@@ -49,6 +49,22 @@
     )
 {%- endmacro %}
 
+{% macro bigquery__get_create_test_executions_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
+    create table {{database_name}}.{{schema_name}}.{{table_name}} (
+        command_invocation_id STRING,
+        node_id STRING,
+        run_started_at TIMESTAMP,
+        was_full_refresh BOOLEAN,
+        thread_id STRING,
+        status STRING,
+        compile_started_at TIMESTAMP,
+        query_completed_at TIMESTAMP,
+        total_node_runtime FLOAT64,
+        rows_affected INTEGER,
+        failures INTEGER
+    )
+{%- endmacro %}
+
 {% macro default__get_create_test_executions_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
     create table {{database_name}}.{{schema_name}}.{{table_name}} (
         command_invocation_id STRING,

macros/create_tests_table_if_not_exists.sql

@@ -43,6 +43,19 @@
     )
 {%- endmacro %}
 
+{% macro bigquery__get_create_tests_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
+    create table {{database_name}}.{{schema_name}}.{{table_name}} (
+        command_invocation_id STRING,
+        node_id STRING,
+        run_started_at TIMESTAMP,
+        name STRING,
+        depends_on_nodes ARRAY<STRING>,
+        package_name STRING,
+        test_path STRING,
+        tags ARRAY<STRING>
+    )
+{%- endmacro %}
+
 {% macro default__get_create_tests_table_if_not_exists_statement(database_name, schema_name, table_name) -%}
     create table {{database_name}}.{{schema_name}}.{{table_name}} (
         command_invocation_id STRING,

macros/insert_into_metadata_table.sql

@@ -1,5 +1,7 @@
 {% macro insert_into_metadata_table(database_name, schema_name, table_name, content) -%}
-    {{ return(adapter.dispatch('insert_into_metadata_table', 'dbt_artifacts')(database_name, schema_name, table_name, content)) }}
+    {% if content != "" %}
+        {{ return(adapter.dispatch('insert_into_metadata_table', 'dbt_artifacts')(database_name, schema_name, table_name, content)) }}
+    {% endif %}
 {%- endmacro %}
 
 {% macro spark__insert_into_metadata_table(database_name, schema_name, table_name, content) -%}
@@ -19,3 +21,16 @@
 
     {% do run_query(insert_into_table_query) %}
 {%- endmacro %}
+
+{% macro bigquery__insert_into_metadata_table(database_name, schema_name, table_name, content) -%}
+
+        {% set insert_into_table_query %}
+        insert into {{database_name}}.{{ schema_name }}.{{ table_name }}
+        VALUES
+        {{ content }}
+        {% endset %}
+
+        {% do run_query(insert_into_table_query) %}
+
+{%- endmacro %}
+

macros/parse_json.sql

@@ -9,3 +9,8 @@
 {% macro snowflake__parse_json(field) -%}
     parse_json({{ field }})
 {%- endmacro %}
+
+{% macro bigquery__parse_json(field) -%}
+    parse_json({{ field }})
+{%- endmacro %}
+