feat(bigquery): only add 'bytes_processed' column in case the adapter…

… type is BigQuery. Remove as well unncessary helper logs
brooklyn-data · Aug 28, 2022 · 15ef765 · 15ef765
1 parent 66443f5
commit 15ef765
Show file tree

Hide file tree

Showing 8 changed files with 16 additions and 6 deletions.
diff --git a/macros/create_model_executions_table_if_not_exists.sql b/macros/create_model_executions_table_if_not_exists.sql
@@ -28,7 +28,6 @@
         query_completed_at TIMESTAMP,
         total_node_runtime DOUBLE,
         rows_affected INTEGER,
-        bytes_processed INTEGER,
         materialization STRING,
         schema STRING,
         name STRING
@@ -48,7 +47,6 @@
         query_completed_at TIMESTAMP_TZ,
         total_node_runtime DOUBLE,
         rows_affected INTEGER,
-        bytes_processed INTEGER,
         materialization STRING,
         schema STRING,
         name STRING

diff --git a/macros/upload_model_executions.sql b/macros/upload_model_executions.sql
@@ -65,7 +65,6 @@
 
                 {{ model.execution_time }}, {# total_node_runtime #}
                 null, -- rows_affected not available {# Only available in BigQuery #}
-                null, -- bytes_processed not available {# Only available in BigQuery #}
                 '{{ model.node.config.materialized }}', {# materialization #}
                 '{{ model.node.schema }}', {# schema #}
                 '{{ model.node.name }}' {# name #}

diff --git a/macros/upload_results.sql b/macros/upload_results.sql
@@ -15,7 +15,6 @@
             {% do log("Uploading seed executions", true) %}
             {% set src_dbt_seed_executions = source('dbt_artifacts', 'seed_executions') %}
             {% set content_seed_executions = dbt_artifacts.upload_seed_executions(results) %}
-            {% do log("FROUFROU " ~ content_seed_executions) %}
             {{ dbt_artifacts.insert_into_metadata_table(
                 database_name=src_dbt_seed_executions.database,
                 schema_name=src_dbt_seed_executions.schema,

diff --git a/macros/upload_seed_executions.sql b/macros/upload_seed_executions.sql
@@ -122,6 +122,7 @@
                 '{{ model.node.schema }}', {# schema #}
                 '{{ model.node.name }}' {# name #}
             )
+            {%- if not loop.last %},{%- endif %}
         {%- endfor %}
         {% endset %}
         {{ seed_execution_values }}

diff --git a/macros/upload_seeds.sql b/macros/upload_seeds.sql
@@ -3,7 +3,6 @@
     {% for node in graph.nodes.values() | selectattr("resource_type", "equalto", "seed") %}
         {% do seeds.append(node) %}
     {% endfor %}
-    {% do log("TUTU " ~ seeds) %}
     {{ return(adapter.dispatch('get_seeds_dml_sql', 'dbt_artifacts')(seeds)) }}
 {%- endmacro %}
 

diff --git a/models/dim_dbt__current_models.sql b/models/dim_dbt__current_models.sql
@@ -24,7 +24,9 @@ latest_models_runs as (
         , model_executions.query_completed_at
         , model_executions.total_node_runtime
         , model_executions.rows_affected
+        {% if target.type == 'bigquery' %}
         , model_executions.bytes_processed
+        {% endif %}
         , row_number() over (
             partition by latest_models.node_id, model_executions.was_full_refresh
             order by model_executions.query_completed_at desc /* most recent ranked first */
@@ -40,11 +42,15 @@ latest_model_stats as (
         , max(case when was_full_refresh then query_completed_at end) as last_full_refresh_run_completed_at
         , max(case when was_full_refresh then total_node_runtime end) as last_full_refresh_run_total_runtime
         , max(case when was_full_refresh then rows_affected end) as last_full_refresh_run_rows_affected
+        {% if target.type == 'bigquery' %}
         , max(case when was_full_refresh then bytes_processed end) as last_full_refresh_run_bytes_processed
+        {% endif %}
         , max(query_completed_at) as last_run_completed_at
         , max(total_node_runtime) as last_run_total_runtime
         , max(rows_affected) as last_run_rows_affected
+        {% if target.type == 'bigquery' %}
         , max(bytes_processed) as last_run_bytes_processed
+        {% endif %}
     from latest_models_runs
     where run_idx = 1
     group by 1
@@ -56,11 +62,15 @@ final as (
         , latest_model_stats.last_full_refresh_run_completed_at
         , latest_model_stats.last_full_refresh_run_total_runtime
         , latest_model_stats.last_full_refresh_run_rows_affected
+        {% if target.type == 'bigquery' %}
         , latest_model_stats.last_full_refresh_run_bytes_processed
+        {% endif %}
         , latest_model_stats.last_run_completed_at
         , latest_model_stats.last_run_total_runtime
         , latest_model_stats.last_run_rows_affected
+        {% if target.type == 'bigquery' %}
         , latest_model_stats.last_run_bytes_processed
+        {% endif %}
     from latest_models
     left join latest_model_stats
         on latest_models.node_id = latest_model_stats.node_id

diff --git a/models/fct_dbt__model_executions.sql b/models/fct_dbt__model_executions.sql
@@ -19,7 +19,9 @@ model_executions as (
         query_completed_at,
         total_node_runtime,
         rows_affected,
+        {% if target.type == 'bigquery' %}
         bytes_processed,
+        {% endif %}
         materialization,
         schema,
         name

diff --git a/models/staging/stg_dbt__model_executions.sql b/models/staging/stg_dbt__model_executions.sql
@@ -19,7 +19,9 @@ enhanced as (
         query_completed_at,
         total_node_runtime,
         rows_affected,
-        bytes_processed,
+        {% if target.type == 'bigquery' %}
+            bytes_processed,
+        {% endif %}
         materialization,
         schema,
         name