Feature/add dbt unit tests

.github/pull_request_template.md

@@ -7,4 +7,4 @@ Describe your changes, and why you're making them.
 - [ ] I have verified that these changes work locally
 - [ ] I have updated the README.md (if applicable)
 - [ ] I have added tests & descriptions to my models (and macros if applicable)
-- [ ] I have run `dbt test` and `python -m pytest .` to validate existing tests
+- [ ] I have run `dbt test` to validate existing tests

.github/workflows/run_unit_tests_on_pr.yml

@@ -1,8 +1,20 @@
 name: Run Unit Tests on Pull Request
 
-on: [pull_request_target,workflow_dispatch]
+# on: [pull_request_target,workflow_dispatch]
+on:
+  push:
+    branches:
+      - 'feature/add_dbt_unit_tests'
 env:
   BIGQUERY_PROJECT: ${{ secrets.BIGQUERY_PROJECT }}
+  BIGQUERY_PROPERTY_ID: ${{ secrets.BIGQUERY_PROPERTY_ID }}
+  BIGQUERY_DATASET: ${{ secrets.BIGQUERY_DATASET }}
+  BIGQUERY_KEYFILE: ./unit_tests/dbt-service-account.json
+  GA4_CONVERSION_EVENTS: ${{ vars.GA4_CONVERSION_EVENTS }}
+  GA4_DERIVED_SESSION_PROPERTIES: ${{ vars.GA4_DERIVED_SESSION_PROPERTIES }}
+  GA4_DERIVED_USER_PROPERTIES: ${{ vars.GA4_DERIVED_USER_PROPERTIES }}
+  GA4_INCREMENTAL_DAYS: ${{ vars.GA4_INCREMENTAL_DAYS }}
+  GA4_START_DATE: ${{ vars.GA4_START_DATE }}
 
 jobs:
   pytest_run_all:
@@ -35,3 +47,36 @@ jobs:
 
       - name: Run tests
         run: python -m pytest .
+
+  run_dbt_unit_tests:
+    name: Run dbt Unit Tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - uses: actions/setup-python@v1
+        with:
+          python-version: "3.11.x"
+
+      - name: Authenticate using service account
+        run: 'echo "$KEYFILE" > ./unit_tests/dbt-service-account.json'
+        shell: bash
+        env:
+          KEYFILE: ${{ secrets.GCP_BIGQUERY_USER_KEYFILE }}
+
+      - name: Install dbt
+        run: |
+          pip install dbt-core
+          pip install dbt-bigquery
+          dbt deps
+
+      - name: Materialize necessary dbt resources
+        run: |
+          dbt seed -f
+          dbt run -s +test_type:unit -f --empty
+
+      - name: Run dbt unit tests
+        run: dbt test -s test_type:unit

.gitignore

@@ -2,6 +2,8 @@
 target/
 dbt_packages/
 logs/
+package-lock.yml
+.user.yml
 
 google-cloud-sdk/
 unit_tests/.env

README.md

@@ -304,7 +304,26 @@ gcloud auth application-default login --scopes=https://www.googleapis.com/auth/b
 ```
 # Unit Testing
 
-This package uses `pytest` as a method of unit testing individual models. More details can be found in the [unit_tests/README.md](unit_tests) folder.
+The dbt-ga4 package treats each model and macro as a 'unit' of code. If we fix the input to each unit, we can test that we received the expected output.
+
+This package currently uses a combination of dbt unit tests and `pytest` as a method of unit testing individual models. The remaining `pytest` unit test will be refactored to a dbt unit test when possible - progress on the bug preventing that work can be tracked [here](https://github.com/dbt-labs/dbt-core/issues/10353).
+
+### dbt unit tests
+
+dbt's documentation on unit tests can be found [here](https://docs.getdbt.com/docs/build/unit-tests). Unit tests are performed the same way other types of dbt tests are executed.
+
+Execute a specific test:
+```
+dbt test -s <test_name>
+```
+Execute all tests configured for a model:
+```
+dbt test -s <model_name>
+```
+
+### pytest
+
+More details on using `pytest` for unit testing can be found in the [unit_tests/README.md](unit_tests) folder.
 
 # Overriding Default Channel Groupings
 

dbt_project.yml

@@ -8,6 +8,8 @@ seed-paths: ["seeds"]
 macro-paths: ["macros"]
 snapshot-paths: ["snapshots"]
 
+profile: 'default'
+
 target-path: "target"  # directory which will store compiled SQL files
 clean-targets:         # directories to be removed by `dbt clean`
   - "target"

macros/base_select.sql

@@ -36,7 +36,8 @@
     , ecommerce.transaction_id
     , items
     , {%- if  var('combined_dataset', false) != false %} cast(left(regexp_replace(_table_suffix, r'^(intraday_)?\d{8}', ''), 100) as int64)
-        {%- else %} {{ var('property_ids')[0] }}
+        {%- elif var('property_ids', false) != false %} {{ var('property_ids')[0] }}
+        {%- else %} {{ env_var('BIGQUERY_PROPERTY_ID') }}
         {%- endif %} as property_id
 {% endmacro %}
 

models/marts/core/dim_ga4__sessions_daily.sql

@@ -1,5 +1,5 @@
 {% set partitions_to_replace = ['current_date'] %}
-{% for i in range(var('static_incremental_days')) %}
+{% for i in range(env_var('GA4_INCREMENTAL_DAYS')|int if env_var('GA4_INCREMENTAL_DAYS', false) else var('static_incremental_days')) %}
     {% set partitions_to_replace = partitions_to_replace.append('date_sub(current_date, interval ' + (i+1)|string + ' day)') %}
 {% endfor %}
 {{

models/marts/core/fct_ga4__pages.sql

@@ -1,5 +1,5 @@
 {% set partitions_to_replace = ['current_date'] %}
-{% for i in range(var('static_incremental_days')) %}
+{% for i in range(env_var('GA4_INCREMENTAL_DAYS')|int if env_var('GA4_INCREMENTAL_DAYS', false) else var('static_incremental_days')) %}
     {% set partitions_to_replace = partitions_to_replace.append('date_sub(current_date, interval ' + (i+1)|string + ' day)') %}
 {% endfor %}
 {{

models/marts/core/fct_ga4__sessions_daily.sql

@@ -1,5 +1,5 @@
 {% set partitions_to_replace = ['current_date'] %}
-{% for i in range(var('static_incremental_days')) %}
+{% for i in range(env_var('GA4_INCREMENTAL_DAYS')|int if env_var('GA4_INCREMENTAL_DAYS', false) else var('static_incremental_days')) %}
     {% set partitions_to_replace = partitions_to_replace.append('date_sub(current_date, interval ' + (i+1)|string + ' day)') %}
 {% endfor %}
 {{

models/staging/base/base_ga4__events.sql

@@ -1,5 +1,5 @@
 {% set partitions_to_replace = ['current_date'] %}
-{% for i in range(var('static_incremental_days')) %}
+{% for i in range(env_var('GA4_INCREMENTAL_DAYS')|int if env_var('GA4_INCREMENTAL_DAYS', false) else var('static_incremental_days')) %}
     {% set partitions_to_replace = partitions_to_replace.append('date_sub(current_date, interval ' + (i+1)|string + ' day)') %}
 {% endfor %}
 
@@ -21,9 +21,11 @@ with source as (
     select
         {{ ga4.base_select_source() }}
     from {{ source('ga4', 'events') }}
-    where cast(left(replace(_table_suffix, 'intraday_', ''), 8) as int64) >= {{var('start_date')}}
-    {% if is_incremental() %}
-        and parse_date('%Y%m%d', left(replace(_table_suffix, 'intraday_', ''), 8)) in ({{ partitions_to_replace | join(',') }})
+    {% if not flags.EMPTY %}
+        where cast(left(replace(_table_suffix, 'intraday_', ''), 8) as int64) >= {{ env_var('GA4_START_DATE') if env_var('GA4_START_DATE', false) else var('start_date') }}
+        {% if is_incremental() %}
+            and parse_date('%Y%m%d', left(replace(_table_suffix, 'intraday_', ''), 8)) in ({{ partitions_to_replace | join(',') }})
+        {% endif %}
     {% endif %}
 ),
 renamed as (

models/staging/recommended_events/stg_ga4__event_purchase_deduplicated.sql

@@ -1,6 +1,6 @@
 {% if not flags.FULL_REFRESH %}
     {% set partitions_to_query = ['current_date'] %}
-    {% for i in range(var('static_incremental_days', 1)) %}
+    {% for i in range(env_var('GA4_INCREMENTAL_DAYS')|int if env_var('GA4_INCREMENTAL_DAYS', false) else var('static_incremental_days')) %}
         {% set partitions_to_query = partitions_to_query.append('date_sub(current_date, interval ' + (i+1)|string + ' day)') %}
     {% endfor %}
 {% endif %}

models/staging/src_ga4.yml

@@ -4,11 +4,13 @@ sources:
   - name: ga4
     database: | # Source from target.project if multi-property, otherwise source from source_project
       {%- if  var('combined_dataset', false) != false -%} {{target.project}}
-      {%- else -%} {{var('source_project')}}
+      {%- elif var('source_project', false) != false -%} {{var('source_project')}}
+      {%- else -%} {{env_var('BIGQUERY_PROJECT')}}
       {%- endif -%}
     schema: | # Source from combined property dataset if set, otherwise source from original GA4 property
       {%- if  var('combined_dataset', false) != false -%} {{var('combined_dataset')}}
-      {%- else -%} analytics_{{var('property_ids')[0]}}
+      {%- elif var('property_ids', false) != false -%} analytics_{{var('property_ids')[0]}}
+      {%- else -%} analytics_{{env_var('BIGQUERY_PROPERTY_ID')}}
       {%- endif -%}
     tables:
       - name: events

models/staging/stg_ga4__client_key_first_last_events.yml

@@ -7,4 +7,20 @@ models:
       - name: client_key
         description: Hashed combination of user_pseudo_id and stream_id
         tests: 
-          - unique
+          - unique
+unit_tests:
+  - name: test_stg_ga4__client_key_first_last_events
+    description: Test pulling the first and last event per client key
+    model: stg_ga4__client_key_first_last_events
+    given:
+      - input: ref('stg_ga4__events')
+        format: csv
+        rows: |
+          stream_id,client_key,event_key,event_timestamp
+          1,IX+OyYJBgjwqML19GB/XIQ==,H06dLW6OhNJJ6SoEPFsSyg==,1661339279816517
+          1,IX+OyYJBgjwqML19GB/XIQ==,gt1SoAtrxDv33uDGwVeMVA==,1661339279816518
+    expect:
+      format: csv
+      rows: |
+        client_key,first_event,last_event
+        IX+OyYJBgjwqML19GB/XIQ==,H06dLW6OhNJJ6SoEPFsSyg==,gt1SoAtrxDv33uDGwVeMVA==

models/staging/stg_ga4__client_key_first_last_pageviews.yml

@@ -7,4 +7,20 @@ models:
       - name: client_key
         description: Hashed combination of user_pseudo_id and stream_id
         tests:
-          - unique
+          - unique
+unit_tests:
+  - name: test_stg_ga4__client_key_first_last_pageviews
+    description: Test pulling the first and last page view per client key
+    model: stg_ga4__client_key_first_last_pageviews
+    given:
+      - input: ref('stg_ga4__event_page_view')
+        format: csv
+        rows: |
+          stream_id,client_key,event_key,event_timestamp,page_location
+          1,IX+OyYJBgjwqML19GB/XIQ==,H06dLW6OhNJJ6SoEPFsSyg==,1661339279816517,A
+          1,IX+OyYJBgjwqML19GB/XIQ==,gt1SoAtrxDv33uDGwVeMVA==,1661339279816518,B
+    expect:
+      format: csv
+      rows: |
+        client_key,first_page_view_event_key,last_page_view_event_key,first_page_location,last_page_location
+        IX+OyYJBgjwqML19GB/XIQ==,H06dLW6OhNJJ6SoEPFsSyg==,gt1SoAtrxDv33uDGwVeMVA==,A,B

models/staging/stg_ga4__derived_session_properties.sql

@@ -1,5 +1,5 @@
 {{ config(
-  enabled = true if var('derived_session_properties', false) else false,
+  enabled = true if var('derived_session_properties', false) or env_var('GA4_DERIVED_SESSION_PROPERTIES', false) else false,
   materialized = "table"
 ) }}
 

models/staging/stg_ga4__derived_session_properties.yml

@@ -8,4 +8,39 @@ models:
     columns:
       - name: session_key
         tests:
-          - unique  
+          - unique
+unit_tests:
+  - name: test_derived_session_properties
+    description: Test whether a derived property is successfully retrieved from multiple event payloads
+    model: stg_ga4__derived_session_properties
+    given:
+      - input: ref('stg_ga4__events')
+        format: sql
+        rows: |
+          select 
+          'AAA' as session_key
+          , 1617691790431476 as event_timestamp
+          , 'first_visit' as event_name
+          , ARRAY[STRUCT('my_param' as key, STRUCT(1 as int_value) as value)] as event_params
+          , ARRAY[STRUCT('my_property' as key, STRUCT('value1' as string_value) as value)] as user_properties
+          union all
+          select
+          'AAA' as session_key
+          , 1617691790431477 as event_timestamp
+          , 'first_visit' as event_name
+          , ARRAY[STRUCT('my_param' as key, STRUCT(2 as int_value) as value)] as event_params
+          , ARRAY[] as user_properties
+          union all
+          select 
+          'BBB' as session_key
+          , 1617691790431477 as event_timestamp
+          , 'first_visit' as event_name
+          , ARRAY[STRUCT('my_param' as key, STRUCT(1 as int_value) as value)] as event_params
+          , ARRAY[STRUCT('my_property' as key, STRUCT('value2' as string_value) as value)] as user_properties
+    expect:
+      format: dict
+      rows:
+        - {session_key: AAA, my_derived_property: 2, my_derived_property2: value1}
+        - {session_key: BBB, my_derived_property: 1, my_derived_property2: value2}
+    overrides:
+      vars: {derived_session_properties: [{event_parameter: 'my_param',session_property_name: 'my_derived_property',value_type: 'int_value'},{user_property: 'my_property',session_property_name: 'my_derived_property2',value_type: 'string_value'}]}

models/staging/stg_ga4__derived_session_properties_daily.sql

@@ -1,10 +1,10 @@
 {% set partitions_to_replace = ['current_date'] %}
-{% for i in range(var('static_incremental_days')) %}
+{% for i in range(env_var('GA4_INCREMENTAL_DAYS')|int if env_var('GA4_INCREMENTAL_DAYS', false) else var('static_incremental_days')) %}
     {% set partitions_to_replace = partitions_to_replace.append('date_sub(current_date, interval ' + (i+1)|string + ' day)') %}
 {% endfor %}
 {{
     config(
-        enabled = true if var('derived_session_properties', false) else false,
+        enabled = true if var('derived_session_properties', false) or env_var('GA4_DERIVED_SESSION_PROPERTIES', false) else false,
         materialized = 'incremental',
         incremental_strategy = 'insert_overwrite',
         tags = ["incremental"],

models/staging/stg_ga4__derived_user_properties.sql

@@ -1,5 +1,5 @@
 {{ config(
-  enabled = true if var('derived_user_properties', false) else false,
+  enabled = true if var('derived_user_properties', false) or env_var('GA4_DERIVED_USER_PROPERTIES', false) else false,
   materialized = "table"
 ) }}
 

models/staging/stg_ga4__derived_user_properties.yml

@@ -7,4 +7,36 @@ models:
       - name: client_key
         description: Hashed combination of user_pseudo_id and stream_id
         tests:
-          - unique  
+          - unique  
+unit_tests:
+  - name: test_derived_user_properties
+    description: Test whether a derived user property is successfully retrieved from multiple event payloads
+    model: stg_ga4__derived_user_properties
+    given:
+      - input: ref('stg_ga4__events')
+        format: sql
+        rows: |
+          select 
+          'AAA' as client_key
+          , 1617691790431476 as event_timestamp
+          , 'first_visit' as event_name
+          , ARRAY[STRUCT('my_param' as key, STRUCT(1 as int_value) as value)] as event_params
+          union all
+          select
+          'AAA' as client_key
+          , 1617691790431477 as event_timestamp
+          , 'first_visit' as event_name
+          , ARRAY[STRUCT('my_param' as key, STRUCT(2 as int_value) as value)] as event_params
+          union all
+          select 
+          'BBB' as client_key
+          , 1617691790431477 as event_timestamp
+          , 'first_visit' as event_name
+          , ARRAY[STRUCT('my_param' as key, STRUCT(1 as int_value) as value)] as event_params
+    expect:
+      format: dict
+      rows:
+        - {client_key: AAA, my_derived_property: 2}
+        - {client_key: BBB, my_derived_property: 1}
+    overrides:
+      vars: {derived_user_properties: [{event_parameter: 'my_param',user_property_name: 'my_derived_property',value_type: 'int_value'}]}

models/staging/stg_ga4__event_to_query_string_params.yml

@@ -3,4 +3,23 @@ version: 2
 models:  
   - name: stg_ga4__event_to_query_string_params
     description: This model pivots the query string parameters contained within the event's page_location field to become rows. Each row is a single parameter/value combination contained in a single event's query string.
-
+unit_tests:
+  - name: test_stg_ga4__event_to_query_string_params
+    description: Test whether event query strings are flattened for each query string parameter
+    model: stg_ga4__event_to_query_string_params
+    given:
+      - input: ref('stg_ga4__events')
+        format: csv
+        rows: |
+          event_key,page_query_string
+          aaa,param1=value1&param2=value2
+          bbb,param1
+          ccc,param1=
+    expect:
+      format: csv
+      rows: |
+        event_key,param,value
+        aaa,param1,value1
+        aaa,param2,value2
+        bbb,param1,
+        ccc,param1,