dbt_test.yml

name: Test DBT package

on:
  workflow_dispatch:
  push:
    branches: [ "main", "develop" ]
    paths:
      - macros/**
      - dbt_project.yml
      - .github/workflows/dbt_test.yml

permissions:
  contents: read

defaults:
  run:
    working-directory: integration_tests

jobs:
  test-duckdb:
    name: Test on DuckDB
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.8", "3.9", "3.10", "3.11"]

    env:
      DBT_TARGET: duckdb

    steps:
    - uses: actions/checkout@v4
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}

    - name: pip install
      run: pip install dbt-core duckdb dbt-${DBT_TARGET}

    - name: Install DBT deps
      run: dbt deps --target ${DBT_TARGET}

    - name: load test data
      run: dbt seed --target ${DBT_TARGET}

    - name: run models
      run: dbt run --target ${DBT_TARGET}
    
    - name: dbt test macros
      run: dbt test --target ${DBT_TARGET}

    - name: Create seed tests
      run: bash tests/create_seeds_tests_yml.sh

    - name: dbt test seeds
      run: dbt test --target ${DBT_TARGET} --select config.materialized:seed --exclude "dbt_utils_recency*"

  test-postgres:
    name: Test on Postgres
    runs-on: ubuntu-latest
    container:
      image: python:3.11

    services:
      postgres:
        image: postgres
        env:
          POSTGRES_PASSWORD: postgres

    env:
      DBT_TARGET: postgres

    steps:
    - uses: actions/checkout@v4

    - name: pip install
      run: pip install dbt-core dbt-${DBT_TARGET}

    - name: Install DBT deps
      run: dbt deps --target ${DBT_TARGET}

    - name: load test data
      run: dbt seed --target ${DBT_TARGET}
    
    - name: run models
      run: dbt run --target ${DBT_TARGET}

    - name: dbt test macros
      run: dbt test --target ${DBT_TARGET}

    - name: Create seed tests
      run: bash tests/create_seeds_tests_yml.sh

    - name: dbt test seeds
      run: dbt test --target ${DBT_TARGET} --select config.materialized:seed --exclude "dbt_utils_recency*"

  test-snowflake:
    name: Test on Snowflake
    runs-on: ubuntu-latest
    container:
      image: python:3.11

    env:
      SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
      DBT_TARGET: snowflake

    steps:
    - uses: actions/checkout@v4

    - name: pip install
      run: pip install dbt-core dbt-${DBT_TARGET}

    - name: Install DBT deps
      run: dbt deps --target ${DBT_TARGET}

    - name: load test data
      run: dbt seed --target ${DBT_TARGET}
    
    - name: run models
      run: dbt run --target ${DBT_TARGET}

    - name: dbt test macros
      run: dbt test --target ${DBT_TARGET}

    - name: Create seed tests
      run: bash tests/create_seeds_tests_yml.sh

    - name: dbt test seeds
      run: dbt test --target ${DBT_TARGET} --select config.materialized:seed --exclude "dbt_utils_recency*"

  test-redshift:
    name: Test on RedShift
    runs-on: ubuntu-latest
    container:
      image: python:3.11

    env:
      REDSHIFT_PASSWORD: ${{ secrets.REDSHIFT_PASSWORD }}
      DBT_TARGET: redshift

    steps:
    - uses: actions/checkout@v4

    - name: pip install
      run: pip install dbt-core dbt-${DBT_TARGET}

    - name: Install DBT deps
      run: dbt deps --target ${DBT_TARGET}

    - name: load test data
      run: dbt seed --target ${DBT_TARGET}

    - name: run models
      run: dbt run --target ${DBT_TARGET}

    - name: dbt test macros
      run: dbt test --target ${DBT_TARGET}

    - name: Create seed tests
      run: bash tests/create_seeds_tests_yml.sh

    - name: dbt test seeds
      run: dbt test --target ${DBT_TARGET} --select config.materialized:seed --exclude "dbt_utils_recency*"

  test-bigquery:
    name: Test on BigQuery
    runs-on: ubuntu-latest
    container:
      image: python:3.11

    env:
      DBT_TARGET: bigquery

    steps:
    - uses: actions/checkout@v4

    - name: create-json
      id: create-json
      uses: jsdaniell/create-json@1.1.2
      with:
        name: "gcp_keyfile.json"
        dir: 'integration_tests/'
        json: ${{ secrets.GCP_KEYFILE }}

    #- run: echo '${{ secrets.GCP_KEYFILE }}' > gcp_keyfile.json

    - name: pip install
      run: pip install dbt-core dbt-${DBT_TARGET}

    - name: Install DBT deps
      run: dbt deps --target ${DBT_TARGET}
      
    - name: load test data
      run: dbt seed --target ${DBT_TARGET} --exclude colnames_with_spaces
    
    - name: run models
      run: dbt run --target ${DBT_TARGET}

    - name: dbt test macros
      run: dbt test --target ${DBT_TARGET} --exclude tag:dataset-colnames_with_spaces
      
    - name: Create seed tests
      run: |
        dbt compile --target ${DBT_TARGET} -q \
        --inline "{{ testgen.get_test_suggestions(ref('users'), resource_type='seeds', column_config={'quote': true, 'tags': ['dataset-users']} ) }}" \
        > seeds/users_test_suggestions.yml

    - name: dbt test seeds
      run: dbt test --target ${DBT_TARGET} --select config.materialized:seed --exclude "dbt_utils_recency*" --exclude tag:dataset-colnames_with_spaces

  test-databricks:
    name: Test on Databricks
    runs-on: ubuntu-latest
    container:
      image: python:3.11

    env:
      DBT_TARGET: databricks
      DATABRICKS_CLIENT_ID: ${{ secrets.DATABRICKS_CLIENT_ID }}
      DATABRICKS_CLIENT_SECRET: ${{ secrets.DATABRICKS_CLIENT_SECRET }}

    steps:
    - uses: actions/checkout@v4

    - name: pip install
      run: pip install dbt-core dbt-${DBT_TARGET}

    - name: Install DBT deps
      run: dbt deps --target ${DBT_TARGET}
      
    - name: load test data
      run: dbt seed --target ${DBT_TARGET} --exclude colnames_with_spaces

    - name: run models
      run: dbt run --target ${DBT_TARGET}

    - name: dbt test macros
      run: dbt test --target ${DBT_TARGET} --exclude tag:dataset-colnames_with_spaces
      
    - name: Create seed tests
      run: |
        dbt compile --target ${DBT_TARGET} -q \
        --inline "{{ testgen.get_test_suggestions(ref('users'), resource_type='seeds', column_config={'quote': true, 'tags': ['dataset-users']} ) }}" \
        > seeds/users_test_suggestions.yml

    - name: dbt test seeds
      run: dbt test --target ${DBT_TARGET} --select config.materialized:seed --exclude "dbt_utils_recency*" --exclude tag:dataset-colnames_with_spaces