Upgrade dependencies and align to dbt 1.8 syntax

billiam_database/dbt_project.yml

@@ -33,6 +33,11 @@ on-run-start:
 on-run-end:
   - "{% if target.name == 'prod' %}DROP SCHEMA IF EXISTS dbt_test__audit CASCADE;{% endif %}"
 
+flags:
+  require_explicit_package_overrides_for_builtin_materializations: true
+  require_model_names_without_spaces: true
+  source_freshness_run_project_hooks: false
+
 seeds:
   billiam_database:
     +enabled: true
@@ -76,13 +81,11 @@ models:
 #    +docs:
 #      show: false
 
-# https://docs.getdbt.com/reference/test-configs
-tests:
+# https://docs.getdbt.com/reference/data-test-configs
+data_tests:
   +docs:
     node_color: Indigo
     show: false
   billiam_database:
     +enabled: true
     +group: billiam
-    unit:
-      +tags: ["unit-test"]

billiam_database/models/intermediate/_schema.yml

@@ -12,14 +12,14 @@ models:
 
       This may be decommissioned in the future as it's not really doing much.
     columns:
-      - { name: row_id, tests: ["not_null"] }
-      - { name: transaction_date, tests: ["not_null"] }
-      - { name: transaction_id, tests: ["not_null"] }
-      - { name: item, tests: ["not_null"] }
-      - { name: cost, tests: ["not_null"] }
-      - { name: category, tests: ["not_null"] }
-      - { name: counterparty, tests: ["not_null"] }
-      - { name: exclusion_flag, tests: ["not_null"] }
+      - { name: row_id, data_tests: ["not_null"] }
+      - { name: transaction_date, data_tests: ["not_null"] }
+      - { name: transaction_id, data_tests: ["not_null"] }
+      - { name: item, data_tests: ["not_null"] }
+      - { name: cost, data_tests: ["not_null"] }
+      - { name: category, data_tests: ["not_null"] }
+      - { name: counterparty, data_tests: ["not_null"] }
+      - { name: exclusion_flag, data_tests: ["not_null"] }
   - name: int__transactions
     description: >
       The transactions and their core measures.
@@ -28,12 +28,12 @@ models:
       overall balances -- for example, paying off a credit card is being
       considered as not changing my net balance, and has been excluded.
     columns:
-      - { name: transaction_id, tests: ["not_null", "unique"] }
+      - { name: transaction_id, data_tests: ["not_null", "unique"] }
       - { name: transaction_date }
-      - { name: cost, tests: ["not_null"] }
-      - { name: item_count, tests: ["not_null"] }
+      - { name: cost, data_tests: ["not_null"] }
+      - { name: item_count, data_tests: ["not_null"] }
       - name: counterparty
-        tests:
+        data_tests:
           - not_null
           - dbt_expectations.expect_column_values_to_not_match_like_pattern:
               like_pattern: "%||%"

billiam_database/models/intermediate/tests/test__int__transaction_items.yml

@@ -0,0 +1,19 @@
+---
+unit_tests:
+  - name: test__int__transaction_items
+    model: int__transaction_items
+    given:
+      - input: ref("stg__finances")
+        rows:
+          # prettier-ignore
+          - {row_id: 1, transaction_id: 1, transaction_date: "2020-01-01", item: "Item 1", cost: 2.99, category: "Food", counterparty: "Tesco", payment_method: , exclusion_flag: , reimbursement_transaction_id: }
+          - {row_id: 2, transaction_id: 1, transaction_date: "2020-01-01", item: "Item 2", cost: 3.99, category: "Food", counterparty: "Tesco", payment_method: , exclusion_flag: , reimbursement_transaction_id: }
+      - input: ref("int__transactions")
+        rows:
+          # prettier-ignore
+          - {transaction_id: 1, transaction_date: "2020-01-01", cost: 6.98, item_count: 2, counterparty: "Tesco"}
+    expect:
+      rows:
+        # prettier-ignore
+        - {row_id: 1, transaction_id: 1, transaction_date: "2020-01-01", item: "Item 1", cost: 2.99, category: "Food", counterparty: "Tesco", exclusion_flag: }
+        - {row_id: 2, transaction_id: 1, transaction_date: "2020-01-01", item: "Item 2", cost: 3.99, category: "Food", counterparty: "Tesco", exclusion_flag: }

billiam_database/models/intermediate/tests/test__int__transactions.yml

@@ -0,0 +1,14 @@
+---
+unit_tests:
+  - name: test__int__transactions
+    model: int__transactions
+    given:
+      - input: ref("stg__finances")
+        rows:
+          # prettier-ignore
+          - {row_id: 1, transaction_id: 1, transaction_date: "2020-01-01", item: "Item 1", cost: 2.99, category: "Food", counterparty: "Tesco", payment_method: , exclusion_flag: , reimbursement_transaction_id: }
+          - {row_id: 2, transaction_id: 1, transaction_date: "2020-01-01", item: "Item 2", cost: 3.99, category: "Food", counterparty: "Tesco", payment_method: , exclusion_flag: , reimbursement_transaction_id: }
+    expect:
+      rows:
+        # prettier-ignore
+        - {transaction_id: 1, transaction_date: "2020-01-01", cost: 6.98, item_count: 2, counterparty: "Tesco"}

billiam_database/models/presentation/tests/test__pl__daily_metrics.yml

@@ -0,0 +1,27 @@
+# TODO: Make the axis unit-testable
+#
+#unit_tests:
+#  - name: test__pl__daily_metrics
+#    model: pl__daily_metrics
+#    given:
+#      - input: ref("stg__daily_tracker")
+#        rows:
+#          # prettier-ignore
+#          - {date_time: "2018-04-23 08:15:00", project: "BAU Task", detail: "Task 1", minutes: 15, company: "TSB"}
+#          - {date_time: "2018-04-23 08:30:00", project: "BAU Task", detail: "Task 2", minutes: 15, company: "Jaja"}
+#          - {date_time: "2018-04-23 08:45:00", project: "BAU Task", detail: "Task 3", minutes: 15, company: "Allica"}
+#          - {date_time: "2018-04-23 09:00:00", project: "BAU Task", detail: "Task 4", minutes: 15, company: "Sainsbury's"}
+#      - input: ref("int__transaction_items")
+#        rows:
+#          # prettier-ignore
+#          - {row_id: 1, transaction_id: 1, transaction_date: "2018-01-01", item: "Item 1", cost: 2.99, category: "Food", counterparty: "Tesco",    exclusion_flag: false}
+#          - {row_id: 2, transaction_id: 1, transaction_date: "2018-01-01", item: "Item 2", cost: 3.99, category: "Rent", counterparty: "Landlord", exclusion_flag: false}
+#          - {row_id: 3, transaction_id: 2, transaction_date: "2018-01-02", item: "Item 3", cost: 4.99, category: "Food", counterparty: "Tesco",    exclusion_flag: false}
+#          - {row_id: 4, transaction_id: 3, transaction_date: "2018-01-03", item: "Item 4", cost: 5.99, category: "Food", counterparty: "Tesco",    exclusion_flag: true}
+#          - {row_id: 5, transaction_id: 4, transaction_date: "2018-01-04", item: "Item 5", cost: 6.99, category: "Rent", counterparty: "Landlord", exclusion_flag: false}
+#    expect:
+#      rows:
+#        # prettier-ignore
+#        - {metric_date: "2018-01-01", total_cost: 6.98, non_essential_cost: 2.99, non_essential_cost_proportion: 42.8367, total_working_time: 30, meeting_time: 15, meeting_proportion:  50.0, working_day_meeting_proportion:  50.0}
+#        - {metric_date: "2018-01-02", total_cost: 4.99, non_essential_cost: 4.99, non_essential_cost_proportion: 100.0,   total_working_time: 15, meeting_time:   , meeting_proportion:      , working_day_meeting_proportion:      }
+#        - {metric_date: "2018-01-04", total_cost: 6.99, non_essential_cost: ,     non_essential_cost_proportion: 100.0,   total_working_time: 15, meeting_time: 15, meeting_proportion: 100.0, working_day_meeting_proportion: 100.0}

billiam_database/models/presentation/tests/test__pl__daily_tracker.yml

@@ -0,0 +1,19 @@
+---
+unit_tests:
+  - name: test__pl__daily_tracker
+    model: pl__daily_tracker
+    given:
+      - input: ref("stg__daily_tracker")
+        rows:
+          # prettier-ignore
+          - {date_time: "2019-04-23 08:15:00", project: "BAU Task", detail: "Task 1", minutes: 15, company: "TSB"}
+          - {date_time: "2019-04-23 08:30:00", project: "BAU Task", detail: "Task 2", minutes: 15, company: "Jaja"}
+          - {date_time: "2019-04-23 08:45:00", project: "BAU Task", detail: "Task 3", minutes: 15, company: "Allica"}
+          - {date_time: "2019-04-23 09:00:00", project: "BAU Task", detail: "Task 4", minutes: 15, company: "Sainsbury's"}
+    expect:
+      rows:
+        # prettier-ignore
+        - {date_time: "2019-04-23 08:15:00", project: "BAU Task", detail: "Task 1", minutes: 15, company: "TSB"}
+        - {date_time: "2019-04-23 08:30:00", project: "BAU Task", detail: "Task 2", minutes: 15, company: "Jaja"}
+        - {date_time: "2019-04-23 08:45:00", project: "BAU Task", detail: "Task 3", minutes: 15, company: "Allica"}
+        - {date_time: "2019-04-23 09:00:00", project: "BAU Task", detail: "Task 4", minutes: 15, company: "Sainsbury's"}

billiam_database/models/presentation/tests/test__pl__task_details.yml

@@ -0,0 +1,20 @@
+---
+unit_tests:
+  - name: test__pl__task_details
+    model: pl__task_details
+    given:
+      - input: ref("stg__daily_tracker")
+        rows:
+          # prettier-ignore
+          - {date_time: "2019-04-23 08:15:00", project: "BAU Task", detail: "Task 1", minutes: 15, company: "TSB"}
+          - {date_time: "2019-04-23 08:30:00", project: "BAU Task", detail: "Task 2", minutes: 15, company: "Jaja"}
+          - {date_time: "2019-04-23 08:45:00", project: "BAU Task", detail: "Task 3", minutes: 15, company: "Allica"}
+          - {date_time: "2019-04-23 09:00:00", project: "BAU Task", detail: "Task 4", minutes: 15, company: "Sainsbury's"}
+    expect:
+      rows:
+        # prettier-ignore
+        - {group_id: 0, group_description: "Task and detail", project: "BAU Task", detail: "Task 1", total_records: 1, total_time: 15}
+        - {group_id: 0, group_description: "Task and detail", project: "BAU Task", detail: "Task 2", total_records: 1, total_time: 15}
+        - {group_id: 0, group_description: "Task and detail", project: "BAU Task", detail: "Task 3", total_records: 1, total_time: 15}
+        - {group_id: 0, group_description: "Task and detail", project: "BAU Task", detail: "Task 4", total_records: 1, total_time: 15}
+        - {group_id: 1, group_description: "Task only",       project: "BAU Task", detail: "",       total_records: 4, total_time: 60}

billiam_database/models/staging/_schema.yml

@@ -15,21 +15,21 @@ models:
 
           The local timestamp isn't recorded, but it can be assumed to be GMT
           (and BST over the Summertime).
-        tests:
+        data_tests:
           - unique
           - not_null
       - name: project
         data_type: text
         description: >
           The task (project) that was being worked on at the timestamp.
-        tests:
+        data_tests:
           - not_null
       - name: detail
         data_type: text
         description: >
           Additional details about the task that was being worked on at the
           timestamp.
-        tests:
+        data_tests:
           - not_null
       - name: minutes
         data_type: integer
@@ -52,7 +52,7 @@ models:
 
           - The entry has over-stated how much time was spent on the task
             because time was spent on other tasks before 10:00 and after 10:00.
-        tests:
+        data_tests:
           - not_null
       - name: company
         data_type: text
@@ -61,7 +61,7 @@ models:
 
           In a future iteration of this project, this will instead be captured
           in a seed with the from- and to- employment dates.
-        tests:
+        data_tests:
           - not_null
           - accepted_values:
               values: ["TSB", "Jaja", "Allica", "Sainsbury''s"]
@@ -72,13 +72,13 @@ models:
       - name: row_id
         data_type: integer
         description: A generated identifier to uniquely identify rows.
-        tests:
+        data_tests:
           - unique
           - not_null
       - name: transaction_id
         data_type: integer
         description: The transaction identifier.
-        tests:
+        data_tests:
           - not_null
       - name: transaction_date
         data_type: date
@@ -88,12 +88,12 @@ models:
           This will be when the transaction initiation happened in the local
           timezone (which isn't recorded), not when the money was physically
           moved (where there's a difference).
-        tests:
+        data_tests:
           - not_null
       - name: item
         data_type: text
         description: The name of the item.
-        tests:
+        data_tests:
           - not_null
       - name: cost
         data_type: decimal(18, 3)
@@ -102,12 +102,12 @@ models:
 
           A positive value represents money sent to someone else, and a negative
           value represents money sent to me (or is a discount, etc).
-        tests:
+        data_tests:
           - not_null
       - name: category
         data_type: text
         description: The category of the item.
-        tests:
+        data_tests:
           - not_null
       - name: counterparty
         data_type: text
@@ -117,22 +117,22 @@ models:
           Note that this can also be a person (where the transaction is between
           me and someone else), and will always be the entity that I transacted
           with directly (so a third party will not be shown here).
-        tests:
+        data_tests:
           - not_null
       - name: payment_method
         data_type: text
         description: The method used to pay for the transaction.
-        tests:
+        data_tests:
           - not_null
       - name: exclusion_flag
         data_type: boolean
         description: Whether to exclude the item from reporting.
-        tests:
+        data_tests:
           - not_null
       - name: reimbursement_transaction_id
         data_type: integer
         description: The transaction ID of the transaction that this repays.
-        tests:
+        data_tests:
           - relationships:
               to: ref("stg__finances")
               field: transaction_id
@@ -145,18 +145,18 @@ models:
       - name: transaction_id
         data_type: text
         description: The unique identifier for each transaction.
-        tests:
+        data_tests:
           - unique
           - not_null
       - name: transaction_date
         data_type: date
         description: The date of the transaction.
-        tests:
+        data_tests:
           - not_null
       - name: transaction_time
         data_type: time
         description: The time of the transaction.
-        tests:
+        data_tests:
           - not_null
       - name: type
         data_type: text
@@ -166,7 +166,7 @@ models:
           This is typically to distinguish between transaction methods, such as
           internal transfers (`Pot transfer`) and payment schemes like faster
           payments (`Faster payment`).
-        tests:
+        data_tests:
           - not_null
       - name: counterparty
         data_type: text

billiam_database/models/staging/tests/test__stg__daily_tracker.yml

@@ -0,0 +1,20 @@
+# Currently disabled as the test doesn't work on non-database objects
+#
+#unit_tests:
+#  - name: test__stg__daily_tracker
+#    model: stg__daily_tracker
+#    given:
+#      - input: source("raw", "daily_tracker")
+#        rows:
+#          # prettier-ignore
+#          - {date_time: "2019-04-23 08:15:00", task: "BAU Task     ", detail: "Running Unsecured Master Code  ", interval: 15, company: TSB}
+#          - {date_time: "2019-04-23 08:30:00", task: "Adhoc Task   ", detail: "Working on this workbook       ", interval: 15, company: TSB}
+#          - {date_time: "2019-04-23 08:45:00", task: "Documentation", detail: "Weekly Wellness Procedure Guide", interval: 15, company: TSB}
+#          - {date_time: "2019-04-23 09:00:00", task: "Something    ", detail: "Weekly Wellness                ", interval: 15, company: TSB}
+#    expect:
+#      rows:
+#        # prettier-ignore
+#        - {date_time: "2019-04-23 08:15:00", task: "BAU Task",      detail: "Running Unsecured Master Code",   interval: 15, company: TSB}
+#        - {date_time: "2019-04-23 08:30:00", task: "Adhoc Task",    detail: "Working on this workbook",        interval: 15, company: TSB}
+#        - {date_time: "2019-04-23 08:45:00", task: "Documentation", detail: "Weekly Wellness Procedure Guide", interval: 15, company: TSB}
+#        - {date_time: "2019-04-23 09:00:00", task: "Something",     detail: "Weekly Wellness",                 interval: 15, company: TSB}

billiam_database/models/staging/tests/test__stg__finances.yml

@@ -0,0 +1,20 @@
+# Currently disabled as the test doesn't work on non-database objects
+#
+#unit_tests:
+#  - name: test__stg__finances
+#    model: stg__finances
+#    given:
+#      - input: source("raw", "finances")
+#        rows:
+#          # prettier-ignore
+#          - {"Transaction": 1, "Date": "2018-01-18", Item: "Cheese  ", Cost: "£5.24",    Category: "   Food ", Retailer: "Tesco  ", "Payment Method": "Santander", Exclusion: 0, "Repayment Identifier": , "Reimbursement Transaction": , "Expenditure Identifier": "Expenditure", "Regular Spending": 0, "Small Transaction": 1, "Week Ending": "20/01/2018", Month: "01/01/2018", "Pay Date": "00/01/1900", "Overall Category": "Food        ", "Item Roundup": 9.04, "Transaction Roundup": 8.44}
+#          - {"Transaction": 1, "Date": "2018-01-18", Item: "Chillies", Cost: "£0.90",    Category: "   Food ", Retailer: "Tesco  ", "Payment Method": "Santander", Exclusion: 0, "Repayment Identifier": , "Reimbursement Transaction": , "Expenditure Identifier": "Expenditure", "Regular Spending": 0, "Small Transaction": 1, "Week Ending": "20/01/2018", Month: "01/01/2018", "Pay Date": "00/01/1900", "Overall Category": "Food        ", "Item Roundup":  1.4, "Transaction Roundup": 8.44}
+#          - {"Transaction": 1, "Date": "2018-01-18", Item: "Lettuce ", Cost: "£1000.50", Category: "Food ",    Retailer: "Tesco  ", "Payment Method": "Santander", Exclusion: 0, "Repayment Identifier": , "Reimbursement Transaction": , "Expenditure Identifier": "Expenditure", "Regular Spending": 0, "Small Transaction": 1, "Week Ending": "20/01/2018", Month: "01/01/2018", "Pay Date": "00/01/1900", "Overall Category": "Food        ", "Item Roundup":    3, "Transaction Roundup": 8.44}
+#          - {"Transaction": 2, "Date": "2018-01-18", Item: "Internet", Cost: "-£10.00",  Category: " Bills",   Retailer: "Someone", "Payment Method": "Cash     ", Exclusion: 1, "Repayment Identifier": , "Reimbursement Transaction": , "Expenditure Identifier": "Income     ", "Regular Spending": 0, "Small Transaction": 1, "Week Ending": "20/01/2018", Month: "01/01/2018", "Pay Date": "00/01/1900", "Overall Category": "Living Costs", "Item Roundup":    0, "Transaction Roundup": 0}
+#    expect:
+#      rows:
+#        # prettier-ignore
+#        - {row_id: 1, transaction_id: 1, transaction_date: "2018-01-18", item: "Cheese",   cost:    5.24, category: "Food",  counterparty: "Tesco",   payment_method: "Santander", exclusion_flag: false, reimbursement_transaction_id: }
+#        - {row_id: 2, transaction_id: 1, transaction_date: "2018-01-18", item: "Chillies", cost:    0.90, category: "Food",  counterparty: "Tesco",   payment_method: "Santander", exclusion_flag: false, reimbursement_transaction_id: }
+#        - {row_id: 3, transaction_id: 1, transaction_date: "2018-01-18", item: "Lettuce",  cost: 1000.50, category: "Food",  counterparty: "Tesco",   payment_method: "Santander", exclusion_flag: false, reimbursement_transaction_id: }
+#        - {row_id: 4, transaction_id: 2, transaction_date: "2018-01-18", item: "Internet", cost:  -10.00, category: "Bills", counterparty: "Someone", payment_method: "Cash",      exclusion_flag: true,  reimbursement_transaction_id: }