From c5c09d077f97bcac1df32922e4deb65a03709110 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Thu, 19 Jul 2018 08:07:54 -0600 Subject: [PATCH] Add generated_at field to catalog and manifest (#864) --- dbt/contracts/graph/parsed.py | 14 ++++++++++--- dbt/task/generate.py | 1 + dbt/utils.py | 7 +++++++ .../test_docs_generate.py | 11 +++++++++- .../test_late_binding_view.py | 21 +++++++++++++++++++ test/unit/test_manifest.py | 8 ++++++- test/unit/test_parser.py | 2 ++ 7 files changed, 59 insertions(+), 5 deletions(-) create mode 100644 test/integration/034_redshift_test/test_late_binding_view.py diff --git a/dbt/contracts/graph/parsed.py b/dbt/contracts/graph/parsed.py index 4c9eb5457f2..61ae0b03e9d 100644 --- a/dbt/contracts/graph/parsed.py +++ b/dbt/contracts/graph/parsed.py @@ -1,5 +1,5 @@ from dbt.api import APIObject -from dbt.utils import deep_merge +from dbt.utils import deep_merge, timestring from dbt.node_types import NodeType import dbt.clients.jinja @@ -252,6 +252,9 @@ 'properties': { 'nodes': PARSED_NODES_CONTRACT, 'macros': PARSED_MACROS_CONTRACT, + 'generated_at': { + 'type': 'date-time' + } }, 'required': ['nodes', 'macros'], } @@ -333,12 +336,16 @@ def build_edges(nodes): class ParsedManifest(object): """The final result of parsing all macros and nodes in a graph.""" - def __init__(self, nodes, macros): + def __init__(self, nodes, macros, generated_at=None): """The constructor. nodes and macros are dictionaries mapping unique - IDs to ParsedNode and ParsedMacro objects, respectively. + IDs to ParsedNode and ParsedMacro objects, respectively. generated_at + is a text timestamp in RFC 3339 format. """ self.nodes = nodes self.macros = macros + if generated_at is None: + generated_at = timestring() + self.generated_at = generated_at def serialize(self): """Convert the parsed manifest to a nested dict structure that we can @@ -351,6 +358,7 @@ def serialize(self): 'macros': {k: v.serialize() for k, v in self.macros.items()}, 'parent_map': backward_edges, 'child_map': forward_edges, + 'generated_at': self.generated_at, } def _find_by_name(self, name, package, subgraph, nodetype): diff --git a/dbt/task/generate.py b/dbt/task/generate.py index aa1a295c4c0..e63f805eddf 100644 --- a/dbt/task/generate.py +++ b/dbt/task/generate.py @@ -115,6 +115,7 @@ def run(self): for row in results ] results = unflatten(results) + results['generated_at'] = dbt.utils.timestring() path = os.path.join(self.project['target-path'], CATALOG_FILENAME) write_file(path, json.dumps(results)) diff --git a/dbt/utils.py b/dbt/utils.py index b6d24668883..27b71d582d4 100644 --- a/dbt/utils.py +++ b/dbt/utils.py @@ -1,3 +1,4 @@ +from datetime import datetime import os import hashlib import itertools @@ -460,3 +461,9 @@ def filter_null_values(input): def add_ephemeral_model_prefix(s): return '__dbt__CTE__{}'.format(s) + + +def timestring(): + """Get the current datetime as an RFC 3339-compliant string""" + # isoformat doesn't include the mandatory trailing 'Z' for UTC. + return datetime.utcnow().isoformat() + 'Z' diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py index 91effc026f7..8fc1a497ef2 100644 --- a/test/integration/029_docs_generate_tests/test_docs_generate.py +++ b/test/integration/029_docs_generate_tests/test_docs_generate.py @@ -3,6 +3,8 @@ from test.integration.base import DBTIntegrationTest, use_profile +from freezegun import freeze_time + class TestDocsGenerate(DBTIntegrationTest): def setUp(self): @@ -51,6 +53,7 @@ def verify_catalog(self, expected): my_schema_name = self.unique_schema() self.assertIn(my_schema_name, catalog) + self.assertEqual(catalog['generated_at'], '2017-08-16T10:11:12Z') my_schema = catalog[my_schema_name] self.assertEqual(expected, my_schema) @@ -152,6 +155,7 @@ def expected_seeded_manifest(self): 'model.test.model': [], 'seed.test.seed': ['model.test.model'], }, + 'generated_at': '2017-08-16T10:11:12Z', } def verify_manifest(self, expected_manifest): @@ -162,7 +166,7 @@ def verify_manifest(self, expected_manifest): self.assertEqual( set(manifest), - {'nodes', 'macros', 'parent_map', 'child_map'} + {'nodes', 'macros', 'parent_map', 'child_map', 'generated_at'} ) self.verify_manifest_macros(manifest) @@ -172,6 +176,7 @@ def verify_manifest(self, expected_manifest): self.assertEqual(manifest_without_macros, expected_manifest) @use_profile('postgres') + @freeze_time('2017-08-16T10:11:12Z') def test__postgres__run_and_generate(self): self.run_and_generate() my_schema_name = self.unique_schema() @@ -231,6 +236,7 @@ def test__postgres__run_and_generate(self): self.verify_manifest(self.expected_seeded_manifest()) @use_profile('snowflake') + @freeze_time('2017-08-16T10:11:12Z') def test__snowflake__run_and_generate(self): self.run_and_generate() my_schema_name = self.unique_schema() @@ -291,6 +297,7 @@ def test__snowflake__run_and_generate(self): self.verify_manifest(self.expected_seeded_manifest()) @use_profile('bigquery') + @freeze_time('2017-08-16T10:11:12Z') def test__bigquery__run_and_generate(self): self.run_and_generate() my_schema_name = self.unique_schema() @@ -350,6 +357,7 @@ def test__bigquery__run_and_generate(self): self.verify_manifest(self.expected_seeded_manifest()) @use_profile('bigquery') + @freeze_time('2017-08-16T10:11:12Z') def test__bigquery__nested_models(self): self.use_default_project({'source-paths': [self.dir('bq_models')]}) @@ -481,6 +489,7 @@ def test__bigquery__nested_models(self): 'model.test.model': ['model.test.seed'], 'model.test.seed': [] }, + 'generated_at': '2017-08-16T10:11:12Z', } self.verify_manifest(expected_manifest) diff --git a/test/integration/034_redshift_test/test_late_binding_view.py b/test/integration/034_redshift_test/test_late_binding_view.py new file mode 100644 index 00000000000..7af7275632d --- /dev/null +++ b/test/integration/034_redshift_test/test_late_binding_view.py @@ -0,0 +1,21 @@ +import json +import os + +from nose.plugins.attrib import attr +from test.integration.base import DBTIntegrationTest + + +class TestLateBindingView(DBTIntegrationTest): + @property + def schema(self): + return 'late_binding_view_033' + + @staticmethod + def dir(path): + return os.path.normpath( + os.path.join('test/integration/033_redshift_test', path) + ) + + @property + def models(self): + return self.dir("models") diff --git a/test/unit/test_manifest.py b/test/unit/test_manifest.py index 60d4440a7ca..b99a910d28c 100644 --- a/test/unit/test_manifest.py +++ b/test/unit/test_manifest.py @@ -5,6 +5,8 @@ import dbt.flags from dbt.contracts.graph.parsed import ParsedNode, ParsedManifest +from dbt.utils import timestring +import freezegun class ManifestTest(unittest.TestCase): def setUp(self): @@ -151,17 +153,21 @@ def setUp(self): ), } + @freezegun.freeze_time('2018-02-14T09:15:13Z') def test__no_nodes(self): manifest = ParsedManifest(nodes={}, macros={}) self.assertEqual( manifest.serialize(), - {'nodes': {}, 'macros': {}, 'parent_map': {}, 'child_map': {}} + {'nodes': {}, 'macros': {}, 'parent_map': {}, 'child_map': {}, + 'generated_at': '2018-02-14T09:15:13Z'} ) + @freezegun.freeze_time('2018-02-14T09:15:13Z') def test__nested_nodes(self): nodes = copy.copy(self.nested_nodes) manifest = ParsedManifest(nodes=nodes, macros={}) serialized = manifest.serialize() + self.assertEqual(serialized['generated_at'], '2018-02-14T09:15:13Z') parent_map = serialized['parent_map'] child_map = serialized['child_map'] # make sure there aren't any extra/missing keys. diff --git a/test/unit/test_parser.py b/test/unit/test_parser.py index 56b9b6407cf..0df20511fef 100644 --- a/test/unit/test_parser.py +++ b/test/unit/test_parser.py @@ -4,6 +4,7 @@ import dbt.flags from dbt.parser import ModelParser, MacroParser, DataTestParser, SchemaParser, ParserUtils +from dbt.utils import timestring from dbt.node_types import NodeType from dbt.contracts.graph.parsed import ParsedManifest, ParsedNode, ParsedMacro @@ -702,6 +703,7 @@ def test__process_refs__packages(self): manifest = ParsedManifest( nodes={k: ParsedNode(**v) for (k,v) in graph['nodes'].items()}, macros={k: ParsedMacro(**v) for (k,v) in graph['macros'].items()}, + generated_at=timestring(), ) processed_manifest = ParserUtils.process_refs(manifest, 'root')