Add generated_at field to catalog and manifest (#864)

dbt/contracts/graph/parsed.py

@@ -1,5 +1,5 @@
 from dbt.api import APIObject
-from dbt.utils import deep_merge
+from dbt.utils import deep_merge, timestring
 from dbt.node_types import NodeType
 
 import dbt.clients.jinja
@@ -252,6 +252,9 @@
     'properties': {
         'nodes': PARSED_NODES_CONTRACT,
         'macros': PARSED_MACROS_CONTRACT,
+        'generated_at': {
+            'type': 'date-time'
+        }
     },
     'required': ['nodes', 'macros'],
 }
@@ -333,12 +336,16 @@ def build_edges(nodes):
 
 class ParsedManifest(object):
     """The final result of parsing all macros and nodes in a graph."""
-    def __init__(self, nodes, macros):
+    def __init__(self, nodes, macros, generated_at=None):
         """The constructor. nodes and macros are dictionaries mapping unique
-        IDs to ParsedNode and ParsedMacro objects, respectively.
+        IDs to ParsedNode and ParsedMacro objects, respectively. generated_at
+        is a text timestamp in RFC 3339 format.
         """
         self.nodes = nodes
         self.macros = macros
+        if generated_at is None:
+            generated_at = timestring()
+        self.generated_at = generated_at
 
     def serialize(self):
         """Convert the parsed manifest to a nested dict structure that we can
@@ -351,6 +358,7 @@ def serialize(self):
             'macros': {k: v.serialize() for k, v in self.macros.items()},
             'parent_map': backward_edges,
             'child_map': forward_edges,
+            'generated_at': self.generated_at,
         }
 
     def _find_by_name(self, name, package, subgraph, nodetype):

dbt/task/generate.py

@@ -115,6 +115,7 @@ def run(self):
             for row in results
         ]
         results = unflatten(results)
+        results['generated_at'] = dbt.utils.timestring()
 
         path = os.path.join(self.project['target-path'], CATALOG_FILENAME)
         write_file(path, json.dumps(results))

dbt/utils.py

@@ -1,3 +1,4 @@
+from datetime import datetime
 import os
 import hashlib
 import itertools
@@ -460,3 +461,9 @@ def filter_null_values(input):
 
 def add_ephemeral_model_prefix(s):
     return '__dbt__CTE__{}'.format(s)
+
+
+def timestring():
+    """Get the current datetime as an RFC 3339-compliant string"""
+    # isoformat doesn't include the mandatory trailing 'Z' for UTC.
+    return datetime.utcnow().isoformat() + 'Z'

test/integration/029_docs_generate_tests/test_docs_generate.py

@@ -3,6 +3,8 @@
 
 from test.integration.base import DBTIntegrationTest, use_profile
 
+from freezegun import freeze_time
+
 
 class TestDocsGenerate(DBTIntegrationTest):
     def setUp(self):
@@ -48,6 +50,7 @@ def verify_catalog(self, expected):
 
         my_schema_name = self.unique_schema()
         self.assertIn(my_schema_name, catalog)
+        self.assertEqual(catalog['generated_at'], '2017-08-16T10:11:12Z')
         my_schema = catalog[my_schema_name]
         self.assertEqual(expected, my_schema)
 
@@ -148,6 +151,7 @@ def expected_seeded_manifest(self):
                 'model.test.model': [],
                 'seed.test.seed': ['model.test.model'],
             },
+            'generated_at': '2017-08-16T10:11:12Z',
         }
 
     def verify_manifest(self, expected_manifest):
@@ -158,7 +162,7 @@ def verify_manifest(self, expected_manifest):
 
         self.assertEqual(
             set(manifest),
-            {'nodes', 'macros', 'parent_map', 'child_map'}
+            {'nodes', 'macros', 'parent_map', 'child_map', 'generated_at'}
         )
 
         self.verify_manifest_macros(manifest)
@@ -168,6 +172,7 @@ def verify_manifest(self, expected_manifest):
         self.assertEqual(manifest_without_macros, expected_manifest)
 
     @use_profile('postgres')
+    @freeze_time('2017-08-16T10:11:12Z')
     def test__postgres__run_and_generate(self):
         self.run_and_generate()
         my_schema_name = self.unique_schema()
@@ -227,6 +232,7 @@ def test__postgres__run_and_generate(self):
         self.verify_manifest(self.expected_seeded_manifest())
 
     @use_profile('snowflake')
+    @freeze_time('2017-08-16T10:11:12Z')
     def test__snowflake__run_and_generate(self):
         self.run_and_generate()
         my_schema_name = self.unique_schema()
@@ -287,6 +293,7 @@ def test__snowflake__run_and_generate(self):
         self.verify_manifest(self.expected_seeded_manifest())
 
     @use_profile('bigquery')
+    @freeze_time('2017-08-16T10:11:12Z')
     def test__bigquery__run_and_generate(self):
         self.run_and_generate()
         my_schema_name = self.unique_schema()
@@ -346,6 +353,7 @@ def test__bigquery__run_and_generate(self):
         self.verify_manifest(self.expected_seeded_manifest())
 
     @use_profile('bigquery')
+    @freeze_time('2017-08-16T10:11:12Z')
     def test__bigquery__nested_models(self):
         self.use_default_project({'source-paths': [self.dir('bq_models')]})
 
@@ -477,5 +485,6 @@ def test__bigquery__nested_models(self):
                 'model.test.model': ['model.test.seed'],
                 'model.test.seed': []
             },
+            'generated_at': '2017-08-16T10:11:12Z',
         }
         self.verify_manifest(expected_manifest)

test/integration/034_redshift_test/test_late_binding_view.py

@@ -0,0 +1,21 @@
+import json
+import os
+
+from nose.plugins.attrib import attr
+from test.integration.base import DBTIntegrationTest
+
+
+class TestLateBindingView(DBTIntegrationTest):
+    @property
+    def schema(self):
+        return 'late_binding_view_033'
+
+    @staticmethod
+    def dir(path):
+        return os.path.normpath(
+            os.path.join('test/integration/033_redshift_test', path)
+        )
+
+    @property
+    def models(self):
+        return self.dir("models")

test/unit/test_manifest.py

@@ -5,6 +5,8 @@
 
 import dbt.flags
 from dbt.contracts.graph.parsed import ParsedNode, ParsedManifest
+from dbt.utils import timestring
+import freezegun
 
 class ManifestTest(unittest.TestCase):
     def setUp(self):
@@ -151,17 +153,21 @@ def setUp(self):
             ),
         }
 
+    @freezegun.freeze_time('2018-02-14T09:15:13Z')
     def test__no_nodes(self):
         manifest = ParsedManifest(nodes={}, macros={})
         self.assertEqual(
             manifest.serialize(),
-            {'nodes': {}, 'macros': {}, 'parent_map': {}, 'child_map': {}}
+            {'nodes': {}, 'macros': {}, 'parent_map': {}, 'child_map': {},
+             'generated_at': '2018-02-14T09:15:13Z'}
         )
 
+    @freezegun.freeze_time('2018-02-14T09:15:13Z')
     def test__nested_nodes(self):
         nodes = copy.copy(self.nested_nodes)
         manifest = ParsedManifest(nodes=nodes, macros={})
         serialized = manifest.serialize()
+        self.assertEqual(serialized['generated_at'], '2018-02-14T09:15:13Z')
         parent_map = serialized['parent_map']
         child_map = serialized['child_map']
         # make sure there aren't any extra/missing keys.

test/unit/test_parser.py

@@ -4,6 +4,7 @@
 
 import dbt.flags
 from dbt.parser import ModelParser, MacroParser, DataTestParser, SchemaParser, ParserUtils
+from dbt.utils import timestring
 
 from dbt.node_types import NodeType
 from dbt.contracts.graph.parsed import ParsedManifest, ParsedNode, ParsedMacro
@@ -702,6 +703,7 @@ def test__process_refs__packages(self):
         manifest = ParsedManifest(
             nodes={k: ParsedNode(**v) for (k,v) in graph['nodes'].items()},
             macros={k: ParsedMacro(**v) for (k,v) in graph['macros'].items()},
+            generated_at=timestring(),
         )
 
         processed_manifest = ParserUtils.process_refs(manifest, 'root')