Merge pull request #877 from fishtown-analytics/add-generated-dates

Add generated dates (#864)

dbt/contracts/graph/parsed.py

@@ -1,5 +1,5 @@
 from dbt.api import APIObject
-from dbt.utils import deep_merge
+from dbt.utils import deep_merge, timestring
 from dbt.node_types import NodeType
 
 import dbt.clients.jinja
@@ -242,6 +242,23 @@
     },
 }
 
+
+NODE_EDGE_MAP = {
+    'type': 'object',
+    'additionalProperties': False,
+    'description': 'A map of node relationships',
+    'patternProperties': {
+        '.*': {
+            'type': 'array',
+            'items': {
+                'type': 'string',
+                'description': 'A node name',
+            }
+        }
+    }
+}
+
+
 PARSED_MANIFEST_CONTRACT = {
     'type': 'object',
     'additionalProperties': False,
@@ -252,6 +269,12 @@
     'properties': {
         'nodes': PARSED_NODES_CONTRACT,
         'macros': PARSED_MACROS_CONTRACT,
+        'generated_at': {
+            'type': 'string',
+            'format': 'date-time',
+        },
+        'parent_map': NODE_EDGE_MAP,
+        'child_map': NODE_EDGE_MAP,
     },
     'required': ['nodes', 'macros'],
 }
@@ -331,14 +354,19 @@ def build_edges(nodes):
     return forward_edges, backward_edges
 
 
-class ParsedManifest(object):
+class ParsedManifest(APIObject):
+    SCHEMA = PARSED_MANIFEST_CONTRACT
     """The final result of parsing all macros and nodes in a graph."""
-    def __init__(self, nodes, macros):
+    def __init__(self, nodes, macros, generated_at):
         """The constructor. nodes and macros are dictionaries mapping unique
-        IDs to ParsedNode and ParsedMacro objects, respectively.
+        IDs to ParsedNode and ParsedMacro objects, respectively. generated_at
+        is a text timestamp in RFC 3339 format.
         """
         self.nodes = nodes
         self.macros = macros
+        self.generated_at = generated_at
+        self._contents = {}
+        super(ParsedManifest, self).__init__()
 
     def serialize(self):
         """Convert the parsed manifest to a nested dict structure that we can
@@ -351,6 +379,7 @@ def serialize(self):
             'macros': {k: v.serialize() for k, v in self.macros.items()},
             'parent_map': backward_edges,
             'child_map': forward_edges,
+            'generated_at': self.generated_at,
         }
 
     def _find_by_name(self, name, package, subgraph, nodetype):

dbt/loader.py

@@ -2,6 +2,7 @@
 
 from dbt.node_types import NodeType
 from dbt.contracts.graph.parsed import ParsedManifest
+from dbt.utils import timestring
 
 import dbt.parser
 
@@ -18,7 +19,8 @@ def load_all(cls, root_project, all_projects):
         for loader in cls._LOADERS:
             nodes.update(loader.load_all(root_project, all_projects, macros))
 
-        manifest = ParsedManifest(nodes=nodes, macros=macros)
+        manifest = ParsedManifest(nodes=nodes, macros=macros,
+                                  generated_at=timestring())
         manifest = dbt.parser.ParserUtils.process_refs(manifest, root_project)
         return manifest
 

dbt/task/generate.py

@@ -1,7 +1,6 @@
 import json
 import os
 
-from dbt.contracts.graph.parsed import ParsedManifest, ParsedNode, ParsedMacro
 from dbt.adapters.factory import get_adapter
 from dbt.clients.system import write_file
 from dbt.compat import bigint
@@ -115,6 +114,7 @@ def run(self):
             for row in results
         ]
         results = unflatten(results)
+        results['generated_at'] = dbt.utils.timestring()
 
         path = os.path.join(self.project['target-path'], CATALOG_FILENAME)
         write_file(path, json.dumps(results))

dbt/utils.py

@@ -1,3 +1,4 @@
+from datetime import datetime
 import os
 import hashlib
 import itertools
@@ -460,3 +461,9 @@ def filter_null_values(input):
 
 def add_ephemeral_model_prefix(s):
     return '__dbt__CTE__{}'.format(s)
+
+
+def timestring():
+    """Get the current datetime as an RFC 3339-compliant string"""
+    # isoformat doesn't include the mandatory trailing 'Z' for UTC.
+    return datetime.utcnow().isoformat() + 'Z'

test/integration/029_docs_generate_tests/test_docs_generate.py

@@ -1,5 +1,6 @@
 import json
 import os
+from datetime import datetime, timedelta
 
 from test.integration.base import DBTIntegrationTest, use_profile
 
@@ -43,6 +44,25 @@ def run_and_generate(self, extra=None):
         self.assertEqual(len(self.run_dbt()), 1)
         self.run_dbt(['docs', 'generate'])
 
+    def assertRecent(self, timestr):
+        """Given a timestring in '%Y-%m-%dT%H:%M:%SZ' format (ISO8601), assert
+        that it represents a time before now and a time after 24h ago.
+
+        We can't just set the time via freezegun.freeze_time because that
+        breaks SSL, and a lot of these tests use SSL.
+        """
+        now = datetime.utcnow()
+        yesterday = now + timedelta(days=-1)
+        parsed = datetime.strptime(timestr, '%Y-%m-%dT%H:%M:%S.%fZ')
+        self.assertLess(
+            yesterday, parsed,
+            'parsed date {} happened over 24h ago'.format(parsed)
+        )
+        self.assertGreaterEqual(
+            now, parsed,
+            'parsed date {} happened in the future'.format(parsed)
+        )
+
     def verify_catalog(self, expected):
         self.assertTrue(os.path.exists('./target/catalog.json'))
 
@@ -51,6 +71,8 @@ def verify_catalog(self, expected):
 
         my_schema_name = self.unique_schema()
         self.assertIn(my_schema_name, catalog)
+        self.assertIn('generated_at', catalog)
+        self.assertRecent(catalog.pop('generated_at'))
         my_schema = catalog[my_schema_name]
         self.assertEqual(expected, my_schema)
 
@@ -162,14 +184,15 @@ def verify_manifest(self, expected_manifest):
 
         self.assertEqual(
             set(manifest),
-            {'nodes', 'macros', 'parent_map', 'child_map'}
+            {'nodes', 'macros', 'parent_map', 'child_map', 'generated_at'}
         )
 
         self.verify_manifest_macros(manifest)
-        manifest_without_macros = {
-            k: v for k, v in manifest.items() if k != 'macros'
+        manifest_without_extras = {
+            k: v for k, v in manifest.items()
+            if k not in {'macros', 'generated_at'}
         }
-        self.assertEqual(manifest_without_macros, expected_manifest)
+        self.assertEqual(manifest_without_extras, expected_manifest)
 
     @use_profile('postgres')
     def test__postgres__run_and_generate(self):
@@ -298,7 +321,7 @@ def test__bigquery__run_and_generate(self):
             {
                 'name': 'id',
                 'index': 1,
-                'type': 'INTEGER',
+                'type': 'INT64',
                 'comment': None,
             },
             {
@@ -361,31 +384,31 @@ def test__bigquery__nested_models(self):
             {
                 "name": "field_1",
                 "index": 1,
-                "type": "INTEGER",
+                "type": "INT64",
                 "comment": None
             },
             {
                 "name": "field_2",
                 "index": 2,
-                "type": "INTEGER",
+                "type": "INT64",
                 "comment": None
             },
             {
                 "name": "field_3",
                 "index": 3,
-                "type": "INTEGER",
+                "type": "INT64",
                 "comment": None
             },
             {
                 "name": "nested_field.field_4",
                 "index": 4,
-                "type": "INTEGER",
+                "type": "INT64",
                 "comment": None
             },
             {
                 "name": "nested_field.field_5",
                 "index": 5,
-                "type": "INTEGER",
+                "type": "INT64",
                 "comment": None
             }
         ]

test/integration/034_redshift_test/test_late_binding_view.py

@@ -0,0 +1,21 @@
+import json
+import os
+
+from nose.plugins.attrib import attr
+from test.integration.base import DBTIntegrationTest
+
+
+class TestLateBindingView(DBTIntegrationTest):
+    @property
+    def schema(self):
+        return 'late_binding_view_033'
+
+    @staticmethod
+    def dir(path):
+        return os.path.normpath(
+            os.path.join('test/integration/033_redshift_test', path)
+        )
+
+    @property
+    def models(self):
+        return self.dir("models")

test/unit/test_manifest.py

@@ -5,6 +5,8 @@
 
 import dbt.flags
 from dbt.contracts.graph.parsed import ParsedNode, ParsedManifest
+from dbt.utils import timestring
+import freezegun
 
 class ManifestTest(unittest.TestCase):
     def setUp(self):
@@ -151,17 +153,23 @@ def setUp(self):
             ),
         }
 
+    @freezegun.freeze_time('2018-02-14T09:15:13Z')
     def test__no_nodes(self):
-        manifest = ParsedManifest(nodes={}, macros={})
+        manifest = ParsedManifest(nodes={}, macros={},
+                                  generated_at=timestring())
         self.assertEqual(
             manifest.serialize(),
-            {'nodes': {}, 'macros': {}, 'parent_map': {}, 'child_map': {}}
+            {'nodes': {}, 'macros': {}, 'parent_map': {}, 'child_map': {},
+             'generated_at': '2018-02-14T09:15:13Z'}
         )
 
+    @freezegun.freeze_time('2018-02-14T09:15:13Z')
     def test__nested_nodes(self):
         nodes = copy.copy(self.nested_nodes)
-        manifest = ParsedManifest(nodes=nodes, macros={})
+        manifest = ParsedManifest(nodes=nodes, macros={},
+                                  generated_at=timestring())
         serialized = manifest.serialize()
+        self.assertEqual(serialized['generated_at'], '2018-02-14T09:15:13Z')
         parent_map = serialized['parent_map']
         child_map = serialized['child_map']
         # make sure there aren't any extra/missing keys.
@@ -220,7 +228,8 @@ def test__nested_nodes(self):
 
     def test__to_flat_graph(self):
         nodes = copy.copy(self.nested_nodes)
-        manifest = ParsedManifest(nodes=nodes, macros={})
+        manifest = ParsedManifest(nodes=nodes, macros={},
+                                  generated_at=timestring())
         flat_graph = manifest.to_flat_graph()
         flat_nodes = flat_graph['nodes']
         self.assertEqual(set(flat_graph), set(['nodes', 'macros']))

test/unit/test_parser.py

@@ -4,6 +4,7 @@
 
 import dbt.flags
 from dbt.parser import ModelParser, MacroParser, DataTestParser, SchemaParser, ParserUtils
+from dbt.utils import timestring
 
 from dbt.node_types import NodeType
 from dbt.contracts.graph.parsed import ParsedManifest, ParsedNode, ParsedMacro
@@ -702,6 +703,7 @@ def test__process_refs__packages(self):
         manifest = ParsedManifest(
             nodes={k: ParsedNode(**v) for (k,v) in graph['nodes'].items()},
             macros={k: ParsedMacro(**v) for (k,v) in graph['macros'].items()},
+            generated_at=timestring(),
         )
 
         processed_manifest = ParserUtils.process_refs(manifest, 'root')