Added Calendar Heatmap (#475)

* Added sqlite Grains

* Calendar heatmap visualization

* Linting

* Explicit metric setting was breaking tests

* Python linting

* Code cleanup + review

* [fixing the build] a new version of eslint is more picky

* Linting

* Added sqlite Grains

* Calendar heatmap visualization

* Linting

* Linting

* Explicit metric setting was breaking tests

* Python linting

* Code cleanup + review

caravel/assets/javascripts/modules/caravel.js

@@ -27,7 +27,8 @@ var sourceMap = {
   table: 'table.js',
   word_cloud: 'word_cloud.js',
   world_map: 'world_map.js',
-  treemap: 'treemap.js'
+  treemap: 'treemap.js',
+  cal_heatmap: 'cal_heatmap.js'
 };
 
 var color = function () {

caravel/assets/visualizations/cal_heatmap.css

@@ -0,0 +1,3 @@
+.cal_heatmap .slice_container {
+  padding: 10px;
+}

caravel/assets/visualizations/cal_heatmap.js

@@ -0,0 +1,58 @@
+// JS
+var d3 = window.d3 || require('d3');
+
+// CSS
+require('./cal_heatmap.css');
+require('../node_modules/cal-heatmap/cal-heatmap.css');
+
+var CalHeatMap = require('cal-heatmap');
+
+function calHeatmap(slice) {
+
+  var div = d3.select(slice.selector);
+  var cal = new CalHeatMap();
+
+  var render = function () {
+    d3.json(slice.jsonEndpoint(), function (error, json) {
+
+      if (error !== null) {
+        slice.error(error.responseText);
+        return '';
+      }
+
+      div.selectAll("*").remove();
+      cal = new CalHeatMap();
+
+      var timestamps = json.data["timestamps"],
+          extents = d3.extent(Object.keys(timestamps), function (key) {
+            return timestamps[key];
+          }),
+          step = (extents[1] - extents[0]) / 5;
+
+      try {
+        cal.init({
+          start: json.data["start"],
+          data: timestamps,
+          itemSelector: slice.selector,
+          tooltip: true,
+          domain: json.data["domain"],
+          subDomain: json.data["subdomain"],
+          range: json.data["range"],
+          browsing: true,
+          legend: [extents[0], extents[0]+step, extents[0]+step*2, extents[0]+step*3]
+        });
+      } catch (e) {
+        slice.error(e);
+      }
+
+      slice.done(json);
+    });
+  };
+
+  return {
+    render: render,
+    resize: render
+  };
+}
+
+module.exports = calHeatmap;

caravel/bin/caravel

@@ -85,6 +85,9 @@ def load_examples(load_test_data):
     print("Loading [Birth names]")
     data.load_birth_names()
 
+    print("Loading [Random time series data]")
+    data.load_random_time_series_data()
+
     if load_test_data:
         print("Loading [Unicode test data]")
         data.load_unicode_test_data()

caravel/data/__init__.py

@@ -895,3 +895,57 @@ def load_unicode_test_data():
     dash.slices = [slc]
     db.session.merge(dash)
     db.session.commit()
+
+
+def load_random_time_series_data():
+    """Loading random time series data from a zip file in the repo"""
+    with gzip.open(os.path.join(DATA_FOLDER, 'random_time_series.json.gz')) as f:
+        pdf = pd.read_json(f)
+    pdf.ds = pd.to_datetime(pdf.ds, unit='s')
+    pdf.to_sql(
+        'random_time_series',
+        db.engine,
+        if_exists='replace',
+        chunksize=500,
+        dtype={
+            'ds': DateTime,
+        },
+        index=False)
+    print("Done loading table!")
+    print("-" * 80)
+
+    print("Creating table reference")
+    obj = db.session.query(TBL).filter_by(table_name='random_time_series').first()
+    if not obj:
+        obj = TBL(table_name='random_time_series')
+    obj.main_dttm_col = 'ds'
+    obj.database = get_or_create_db(db.session)
+    obj.is_featured = False
+    db.session.merge(obj)
+    db.session.commit()
+    obj.fetch_metadata()
+    tbl = obj
+
+    slice_data = {
+        "datasource_id": "6",
+        "datasource_name": "random_time_series",
+        "datasource_type": "table",
+        "granularity": "day",
+        "row_limit": config.get("ROW_LIMIT"),
+        "since": "1 year ago",
+        "until": "now",
+        "where": "",
+        "viz_type": "cal_heatmap",
+        "domain_granularity": "month",
+        "subdomain_granularity": "day",
+    }
+
+    print("Creating a slice")
+    slc = Slice(
+        slice_name="Calendar Heatmap",
+        viz_type='cal_heatmap',
+        datasource_type='table',
+        table=tbl,
+        params=get_slice_json(slice_data),
+    )
+    merge_slice(slc)

caravel/forms.py

@@ -253,6 +253,29 @@ def __init__(self, viz):
                     "The time granularity for the visualization. Note that you "
                     "can type and use simple natural language as in '10 seconds', "
                     "'1 day' or '56 weeks'")),
+            'domain_granularity': SelectField(
+                'Domain', default="month",
+                choices=self.choicify([
+                    'hour',
+                    'day',
+                    'week',
+                    'month',
+                    'year',
+                ]),
+                description=(
+                    "The time unit used for the grouping of blocks")),
+            'subdomain_granularity': SelectField(
+                'Subdomain', default="day",
+                choices=self.choicify([
+                    'min',
+                    'hour',
+                    'day',
+                    'week',
+                    'month',
+                ]),
+                description=(
+                    "The time unit for each block. Should be a smaller unit than "
+                    "domain_granularity. Should be larger or equal to Time Grain")),
             'link_length': FreeFormSelectField(
                 'Link Length', default="200",
                 choices=self.choicify([

caravel/models.py

@@ -406,6 +406,12 @@ def grains(self):
                 Grain("month", "DATE(DATE_SUB({col}, "
                       "INTERVAL DAYOFMONTH({col}) - 1 DAY))"),
             ),
+            'sqlite': (
+                Grain('Time Column', '{col}'),
+                Grain('day', 'DATE({col})'),
+                Grain("week", "DATE({col}, -strftime('%w', {col}) || ' days')"),
+                Grain("month", "DATE({col}, -strftime('%d', {col}) || ' days')"),
+            ),
             'postgresql': (
                 Grain("Time Column", "{col}"),
                 Grain("second", "DATE_TRUNC('second', {col})"),

caravel/viz.py

@@ -23,6 +23,7 @@
 from six import string_types
 from werkzeug.datastructures import ImmutableMultiDict
 from werkzeug.urls import Href
+from dateutil import relativedelta as rdelta
 
 from caravel import app, utils, cache
 from caravel.forms import FormFactory
@@ -541,6 +542,67 @@ def get_data(self):
         return chart_data
 
 
+class CalHeatmapViz(BaseViz):
+
+    """Calendar heatmap."""
+
+    viz_type = "cal_heatmap"
+    verbose_name = "Calender Heatmap"
+    credits = (
+        '<a href=https://github.com/wa0x6e/cal-heatmap>cal-heatmap</a>')
+    is_timeseries = True
+    fieldsets = ({
+        'label': None,
+        'fields': (
+            'metric',
+            'domain_granularity',
+            'subdomain_granularity',
+        ),
+    },)
+
+    def get_df(self, query_obj=None):
+        df = super(CalHeatmapViz, self).get_df(query_obj)
+        return df
+
+    def get_data(self):
+        df = self.get_df()
+        form_data = self.form_data
+
+        df.columns = ["timestamp", "metric"]
+        timestamps = {str(obj["timestamp"].value / 10**9):
+                      obj.get("metric") for obj in df.to_dict("records")}
+
+        start = utils.parse_human_datetime(form_data.get("since"))
+        end = utils.parse_human_datetime(form_data.get("until"))
+        domain = form_data.get("domain_granularity")
+        diff_delta = rdelta.relativedelta(end, start)
+        diff_secs = (end - start).total_seconds()
+
+        if domain == "year":
+            range_ = diff_delta.years + 1
+        elif domain == "month":
+            range_ = diff_delta.years * 12 + diff_delta.months + 1
+        elif domain == "week":
+            range_ = diff_delta.years * 53 + diff_delta.weeks + 1
+        elif domain == "day":
+            range_ = diff_secs // (24*60*60) + 1
+        else:
+            range_ = diff_secs // (60*60) + 1
+
+        return {
+            "timestamps": timestamps,
+            "start": start,
+            "domain": domain,
+            "subdomain": form_data.get("subdomain_granularity"),
+            "range": range_,
+        }
+
+    def query_obj(self):
+        qry = super(CalHeatmapViz, self).query_obj()
+        qry["metrics"] = [self.form_data["metric"]]
+        return qry
+
+
 class NVD3Viz(BaseViz):
 
     """Base class for all nvd3 vizs"""
@@ -1572,6 +1634,7 @@ def get_data(self):
     HeatmapViz,
     BoxPlotViz,
     TreemapViz,
+    CalHeatmapViz,
 ]
 
 viz_types = OrderedDict([(v.viz_type, v) for v in viz_types_list

docs/gallery.rst

@@ -70,3 +70,6 @@ Gallery
 .. image:: _static/img/viz_thumbnails/treemap.png
    :scale: 25 %
 
+.. image:: _static/img/viz_thumbnails/cal_heatmap.png
+   :scale: 25 %
+