Stable

panoramix/TODO.md

@@ -1,3 +1,3 @@
 # TODO
-* Default URL params per datasource
-* Get config metrics to work
+* Multi-filters
+* multi-metrics

panoramix/app/models.py

@@ -5,14 +5,9 @@
 from sqlalchemy.orm import relationship
 from app import db, utils
 from dateutil.parser import parse
-"""
+import json
 
-You can use the extra Flask-AppBuilder fields and Mixin's
 
-AuditMixin will add automatic timestamp of created and modified by who
-
-
-"""
 client = utils.get_pydruid_client()
 
 class Datasource(Model, AuditMixin):
@@ -22,12 +17,11 @@ class Datasource(Model, AuditMixin):
     is_featured = Column(Boolean, default=False)
     is_hidden = Column(Boolean, default=False)
     description = Column(Text)
-    columns = relationship('Column', backref='datasource')
-    udfs = relationship('JavascriptUdf', backref='datasource')
+    default_endpoint = Column(Text)
 
     @property
-    def metrics(self):
-        return [col.column_name for col in self.columns if not col.groupby]
+    def metrics_combo(self):
+        return [(m.metric_name, m.verbose_name) for m in self.metrics]
 
     def __repr__(self):
         return self.datasource_name
@@ -37,24 +31,41 @@ def datasource_link(self):
         url = "/panoramix/datasource/{}/".format(self.datasource_name)
         return '<a href="{url}">{self.datasource_name}</a>'.format(**locals())
 
+    def get_metric_obj(self, metric_name):
+        return [
+            m.json_obj for m in self.metrics
+            if m.metric_name == metric_name
+        ][0]
+
     @classmethod
     def latest_metadata(cls, name):
         results = client.time_boundary(datasource=name)
+        print "---" * 100
+        print name
+        print results
         max_time = results[0]['result']['maxTime']
         max_time = parse(max_time)
         intervals = (max_time - timedelta(seconds=1)).isoformat() + '/'
         intervals += (max_time + timedelta(seconds=1)).isoformat()
         segment_metadata = client.segment_metadata(
             datasource=name,
             intervals=intervals)
-        return segment_metadata[-1]['columns']
+        print segment_metadata
+        if segment_metadata:
+            return segment_metadata[-1]['columns']
+
+    def generate_metrics(self):
+        for col in self.columns:
+            col.generate_metrics()
 
     @classmethod
     def sync_to_db(cls, name):
         datasource = db.session.query(cls).filter_by(datasource_name=name).first()
         if not datasource:
             db.session.add(cls(datasource_name=name))
         cols = cls.latest_metadata(name)
+        if not cols:
+            return
         for col in cols:
             col_obj = (
                 db.session
@@ -71,7 +82,7 @@ def sync_to_db(cls, name):
                 col_obj.filterable = True
             if col_obj:
                 col_obj.type = cols[col]['type']
-
+            col_obj.generate_metrics()
         db.session.commit()
 
     @property
@@ -87,26 +98,29 @@ def filterable_column_names(self):
         return sorted([c.column_name for c in self.columns if c.filterable])
 
 
-class JavascriptUdf(Model, AuditMixin):
-    __tablename__ = 'udfs'
+class Metric(Model):
+    __tablename__ = 'metrics'
     id = Column(Integer, primary_key=True)
+    metric_name = Column(String(512))
+    verbose_name = Column(String(1024))
+    metric_type = Column(String(32))
     datasource_name = Column(
         String(256),
         ForeignKey('datasources.datasource_name'))
-    udf_name = Column(String(256))
-    column_list = Column(String(1024))
-    code = Column(Text)
-
-    def __repr__(self):
-        return self.udf_name
+    datasource = relationship('Datasource', backref='metrics')
+    json = Column(Text)
 
+    @property
+    def json_obj(self):
+        return json.loads(self.json)
 
 class Column(Model, AuditMixin):
     __tablename__ = 'columns'
     id = Column(Integer, primary_key=True)
     datasource_name = Column(
         String(256),
         ForeignKey('datasources.datasource_name'))
+    datasource = relationship('Datasource', backref='columns')
     column_name = Column(String(256))
     is_active = Column(Boolean, default=True)
     type = Column(String(32))
@@ -120,3 +134,73 @@ class Column(Model, AuditMixin):
     def __repr__(self):
         return self.column_name
 
+    @property
+    def isnum(self):
+        return self.type in ('LONG', 'DOUBLE')
+
+    def generate_metrics(self):
+        M = Metric
+        metrics = []
+        metrics.append(Metric(
+            metric_name='count',
+            verbose_name='COUNT(*)',
+            metric_type='count',
+            json=json.dumps({
+                'type': 'count', 'name': 'count'})
+        ))
+        if self.datasource.datasource_name == 'platform' and self.column_name=='subject_id':
+            print((self.column_name, self.type, self.isnum))
+
+        if self.sum and self.isnum:
+            mt = self.type.lower() + 'Sum'
+            name='sum__' + self.column_name
+            metrics.append(Metric(
+                metric_name=name,
+                metric_type='sum',
+                verbose_name='SUM({})'.format(self.column_name),
+                json=json.dumps({
+                    'type': mt, 'name': name, 'fieldName': self.column_name})
+            ))
+        if self.min and self.isnum:
+            mt = self.type.lower() + 'Min'
+            name='min__' + self.column_name
+            metrics.append(Metric(
+                metric_name=name,
+                metric_type='min',
+                verbose_name='MIN({})'.format(self.column_name),
+                json=json.dumps({
+                    'type': mt, 'name': name, 'fieldName': self.column_name})
+            ))
+        if self.max and self.isnum:
+            mt = self.type.lower() + 'Max'
+            name='max__' + self.column_name
+            metrics.append(Metric(
+                metric_name=name,
+                metric_type='max',
+                verbose_name='MAX({})'.format(self.column_name),
+                json=json.dumps({
+                    'type': mt, 'name': name, 'fieldName': self.column_name})
+            ))
+        if self.count_distinct:
+            mt = 'count_distinct'
+            name='count_distinct__' + self.column_name
+            metrics.append(Metric(
+                metric_name=name,
+                verbose_name='COUNT(DISTINCT {})'.format(self.column_name),
+                metric_type='count_distinct',
+                json=json.dumps({
+                    'type': 'cardinality',
+                    'name': name,
+                    'fieldNames': [self.column_name]})
+            ))
+        for metric in metrics:
+            m = (
+                db.session.query(M)
+                .filter(M.datasource_name==self.datasource_name)
+                .filter(M.metric_name==metric.metric_name)
+                .first()
+            )
+            metric.datasource_name = self.datasource_name
+            if not m:
+                db.session.add(metric)
+                db.session.commit()

panoramix/app/templates/appbuilder/navbar.html

@@ -0,0 +1,35 @@
+
+
+
+{% set menu = appbuilder.menu %}
+{% set languages = appbuilder.languages %}
+
+<div class="navbar {{menu.extra_classes}}" role="navigation">
+   <div class="container">
+        <div class="navbar-header">
+            <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse">
+                <span class="icon-bar"></span>
+                <span class="icon-bar"></span>
+                <span class="icon-bar"></span>
+            </button>
+            {% if appbuilder.app_icon %}
+                <a class="navbar-brand" style="padding:10px;" href="{{appbuilder.get_url_for_index}}">
+                <img width="30" src="{{appbuilder.app_icon}}" >
+                </a>
+            {% endif %}                
+                <span class="navbar-brand">
+                <a href="{{appbuilder.get_url_for_index}}">
+                {{ appbuilder.app_name }}
+                </a>
+                </span>
+        </div>
+        <div class="navbar-collapse collapse">
+            <ul class="nav navbar-nav">
+                {% include 'appbuilder/navbar_menu.html' %}
+            </ul>
+            <ul class="nav navbar-nav navbar-right">
+                {% include 'appbuilder/navbar_right.html' %}
+            </ul>
+        </div>
+   </div>
+</div>

panoramix/app/templates/panoramix/base.html

@@ -4,5 +4,8 @@
 {{super()}}
 <link rel="shortcut icon" href="{{ url_for('static', filename='chaudron.png') }}">
 <style>
+.navbar-brand a {
+  color: white;
+}
 </style>
 {% endblock %}

panoramix/app/views.py

@@ -38,8 +38,7 @@ class QueryForm(OmgWtForm):
         viz_type = SelectField(
             'Viz',
             choices=[(k, v.verbose_name) for k, v in viz.viz_types.items()])
-        metric = SelectField(
-            'Metric', choices=[(m, m) for m in datasource.metrics])
+        metric = SelectField('Metric', choices=datasource.metrics_combo)
         groupby = SelectMultipleField(
             'Group by', choices=[
                 (s, s) for s in datasource.groupby_column_names])
@@ -61,22 +60,33 @@ class QueryForm(OmgWtForm):
 
 class ColumnInlineView(CompactCRUDMixin, ModelView):
     datamodel = SQLAInterface(models.Column)
-    edit_columns = ['column_name', 'groupby', 'count_distinct', 'sum', 'min', 'max']
-    list_columns = ['column_name', 'groupby', 'count_distinct', 'sum', 'min', 'max']
+    edit_columns = [
+        'column_name', 'datasource', 'groupby', 'count_distinct',
+        'sum', 'min', 'max']
+    list_columns = [
+        'column_name', 'type', 'groupby', 'count_distinct',
+        'sum', 'min', 'max']
     can_delete = False
 appbuilder.add_view_no_menu(ColumnInlineView)
 
-class JavascriptUdfInlineView(CompactCRUDMixin, ModelView):
-    datamodel = SQLAInterface(models.JavascriptUdf)
-    edit_columns = ['udf_name', 'column_list', 'code']
-appbuilder.add_view_no_menu(JavascriptUdfInlineView)
+
+class MetricInlineView(CompactCRUDMixin, ModelView):
+    datamodel = SQLAInterface(models.Metric)
+    list_columns = ['metric_name', 'verbose_name', 'metric_type' ]
+    edit_columns = [
+        'metric_name', 'verbose_name', 'metric_type', 'datasource', 'json']
+    add_columns = [
+        'metric_name', 'verbose_name', 'metric_type', 'datasource', 'json']
+appbuilder.add_view_no_menu(MetricInlineView)
 
 
 class DatasourceModelView(ModelView):
     datamodel = SQLAInterface(models.Datasource)
-    list_columns = ['datasource_link', 'is_featured' ]
-    related_views = [ColumnInlineView, JavascriptUdfInlineView]
-    edit_columns = ['datasource_name', 'description', 'is_featured', 'is_hidden']
+    list_columns = ['datasource_link', 'is_featured', 'is_hidden']
+    related_views = [ColumnInlineView, MetricInlineView]
+    edit_columns = [
+        'datasource_name', 'description', 'is_featured', 'is_hidden',
+        'default_endpoint']
     page_size = 100
 
 
@@ -90,13 +100,18 @@ class DatasourceModelView(ModelView):
 class Panoramix(BaseView):
     @expose("/datasource/<datasource_name>/")
     def datasource(self, datasource_name):
-        viz_type = request.args.get("viz_type", "table")
+        viz_type = request.args.get("viz_type")
+
         datasource = (
             db.session
             .query(models.Datasource)
             .filter_by(datasource_name=datasource_name)
             .first()
         )
+        if not viz_type and datasource.default_endpoint:
+            return redirect(datasource.default_endpoint)
+        if not viz_type:
+            viz_type = "table"
         obj = viz.viz_types[viz_type](
             datasource,
             form_class=form_factory(datasource, request.args),
@@ -107,7 +122,7 @@ def datasource(self, datasource_name):
 
 
     @expose("/refresh_datasources/")
-    def datasources(self):
+    def refresh_datasources(self):
         import requests
         import json
         endpoint = (
@@ -126,4 +141,6 @@ def datasources(self):
     href='/panoramix/refresh_datasources/',
     category='Admin',
     icon="fa-cogs")
+
+#models.Metric.__table__.drop(db.engine)
 db.create_all()

panoramix/app/viz.py

@@ -23,7 +23,7 @@ def __init__(self, datasource, form_class, form_data, view):
         self.datasource = datasource
         self.form_class = form_class
         self.form_data = form_data
-        self.metric = form_data.get('metric')
+        self.metric = form_data.get('metric', 'count')
         self.df = self.bake_query()
         self.view = view
         if self.df is not None:
@@ -67,7 +67,10 @@ def query_obj(self):
         args = self.form_data
         groupby = args.getlist("groupby") or []
         granularity = args.get("granularity")
-        metric = "count"
+        aggregations = {
+            m.metric_name: m.json_obj
+            for m in ds.metrics if m.metric_name == self.metric
+        }
         limit = int(
             args.get("limit", config.ROW_LIMIT)) or config.ROW_LIMIT
         since = args.get("since", "all")
@@ -77,12 +80,12 @@ def query_obj(self):
             'granularity': granularity or 'all',
             'intervals': from_dttm + '/' + datetime.now().isoformat(),
             'dimensions': groupby,
-            'aggregations': {"count": agg.doublesum(metric)},
+            'aggregations': aggregations,
             'limit_spec': {
                 "type": "default",
                 "limit": limit,
                 "columns": [{
-                    "dimension": metric,
+                    "dimension": self.metric,
                     "direction": "descending",
                 }],
             },
@@ -151,9 +154,9 @@ def render(self):
             columns=[
                 col for col in df.columns if col not in ["timestamp", metric]],
             values=[metric])
+
         chart_js = serialize(
             df, kind=self.chart_kind, stacked=self.stacked, **CHART_ARGS)
-        print self.stacked
         return super(TimeSeriesViz, self).render(chart_js=chart_js)
 
     def bake_query(self):

panoramix/config.py

@@ -34,7 +34,7 @@
 APP_NAME = "Panoramix"
 
 # Uncomment to setup Setup an App icon
-#APP_ICON = "static/img/logo.jpg"
+APP_ICON = "/static/chaudron_white.png"
 
 #----------------------------------------------------
 # AUTHENTICATION CONFIG