diff --git a/base.html b/base.html new file mode 100644 index 0000000000000..12f0245b0925b --- /dev/null +++ b/base.html @@ -0,0 +1,86 @@ +{% import 'admin/layout.html' as layout with context -%} +{% import 'admin/static.html' as admin_static with context %} + + + + {% block title %}{% if admin_view.category %}{{ admin_view.category }} - {% endif %}{{ admin_view.name }} - {{ admin_view.admin.name }}{% endblock %} + {% block head_meta %} + + + + + + {% endblock %} + {% block head_css %} + + + + + + + {% endblock %} + {% block head %} + {% endblock %} + {% block head_tail %} + {% endblock %} + + + {% block page_body %} +
+ + + {% block messages %} + {{ layout.messages() }} + {% endblock %} + + {% set render_ctx = h.resolve_ctx() %} + + {% block body %}{% endblock %} +
+ {% endblock %} + + {% block tail_js %} + + + + + + {% endblock %} + + {% block tail %} + {% endblock %} + + diff --git a/panoramix/app.py b/panoramix/app.py index 2737c6cf9bb6a..d65f4b76ba1b6 100644 --- a/panoramix/app.py +++ b/panoramix/app.py @@ -1,40 +1,78 @@ from dateutil.parser import parse from datetime import timedelta -from flask import Flask, request, Blueprint -from panoramix import settings, viz +from flask import Flask, request, Blueprint, url_for, Markup +from flask.ext.sqlalchemy import SQLAlchemy +from flask.ext.admin import Admin, BaseView, expose, AdminIndexView +from panoramix import settings, viz, models from flask_bootstrap import Bootstrap from wtforms import Form, SelectMultipleField, SelectField, TextField +from wtforms.fields import Field import pandas as pd +from flask_admin.contrib import sqla pd.set_option('display.max_colwidth', -1) -since_l = { - '1hour': timedelta(hours=1), - '1day': timedelta(days=1), - '7days': timedelta(days=7), - '28days': timedelta(days=28), - 'all': timedelta(days=365*100) -} client = settings.get_pydruid_client() +class OmgWtForm(Form): + field_order = ( + 'viz_type', 'granularity', 'since', 'group_by', 'limit') + def fields(self): + fields = [] + for field in self.field_order: + if hasattr(self, field): + obj = getattr(self, field) + if isinstance(obj, Field): + fields.append(getattr(self, field)) + return fields + + class DruidDataSource(object): def __init__(self, name): self.name = name self.cols = self.latest_metadata() - self.col_names = sorted([col for col in self.cols.keys()]) + self.col_names = sorted([ + col for col in self.cols.keys() + if not col.startswith("_") and col not in self.metrics]) def latest_metadata(self): - max_time = client.time_boundary( - datasource=self.name)[0]['result']['maxTime'] + results = client.time_boundary(datasource=self.name) + max_time = results[0]['result']['maxTime'] max_time = parse(max_time) intervals = (max_time - timedelta(seconds=1)).isoformat() + '/' - intervals += max_time.isoformat() - return client.segment_metadata( + intervals += (max_time + timedelta(seconds=1)).isoformat() + segment_metadata = client.segment_metadata( datasource=self.name, - intervals=intervals)[-1]['columns'] + intervals=intervals) + return segment_metadata[-1]['columns'] + + @property + def metrics(self): + return [ + k for k, v in self.cols.items() + if v['type'] != 'STRING' and not k.startswith('_')] + + def sync_to_db(self): + DS = Datasource + datasource = DS.query.filter_by(datasource_name=self.name).first() + if not datasource: + db.session.add(DS(datasource_name=self.name)) + for col in self.cols: + col_obj = Column.query.filter_by(datasource_name=self.name, column_name=col).first() + datatype = self.cols[col]['type'] + if not col_obj: + col_obj = Column(datasource_name=self.name, column_name=col) + db.session.add(col_obj) + if datatype == "STRING": + col_obj.groupby = True + if col_obj: + col_obj.type = self.cols[col]['type'] + + db.session.commit() + def form_factory(datasource, form_args=None): grain = ['all', 'none', 'minute', 'hour', 'day'] @@ -50,48 +88,154 @@ def form_factory(datasource, form_args=None): except: pass - class QueryForm(Form): + class QueryForm(OmgWtForm): viz_type = SelectField( 'Viz', choices=[(k, v.verbose_name) for k, v in viz.viz_types.items()]) + metric = SelectField( + 'Metric', choices=[(m, m) for m in datasource.metrics]) groupby = SelectMultipleField( 'Group by', choices=[(m, m) for m in datasource.col_names]) granularity = SelectField( 'Granularity', choices=[(g, g) for g in grain]) since = SelectField( - 'Since', choices=[(s, s) for s in since_l.keys()]) + 'Since', choices=[(s, s) for s in settings.since_l.keys()], + default="all") limit = SelectField( 'Limit', choices=[(s, s) for s in limits]) - flt_col_1 = SelectField( - 'Filter 1', choices=[(m, m) for m in datasource.col_names]) - flt_op_1 = SelectField( - 'Filter 1', choices=[(m, m) for m in ['==', '!=', 'in',]]) - flt_eq_1 = TextField("Super") + for i in range(10): + setattr(QueryForm, 'flt_col_' + str(i), SelectField( + 'Filter 1', choices=[(m, m) for m in datasource.col_names])) + setattr(QueryForm, 'flt_op_' + str(i), SelectField( + 'Filter 1', choices=[(m, m) for m in ['==', '!=', 'in',]])) + setattr(QueryForm, 'flt_eq_' + str(i), TextField("Super")) return QueryForm - +""" bp = Blueprint( 'panoramix', __name__, template_folder='templates', static_folder='static') - - -@bp.route("/datasource//") -def datasource(name): - viz_type = request.args.get("viz_type", "table") - datasource = DruidDataSource(name) - obj = viz.viz_types[viz_type]( - datasource, - form_class=form_factory(datasource, request.args), - form_data=request.args) - return obj.render() - +""" + + + +app = Flask(__name__) +app.config['SQLALCHEMY_DATABASE_URI'] = settings.SQLALCHEMY_DATABASE_URI +db = SQLAlchemy(app) +app.secret_key = "monkeys" +#app.register_blueprint(bp, url_prefix='/panoramix') +Bootstrap(app) +admin = Admin( + app, name = "Panoramix", + template_mode='bootstrap3') + + + +class Datasource(db.Model): + __tablename__ = 'datasources' + id = db.Column(db.Integer, primary_key=True) + datasource_name = db.Column(db.String(256), unique=True) + is_featured = db.Column(db.Boolean, default=False) + is_hidden = db.Column(db.Boolean, default=False) + description = db.Column(db.Text) + created_dttm = db.Column(db.DateTime, default=db.func.now()) + + +class Column(db.Model): + __tablename__ = 'columns' + id = db.Column(db.Integer, primary_key=True) + datasource_name = db.Column( + db.String(256), + db.ForeignKey('datasources.datasource_name')) + column_name = db.Column(db.String(256)) + is_active = db.Column(db.Boolean, default=True) + type = db.Column(db.String(32)) + groupby = db.Column(db.Boolean, default=False) + count_distinct = db.Column(db.Boolean, default=False) + sum = db.Column(db.Boolean, default=False) + max = db.Column(db.Boolean, default=False) + min = db.Column(db.Boolean, default=False) + datasource = db.relationship('Datasource', + backref=db.backref('columns', lazy='dynamic')) + + def __repr__(self): + return self.column_name + + +class JsUdf(db.Model): + __tablename__ = 'udfs' + id = db.Column(db.Integer, primary_key=True) + datasource_name = db.Column( + db.String(256), + db.ForeignKey('datasources.datasource_name')) + udf_name = db.Column(db.String(256)) + column_list = db.Column(db.String(1024)) + code = db.Column(db.Text) + datasource = db.relationship('Datasource', + backref=db.backref('udfs', lazy='dynamic')) + + +def datasource_link(v, c, m, p): + url = '/admin/datasourceview/datasource/{}/'.format(m.datasource_name) + return Markup('{m.datasource_name}'.format(**locals())) + + +class DatasourceAdmin(sqla.ModelView): + inline_models = (Column, JsUdf,) + column_formatters = dict(datasource_name=datasource_link) + + +class DatasourceView(BaseView): + @expose('/') + def index(self): + return "" + @expose("/datasource//") + def datasource(self, datasource_name): + viz_type = request.args.get("viz_type", "table") + datasource = DruidDataSource(datasource_name) + obj = viz.viz_types[viz_type]( + datasource, + form_class=form_factory(datasource, request.args), + form_data=request.args, + admin_view=self) + if obj.df is None or obj.df.empty: + return obj.render_no_data() + return obj.render() + + + @expose("/datasources/") + def datasources(): + import requests + import json + endpoint = ( + "http://{COORDINATOR_HOST}:{COORDINATOR_PORT}/" + "{COORDINATOR_BASE_ENDPOINT}/datasources" + ).format(**settings.__dict__) + datasources = json.loads(requests.get(endpoint).text) + for datasource in datasources: + ds = DruidDataSource(datasource) + ds.sync_to_db() + + return json.dumps(datasources, indent=4) + + + @expose("/datasource_metadata//") + def datasource_metadata(name): + import requests + import json + endpoint = ( + "http://{COORDINATOR_HOST}:{COORDINATOR_PORT}/" + "{COORDINATOR_BASE_ENDPOINT}/datasource" + ).format(**settings.__dict__) + + return str(datasources) + +admin.add_view(DatasourceView(name="Datasource")) if __name__ == '__main__': - app = Flask(__name__) - app.secret_key = "monkeys" - app.register_blueprint(bp, url_prefix='/panoramix') - Bootstrap(app) + db.create_all() + admin.add_view(DatasourceAdmin(Datasource, db.session, name="Datasources")) app.debug = True app.run(host='0.0.0.0', port=settings.FLASK_APP_PORT) diff --git a/panoramix/models.py b/panoramix/models.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/panoramix/settings.py b/panoramix/settings.py index dc235b89d5b92..59215bc27018a 100644 --- a/panoramix/settings.py +++ b/panoramix/settings.py @@ -1,11 +1,26 @@ +from datetime import timedelta + FLASK_APP_PORT = 8088 ROW_LIMIT = 10000 +SQLALCHEMY_DATABASE_URI = "sqlite:////tmp/panoramix.db" DRUID_HOST = '10.181.47.80' -DRUID_PORT = 8088 +DRUID_PORT = 8080 DRUID_BASE_ENDPOINT = 'druid/v2' +COORDINATOR_HOST = '10.168.176.249' +COORDINATOR_PORT = '8080' +COORDINATOR_BASE_ENDPOINT = 'druid/coordinator/v1' + +since_l = { + '1hour': timedelta(hours=1), + '1day': timedelta(days=1), + '7days': timedelta(days=7), + '28days': timedelta(days=28), + 'all': timedelta(days=365*100) +} + def get_pydruid_client(): from pydruid import client return client.PyDruid( diff --git a/panoramix/static/chaudron.png b/panoramix/static/chaudron.png new file mode 100644 index 0000000000000..c1dd4ed7495fd Binary files /dev/null and b/panoramix/static/chaudron.png differ diff --git a/panoramix/static/main.css b/panoramix/static/main.css index a6d67a4711909..19642c9eb791e 100644 --- a/panoramix/static/main.css +++ b/panoramix/static/main.css @@ -19,53 +19,10 @@ button.btn { border: 1px solid black; } -div.rich_doc { - padding: 5px 10px; - border: 1px solid #dddddd; - background: white; - border-radius: 4px; -} - -span.status_square { - width:10px; - height:10px; - border:1px solid grey; - display:inline-block; - padding-left: 0px; - cursor: pointer; -} -div.squares{ - float:right; - font-size: 1; -} -div.task_row{ -} -span.success{ - background-color:green; -} -span.up_for_retry{ - background-color:yellow; -} -span.started{ - background-color:lime; -} -span.error{ - background-color:red; -} -span.queued{ - background-color:gray; -} .tooltip-inner { text-align:left !important; font-size: 12px; } -input#execution_date { - margin-bottom: 0px; -} -table.highlighttable{ - width: 100%; - table-layout:fixed; -} div.linenodiv { padding-right: 1px !important; } @@ -97,18 +54,26 @@ input, select { font-family: monospace; } -#sql { - border: 1px solid #CCC; - border-radius: 5px; +table.dataframe { + font-size: 12px; +} +table.dataframe tbody tr td { + padding: 2px; +} +table thead th { + background-color: #F3F3F3; } -.ace_editor div { - font: inherit!important +table.dataframe.dataTable thead > tr > th { + padding: 10px 20px 10px 10px; } -#ace_container { - margin: 10px 0px; +table.dataTable.dataframe thead .sorting { + background: url('sort_both.png') no-repeat center right } -#sql_ace { - visibility: hidden; +table.dataTable.dataframe thead .sorting_desc { + background: url('sort_desc.png') no-repeat center right +} +table.dataTable.dataframe thead .sorting_asc { + background: url('sort_asc.png') no-repeat center right } .no-wrap { white-space: nowrap; @@ -123,85 +88,3 @@ body div.panel { .blur { filter:url(#blur-effect-1); } -div.legend_item { - -moz-border-radius: 5px/5px; - -webkit-border-radius: 5px 5px; - border-radius: 5px/5px; - float:right; - margin: 0px 10px 0px 0px; - padding:0px 5px; - border:solid 2px grey; - font-size: 12px; -} -div.legend_circle{ - -moz-border-radius: 10px/10px; - -webkit-border-radius: 10px 10px; - border-radius: 10px/10px; - width:15px; - height:15px; - border:1px solid grey; - float:left; - margin-top: 2px; - margin-right: 0px; -} -div.square { - width:12px; - height:12px; - float: right; - margin-top: 2px; - border:1px solid black; -} -.btn:active, .btn.active { - box-shadow: inset 0 6px 6px rgba(0, 0, 0, 0.4); -} - -.hll { background-color: #ffffcc } -.c { color: #408080; font-style: italic } /* Comment */ -.err { border: 1px solid #FF0000 } /* Error */ -.k { color: #008000; font-weight: bold } /* Keyword */ -.o { color: #666666 } /* Operator */ -.cm { color: #408080; font-style: italic } /* Comment.Multiline */ -.cp { color: #BC7A00 } /* Comment.Preproc */ -.c1 { color: #408080; font-style: italic } /* Comment.Single */ -.cs { color: #408080; font-style: italic } /* Comment.Special */ -.gd { color: #A00000 } /* Generic.Deleted */ -.ge { font-style: italic } /* Generic.Emph */ -.gr { color: #FF0000 } /* Generic.Error */ -.gh { color: #000080; font-weight: bold } /* Generic.Heading */ -.gi { color: #00A000 } /* Generic.Inserted */ -.go { color: #888888 } /* Generic.Output */ -.gp { color: #000080; font-weight: bold } /* Generic.Prompt */ -.gs { font-weight: bold } /* Generic.Strong */ -.gu { color: #800080; font-weight: bold } /* Generic.Subheading */ -.gt { color: #0044DD } /* Generic.Traceback */ -.kc { color: #008000; font-weight: bold } /* Keyword.Constant */ -.kd { color: #008000; font-weight: bold } /* Keyword.Declaration */ -.kn { color: #008000; font-weight: bold } /* Keyword.Namespace */ -.kp { color: #008000 } /* Keyword.Pseudo */ -.kr { color: #008000; font-weight: bold } /* Keyword.Reserved */ -.kt { color: #B00040 } /* Keyword.Type */ -.m { color: #666666 } /* Literal.Number */ -.s { color: #BA2121 } /* Literal.String */ -.na { color: #7D9029 } /* Name.Attribute */ -.nb { color: #008000 } /* Name.Builtin */ -.nc { color: #0000FF; font-weight: bold } /* Name.Class */ -.no { color: #880000 } /* Name.Constant */ -.nd { color: #AA22FF } /* Name.Decorator */ -.ni { color: #999999; font-weight: bold } /* Name.Entity */ -.ne { color: #D2413A; font-weight: bold } /* Name.Exception */ -.nf { color: #0000FF } /* Name.Function */ -.nl { color: #A0A000 } /* Name.Label */ -.nn { color: #0000FF; font-weight: bold } /* Name.Namespace */ -.nt { color: #008000; font-weight: bold } /* Name.Tag */ -.nv { color: #19177C } /* Name.Variable */ -.ow { color: #AA22FF; font-weight: bold } /* Operator.Word */ -.w { color: #bbbbbb } /* Text.Whitespace */ -.mb { color: #666666 } /* Literal.Number.Bin */ -.mf { color: #666666 } /* Literal.Number.Float */ -.mh { color: #666666 } /* Literal.Number.Hex */ -.mi { color: #666666 } /* Literal.Number.Integer */ -.mo { color: #666666 } /* Literal.Number.Oct */ -.sb { color: #BA2121 } /* Literal.String.Backtick */ -.sc { color: #BA2121 } /* Literal.String.Char */ -.sd { color: #BA2121; font-style: italic } /* Literal.String.Doc */ -.s2 { color: #BA2121 } /* Literal.String.Double */ diff --git a/panoramix/static/panoramix.jpg b/panoramix/static/panoramix.jpg new file mode 100644 index 0000000000000..b6eb231caed83 Binary files /dev/null and b/panoramix/static/panoramix.jpg differ diff --git a/panoramix/static/panoramix.png b/panoramix/static/panoramix.png new file mode 100644 index 0000000000000..3ce04c348b74c Binary files /dev/null and b/panoramix/static/panoramix.png differ diff --git a/panoramix/templates/index.html b/panoramix/templates/index.html new file mode 100644 index 0000000000000..1d75697efa533 --- /dev/null +++ b/panoramix/templates/index.html @@ -0,0 +1,2 @@ +{% extends "admin/base.html" %} + diff --git a/panoramix/templates/panoramix/base.html b/panoramix/templates/panoramix/base.html index 189ea81d9c332..c17cf1f5d2f6b 100644 --- a/panoramix/templates/panoramix/base.html +++ b/panoramix/templates/panoramix/base.html @@ -1,64 +1,2 @@ -{% extends "bootstrap/base.html" %} +{% extends "index.html" %} -{% block title %}Panoramix - A Druid UI{% endblock %} -{% block html_attribs %} lang="en"{% endblock %} - -{% block styles %} -{{super()}} - - - - -{% endblock %} - -{% block navbar %} - -{% endblock %} - -{% block scripts %} -{{ super() }} - - -{% endblock %} diff --git a/panoramix/templates/panoramix/datasource.html b/panoramix/templates/panoramix/datasource.html index 58fa6a939924d..ea8ce0b467155 100644 --- a/panoramix/templates/panoramix/datasource.html +++ b/panoramix/templates/panoramix/datasource.html @@ -1,44 +1,89 @@ {% extends "panoramix/base.html" %} -{% block content %} +{% block styles %} +{{super()}} + +{% endblock %} +{% block body %}
-
-

{{ datasource }}

-
-
{{ form.viz_type.label }}: {{ form.viz_type(class_="form-control select2") }}
-
{{ form.granularity.label }}: {{ form.granularity(class_="form-control select2") }}
-
{{ form.since.label }}: {{ form.since(class_="form-control select2") }}
-
{{ form.groupby.label }}: {{ form.groupby(class_="form-control select2") }}
-
{{ form.limit.label }}: {{ form.limit(class_="form-control select2_tags") }}
-
-

Filters

-
- {{ form.flt_col_1(class_="form-control select2") }} - {{ form.flt_op_1(class_="form-control select2 input-sm") }} - {{ form.flt_eq_1(class_="form-control") }} +
+

+ {{ datasource.name }} + +

+ +
+ +
{{ form.viz_type.label }}: {{ form.viz_type(class_="form-control") }}
+
{{ form.metric.label }}: {{ form.metric(class_="form-control select2") }}
+
{{ form.granularity.label }}: {{ form.granularity(class_="form-control") }}
+
{{ form.since.label }}: {{ form.since(class_="form-control") }}
+
{{ form.groupby.label }}: {{ form.groupby(class_="form-control") }}
+
{{ form.limit.label }}: {{ form.limit(class_="form-control") }}
+
+

Filters

+
+ {% for i in range(10) %} +
+
+ {{ form['flt_col_' ~ i](class_="form-control select2 inc") }} +
+
+ {{ form['flt_op_' ~ i](class_="form-control select2 input-sm inc") }} + {{ form['flt_eq_' ~ i](class_="form-control inc") }} +
-
- -
- -
-
+
+
+ {% endfor %} +
+ +
+ +
+ +
+
-
-

{{ verbose_viz_type }}

- {% block viz %} - {% endblock %} +
+

{{ viz.verbose_name }}

+
+ {% block viz %} + {% endblock %} - {% if debug %} -

Results

- 
-               {{ results }} 
-
+ {% if debug %} +

Results

+ 
+      {{ results }} 
+
-

Latest Segment Metadata

- 
+    
Latest Segment Metadata

+    
                {{ latest_metadata }} 
-            

-        {% endif %}
-
+ + {% endif %} +
{% endblock %} + +{% block tail %} +{{ super() }} + +{% endblock %} diff --git a/panoramix/templates/panoramix/no_data.html b/panoramix/templates/panoramix/no_data.html new file mode 100644 index 0000000000000..d2e30a6d803f6 --- /dev/null +++ b/panoramix/templates/panoramix/no_data.html @@ -0,0 +1,5 @@ +{% extends "panoramix/datasource.html" %} + +{% block viz %} +No data: review your incantations. +{% endblock %} diff --git a/panoramix/templates/panoramix/noadmin.html b/panoramix/templates/panoramix/noadmin.html new file mode 100644 index 0000000000000..679c0ba5592ac --- /dev/null +++ b/panoramix/templates/panoramix/noadmin.html @@ -0,0 +1,56 @@ +{% extends "bootstrap/base.html" %} + +{% block title %}Panoramix - A Druid UI{% endblock %} +{% block html_attribs %} lang="en"{% endblock %} + +{% block head %} +{{super()}} + +{% endblock %} + +{% block styles %} +{{super()}} + + + + +{% endblock %} + +{% block navbar %} + +{% endblock %} diff --git a/panoramix/templates/panoramix/viz_highcharts.html b/panoramix/templates/panoramix/viz_highcharts.html index efadd26fc9bf9..0276ed7e414c9 100644 --- a/panoramix/templates/panoramix/viz_highcharts.html +++ b/panoramix/templates/panoramix/viz_highcharts.html @@ -3,7 +3,7 @@
{% endblock %} -{% block scripts %} +{% block tail %} {{ super() }} {% endblock %} diff --git a/panoramix/viz.py b/panoramix/viz.py index 3a354e28749c9..0c9285a290d02 100644 --- a/panoramix/viz.py +++ b/panoramix/viz.py @@ -1,9 +1,11 @@ -from pydruid import client from pydruid.utils.filters import Dimension, Filter from datetime import datetime from flask import render_template, flash import pandas as pd from pandas_highcharts.core import serialize +from panoramix import settings +from pydruid.utils import aggregators as agg +from collections import OrderedDict CHART_ARGS = { @@ -12,33 +14,24 @@ 'render_to': 'chart', } -# temp hack -metric = "count" - class BaseViz(object): verbose_name = "Base Viz" template = "panoramix/datasource.html" - def __init__(self, datasource, form_class, form_data): + def __init__(self, datasource, form_class, form_data, admin_view): self.datasource = datasource self.form_class = form_class self.form_data = form_data + self.metric = form_data.get('metric') + self.admin_view = admin_view self.df = self.bake_query() if self.df is not None: self.df.timestamp = pd.to_datetime(self.df.timestamp) self.df_prep() self.form_prep() - def bake_query(self): - ds = self.datasource + def query_filters(self): args = self.form_data - groupby = args.getlist("groupby") or [] - granularity = args.get("granularity") - metric = "count" - limit = int(args.get("limit", ROW_LIMIT)) or ROW_LIMIT - since = args.get("since", "all") - from_dttm = (datetime.now() - since_l[since]).isoformat() - # Building filters i = 1 filters = None @@ -57,8 +50,7 @@ def bake_query(self): for s in eq.split(','): s = s.strip() fields.append(Filter.build_filter(Dimension(col)==s)) - cond = Filter(type="or", fields=fields) - + cond = Filter(type="and", fields=fields) if filters: filters = cond and filters @@ -67,29 +59,42 @@ def bake_query(self): else: break i += 1 + return filters - kw = {} - if filters: - kw['filter'] = filters - query.groupby( - datasource=ds.name, - granularity=granularity or 'all', - intervals=from_dttm + '/' + datetime.now().isoformat(), - dimensions=groupby, - aggregations={"count": client.doublesum(metric)}, - #filter=filters, - limit_spec={ + def query_obj(self): + ds = self.datasource + args = self.form_data + groupby = args.getlist("groupby") or [] + granularity = args.get("granularity") + metric = "count" + limit = int( + args.get("limit", settings.ROW_LIMIT)) or settings.ROW_LIMIT + since = args.get("since", "all") + from_dttm = (datetime.now() - settings.since_l[since]).isoformat() + d = { + 'datasource': ds.name, + 'granularity': granularity or 'all', + 'intervals': from_dttm + '/' + datetime.now().isoformat(), + 'dimensions': groupby, + 'aggregations': {"count": agg.doublesum(metric)}, + 'limit_spec': { "type": "default", "limit": limit, "columns": [{ - "dimension" : metric, - "direction" : "descending", - },], + "dimension": metric, + "direction": "descending", + }], }, - **kw - ) - return query.export_pandas() + } + filters = self.query_filters() + if filters: + d['filter'] = filters + return d + def bake_query(self): + client = settings.get_pydruid_client() + client.groupby(**self.query_obj()) + return client.export_pandas() def df_prep(self, ): pass @@ -97,17 +102,21 @@ def df_prep(self, ): def form_prep(self): pass + def render_no_data(self): + self.template = "panoramix/no_data.html" + return BaseViz.render(self) + def render(self, *args, **kwargs): form = self.form_class(self.form_data) - return render_template( - self.template, form=form) + return self.admin_view.render( + self.template, form=form, viz=self, datasource=self.datasource, + *args, **kwargs) class TableViz(BaseViz): verbose_name = "Table View" template = 'panoramix/viz_table.html' def render(self): - form = self.form_class(self.form_data) if self.df is None or self.df.empty: flash("No data.", "error") table = None @@ -115,53 +124,85 @@ def render(self): if self.form_data.get("granularity") == "all": del self.df['timestamp'] table = self.df.to_html( - classes=["table", "table-striped", 'table-bordered'], + classes=[ + 'table', 'table-striped', 'table-bordered', + 'table-condensed'], index=False) - return render_template( - self.template, form=form, table=table) + return super(TableViz, self).render(table=table) class HighchartsViz(BaseViz): verbose_name = "Base Highcharts Viz" template = 'panoramix/viz_highcharts.html' chart_kind = 'line' - def render(self, *args, **kwargs): - form = self.form_class(self.form_data) - if self.df is None or self.df.empty: - flash("No data.", "error") - else: - table = self.df.to_html( - classes=["table", "table-striped", 'table-bordered'], - index=False) - return render_template( - self.template, form=form, table=table, - *args, **kwargs) class TimeSeriesViz(HighchartsViz): verbose_name = "Time Series - Line Chart" chart_kind = "line" + def render(self): + metric = self.metric df = self.df df = df.pivot_table( index="timestamp", columns=[ col for col in df.columns if col not in ["timestamp", metric]], values=[metric]) - chart_js = serialize( - df, kind=self.chart_kind, **CHART_ARGS) + chart_js = serialize(df, kind=self.chart_kind, **CHART_ARGS) return super(TimeSeriesViz, self).render(chart_js=chart_js) + def bake_query(self): + """ + Doing a 2 phase query where we limit the number of series. + """ + client = settings.get_pydruid_client() + qry = self.query_obj() + qry['granularity'] = "all" + client.groupby(**qry) + df = client.export_pandas() + dims = qry['dimensions'] + filters = [] + for index, row in df.iterrows(): + fields = [] + for dim in dims: + f = Filter.build_filter(Dimension(dim) == row[dim]) + fields.append(f) + if len(fields) > 1: + filters.append(Filter.build_filter(Filter(type="and", fields=fields))) + elif fields: + filters.append(fields[0]) + + qry = self.query_obj() + if filters: + ff = Filter(type="or", fields=filters) + qry['filter'] = ff + del qry['limit_spec'] + client.groupby(**qry) + return client.export_pandas() + class TimeSeriesAreaViz(TimeSeriesViz): verbose_name = "Time Series - Area Chart" chart_kind = "area" +class TimeSeriesBarViz(TimeSeriesViz): + verbose_name = "Time Series - Bar Chart" + chart_kind = "bar" + + class DistributionBarViz(HighchartsViz): verbose_name = "Distribution - Bar Chart" chart_kind = "bar" + + def query_obj(self): + d = super(DistributionBarViz, self).query_obj() + d['granularity'] = "all" + return d + def render(self): + metric = self.metric df = self.df df = df.pivot_table( index=[ @@ -172,9 +213,33 @@ def render(self): df, kind=self.chart_kind, **CHART_ARGS) return super(DistributionBarViz, self).render(chart_js=chart_js) -viz_types = { - 'table': TableViz, - 'line': TimeSeriesViz, - 'area': TimeSeriesAreaViz, - 'dist_bar': DistributionBarViz, -} + +class DistributionPieViz(HighchartsViz): + verbose_name = "Distribution - Pie Chart" + chart_kind = "pie" + + def query_obj(self): + d = super(DistributionPieViz, self).query_obj() + d['granularity'] = "all" + return d + + def render(self): + metric = self.metric + df = self.df + df = df.pivot_table( + index=[ + col for col in df.columns if col not in ['timestamp', metric]], + values=[metric]) + df = df.sort(metric, ascending=False) + chart_js = serialize( + df, kind=self.chart_kind, **CHART_ARGS) + return super(DistributionPieViz, self).render(chart_js=chart_js) + +viz_types = OrderedDict([ + ['table', TableViz], + ['line', TimeSeriesViz], + ['area', TimeSeriesAreaViz], + ['bar', TimeSeriesBarViz], + ['dist_bar', DistributionBarViz], + ['pie', DistributionPieViz], +]) diff --git a/requirements.txt b/requirements.txt index e1e0740235121..d74844803ef93 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,10 @@ flask +flask-admin flask-bootstrap +flask-sqlalchemy pandas pandas-highcharts pydruid python-dateutil +requests wtforms