diff --git a/.gitignore b/.gitignore index 0d20b6487c61e..7449a91d059b0 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ *.pyc +tmp diff --git a/base.html b/base.html deleted file mode 100644 index 12f0245b0925b..0000000000000 --- a/base.html +++ /dev/null @@ -1,86 +0,0 @@ -{% import 'admin/layout.html' as layout with context -%} -{% import 'admin/static.html' as admin_static with context %} - - - - {% block title %}{% if admin_view.category %}{{ admin_view.category }} - {% endif %}{{ admin_view.name }} - {{ admin_view.admin.name }}{% endblock %} - {% block head_meta %} - - - - - - {% endblock %} - {% block head_css %} - - - - - - - {% endblock %} - {% block head %} - {% endblock %} - {% block head_tail %} - {% endblock %} - - - {% block page_body %} -
- - - {% block messages %} - {{ layout.messages() }} - {% endblock %} - - {% set render_ctx = h.resolve_ctx() %} - - {% block body %}{% endblock %} -
- {% endblock %} - - {% block tail_js %} - - - - - - {% endblock %} - - {% block tail %} - {% endblock %} - - diff --git a/init.sh b/init.sh deleted file mode 100644 index daaa10cb2d88a..0000000000000 --- a/init.sh +++ /dev/null @@ -1 +0,0 @@ -export PYTHONPATH=/home/maxime_beauchemin/code/panoramix:/home/maxime_beauchemin/code/pydruid diff --git a/panoramix/.gitignore b/panoramix/.gitignore new file mode 100644 index 0000000000000..1c3f9231c3755 --- /dev/null +++ b/panoramix/.gitignore @@ -0,0 +1,9 @@ +*.pyc +app.db +./tmp +./build/* +./.idea +./.idea/* +env +venv +*.sublime* diff --git a/panoramix/README.rst b/panoramix/README.rst new file mode 100644 index 0000000000000..bc267194cd6a1 --- /dev/null +++ b/panoramix/README.rst @@ -0,0 +1,15 @@ +Base Skeleton to start your application using Flask-AppBuilder +-------------------------------------------------------------- + +- Install it:: + + pip install flask-appbuilder + git clone https://github.com/dpgaspar/Flask-AppBuilder-Skeleton.git + +- Run it:: + + fabmanager run + + +That's it!! + diff --git a/panoramix/TODO.md b/panoramix/TODO.md new file mode 100644 index 0000000000000..192021631f3ea --- /dev/null +++ b/panoramix/TODO.md @@ -0,0 +1,3 @@ +# TODO +* Default URL params per datasource +* Get config metrics to work diff --git a/panoramix/__init__.py b/panoramix/__init__.py deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/panoramix/app.py b/panoramix/app.py deleted file mode 100644 index eb3aac7ea5014..0000000000000 --- a/panoramix/app.py +++ /dev/null @@ -1,251 +0,0 @@ -from dateutil.parser import parse -from datetime import timedelta -from flask import Flask, request, Blueprint, url_for, Markup -from flask.ext.sqlalchemy import SQLAlchemy -from flask.ext.admin import Admin, BaseView, expose, AdminIndexView -from panoramix import settings, viz, models -from flask_bootstrap import Bootstrap -from wtforms import Form, SelectMultipleField, SelectField, TextField -from wtforms.fields import Field -import pandas as pd -from flask_admin.contrib import sqla - - -pd.set_option('display.max_colwidth', -1) - -client = settings.get_pydruid_client() - - -class OmgWtForm(Form): - field_order = ( - 'viz_type', 'granularity', 'since', 'group_by', 'limit') - def fields(self): - fields = [] - for field in self.field_order: - if hasattr(self, field): - obj = getattr(self, field) - if isinstance(obj, Field): - fields.append(getattr(self, field)) - return fields - - -def form_factory(datasource, form_args=None): - grain = ['all', 'none', 'minute', 'hour', 'day'] - limits = [0, 5, 10, 25, 50, 100, 500] - - if form_args: - limit = form_args.get("limit") - try: - limit = int(limit) - if limit not in limits: - limits.append(limit) - limits = sorted(limits) - except: - pass - - class QueryForm(OmgWtForm): - viz_type = SelectField( - 'Viz', - choices=[(k, v.verbose_name) for k, v in viz.viz_types.items()]) - metric = SelectField( - 'Metric', choices=[(m, m) for m in datasource.metrics]) - groupby = SelectMultipleField( - 'Group by', choices=[ - (s, s) for s in datasource.groupby_column_names]) - granularity = SelectField( - 'Time Granularity', choices=[(g, g) for g in grain]) - since = SelectField( - 'Since', choices=[(s, s) for s in settings.since_l.keys()], - default="all") - limit = SelectField( - 'Limit', choices=[(s, s) for s in limits]) - for i in range(10): - setattr(QueryForm, 'flt_col_' + str(i), SelectField( - 'Filter 1', choices=[(s, s) for s in datasource.filterable_column_names])) - setattr(QueryForm, 'flt_op_' + str(i), SelectField( - 'Filter 1', choices=[(m, m) for m in ['==', '!=', 'in',]])) - setattr(QueryForm, 'flt_eq_' + str(i), TextField("Super")) - return QueryForm - -""" -bp = Blueprint( - 'panoramix', __name__, - template_folder='templates', - static_folder='static') -""" - - - -app = Flask(__name__) -app.config['SQLALCHEMY_DATABASE_URI'] = settings.SQLALCHEMY_DATABASE_URI -db = SQLAlchemy(app) -app.secret_key = "monkeys" -#app.register_blueprint(bp, url_prefix='/panoramix') -Bootstrap(app) -admin = Admin( - app, name = "Panoramix", - template_mode='bootstrap3') - - - -class Datasource(db.Model): - __tablename__ = 'datasources' - id = db.Column(db.Integer, primary_key=True) - datasource_name = db.Column(db.String(256), unique=True) - is_featured = db.Column(db.Boolean, default=False) - is_hidden = db.Column(db.Boolean, default=False) - description = db.Column(db.Text) - created_dttm = db.Column(db.DateTime, default=db.func.now()) - - @property - def metrics(self): - return [col.column_name for col in self.columns if not col.groupby] - - @classmethod - def latest_metadata(cls, name): - results = client.time_boundary(datasource=name) - max_time = results[0]['result']['maxTime'] - max_time = parse(max_time) - intervals = (max_time - timedelta(seconds=1)).isoformat() + '/' - intervals += (max_time + timedelta(seconds=1)).isoformat() - segment_metadata = client.segment_metadata( - datasource=name, - intervals=intervals) - return segment_metadata[-1]['columns'] - - @classmethod - def sync_to_db(cls, name): - datasource = cls.query.filter_by(datasource_name=name).first() - if not datasource: - db.session.add(cls(datasource_name=name)) - cols = cls.latest_metadata(name) - for col in cols: - col_obj = Column.query.filter_by(datasource_name=name, column_name=col).first() - datatype = cols[col]['type'] - if not col_obj: - col_obj = Column(datasource_name=name, column_name=col) - db.session.add(col_obj) - if datatype == "STRING": - col_obj.groupby = True - col_obj.filterable = True - if col_obj: - col_obj.type = cols[col]['type'] - - db.session.commit() - - @property - def column_names(self): - return sorted([c.column_name for c in self.columns]) - - @property - def groupby_column_names(self): - return sorted([c.column_name for c in self.columns if c.groupby]) - - @property - def filterable_column_names(self): - return sorted([c.column_name for c in self.columns if c.filterable]) - - - -class Column(db.Model): - __tablename__ = 'columns' - id = db.Column(db.Integer, primary_key=True) - datasource_name = db.Column( - db.String(256), - db.ForeignKey('datasources.datasource_name')) - column_name = db.Column(db.String(256)) - is_active = db.Column(db.Boolean, default=True) - type = db.Column(db.String(32)) - groupby = db.Column(db.Boolean, default=False) - count_distinct = db.Column(db.Boolean, default=False) - sum = db.Column(db.Boolean, default=False) - max = db.Column(db.Boolean, default=False) - min = db.Column(db.Boolean, default=False) - filterable = db.Column(db.Boolean, default=False) - datasource = db.relationship('Datasource', - backref=db.backref('columns', lazy='dynamic')) - - def __repr__(self): - return self.column_name - - -class JsUdf(db.Model): - __tablename__ = 'udfs' - id = db.Column(db.Integer, primary_key=True) - datasource_name = db.Column( - db.String(256), - db.ForeignKey('datasources.datasource_name')) - udf_name = db.Column(db.String(256)) - column_list = db.Column(db.String(1024)) - code = db.Column(db.Text) - datasource = db.relationship('Datasource', - backref=db.backref('udfs', lazy='dynamic')) - - -def datasource_link(v, c, m, p): - url = '/admin/datasourceview/datasource/{}/'.format(m.datasource_name) - return Markup('{m.datasource_name}'.format(**locals())) - - -class DatasourceAdmin(sqla.ModelView): - inline_models = (Column, JsUdf,) - column_formatters = dict(datasource_name=datasource_link) - - -class DatasourceView(BaseView): - @expose('/') - def index(self): - return "" - @expose("/datasource//") - def datasource(self, datasource_name): - viz_type = request.args.get("viz_type", "table") - datasource = ( - Datasource - .query - .filter_by(datasource_name=datasource_name) - .first() - ) - obj = viz.viz_types[viz_type]( - datasource, - form_class=form_factory(datasource, request.args), - form_data=request.args, - admin_view=self) - if obj.df is None or obj.df.empty: - return obj.render_no_data() - return obj.render() - - - @expose("/datasources/") - def datasources(self): - import requests - import json - endpoint = ( - "http://{COORDINATOR_HOST}:{COORDINATOR_PORT}/" - "{COORDINATOR_BASE_ENDPOINT}/datasources" - ).format(**settings.__dict__) - datasources = json.loads(requests.get(endpoint).text) - for datasource in datasources: - Datasource.sync_to_db(datasource) - - return json.dumps(datasources, indent=4) - - - @expose("/datasource_metadata//") - def datasource_metadata(name): - import requests - import json - endpoint = ( - "http://{COORDINATOR_HOST}:{COORDINATOR_PORT}/" - "{COORDINATOR_BASE_ENDPOINT}/datasource" - ).format(**settings.__dict__) - - return str(datasources) - -admin.add_view(DatasourceView(name="Datasource")) - -if __name__ == '__main__': - - db.create_all() - admin.add_view(DatasourceAdmin(Datasource, db.session, name="Datasources")) - app.debug = True - app.run(host='0.0.0.0', port=settings.FLASK_APP_PORT) diff --git a/panoramix/app/__init__.py b/panoramix/app/__init__.py new file mode 100644 index 0000000000000..ccc0a50e31c30 --- /dev/null +++ b/panoramix/app/__init__.py @@ -0,0 +1,34 @@ +import logging +from flask import Flask +from flask.ext.appbuilder import SQLA, AppBuilder + +""" + Logging configuration +""" + +logging.basicConfig(format='%(asctime)s:%(levelname)s:%(name)s:%(message)s') +logging.getLogger().setLevel(logging.DEBUG) + +app = Flask(__name__) +app.config.from_object('config') +db = SQLA(app) +appbuilder = AppBuilder( + app, db.session, base_template='panoramix/base.html') +#appbuilder.app_name = 'Panoramix' + + +""" +from sqlalchemy.engine import Engine +from sqlalchemy import event + +#Only include this for SQLLite constraints +@event.listens_for(Engine, "connect") +def set_sqlite_pragma(dbapi_connection, connection_record): + # Will force sqllite contraint foreign keys + cursor = dbapi_connection.cursor() + cursor.execute("PRAGMA foreign_keys=ON") + cursor.close() +""" + +from app import views + diff --git a/panoramix/app/models.py b/panoramix/app/models.py new file mode 100644 index 0000000000000..b9b81d29e6c31 --- /dev/null +++ b/panoramix/app/models.py @@ -0,0 +1,122 @@ +from flask.ext.appbuilder import Model +from datetime import datetime, timedelta +from flask.ext.appbuilder.models.mixins import AuditMixin, FileColumn, ImageColumn +from sqlalchemy import Column, Integer, String, ForeignKey, Text, Boolean +from sqlalchemy.orm import relationship +from app import db, utils +from dateutil.parser import parse +""" + +You can use the extra Flask-AppBuilder fields and Mixin's + +AuditMixin will add automatic timestamp of created and modified by who + + +""" +client = utils.get_pydruid_client() + +class Datasource(Model, AuditMixin): + __tablename__ = 'datasources' + id = Column(Integer, primary_key=True) + datasource_name = Column(String(256), unique=True) + is_featured = Column(Boolean, default=False) + is_hidden = Column(Boolean, default=False) + description = Column(Text) + columns = relationship('Column', backref='datasource') + udfs = relationship('JavascriptUdf', backref='datasource') + + @property + def metrics(self): + return [col.column_name for col in self.columns if not col.groupby] + + def __repr__(self): + return self.datasource_name + + @property + def datasource_link(self): + url = "/panoramix/datasource/{}/".format(self.datasource_name) + return '{self.datasource_name}'.format(**locals()) + + @classmethod + def latest_metadata(cls, name): + results = client.time_boundary(datasource=name) + max_time = results[0]['result']['maxTime'] + max_time = parse(max_time) + intervals = (max_time - timedelta(seconds=1)).isoformat() + '/' + intervals += (max_time + timedelta(seconds=1)).isoformat() + segment_metadata = client.segment_metadata( + datasource=name, + intervals=intervals) + return segment_metadata[-1]['columns'] + + @classmethod + def sync_to_db(cls, name): + datasource = db.session.query(cls).filter_by(datasource_name=name).first() + if not datasource: + db.session.add(cls(datasource_name=name)) + cols = cls.latest_metadata(name) + for col in cols: + col_obj = ( + db.session + .query(Column) + .filter_by(datasource_name=name, column_name=col) + .first() + ) + datatype = cols[col]['type'] + if not col_obj: + col_obj = Column(datasource_name=name, column_name=col) + db.session.add(col_obj) + if datatype == "STRING": + col_obj.groupby = True + col_obj.filterable = True + if col_obj: + col_obj.type = cols[col]['type'] + + db.session.commit() + + @property + def column_names(self): + return sorted([c.column_name for c in self.columns]) + + @property + def groupby_column_names(self): + return sorted([c.column_name for c in self.columns if c.groupby]) + + @property + def filterable_column_names(self): + return sorted([c.column_name for c in self.columns if c.filterable]) + + +class JavascriptUdf(Model, AuditMixin): + __tablename__ = 'udfs' + id = Column(Integer, primary_key=True) + datasource_name = Column( + String(256), + ForeignKey('datasources.datasource_name')) + udf_name = Column(String(256)) + column_list = Column(String(1024)) + code = Column(Text) + + def __repr__(self): + return self.udf_name + + +class Column(Model, AuditMixin): + __tablename__ = 'columns' + id = Column(Integer, primary_key=True) + datasource_name = Column( + String(256), + ForeignKey('datasources.datasource_name')) + column_name = Column(String(256)) + is_active = Column(Boolean, default=True) + type = Column(String(32)) + groupby = Column(Boolean, default=False) + count_distinct = Column(Boolean, default=False) + sum = Column(Boolean, default=False) + max = Column(Boolean, default=False) + min = Column(Boolean, default=False) + filterable = Column(Boolean, default=False) + + def __repr__(self): + return self.column_name + diff --git a/panoramix/static/bootstrap-theme.css b/panoramix/app/static/bootstrap-theme.css similarity index 100% rename from panoramix/static/bootstrap-theme.css rename to panoramix/app/static/bootstrap-theme.css diff --git a/panoramix/static/chaudron.png b/panoramix/app/static/chaudron.png similarity index 100% rename from panoramix/static/chaudron.png rename to panoramix/app/static/chaudron.png diff --git a/panoramix/static/favicon.png b/panoramix/app/static/favicon.png similarity index 100% rename from panoramix/static/favicon.png rename to panoramix/app/static/favicon.png diff --git a/panoramix/static/highcharts.js b/panoramix/app/static/highcharts.js similarity index 100% rename from panoramix/static/highcharts.js rename to panoramix/app/static/highcharts.js diff --git a/panoramix/static/main.css b/panoramix/app/static/main.css similarity index 100% rename from panoramix/static/main.css rename to panoramix/app/static/main.css diff --git a/panoramix/static/panoramix.jpg b/panoramix/app/static/panoramix.jpg similarity index 100% rename from panoramix/static/panoramix.jpg rename to panoramix/app/static/panoramix.jpg diff --git a/panoramix/static/panoramix.png b/panoramix/app/static/panoramix.png similarity index 100% rename from panoramix/static/panoramix.png rename to panoramix/app/static/panoramix.png diff --git a/panoramix/static/select2-bootstrap.css b/panoramix/app/static/select2-bootstrap.css similarity index 100% rename from panoramix/static/select2-bootstrap.css rename to panoramix/app/static/select2-bootstrap.css diff --git a/panoramix/static/select2.min.css b/panoramix/app/static/select2.min.css similarity index 100% rename from panoramix/static/select2.min.css rename to panoramix/app/static/select2.min.css diff --git a/panoramix/static/select2.min.js b/panoramix/app/static/select2.min.js similarity index 100% rename from panoramix/static/select2.min.js rename to panoramix/app/static/select2.min.js diff --git a/panoramix/static/serpe.jpg b/panoramix/app/static/serpe.jpg similarity index 100% rename from panoramix/static/serpe.jpg rename to panoramix/app/static/serpe.jpg diff --git a/panoramix/static/tux_panoramix.png b/panoramix/app/static/tux_panoramix.png similarity index 100% rename from panoramix/static/tux_panoramix.png rename to panoramix/app/static/tux_panoramix.png diff --git a/panoramix/app/templates/index.html b/panoramix/app/templates/index.html new file mode 100644 index 0000000000000..2970ab1592018 --- /dev/null +++ b/panoramix/app/templates/index.html @@ -0,0 +1,2 @@ +{% extends "appbuilder/baselayout.html" %} + diff --git a/panoramix/app/templates/panoramix/base.html b/panoramix/app/templates/panoramix/base.html new file mode 100644 index 0000000000000..b93e5d1f05946 --- /dev/null +++ b/panoramix/app/templates/panoramix/base.html @@ -0,0 +1,8 @@ +{% extends "appbuilder/baselayout.html" %} + +{% block head_css %} +{{super()}} + + +{% endblock %} diff --git a/panoramix/templates/panoramix/datasource.html b/panoramix/app/templates/panoramix/datasource.html similarity index 86% rename from panoramix/templates/panoramix/datasource.html rename to panoramix/app/templates/panoramix/datasource.html index 8f6a45fe92aba..0c5506f616814 100644 --- a/panoramix/templates/panoramix/datasource.html +++ b/panoramix/app/templates/panoramix/datasource.html @@ -1,5 +1,5 @@ {% extends "panoramix/base.html" %} -{% block styles %} +{% block head_css %} {{super()}} {% endblock %} -{% block body %} +{% block content %}

{{ datasource.datasource_name }} - +

-
{{ form.viz_type.label }}: {{ form.viz_type(class_="form-control") }}
+
{{ form.viz_type.label }}: {{ form.viz_type(class_="form-control select2") }}
{{ form.metric.label }}: {{ form.metric(class_="form-control select2") }}
-
{{ form.granularity.label }}: {{ form.granularity(class_="form-control") }}
-
{{ form.since.label }}: {{ form.since(class_="form-control") }}
-
{{ form.groupby.label }}: {{ form.groupby(class_="form-control") }}
-
{{ form.limit.label }}: {{ form.limit(class_="form-control") }}
+
{{ form.granularity.label }}: {{ form.granularity(class_="form-control select2") }}
+
{{ form.since.label }}: {{ form.since(class_="form-control select2") }}
+
{{ form.groupby.label }}: {{ form.groupby(class_="form-control select2") }}
+
{{ form.limit.label }}: {{ form.limit(class_="form-control select2") }}

Filters

@@ -78,12 +78,12 @@

Latest Segment Metadata

{% endblock %} -{% block tail %} +{% block tail_js %} {{ super() }} {% endblock %} diff --git a/panoramix/templates/panoramix/no_data.html b/panoramix/app/templates/panoramix/no_data.html similarity index 100% rename from panoramix/templates/panoramix/no_data.html rename to panoramix/app/templates/panoramix/no_data.html diff --git a/panoramix/templates/panoramix/noadmin.html b/panoramix/app/templates/panoramix/noadmin.html similarity index 100% rename from panoramix/templates/panoramix/noadmin.html rename to panoramix/app/templates/panoramix/noadmin.html diff --git a/panoramix/templates/panoramix/viz_highcharts.html b/panoramix/app/templates/panoramix/viz_highcharts.html similarity index 100% rename from panoramix/templates/panoramix/viz_highcharts.html rename to panoramix/app/templates/panoramix/viz_highcharts.html diff --git a/panoramix/templates/panoramix/viz_table.html b/panoramix/app/templates/panoramix/viz_table.html similarity index 100% rename from panoramix/templates/panoramix/viz_table.html rename to panoramix/app/templates/panoramix/viz_table.html diff --git a/panoramix/app/utils.py b/panoramix/app/utils.py new file mode 100644 index 0000000000000..b92ebfe224cee --- /dev/null +++ b/panoramix/app/utils.py @@ -0,0 +1,18 @@ +import config +from datetime import timedelta + +since_l = { + '1hour': timedelta(hours=1), + '1day': timedelta(days=1), + '7days': timedelta(days=7), + '28days': timedelta(days=28), + 'all': timedelta(days=365*100) +} + +def get_pydruid_client(): + from pydruid import client + return client.PyDruid( + "http://{0}:{1}/".format(config.DRUID_HOST, config.DRUID_PORT), + config.DRUID_BASE_ENDPOINT) + + diff --git a/panoramix/app/views.py b/panoramix/app/views.py new file mode 100644 index 0000000000000..1f4fb077528a5 --- /dev/null +++ b/panoramix/app/views.py @@ -0,0 +1,129 @@ +from flask import request, redirect, flash +from flask.ext.appbuilder.models.sqla.interface import SQLAInterface +from flask.ext.appbuilder import ModelView, CompactCRUDMixin, BaseView, expose +from app import appbuilder, db, models, viz, utils +import config +from wtforms import Form, SelectMultipleField, SelectField, TextField +from wtforms.fields import Field +from datetime import timedelta + +class OmgWtForm(Form): + field_order = ( + 'viz_type', 'granularity', 'since', 'group_by', 'limit') + def fields(self): + fields = [] + for field in self.field_order: + if hasattr(self, field): + obj = getattr(self, field) + if isinstance(obj, Field): + fields.append(getattr(self, field)) + return fields + + +def form_factory(datasource, form_args=None): + grain = ['all', 'none', 'minute', 'hour', 'day'] + limits = [0, 5, 10, 25, 50, 100, 500] + + if form_args: + limit = form_args.get("limit") + try: + limit = int(limit) + if limit not in limits: + limits.append(limit) + limits = sorted(limits) + except: + pass + + class QueryForm(OmgWtForm): + viz_type = SelectField( + 'Viz', + choices=[(k, v.verbose_name) for k, v in viz.viz_types.items()]) + metric = SelectField( + 'Metric', choices=[(m, m) for m in datasource.metrics]) + groupby = SelectMultipleField( + 'Group by', choices=[ + (s, s) for s in datasource.groupby_column_names]) + granularity = SelectField( + 'Time Granularity', choices=[(g, g) for g in grain]) + since = SelectField( + 'Since', choices=[(s, s) for s in utils.since_l.keys()], + default="all") + limit = SelectField( + 'Limit', choices=[(s, s) for s in limits]) + for i in range(10): + setattr(QueryForm, 'flt_col_' + str(i), SelectField( + 'Filter 1', choices=[(s, s) for s in datasource.filterable_column_names])) + setattr(QueryForm, 'flt_op_' + str(i), SelectField( + 'Filter 1', choices=[(m, m) for m in ['==', '!=', 'in',]])) + setattr(QueryForm, 'flt_eq_' + str(i), TextField("Super")) + return QueryForm + + +class ColumnInlineView(CompactCRUDMixin, ModelView): + datamodel = SQLAInterface(models.Column) + edit_columns = ['column_name', 'groupby', 'count_distinct', 'sum', 'min', 'max'] + list_columns = ['column_name', 'groupby', 'count_distinct', 'sum', 'min', 'max'] + can_delete = False +appbuilder.add_view_no_menu(ColumnInlineView) + +class JavascriptUdfInlineView(CompactCRUDMixin, ModelView): + datamodel = SQLAInterface(models.JavascriptUdf) + edit_columns = ['udf_name', 'column_list', 'code'] +appbuilder.add_view_no_menu(JavascriptUdfInlineView) + + +class DatasourceModelView(ModelView): + datamodel = SQLAInterface(models.Datasource) + list_columns = ['datasource_link', 'is_featured' ] + related_views = [ColumnInlineView, JavascriptUdfInlineView] + edit_columns = ['datasource_name', 'description', 'is_featured', 'is_hidden'] + page_size = 100 + + +appbuilder.add_view( + DatasourceModelView, + "Datasources", + icon="fa-cube", + category_icon='fa-envelope') + + +class Panoramix(BaseView): + @expose("/datasource//") + def datasource(self, datasource_name): + viz_type = request.args.get("viz_type", "table") + datasource = ( + db.session + .query(models.Datasource) + .filter_by(datasource_name=datasource_name) + .first() + ) + obj = viz.viz_types[viz_type]( + datasource, + form_class=form_factory(datasource, request.args), + form_data=request.args, view=self) + if obj.df is None or obj.df.empty: + return obj.render_no_data() + return obj.render() + + + @expose("/refresh_datasources/") + def datasources(self): + import requests + import json + endpoint = ( + "http://{COORDINATOR_HOST}:{COORDINATOR_PORT}/" + "{COORDINATOR_BASE_ENDPOINT}/datasources" + ).format(**config.__dict__) + datasources = json.loads(requests.get(endpoint).text) + for datasource in datasources: + models.Datasource.sync_to_db(datasource) + flash("Refreshed metadata from Druid!", 'info') + return redirect("/datasourcemodelview/list/") + +appbuilder.add_view_no_menu(Panoramix) +appbuilder.add_link( + "Refresh Metadata", + href='/panoramix/refresh_datasources/', + category='Admin', + icon="fa-cogs") +db.create_all() diff --git a/panoramix/viz.py b/panoramix/app/viz.py similarity index 85% rename from panoramix/viz.py rename to panoramix/app/viz.py index 7e4a9c1c9c9f6..c3f846e6efd30 100644 --- a/panoramix/viz.py +++ b/panoramix/app/viz.py @@ -3,9 +3,10 @@ from flask import render_template, flash import pandas as pd from pandas_highcharts.core import serialize -from panoramix import settings from pydruid.utils import aggregators as agg from collections import OrderedDict +from app import utils +import config CHART_ARGS = { @@ -18,13 +19,13 @@ class BaseViz(object): verbose_name = "Base Viz" template = "panoramix/datasource.html" - def __init__(self, datasource, form_class, form_data, admin_view): + def __init__(self, datasource, form_class, form_data, view): self.datasource = datasource self.form_class = form_class self.form_data = form_data self.metric = form_data.get('metric') - self.admin_view = admin_view self.df = self.bake_query() + self.view = view if self.df is not None: self.df.timestamp = pd.to_datetime(self.df.timestamp) self.df_prep() @@ -68,9 +69,9 @@ def query_obj(self): granularity = args.get("granularity") metric = "count" limit = int( - args.get("limit", settings.ROW_LIMIT)) or settings.ROW_LIMIT + args.get("limit", config.ROW_LIMIT)) or config.ROW_LIMIT since = args.get("since", "all") - from_dttm = (datetime.now() - settings.since_l[since]).isoformat() + from_dttm = (datetime.now() - utils.since_l[since]).isoformat() d = { 'datasource': ds.datasource_name, 'granularity': granularity or 'all', @@ -92,7 +93,7 @@ def query_obj(self): return d def bake_query(self): - client = settings.get_pydruid_client() + client = utils.get_pydruid_client() client.groupby(**self.query_obj()) return client.export_pandas() @@ -108,7 +109,7 @@ def render_no_data(self): def render(self, *args, **kwargs): form = self.form_class(self.form_data) - return self.admin_view.render( + return self.view.render_template( self.template, form=form, viz=self, datasource=self.datasource, *args, **kwargs) @@ -159,29 +160,30 @@ def bake_query(self): """ Doing a 2 phase query where we limit the number of series. """ - client = settings.get_pydruid_client() + client = utils.get_pydruid_client() qry = self.query_obj() qry['granularity'] = "all" client.groupby(**qry) df = client.export_pandas() - dims = qry['dimensions'] - filters = [] - for index, row in df.iterrows(): - fields = [] - for dim in dims: - f = Filter.build_filter(Dimension(dim) == row[dim]) - fields.append(f) - if len(fields) > 1: - filters.append(Filter.build_filter(Filter(type="and", fields=fields))) - elif fields: - filters.append(fields[0]) - - qry = self.query_obj() - if filters: - ff = Filter(type="or", fields=filters) - qry['filter'] = ff - del qry['limit_spec'] - client.groupby(**qry) + if not df is None: + dims = qry['dimensions'] + filters = [] + for index, row in df.iterrows(): + fields = [] + for dim in dims: + f = Filter.build_filter(Dimension(dim) == row[dim]) + fields.append(f) + if len(fields) > 1: + filters.append(Filter.build_filter(Filter(type="and", fields=fields))) + elif fields: + filters.append(fields[0]) + + qry = self.query_obj() + if filters: + ff = Filter(type="or", fields=filters) + qry['filter'] = ff + del qry['limit_spec'] + client.groupby(**qry) return client.export_pandas() diff --git a/panoramix/babel/babel.cfg b/panoramix/babel/babel.cfg new file mode 100644 index 0000000000000..70e23ac634f1a --- /dev/null +++ b/panoramix/babel/babel.cfg @@ -0,0 +1,3 @@ +[python: **.py] +[jinja2: **/templates/**.html] +encoding = utf-8 diff --git a/panoramix/babel/messages.pot b/panoramix/babel/messages.pot new file mode 100644 index 0000000000000..8b137891791fe --- /dev/null +++ b/panoramix/babel/messages.pot @@ -0,0 +1 @@ + diff --git a/panoramix/config.py b/panoramix/config.py new file mode 100644 index 0000000000000..0e43f631f3018 --- /dev/null +++ b/panoramix/config.py @@ -0,0 +1,117 @@ +import os +from flask_appbuilder.security.manager import AUTH_OID, AUTH_REMOTE_USER, AUTH_DB, AUTH_LDAP, AUTH_OAUTH +basedir = os.path.abspath(os.path.dirname(__file__)) + +#--------------------------------------------------------- +# Panoramix specifix config +#--------------------------------------------------------- +ROW_LIMIT = 5000 + +DRUID_HOST = '10.181.47.80' +DRUID_PORT = 8080 +DRUID_BASE_ENDPOINT = 'druid/v2' + +COORDINATOR_HOST = '10.168.176.249' +COORDINATOR_PORT = '8080' +COORDINATOR_BASE_ENDPOINT = 'druid/coordinator/v1' +#--------------------------------------------------------- + +# Your App secret key +SECRET_KEY = '\2\1thisismyscretkey\1\2\e\y\y\h' + +# The SQLAlchemy connection string. +SQLALCHEMY_DATABASE_URI = 'sqlite:///' + os.path.join(basedir, 'app.db') +#SQLALCHEMY_DATABASE_URI = 'mysql://myapp@localhost/myapp' +#SQLALCHEMY_DATABASE_URI = 'postgresql://root:password@localhost/myapp' + +# Flask-WTF flag for CSRF +CSRF_ENABLED = True + +#------------------------------ +# GLOBALS FOR APP Builder +#------------------------------ +# Uncomment to setup Your App name +APP_NAME = "Panoramix" + +# Uncomment to setup Setup an App icon +#APP_ICON = "static/img/logo.jpg" + +#---------------------------------------------------- +# AUTHENTICATION CONFIG +#---------------------------------------------------- +# The authentication type +# AUTH_OID : Is for OpenID +# AUTH_DB : Is for database (username/password() +# AUTH_LDAP : Is for LDAP +# AUTH_REMOTE_USER : Is for using REMOTE_USER from web server +AUTH_TYPE = AUTH_DB + +# Uncomment to setup Full admin role name +#AUTH_ROLE_ADMIN = 'Admin' + +# Uncomment to setup Public role name, no authentication needed +#AUTH_ROLE_PUBLIC = 'Public' + +# Will allow user self registration +#AUTH_USER_REGISTRATION = True + +# The default user self registration role +#AUTH_USER_REGISTRATION_ROLE = "Public" + +# When using LDAP Auth, setup the ldap server +#AUTH_LDAP_SERVER = "ldap://ldapserver.new" + +# Uncomment to setup OpenID providers example for OpenID authentication +#OPENID_PROVIDERS = [ +# { 'name': 'Yahoo', 'url': 'https://me.yahoo.com' }, +# { 'name': 'AOL', 'url': 'http://openid.aol.com/' }, +# { 'name': 'Flickr', 'url': 'http://www.flickr.com/' }, +# { 'name': 'MyOpenID', 'url': 'https://www.myopenid.com' }] +#--------------------------------------------------- +# Babel config for translations +#--------------------------------------------------- +# Setup default language +BABEL_DEFAULT_LOCALE = 'en' +# Your application default translation path +BABEL_DEFAULT_FOLDER = 'translations' +# The allowed translation for you app +LANGUAGES = { + 'en': {'flag':'gb', 'name':'English'}, + 'pt': {'flag':'pt', 'name':'Portuguese'}, + 'pt_BR': {'flag':'br', 'name': 'Pt Brazil'}, + 'es': {'flag':'es', 'name':'Spanish'}, + 'de': {'flag':'de', 'name':'German'}, + 'zh': {'flag':'cn', 'name':'Chinese'}, + 'ru': {'flag':'ru', 'name':'Russian'} +} +#--------------------------------------------------- +# Image and file configuration +#--------------------------------------------------- +# The file upload folder, when using models with files +UPLOAD_FOLDER = basedir + '/app/static/uploads/' + +# The image upload folder, when using models with images +IMG_UPLOAD_FOLDER = basedir + '/app/static/uploads/' + +# The image upload url, when using models with images +IMG_UPLOAD_URL = '/static/uploads/' +# Setup image size default is (300, 200, True) +#IMG_SIZE = (300, 200, True) + +# Theme configuration +# these are located on static/appbuilder/css/themes +# you can create your own and easily use them placing them on the same dir structure to override +APP_THEME = "bootstrap-theme.css" # default bootstrap +#APP_THEME = "cerulean.css" +#APP_THEME = "amelia.css" +#APP_THEME = "cosmo.css" +#APP_THEME = "cyborg.css" +#APP_THEME = "flatly.css" +#APP_THEME = "journal.css" +#APP_THEME = "readable.css" +#APP_THEME = "simplex.css" +#APP_THEME = "slate.css" +#APP_THEME = "spacelab.css" +#APP_THEME = "united.css" +#APP_THEME = "yeti.css" + diff --git a/panoramix/models.py b/panoramix/models.py deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/panoramix/run.py b/panoramix/run.py new file mode 100644 index 0000000000000..2218f48247684 --- /dev/null +++ b/panoramix/run.py @@ -0,0 +1,4 @@ +from app import app + +app.run(host='0.0.0.0', port=8081, debug=True) + diff --git a/panoramix/settings.py b/panoramix/settings.py deleted file mode 100644 index 59215bc27018a..0000000000000 --- a/panoramix/settings.py +++ /dev/null @@ -1,28 +0,0 @@ -from datetime import timedelta - -FLASK_APP_PORT = 8088 - -ROW_LIMIT = 10000 -SQLALCHEMY_DATABASE_URI = "sqlite:////tmp/panoramix.db" - -DRUID_HOST = '10.181.47.80' -DRUID_PORT = 8080 -DRUID_BASE_ENDPOINT = 'druid/v2' - -COORDINATOR_HOST = '10.168.176.249' -COORDINATOR_PORT = '8080' -COORDINATOR_BASE_ENDPOINT = 'druid/coordinator/v1' - -since_l = { - '1hour': timedelta(hours=1), - '1day': timedelta(days=1), - '7days': timedelta(days=7), - '28days': timedelta(days=28), - 'all': timedelta(days=365*100) -} - -def get_pydruid_client(): - from pydruid import client - return client.PyDruid( - "http://{0}:{1}/".format(DRUID_HOST, DRUID_PORT), - DRUID_BASE_ENDPOINT) diff --git a/panoramix/templates/index.html b/panoramix/templates/index.html deleted file mode 100644 index 1d75697efa533..0000000000000 --- a/panoramix/templates/index.html +++ /dev/null @@ -1,2 +0,0 @@ -{% extends "admin/base.html" %} - diff --git a/panoramix/templates/panoramix/base.html b/panoramix/templates/panoramix/base.html deleted file mode 100644 index c17cf1f5d2f6b..0000000000000 --- a/panoramix/templates/panoramix/base.html +++ /dev/null @@ -1,2 +0,0 @@ -{% extends "index.html" %} - diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index d74844803ef93..0000000000000 --- a/requirements.txt +++ /dev/null @@ -1,10 +0,0 @@ -flask -flask-admin -flask-bootstrap -flask-sqlalchemy -pandas -pandas-highcharts -pydruid -python-dateutil -requests -wtforms