diff --git a/README.md b/README.md index b905d6489b0e0..a83adc5d0e3d0 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ and interactive. Buzz Phrases ------------ -* Analytics at the speed of thought! +* Analytics at the speed of thought! * Instantaneous learning curve * Realtime analytics when querying [Druid.io](http://druid.io) * Extentsible to infinity @@ -67,11 +67,11 @@ pip install panoramix # Create an admin user fabmanager create-admin --app panoramix -# Clone the github repo -git clone https://github.com/mistercrunch/panoramix.git +# Load some data to play with +panoramix load_examples -# Start the web server -panoramix +# Start the development web server +panoramix runserver -d ``` After installation, you should be able to point your browser to the right diff --git a/panoramix/bin/panoramix b/panoramix/bin/panoramix index cc17f8512a72d..12a8ee44e9dc6 100755 --- a/panoramix/bin/panoramix +++ b/panoramix/bin/panoramix @@ -4,6 +4,12 @@ from flask.ext.script import Manager from panoramix import app, config from subprocess import Popen from flask.ext.migrate import MigrateCommand +from panoramix import db +from flask.ext.appbuilder import Base +from sqlalchemy import Column, Integer, String +from panoramix import config, models +import csv +import gzip manager = Manager(app) @@ -33,9 +39,66 @@ def runserver(debug, port): Popen(cmd, shell=True).wait() @manager.command -def load_examples(self): +def load_examples(): """Loads a set of Slices and Dashboards and a supporting dataset """ - print("Loading examples") + print("Loading examples into {}".format(db)) + class BirthNames(Base): + __tablename__ = "birth_names" + id = Column(Integer, primary_key=True) + state = Column(String(10)) + year = Column(Integer) + name = Column(String(128)) + num = Column(Integer) + ds = Column(String(20)) + gender = Column(String(10)) + try: + BirthNames.__table__.drop(db.engine) + except: + pass + Base.metadata.create_all(db.engine) + session = db.session() + + with gzip.open(config.basedir + '/data/birth_names.csv.gz') as f: + bb_csv = csv.reader(f) + for i, (state, year, name, num, gender) in enumerate(bb_csv): + if i == 0 or not name or name=="\xc2\xa0": + continue + if num == "NA": + num = 0 + ds = str(year) + '-01-01' + session.add( + BirthNames( + state=state, year=year, + ds=ds, + name=name, num=num, gender=gender)) + if i % 1000 == 0: + print("{} loaded out of 502619 rows".format(i)) + session.commit() + session.commit() + print("Done loading table!") + DB = models.Database + dbobj = session.query(DB).filter_by(database_name='main').first() + if not dbobj: + dbobj = DB() + dbobj.database_name = "main" + dbobj.sqlalchemy_uri = config.SQLALCHEMY_DATABASE_URI + session.merge(dbobj) + session.commit() + + TBL = models.Table + obj = session.query(TBL).filter_by(table_name='birth_names').first() + if not obj: + obj = TBL() + obj.table_name = 'birth_names' + obj.main_dttm_col = 'ds' + obj.default_endpoint = "/panoramix/datasource/table/1/?viz_type=table&granularity=one+day&since=100+years&until=now&row_limit=10&where=&flt_col_0=ds&flt_op_0=in&flt_eq_0=&flt_col_1=ds&flt_op_1=in&flt_eq_1=&slice_name=TEST&datasource_name=birth_names&datasource_id=1&datasource_type=table" + obj.database = dbobj + obj.fetch_metadata() + session.merge(obj) + session.commit() + + session.close() + if __name__ == "__main__": manager.run() diff --git a/panoramix/data/birth_names.csv.gz b/panoramix/data/birth_names.csv.gz new file mode 100644 index 0000000000000..57acdf74e9c43 Binary files /dev/null and b/panoramix/data/birth_names.csv.gz differ diff --git a/panoramix/models.py b/panoramix/models.py index 7ef896f07ad40..037b54b69c0c7 100644 --- a/panoramix/models.py +++ b/panoramix/models.py @@ -24,11 +24,23 @@ from panoramix import db, get_session, config, utils from panoramix.viz import viz_types +from sqlalchemy.ext.declarative import declared_attr QueryResult = namedtuple('namedtuple', ['df', 'query', 'duration']) -class Slice(Model, AuditMixin): +class AuditMixinNullable(AuditMixin): + @declared_attr + def created_by_fk(cls): + return Column(Integer, ForeignKey('ab_user.id'), + default=cls.get_user_id, nullable=True) + @declared_attr + def changed_by_fk(cls): + return Column(Integer, ForeignKey('ab_user.id'), + default=cls.get_user_id, onupdate=cls.get_user_id, nullable=True) + + +class Slice(Model, AuditMixinNullable): """A slice is essentially a report or a view on data""" __tablename__ = 'slices' id = Column(Integer, primary_key=True) @@ -75,6 +87,10 @@ def slice_url(self): "{self.datasource_id}/".format(self=self)) return href(d) + @property + def edit_url(self): + return "/slicemodelview/edit/{}".format(self.id) + @property def slice_link(self): url = self.slice_url @@ -101,7 +117,7 @@ def get_viz(self): ) -class Dashboard(Model, AuditMixin): +class Dashboard(Model, AuditMixinNullable): """A dash to slash""" __tablename__ = 'dashboards' id = Column(Integer, primary_key=True) @@ -146,7 +162,7 @@ def filterable_column_names(self): return sorted([c.column_name for c in self.columns if c.filterable]) -class Database(Model, AuditMixin): +class Database(Model, AuditMixinNullable): __tablename__ = 'dbs' id = Column(Integer, primary_key=True) database_name = Column(String(250), unique=True) @@ -166,15 +182,13 @@ def get_table(self, table_name): autoload_with=self.get_sqla_engine()) -class Table(Model, Queryable, AuditMixin): +class Table(Model, Queryable, AuditMixinNullable): type = "table" __tablename__ = 'tables' id = Column(Integer, primary_key=True) - table_name = Column(String(255), unique=True) - main_datetime_column_id = Column(Integer, ForeignKey('table_columns.id')) - main_datetime_column = relationship( - 'TableColumn', foreign_keys=[main_datetime_column_id]) + table_name = Column(String(250), unique=True) + main_dttm_col = Column(String(250)) default_endpoint = Column(Text) database_id = Column(Integer, ForeignKey('dbs.id'), nullable=False) database = relationship( @@ -308,8 +322,11 @@ def query( extras=None): qry_start_dttm = datetime.now() + if not self.main_dttm_col: + raise Exception( + "Datetime column not provided as part table configuration") timestamp = literal_column( - self.main_datetime_column.column_name).label('timestamp') + self.main_dttm_col).label('timestamp') metrics_exprs = [ literal_column(m.expression).label(m.metric_name) for m in self.metrics if m.metric_name in metrics] @@ -420,7 +437,7 @@ def fetch_metadata(self): self.columns.append(dbcol) if not any_date_col and 'date' in datatype.lower(): - any_date_col = dbcol + any_date_col = col.name if dbcol.sum: metrics.append(M( @@ -464,18 +481,18 @@ def fetch_metadata(self): m = ( db.session.query(M) .filter(M.metric_name == metric.metric_name) - .filter(M.table == self) + .filter(M.table_id == self.id) .first() ) - metric.table = self + metric.table_id = self.id if not m: db.session.add(metric) db.session.commit() - if not self.main_datetime_column: - self.main_datetime_column = any_date_col + if not self.main_dttm_col: + self.main_dttm_col = any_date_col -class SqlMetric(Model, AuditMixin): +class SqlMetric(Model, AuditMixinNullable): __tablename__ = 'sql_metrics' id = Column(Integer, primary_key=True) metric_name = Column(String(512)) @@ -488,7 +505,7 @@ class SqlMetric(Model, AuditMixin): description = Column(Text) -class TableColumn(Model, AuditMixin): +class TableColumn(Model, AuditMixinNullable): __tablename__ = 'table_columns' id = Column(Integer, primary_key=True) table_id = Column(Integer, ForeignKey('tables.id')) @@ -513,7 +530,7 @@ def isnum(self): return self.type in ('LONG', 'DOUBLE', 'FLOAT') -class Cluster(Model, AuditMixin): +class Cluster(Model, AuditMixinNullable): __tablename__ = 'clusters' id = Column(Integer, primary_key=True) cluster_name = Column(String(250), unique=True) @@ -560,7 +577,7 @@ class Datasource(Model, AuditMixin, Queryable): user_id = Column(Integer, ForeignKey('ab_user.id')) owner = relationship('User', backref='datasources', foreign_keys=[user_id]) cluster_name = Column( - String(255), ForeignKey('clusters.cluster_name')) + String(250), ForeignKey('clusters.cluster_name')) cluster = relationship( 'Cluster', backref='datasources', foreign_keys=[cluster_name]) @@ -783,7 +800,7 @@ def json_obj(self): return obj -class Column(Model, AuditMixin): +class Column(Model, AuditMixinNullable): __tablename__ = 'columns' id = Column(Integer, primary_key=True) datasource_name = Column( diff --git a/panoramix/templates/panoramix/dashboard.html b/panoramix/templates/panoramix/dashboard.html index 28600eda4366d..a067c9641634d 100644 --- a/panoramix/templates/panoramix/dashboard.html +++ b/panoramix/templates/panoramix/dashboard.html @@ -95,7 +95,7 @@

- +
diff --git a/panoramix/templates/panoramix/viz_table.html b/panoramix/templates/panoramix/viz_table.html index 20efc23e9f7ab..42f7c51244b09 100644 --- a/panoramix/templates/panoramix/viz_table.html +++ b/panoramix/templates/panoramix/viz_table.html @@ -26,8 +26,7 @@ {% else %} - + {% endif %} {% endmacro %} @@ -37,14 +36,22 @@ {% endif %} diff --git a/panoramix/views.py b/panoramix/views.py index e16c512b9b7e9..47263910f6871 100644 --- a/panoramix/views.py +++ b/panoramix/views.py @@ -86,6 +86,44 @@ class MetricInlineView(CompactCRUDMixin, ModelView): appbuilder.add_view_no_menu(MetricInlineView) +class DatabaseView(ModelView, DeleteMixin): + datamodel = SQLAInterface(models.Database) + list_columns = ['database_name'] + add_columns = ['database_name', 'sqlalchemy_uri'] + edit_columns = add_columns + +appbuilder.add_view( + DatabaseView, + "Databases", + icon="fa-database", + category="Sources", + category_icon='fa-cogs',) + + +class TableView(ModelView, DeleteMixin): + datamodel = SQLAInterface(models.Table) + list_columns = ['table_link', 'database'] + add_columns = ['table_name', 'database', 'default_endpoint'] + edit_columns = [ + 'table_name', 'database', 'main_dttm_col', 'default_endpoint'] + related_views = [TableColumnInlineView, SqlMetricInlineView] + + def post_add(self, table): + table.fetch_metadata() + + def post_update(self, table): + table.fetch_metadata() + +appbuilder.add_view( + TableView, + "Tables", + category="Sources", + icon='fa-table',) + + +appbuilder.add_separator("Sources") + + class ClusterModelView(ModelView, DeleteMixin): datamodel = SQLAInterface(models.Cluster) add_columns = [ @@ -100,13 +138,16 @@ class ClusterModelView(ModelView, DeleteMixin): ClusterModelView, "Druid Clusters", icon="fa-server", - category="Admin", + category="Sources", category_icon='fa-cogs',) class SliceModelView(ModelView, DeleteMixin): datamodel = SQLAInterface(models.Slice) list_columns = ['slice_link', 'viz_type', 'datasource', 'created_by'] + edit_columns = [ + 'slice_name', 'viz_type', 'druid_datasource', 'table', + 'dashboards', 'params'] appbuilder.add_view( SliceModelView, @@ -131,40 +172,6 @@ class DashboardModelView(ModelView, DeleteMixin): category_icon='',) -class DatabaseView(ModelView, DeleteMixin): - datamodel = SQLAInterface(models.Database) - list_columns = ['database_name'] - add_columns = ['database_name', 'sqlalchemy_uri'] - edit_columns = add_columns - -appbuilder.add_view( - DatabaseView, - "Databases", - icon="fa-database", - category="Admin", - category_icon='fa-cogs',) - - -class TableView(ModelView, DeleteMixin): - datamodel = SQLAInterface(models.Table) - list_columns = ['table_link', 'database'] - add_columns = ['table_name', 'database', 'default_endpoint'] - edit_columns = [ - 'table_name', 'database', 'main_datetime_column', 'default_endpoint'] - related_views = [TableColumnInlineView, SqlMetricInlineView] - - def post_add(self, table): - table.fetch_metadata() - - def post_update(self, table): - table.fetch_metadata() - -appbuilder.add_view( - TableView, - "Tables", - icon='fa-table',) - - class DatasourceModelView(ModelView, DeleteMixin): datamodel = SQLAInterface(models.Datasource) list_columns = [ @@ -186,6 +193,7 @@ def post_update(self, datasource): appbuilder.add_view( DatasourceModelView, "Druid Datasources", + category="Sources", icon="fa-cubes") @@ -241,7 +249,14 @@ def datasource(self, datasource_type, datasource_id): status=status, mimetype="application/json") else: - return self.render_template("panoramix/viz.html", viz=obj) + try: + resp = self.render_template("panoramix/viz.html", viz=obj) + except Exception as e: + return Response( + str(e), + status=500, + mimetype="application/json") + return resp @has_access @expose("/save_dash//", methods=['GET', 'POST']) @@ -303,8 +318,10 @@ def dashboard(self, id_): .filter(models.Dashboard.id == id_) .first() ) - pos_dict = { - int(o['slice_id']):o for o in json.loads(dashboard.position_json)} + pos_dict = {} + if dashboard.position_json: + pos_dict = { + int(o['slice_id']):o for o in json.loads(dashboard.position_json)} return self.render_template( "panoramix/dashboard.html", dashboard=dashboard, pos_dict=pos_dict) @@ -342,6 +359,6 @@ def autocomplete(self, datasource, column): appbuilder.add_link( "Refresh Druid Metadata", href='/panoramix/refresh_datasources/', - category='Admin', + category='Sources', category_icon='fa-cogs', icon="fa-cog")