From c2049f837c0152a400fbba0a075389c8f2812158 Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Tue, 22 Sep 2015 11:36:54 -0700 Subject: [PATCH 1/7] Automate Panoramix role creation --- panoramix/bin/panoramix | 13 ++++++++----- panoramix/utils.py | 20 +++++++++++++++++++- tests/core_tests.py | 6 ++++-- 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/panoramix/bin/panoramix b/panoramix/bin/panoramix index 461341e8ca198..cc96fc4d0f1ba 100755 --- a/panoramix/bin/panoramix +++ b/panoramix/bin/panoramix @@ -7,20 +7,18 @@ import json from subprocess import Popen from flask.ext.script import Manager +from panoramix import app from flask.ext.migrate import MigrateCommand from panoramix import db +from flask.ext.appbuilder import Base from sqlalchemy import Column, Integer, String, Table, DateTime - -from panoramix import app -from panoramix import models - +from panoramix import models, utils config = app.config manager = Manager(app) manager.add_command('db', MigrateCommand) -from flask.ext.appbuilder import Base @manager.option( '-d', '--debug', action='store_true', @@ -45,6 +43,11 @@ def runserver(debug, port): print("Starting server with command: " + cmd) Popen(cmd, shell=True).wait() +@manager.command +def init(): + """Inits the Panoramix application""" + utils.init() + @manager.option( '-s', '--sample', action='store_true', help="Only load 1000 rows (faster, used for testing)") diff --git a/panoramix/utils.py b/panoramix/utils.py index 3fc1645173fe3..31517e8ce2dc0 100644 --- a/panoramix/utils.py +++ b/panoramix/utils.py @@ -5,6 +5,7 @@ import json import parsedatetime import functools +from panoramix import db class memoized(object): @@ -78,7 +79,6 @@ def parse_human_timedelta(s): return d - dttm - class JSONEncodedDict(TypeDecorator): """Represents an immutable structure as a json-encoded string.""" impl = TEXT @@ -93,6 +93,7 @@ def process_result_value(self, value, dialect): value = json.loads(value) return value + def color(s): """ Get a consistent color from the same string using a hash function @@ -109,3 +110,20 @@ def color(s): h = hashlib.md5(s) i = int(h.hexdigest(), 16) return colors[i % len(colors)] + + +def init(): + """ + Inits the Panoramix application with security roles and such + """ + from panoramix import appbuilder + sm = appbuilder.sm + alpha = sm.add_role("Alpha") + from flask_appbuilder.security.sqla import models + perms = db.session.query(models.PermissionView).all() + for perm in perms: + if perm.view_menu.name not in ( + 'UserDBModelView', 'RoleModelView', 'ResetPasswordView', + 'Security'): + sm.add_permission_role(alpha, perm) + sm.add_role("Gamma") diff --git a/tests/core_tests.py b/tests/core_tests.py index 24848d8ba0f7c..49d10d4ae5a47 100644 --- a/tests/core_tests.py +++ b/tests/core_tests.py @@ -4,9 +4,8 @@ import urllib2 os.environ['PANORAMIX_CONFIG'] = 'tests.panoramix_test_config' from flask.ext.testing import LiveServerTestCase, TestCase -from flask_login import login_user -from panoramix import app, appbuilder, db, models +from panoramix import app, db, models, utils BASE_DIR = app.config.get("BASE_DIR") cli = imp.load_source('cli', BASE_DIR + "/bin/panoramix") @@ -21,6 +20,9 @@ def create_app(self): def setUp(self): pass + def test_init(self): + utils.init() + def test_load_examples(self): cli.load_examples(sample=True) From 45b595001986d68f48278e1c217eb4ab5e0c5a40 Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Tue, 29 Sep 2015 22:45:43 -0700 Subject: [PATCH 2/7] Definining a Gamma role --- panoramix/utils.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/panoramix/utils.py b/panoramix/utils.py index 31517e8ce2dc0..4240eba1c3aaa 100644 --- a/panoramix/utils.py +++ b/panoramix/utils.py @@ -126,4 +126,20 @@ def init(): 'UserDBModelView', 'RoleModelView', 'ResetPasswordView', 'Security'): sm.add_permission_role(alpha, perm) - sm.add_role("Gamma") + gamma = sm.add_role("Gamma") + for perm in perms: + s = perm.permission.name + if( + perm.view_menu.name not in ( + 'UserDBModelView', + 'RoleModelView', + 'ResetPasswordView', + 'Security') and + perm.permission.name not in ( + 'can_edit', + 'can_add', + 'can_save', + 'can_download', + 'muldelete', + )): + sm.add_permission_role(gamma, perm) From e3cdf5bb2c1b699d8c747f1157f425373fd9d871 Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Sun, 4 Oct 2015 08:27:09 -0700 Subject: [PATCH 3/7] Progress --- panoramix/models.py | 15 ++++++++------- panoramix/utils.py | 5 +++++ panoramix/views.py | 4 ++-- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/panoramix/models.py b/panoramix/models.py index 3c3a24b4878b5..30a14e80e59fb 100644 --- a/panoramix/models.py +++ b/panoramix/models.py @@ -8,8 +8,8 @@ from pydruid.utils.filters import Dimension, Filter from sqlalchemy import ( Column, Integer, String, ForeignKey, Text, Boolean, DateTime) -from sqlalchemy import Table as sqlaTable -from sqlalchemy import create_engine, MetaData, desc, select, and_, Table +from sqlalchemy import Table +from sqlalchemy import create_engine, MetaData, desc, select, and_ from sqlalchemy.orm import relationship from sqlalchemy.sql import table, literal_column, text from flask import request @@ -55,7 +55,7 @@ class Slice(Model, AuditMixinNullable): params = Column(Text) table = relationship( - 'Table', foreign_keys=[table_id], backref='slices') + 'SqlaTable', foreign_keys=[table_id], backref='slices') druid_datasource = relationship( 'Datasource', foreign_keys=[druid_datasource_id], backref='slices') @@ -184,13 +184,13 @@ def get_sqla_engine(self): def get_table(self, table_name): meta = MetaData() - return sqlaTable( + return Table( table_name, meta, autoload=True, autoload_with=self.get_sqla_engine()) -class Table(Model, Queryable, AuditMixinNullable): +class SqlaTable(Model, Queryable, AuditMixinNullable): type = "table" __tablename__ = 'tables' @@ -519,7 +519,7 @@ class SqlMetric(Model, AuditMixinNullable): metric_type = Column(String(32)) table_id = Column(Integer, ForeignKey('tables.id')) table = relationship( - 'Table', backref='metrics', foreign_keys=[table_id]) + 'SqlaTable', backref='metrics', foreign_keys=[table_id]) expression = Column(Text) description = Column(Text) @@ -528,7 +528,8 @@ class TableColumn(Model, AuditMixinNullable): __tablename__ = 'table_columns' id = Column(Integer, primary_key=True) table_id = Column(Integer, ForeignKey('tables.id')) - table = relationship('Table', backref='columns', foreign_keys=[table_id]) + table = relationship( + 'SqlaTable', backref='columns', foreign_keys=[table_id]) column_name = Column(String(256)) is_dttm = Column(Boolean, default=True) is_active = Column(Boolean, default=True) diff --git a/panoramix/utils.py b/panoramix/utils.py index 4240eba1c3aaa..25f5181484730 100644 --- a/panoramix/utils.py +++ b/panoramix/utils.py @@ -143,3 +143,8 @@ def init(): 'muldelete', )): sm.add_permission_role(gamma, perm) + session = db.session() + for i in range(100): + print(type(models.Table)) + for table in session.query(models.Table).all(): + print table diff --git a/panoramix/views.py b/panoramix/views.py index 91771f76722d1..89d3ff455f740 100644 --- a/panoramix/views.py +++ b/panoramix/views.py @@ -116,7 +116,7 @@ class DatabaseView(PanoramixModelView, DeleteMixin): class TableView(PanoramixModelView, DeleteMixin): - datamodel = SQLAInterface(models.Table) + datamodel = SQLAInterface(models.SqlaTable) list_columns = ['table_link', 'database'] add_columns = ['table_name', 'database', 'default_endpoint'] edit_columns = [ @@ -268,7 +268,7 @@ def datasource(self, datasource_type, datasource_id): if datasource_type == "table": datasource = ( db.session - .query(models.Table) + .query(models.SqlaTable) .filter_by(id=datasource_id) .first() ) From 71cf515d0d48cb26467eaed08ee613afd77e170d Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Sun, 4 Oct 2015 09:33:08 -0700 Subject: [PATCH 4/7] Getting this thing to work --- panoramix/models.py | 6 ++++++ panoramix/utils.py | 15 ++++++++------- panoramix/views.py | 41 +++++++++++++++++++++++++---------------- 3 files changed, 39 insertions(+), 23 deletions(-) diff --git a/panoramix/models.py b/panoramix/models.py index 30a14e80e59fb..14fb39b0d6a78 100644 --- a/panoramix/models.py +++ b/panoramix/models.py @@ -207,6 +207,12 @@ class SqlaTable(Model, Queryable, AuditMixinNullable): def __repr__(self): return self.table_name + @property + def perm(self): + return ( + "[{self.database}].[{self.table_name}]" + "(id:{self.id})").format(self=self) + @property def name(self): return self.table_name diff --git a/panoramix/utils.py b/panoramix/utils.py index 25f5181484730..99e1c8230afe9 100644 --- a/panoramix/utils.py +++ b/panoramix/utils.py @@ -1,4 +1,4 @@ -from datetime import date, datetime, timedelta +from datetime import datetime from dateutil.parser import parse import hashlib from sqlalchemy.types import TypeDecorator, TEXT @@ -117,10 +117,11 @@ def init(): Inits the Panoramix application with security roles and such """ from panoramix import appbuilder + from panoramix import models sm = appbuilder.sm alpha = sm.add_role("Alpha") - from flask_appbuilder.security.sqla import models - perms = db.session.query(models.PermissionView).all() + from flask_appbuilder.security.sqla import models as ab_models + perms = db.session.query(ab_models.PermissionView).all() for perm in perms: if perm.view_menu.name not in ( 'UserDBModelView', 'RoleModelView', 'ResetPasswordView', @@ -144,7 +145,7 @@ def init(): )): sm.add_permission_role(gamma, perm) session = db.session() - for i in range(100): - print(type(models.Table)) - for table in session.query(models.Table).all(): - print table + for table in session.query(models.SqlaTable).all(): + sm.add_permission_view_menu('datasource_access', table.perm) + for druid_datasource in session.query(models.Datasource).all(): + sm.add_permission_view_menu('datasource_access', druid_datasource.perm) diff --git a/panoramix/views.py b/panoramix/views.py index 89d3ff455f740..6470398ca0db4 100644 --- a/panoramix/views.py +++ b/panoramix/views.py @@ -2,7 +2,7 @@ import json import logging -from flask import request, redirect, flash, Response +from flask import request, redirect, flash, Response, g from flask.ext.appbuilder import ModelView, CompactCRUDMixin, BaseView, expose from flask.ext.appbuilder.actions import action from flask.ext.appbuilder.models.sqla.interface import SQLAInterface @@ -229,6 +229,29 @@ class Panoramix(BaseView): @has_access @expose("/datasource///") def datasource(self, datasource_type, datasource_id): + if datasource_type == "table": + datasource = ( + db.session + .query(models.SqlaTable) + .filter_by(id=datasource_id) + .first() + ) + else: + datasource = ( + db.session + .query(models.Datasource) + .filter_by(id=datasource_id) + .first() + ) + + if 'Gamma' in [r.name for r in g.user.roles]: + datasource_access = self.appbuilder.sm.has_access( + 'datasource_access', datasource.perm) + if not datasource_access: + flash( + "You don't seem to have access to this datasource", + "danger") + return redirect('/') action = request.args.get('action') if action == 'save': session = db.session() @@ -263,22 +286,8 @@ def datasource(self, datasource_type, datasource_id): session.add(obj) session.commit() flash("Slice <{}> has been added to the pie".format(slice_name), "info") - redirect(obj.slice_url) + return redirect(obj.slice_url) - if datasource_type == "table": - datasource = ( - db.session - .query(models.SqlaTable) - .filter_by(id=datasource_id) - .first() - ) - else: - datasource = ( - db.session - .query(models.Datasource) - .filter_by(id=datasource_id) - .first() - ) if not datasource: flash("The datasource seem to have been deleted", "alert") From 459048abe0f8873ada861edc712d5abcf0456216 Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Sun, 4 Oct 2015 09:35:05 -0700 Subject: [PATCH 5/7] Bugfix --- panoramix/bin/panoramix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/panoramix/bin/panoramix b/panoramix/bin/panoramix index cc96fc4d0f1ba..47b41d5627429 100755 --- a/panoramix/bin/panoramix +++ b/panoramix/bin/panoramix @@ -111,7 +111,7 @@ def load_examples(sample): session.commit() print("Creating table reference") - TBL = models.Table + TBL = models.SqlaTable obj = session.query(TBL).filter_by(table_name='birth_names').first() if not obj: obj = TBL(table_name = 'birth_names') From b4c8d7a81e0062608d0d666e412697cae74e8e38 Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Sun, 4 Oct 2015 09:42:49 -0700 Subject: [PATCH 6/7] Cosmetics --- panoramix/static/panoramix.css | 2 +- panoramix/templates/index.html | 10 +++++----- panoramix/views.py | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/panoramix/static/panoramix.css b/panoramix/static/panoramix.css index 2cdf80dc592a9..a75335c163768 100644 --- a/panoramix/static/panoramix.css +++ b/panoramix/static/panoramix.css @@ -31,7 +31,7 @@ form div { font-size: 80px; } .index .carousel-caption p { - font-size: 25px; + font-size: 20px; } .index div.carousel-caption{ background: rgba(0,0,0,0.5); diff --git a/panoramix/templates/index.html b/panoramix/templates/index.html index 9551ad490012b..ab2339fe91e15 100644 --- a/panoramix/templates/index.html +++ b/panoramix/templates/index.html @@ -28,8 +28,8 @@

Panoramix

-

Explore your data -

+

Explore your data +

Intuitively navigate your data while slicing, dicing, and visualizing through a rich set of widgets @@ -39,21 +39,21 @@

Explore your data
-

Create and share dashboards

+

Create and share dashboards

Assemble many data visualization "slices" into a rich collection

-

Extend

+

Extend

Join the community and take part in extending the widget library

-

Connect

+

Connect

Access data from MySql, Presto.db, Postgres, RedShift, Oracle, MsSql, SQLite, and more through the SqlAlchemy integration. You can also diff --git a/panoramix/views.py b/panoramix/views.py index 6470398ca0db4..9c803c9672fa1 100644 --- a/panoramix/views.py +++ b/panoramix/views.py @@ -251,7 +251,7 @@ def datasource(self, datasource_type, datasource_id): flash( "You don't seem to have access to this datasource", "danger") - return redirect('/') + return redirect('/slicemodelview/list/') action = request.args.get('action') if action == 'save': session = db.session() From 6ea178eced0dcb512915c042aa8db2a6114cfb5d Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Sun, 4 Oct 2015 16:18:47 -0700 Subject: [PATCH 7/7] Perms are kept from getting duped, created on save --- panoramix/static/panoramix.js | 2 +- panoramix/templates/panoramix/datasource.html | 15 ++++++++----- panoramix/utils.py | 22 ++++++++++++++----- panoramix/views.py | 22 +++++++++++++------ 4 files changed, 43 insertions(+), 18 deletions(-) diff --git a/panoramix/static/panoramix.js b/panoramix/static/panoramix.js index a7a59271cf4fe..7d97d273e1b98 100644 --- a/panoramix/static/panoramix.js +++ b/panoramix/static/panoramix.js @@ -64,7 +64,7 @@ function initializeDatasourceView() { } }) add_filter(); - $("#druidify").click(druidify); + $(".druidify").click(druidify); function create_choices(term, data) { var filtered = $(data).filter(function() { diff --git a/panoramix/templates/panoramix/datasource.html b/panoramix/templates/panoramix/datasource.html index cedb23deaaf48..2838585d6ca3c 100644 --- a/panoramix/templates/panoramix/datasource.html +++ b/panoramix/templates/panoramix/datasource.html @@ -8,11 +8,16 @@

{{ datasource.name }} {% if datasource.description %} - + {% endif %} - - - +
+ + + + +

@@ -68,7 +73,7 @@

Filters

- diff --git a/panoramix/utils.py b/panoramix/utils.py index 99e1c8230afe9..0a9fa621f34aa 100644 --- a/panoramix/utils.py +++ b/panoramix/utils.py @@ -63,6 +63,12 @@ def dttm_from_timtuple(d): d.tm_year, d.tm_mon, d.tm_mday, d.tm_hour, d.tm_min, d.tm_sec) +def merge_perm(sm, permission_name, view_menu_name): + pv = sm.find_permission_view_menu(permission_name, view_menu_name) + if not pv: + sm.add_permission_view_menu(permission_name, view_menu_name) + + def parse_human_timedelta(s): """ Use the parsedatetime lib to return ``datetime.datetime`` from human @@ -118,9 +124,12 @@ def init(): """ from panoramix import appbuilder from panoramix import models + from flask_appbuilder.security.sqla import models as ab_models sm = appbuilder.sm alpha = sm.add_role("Alpha") - from flask_appbuilder.security.sqla import models as ab_models + + merge_perm(sm, 'all_datasource_access', 'all_datasource_access') + perms = db.session.query(ab_models.PermissionView).all() for perm in perms: if perm.view_menu.name not in ( @@ -142,10 +151,13 @@ def init(): 'can_save', 'can_download', 'muldelete', + 'all_datasource_access', )): sm.add_permission_role(gamma, perm) session = db.session() - for table in session.query(models.SqlaTable).all(): - sm.add_permission_view_menu('datasource_access', table.perm) - for druid_datasource in session.query(models.Datasource).all(): - sm.add_permission_view_menu('datasource_access', druid_datasource.perm) + table_perms = [ + table.perm for table in session.query(models.SqlaTable).all()] + table_perms += [ + table.perm for table in session.query(models.Datasource).all()] + for table_perm in table_perms: + merge_perm(sm, 'datasource_access', table.perm) diff --git a/panoramix/views.py b/panoramix/views.py index 9c803c9672fa1..bb5b4357225df 100644 --- a/panoramix/views.py +++ b/panoramix/views.py @@ -11,7 +11,7 @@ from sqlalchemy import create_engine from wtforms.validators import ValidationError -from panoramix import appbuilder, db, models, viz, utils, app +from panoramix import appbuilder, db, models, viz, utils, app, sm config = app.config @@ -124,10 +124,17 @@ class TableView(PanoramixModelView, DeleteMixin): related_views = [TableColumnInlineView, SqlMetricInlineView] def post_add(self, table): - table.fetch_metadata() + try: + table.fetch_metadata() + except Exception as e: + flash( + "Table [{}] doesn't seem to exist, " + "couldn't fetch metadata".format(table.table_name), + "danger") + utils.merge_perm(sm, 'datasource_access', table.perm) def post_update(self, table): - table.fetch_metadata() + self.post_add(table) appbuilder.add_view( TableView, @@ -203,10 +210,10 @@ class DatasourceModelView(PanoramixModelView, DeleteMixin): def post_add(self, datasource): datasource.generate_metrics() + utils.merge_perm(sm, 'datasource_access', table.perm) def post_update(self, datasource): - datasource.generate_metrics() - + self.post_add(datasource) appbuilder.add_view( DatasourceModelView, @@ -244,10 +251,11 @@ def datasource(self, datasource_type, datasource_id): .first() ) - if 'Gamma' in [r.name for r in g.user.roles]: + all_datasource_access = self.appbuilder.sm.has_access( + 'all_datasource_access', 'all_datasource_access') datasource_access = self.appbuilder.sm.has_access( 'datasource_access', datasource.perm) - if not datasource_access: + if not all_datasource_access or not datasource_access: flash( "You don't seem to have access to this datasource", "danger")