From c6fe2df7e656213308217673b87afe038602c28e Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Thu, 27 Sep 2018 17:01:06 +0530 Subject: [PATCH 1/6] feat: import/export dashboards via cli --- superset/cli.py | 49 +++++++++++++++++++++++- superset/dashboard_import_export_util.py | 36 +++++++++++++++++ superset/views/core.py | 13 +------ 3 files changed, 85 insertions(+), 13 deletions(-) create mode 100644 superset/dashboard_import_export_util.py diff --git a/superset/cli.py b/superset/cli.py index 6183601d02536..1e62a7a9f4c6a 100755 --- a/superset/cli.py +++ b/superset/cli.py @@ -18,7 +18,7 @@ import yaml from superset import ( - app, data, db, dict_import_export_util, security_manager, utils, + app, dashboard_import_export_util, data, db, dict_import_export_util, security_manager, utils, ) config = app.config @@ -223,6 +223,51 @@ def refresh_druid(datasource, merge): '[' + cluster.cluster_name + ']') session.commit() +@app.cli.command() +@click.option( + '--path', '-p', + help='Path to a single JSON file or path containing multiple JSON ' + 'files to import (*.json)') +@click.option( + '--recursive', '-r', + help='recursively search the path for json files') +def import_dashboards(path, recursive=False): + """Import dashboards from JSON""" + p = Path(path) + files = [] + if p.is_file(): + files.append(p) + elif p.exists() and not recursive: + files.extend(p.glob('*.json')) + elif p.exists() and recursive: + files.extend(p.rglob('*.json')) + for f in files: + logging.info('Importing dashboard from file %s', f) + try: + with f.open() as data_stream: + dashboard_import_export_util.import_dashboards( + db.session, data_stream ) + except Exception as e: + logging.error('Error when importing dashboard from file %s', f) + logging.error(e) + +@app.cli.command() +@click.option( + '--dashboard-file', '-f', default=None, + help='Specify the the file to export to') +@click.option( + '--print_stdout', '-p', + help='Print JSON to stdout') +def export_dashboards(print_stdout, dashboard_file): + """Export dashboards to json""" + data = dashboard_import_export_util.export_dashboards(db.session) + if print_stdout or not dashboard_file: + print(data) + if dashboard_file: + logging.info('Exporting dashboards to %s', dashboard_file) + with open(dashboard_file, 'w') as data_stream: + data_stream.write(data) + @app.cli.command() @click.option( @@ -268,7 +313,7 @@ def import_datasources(path, sync, recursive=False): '--datasource-file', '-f', default=None, help='Specify the the file to export to') @click.option( - '--print', '-p', + '--print_stdout', '-p', help='Print YAML to stdout') @click.option( '--back-references', '-b', diff --git a/superset/dashboard_import_export_util.py b/superset/dashboard_import_export_util.py new file mode 100644 index 0000000000000..59502270485e6 --- /dev/null +++ b/superset/dashboard_import_export_util.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +# pylint: disable=C,R,W +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import logging +import time +import json + +from superset import utils +from superset.models.core import Dashboard + +def import_dashboards(session, data_stream): + """Imports dashboards""" + current_tt = int(time.time()) + data = json.loads(data_stream.read(), object_hook=utils.decode_dashboards) + # TODO: import DRUID datasources + for table in data['datasources']: + type(table).import_obj(table, import_time=current_tt) + session.commit() + for dashboard in data['dashboards']: + Dashboard.import_obj( + dashboard, import_time=current_tt) + session.commit() + +def export_dashboards(session): + logging.info('Starting export') + dashboards = session.query(Dashboard) + dashboard_ids = [] + for dashboard in dashboards: + dashboard_ids.append(dashboard.id) + data = Dashboard.export_dashboards(dashboard_ids) + logging.info('Starting export %s', data) + return data \ No newline at end of file diff --git a/superset/views/core.py b/superset/views/core.py index f811fd8733f4a..b8d7d1aef1f8e 100755 --- a/superset/views/core.py +++ b/superset/views/core.py @@ -34,7 +34,7 @@ from werkzeug.utils import secure_filename from superset import ( - app, appbuilder, cache, db, results_backend, security_manager, sql_lab, utils, + app, appbuilder, cache, dashboard_import_export_util ,db, results_backend, security_manager, sql_lab, utils, viz, ) from superset.connectors.connector_registry import ConnectorRegistry @@ -1238,16 +1238,7 @@ def import_dashboards(self): """Overrides the dashboards using json instances from the file.""" f = request.files.get('file') if request.method == 'POST' and f: - current_tt = int(time.time()) - data = json.loads(f.stream.read(), object_hook=utils.decode_dashboards) - # TODO: import DRUID datasources - for table in data['datasources']: - type(table).import_obj(table, import_time=current_tt) - db.session.commit() - for dashboard in data['dashboards']: - models.Dashboard.import_obj( - dashboard, import_time=current_tt) - db.session.commit() + dashboard_import_export_util.import_dashboards(db.session, f.stream) return redirect('/dashboard/list/') return self.render_template('superset/import_dashboards.html') From 627fdda58e9653011a063cbcb99d4d27d52020bd Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Fri, 28 Sep 2018 11:19:40 +0530 Subject: [PATCH 2/6] style: fixed lint error --- superset/cli.py | 7 +++++-- superset/dashboard_import_export_util.py | 20 +++++++++++--------- superset/views/core.py | 5 ++--- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/superset/cli.py b/superset/cli.py index 1e62a7a9f4c6a..e7764bd978254 100755 --- a/superset/cli.py +++ b/superset/cli.py @@ -18,7 +18,8 @@ import yaml from superset import ( - app, dashboard_import_export_util, data, db, dict_import_export_util, security_manager, utils, + app, dashboard_import_export_util, data, db, + dict_import_export_util, security_manager, utils, ) config = app.config @@ -223,6 +224,7 @@ def refresh_druid(datasource, merge): '[' + cluster.cluster_name + ']') session.commit() + @app.cli.command() @click.option( '--path', '-p', @@ -246,11 +248,12 @@ def import_dashboards(path, recursive=False): try: with f.open() as data_stream: dashboard_import_export_util.import_dashboards( - db.session, data_stream ) + db.session, data_stream) except Exception as e: logging.error('Error when importing dashboard from file %s', f) logging.error(e) + @app.cli.command() @click.option( '--dashboard-file', '-f', default=None, diff --git a/superset/dashboard_import_export_util.py b/superset/dashboard_import_export_util.py index 59502270485e6..f9c852da4a041 100644 --- a/superset/dashboard_import_export_util.py +++ b/superset/dashboard_import_export_util.py @@ -5,13 +5,14 @@ from __future__ import print_function from __future__ import unicode_literals +import json import logging import time -import json from superset import utils from superset.models.core import Dashboard + def import_dashboards(session, data_stream): """Imports dashboards""" current_tt = int(time.time()) @@ -25,12 +26,13 @@ def import_dashboards(session, data_stream): dashboard, import_time=current_tt) session.commit() + def export_dashboards(session): - logging.info('Starting export') - dashboards = session.query(Dashboard) - dashboard_ids = [] - for dashboard in dashboards: - dashboard_ids.append(dashboard.id) - data = Dashboard.export_dashboards(dashboard_ids) - logging.info('Starting export %s', data) - return data \ No newline at end of file + logging.info('Starting export') + dashboards = session.query(Dashboard) + dashboard_ids = [] + for dashboard in dashboards: + dashboard_ids.append(dashboard.id) + data = Dashboard.export_dashboards(dashboard_ids) + logging.info('Starting export %s', data) + return data diff --git a/superset/views/core.py b/superset/views/core.py index b8d7d1aef1f8e..6d3757080f8a3 100755 --- a/superset/views/core.py +++ b/superset/views/core.py @@ -34,9 +34,8 @@ from werkzeug.utils import secure_filename from superset import ( - app, appbuilder, cache, dashboard_import_export_util ,db, results_backend, security_manager, sql_lab, utils, - viz, -) + app, appbuilder, cache, dashboard_import_export_util, db, results_backend, + security_manager, sql_lab, utils, viz) from superset.connectors.connector_registry import ConnectorRegistry from superset.connectors.sqla.models import AnnotationDatasource, SqlaTable from superset.exceptions import SupersetException From f5d47f21724a75160ce665d2b6c045857659724e Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Mon, 1 Oct 2018 14:42:37 +0530 Subject: [PATCH 3/6] test: added test for import and export util --- superset/cli.py | 4 +- superset/dashboard_import_export_util.py | 11 +- tests/births_dash_test.json | 529 +++++++++++++++++++++++ tests/import_export_tests.py | 46 +- 4 files changed, 582 insertions(+), 8 deletions(-) create mode 100644 tests/births_dash_test.json diff --git a/superset/cli.py b/superset/cli.py index e7764bd978254..4a34bdcfa4dae 100755 --- a/superset/cli.py +++ b/superset/cli.py @@ -228,7 +228,7 @@ def refresh_druid(datasource, merge): @app.cli.command() @click.option( '--path', '-p', - help='Path to a single JSON file or path containing multiple JSON ' + help='Path to a single JSON file or path containing multiple JSON files' 'files to import (*.json)') @click.option( '--recursive', '-r', @@ -262,7 +262,7 @@ def import_dashboards(path, recursive=False): '--print_stdout', '-p', help='Print JSON to stdout') def export_dashboards(print_stdout, dashboard_file): - """Export dashboards to json""" + """Export dashboards to JSON""" data = dashboard_import_export_util.export_dashboards(db.session) if print_stdout or not dashboard_file: print(data) diff --git a/superset/dashboard_import_export_util.py b/superset/dashboard_import_export_util.py index f9c852da4a041..0a8fd25c2b2e3 100644 --- a/superset/dashboard_import_export_util.py +++ b/superset/dashboard_import_export_util.py @@ -13,26 +13,27 @@ from superset.models.core import Dashboard -def import_dashboards(session, data_stream): - """Imports dashboards""" +def import_dashboards(session, data_stream, import_time=None): + """Imports dashboards from a stream to databases""" current_tt = int(time.time()) + import_time = current_tt if import_time is None else import_time data = json.loads(data_stream.read(), object_hook=utils.decode_dashboards) # TODO: import DRUID datasources for table in data['datasources']: - type(table).import_obj(table, import_time=current_tt) + type(table).import_obj(table, import_time=import_time) session.commit() for dashboard in data['dashboards']: Dashboard.import_obj( - dashboard, import_time=current_tt) + dashboard, import_time=import_time) session.commit() def export_dashboards(session): + """Returns all dashboards metadata as a json dump""" logging.info('Starting export') dashboards = session.query(Dashboard) dashboard_ids = [] for dashboard in dashboards: dashboard_ids.append(dashboard.id) data = Dashboard.export_dashboards(dashboard_ids) - logging.info('Starting export %s', data) return data diff --git a/tests/births_dash_test.json b/tests/births_dash_test.json new file mode 100644 index 0000000000000..e680333da3f6d --- /dev/null +++ b/tests/births_dash_test.json @@ -0,0 +1,529 @@ +{ + "dashboards": [ + { + "__Dashboard__": { + "dashboard_title": "Births for test", + "slug": "test_birth_import", + "json_metadata": "{\"filter_immune_slice_fields\": {}, \"filter_immune_slices\": [], \"remote_id\": 33333, \"timed_refresh_immune_slices\": [], \"expanded_slices\": {}, \"default_filters\": \"{}\", \"import_time\": 2018}", + "description": null, + "slices": [ + { + "__Slice__": { + "created_by_fk": 1, + "datasource_id": 3, + "description": null, + "viz_type": "pie", + "perm": "[main].[birth_names](id:3)", + "datasource_type": "table", + "created_on": { + "__datetime__": "2018-10-01T10:00:22" + }, + "params": "{\"since\": \"100 years ago\", \"row_limit\": 50000, \"metric\": \"sum__num\", \"metrics\": [\"sum__num\"], \"markup_type\": \"markdown\", \"granularity_sqla\": \"ds\", \"viz_type\": \"pie\", \"database_name\": \"main\", \"remote_id\": 280, \"compare_suffix\": \"o10Y\", \"compare_lag\": \"10\", \"limit\": \"25\", \"until\": \"now\", \"datasource_name\": \"birth_names\", \"where\": \"\", \"groupby\": [\"gender\"], \"import_time\": 2018, \"schema\": \"birth_names\"}", + "changed_by_fk": 1, + "datasource_name": null, + "slice_name": "Genders", + "cache_timeout": null, + "id": 280, + "changed_on": { + "__datetime__": "2018-10-01T10:36:24" + } + } + }, + { + "__Slice__": { + "created_by_fk": 1, + "datasource_id": 3, + "description": null, + "viz_type": "big_number_total", + "perm": "[main].[birth_names](id:3)", + "datasource_type": "table", + "created_on": { + "__datetime__": "2018-10-01T10:00:22" + }, + "params": "{\"subheader\": \"total female participants\", \"since\": \"100 years ago\", \"row_limit\": 50000, \"metric\": \"sum__num\", \"metrics\": [\"sum__num\"], \"markup_type\": \"markdown\", \"filters\": [{\"col\": \"gender\", \"val\": [\"girl\"], \"op\": \"in\"}], \"granularity_sqla\": \"ds\", \"viz_type\": \"big_number_total\", \"database_name\": \"main\", \"remote_id\": 286, \"compare_suffix\": \"o10Y\", \"compare_lag\": \"10\", \"limit\": \"25\", \"until\": \"now\", \"datasource_name\": \"birth_names\", \"where\": \"\", \"groupby\": [], \"import_time\": 2018, \"schema\": \"birth_names\"}", + "changed_by_fk": 1, + "datasource_name": null, + "slice_name": "Number of Girls", + "cache_timeout": null, + "id": 286, + "changed_on": { + "__datetime__": "2018-10-01T10:36:24" + } + } + } + ], + "position_json": "{\"CHART-58575537\":{\"children\":[],\"id\":\"CHART-58575537\",\"meta\":{\"chartId\":280,\"height\":69,\"sliceName\":\"Genders\",\"width\":4},\"type\":\"CHART\"},\"CHART-985bfd1e\":{\"children\":[],\"id\":\"CHART-985bfd1e\",\"meta\":{\"chartId\":286,\"height\":66,\"sliceName\":\"Number of Girls\",\"width\":8},\"type\":\"CHART\"},\"COLUMN-25a865d6\":{\"children\":[\"ROW-cc97c6ac\"],\"id\":\"COLUMN-25a865d6\",\"meta\":{\"background\":\"BACKGROUND_TRANSPARENT\",\"width\":4},\"type\":\"COLUMN\"},\"DASHBOARD_VERSION_KEY\":\"v2\",\"GRID_ID\":{\"children\":[\"ROW-8515ace3\"],\"id\":\"GRID_ID\",\"type\":\"GRID\"},\"HEADER_ID\":{\"id\":\"HEADER_ID\",\"meta\":{\"text\":\"Births for test [copy]\"},\"type\":\"HEADER\"},\"ROOT_ID\":{\"children\":[\"GRID_ID\"],\"id\":\"ROOT_ID\",\"type\":\"ROOT\"},\"ROW-8515ace3\":{\"children\":[\"COLUMN-25a865d6\",\"CHART-985bfd1e\"],\"id\":\"ROW-8515ace3\",\"meta\":{\"background\":\"BACKGROUND_TRANSPARENT\"},\"type\":\"ROW\"},\"ROW-cc97c6ac\":{\"children\":[\"CHART-58575537\"],\"id\":\"ROW-cc97c6ac\",\"meta\":{\"background\":\"BACKGROUND_TRANSPARENT\"},\"type\":\"ROW\"}}", + "created_on": { + "__datetime__": "2018-10-01T10:34:58" + }, + "css": "", + "changed_by_fk": 1, + "created_by_fk": 1, + "id": 3, + "changed_on": { + "__datetime__": "2018-10-01T10:35:46" + } + } + } + ], + "datasources": [ + { + "__SqlaTable__": { + "is_featured": false, + "offset": 0, + "main_dttm_col": "ds", + "created_on": { + "__datetime__": "2018-08-13T13:07:16" + }, + "cache_timeout": null, + "id": 3, + "changed_on": { + "__datetime__": "2018-10-01T10:00:22" + }, + "template_params": null, + "created_by_fk": null, + "user_id": null, + "default_endpoint": null, + "perm": "[main].[birth_names](id:3)", + "params": "{\"database_name\": \"main\", \"remote_id\": 3, \"import_time\": 1538368222}", + "filter_select_enabled": true, + "changed_by_fk": 1, + "columns": [ + { + "__TableColumn__": { + "python_date_format": null, + "created_on": { + "__datetime__": "2018-08-13T13:07:16" + }, + "avg": false, + "id": 332, + "changed_on": { + "__datetime__": "2018-08-13T13:07:16" + }, + "created_by_fk": null, + "min": false, + "filterable": false, + "is_dttm": false, + "sum": false, + "database_expression": null, + "changed_by_fk": null, + "type": null, + "column_name": "num_california", + "description": null, + "max": false, + "is_active": true, + "verbose_name": null, + "table_id": 3, + "expression": "CASE WHEN state = 'CA' THEN num ELSE 0 END", + "groupby": false, + "count_distinct": false + } + }, + { + "__TableColumn__": { + "python_date_format": null, + "created_on": { + "__datetime__": "2018-08-13T13:07:16" + }, + "avg": false, + "id": 333, + "changed_on": { + "__datetime__": "2018-08-13T13:07:16" + }, + "created_by_fk": null, + "min": false, + "filterable": false, + "is_dttm": true, + "sum": false, + "database_expression": null, + "changed_by_fk": null, + "type": "DATETIME", + "column_name": "ds", + "description": null, + "max": false, + "is_active": true, + "verbose_name": null, + "table_id": 3, + "expression": "", + "groupby": false, + "count_distinct": false + } + }, + { + "__TableColumn__": { + "python_date_format": null, + "created_on": { + "__datetime__": "2018-08-13T13:07:16" + }, + "avg": false, + "id": 334, + "changed_on": { + "__datetime__": "2018-08-13T13:07:16" + }, + "created_by_fk": null, + "min": false, + "filterable": true, + "is_dttm": false, + "sum": false, + "database_expression": null, + "changed_by_fk": null, + "type": "VARCHAR(16)", + "column_name": "gender", + "description": null, + "max": false, + "is_active": true, + "verbose_name": null, + "table_id": 3, + "expression": "", + "groupby": true, + "count_distinct": false + } + }, + { + "__TableColumn__": { + "python_date_format": null, + "created_on": { + "__datetime__": "2018-08-13T13:07:16" + }, + "avg": false, + "id": 335, + "changed_on": { + "__datetime__": "2018-08-13T13:07:16" + }, + "created_by_fk": null, + "min": false, + "filterable": true, + "is_dttm": false, + "sum": false, + "database_expression": null, + "changed_by_fk": null, + "type": "VARCHAR(255)", + "column_name": "name", + "description": null, + "max": false, + "is_active": true, + "verbose_name": null, + "table_id": 3, + "expression": "", + "groupby": true, + "count_distinct": false + } + }, + { + "__TableColumn__": { + "python_date_format": null, + "created_on": { + "__datetime__": "2018-08-13T13:07:16" + }, + "avg": true, + "id": 336, + "changed_on": { + "__datetime__": "2018-08-13T13:07:16" + }, + "created_by_fk": null, + "min": false, + "filterable": false, + "is_dttm": false, + "sum": true, + "database_expression": null, + "changed_by_fk": null, + "type": "BIGINT", + "column_name": "num", + "description": null, + "max": false, + "is_active": true, + "verbose_name": null, + "table_id": 3, + "expression": "", + "groupby": false, + "count_distinct": false + } + }, + { + "__TableColumn__": { + "python_date_format": null, + "created_on": { + "__datetime__": "2018-08-13T13:07:16" + }, + "avg": false, + "id": 337, + "changed_on": { + "__datetime__": "2018-08-13T13:07:16" + }, + "created_by_fk": null, + "min": false, + "filterable": true, + "is_dttm": false, + "sum": false, + "database_expression": null, + "changed_by_fk": null, + "type": "VARCHAR(10)", + "column_name": "state", + "description": null, + "max": false, + "is_active": true, + "verbose_name": null, + "table_id": 3, + "expression": "", + "groupby": true, + "count_distinct": false + } + }, + { + "__TableColumn__": { + "python_date_format": null, + "created_on": { + "__datetime__": "2018-08-13T13:07:16" + }, + "avg": true, + "id": 338, + "changed_on": { + "__datetime__": "2018-08-13T13:07:16" + }, + "created_by_fk": null, + "min": false, + "filterable": false, + "is_dttm": false, + "sum": true, + "database_expression": null, + "changed_by_fk": null, + "type": "BIGINT", + "column_name": "sum_boys", + "description": null, + "max": false, + "is_active": true, + "verbose_name": null, + "table_id": 3, + "expression": "", + "groupby": false, + "count_distinct": false + } + }, + { + "__TableColumn__": { + "python_date_format": null, + "created_on": { + "__datetime__": "2018-08-13T13:07:16" + }, + "avg": true, + "id": 339, + "changed_on": { + "__datetime__": "2018-08-13T13:07:16" + }, + "created_by_fk": null, + "min": false, + "filterable": false, + "is_dttm": false, + "sum": true, + "database_expression": null, + "changed_by_fk": null, + "type": "BIGINT", + "column_name": "sum_girls", + "description": null, + "max": false, + "is_active": true, + "verbose_name": null, + "table_id": 3, + "expression": "", + "groupby": false, + "count_distinct": false + } + } + ], + "schema": null, + "fetch_values_predicate": null, + "description": null, + "metrics": [ + { + "__SqlMetric__": { + "description": null, + "d3format": null, + "verbose_name": null, + "created_on": { + "__datetime__": "2018-08-13T13:07:16" + }, + "warning_text": null, + "table_id": 3, + "changed_by_fk": null, + "metric_type": "avg", + "is_restricted": false, + "created_by_fk": null, + "expression": "AVG(num)", + "id": 653, + "changed_on": { + "__datetime__": "2018-08-13T13:07:16" + }, + "metric_name": "avg__num" + } + }, + { + "__SqlMetric__": { + "description": null, + "d3format": null, + "verbose_name": null, + "created_on": { + "__datetime__": "2018-08-13T13:07:16" + }, + "warning_text": null, + "table_id": 3, + "changed_by_fk": null, + "metric_type": "sum", + "is_restricted": false, + "created_by_fk": null, + "expression": "SUM(num)", + "id": 654, + "changed_on": { + "__datetime__": "2018-08-13T13:07:16" + }, + "metric_name": "sum__num" + } + }, + { + "__SqlMetric__": { + "description": null, + "d3format": null, + "verbose_name": null, + "created_on": { + "__datetime__": "2018-08-13T13:07:16" + }, + "warning_text": null, + "table_id": 3, + "changed_by_fk": null, + "metric_type": "avg", + "is_restricted": false, + "created_by_fk": null, + "expression": "AVG(sum_boys)", + "id": 655, + "changed_on": { + "__datetime__": "2018-08-13T13:07:16" + }, + "metric_name": "avg__sum_boys" + } + }, + { + "__SqlMetric__": { + "description": null, + "d3format": null, + "verbose_name": null, + "created_on": { + "__datetime__": "2018-08-13T13:07:16" + }, + "warning_text": null, + "table_id": 3, + "changed_by_fk": null, + "metric_type": "sum", + "is_restricted": false, + "created_by_fk": null, + "expression": "SUM(sum_boys)", + "id": 656, + "changed_on": { + "__datetime__": "2018-08-13T13:07:16" + }, + "metric_name": "sum__sum_boys" + } + }, + { + "__SqlMetric__": { + "description": null, + "d3format": null, + "verbose_name": null, + "created_on": { + "__datetime__": "2018-08-13T13:07:16" + }, + "warning_text": null, + "table_id": 3, + "changed_by_fk": null, + "metric_type": "avg", + "is_restricted": false, + "created_by_fk": null, + "expression": "AVG(sum_girls)", + "id": 657, + "changed_on": { + "__datetime__": "2018-08-13T13:07:16" + }, + "metric_name": "avg__sum_girls" + } + }, + { + "__SqlMetric__": { + "description": null, + "d3format": null, + "verbose_name": null, + "created_on": { + "__datetime__": "2018-08-13T13:07:16" + }, + "warning_text": null, + "table_id": 3, + "changed_by_fk": null, + "metric_type": "sum", + "is_restricted": false, + "created_by_fk": null, + "expression": "SUM(sum_girls)", + "id": 658, + "changed_on": { + "__datetime__": "2018-08-13T13:07:16" + }, + "metric_name": "sum__sum_girls" + } + }, + { + "__SqlMetric__": { + "description": null, + "d3format": null, + "verbose_name": "COUNT(*)", + "created_on": { + "__datetime__": "2018-08-13T13:07:16" + }, + "warning_text": null, + "table_id": 3, + "changed_by_fk": null, + "metric_type": "count", + "is_restricted": false, + "created_by_fk": null, + "expression": "COUNT(*)", + "id": 659, + "changed_on": { + "__datetime__": "2018-08-13T13:07:16" + }, + "metric_name": "count" + } + } + ], + "sql": null, + "database": { + "__Database__": { + "allow_run_sync": true, + "allow_ctas": false, + "allow_multi_schema_metadata_fetch": true, + "created_on": { + "__datetime__": "2018-08-13T13:07:04" + }, + "created_by_fk": null, + "id": 1, + "allow_csv_upload": true, + "cache_timeout": null, + "extra": "{\n \"metadata_params\": {},\n \"engine_params\": {}\n}\n", + "perm": "[main].(id:1)", + "changed_on": { + "__datetime__": "2018-08-13T13:07:04" + }, + "changed_by_fk": null, + "verbose_name": null, + "expose_in_sqllab": true, + "allow_dml": false, + "allow_run_async": false, + "password": null, + "select_as_create_table_as": false, + "database_name": "main", + "sqlalchemy_uri": "sqlite:////Users/arpit.agarwal/.superset/superset.db", + "impersonate_user": false, + "force_ctas_schema": null + } + }, + "database_id": 1, + "table_name": "birth_names", + "is_sqllab_view": false + } + } + ] +} \ No newline at end of file diff --git a/tests/import_export_tests.py b/tests/import_export_tests.py index 3a3d5f93ae73b..83e9d671ed543 100644 --- a/tests/import_export_tests.py +++ b/tests/import_export_tests.py @@ -10,7 +10,7 @@ from sqlalchemy.orm.session import make_transient -from superset import db, utils +from superset import dashboard_import_export_util, db, utils from superset.connectors.druid.models import ( DruidColumn, DruidDatasource, DruidMetric, ) @@ -149,6 +149,9 @@ def get_table_by_name(self, name): return db.session.query(SqlaTable).filter_by( table_name=name).first() + def get_num_dashboards(self): + return db.session.query(models.Dashboard).count() + def assert_dash_equals(self, expected_dash, actual_dash, check_position=True): self.assertEquals(expected_dash.slug, actual_dash.slug) @@ -547,6 +550,47 @@ def test_import_druid_override_identical(self): self.assert_datasource_equals( copy_datasource, self.get_datasource(imported_id)) + def test_import_util_dashboard(self): + dashboard_json = None + with open('tests/births_dash_test.json') as data_stream: + dashboard_import_export_util.import_dashboards( + db.session, data_stream, import_time=2018) + data_stream.seek(0, 0) + dashboard_json = json.loads( + data_stream.read(), + object_hook=utils.decode_dashboards) + imported_dash = self.get_dash_by_slug('test_birth_import') + expected_dash = dashboard_json['dashboards'][0] + self.assert_dash_equals(expected_dash, imported_dash, check_position=False) + + def test_export_dashboards_util(self): + dashboards_json_dump = dashboard_import_export_util.export_dashboards( + db.session) + dashboards_objects = json.loads( + dashboards_json_dump, + object_hook=utils.decode_dashboards, + ) + + exported_dashboards = dashboards_objects['dashboards'] + for dashboard in exported_dashboards: + id = dashboard.id + dash = self.get_dash(id) + self.assert_dash_equals(dash, dashboard) + self.assertEquals( + dash.id, json.loads( + dashboard.json_metadata, + object_hook=utils.decode_dashboards, + )['remote_id'], + ) + numDasboards = self.get_num_dashboards() + self.assertEquals(numDasboards, len(exported_dashboards)) + + exported_tables = dashboards_objects['datasources'] + for exported_table in exported_tables: + id = exported_table.id + table = self.get_table(id) + self.assert_table_equals(table, exported_table) + if __name__ == '__main__': unittest.main() From a1914dd36ce2696898f5544932c9c09d43438d0d Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Mon, 1 Oct 2018 16:27:40 +0530 Subject: [PATCH 4/6] test: removing import test as it is causing integrity issues Import is a wrapper around exist functionality so we can go ahead without a test or mock the actual db operation using https://docs.python.org/3/library/unittest.mock.html And validate the wrapper operations only. --- tests/import_export_tests.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/tests/import_export_tests.py b/tests/import_export_tests.py index 83e9d671ed543..5b1aee7443e1f 100644 --- a/tests/import_export_tests.py +++ b/tests/import_export_tests.py @@ -550,19 +550,6 @@ def test_import_druid_override_identical(self): self.assert_datasource_equals( copy_datasource, self.get_datasource(imported_id)) - def test_import_util_dashboard(self): - dashboard_json = None - with open('tests/births_dash_test.json') as data_stream: - dashboard_import_export_util.import_dashboards( - db.session, data_stream, import_time=2018) - data_stream.seek(0, 0) - dashboard_json = json.loads( - data_stream.read(), - object_hook=utils.decode_dashboards) - imported_dash = self.get_dash_by_slug('test_birth_import') - expected_dash = dashboard_json['dashboards'][0] - self.assert_dash_equals(expected_dash, imported_dash, check_position=False) - def test_export_dashboards_util(self): dashboards_json_dump = dashboard_import_export_util.export_dashboards( db.session) From f5922f521c2de08000fe0a5ea5d2e1fc9296fdf8 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Mon, 1 Oct 2018 20:43:08 +0530 Subject: [PATCH 5/6] test: remove test data file --- tests/births_dash_test.json | 529 ------------------------------------ 1 file changed, 529 deletions(-) delete mode 100644 tests/births_dash_test.json diff --git a/tests/births_dash_test.json b/tests/births_dash_test.json deleted file mode 100644 index e680333da3f6d..0000000000000 --- a/tests/births_dash_test.json +++ /dev/null @@ -1,529 +0,0 @@ -{ - "dashboards": [ - { - "__Dashboard__": { - "dashboard_title": "Births for test", - "slug": "test_birth_import", - "json_metadata": "{\"filter_immune_slice_fields\": {}, \"filter_immune_slices\": [], \"remote_id\": 33333, \"timed_refresh_immune_slices\": [], \"expanded_slices\": {}, \"default_filters\": \"{}\", \"import_time\": 2018}", - "description": null, - "slices": [ - { - "__Slice__": { - "created_by_fk": 1, - "datasource_id": 3, - "description": null, - "viz_type": "pie", - "perm": "[main].[birth_names](id:3)", - "datasource_type": "table", - "created_on": { - "__datetime__": "2018-10-01T10:00:22" - }, - "params": "{\"since\": \"100 years ago\", \"row_limit\": 50000, \"metric\": \"sum__num\", \"metrics\": [\"sum__num\"], \"markup_type\": \"markdown\", \"granularity_sqla\": \"ds\", \"viz_type\": \"pie\", \"database_name\": \"main\", \"remote_id\": 280, \"compare_suffix\": \"o10Y\", \"compare_lag\": \"10\", \"limit\": \"25\", \"until\": \"now\", \"datasource_name\": \"birth_names\", \"where\": \"\", \"groupby\": [\"gender\"], \"import_time\": 2018, \"schema\": \"birth_names\"}", - "changed_by_fk": 1, - "datasource_name": null, - "slice_name": "Genders", - "cache_timeout": null, - "id": 280, - "changed_on": { - "__datetime__": "2018-10-01T10:36:24" - } - } - }, - { - "__Slice__": { - "created_by_fk": 1, - "datasource_id": 3, - "description": null, - "viz_type": "big_number_total", - "perm": "[main].[birth_names](id:3)", - "datasource_type": "table", - "created_on": { - "__datetime__": "2018-10-01T10:00:22" - }, - "params": "{\"subheader\": \"total female participants\", \"since\": \"100 years ago\", \"row_limit\": 50000, \"metric\": \"sum__num\", \"metrics\": [\"sum__num\"], \"markup_type\": \"markdown\", \"filters\": [{\"col\": \"gender\", \"val\": [\"girl\"], \"op\": \"in\"}], \"granularity_sqla\": \"ds\", \"viz_type\": \"big_number_total\", \"database_name\": \"main\", \"remote_id\": 286, \"compare_suffix\": \"o10Y\", \"compare_lag\": \"10\", \"limit\": \"25\", \"until\": \"now\", \"datasource_name\": \"birth_names\", \"where\": \"\", \"groupby\": [], \"import_time\": 2018, \"schema\": \"birth_names\"}", - "changed_by_fk": 1, - "datasource_name": null, - "slice_name": "Number of Girls", - "cache_timeout": null, - "id": 286, - "changed_on": { - "__datetime__": "2018-10-01T10:36:24" - } - } - } - ], - "position_json": "{\"CHART-58575537\":{\"children\":[],\"id\":\"CHART-58575537\",\"meta\":{\"chartId\":280,\"height\":69,\"sliceName\":\"Genders\",\"width\":4},\"type\":\"CHART\"},\"CHART-985bfd1e\":{\"children\":[],\"id\":\"CHART-985bfd1e\",\"meta\":{\"chartId\":286,\"height\":66,\"sliceName\":\"Number of Girls\",\"width\":8},\"type\":\"CHART\"},\"COLUMN-25a865d6\":{\"children\":[\"ROW-cc97c6ac\"],\"id\":\"COLUMN-25a865d6\",\"meta\":{\"background\":\"BACKGROUND_TRANSPARENT\",\"width\":4},\"type\":\"COLUMN\"},\"DASHBOARD_VERSION_KEY\":\"v2\",\"GRID_ID\":{\"children\":[\"ROW-8515ace3\"],\"id\":\"GRID_ID\",\"type\":\"GRID\"},\"HEADER_ID\":{\"id\":\"HEADER_ID\",\"meta\":{\"text\":\"Births for test [copy]\"},\"type\":\"HEADER\"},\"ROOT_ID\":{\"children\":[\"GRID_ID\"],\"id\":\"ROOT_ID\",\"type\":\"ROOT\"},\"ROW-8515ace3\":{\"children\":[\"COLUMN-25a865d6\",\"CHART-985bfd1e\"],\"id\":\"ROW-8515ace3\",\"meta\":{\"background\":\"BACKGROUND_TRANSPARENT\"},\"type\":\"ROW\"},\"ROW-cc97c6ac\":{\"children\":[\"CHART-58575537\"],\"id\":\"ROW-cc97c6ac\",\"meta\":{\"background\":\"BACKGROUND_TRANSPARENT\"},\"type\":\"ROW\"}}", - "created_on": { - "__datetime__": "2018-10-01T10:34:58" - }, - "css": "", - "changed_by_fk": 1, - "created_by_fk": 1, - "id": 3, - "changed_on": { - "__datetime__": "2018-10-01T10:35:46" - } - } - } - ], - "datasources": [ - { - "__SqlaTable__": { - "is_featured": false, - "offset": 0, - "main_dttm_col": "ds", - "created_on": { - "__datetime__": "2018-08-13T13:07:16" - }, - "cache_timeout": null, - "id": 3, - "changed_on": { - "__datetime__": "2018-10-01T10:00:22" - }, - "template_params": null, - "created_by_fk": null, - "user_id": null, - "default_endpoint": null, - "perm": "[main].[birth_names](id:3)", - "params": "{\"database_name\": \"main\", \"remote_id\": 3, \"import_time\": 1538368222}", - "filter_select_enabled": true, - "changed_by_fk": 1, - "columns": [ - { - "__TableColumn__": { - "python_date_format": null, - "created_on": { - "__datetime__": "2018-08-13T13:07:16" - }, - "avg": false, - "id": 332, - "changed_on": { - "__datetime__": "2018-08-13T13:07:16" - }, - "created_by_fk": null, - "min": false, - "filterable": false, - "is_dttm": false, - "sum": false, - "database_expression": null, - "changed_by_fk": null, - "type": null, - "column_name": "num_california", - "description": null, - "max": false, - "is_active": true, - "verbose_name": null, - "table_id": 3, - "expression": "CASE WHEN state = 'CA' THEN num ELSE 0 END", - "groupby": false, - "count_distinct": false - } - }, - { - "__TableColumn__": { - "python_date_format": null, - "created_on": { - "__datetime__": "2018-08-13T13:07:16" - }, - "avg": false, - "id": 333, - "changed_on": { - "__datetime__": "2018-08-13T13:07:16" - }, - "created_by_fk": null, - "min": false, - "filterable": false, - "is_dttm": true, - "sum": false, - "database_expression": null, - "changed_by_fk": null, - "type": "DATETIME", - "column_name": "ds", - "description": null, - "max": false, - "is_active": true, - "verbose_name": null, - "table_id": 3, - "expression": "", - "groupby": false, - "count_distinct": false - } - }, - { - "__TableColumn__": { - "python_date_format": null, - "created_on": { - "__datetime__": "2018-08-13T13:07:16" - }, - "avg": false, - "id": 334, - "changed_on": { - "__datetime__": "2018-08-13T13:07:16" - }, - "created_by_fk": null, - "min": false, - "filterable": true, - "is_dttm": false, - "sum": false, - "database_expression": null, - "changed_by_fk": null, - "type": "VARCHAR(16)", - "column_name": "gender", - "description": null, - "max": false, - "is_active": true, - "verbose_name": null, - "table_id": 3, - "expression": "", - "groupby": true, - "count_distinct": false - } - }, - { - "__TableColumn__": { - "python_date_format": null, - "created_on": { - "__datetime__": "2018-08-13T13:07:16" - }, - "avg": false, - "id": 335, - "changed_on": { - "__datetime__": "2018-08-13T13:07:16" - }, - "created_by_fk": null, - "min": false, - "filterable": true, - "is_dttm": false, - "sum": false, - "database_expression": null, - "changed_by_fk": null, - "type": "VARCHAR(255)", - "column_name": "name", - "description": null, - "max": false, - "is_active": true, - "verbose_name": null, - "table_id": 3, - "expression": "", - "groupby": true, - "count_distinct": false - } - }, - { - "__TableColumn__": { - "python_date_format": null, - "created_on": { - "__datetime__": "2018-08-13T13:07:16" - }, - "avg": true, - "id": 336, - "changed_on": { - "__datetime__": "2018-08-13T13:07:16" - }, - "created_by_fk": null, - "min": false, - "filterable": false, - "is_dttm": false, - "sum": true, - "database_expression": null, - "changed_by_fk": null, - "type": "BIGINT", - "column_name": "num", - "description": null, - "max": false, - "is_active": true, - "verbose_name": null, - "table_id": 3, - "expression": "", - "groupby": false, - "count_distinct": false - } - }, - { - "__TableColumn__": { - "python_date_format": null, - "created_on": { - "__datetime__": "2018-08-13T13:07:16" - }, - "avg": false, - "id": 337, - "changed_on": { - "__datetime__": "2018-08-13T13:07:16" - }, - "created_by_fk": null, - "min": false, - "filterable": true, - "is_dttm": false, - "sum": false, - "database_expression": null, - "changed_by_fk": null, - "type": "VARCHAR(10)", - "column_name": "state", - "description": null, - "max": false, - "is_active": true, - "verbose_name": null, - "table_id": 3, - "expression": "", - "groupby": true, - "count_distinct": false - } - }, - { - "__TableColumn__": { - "python_date_format": null, - "created_on": { - "__datetime__": "2018-08-13T13:07:16" - }, - "avg": true, - "id": 338, - "changed_on": { - "__datetime__": "2018-08-13T13:07:16" - }, - "created_by_fk": null, - "min": false, - "filterable": false, - "is_dttm": false, - "sum": true, - "database_expression": null, - "changed_by_fk": null, - "type": "BIGINT", - "column_name": "sum_boys", - "description": null, - "max": false, - "is_active": true, - "verbose_name": null, - "table_id": 3, - "expression": "", - "groupby": false, - "count_distinct": false - } - }, - { - "__TableColumn__": { - "python_date_format": null, - "created_on": { - "__datetime__": "2018-08-13T13:07:16" - }, - "avg": true, - "id": 339, - "changed_on": { - "__datetime__": "2018-08-13T13:07:16" - }, - "created_by_fk": null, - "min": false, - "filterable": false, - "is_dttm": false, - "sum": true, - "database_expression": null, - "changed_by_fk": null, - "type": "BIGINT", - "column_name": "sum_girls", - "description": null, - "max": false, - "is_active": true, - "verbose_name": null, - "table_id": 3, - "expression": "", - "groupby": false, - "count_distinct": false - } - } - ], - "schema": null, - "fetch_values_predicate": null, - "description": null, - "metrics": [ - { - "__SqlMetric__": { - "description": null, - "d3format": null, - "verbose_name": null, - "created_on": { - "__datetime__": "2018-08-13T13:07:16" - }, - "warning_text": null, - "table_id": 3, - "changed_by_fk": null, - "metric_type": "avg", - "is_restricted": false, - "created_by_fk": null, - "expression": "AVG(num)", - "id": 653, - "changed_on": { - "__datetime__": "2018-08-13T13:07:16" - }, - "metric_name": "avg__num" - } - }, - { - "__SqlMetric__": { - "description": null, - "d3format": null, - "verbose_name": null, - "created_on": { - "__datetime__": "2018-08-13T13:07:16" - }, - "warning_text": null, - "table_id": 3, - "changed_by_fk": null, - "metric_type": "sum", - "is_restricted": false, - "created_by_fk": null, - "expression": "SUM(num)", - "id": 654, - "changed_on": { - "__datetime__": "2018-08-13T13:07:16" - }, - "metric_name": "sum__num" - } - }, - { - "__SqlMetric__": { - "description": null, - "d3format": null, - "verbose_name": null, - "created_on": { - "__datetime__": "2018-08-13T13:07:16" - }, - "warning_text": null, - "table_id": 3, - "changed_by_fk": null, - "metric_type": "avg", - "is_restricted": false, - "created_by_fk": null, - "expression": "AVG(sum_boys)", - "id": 655, - "changed_on": { - "__datetime__": "2018-08-13T13:07:16" - }, - "metric_name": "avg__sum_boys" - } - }, - { - "__SqlMetric__": { - "description": null, - "d3format": null, - "verbose_name": null, - "created_on": { - "__datetime__": "2018-08-13T13:07:16" - }, - "warning_text": null, - "table_id": 3, - "changed_by_fk": null, - "metric_type": "sum", - "is_restricted": false, - "created_by_fk": null, - "expression": "SUM(sum_boys)", - "id": 656, - "changed_on": { - "__datetime__": "2018-08-13T13:07:16" - }, - "metric_name": "sum__sum_boys" - } - }, - { - "__SqlMetric__": { - "description": null, - "d3format": null, - "verbose_name": null, - "created_on": { - "__datetime__": "2018-08-13T13:07:16" - }, - "warning_text": null, - "table_id": 3, - "changed_by_fk": null, - "metric_type": "avg", - "is_restricted": false, - "created_by_fk": null, - "expression": "AVG(sum_girls)", - "id": 657, - "changed_on": { - "__datetime__": "2018-08-13T13:07:16" - }, - "metric_name": "avg__sum_girls" - } - }, - { - "__SqlMetric__": { - "description": null, - "d3format": null, - "verbose_name": null, - "created_on": { - "__datetime__": "2018-08-13T13:07:16" - }, - "warning_text": null, - "table_id": 3, - "changed_by_fk": null, - "metric_type": "sum", - "is_restricted": false, - "created_by_fk": null, - "expression": "SUM(sum_girls)", - "id": 658, - "changed_on": { - "__datetime__": "2018-08-13T13:07:16" - }, - "metric_name": "sum__sum_girls" - } - }, - { - "__SqlMetric__": { - "description": null, - "d3format": null, - "verbose_name": "COUNT(*)", - "created_on": { - "__datetime__": "2018-08-13T13:07:16" - }, - "warning_text": null, - "table_id": 3, - "changed_by_fk": null, - "metric_type": "count", - "is_restricted": false, - "created_by_fk": null, - "expression": "COUNT(*)", - "id": 659, - "changed_on": { - "__datetime__": "2018-08-13T13:07:16" - }, - "metric_name": "count" - } - } - ], - "sql": null, - "database": { - "__Database__": { - "allow_run_sync": true, - "allow_ctas": false, - "allow_multi_schema_metadata_fetch": true, - "created_on": { - "__datetime__": "2018-08-13T13:07:04" - }, - "created_by_fk": null, - "id": 1, - "allow_csv_upload": true, - "cache_timeout": null, - "extra": "{\n \"metadata_params\": {},\n \"engine_params\": {}\n}\n", - "perm": "[main].(id:1)", - "changed_on": { - "__datetime__": "2018-08-13T13:07:04" - }, - "changed_by_fk": null, - "verbose_name": null, - "expose_in_sqllab": true, - "allow_dml": false, - "allow_run_async": false, - "password": null, - "select_as_create_table_as": false, - "database_name": "main", - "sqlalchemy_uri": "sqlite:////Users/arpit.agarwal/.superset/superset.db", - "impersonate_user": false, - "force_ctas_schema": null - } - }, - "database_id": 1, - "table_name": "birth_names", - "is_sqllab_view": false - } - } - ] -} \ No newline at end of file From a6545cc7818af9816403db068a930f4c32240c4e Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Mon, 1 Oct 2018 22:13:02 +0530 Subject: [PATCH 6/6] test: removed usage of reserved keyword id --- tests/import_export_tests.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/import_export_tests.py b/tests/import_export_tests.py index 5b1aee7443e1f..932f9e0f9d3d0 100644 --- a/tests/import_export_tests.py +++ b/tests/import_export_tests.py @@ -560,8 +560,8 @@ def test_export_dashboards_util(self): exported_dashboards = dashboards_objects['dashboards'] for dashboard in exported_dashboards: - id = dashboard.id - dash = self.get_dash(id) + id_ = dashboard.id + dash = self.get_dash(id_) self.assert_dash_equals(dash, dashboard) self.assertEquals( dash.id, json.loads( @@ -574,8 +574,8 @@ def test_export_dashboards_util(self): exported_tables = dashboards_objects['datasources'] for exported_table in exported_tables: - id = exported_table.id - table = self.get_table(id) + id_ = exported_table.id + table = self.get_table(id_) self.assert_table_equals(table, exported_table)