feat: import/export dashboards via cli (apache#6061)

* feat: import/export dashboards via cli

* style: fixed lint error

* test: added test for import and export util

* test: removing import test as it is causing integrity issues

Import is a wrapper around exist functionality so we can go ahead without a test or mock the actual db operation using https://docs.python.org/3/library/unittest.mock.html

And validate the wrapper operations only.

* test: remove test data file

* test: removed usage of reserved keyword id

* Fix: remove test that is causing unintended impact

superset/cli.py

@@ -18,7 +18,8 @@
 import yaml
 
 from superset import (
-    app, data, db, dict_import_export_util, security_manager, utils,
+    app, dashboard_import_export_util, data, db,
+    dict_import_export_util, security_manager, utils,
 )
 
 config = app.config
@@ -224,6 +225,53 @@ def refresh_druid(datasource, merge):
     session.commit()
 
 
+@app.cli.command()
+@click.option(
+    '--path', '-p',
+    help='Path to a single JSON file or path containing multiple JSON files'
+         'files to import (*.json)')
+@click.option(
+    '--recursive', '-r',
+    help='recursively search the path for json files')
+def import_dashboards(path, recursive=False):
+    """Import dashboards from JSON"""
+    p = Path(path)
+    files = []
+    if p.is_file():
+        files.append(p)
+    elif p.exists() and not recursive:
+        files.extend(p.glob('*.json'))
+    elif p.exists() and recursive:
+        files.extend(p.rglob('*.json'))
+    for f in files:
+        logging.info('Importing dashboard from file %s', f)
+        try:
+            with f.open() as data_stream:
+                dashboard_import_export_util.import_dashboards(
+                    db.session, data_stream)
+        except Exception as e:
+            logging.error('Error when importing dashboard from file %s', f)
+            logging.error(e)
+
+
+@app.cli.command()
+@click.option(
+    '--dashboard-file', '-f', default=None,
+    help='Specify the the file to export to')
+@click.option(
+    '--print_stdout', '-p',
+    help='Print JSON to stdout')
+def export_dashboards(print_stdout, dashboard_file):
+    """Export dashboards to JSON"""
+    data = dashboard_import_export_util.export_dashboards(db.session)
+    if print_stdout or not dashboard_file:
+        print(data)
+    if dashboard_file:
+        logging.info('Exporting dashboards to %s', dashboard_file)
+        with open(dashboard_file, 'w') as data_stream:
+            data_stream.write(data)
+
+
 @app.cli.command()
 @click.option(
     '--path', '-p',
@@ -268,7 +316,7 @@ def import_datasources(path, sync, recursive=False):
     '--datasource-file', '-f', default=None,
     help='Specify the the file to export to')
 @click.option(
-    '--print', '-p',
+    '--print_stdout', '-p',
     help='Print YAML to stdout')
 @click.option(
     '--back-references', '-b',

superset/dashboard_import_export_util.py

@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+# pylint: disable=C,R,W
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import json
+import logging
+import time
+
+from superset import utils
+from superset.models.core import Dashboard
+
+
+def import_dashboards(session, data_stream, import_time=None):
+    """Imports dashboards from a stream to databases"""
+    current_tt = int(time.time())
+    import_time = current_tt if import_time is None else import_time
+    data = json.loads(data_stream.read(), object_hook=utils.decode_dashboards)
+    # TODO: import DRUID datasources
+    for table in data['datasources']:
+        type(table).import_obj(table, import_time=import_time)
+    session.commit()
+    for dashboard in data['dashboards']:
+        Dashboard.import_obj(
+            dashboard, import_time=import_time)
+    session.commit()
+
+
+def export_dashboards(session):
+    """Returns all dashboards metadata as a json dump"""
+    logging.info('Starting export')
+    dashboards = session.query(Dashboard)
+    dashboard_ids = []
+    for dashboard in dashboards:
+        dashboard_ids.append(dashboard.id)
+    data = Dashboard.export_dashboards(dashboard_ids)
+    return data

superset/views/core.py

@@ -35,9 +35,8 @@
 from werkzeug.utils import secure_filename
 
 from superset import (
-    app, appbuilder, cache, db, results_backend, security_manager, sql_lab, utils,
-    viz,
-)
+    app, appbuilder, cache, dashboard_import_export_util, db, results_backend,
+    security_manager, sql_lab, utils, viz)
 from superset.connectors.connector_registry import ConnectorRegistry
 from superset.connectors.sqla.models import AnnotationDatasource, SqlaTable
 from superset.exceptions import SupersetException
@@ -1261,16 +1260,7 @@ def import_dashboards(self):
         """Overrides the dashboards using json instances from the file."""
         f = request.files.get('file')
         if request.method == 'POST' and f:
-            current_tt = int(time.time())
-            data = json.loads(f.stream.read(), object_hook=utils.decode_dashboards)
-            # TODO: import DRUID datasources
-            for table in data['datasources']:
-                type(table).import_obj(table, import_time=current_tt)
-            db.session.commit()
-            for dashboard in data['dashboards']:
-                models.Dashboard.import_obj(
-                    dashboard, import_time=current_tt)
-            db.session.commit()
+            dashboard_import_export_util.import_dashboards(db.session, f.stream)
             return redirect('/dashboard/list/')
         return self.render_template('superset/import_dashboards.html')