From 9d629e8364cb82ea5b0f2eb647016784f1fd08a8 Mon Sep 17 00:00:00 2001 From: Dave Smith Date: Wed, 6 Nov 2019 15:35:33 -0800 Subject: [PATCH 1/3] LYFT HACK: prepare and pass req json directly to sql_json_call --- superset/views/core.py | 31 ++++++++++++++++--------------- superset/views/lyft.py | 19 ++++++++++++++++++- 2 files changed, 34 insertions(+), 16 deletions(-) diff --git a/superset/views/core.py b/superset/views/core.py index 19c30c2ffd849..6d219f15c5e77 100755 --- a/superset/views/core.py +++ b/superset/views/core.py @@ -2681,36 +2681,37 @@ def _sql_json_sync( @expose("/sql_json/", methods=["POST"]) @event_logger.log_this def sql_json(self): - return self.sql_json_call(request) + request_json = request.json + return self.sql_json_call(request_json) - def sql_json_call(self, request): + def sql_json_call(self, request_json): """Runs arbitrary sql and returns data as json""" # Collect Values - database_id: int = request.json.get("database_id") - schema: str = request.json.get("schema") - sql: str = request.json.get("sql") + database_id: int = request_json.get("database_id") + schema: str = request_json.get("schema") + sql: str = request_json.get("sql") try: template_params: dict = json.loads( - request.json.get("templateParams") or "{}" + request_json.get("templateParams") or "{}" ) except json.decoder.JSONDecodeError: logging.warning( - f"Invalid template parameter {request.json.get('templateParams')}" + f"Invalid template parameter {request_json.get('templateParams')}" " specified. Defaulting to empty dict" ) template_params = {} - limit = request.json.get("queryLimit") or app.config["SQL_MAX_ROW"] - async_flag: bool = request.json.get("runAsync") + limit = request_json.get("queryLimit") or app.config["SQL_MAX_ROW"] + async_flag: bool = request_json.get("runAsync") if limit < 0: logging.warning( f"Invalid limit of {limit} specified. Defaulting to max limit." ) limit = 0 - select_as_cta: bool = request.json.get("select_as_cta") - tmp_table_name: str = request.json.get("tmp_table_name") - client_id: str = request.json.get("client_id") or utils.shortid()[:10] - sql_editor_id: str = request.json.get("sql_editor_id") - tab_name: str = request.json.get("tab") + select_as_cta: bool = request_json.get("select_as_cta") + tmp_table_name: str = request_json.get("tmp_table_name") + client_id: str = request_json.get("client_id") or utils.shortid()[:10] + sql_editor_id: str = request_json.get("sql_editor_id") + tab_name: str = request_json.get("tab") status: bool = QueryStatus.PENDING if async_flag else QueryStatus.RUNNING session = db.session() @@ -2781,7 +2782,7 @@ def sql_json_call(self, request): # (feature that will expand Presto row objects and arrays) expand_data: bool = is_feature_enabled( "PRESTO_EXPAND_DATA" - ) and request.json.get("expand_data") + ) and request_json.get("expand_data") # Async request. if async_flag: diff --git a/superset/views/lyft.py b/superset/views/lyft.py index f811c05e1e532..a18ebccf6e8a5 100644 --- a/superset/views/lyft.py +++ b/superset/views/lyft.py @@ -63,7 +63,24 @@ def sql_json(self): return json_error_response("{}".format(e), status=412) except SupersetException as e: return json_error_response("{}".format(e)) - return self.sql_json_call(request) + + request_json = request.get_json() + if not request_json: + request_json = { + "database_id": int(request.form.get("database_id")), + "schema": request.form.get("schema"), + "sql": request.form.get("sql"), + "templateParams": request.form.get("templateParams", "{}"), + "queryLimit": int(request.form.get("queryLimit", app.config.get("SQL_MAX_ROW"))), + "runAsync": request.form.get("runAsync") == "true", + "select_as_cta": request.form.get("select_as_cta") == "true", + "tmp_table_name": request.form.get("tmp_table_name"), + "client_id": request.form.get("client_id") or None, + "sql_editor_id": request.form.get("sql_editor_id"), + "tab": request.form.get("tab") + } + + return self.sql_json_call(request_json) @event_logger.log_this @expose("/queries/") From cbbfb862dc6682904190903c97bf61a65c290ce4 Mon Sep 17 00:00:00 2001 From: Dave Smith Date: Wed, 6 Nov 2019 15:44:05 -0800 Subject: [PATCH 2/3] update lyft_changes.txt --- lyft_changes.txt | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lyft_changes.txt b/lyft_changes.txt index 3eaf88d9520ea..e2ec78ae0ac50 100644 --- a/lyft_changes.txt +++ b/lyft_changes.txt @@ -35,4 +35,11 @@ file to help us track how close or far we are from apache/incubator-superset mas - superset/assets/src/visualizations/Kepler/Kepler.css has been removed - superset/assets/src/visualizations/Kepler/Kepler.jsx has been removed - superset/assets/src/visualizations/Kepler/KeplerChartPlugin.js has been removed -- superset/assets/src/visualizations/presets/DeckGLChartPreset.js has been removed \ No newline at end of file +- superset/assets/src/visualizations/presets/DeckGLChartPreset.js has been removed + +2.superset/views/lyft.py and core.py +We are "intercepting" sql_json and other calls to authenticate then passing down into actual superset functions. To do this we have added endpoints in lyft.py, but also have broken the apache endpoints (like sql_json) into the request handler itself (ie sql_json) and a shared library function (sql_json_call) that is used by both the apache and lyft version of the endoint. + +3. superset/views/core.py again +in sql_json_call, we have changed the arg to take a json object named request_json (instead of original request) and string replaced all of the usages of request.json to use request_json. This lets us create a dictionary of request param to pass into the sql_json_call method. We need this because our callers such as amundsen are not json encoding data in calls, and superset had a breaking API change that expects them to be encoded. + From 1b360e4fceea5e950ae7a98f5615cefe82dca57f Mon Sep 17 00:00:00 2001 From: khtruong Date: Thu, 7 Nov 2019 09:10:34 -0800 Subject: [PATCH 3/3] fix: lint issues --- superset/views/lyft.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/superset/views/lyft.py b/superset/views/lyft.py index a18ebccf6e8a5..1d88a408fd942 100644 --- a/superset/views/lyft.py +++ b/superset/views/lyft.py @@ -64,20 +64,22 @@ def sql_json(self): except SupersetException as e: return json_error_response("{}".format(e)) - request_json = request.get_json() + request_json = request.get_json() if not request_json: request_json = { "database_id": int(request.form.get("database_id")), "schema": request.form.get("schema"), "sql": request.form.get("sql"), "templateParams": request.form.get("templateParams", "{}"), - "queryLimit": int(request.form.get("queryLimit", app.config.get("SQL_MAX_ROW"))), + "queryLimit": int( + request.form.get("queryLimit", app.config.get("SQL_MAX_ROW")) + ), "runAsync": request.form.get("runAsync") == "true", "select_as_cta": request.form.get("select_as_cta") == "true", "tmp_table_name": request.form.get("tmp_table_name"), "client_id": request.form.get("client_id") or None, "sql_editor_id": request.form.get("sql_editor_id"), - "tab": request.form.get("tab") + "tab": request.form.get("tab"), } return self.sql_json_call(request_json)