From 159164ab4df960bfe75256f3a5d4613420af1f42 Mon Sep 17 00:00:00 2001 From: Grace Date: Mon, 10 Feb 2020 17:35:34 -0800 Subject: [PATCH] [migration] metadata for dashboard filters --- .../3325d4caccc8_dashboard_scoped_filters.py | 114 ++++++++++++++++++ .../dashboard_filter_scopes_converter.py | 73 +++++++++++ 2 files changed, 187 insertions(+) create mode 100644 superset/migrations/versions/3325d4caccc8_dashboard_scoped_filters.py create mode 100644 superset/utils/dashboard_filter_scopes_converter.py diff --git a/superset/migrations/versions/3325d4caccc8_dashboard_scoped_filters.py b/superset/migrations/versions/3325d4caccc8_dashboard_scoped_filters.py new file mode 100644 index 0000000000000..bbec805d67f43 --- /dev/null +++ b/superset/migrations/versions/3325d4caccc8_dashboard_scoped_filters.py @@ -0,0 +1,114 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""empty message + +Revision ID: 3325d4caccc8 +Revises: e96dbf2cfef0 +Create Date: 2020-02-07 14:13:51.714678 + +""" + +# revision identifiers, used by Alembic. +import json +import logging + +from alembic import op +from sqlalchemy import and_, Column, ForeignKey, Integer, String, Table, Text +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import relationship + +from superset import db +from superset.utils.dashboard_filter_scopes_converter import convert_filter_scopes + +revision = "3325d4caccc8" +down_revision = "e96dbf2cfef0" + +Base = declarative_base() + + +class Slice(Base): + """Declarative class to do query in upgrade""" + + __tablename__ = "slices" + id = Column(Integer, primary_key=True) + slice_name = Column(String(250)) + params = Column(Text) + viz_type = Column(String(250)) + + +dashboard_slices = Table( + "dashboard_slices", + Base.metadata, + Column("id", Integer, primary_key=True), + Column("dashboard_id", Integer, ForeignKey("dashboards.id")), + Column("slice_id", Integer, ForeignKey("slices.id")), +) + + +class Dashboard(Base): + __tablename__ = "dashboards" + id = Column(Integer, primary_key=True) + json_metadata = Column(Text) + slices = relationship("Slice", secondary=dashboard_slices, backref="dashboards") + + +def upgrade(): + bind = op.get_bind() + session = db.Session(bind=bind) + + dashboards = session.query(Dashboard).all() + for i, dashboard in enumerate(dashboards): + print("scanning dashboard ({}/{}) >>>>".format(i + 1, len(dashboards))) + try: + json_metadata = json.loads(dashboard.json_metadata or "{}") + if "filter_scopes" in json_metadata: + continue + + filter_scopes = {} + slice_ids = [slice.id for slice in dashboard.slices] + filters = ( + session.query(Slice) + .filter(and_(Slice.id.in_(slice_ids), Slice.viz_type == "filter_box")) + .all() + ) + + # if dashboard has filter_box + if len(filters): + filter_scopes = convert_filter_scopes(json_metadata, filters) + + json_metadata.pop("filter_immune_slices", None) + json_metadata.pop("filter_immune_slice_fields", None) + if filter_scopes: + json_metadata["filter_scopes"] = filter_scopes + logging.info( + f"Adding filter_scopes for dashboard {dashboard.id}: {json.dumps(filter_scopes)}" + ) + if json_metadata: + dashboard.json_metadata = json.dumps( + json_metadata, indent=None, separators=(",", ":"), sort_keys=True + ) + + session.merge(dashboard) + except Exception as e: + logging.exception(f"dashboard {dashboard.id} has error: {e}") + + session.commit() + session.close() + + +def downgrade(): + pass diff --git a/superset/utils/dashboard_filter_scopes_converter.py b/superset/utils/dashboard_filter_scopes_converter.py new file mode 100644 index 0000000000000..96c080ed7c26c --- /dev/null +++ b/superset/utils/dashboard_filter_scopes_converter.py @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import json +import logging +from typing import Dict, List + +from superset.models.slice import Slice + +logger = logging.getLogger(__name__) + + +def convert_filter_scopes(json_metadata: Dict, filters: List[Slice]): + filter_scopes = {} + immuned_by_id: List[int] = json_metadata.get("filter_immune_slices") or [] + immuned_by_column: Dict = {} + for slice_id, columns in json_metadata.get( + "filter_immune_slice_fields", {} + ).items(): + for column in columns: + if immuned_by_column.get(column, None) is None: + immuned_by_column[column] = [] + immuned_by_column[column].append(int(slice_id)) + + def add_filter_scope(filter_field, filter_id): + # in case filter field is invalid + if isinstance(filter_field, str): + current_filter_immune = list( + set(immuned_by_id + immuned_by_column.get(filter_field, [])) + ) + filter_fields[filter_field] = { + "scope": ["ROOT_ID"], + "immune": current_filter_immune, + } + else: + logging.info(f"slice [{filter_id}] has invalid field: {filter_field}") + + for filter_slice in filters: + filter_fields: Dict = {} + filter_id = filter_slice.id + slice_params = json.loads(filter_slice.params or "{}") + configs = slice_params.get("filter_configs") or [] + + if slice_params.get("date_filter", False): + add_filter_scope("__time_range", filter_id) + if slice_params.get("show_sqla_time_column", False): + add_filter_scope("__time_col", filter_id) + if slice_params.get("show_sqla_time_granularity", False): + add_filter_scope("__time_grain", filter_id) + if slice_params.get("show_druid_time_granularity", False): + add_filter_scope("__granularity", filter_id) + if slice_params.get("show_druid_time_origin", False): + add_filter_scope("druid_time_origin", filter_id) + for config in configs: + add_filter_scope(config.get("column"), filter_id) + + if filter_fields: + filter_scopes[filter_id] = filter_fields + + return filter_scopes