[migration] metadata for dashboard filters

superset/migrations/versions/3325d4caccc8_dashboard_scoped_filters.py

@@ -0,0 +1,114 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""empty message
+
+Revision ID: 3325d4caccc8
+Revises: e96dbf2cfef0
+Create Date: 2020-02-07 14:13:51.714678
+
+"""
+
+# revision identifiers, used by Alembic.
+import json
+import logging
+
+from alembic import op
+from sqlalchemy import and_, Column, ForeignKey, Integer, String, Table, Text
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import relationship
+
+from superset import db
+from superset.utils.dashboard_filter_scopes_converter import convert_filter_scopes
+
+revision = "3325d4caccc8"
+down_revision = "e96dbf2cfef0"
+
+Base = declarative_base()
+
+
+class Slice(Base):
+    """Declarative class to do query in upgrade"""
+
+    __tablename__ = "slices"
+    id = Column(Integer, primary_key=True)
+    slice_name = Column(String(250))
+    params = Column(Text)
+    viz_type = Column(String(250))
+
+
+dashboard_slices = Table(
+    "dashboard_slices",
+    Base.metadata,
+    Column("id", Integer, primary_key=True),
+    Column("dashboard_id", Integer, ForeignKey("dashboards.id")),
+    Column("slice_id", Integer, ForeignKey("slices.id")),
+)
+
+
+class Dashboard(Base):
+    __tablename__ = "dashboards"
+    id = Column(Integer, primary_key=True)
+    json_metadata = Column(Text)
+    slices = relationship("Slice", secondary=dashboard_slices, backref="dashboards")
+
+
+def upgrade():
+    bind = op.get_bind()
+    session = db.Session(bind=bind)
+
+    dashboards = session.query(Dashboard).all()
+    for i, dashboard in enumerate(dashboards):
+        print("scanning dashboard ({}/{}) >>>>".format(i + 1, len(dashboards)))
+        try:
+            json_metadata = json.loads(dashboard.json_metadata or "{}")
+            if "filter_scopes" in json_metadata:
+                continue
+
+            filter_scopes = {}
+            slice_ids = [slice.id for slice in dashboard.slices]
+            filters = (
+                session.query(Slice)
+                .filter(and_(Slice.id.in_(slice_ids), Slice.viz_type == "filter_box"))
+                .all()
+            )
+
+            # if dashboard has filter_box
+            if len(filters):
+                filter_scopes = convert_filter_scopes(json_metadata, filters)
+
+            json_metadata.pop("filter_immune_slices", None)
+            json_metadata.pop("filter_immune_slice_fields", None)
+            if filter_scopes:
+                json_metadata["filter_scopes"] = filter_scopes
+                logging.info(
+                    f"Adding filter_scopes for dashboard {dashboard.id}: {json.dumps(filter_scopes)}"
+                )
+            if json_metadata:
+                dashboard.json_metadata = json.dumps(
+                    json_metadata, indent=None, separators=(",", ":"), sort_keys=True
+                )
+
+            session.merge(dashboard)
+        except Exception as e:
+            logging.exception(f"dashboard {dashboard.id} has error: {e}")
+
+    session.commit()
+    session.close()
+
+
+def downgrade():
+    pass

superset/utils/dashboard_filter_scopes_converter.py

@@ -0,0 +1,73 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import json
+import logging
+from typing import Dict, List
+
+from superset.models.slice import Slice
+
+logger = logging.getLogger(__name__)
+
+
+def convert_filter_scopes(json_metadata: Dict, filters: List[Slice]):
+    filter_scopes = {}
+    immuned_by_id: List[int] = json_metadata.get("filter_immune_slices") or []
+    immuned_by_column: Dict = {}
+    for slice_id, columns in json_metadata.get(
+        "filter_immune_slice_fields", {}
+    ).items():
+        for column in columns:
+            if immuned_by_column.get(column, None) is None:
+                immuned_by_column[column] = []
+            immuned_by_column[column].append(int(slice_id))
+
+    def add_filter_scope(filter_field, filter_id):
+        # in case filter field is invalid
+        if isinstance(filter_field, str):
+            current_filter_immune = list(
+                set(immuned_by_id + immuned_by_column.get(filter_field, []))
+            )
+            filter_fields[filter_field] = {
+                "scope": ["ROOT_ID"],
+                "immune": current_filter_immune,
+            }
+        else:
+            logging.info(f"slice [{filter_id}] has invalid field: {filter_field}")
+
+    for filter_slice in filters:
+        filter_fields: Dict = {}
+        filter_id = filter_slice.id
+        slice_params = json.loads(filter_slice.params or "{}")
+        configs = slice_params.get("filter_configs") or []
+
+        if slice_params.get("date_filter", False):
+            add_filter_scope("__time_range", filter_id)
+        if slice_params.get("show_sqla_time_column", False):
+            add_filter_scope("__time_col", filter_id)
+        if slice_params.get("show_sqla_time_granularity", False):
+            add_filter_scope("__time_grain", filter_id)
+        if slice_params.get("show_druid_time_granularity", False):
+            add_filter_scope("__granularity", filter_id)
+        if slice_params.get("show_druid_time_origin", False):
+            add_filter_scope("druid_time_origin", filter_id)
+        for config in configs:
+            add_filter_scope(config.get("column"), filter_id)
+
+        if filter_fields:
+            filter_scopes[filter_id] = filter_fields
+
+    return filter_scopes