This is an automated email from the ASF dual-hosted git repository.
graceguo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-superset.git
The following commit(s) were added to refs/heads/master by this push:
new adebd40 [cache warm_up] warm_up slice with dashboard default_filters
(#9311)
adebd40 is described below
commit adebd40d300fb86a53220ee99b453fb853fb2ef5
Author: Grace Guo <grace....@airbnb.com>
AuthorDate: Wed Mar 18 08:21:10 2020 -0700
[cache warm_up] warm_up slice with dashboard default_filters (#9311)
* [cache warm_up] warm_up slice with dashboard default_filters
* update Celery warmup tasks
* fix code review comments
* add try catch and type checking for parsed dash metadata
* extra code review fix
---
superset/tasks/cache.py | 31 +++---
superset/views/core.py | 6 ++
superset/views/utils.py | 88 ++++++++++++++++
tests/strategy_tests.py | 22 ++++
tests/utils_tests.py | 265 ++++++++++++++++++++++++++++++++++++++++++++++++
5 files changed, 396 insertions(+), 16 deletions(-)
diff --git a/superset/tasks/cache.py b/superset/tasks/cache.py
index ae95f20..67c366b 100644
--- a/superset/tasks/cache.py
+++ b/superset/tasks/cache.py
@@ -18,6 +18,7 @@
import json
import logging
+from typing import Any, Dict, Optional
from urllib import request
from urllib.error import URLError
@@ -31,6 +32,7 @@ from superset.models.dashboard import Dashboard
from superset.models.slice import Slice
from superset.models.tags import Tag, TaggedObject
from superset.utils.core import parse_human_datetime
+from superset.views.utils import build_extra_filters
logger = get_task_logger(__name__)
logger.setLevel(logging.INFO)
@@ -54,27 +56,23 @@ def get_form_data(chart_id, dashboard=None):
if not default_filters:
return form_data
- # do not apply filters if chart is immune to them
- immune_fields = []
filter_scopes = json_metadata.get("filter_scopes", {})
- if filter_scopes:
- for scopes in filter_scopes.values():
- for (field, scope) in scopes.items():
- if chart_id in scope.get("immune", []):
- immune_fields.append(field)
-
- extra_filters = []
- for filters in default_filters.values():
- for col, val in filters.items():
- if col not in immune_fields:
- extra_filters.append({"col": col, "op": "in", "val": val})
+ layout = json.loads(dashboard.position_json or "{}")
+ if (
+ isinstance(layout, dict)
+ and isinstance(filter_scopes, dict)
+ and isinstance(default_filters, dict)
+ ):
+ extra_filters = build_extra_filters(
+ layout, filter_scopes, default_filters, chart_id
+ )
if extra_filters:
form_data["extra_filters"] = extra_filters
return form_data
-def get_url(chart):
+def get_url(chart, extra_filters: Optional[Dict[str, Any]] = None):
"""Return external URL for warming up a given chart/table cache."""
with app.test_request_context():
baseurl = (
@@ -82,7 +80,7 @@ def get_url(chart):
"{SUPERSET_WEBSERVER_ADDRESS}:"
"{SUPERSET_WEBSERVER_PORT}".format(**app.config)
)
- return f"{baseurl}{chart.url}"
+ return f"{baseurl}{chart.get_explore_url(overrides=extra_filters)}"
class Strategy:
@@ -181,7 +179,8 @@ class TopNDashboardsStrategy(Strategy):
dashboards =
session.query(Dashboard).filter(Dashboard.id.in_(dash_ids)).all()
for dashboard in dashboards:
for chart in dashboard.slices:
- urls.append(get_url(chart))
+ form_data_with_filters = get_form_data(chart.id, dashboard)
+ urls.append(get_url(chart, form_data_with_filters))
return urls
diff --git a/superset/views/core.py b/superset/views/core.py
index ff6dcbf..447461e 100755
--- a/superset/views/core.py
+++ b/superset/views/core.py
@@ -90,6 +90,7 @@ from superset.utils.dashboard_filter_scopes_converter import
copy_filter_scopes
from superset.utils.dates import now_as_float
from superset.utils.decorators import etag_cache, stats_timing
from superset.views.database.filters import DatabaseFilter
+from superset.views.utils import get_dashboard_extra_filters
from .base import (
api,
@@ -1651,6 +1652,7 @@ class Superset(BaseSupersetView):
slices = None
session = db.session()
slice_id = request.args.get("slice_id")
+ dashboard_id = request.args.get("dashboard_id")
table_name = request.args.get("table_name")
db_name = request.args.get("db_name")
@@ -1696,6 +1698,10 @@ class Superset(BaseSupersetView):
for slc in slices:
try:
form_data = get_form_data(slc.id, use_slice_data=True)[0]
+ if dashboard_id:
+ form_data["extra_filters"] = get_dashboard_extra_filters(
+ slc.id, dashboard_id
+ )
obj = get_viz(
datasource_type=slc.datasource.type,
datasource_id=slc.datasource.id,
diff --git a/superset/views/utils.py b/superset/views/utils.py
index 9bcfc6e..40de71e 100644
--- a/superset/views/utils.py
+++ b/superset/views/utils.py
@@ -27,6 +27,7 @@ from superset import app, db, viz
from superset.connectors.connector_registry import ConnectorRegistry
from superset.exceptions import SupersetException
from superset.legacy import update_time_range
+from superset.models.dashboard import Dashboard
from superset.models.slice import Slice
from superset.utils.core import QueryStatus, TimeRangeEndpoint
@@ -262,3 +263,90 @@ def get_time_range_endpoints(
return (TimeRangeEndpoint(start), TimeRangeEndpoint(end))
return (TimeRangeEndpoint.INCLUSIVE, TimeRangeEndpoint.EXCLUSIVE)
+
+
+# see all dashboard components type in
+# /superset-frontend/src/dashboard/util/componentTypes.js
+CONTAINER_TYPES = ["COLUMN", "GRID", "TABS", "TAB", "ROW"]
+
+
+def get_dashboard_extra_filters(
+ slice_id: int, dashboard_id: int
+) -> List[Dict[str, Any]]:
+ session = db.session()
+ dashboard =
session.query(Dashboard).filter_by(id=dashboard_id).one_or_none()
+
+ # is chart in this dashboard?
+ if (
+ dashboard is None
+ or not dashboard.json_metadata
+ or not dashboard.slices
+ or not any([slc for slc in dashboard.slices if slc.id == slice_id])
+ ):
+ return []
+
+ try:
+ # does this dashboard have default filters?
+ json_metadata = json.loads(dashboard.json_metadata)
+ default_filters = json.loads(json_metadata.get("default_filters",
"null"))
+ if not default_filters:
+ return []
+
+ # are default filters applicable to the given slice?
+ filter_scopes = json_metadata.get("filter_scopes", {})
+ layout = json.loads(dashboard.position_json or "{}")
+
+ if (
+ isinstance(layout, dict)
+ and isinstance(filter_scopes, dict)
+ and isinstance(default_filters, dict)
+ ):
+ return build_extra_filters(layout, filter_scopes, default_filters,
slice_id)
+ except json.JSONDecodeError:
+ pass
+
+ return []
+
+
+def build_extra_filters(
+ layout: Dict,
+ filter_scopes: Dict,
+ default_filters: Dict[str, Dict[str, List]],
+ slice_id: int,
+) -> List[Dict[str, Any]]:
+ extra_filters = []
+
+ # do not apply filters if chart is not in filter's scope or
+ # chart is immune to the filter
+ for filter_id, columns in default_filters.items():
+ scopes_by_filter_field = filter_scopes.get(filter_id, {})
+ for col, val in columns.items():
+ current_field_scopes = scopes_by_filter_field.get(col, {})
+ scoped_container_ids = current_field_scopes.get("scope",
["ROOT_ID"])
+ immune_slice_ids = current_field_scopes.get("immune", [])
+
+ for container_id in scoped_container_ids:
+ if slice_id not in immune_slice_ids and is_slice_in_container(
+ layout, container_id, slice_id
+ ):
+ extra_filters.append({"col": col, "op": "in", "val": val})
+
+ return extra_filters
+
+
+def is_slice_in_container(layout: Dict, container_id: str, slice_id: int) ->
bool:
+ if container_id == "ROOT_ID":
+ return True
+
+ node = layout[container_id]
+ node_type = node.get("type")
+ if node_type == "CHART" and node.get("meta", {}).get("chartId") ==
slice_id:
+ return True
+
+ if node_type in CONTAINER_TYPES:
+ children = node.get("children", [])
+ return any(
+ is_slice_in_container(layout, child_id, slice_id) for child_id in
children
+ )
+
+ return False
diff --git a/tests/strategy_tests.py b/tests/strategy_tests.py
index 646931f..08d25b2 100644
--- a/tests/strategy_tests.py
+++ b/tests/strategy_tests.py
@@ -33,6 +33,22 @@ from .base_tests import SupersetTestCase
URL_PREFIX = "http://0.0.0.0:8081";
+mock_positions = {
+ "DASHBOARD_VERSION_KEY": "v2",
+ "DASHBOARD_CHART_TYPE-1": {
+ "type": "CHART",
+ "id": "DASHBOARD_CHART_TYPE-1",
+ "children": [],
+ "meta": {"width": 4, "height": 50, "chartId": 1},
+ },
+ "DASHBOARD_CHART_TYPE-2": {
+ "type": "CHART",
+ "id": "DASHBOARD_CHART_TYPE-2",
+ "children": [],
+ "meta": {"width": 4, "height": 50, "chartId": 2},
+ },
+}
+
class CacheWarmUpTests(SupersetTestCase):
def __init__(self, *args, **kwargs):
@@ -48,6 +64,7 @@ class CacheWarmUpTests(SupersetTestCase):
chart_id = 1
dashboard = MagicMock()
dashboard.json_metadata = None
+ dashboard.position_json = json.dumps(mock_positions)
result = get_form_data(chart_id, dashboard)
expected = {"slice_id": chart_id}
self.assertEqual(result, expected)
@@ -56,6 +73,7 @@ class CacheWarmUpTests(SupersetTestCase):
chart_id = 1
filter_box_id = 2
dashboard = MagicMock()
+ dashboard.position_json = json.dumps(mock_positions)
dashboard.json_metadata = json.dumps(
{
"filter_scopes": {
@@ -76,6 +94,7 @@ class CacheWarmUpTests(SupersetTestCase):
chart_id = 1
dashboard = MagicMock()
dashboard.json_metadata = json.dumps({})
+ dashboard.position_json = json.dumps(mock_positions)
result = get_form_data(chart_id, dashboard)
expected = {"slice_id": chart_id}
self.assertEqual(result, expected)
@@ -84,6 +103,7 @@ class CacheWarmUpTests(SupersetTestCase):
chart_id = 1
filter_box_id = 2
dashboard = MagicMock()
+ dashboard.position_json = json.dumps(mock_positions)
dashboard.json_metadata = json.dumps(
{
"default_filters": json.dumps(
@@ -112,6 +132,7 @@ class CacheWarmUpTests(SupersetTestCase):
chart_id = 1
filter_box_id = 2
dashboard = MagicMock()
+ dashboard.position_json = json.dumps(mock_positions)
dashboard.json_metadata = json.dumps(
{
"default_filters": json.dumps(
@@ -132,6 +153,7 @@ class CacheWarmUpTests(SupersetTestCase):
chart_id = 1
filter_box_id = 2
dashboard = MagicMock()
+ dashboard.position_json = json.dumps(mock_positions)
dashboard.json_metadata = json.dumps(
{
"default_filters": json.dumps(
diff --git a/tests/utils_tests.py b/tests/utils_tests.py
index cc19c48..5332007 100644
--- a/tests/utils_tests.py
+++ b/tests/utils_tests.py
@@ -56,6 +56,7 @@ from superset.utils.core import (
zlib_decompress,
)
from superset.views.utils import get_time_range_endpoints
+from superset.views.utils import build_extra_filters
from tests.base_tests import SupersetTestCase
@@ -956,3 +957,267 @@ class UtilsTestCase(SupersetTestCase):
self.assertListEqual(get_iterable(123), [123])
self.assertListEqual(get_iterable([123]), [123])
self.assertListEqual(get_iterable("foo"), ["foo"])
+
+ def test_build_extra_filters(self):
+ layout = {
+ "CHART-2ee52f30": {
+ "children": [],
+ "id": "CHART-2ee52f30",
+ "meta": {
+ "chartId": 1020,
+ "height": 38,
+ "sliceName": "Chart 927",
+ "width": 6,
+ },
+ "parents": [
+ "ROOT_ID",
+ "TABS-Qq4sdkANSY",
+ "TAB-VrhTX2WUlO",
+ "TABS-N1zN4CIZP0",
+ "TAB-asWdJzKmTN",
+ "ROW-i_sG4ccXE",
+ ],
+ "type": "CHART",
+ },
+ "CHART-36bfc934": {
+ "children": [],
+ "id": "CHART-36bfc934",
+ "meta": {
+ "chartId": 1018,
+ "height": 26,
+ "sliceName": "Region Filter",
+ "width": 2,
+ },
+ "parents": [
+ "ROOT_ID",
+ "TABS-Qq4sdkANSY",
+ "TAB-W62P60D88",
+ "ROW-1e064e3c",
+ "COLUMN-fe3914b8",
+ ],
+ "type": "CHART",
+ },
+ "CHART-E_y2cuNHTv": {
+ "children": [],
+ "id": "CHART-E_y2cuNHTv",
+ "meta": {"chartId": 998, "height": 55, "sliceName": "MAP",
"width": 6},
+ "parents": [
+ "ROOT_ID",
+ "TABS-Qq4sdkANSY",
+ "TAB-W62P60D88",
+ "ROW-1e064e3c",
+ ],
+ "type": "CHART",
+ },
+ "CHART-JNxDOsAfEb": {
+ "children": [],
+ "id": "CHART-JNxDOsAfEb",
+ "meta": {
+ "chartId": 1015,
+ "height": 27,
+ "sliceName": "Population",
+ "width": 4,
+ },
+ "parents": [
+ "ROOT_ID",
+ "TABS-Qq4sdkANSY",
+ "TAB-W62P60D88",
+ "ROW-1e064e3c",
+ "COLUMN-fe3914b8",
+ ],
+ "type": "CHART",
+ },
+ "CHART-KoOwqalV80": {
+ "children": [],
+ "id": "CHART-KoOwqalV80",
+ "meta": {
+ "chartId": 927,
+ "height": 20,
+ "sliceName": "Chart 927",
+ "width": 4,
+ },
+ "parents": [
+ "ROOT_ID",
+ "TABS-Qq4sdkANSY",
+ "TAB-VrhTX2WUlO",
+ "TABS-N1zN4CIZP0",
+ "TAB-cHNWcBZC9",
+ "ROW-9b9vrWKPY",
+ ],
+ "type": "CHART",
+ },
+ "CHART-YCQAPVK7mQ": {
+ "children": [],
+ "id": "CHART-YCQAPVK7mQ",
+ "meta": {
+ "chartId": 1023,
+ "height": 38,
+ "sliceName": "World's Population",
+ "width": 4,
+ },
+ "parents": [
+ "ROOT_ID",
+ "TABS-Qq4sdkANSY",
+ "TAB-VrhTX2WUlO",
+ "ROW-UfxFT36oV5",
+ ],
+ "type": "CHART",
+ },
+ "COLUMN-fe3914b8": {
+ "children": ["CHART-36bfc934", "CHART-JNxDOsAfEb"],
+ "id": "COLUMN-fe3914b8",
+ "meta": {"background": "BACKGROUND_TRANSPARENT", "width": 6},
+ "parents": [
+ "ROOT_ID",
+ "TABS-Qq4sdkANSY",
+ "TAB-W62P60D88",
+ "ROW-1e064e3c",
+ ],
+ "type": "COLUMN",
+ },
+ "DASHBOARD_VERSION_KEY": "v2",
+ "GRID_ID": {
+ "children": [],
+ "id": "GRID_ID",
+ "parents": ["ROOT_ID"],
+ "type": "GRID",
+ },
+ "HEADER_ID": {
+ "id": "HEADER_ID",
+ "meta": {"text": "Test warmup 1023"},
+ "type": "HEADER",
+ },
+ "ROOT_ID": {
+ "children": ["TABS-Qq4sdkANSY"],
+ "id": "ROOT_ID",
+ "type": "ROOT",
+ },
+ "ROW-1e064e3c": {
+ "children": ["COLUMN-fe3914b8", "CHART-E_y2cuNHTv"],
+ "id": "ROW-1e064e3c",
+ "meta": {"background": "BACKGROUND_TRANSPARENT"},
+ "parents": ["ROOT_ID", "TABS-Qq4sdkANSY", "TAB-W62P60D88"],
+ "type": "ROW",
+ },
+ "ROW-9b9vrWKPY": {
+ "children": ["CHART-KoOwqalV80"],
+ "id": "ROW-9b9vrWKPY",
+ "meta": {"background": "BACKGROUND_TRANSPARENT"},
+ "parents": [
+ "ROOT_ID",
+ "TABS-Qq4sdkANSY",
+ "TAB-VrhTX2WUlO",
+ "TABS-N1zN4CIZP0",
+ "TAB-cHNWcBZC9",
+ ],
+ "type": "ROW",
+ },
+ "ROW-UfxFT36oV5": {
+ "children": ["CHART-YCQAPVK7mQ"],
+ "id": "ROW-UfxFT36oV5",
+ "meta": {"background": "BACKGROUND_TRANSPARENT"},
+ "parents": ["ROOT_ID", "TABS-Qq4sdkANSY", "TAB-VrhTX2WUlO"],
+ "type": "ROW",
+ },
+ "ROW-i_sG4ccXE": {
+ "children": ["CHART-2ee52f30"],
+ "id": "ROW-i_sG4ccXE",
+ "meta": {"background": "BACKGROUND_TRANSPARENT"},
+ "parents": [
+ "ROOT_ID",
+ "TABS-Qq4sdkANSY",
+ "TAB-VrhTX2WUlO",
+ "TABS-N1zN4CIZP0",
+ "TAB-asWdJzKmTN",
+ ],
+ "type": "ROW",
+ },
+ "TAB-VrhTX2WUlO": {
+ "children": ["ROW-UfxFT36oV5", "TABS-N1zN4CIZP0"],
+ "id": "TAB-VrhTX2WUlO",
+ "meta": {"text": "New Tab"},
+ "parents": ["ROOT_ID", "TABS-Qq4sdkANSY"],
+ "type": "TAB",
+ },
+ "TAB-W62P60D88": {
+ "children": ["ROW-1e064e3c"],
+ "id": "TAB-W62P60D88",
+ "meta": {"text": "Tab 2"},
+ "parents": ["ROOT_ID", "TABS-Qq4sdkANSY"],
+ "type": "TAB",
+ },
+ "TAB-asWdJzKmTN": {
+ "children": ["ROW-i_sG4ccXE"],
+ "id": "TAB-asWdJzKmTN",
+ "meta": {"text": "nested tab 1"},
+ "parents": [
+ "ROOT_ID",
+ "TABS-Qq4sdkANSY",
+ "TAB-VrhTX2WUlO",
+ "TABS-N1zN4CIZP0",
+ ],
+ "type": "TAB",
+ },
+ "TAB-cHNWcBZC9": {
+ "children": ["ROW-9b9vrWKPY"],
+ "id": "TAB-cHNWcBZC9",
+ "meta": {"text": "test2d tab 2"},
+ "parents": [
+ "ROOT_ID",
+ "TABS-Qq4sdkANSY",
+ "TAB-VrhTX2WUlO",
+ "TABS-N1zN4CIZP0",
+ ],
+ "type": "TAB",
+ },
+ "TABS-N1zN4CIZP0": {
+ "children": ["TAB-asWdJzKmTN", "TAB-cHNWcBZC9"],
+ "id": "TABS-N1zN4CIZP0",
+ "meta": {},
+ "parents": ["ROOT_ID", "TABS-Qq4sdkANSY", "TAB-VrhTX2WUlO"],
+ "type": "TABS",
+ },
+ "TABS-Qq4sdkANSY": {
+ "children": ["TAB-VrhTX2WUlO", "TAB-W62P60D88"],
+ "id": "TABS-Qq4sdkANSY",
+ "meta": {},
+ "parents": ["ROOT_ID"],
+ "type": "TABS",
+ },
+ }
+ filter_scopes = {
+ "1018": {
+ "region": {"scope": ["TAB-W62P60D88"], "immune": [998]},
+ "country_name": {"scope": ["ROOT_ID"], "immune": [927, 998]},
+ }
+ }
+ default_filters = {
+ "1018": {"region": ["North America"], "country_name": ["United
States"]}
+ }
+
+ # immune to all filters
+ slice_id = 998
+ extra_filters = build_extra_filters(
+ layout, filter_scopes, default_filters, slice_id
+ )
+ expected = []
+ self.assertEqual(extra_filters, expected)
+
+ # in scope
+ slice_id = 1015
+ extra_filters = build_extra_filters(
+ layout, filter_scopes, default_filters, slice_id
+ )
+ expected = [
+ {"col": "region", "op": "in", "val": ["North America"]},
+ {"col": "country_name", "op": "in", "val": ["United States"]},
+ ]
+ self.assertEqual(extra_filters, expected)
+
+ # not in scope
+ slice_id = 927
+ extra_filters = build_extra_filters(
+ layout, filter_scopes, default_filters, slice_id
+ )
+ expected = []
+ self.assertEqual(extra_filters, expected)
