This is an automated email from the ASF dual-hosted git repository.
rusackas pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/superset.git
The following commit(s) were added to refs/heads/master by this push:
new afb863684b1 feat(cache): add DATA_CACHE_MAX_VALUE_SIZE to skip caching
oversized results (#42570)
afb863684b1 is described below
commit afb863684b1f23ae11c0db7058c110d75e51a748
Author: Luiz Otavio <[email protected]>
AuthorDate: Wed Jul 29 15:35:42 2026 -0300
feat(cache): add DATA_CACHE_MAX_VALUE_SIZE to skip caching oversized
results (#42570)
---
docs/admin_docs/configuration/cache.mdx | 15 +++
superset/config.py | 9 ++
superset/utils/cache.py | 21 +++
tests/unit_tests/utils/cache_test.py | 225 ++++++++++++++++++++++++++++++++
4 files changed, 270 insertions(+)
diff --git a/docs/admin_docs/configuration/cache.mdx
b/docs/admin_docs/configuration/cache.mdx
index d4a5f36d087..ba6f842e6aa 100644
--- a/docs/admin_docs/configuration/cache.mdx
+++ b/docs/admin_docs/configuration/cache.mdx
@@ -76,6 +76,21 @@ value defined in `DATA_CACHE_CONFIG`.
Note, that by setting the cache timeout to `-1`, caching for charting data can
be disabled, either
per chart, dataset or database, or by default if set in `DATA_CACHE_CONFIG`.
+## Limiting Cached Result Size
+
+Very large chart or SQL query results can flood the cache backend
(Redis/Memcached), evicting many
+smaller useful entries or exhausting memory. To cap the size of any single
value written to the data
+cache, set `DATA_CACHE_MAX_VALUE_SIZE` (in bytes) in `superset_config.py`:
+
+```python
+DATA_CACHE_MAX_VALUE_SIZE = 10 * 1024 * 1024 # 10 MB
+```
+
+When a result's serialized size exceeds this threshold it is not written to
the data cache — the
+chart still renders, but the next load re-queries the datasource instead of
getting a cache hit. The
+`skip_cache_value_too_large` statsd metric is incremented each time this
happens. Set to `None` (the
+default) to disable the check.
+
## SQL Lab Query Results
Caching for SQL Lab query results is used when async queries are enabled and
is configured using
diff --git a/superset/config.py b/superset/config.py
index d77c26c1002..70c932f611f 100644
--- a/superset/config.py
+++ b/superset/config.py
@@ -1350,6 +1350,15 @@ CACHE_CONFIG: CacheConfig = {"CACHE_TYPE": "NullCache"}
# Cache for datasource metadata and query results
DATA_CACHE_CONFIG: CacheConfig = {"CACHE_TYPE": "NullCache"}
+# Upper bound, in bytes, on the serialized size of a single value written to
the
+# data cache (chart and SQL query results). When a result's pickled size
exceeds
+# this threshold the value is NOT written to the cache: the chart still
renders,
+# but the next load re-queries the datasource instead of getting a cache hit.
This
+# protects the cache backend (e.g. Redis/Memcached) from being flooded by very
+# large result sets. Set to ``None`` to disable the check (the default).
Example:
+# 10 * 1024 * 1024 for a 10 MB limit.
+DATA_CACHE_MAX_VALUE_SIZE: int | None = None
+
# Cache for dashboard filter state. `CACHE_TYPE` defaults to
`SupersetMetastoreCache`
# that stores the values in the key-value table in the Superset metastore, as
it's
# required for Superset to operate correctly, but can be replaced by any
diff --git a/superset/utils/cache.py b/superset/utils/cache.py
index d138a7e9326..161b40832d2 100644
--- a/superset/utils/cache.py
+++ b/superset/utils/cache.py
@@ -18,6 +18,7 @@ from __future__ import annotations
import inspect
import logging
+import pickle
from datetime import datetime, timedelta, timezone
from functools import wraps
from typing import Any, Callable
@@ -77,6 +78,26 @@ def set_and_log_cache(
datetime.now(timezone.utc).replace(tzinfo=None).isoformat().split(".")[0]
)
value = {**cache_value, "dttm": dttm}
+
+ # Skip caching results that are too large to protect the cache backend
+ # (e.g. Redis/Memcached) from being flooded by huge result sets. The
chart
+ # still renders; the value is simply not cached, causing a re-query on
the
+ # next load instead of a cache hit. Disabled when
DATA_CACHE_MAX_VALUE_SIZE
+ # is None (the default), in which case no serialization overhead is
incurred.
+ max_value_size = app.config.get("DATA_CACHE_MAX_VALUE_SIZE")
+ if max_value_size is not None:
+ value_size = len(pickle.dumps(value,
protocol=pickle.HIGHEST_PROTOCOL))
+ if value_size > max_value_size:
+ logger.warning(
+ "Skipping cache set for key %s: serialized value size %d
bytes "
+ "exceeds DATA_CACHE_MAX_VALUE_SIZE (%d bytes)",
+ cache_key,
+ value_size,
+ max_value_size,
+ )
+ app.config["STATS_LOGGER"].incr("skip_cache_value_too_large")
+ return
+
cache_instance.set(cache_key, value, timeout=timeout)
stats_logger = app.config["STATS_LOGGER"]
stats_logger.incr("set_cache_key")
diff --git a/tests/unit_tests/utils/cache_test.py
b/tests/unit_tests/utils/cache_test.py
index bd6179957e4..2d0d48e5983 100644
--- a/tests/unit_tests/utils/cache_test.py
+++ b/tests/unit_tests/utils/cache_test.py
@@ -17,6 +17,9 @@
# pylint: disable=import-outside-toplevel, unused-argument
+from typing import Any
+from unittest.mock import MagicMock
+
from pytest_mock import MockerFixture
@@ -49,3 +52,225 @@ def test_memoized_func(mocker: MockerFixture) -> None:
cache.get.return_value = 43
result = decorated(self, "public", cache=True)
assert result == 43
+
+
+def _make_cache_instance(mocker: MockerFixture) -> MagicMock:
+ """A cache instance whose ``.cache`` is not a ``NullCache``."""
+ cache_instance = mocker.MagicMock()
+ cache_instance.cache = object()
+ return cache_instance
+
+
+def _patch_config(mocker: MockerFixture, **overrides: Any) -> dict[str, Any]:
+ config = {
+ "CACHE_DEFAULT_TIMEOUT": 100,
+ "STATS_LOGGER": mocker.MagicMock(),
+ "STORE_CACHE_KEYS_IN_METADATA_DB": False,
+ "DATA_CACHE_MAX_VALUE_SIZE": None,
+ }
+ config.update(overrides)
+ mocker.patch("superset.utils.cache.app.config", config)
+ return config
+
+
+def test_set_and_log_cache_under_threshold(mocker: MockerFixture) -> None:
+ """A value under DATA_CACHE_MAX_VALUE_SIZE is cached normally."""
+ from superset.utils.cache import set_and_log_cache
+
+ config = _patch_config(mocker, DATA_CACHE_MAX_VALUE_SIZE=10 * 1024 * 1024)
+ cache_instance = _make_cache_instance(mocker)
+
+ set_and_log_cache(cache_instance, "my_key", {"df": "small"})
+
+ cache_instance.set.assert_called_once()
+ config["STATS_LOGGER"].incr.assert_any_call("set_cache_key")
+ assert (
+ mocker.call("skip_cache_value_too_large")
+ not in config["STATS_LOGGER"].incr.mock_calls
+ )
+
+
+def test_set_and_log_cache_over_threshold(mocker: MockerFixture) -> None:
+ """A value exceeding DATA_CACHE_MAX_VALUE_SIZE is not cached."""
+ from superset.utils.cache import set_and_log_cache
+
+ config = _patch_config(
+ mocker,
+ DATA_CACHE_MAX_VALUE_SIZE=10,
+ STORE_CACHE_KEYS_IN_METADATA_DB=True,
+ )
+ cache_instance = _make_cache_instance(mocker)
+ mock_session = mocker.patch("superset.utils.cache.db.session")
+
+ set_and_log_cache(
+ cache_instance,
+ "my_key",
+ {"df": "a value large enough to exceed the tiny threshold"},
+ datasource_uid="1__table",
+ )
+
+ cache_instance.set.assert_not_called()
+
config["STATS_LOGGER"].incr.assert_called_once_with("skip_cache_value_too_large")
+ assert mocker.call("set_cache_key") not in
config["STATS_LOGGER"].incr.mock_calls
+ mock_session.add.assert_not_called()
+
+
+def test_set_and_log_cache_disabled_no_serialization(mocker: MockerFixture) ->
None:
+ """When the limit is None (default), no pickling overhead is incurred."""
+ from superset.utils.cache import set_and_log_cache
+
+ _patch_config(mocker, DATA_CACHE_MAX_VALUE_SIZE=None)
+ cache_instance = _make_cache_instance(mocker)
+ mock_dumps = mocker.patch("superset.utils.cache.pickle.dumps")
+
+ set_and_log_cache(cache_instance, "my_key", {"df": "small"})
+
+ cache_instance.set.assert_called_once()
+ mock_dumps.assert_not_called()
+
+
+def test_set_and_log_cache_null_cache(mocker: MockerFixture) -> None:
+ """A NullCache backend short-circuits before any set."""
+ from flask_caching.backends import NullCache
+
+ from superset.utils.cache import set_and_log_cache
+
+ _patch_config(mocker, DATA_CACHE_MAX_VALUE_SIZE=10)
+ cache_instance = mocker.MagicMock()
+ cache_instance.cache = NullCache()
+
+ set_and_log_cache(cache_instance, "my_key", {"df": "small"})
+
+ cache_instance.set.assert_not_called()
+
+
+def test_set_and_log_cache_disabled_timeout(mocker: MockerFixture) -> None:
+ """A timeout of -1 (CACHE_DISABLED_TIMEOUT) short-circuits before any
set."""
+ from superset.utils.cache import set_and_log_cache
+
+ _patch_config(mocker)
+ cache_instance = _make_cache_instance(mocker)
+
+ set_and_log_cache(cache_instance, "my_key", {"df": "small"},
cache_timeout=-1)
+
+ cache_instance.set.assert_not_called()
+
+
+def test_set_and_log_cache_equal_threshold(mocker: MockerFixture) -> None:
+ """A value whose size EQUALS the threshold is still cached (guard is
``>``)."""
+ import pickle
+
+ from superset.utils.cache import set_and_log_cache
+
+ cache_value = {"df": "boundary"}
+ # Compute the exact serialized size the function will see, including the
+ # injected ``dttm`` field, so we can set the threshold to that exact value.
+ dttm = "2021-01-01T00:00:00"
+ value = {**cache_value, "dttm": dttm}
+ exact_size = len(pickle.dumps(value, protocol=pickle.HIGHEST_PROTOCOL))
+
+ config = _patch_config(mocker, DATA_CACHE_MAX_VALUE_SIZE=exact_size)
+ cache_instance = _make_cache_instance(mocker)
+ # Pin the timestamp so the pickled size matches ``exact_size``
deterministically.
+ mock_datetime = mocker.patch("superset.utils.cache.datetime")
+ mock_datetime.now.return_value.replace.return_value.isoformat.return_value
= dttm
+
+ set_and_log_cache(cache_instance, "my_key", cache_value)
+
+ cache_instance.set.assert_called_once()
+ config["STATS_LOGGER"].incr.assert_any_call("set_cache_key")
+ assert (
+ mocker.call("skip_cache_value_too_large")
+ not in config["STATS_LOGGER"].incr.mock_calls
+ )
+
+
+def test_set_and_log_cache_over_threshold_no_datasource(mocker: MockerFixture)
-> None:
+ """Over-threshold with no datasource_uid: skipped, and no metadata-DB
write."""
+ from superset.utils.cache import set_and_log_cache
+
+ config = _patch_config(
+ mocker,
+ DATA_CACHE_MAX_VALUE_SIZE=10,
+ STORE_CACHE_KEYS_IN_METADATA_DB=True,
+ )
+ cache_instance = _make_cache_instance(mocker)
+ mock_session = mocker.patch("superset.utils.cache.db.session")
+
+ set_and_log_cache(
+ cache_instance,
+ "my_key",
+ {"df": "a value large enough to exceed the tiny threshold"},
+ )
+
+ cache_instance.set.assert_not_called()
+
config["STATS_LOGGER"].incr.assert_called_once_with("skip_cache_value_too_large")
+ mock_session.add.assert_not_called()
+
+
+def test_set_and_log_cache_over_threshold_warns(mocker: MockerFixture) -> None:
+ """The over-threshold branch emits a warning naming the key and sizes."""
+ from superset.utils.cache import set_and_log_cache
+
+ _patch_config(mocker, DATA_CACHE_MAX_VALUE_SIZE=10)
+ cache_instance = _make_cache_instance(mocker)
+ mock_logger = mocker.patch("superset.utils.cache.logger")
+
+ set_and_log_cache(
+ cache_instance,
+ "my_key",
+ {"df": "a value large enough to exceed the tiny threshold"},
+ )
+
+ mock_logger.warning.assert_called_once()
+ warning_args = mock_logger.warning.call_args.args
+ assert "exceeds DATA_CACHE_MAX_VALUE_SIZE" in warning_args[0]
+ assert "my_key" in warning_args
+
+
+def test_set_and_log_cache_under_threshold_metadata_db(mocker: MockerFixture)
-> None:
+ """Under-threshold with datasource_uid + metadata-DB storage writes a
CacheKey."""
+ from superset.utils.cache import set_and_log_cache
+
+ config = _patch_config(
+ mocker,
+ DATA_CACHE_MAX_VALUE_SIZE=10 * 1024 * 1024,
+ STORE_CACHE_KEYS_IN_METADATA_DB=True,
+ )
+ cache_instance = _make_cache_instance(mocker)
+ mock_session = mocker.patch("superset.utils.cache.db.session")
+ mock_cache_key = mocker.patch("superset.utils.cache.CacheKey")
+
+ set_and_log_cache(
+ cache_instance,
+ "my_key",
+ {"df": "small"},
+ cache_timeout=42,
+ datasource_uid="1__table",
+ )
+
+ cache_instance.set.assert_called_once()
+ config["STATS_LOGGER"].incr.assert_any_call("set_cache_key")
+ mock_cache_key.assert_called_once_with(
+ cache_key="my_key",
+ cache_timeout=42,
+ datasource_uid="1__table",
+ )
+ mock_session.add.assert_called_once_with(mock_cache_key.return_value)
+
+
+def test_set_and_log_cache_set_failure_logs(mocker: MockerFixture) -> None:
+ """A failure inside the try block is caught and logged as 'Could not cache
key'."""
+ from superset.utils.cache import set_and_log_cache
+
+ _patch_config(mocker, DATA_CACHE_MAX_VALUE_SIZE=None)
+ cache_instance = _make_cache_instance(mocker)
+ boom = RuntimeError("backend down")
+ cache_instance.set.side_effect = boom
+ mock_logger = mocker.patch("superset.utils.cache.logger")
+
+ # Should not raise despite the backend failure.
+ set_and_log_cache(cache_instance, "my_key", {"df": "small"})
+
+ mock_logger.warning.assert_called_once_with("Could not cache key %s",
"my_key")
+ mock_logger.exception.assert_called_once_with(boom)
