mikebridge commented on code in PR #42760:
URL: https://github.com/apache/superset/pull/42760#discussion_r4043847732
##########
docs/static/feature-flags.json:
##########
@@ -93,6 +93,12 @@
"lifecycle": "development",
"description": "Enable semantic layers and show semantic views
alongside datasets"
},
+ {
+ "name": "SEMANTIC_LAYER_CONTAINMENT_CACHE",
Review Comment:
addressed in b5b6ed484382aa7a2b0615403d7dfc28f608244b
##########
superset-frontend/src/explore/components/ChartPills.tsx:
##########
@@ -100,11 +112,20 @@ export const ChartPills = forwardRef(
limit={Number(rowLimit ?? 0)}
/>
)}
- {!isLoading && firstQueryResponse?.is_cached && (
- <CachedLabel
- onClick={refreshCachedQuery}
- cachedTimestamp={firstQueryResponse.cached_dttm}
- />
+ {!isLoading &&
+ semanticCacheStatus !== 'MIXED' &&
+ (firstQueryResponse?.is_cached ||
+ semanticCacheStatus === 'HIT') && (
+ <CachedLabel
+ onClick={refreshCachedQuery}
+ cachedTimestamp={firstQueryResponse?.cached_dttm}
+ cacheSource={
+ semanticCacheStatus === 'HIT' ? 'semantic' : 'result'
+ }
+ />
+ )}
+ {!isLoading && semanticCacheStatus === 'MIXED' && (
Review Comment:
addressed in b5b6ed484382aa7a2b0615403d7dfc28f608244b
##########
docs/developer_docs/extensions/contribution-types.md:
##########
@@ -291,3 +291,112 @@ class MySemanticLayer(SemanticLayer[MyConfig,
MySemanticView]):
- **Host context**: Original ID used as-is
The decorator registers the class in the semantic layers registry, making it
available in the UI for users to create connections. The `configuration_class`
should be a Pydantic model that defines the fields needed to connect
(credentials, project, database, etc.). Superset uses the model's JSON schema
to render the configuration form dynamically.
+
+#### Semantic result containment caching
+
+Superset containment caching is experimental and off by default. A provider
must
+explicitly opt in; the safe defaults leave caching with the provider and scope
+results to an execution context:
+
+```python
+from superset_core.semantic_layers.layer import (
+ SemanticCacheCapabilities,
+ SemanticCacheExecutionContext,
+ SemanticCacheIdentityMaterial,
+ SemanticCacheResponsibility,
+ SemanticCacheScope,
+)
+
+class MySemanticLayer(SemanticLayer[MyConfig, MySemanticView]):
+ semantic_cache_responsibility = SemanticCacheResponsibility.SUPERSET
+ semantic_cache_scope = SemanticCacheScope.EXECUTION_CONTEXT
+ semantic_cache_capabilities = SemanticCacheCapabilities(
+ comparisons=True,
+ membership=True,
+ nulls=True,
+ pattern_escape="\\",
+ )
+
+ def get_semantic_cache_provider_identity(self) ->
SemanticCacheIdentityMaterial:
+ return SemanticCacheIdentityMaterial(
+ {"provider_version": "v1", "catalog": self.config.catalog}
+ )
+
+ def get_semantic_cache_context_identity(
+ self,
+ context: SemanticCacheExecutionContext,
+ ) -> SemanticCacheIdentityMaterial:
+ return SemanticCacheIdentityMaterial({"tenant":
self.tenant_id(context)})
+```
+
+Identity material must be secret-free and include every provider setting that
can
+change results. For execution-context scope, Superset also hashes the
principal,
+roles, guest-token claims, and row-level-security cache key. Returning `None`
from
+either identity method bypasses containment. Use `GLOBAL` only when results are
+provably identical across principals and tenants; containment is bypassed for a
+`GLOBAL` view whenever Superset row-level security applies to the request,
since
+that variation is invisible to the provider. Declare only filter capabilities
+whose provider semantics exactly match Superset's post-processing semantics.
+
+Operators enable both `SEMANTIC_LAYERS` and the development feature flag
+`SEMANTIC_LAYER_CONTAINMENT_CACHE`, and configure two backends:
+
+- `DATA_CACHE_CONFIG` holds the cached results and their descriptors. It must
be
+ a persistent cache shared by every web and worker process, such as
`RedisCache`.
+ The default `NullCache` discards every value, so containment would only ever
+ miss; `RedisSentinelCache` reads from replicas that can lag the master. Both
+ disable containment at startup rather than run it ineffectively.
+- `DISTRIBUTED_COORDINATION_CONFIG` must select `RedisCache` or
+ `RedisSentinelCache`; containment requires its atomic owner-token lease
+ operations.
+
+```python
+DATA_CACHE_CONFIG = {
+ "CACHE_TYPE": "RedisCache",
+ "CACHE_DEFAULT_TIMEOUT": 86400,
+ "CACHE_KEY_PREFIX": "superset_results",
+ "CACHE_REDIS_URL": "redis://redis:6379/1",
+}
+DISTRIBUTED_COORDINATION_CONFIG = {
+ "CACHE_TYPE": "RedisCache",
+ "CACHE_REDIS_URL": "redis://redis:6379/2",
Review Comment:
addressed in b5b6ed484382aa7a2b0615403d7dfc28f608244b
##########
superset/charts/data/api.py:
##########
@@ -575,6 +575,9 @@ def _process_data(query_data: Any) -> Any:
)
resp = make_response(response_data, 200)
resp.headers["Content-Type"] = "application/json; charset=utf-8"
+ resp.headers["X-Superset-Semantic-Cache"] =
SemanticCacheStatus.combine(
Review Comment:
addressed in b5b6ed484382aa7a2b0615403d7dfc28f608244b
##########
superset/semantic_layers/cache_transform.py:
##########
@@ -0,0 +1,222 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""In-memory transformations for proven-compatible semantic cache results."""
+
+from collections.abc import Callable
+
+import pandas as pd
+import pyarrow as pa
+from superset_core.semantic_layers.types import (
+ AggregationType,
+ Filter,
+ Operator,
+ SemanticQuery,
+ SemanticResult,
+)
+
+from superset.semantic_layers.cache_policy import compile_like_pattern
+from superset.semantic_layers.cache_types import (
+ ContainmentCapabilities,
+ PatternSemantics,
+ ReuseDecision,
+ ReuseMode,
+ ROLLUP_COMPATIBLE_AGGREGATIONS,
+)
+
+
+def _sql_sum(series: pd.Series) -> object:
+ return series.sum(min_count=1)
+
+
+_ROLLUP_AGGREGATIONS: dict[AggregationType, str | Callable[[pd.Series],
object]] = {
+ AggregationType.SUM: _sql_sum,
+ AggregationType.COUNT: "sum",
+ AggregationType.MIN: "min",
+ AggregationType.MAX: "max",
+}
+assert frozenset(_ROLLUP_AGGREGATIONS) == ROLLUP_COMPATIBLE_AGGREGATIONS
+
+
+class SemanticCacheTransformationError(RuntimeError):
+ """Cached data cannot safely satisfy an otherwise eligible query."""
+
+
+def _comparison_mask(
+ series: pd.Series, operator: Operator, value: object
+) -> pd.Series | None:
+ if value is None:
+ return pd.Series(False, index=series.index, dtype=bool)
+ not_null: pd.Series = series.notna()
+ comparisons: dict[Operator, Callable[[object], pd.Series]] = {
+ Operator.EQUALS: series.eq,
+ Operator.NOT_EQUALS: series.ne,
+ Operator.GREATER_THAN: series.gt,
+ Operator.GREATER_THAN_OR_EQUAL: series.ge,
+ Operator.LESS_THAN: series.lt,
+ Operator.LESS_THAN_OR_EQUAL: series.le,
+ }
+ comparison: Callable[[object], pd.Series] | None =
comparisons.get(operator)
+ return not_null & comparison(value) if comparison else None
+
+
+def _membership_mask(
+ series: pd.Series,
+ operator: Operator,
+ values: set[object] | frozenset[object] | tuple[object, ...],
+) -> pd.Series:
+ not_null: pd.Series = series.notna()
+ non_null_values: list[object] = [item for item in values if item is not
None]
+ membership: pd.Series = not_null & series.isin(non_null_values)
+ if operator is Operator.IN:
+ return membership
+ if None in values:
+ return pd.Series(False, index=series.index, dtype=bool)
+ return not_null & ~membership
+
+
+def _pattern_mask(
+ series: pd.Series,
+ operator: Operator,
+ pattern: str,
+ semantics: PatternSemantics,
+) -> pd.Series:
+ not_null: pd.Series = series.notna()
+ matches: pd.Series = not_null & series.astype("string").str.fullmatch(
+ compile_like_pattern(pattern, semantics.escape), na=False
+ )
+ return matches if operator is Operator.LIKE else not_null & ~matches
+
+
+def mask_for(
+ series: pd.Series,
+ operator: Operator,
+ value: object,
+ *,
+ pattern_semantics: PatternSemantics | None = None,
+) -> pd.Series:
+ """Return a SQL-WHERE-compatible boolean mask for a pandas series."""
+ if operator is Operator.IS_NULL:
+ return series.isna()
+ if operator is Operator.IS_NOT_NULL:
+ return series.notna()
+ comparison: pd.Series | None = _comparison_mask(series, operator, value)
+ if comparison is not None:
+ return comparison
+ if operator in {Operator.IN, Operator.NOT_IN} and isinstance(
+ value, (set, frozenset, tuple)
+ ):
+ return _membership_mask(series, operator, value)
+ if (
+ operator in {Operator.LIKE, Operator.NOT_LIKE}
+ and isinstance(value, str)
+ and pattern_semantics is not None
+ ):
+ return _pattern_mask(series, operator, value, pattern_semantics)
+ raise ValueError(f"Unsupported cached filter operation: {operator.value}")
+
+
+def _apply_leftovers(
+ frame: pd.DataFrame,
+ leftovers: frozenset[Filter],
+ capabilities: ContainmentCapabilities,
+) -> pd.DataFrame:
+ transformed: pd.DataFrame = frame
+ for filter_ in sorted(
+ leftovers,
+ key=lambda item: (
+ item.column.id if item.column else "",
+ item.operator.value,
+ ),
+ ):
+ if filter_.column is None:
+ raise ValueError("Cached post-processing requires a filter column")
+ column_name: str = filter_.column.name
+ transformed = transformed.loc[
+ mask_for(
+ transformed[column_name],
+ filter_.operator,
+ filter_.value,
+ pattern_semantics=capabilities.pattern_semantics,
+ )
+ ]
+ return transformed
+
+
+def _rollup(frame: pd.DataFrame, query: SemanticQuery) -> pd.DataFrame:
+ dimension_names: list[str] = [dimension.name for dimension in
query.dimensions]
+ aggregations: dict[str, str | Callable[[pd.Series], object]] = {}
+ for metric in query.metrics:
+ if metric.aggregation not in _ROLLUP_AGGREGATIONS:
+ raise ValueError(f"Metric {metric.id} is not safely roll-up
compatible")
+ aggregations[metric.name] = _ROLLUP_AGGREGATIONS[metric.aggregation]
+ if dimension_names:
+ return frame.groupby(dimension_names, as_index=False,
dropna=False).agg(
Review Comment:
addressed in b5b6ed484382aa7a2b0615403d7dfc28f608244b
##########
superset/common/query_context_processor.py:
##########
@@ -241,6 +241,12 @@ def get_df_payload_result(
self._resolve_forced_query(query_obj, cache_key)
or timeout == CACHE_DISABLED_TIMEOUT
)
+ if query_obj:
+ # Datasources that run their own caching (semantic containment)
+ # must honor the same resolved chart/custom timeout and force
+ # decision as the result cache, not the datasource-level default.
+ query_obj.force_query = force_query
Review Comment:
addressed in b5b6ed484382aa7a2b0615403d7dfc28f608244b
##########
superset/semantic_layers/mapper.py:
##########
@@ -147,8 +191,24 @@ def get_results(query_object: QueryObject) -> QueryResult:
# Step 2: Execute the main query (first in the list)
main_query = queries[0]
- main_result = dispatcher(main_query)
+ main_outcome: SemanticCacheOutcome = _dispatch_semantic_query(
+ query_object.datasource,
+ dispatcher,
+ main_query,
+ force=query_object.force_query,
+ # Row-count results share the query's logical identity but not its
+ # shape: a cached table result would satisfy a row-count lookup (and
+ # vice versa) and break server-side pagination. Containment reuse is
+ # only defined over tabular results, so row-count dispatches bypass
+ # the cache entirely.
+ cacheable=not query_object.is_rowcount,
Review Comment:
addressed in b5b6ed484382aa7a2b0615403d7dfc28f608244b
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
