This is an automated email from the ASF dual-hosted git repository.
aminghadersohi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/superset.git
The following commit(s) were added to refs/heads/master by this push:
new fc5956b0ca4 fix(semantic-layer): use durable metadata cache identity
(#45544)
fc5956b0ca4 is described below
commit fc5956b0ca4d36e1689a2cc83033a284b9b9e871
Author: Mike Bridge <[email protected]>
AuthorDate: Mon Oct 26 11:37:59 2026 -0600
fix(semantic-layer): use durable metadata cache identity (#45544)
Co-authored-by: Mike Bridge <[email protected]>
---
UPDATING.md | 10 +-
docs/developer_docs/semantic-layer-cache-clear.md | 18 +-
superset/semantic_layers/execution.py | 75 +++-
superset/semantic_layers/models.py | 27 +-
.../commands/report/semantic_view_report_test.py | 1 +
.../common/test_query_context_processor.py | 13 +
tests/unit_tests/datasource/test_compatible_api.py | 10 +-
.../unit_tests/security/semantic_guest_rls_test.py | 5 +-
.../semantic_layers/integer_precision_test.py | 6 +-
.../semantic_layers/metadata_namespace_test.py | 422 +++++++++++++++++++++
tests/unit_tests/semantic_layers/models_test.py | 29 +-
.../semantic_layers/simple_cache_clear_test.py | 18 +-
.../semantic_layers/values_endpoint_test.py | 46 ++-
13 files changed, 632 insertions(+), 48 deletions(-)
diff --git a/UPDATING.md b/UPDATING.md
index 782b938a02d..1410a7666a8 100644
--- a/UPDATING.md
+++ b/UPDATING.md
@@ -217,9 +217,13 @@ assists people when migrating to a new version.
Providers must accept required keyword-only `cache_token` and `context` on
`from_configuration`; the optional `from_configuration_with_cache_token` hook
is removed. Local metadata caches must include `cache_token` in their keys
- before lookup or eager discovery for complete invalidation. When tenants
share
- a metadata database or web, Celery and MCP use different database host
aliases,
- configure
+ before lookup or eager discovery for complete invalidation. The default
namespace
+ uses the active metadata database's durable installation identity rather than
+ its address, so web, Celery and MCP agree across host/port/database aliases.
+ This causes a one-time cache miss for semantic-layer results after deployment
+ when using the default namespace; update all roles together. Existing
explicit
+ namespaces retain their key scheme. When tenants share a metadata database or
+ a full database clone must be isolated from its source, configure
`SEMANTIC_LAYER_CACHE_NAMESPACE` as a globally unique workspace identity that
resolves identically in all three contexts. A metadata session must not span
workspaces. Saves handled by old instances during a rolling deployment do not
diff --git a/docs/developer_docs/semantic-layer-cache-clear.md
b/docs/developer_docs/semantic-layer-cache-clear.md
index b820b423ea8..b2ceb04aa7c 100644
--- a/docs/developer_docs/semantic-layer-cache-clear.md
+++ b/docs/developer_docs/semantic-layer-cache-clear.md
@@ -61,16 +61,24 @@ The context and reserved credentials channel are described
in the
[execution
contract](extensions/extension-points/semantic-layers.md#execution-context).
The host captures a database/workspace-scoped layer UUID and version
once per metadata session (the ordinary request/task session); this is the
layer's
**metadata generation**, passed to providers as `cache_token`. The namespace is
-an opaque digest of the metadata database backend, host, port and database
name;
-credentials, driver adapters, connection query options and `SECRET_KEY` do not
-affect it. `SEMANTIC_LAYER_CACHE_NAMESPACE`, when nonempty, replaces that
database
+an opaque digest of the durable installation identity stored in the active
metadata
+database, not its connection address. Host aliases, proxy/default ports,
database
+aliases, credentials, driver adapters, connection query options and
`SECRET_KEY`
+do not affect it. `SEMANTIC_LAYER_CACHE_NAMESPACE`, when nonempty, replaces
that
identity with an operator-provided, globally unique workspace name. Configure
it
-when routing through different database host aliases or sharing a database
across
-tenants. A callable may return the active workspace, but web, Celery and MCP
+when sharing one metadata database across tenants or cloning a database into an
+isolated workspace: a full clone also copies the installation identity. A
callable
+may return the active workspace, but web, Celery and MCP
contexts must resolve the same value for the same workspace. The host captures
it
once per layer per session. A metadata session must not span workspaces. No URL
or credential is exposed in the token.
+Switching from the address-derived default causes a one-time cache miss for
+semantic-layer results after deployment. Roll web, worker and MCP processes
+together so they use the same key scheme. An existing explicit namespace keeps
+its key scheme. After restoring a backup, clear surviving caches or rotate the
+explicit namespace to prevent reuse of generations captured before the restore.
+
Providers with process-local metadata caches must include the supplied token
alongside existing endpoint, credential, role and view scope **before**
any dictionary lookup or eager discovery. Their returned SDK views echo the
diff --git a/superset/semantic_layers/execution.py
b/superset/semantic_layers/execution.py
index 3d30610bfb2..e261d9297d7 100644
--- a/superset/semantic_layers/execution.py
+++ b/superset/semantic_layers/execution.py
@@ -28,9 +28,10 @@ from uuid import uuid4
from flask import current_app, g, has_request_context
from sqlalchemy import select
-from sqlalchemy.engine import Engine
+from sqlalchemy.engine import Connection, Engine
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import object_session, Session
+from sqlalchemy.pool import SingletonThreadPool, StaticPool
from sqlalchemy.sql import Select
from superset_core.semantic_layers.context import (
ExecutionMethod,
@@ -94,14 +95,71 @@ def semantic_namespace() -> str:
return namespace
+def metadata_namespace(connection: Connection) -> str:
+ """Read the active metadata store's identity, not the app's startup bind.
+
+ Use the caller's connection without ORM autoflush or another pool checkout.
+ Only a missing identity needs an independent, durable bootstrap
transaction.
+ Callers memoize this value only within their metadata session.
+ """
+ value: bytes | None = connection.scalar(_namespace_statement())
+ if value is not None:
+ return _decode_namespace(value)
+ if connection.dialect.name == "sqlite" and getattr(
+ connection.connection.driver_connection, "in_transaction", False
+ ):
+ # A separate writer could wait on this caller's SQLite transaction.
+ # Do not commit or roll back caller work to make bootstrap possible.
+ raise ValueError(
+ "Initialize the semantic execution namespace "
+ "before starting a SQLite transaction"
+ )
+ engine: Engine = connection.engine
+ if connection.dialect.name == "sqlite" and isinstance(
+ engine.pool, (SingletonThreadPool, StaticPool)
+ ):
+ # These pools share the caller's connection; recreating them loses an
+ # in-memory store. The transaction guard above protects caller writes.
+ return _install_namespace(engine)
+ # A separate pool preserves the configured connection creator/on-connect
+ # hooks without waiting for capacity held by this caller in the original
pool.
+ bootstrap: Engine = Engine(
+ engine.pool.recreate(),
+ engine.dialect,
+ engine.url,
+ execution_options=engine.get_execution_options(),
+ )
+ try:
+ return _install_namespace(bootstrap)
+ finally:
+ bootstrap.dispose()
+
+
+def _namespace_statement() -> Select:
+ """Select the existing installation record shared by both identity
readers."""
+ return select(KeyValueEntry.value).where(
+ KeyValueEntry.uuid == SEMANTIC_INSTALL_ID_KEY,
+ KeyValueEntry.resource == KeyValueResource.SEMANTIC_EXECUTION.value,
+ )
+
+
+def _decode_namespace(value: bytes | None) -> str:
+ """Validate persisted identity without substituting a process-local
value."""
+ namespace: Any = (
+ JsonKeyValueCodec().decode(value).get("namespace")
+ if value is not None
+ else None
+ )
+ if not isinstance(namespace, str) or not namespace.strip():
+ raise ValueError("Semantic execution namespace is unavailable")
+ return namespace
+
+
def _install_namespace(engine: Engine) -> str:
"""Create a durable UUID independently, including startup races."""
codec: JsonKeyValueCodec = JsonKeyValueCodec()
with Session(engine) as session:
- statement: Select = select(KeyValueEntry.value).where(
- KeyValueEntry.uuid == SEMANTIC_INSTALL_ID_KEY,
- KeyValueEntry.resource ==
KeyValueResource.SEMANTIC_EXECUTION.value,
- )
+ statement: Select = _namespace_statement()
value: bytes | None = session.scalar(statement)
if value is None:
session.add(
@@ -119,12 +177,7 @@ def _install_namespace(engine: Engine) -> str:
except IntegrityError:
session.rollback() # pylint:
disable=consider-using-transaction
value = session.scalar(statement)
- namespace: Any = (
- codec.decode(value).get("namespace") if value is not None else None
- )
- if not isinstance(namespace, str) or not namespace.strip():
- raise ValueError("Semantic execution namespace is unavailable")
- return namespace
+ return _decode_namespace(value)
def effective_role_ids(user: Any) -> tuple[str, ...]:
diff --git a/superset/semantic_layers/models.py
b/superset/semantic_layers/models.py
index 9e946f7edcd..63790838288 100644
--- a/superset/semantic_layers/models.py
+++ b/superset/semantic_layers/models.py
@@ -71,7 +71,11 @@ from superset.semantic_layers.exceptions import (
execute_semantic_query,
SemanticLayerQueryRejectedError,
)
-from superset.semantic_layers.execution import configured_namespace,
SemanticBindingMemo
+from superset.semantic_layers.execution import (
+ configured_namespace,
+ metadata_namespace,
+ SemanticBindingMemo,
+)
from superset.semantic_layers.mapper import get_results
from superset.semantic_layers.registry import registry
from superset.utils import json
@@ -152,6 +156,7 @@ class _MetadataGenerations:
"""Session-local generations, captured before discovery and forgotten on
Save."""
generations: dict[uuid.UUID, str] = field(default_factory=dict)
+ namespaces: dict[sa.engine.Engine, str] = field(default_factory=dict)
@classmethod
def for_session(cls, session: Session) -> _MetadataGenerations:
@@ -164,18 +169,14 @@ class _MetadataGenerations:
"""Web, task and MCP operations share a credential-independent
identity."""
if layer.uuid not in self.generations:
namespace: str = configured_namespace()
- url: sa.engine.URL =
session.get_bind(mapper=SemanticLayer).engine.url
- driver: str = url.drivername.split("+", 1)[0]
- identity: list[str | int | None] = (
- ["namespace", namespace]
- if namespace
- else [
- driver,
- url.host,
- url.port,
- url.database,
- ]
- )
+ if not namespace:
+ engine: sa.engine.Engine =
session.get_bind(mapper=SemanticLayer).engine
+ if engine not in self.namespaces:
+ self.namespaces[engine] = metadata_namespace(
+ session.connection(bind_arguments={"mapper":
SemanticLayer})
+ )
+ namespace = self.namespaces[engine]
+ identity: list[str] = ["namespace", namespace]
scope: str =
hashlib.sha256(json.dumps(identity).encode()).hexdigest()
self.generations[layer.uuid] =
f"{scope}:{layer.uuid}:{layer.cache_version}"
return self.generations[layer.uuid]
diff --git a/tests/unit_tests/commands/report/semantic_view_report_test.py
b/tests/unit_tests/commands/report/semantic_view_report_test.py
index f40eae3e405..d2172fa8579 100644
--- a/tests/unit_tests/commands/report/semantic_view_report_test.py
+++ b/tests/unit_tests/commands/report/semantic_view_report_test.py
@@ -103,6 +103,7 @@ def semantic_report_state(
mocker.patch.dict(
app.config,
{
+ "SEMANTIC_LAYER_CACHE_NAMESPACE": "semantic-report-tests",
"ALERT_REPORTS_EXECUTORS": [FixedExecutor(username=user.username)],
"THUMBNAIL_EXECUTORS": [FixedExecutor(username=user.username)],
},
diff --git a/tests/unit_tests/common/test_query_context_processor.py
b/tests/unit_tests/common/test_query_context_processor.py
index 752d1f2b1c1..1af98edec69 100644
--- a/tests/unit_tests/common/test_query_context_processor.py
+++ b/tests/unit_tests/common/test_query_context_processor.py
@@ -23,6 +23,8 @@ from unittest.mock import MagicMock, patch, PropertyMock
import numpy as np
import pandas as pd
import pytest
+from flask import Flask
+from pytest_mock import MockerFixture
from superset.common.chart_data import ChartDataResultFormat,
ChartDataResultType
from superset.common.chart_data_timing import QueryDataResult, QueryTiming
@@ -37,6 +39,14 @@ from superset.utils.core import GenericDataType
from superset.utils.date_parser import get_past_or_future
[email protected]
+def semantic_cache_namespace(app: Flask, mocker: MockerFixture) -> None:
+ """Isolate transient semantic datasource cache tests from metadata
storage."""
+ mocker.patch.dict(
+ app.config, {"SEMANTIC_LAYER_CACHE_NAMESPACE": "query-context-tests"}
+ )
+
+
@pytest.fixture
def mock_query_context():
with patch(
@@ -2982,6 +2992,7 @@ def
test_completeness_error_escapes_failed_payload_conversion(
def test_semantic_annotation_generation_versions_sql_parent(
processor: QueryContextProcessor,
+ semantic_cache_namespace: None,
) -> None:
from superset.semantic_layers.models import SemanticLayer, SemanticView
@@ -3046,6 +3057,7 @@ def
test_completeness_annotation_failure_preserves_subtype(app_context: Any) ->
def test_data_generation_separates_late_legacy_writer_and_task_keys(
processor: QueryContextProcessor,
+ semantic_cache_namespace: None,
) -> None:
from unittest.mock import PropertyMock
@@ -3121,6 +3133,7 @@ def
test_get_viz_annotation_data_ignores_source_chart_annotations(
def test_semantic_annotation_null_protocol_versions_parent_cache(
app_context: Any,
monkeypatch: pytest.MonkeyPatch,
+ semantic_cache_namespace: None,
) -> None:
"""Saved semantic annotation rows must not reuse pre-NULL-fix parent
keys."""
from superset.common.query_context import QueryContext
diff --git a/tests/unit_tests/datasource/test_compatible_api.py
b/tests/unit_tests/datasource/test_compatible_api.py
index 4210d2d5876..46e5ff21186 100644
--- a/tests/unit_tests/datasource/test_compatible_api.py
+++ b/tests/unit_tests/datasource/test_compatible_api.py
@@ -22,6 +22,7 @@ from uuid import uuid4
import pyarrow as pa
import pytest
+from flask import Flask
from flask.testing import FlaskClient
from pytest_mock import MockerFixture
from superset_core.semantic_layers.types import Dimension, Grains, Metric
@@ -33,9 +34,16 @@ from superset.semantic_layers.models import SemanticLayer,
SemanticView
@pytest.mark.parametrize("ambiguous", [False, True])
def test_compatible_endpoint_resolves_semantic_variants(
- client: FlaskClient, full_api_access: None, mocker: MockerFixture,
ambiguous: bool
+ app: Flask,
+ client: FlaskClient,
+ full_api_access: None,
+ mocker: MockerFixture,
+ ambiguous: bool,
) -> None:
"""The public endpoint uses the real model resolver on cache misses."""
+ mocker.patch.dict(
+ app.config, {"SEMANTIC_LAYER_CACHE_NAMESPACE":
"compatible-endpoint-tests"}
+ )
raw: Dimension = Dimension("raw", "event_time", pa.timestamp("us"))
month: Dimension = Dimension(
"month", "event_time", pa.timestamp("us"), grain=Grains.MONTH
diff --git a/tests/unit_tests/security/semantic_guest_rls_test.py
b/tests/unit_tests/security/semantic_guest_rls_test.py
index 9f1cbf6633f..846715199cc 100644
--- a/tests/unit_tests/security/semantic_guest_rls_test.py
+++ b/tests/unit_tests/security/semantic_guest_rls_test.py
@@ -23,7 +23,7 @@ from unittest.mock import MagicMock, PropertyMock
import pandas as pd
import pyarrow as pa
import pytest
-from flask import g
+from flask import Flask, g
from flask.testing import FlaskClient
from pytest_mock import MockerFixture
from sqlalchemy import create_engine, literal, select, union_all
@@ -62,8 +62,9 @@ def set_guest(rules: list[GuestTokenRlsRule]) -> None:
@pytest.fixture
-def provider(mocker: MockerFixture) -> MagicMock:
+def provider(app: Flask, mocker: MockerFixture) -> MagicMock:
"""The provider returns rows unless the request is refused."""
+ mocker.patch.dict(app.config, {"SEMANTIC_LAYER_CACHE_NAMESPACE":
"guest-rls-tests"})
implementation: MagicMock = MagicMock()
implementation.features = frozenset()
# An unversioned view, so value suggestions are served normally.
diff --git a/tests/unit_tests/semantic_layers/integer_precision_test.py
b/tests/unit_tests/semantic_layers/integer_precision_test.py
index 8929d0c14be..e7343ca71de 100644
--- a/tests/unit_tests/semantic_layers/integer_precision_test.py
+++ b/tests/unit_tests/semantic_layers/integer_precision_test.py
@@ -22,6 +22,7 @@ from unittest.mock import MagicMock, PropertyMock
import pyarrow as pa
import pytest
+from flask import Flask
from flask.testing import FlaskClient
from pytest_mock import MockerFixture
from superset_core.semantic_layers.types import (
@@ -37,8 +38,11 @@ from superset.semantic_layers.models import SemanticLayer,
SemanticView
@pytest.fixture
-def semantic_layer(mocker: MockerFixture) -> SemanticLayer:
+def semantic_layer(app: Flask, mocker: MockerFixture) -> SemanticLayer:
"""Give chart-data views the registered layer required by cache keys."""
+ mocker.patch.dict(
+ app.config, {"SEMANTIC_LAYER_CACHE_NAMESPACE":
"integer-precision-tests"}
+ )
mocker.patch.dict(
"superset.semantic_layers.models.registry",
{"fixture": MagicMock(result_cache_version=None)},
diff --git a/tests/unit_tests/semantic_layers/metadata_namespace_test.py
b/tests/unit_tests/semantic_layers/metadata_namespace_test.py
new file mode 100644
index 00000000000..deeb308c50c
--- /dev/null
+++ b/tests/unit_tests/semantic_layers/metadata_namespace_test.py
@@ -0,0 +1,422 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import annotations
+
+from collections.abc import Callable, Hashable, Iterator
+from datetime import timedelta
+from pathlib import Path
+from sqlite3 import Connection as SQLiteConnection
+from typing import Any
+from unittest.mock import MagicMock
+from uuid import UUID
+
+import pandas as pd
+import pytest
+from flask import Flask, Response
+from flask_caching import Cache
+from pytest_mock import MockerFixture
+from sqlalchemy import create_engine, event, func, select
+from sqlalchemy.engine import Engine, make_url
+from sqlalchemy.orm import Session
+from sqlalchemy.pool import NullPool, QueuePool, SingletonThreadPool,
StaticPool
+
+from superset.charts.data.api import ChartDataRestApi
+from superset.commands.chart.data.get_data_command import ChartDataCommand
+from superset.common.chart_data import ChartDataResultFormat,
ChartDataResultType
+from superset.common.query_context import QueryContext
+from superset.common.query_object import QueryObject
+from superset.common.utils.query_cache_manager import QueryCacheManager
+from superset.constants import CacheRegion
+from superset.key_value.models import KeyValueEntry
+from superset.key_value.types import (
+ JsonKeyValueCodec,
+ KeyValueResource,
+)
+from superset.models.helpers import QueryResult
+from superset.semantic_layers.execution import _install_namespace
+from superset.semantic_layers.models import SemanticLayer, SemanticView
+from tests.unit_tests.semantic_layers.conftest import LegacyLayer
+
+
[email protected]
+def metadata_engines(tmp_path: Path) -> Iterator[Callable[[str, str], Engine]]:
+ """Real isolated stores with synthetic transport aliases, never network
URLs."""
+ engines: list[Engine] = []
+
+ def create(store: str, address: str) -> Engine:
+ engine: Engine = create_engine(f"sqlite:///{tmp_path / store}")
+ KeyValueEntry.__table__.create(engine, checkfirst=True)
+ # The SQLite pool still reaches the file; only address metadata varies.
+ engine.url = make_url(address)
+ engines.append(engine)
+ return engine
+
+ yield create
+ for engine in engines:
+ engine.dispose()
+
+
[email protected]
+def namespace_app(app: Flask, mocker: MockerFixture) -> Flask:
+ """Do not let an app-global identity hide a metadata-bind isolation
defect."""
+ mocker.patch.dict(app.config, {"SEMANTIC_LAYER_CACHE_NAMESPACE": ""})
+ mocker.patch.dict(app.extensions, {"semantic_execution_namespace":
"wrong-store"})
+ mocker.patch.dict(
+ "superset.semantic_layers.models.registry", {"legacy": LegacyLayer}
+ )
+ mocker.patch(
+
"superset.common.query_context_processor.security_manager.get_rls_cache_key",
+ return_value=[],
+ )
+ return app
+
+
+def context_for_session(session: Session) -> QueryContext:
+ """Use identical layer/view identities and real query-key composition."""
+ layer: SemanticLayer = SemanticLayer(
+ uuid=UUID(int=1), type="legacy", configuration="{}", cache_version=0
+ )
+ session.add(layer)
+ view: SemanticView = SemanticView(
+ id=1, uuid=UUID(int=2), name="legacy", semantic_layer=layer,
configuration="{}"
+ )
+ return QueryContext(
+ datasource=view,
+ queries=[QueryObject(columns=[], metrics=[])],
+ slice_=None,
+ form_data=None,
+ result_type=ChartDataResultType.FULL,
+ result_format=ChartDataResultFormat.JSON,
+ cache_values={},
+ )
+
+
[email protected](
+ "address",
+ [
+ "postgresql://web:old@alias:5432/workspace",
+ "postgresql://web:old@metadata/workspace",
+ "postgresql://web:old@pgbouncer:6432/workspace",
+ "postgresql://web:old@metadata:5432/workspace_alias",
+
"postgresql+psycopg2://worker:new@metadata:5432/workspace?application_name=task",
+ ],
+)
+def test_metadata_aliases_share_result_cache(
+ namespace_app: Flask,
+ metadata_engines: Callable[[str, str], Engine],
+ mocker: MockerFixture,
+ address: str,
+) -> None:
+ """Worker DATA writes are readable by an independent web metadata
session."""
+ worker: Engine = metadata_engines(
+ "workspace.db", "postgresql://web:old@metadata:5432/workspace"
+ )
+ web: Engine = metadata_engines("workspace.db", address)
+ backend: Cache = Cache(namespace_app, config={"CACHE_TYPE": "SimpleCache"})
+ mocker.patch.dict(
+ "superset.common.utils.query_cache_manager._cache", {CacheRegion.DATA:
backend}
+ )
+ with Session(worker) as worker_session, Session(web) as web_session:
+ worker_context: QueryContext = context_for_session(worker_session)
+ worker_key: str | None = worker_context.query_cache_key(
+ worker_context.queries[0]
+ )
+ assert worker_key
+ cached: QueryCacheManager = QueryCacheManager()
+ cached.set_query_result(
+ worker_key,
+ QueryResult(
+ df=pd.DataFrame({"value": [7]}),
+ query="provider result",
+ duration=timedelta(0),
+ ),
+ region=CacheRegion.DATA,
+ timeout=600,
+ )
+ assert cached.result_persisted
+ mocker.patch.dict(namespace_app.config, {"SECRET_KEY":
"rotated-test-key"})
+ web_context: QueryContext = context_for_session(web_session)
+ web_key: str | None =
web_context.query_cache_key(web_context.queries[0])
+ api: ChartDataRestApi = ChartDataRestApi()
+ submit: MagicMock = mocker.patch(
+ "superset.charts.data.api.submit_chart_data_query_tasks",
return_value={}
+ )
+ mocker.patch.object(
+ api, "_send_chart_response", return_value=Response(status=200)
+ )
+ response: Response = api._run_async({}, ChartDataCommand(web_context))
+ assert response.status_code == 200
+ submit.assert_not_called()
+ loaded: QueryCacheManager = QueryCacheManager.get(
+ web_key, region=CacheRegion.DATA, force_cached=True
+ )
+ assert loaded.is_cached
+ pd.testing.assert_frame_equal(loaded.df, pd.DataFrame({"value": [7]}))
+ assert web_key == worker_key
+ assert isinstance(worker_context.datasource, SemanticView)
+ assert isinstance(web_context.datasource, SemanticView)
+ assert worker_context.datasource.metadata_generation == (
+ web_context.datasource.metadata_generation
+ )
+ assert (
+ "semantic-null-filters-v1"
+ in web_context.datasource.get_extra_cache_keys({})
+ )
+
+
+def test_copied_layer_ids_in_distinct_stores_are_isolated(
+ namespace_app: Flask, metadata_engines: Callable[[str, str], Engine]
+) -> None:
+ """Sequential tenant sessions in one app cannot borrow its startup
identity."""
+ address: str = "postgresql://user:password@proxy/tenant"
+ first: Engine = metadata_engines("first.db", address)
+ second: Engine = metadata_engines("second.db", address)
+ with Session(first) as first_session, Session(second) as second_session:
+ first_context: QueryContext = context_for_session(first_session)
+ second_context: QueryContext = context_for_session(second_session)
+ assert first_context.query_cache_key(first_context.queries[0]) != (
+ second_context.query_cache_key(second_context.queries[0])
+ )
+
+
[email protected]("callable_override", [False, True])
+def test_explicit_namespace_needs_no_identity_store(
+ namespace_app: Flask, mocker: MockerFixture, callable_override: bool
+) -> None:
+ """Operator identity takes precedence even when the durable store is
unavailable."""
+ configured: str | Callable[[], str] = (
+ (lambda: "explicit-workspace") if callable_override else
"explicit-workspace"
+ )
+ mocker.patch.dict(
+ namespace_app.config, {"SEMANTIC_LAYER_CACHE_NAMESPACE": configured}
+ )
+ with Session() as session:
+ context: QueryContext = context_for_session(session)
+ assert context.query_cache_key(context.queries[0])
+
+
+def test_identity_is_memoized_without_committing_caller(
+ namespace_app: Flask, metadata_engines: Callable[[str, str], Engine]
+) -> None:
+ """Many layers share one identity lookup; caller work remains
uncommitted."""
+ engine: Engine = metadata_engines("memo.db", "sqlite:///synthetic-address")
+ statements: list[str] = []
+
+ def record(
+ connection: Any,
+ cursor: Any,
+ statement: str,
+ parameters: Any,
+ context: Any,
+ executemany: bool,
+ ) -> None:
+ statements.append(statement)
+
+ event.listen(engine, "before_cursor_execute", record)
+ try:
+ with Session(engine) as session:
+ first: QueryContext = context_for_session(session)
+ assert isinstance(first.datasource, SemanticView)
+ original: str = first.datasource.metadata_generation
+ statements.clear()
+ other: SemanticLayer = SemanticLayer(
+ uuid=UUID(int=3), type="legacy", configuration="{}",
cache_version=0
+ )
+ session.add(other)
+ assert other.metadata_generation != original
+ assert not statements
+ assert other in session.new
+ first.datasource.semantic_layer.cache_version = 1
+ assert first.datasource.metadata_generation == original
+ session.info.clear()
+ assert first.datasource.semantic_layer.metadata_generation !=
original
+ assert len(statements) == 1
+ finally:
+ event.remove(engine, "before_cursor_execute", record)
+
+
[email protected]("seed_identity", [True, False])
+def test_single_connection_pool_never_waits_for_itself(
+ namespace_app: Flask, tmp_path: Path, seed_identity: bool
+) -> None:
+ """Both existing identity reads and first-use bootstrap avoid pool
self-wait."""
+ engine: Engine = create_engine(
+ f"sqlite:///{tmp_path / 'single.db'}",
+ poolclass=QueuePool,
+ pool_size=1,
+ max_overflow=0,
+ pool_timeout=0.05,
+ )
+ try:
+ KeyValueEntry.__table__.create(engine)
+ expected: str | None = _install_namespace(engine) if seed_identity
else None
+ with Session(engine) as session:
+ session.connection()
+ context: QueryContext = context_for_session(session)
+ assert isinstance(context.datasource, SemanticView)
+ generation: str = context.datasource.metadata_generation
+ assert generation
+ assert context.datasource.semantic_layer in session.new
+ durable: str = _install_namespace(engine)
+ if expected is not None:
+ assert durable == expected
+ with Session(engine) as reader:
+ assert (
+
context_for_session(reader).datasource.get_extra_cache_keys({})[0]
+ == generation
+ )
+ finally:
+ engine.dispose()
+
+
+def test_nullpool_identity_read_adds_no_connections(
+ namespace_app: Flask, tmp_path: Path
+) -> None:
+ """Each request's already-open connection suffices for the identity
lookup."""
+ engine: Engine = create_engine(
+ f"sqlite:///{tmp_path / 'nullpool.db'}", poolclass=NullPool
+ )
+ connections: list[object] = []
+
+ def connected(dbapi_connection: object, record: object) -> None:
+ connections.append(dbapi_connection)
+
+ try:
+ KeyValueEntry.__table__.create(engine)
+ _install_namespace(engine)
+ event.listen(engine, "connect", connected)
+ for _ in range(3):
+ with Session(engine) as session:
+ session.connection()
+ assert
context_for_session(session).datasource.get_extra_cache_keys({})
+ assert len(connections) == 3
+ finally:
+ event.remove(engine, "connect", connected)
+ engine.dispose()
+
+
+def test_missing_identity_with_sqlite_write_fails_without_committing(
+ namespace_app: Flask, tmp_path: Path
+) -> None:
+ """Bootstrap must not wait on, or commit, the caller's SQLite write
lock."""
+ engine: Engine = create_engine(
+ f"sqlite:///{tmp_path / 'write-lock.db'}",
+ poolclass=QueuePool,
+ pool_size=1,
+ max_overflow=0,
+ pool_timeout=0.05,
+ )
+ try:
+ KeyValueEntry.__table__.create(engine)
+ with Session(engine) as session:
+ session.connection().execute(
+ KeyValueEntry.__table__.insert().values(
+ uuid=UUID(int=20),
+ resource=KeyValueResource.SEMANTIC_EXECUTION.value,
+ value=JsonKeyValueCodec().encode({"pending": True}),
+ )
+ )
+ context: QueryContext = context_for_session(session)
+ with pytest.raises(
+ ValueError, match="before starting a SQLite transaction"
+ ):
+ context.datasource.get_extra_cache_keys({})
+ session.rollback()
+ with engine.connect() as connection:
+ assert (
+
connection.scalar(select(func.count()).select_from(KeyValueEntry)) == 0
+ )
+ _install_namespace(engine)
+ with Session(engine) as session:
+ assert
context_for_session(session).datasource.get_extra_cache_keys({})
+ finally:
+ engine.dispose()
+
+
[email protected]("poolclass", [SingletonThreadPool, StaticPool])
+def test_missing_identity_in_memory_preserves_store(
+ namespace_app: Flask, poolclass: type[SingletonThreadPool] |
type[StaticPool]
+) -> None:
+ """First use retains the in-memory store and does not flush caller
objects."""
+ engine: Engine = create_engine("sqlite://", poolclass=poolclass)
+ try:
+ KeyValueEntry.__table__.create(engine)
+ with Session(engine) as writer:
+ writer.connection().execute(
+ KeyValueEntry.__table__.insert().values(
+ uuid=UUID(int=20),
+ resource=KeyValueResource.SEMANTIC_EXECUTION.value,
+ value=JsonKeyValueCodec().encode({"pending": True}),
+ )
+ )
+ with pytest.raises(
+ ValueError, match="before starting a SQLite transaction"
+ ):
+ context_for_session(writer).datasource.get_extra_cache_keys({})
+ writer.rollback()
+ with Session(engine) as session:
+ assert
session.scalar(select(func.count()).select_from(KeyValueEntry)) == 0
+ context: QueryContext = context_for_session(session)
+ generation: Hashable =
context.datasource.get_extra_cache_keys({})[0]
+ assert generation
+ assert isinstance(context.datasource, SemanticView)
+ assert context.datasource.semantic_layer in session.new
+ session.rollback()
+ with Session(engine) as reader:
+ assert
reader.scalar(select(func.count()).select_from(KeyValueEntry)) == 1
+ assert (
+
context_for_session(reader).datasource.get_extra_cache_keys({})[0]
+ == generation
+ )
+ finally:
+ engine.dispose()
+
+
+def test_missing_identity_uses_metadata_schema(
+ namespace_app: Flask, tmp_path: Path
+) -> None:
+ """Bootstrap writes the configured schema, never an available default
table."""
+ engine: Engine = create_engine(f"sqlite:///{tmp_path / 'main.db'}")
+
+ def attach_schema(connection: SQLiteConnection, record: object) -> None:
+ connection.execute(
+ "ATTACH DATABASE ? AS tenant", (str(tmp_path / "tenant.db"),)
+ )
+
+ event.listen(engine, "connect", attach_schema)
+ tenant: Engine = engine.execution_options(schema_translate_map={None:
"tenant"})
+ try:
+ KeyValueEntry.__table__.create(engine)
+ KeyValueEntry.__table__.create(tenant)
+ with Session(tenant) as session:
+ generation: Hashable = context_for_session(
+ session
+ ).datasource.get_extra_cache_keys({})[0]
+ with Session(tenant) as reader:
+ assert
reader.scalar(select(func.count()).select_from(KeyValueEntry)) == 1
+ assert (
+
context_for_session(reader).datasource.get_extra_cache_keys({})[0]
+ == generation
+ )
+ with Session(engine) as default_schema:
+ assert (
+
default_schema.scalar(select(func.count()).select_from(KeyValueEntry))
+ == 0
+ )
+ finally:
+ engine.dispose()
diff --git a/tests/unit_tests/semantic_layers/models_test.py
b/tests/unit_tests/semantic_layers/models_test.py
index d2d5240b085..7e0c6d04eaf 100644
--- a/tests/unit_tests/semantic_layers/models_test.py
+++ b/tests/unit_tests/semantic_layers/models_test.py
@@ -21,7 +21,7 @@ from __future__ import annotations
import os
import uuid
-from collections.abc import Hashable
+from collections.abc import Hashable, Iterator
from datetime import date, datetime, timezone
from itertools import permutations
from pathlib import Path
@@ -30,6 +30,7 @@ from unittest.mock import MagicMock, patch, PropertyMock
import pyarrow as pa
import pytest
+from flask import Flask
from flask.testing import FlaskClient
from pytest_mock import MockerFixture
from sqlalchemy import inspect, select
@@ -70,6 +71,19 @@ from superset.utils.core import GenericDataType
# =============================================================================
[email protected]
+def explicit_model_namespace(app: Flask, mocker: MockerFixture) ->
Iterator[None]:
+ """Give transient model tests an identity without a metadata-store
fixture."""
+ mocker.patch.dict(
+ app.config, {"SEMANTIC_LAYER_CACHE_NAMESPACE": "model-unit-tests"}
+ )
+ durable_lookup: MagicMock = mocker.patch(
+ "superset.semantic_layers.models.metadata_namespace"
+ )
+ yield
+ durable_lookup.assert_not_called()
+
+
@pytest.mark.parametrize(
"grains, expected",
[
@@ -358,7 +372,9 @@ def
test_semantic_layer_implementation_is_scoped_to_owning_session(
assert factory.call_count == 2
-def test_semantic_layer_implementation(mocker: MockerFixture) -> None:
+def test_semantic_layer_implementation(
+ mocker: MockerFixture, explicit_model_namespace: None
+) -> None:
"""Test that implementation returns a configured semantic layer."""
from superset.semantic_layers.execution import
current_host_execution_context
@@ -607,7 +623,9 @@ def test_semantic_view_query_endpoint_returns_error(
@pytest.mark.parametrize("version", [None, "producer-v1", "producer-v2"])
-def test_semantic_view_get_extra_cache_keys(version: str | None) -> None:
+def test_semantic_view_get_extra_cache_keys(
+ version: str | None, explicit_model_namespace: None
+) -> None:
"""Keep NULL normalization and producer guarantees in the result
identity."""
from superset.common.query_object import QueryObject
@@ -1446,7 +1464,9 @@ def test_semantic_view_explore_url() -> None:
)
-def test_semantic_view_implementation(mocker: MockerFixture) -> None:
+def test_semantic_view_implementation(
+ mocker: MockerFixture, explicit_model_namespace: None
+) -> None:
"""Views share the layer binding and canonicalize equivalent
configuration."""
from superset.semantic_layers.execution import SemanticBindingMemo
@@ -2884,6 +2904,7 @@ def
test_values_search_translates_provider_completeness_error(
@pytest.mark.parametrize("version", [None, "metricflow-completeness-v1"])
def test_result_generation_reads_class_without_provider_construction(
version: str | None,
+ explicit_model_namespace: None,
) -> None:
provider: MagicMock = MagicMock()
provider.result_cache_version = version
diff --git a/tests/unit_tests/semantic_layers/simple_cache_clear_test.py
b/tests/unit_tests/semantic_layers/simple_cache_clear_test.py
index 59a4002ef4c..8b71bf73231 100644
--- a/tests/unit_tests/semantic_layers/simple_cache_clear_test.py
+++ b/tests/unit_tests/semantic_layers/simple_cache_clear_test.py
@@ -25,6 +25,7 @@ from uuid import uuid4
import pytest
from flask import Flask
from sqlalchemy import select
+from sqlalchemy.engine import Engine
from sqlalchemy.orm import Session
from superset_core.semantic_layers.context import ExecutionMethod
@@ -403,7 +404,7 @@ def
test_migration_backfills_existing_layers_and_downgrades() -> None:
from alembic.migration import MigrationContext
from alembic.operations import Operations
from sqlalchemy import create_engine, text
- from sqlalchemy.engine import Connection, Engine
+ from sqlalchemy.engine import Connection
migration: Any = import_module(
"superset.migrations.versions.2026-10-08_00-00_c179af013f48_add_semantic_layer_cache_version"
@@ -542,15 +543,18 @@ def
test_web_and_task_share_metadata_keys_despite_local_credentials(
from sqlalchemy.engine import make_url
monkeypatch.setitem(app.config, "SEMANTIC_LAYER_CACHE_NAMESPACE",
namespace)
- bind: Mock = Mock()
- bind.engine.url = make_url(
-
"postgresql+psycopg2://web:old@metadata:5432/superset?application_name=web"
- )
+ bind: Engine = session.get_bind().engine
view: SemanticView = SemanticView(semantic_layer=layer)
- with patch.object(session, "get_bind", return_value=bind):
+ with patch.object(
+ bind,
+ "url",
+ make_url(
+
"postgresql+psycopg2://web:old@metadata:5432/superset?application_name=web"
+ ),
+ ):
web_key: list[Hashable] = view.get_extra_cache_keys({})
session.info.clear()
- bind.engine.url = make_url(
+ bind.url = make_url(
"postgresql://worker:new@metadata:5432/superset?application_name=celery"
)
monkeypatch.setitem(app.config, "SECRET_KEY", "rotated-for-task")
diff --git a/tests/unit_tests/semantic_layers/values_endpoint_test.py
b/tests/unit_tests/semantic_layers/values_endpoint_test.py
index ff5358e85c3..43bcbfce650 100644
--- a/tests/unit_tests/semantic_layers/values_endpoint_test.py
+++ b/tests/unit_tests/semantic_layers/values_endpoint_test.py
@@ -23,13 +23,18 @@ header contract.
"""
from datetime import datetime
+from pathlib import Path
from typing import Any, cast
from unittest.mock import MagicMock
import pyarrow as pa
import pytest
+from flask import Flask
from flask.testing import FlaskClient
from pytest_mock import MockerFixture
+from sqlalchemy import create_engine, select
+from sqlalchemy.engine import Engine
+from sqlalchemy.orm import Session
from superset_core.semantic_layers.types import (
Dimension,
SemanticRequest,
@@ -37,11 +42,16 @@ from superset_core.semantic_layers.types import (
)
from werkzeug.test import TestResponse
+from superset.key_value.models import KeyValueEntry
from superset.semantic_layers.models import SemanticLayer, SemanticView
@pytest.fixture
-def semantic_view_datasource(mocker: MockerFixture) -> SemanticView:
+def semantic_view_datasource(app: Flask, mocker: MockerFixture) ->
SemanticView:
+ """Give the transient datasource an explicit, isolated cache namespace."""
+ mocker.patch.dict(
+ app.config, {"SEMANTIC_LAYER_CACHE_NAMESPACE": "values-endpoint-tests"}
+ )
implementation: MagicMock = MagicMock(selection_identity_version=None)
implementation.uid.return_value = "semantic_view_uid_123"
implementation.get_dimensions.return_value = [
@@ -100,6 +110,40 @@ def test_semantic_view_values_endpoint_returns_values(
assert cached.headers["X-Cache-Status"] == "HIT"
+def test_values_endpoint_bootstraps_identity_without_flushing_datasource(
+ app: Flask,
+ client: FlaskClient,
+ full_api_access: None,
+ semantic_view_datasource: SemanticView,
+ mocker: MockerFixture,
+ tmp_path: Path,
+) -> None:
+ """The default request path uses a migrated store, not a fixture
override."""
+ mocker.patch.dict(app.config, {"SEMANTIC_LAYER_CACHE_NAMESPACE": ""})
+ mocker.patch(
+ "superset.datasource.api.cache_manager"
+ ).data_cache.get.return_value = None
+ engine: Engine = create_engine(f"sqlite:///{tmp_path / 'metadata.db'}")
+ try:
+ KeyValueEntry.__table__.create(engine)
+ with Session(engine) as session:
+ session.add(semantic_view_datasource.semantic_layer)
+ response: TestResponse = _get(client, "category/values/")
+ assert response.status_code == 200
+ assert response.json is not None
+ assert response.json["result"] == ["Books", "Clothing"]
+ assert semantic_view_datasource.semantic_layer in session.new
+ identity: bytes | None = session.connection().scalar(
+ select(KeyValueEntry.value)
+ )
+ assert identity is not None
+ # Closing the caller session rolls back its pending work, not the
identity.
+ with engine.connect() as connection:
+ assert connection.scalar(select(KeyValueEntry.value)) == identity
+ finally:
+ engine.dispose()
+
+
def test_semantic_view_values_endpoint_passes_search_to_the_provider(
client: Any,
full_api_access: None,
