codeant-ai-for-open-source[bot] commented on code in PR #44849: URL: https://github.com/apache/superset/pull/44849#discussion_r4159833739
########## tests/unit_tests/semantic_layers/metadata_result_inspection_test.py: ########## @@ -0,0 +1,182 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +from unittest.mock import Mock, patch + +import pytest +from flask import Flask + +from superset.common.query_context import QueryContext +from superset.common.query_object import QueryObject +from superset.semantic_layers.cache_inspection import CacheEntryInfo +from superset.semantic_layers.models import SemanticView +from superset.semantic_layers.registry import registry +from tests.unit_tests.semantic_layers.metadata_identity_test import ( + context_for, + RefreshLayer, + ResultView, + view_for, +) + + +def test_result_inspection_keeps_query_rls_and_never_constructs_provider( + app: Flask, monkeypatch: pytest.MonkeyPatch +) -> None: + from superset.commands.semantic_layer.inspect_query_result import ( + InspectQueryResultCommand, + ) + + provider: ResultView = ResultView("unused", 17) + view: SemanticView = view_for(provider) + monkeypatch.setitem(app.config, "SEMANTIC_LAYER_METADATA_REFRESH_ENABLED", True) + monkeypatch.setitem(registry, "cache-test", RefreshLayer) + context: QueryContext + query: QueryObject + context, query = context_for(view) + manager: Mock = Mock() + keys: list[str] = [] + inspect_entry: Mock + construct: Mock + rls: list[str] + with ( + app.test_request_context(), + patch( + "superset.semantic_layers.metadata_binding.is_feature_enabled", + return_value=True, + ), + patch( + "superset.commands.semantic_layer.inspect_query_result.security_manager", + manager, + ), + patch("superset.semantic_layers.result_inspection.security_manager", manager), + patch("superset.common.query_context_processor.security_manager", manager), + patch( + "superset.commands.semantic_layer.inspect_query_result.inspect_derived_entry" + ) as inspect_entry, + patch( + "superset.semantic_layers.metadata_binding.view_implementation", + return_value=provider, + ) as construct, + ): + assert InspectQueryResultCommand(context, 0).run().state == "unsupported" + construct.assert_not_called() + for rls in (["rule-a"], ["rule-b"]): + manager.get_rls_cache_key.return_value = rls + key: str | None = context.query_cache_key(query) + construct.reset_mock() + InspectQueryResultCommand(context, 0).run() + assert inspect_entry.call_args.args == (key, "query_result") + keys.append(inspect_entry.call_args.args[0]) + construct.assert_not_called() + assert keys[0] != keys[1] + manager.get_rls_cache_key.return_value = ["revoked"] + assert InspectQueryResultCommand(context, 0).run().state == "unsupported" + manager.raise_for_access.side_effect = PermissionError("denied") + inspect_entry.reset_mock() + with pytest.raises(PermissionError): + InspectQueryResultCommand(context, 0).run() + inspect_entry.assert_not_called() + + # A new request cannot reuse identities kept by an earlier request, even + # when an internal caller retains the same Python context object. + with ( + app.test_request_context(), + patch( + "superset.commands.semantic_layer.inspect_query_result.security_manager", + Mock(), + ), + ): + assert InspectQueryResultCommand(context, 0).run().state == "unsupported" + + +def test_result_inspection_does_not_refill_after_concurrent_invalidation( + app: Flask, + monkeypatch: pytest.MonkeyPatch, +) -> None: + from superset.commands.semantic_layer.inspect_query_result import ( + InspectQueryResultCommand, + ) + from superset.semantic_layers.metadata import ScopedMetadataStore + from superset.semantic_layers.metadata_binding import ( + operation_deadline, + request_metadata_budget, + ) + from tests.unit_tests.semantic_layers.metadata_contract_test import OptedInLayer + from tests.unit_tests.semantic_layers.metadata_store_test import MemoryBackend + + manager: Mock = Mock() + provider: OptedInLayer = OptedInLayer() + resolve_provider: Mock + view: SemanticView = view_for(ResultView("unused", 17)) + context: QueryContext + query: QueryObject + context, query = context_for(view) + monkeypatch.setitem(app.config, "SEMANTIC_LAYER_METADATA_REFRESH_ENABLED", True) + monkeypatch.setitem( + app.config, "SEMANTIC_LAYER_METADATA_NAMESPACE", "inspection-test" + ) + monkeypatch.setitem(registry, "cache-test", OptedInLayer) + with ( + app.test_request_context(), + patch( + "superset.semantic_layers.metadata_binding.is_feature_enabled", + return_value=True, + ), + patch( + "superset.commands.semantic_layer.inspect_query_result.security_manager", + manager, + ), + patch( + "superset.semantic_layers.metadata_binding.connection_store" + ) as resolve_provider, + patch( + "superset.commands.semantic_layer.inspect_query_result.inspect_derived_entry" + ), + patch.object(OptedInLayer, "from_configuration", return_value=provider), + ): + manager.get_rls_cache_key.return_value = [] + request_metadata_budget() + deadline: float = operation_deadline() + store: ScopedMetadataStore = ScopedMetadataStore( + MemoryBackend(), "inspection", deadline=deadline + ) + store.read(lambda budget: '["orders"]', deadline=deadline) + # Catalog was visible to the diagnostic, but another authorized caller + # retired it before this request resolves the provider-defined uid. + store.invalidate_catalog() + resolve_provider.return_value = store + result: CacheEntryInfo = InspectQueryResultCommand(context, 0).run() + assert result.state == "unsupported" + assert provider.adapter.fetches == 0 + resolve_provider.assert_not_called() Review Comment: ✅ **CodeAnt verified this suggestion was addressed in subsequent commits and marked this thread resolved** as of `ac2497f`. The test now captures and asserts a non-null query cache key before invalidating the catalog, then verifies inspection uses that key without resolving a new provider. <sub>If that's not right, unresolve this thread and CodeAnt will leave it open.</sub> <!-- codeant-auto-resolve-reply --> ########## superset/semantic_layers/metadata_binding.py: ########## @@ -0,0 +1,227 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Host construction and operation lifetime for optional shared metadata.""" + +from __future__ import annotations + +import math +import time +from collections.abc import Iterator +from contextlib import contextmanager +from contextvars import ContextVar, Token +from dataclasses import dataclass, field +from typing import Any, TYPE_CHECKING + +from flask import current_app, has_app_context, has_request_context, request +from sqlalchemy.orm import Session +from superset_core.semantic_layers.layer import SemanticLayer as LayerABC +from superset_core.semantic_layers.metadata import ( + MetadataRefreshError, + remaining_budget, +) +from superset_core.semantic_layers.view import SemanticView as ViewABC + +from superset import db, is_feature_enabled +from superset.coordination.deadline_backend import DeadlineRedisBackend +from superset.semantic_layers.metadata import ( + FETCH_DEADLINE_SECONDS, + metadata_scope, + ScopedMetadataStore, +) +from superset.semantic_layers.registry import registry +from superset.utils import json + +if TYPE_CHECKING: + from superset.semantic_layers.models import SemanticLayer, SemanticView + + +@dataclass +class MetadataOperation: + """One request or worker operation; nested discovery shares its deadline.""" + + deadline: float + layers: dict[str, LayerABC[Any, ViewABC]] = field(default_factory=dict) + views: dict[tuple[str, str, str], ViewABC] = field(default_factory=dict) + stores: dict[str, ScopedMetadataStore] = field(default_factory=dict) + + +_OPERATION_KEY: str = "superset.semantic_metadata.operation" +_worker_operation: ContextVar[MetadataOperation | None] = ContextVar( + _OPERATION_KEY, default=None +) + + +def request_metadata_budget() -> None: + """Register before authentication hooks; this performs no provider or cache I/O.""" + if current_app.config.get("SEMANTIC_LAYER_METADATA_REFRESH_ENABLED") is True: + request.environ.setdefault( + _OPERATION_KEY, MetadataOperation(time.monotonic() + FETCH_DEADLINE_SECONDS) + ) + + +def _current_operation() -> MetadataOperation | None: + if has_request_context(): + return request.environ.get(_OPERATION_KEY) or _worker_operation.get() + return _worker_operation.get() + + +def _operation(*, require_budget: bool = True) -> MetadataOperation: + state: MetadataOperation | None = _current_operation() + if state is None or not math.isfinite(state.deadline): + raise MetadataRefreshError("configuration") + if require_budget: + remaining_budget(state.deadline, now=time.monotonic()) + return state + + +def operation_deadline() -> float: + return _operation().deadline + + +@contextmanager +def metadata_operation(*, deadline: float | None = None) -> Iterator[None]: + """Workers opt in before access checks; nested calls never replenish the budget.""" + if deadline is not None and not math.isfinite(deadline): + raise MetadataRefreshError("configuration") + if deadline is not None: + remaining_budget(deadline, now=time.monotonic()) + if _current_operation() is not None: + _operation(require_budget=False) + yield + return + if has_request_context(): + # HTTP requests must enter through the registered early request hook. + raise MetadataRefreshError("configuration") + ceiling: float = time.monotonic() + FETCH_DEADLINE_SECONDS + state: MetadataOperation = MetadataOperation( + min(deadline, ceiling) if deadline is not None else ceiling + ) + token: Token[MetadataOperation | None] = _worker_operation.set(state) + try: + operation_deadline() + yield + finally: + _worker_operation.reset(token) + + +def metadata_refresh_enabled() -> bool: + return ( + has_app_context() + and current_app.config.get("SEMANTIC_LAYER_METADATA_REFRESH_ENABLED") is True + and is_feature_enabled("SEMANTIC_LAYERS") + ) + + +def participates(layer: SemanticLayer) -> bool: + return metadata_refresh_enabled() and registry[ + layer.type + ].supports_metadata_refresh(json.loads(layer.configuration)) Review Comment: ✅ **CodeAnt verified this suggestion was addressed in subsequent commits and marked this thread resolved** as of `ac2497f`. `participates` now validates that the parsed configuration is a dictionary and converts unknown-provider, type, and JSON/value parsing failures into `MetadataRefreshError("configuration")`. <sub>If that's not right, unresolve this thread and CodeAnt will leave it open.</sub> <!-- codeant-auto-resolve-reply --> -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
