rebenitez1802 commented on code in PR #45357: URL: https://github.com/apache/superset/pull/45357#discussion_r4328438595
########## superset/mcp_service/dashboard/filter_values.py: ########## @@ -0,0 +1,1241 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Authorized, bounded native dropdown discovery, independent of chart scope.""" + +from __future__ import annotations + +import logging +import math +import re +import time +from collections.abc import Iterator +from contextlib import contextmanager +from copy import deepcopy +from dataclasses import dataclass, field +from decimal import Decimal +from numbers import Integral, Real +from typing import Any, cast, Literal, TYPE_CHECKING + +from fastmcp.exceptions import ToolError +from flask import current_app, g +from marshmallow import ValidationError +from pandas import DataFrame, NA +from sqlalchemy.exc import CompileError, UnsupportedCompilationError + +from superset.charts.data.form_data import set_query_context_form_data +from superset.charts.schemas import ChartDataQueryContextSchema +from superset.commands.chart.data.get_data_command import ChartDataCommand +from superset.constants import EMPTY_FILTER_SQL_EXPRESSION, NO_TIME_RANGE +from superset.daos.dataset import DatasetDAO +from superset.dashboards.filter_scope import _is_divider +from superset.dashboards.permalink.schemas import DashboardPermalinkStateSchema +from superset.exceptions import ( + OAuth2Error, + OAuth2RedirectError, + SupersetSecurityException, +) +from superset.mcp_service.chart.chart_helpers import ( + build_query_dicts_from_form_data, + merge_extra_form_data, +) +from superset.mcp_service.dashboard.schemas import ( + _extract_native_filters, + FilterValueDiscovery, + FilterValueReason, + FilterValueRequest, + FilterValueScalar, + GetDashboardInfoRequest, + NativeFilterAvailableValues, + NativeFilterValueSummary, + ProposedFilterValues, +) +from superset.mcp_service.dashboard.tool.manage_native_filters import ( + _empty_select_data_mask, + _FilterValidationError, + _select_data_mask, + current_native_filter_config, +) +from superset.mcp_service.privacy import user_can_view_data_model_metadata +from superset.utils import json +from superset.utils.core import apply_max_row_limit, FilterOperator, GenericDataType + +if TYPE_CHECKING: + from superset.connectors.sqla.models import SqlaTable, TableColumn + from superset.models.dashboard import Dashboard + +_MAX_QUERIES = 20 +_SCHEDULING_SECONDS = 10 +_PAGE_SIZE = 20 +_MAX_SAFE_INTEGER = 2**53 - 1 +# Mirrors the dashboard's extractUrlParams('regular') exclusions. +_RESERVED_URL_PARAMS = frozenset({"standalone", "edit"}) +logger = logging.getLogger(__name__) + + +class _DiscoveryUnavailableError(Exception): + """A bounded, non-sensitive failure for a single dropdown.""" + + def __init__(self, reason: FilterValueReason) -> None: + super().__init__(reason) + self.reason = reason + + +@dataclass +class _Target: + """A configured table target; this is metadata, not an access grant.""" + + dataset: SqlaTable + column: TableColumn + + +@dataclass +class _Discovery: + """Call-local configuration, hydrated state, and execution accounting.""" + + dashboard: Dashboard + configs: dict[str, dict[str, Any]] + masks: dict[str, dict[str, Any]] + started: float = field(default_factory=time.monotonic) + query_count: int = 0 + proposal_errors: dict[str, FilterValueReason] = field(default_factory=dict) + targets: dict[ + tuple[int, str], + _Target | _DiscoveryUnavailableError | SupersetSecurityException, + ] = field(default_factory=dict) + mask_error_reasons: dict[str, FilterValueReason] = field(default_factory=dict) + normalized_masks: set[str] = field(default_factory=set) + url_params: dict[str, str] = field(default_factory=dict) + + +def _check_budget(context: _Discovery) -> None: + """Stop preparation and authorization after a call reaches either budget.""" + if context.query_count >= _MAX_QUERIES: + raise _DiscoveryUnavailableError("query_budget") + if time.monotonic() - context.started >= _SCHEDULING_SECONDS: + raise _DiscoveryUnavailableError("scheduling_budget") + + +def _configured_filters(dashboard: Dashboard) -> list[dict[str, Any]]: + """Read saved definitions in order, excluding visual dividers.""" + return [c for c in current_native_filter_config(dashboard) if not _is_divider(c)] + + +def _select_target(conf: dict[str, Any], context: _Discovery) -> _Target: + """Memoize a native target's metadata preflight, never its final query gate.""" + _check_budget(context) + controls = conf.get("controlValues") or {} + if ( + conf.get("filterType") != "filter_select" + or not isinstance(controls, dict) + or controls.get("inverseSelection") + or controls.get("operatorType", "exact") != "exact" + ): + raise _DiscoveryUnavailableError("unsupported_filter") + targets = conf.get("targets") + if not isinstance(targets, list) or len(targets) != 1: + raise _DiscoveryUnavailableError("unsupported_target") + target = targets[0] + if ( + not isinstance(target, dict) + or target.get("datasourceType") not in (None, "table") + or target.get("semantic_selection_version") + or type(target.get("datasetId")) is not int + or target["datasetId"] <= 0 + or not isinstance(target.get("column"), dict) + or not isinstance(target["column"].get("name"), str) + or not target["column"]["name"] + ): + raise _DiscoveryUnavailableError("unsupported_target") + key = (target["datasetId"], conf["id"]) + cached = context.targets.get(key) + if isinstance(cached, Exception): + raise cached + if cached is not None: + return cached + dataset = DatasetDAO.find_by_id(target["datasetId"], skip_base_filter=True) + if dataset is None: + raise _DiscoveryUnavailableError("unsupported_target") + column = next( + (col for col in dataset.columns if col.column_name == target["column"]["name"]), + None, + ) + if column is None: + raise _DiscoveryUnavailableError("unsupported_target") + # Native authorization preserves dashboard-only grants and guest limits. + # This memoized context is never executed. Every final query is authorized + # separately with its complete predicates before dataframe cache access. + access_context = ChartDataQueryContextSchema().load( + { + "datasource": {"id": dataset.id, "type": "table"}, + "queries": [ + {"columns": [column.column_name], "metrics": [], "row_limit": 1} + ], + "form_data": { + "type": "NATIVE_FILTER", + "dashboardId": context.dashboard.id, + "native_filter_id": conf["id"], + }, + "result_type": "full", + "result_format": "json", + } + ) + _check_budget(context) + try: + ChartDataCommand(access_context).validate() + except SupersetSecurityException as ex: + context.targets[key] = ex + raise + if column.type_generic not in ( + GenericDataType.STRING, + GenericDataType.NUMERIC, + GenericDataType.BOOLEAN, + ): + error = _DiscoveryUnavailableError("unsupported_target") + context.targets[key] = error + raise error + resolved = _Target(dataset, column) + context.targets[key] = resolved + return resolved + + +def _compatible(value: FilterValueScalar, target: _Target) -> bool: + """Compare JSON scalar families, keeping booleans distinct from numbers.""" + if value is None: + return True + if type(value) not in (str, bool, int, float) or ( + isinstance(value, float) and not math.isfinite(value) + ): + return False + if isinstance(value, str): + return target.column.type_generic == GenericDataType.STRING + if type(value) is bool: + return target.column.type_generic == GenericDataType.BOOLEAN + return target.column.type_generic == GenericDataType.NUMERIC + + +def _ancestors(filter_id: str, configs: dict[str, dict[str, Any]]) -> list[str]: + """Resolve ancestors furthest first and reject missing or cyclic graphs.""" + ordered: list[str] = [] + active: set[str] = set() + visited: set[str] = set() + + def visit(current: str) -> None: + """Walk one configuration, retaining stable parent ordering.""" + if current in active or current not in configs: + raise _DiscoveryUnavailableError("parent_unresolved") + if current in visited: + return + active.add(current) + parents = configs[current].get("cascadeParentIds") or [] + if not isinstance(parents, list) or any( + not isinstance(p, str) for p in parents + ): + raise _DiscoveryUnavailableError("parent_unresolved") + for parent in parents: + visit(parent) + active.remove(current) + visited.add(current) + if current != filter_id: + ordered.append(current) + + visit(filter_id) + return ordered + + +def _requests( + configs: list[dict[str, Any]], + request: GetDashboardInfoRequest, +) -> list[FilterValueRequest]: + """Choose explicit IDs or a bounded page before querying any datasource.""" + if request.filter_value_requests is not None: + return request.filter_value_requests + return [ + FilterValueRequest(filter_id=c["id"]) + for c in configs[ + request.filter_values_offset : request.filter_values_offset + _PAGE_SIZE + ] + ] + + +def validate_discovery_configuration( # noqa: C901 + dashboard: Dashboard, + request: GetDashboardInfoRequest, + *, + started_at: float | None = None, +) -> _Discovery: + """Reject invalid IDs, proposed ancestors, and scalar types before querying.""" + configured = _configured_filters(dashboard) + configs: dict[str, dict[str, Any]] = { + c["id"]: c for c in configured if isinstance(c.get("id"), str) + } + if any(not isinstance(c.get("id"), str) or not c["id"] for c in configured) or len( + configs + ) != len(configured): + raise ValueError("Native filter configuration has invalid or duplicate IDs") + context = _Discovery( + dashboard, + configs, + {}, + started=time.monotonic() if started_at is None else started_at, + ) + requests = _requests(configured, request) + for item in requests: + if item.filter_id not in configs: + raise ValueError(f"Unknown native filter ID: {item.filter_id}") + if "exact_value" in item.model_fields_set: + try: + target = _select_target(configs[item.filter_id], context) + except (_DiscoveryUnavailableError, SupersetSecurityException): + continue + if not _compatible(item.exact_value, target): + raise ValueError( + f"Incompatible exact value for native filter {item.filter_id}" + ) + allowed: set[str] = set() + for item in requests: + try: + allowed.update(_ancestors(item.filter_id, configs)) + except _DiscoveryUnavailableError: + pending = [item.filter_id] + visited: set[str] = set() + while pending: + current = pending.pop() + if current in visited or current not in configs: + continue + visited.add(current) + parents = configs[current].get("cascadeParentIds") + if isinstance(parents, list): + pending.extend(p for p in parents if isinstance(p, str)) + allowed.update(visited - {item.filter_id}) + for proposal in request.proposed_filter_values or []: + if proposal.filter_id not in configs or proposal.filter_id not in allowed: + raise ValueError( + "Proposed filters must be configured ancestors of requested filters" + ) + conf = configs[proposal.filter_id] + controls = conf.get("controlValues") or {} + if ( + isinstance(controls, dict) + and controls.get("multiSelect") is False + and len(proposal.values) > 1 + ): + raise ValueError("A single-select proposal accepts at most one value") + try: + target = _select_target(conf, context) + except (_DiscoveryUnavailableError, SupersetSecurityException): + continue + if any(not _compatible(value, target) for value in proposal.values): + raise ValueError( + f"Incompatible proposed value for native filter {proposal.filter_id}" + ) + return context + + +def _real_value(value: Any) -> bool: + """Mirror fillNativeFilters' empty/all-null-array restoration test.""" + return value is not None and not ( + isinstance(value, list) and all(v is None for v in value) + ) + + +def _explicit_clear(mask: dict[str, Any]) -> bool: + """Recognize only the frontend's canonical match-nothing clear evidence.""" + extra = mask.get("extraFormData") or {} + if not isinstance(extra, dict): + return False + predicates = extra.get("adhoc_filters") + return isinstance(predicates, list) and any( + isinstance(p, dict) + and p.get("expressionType") == "SQL" + and p.get("clause") == "WHERE" + and p.get("sqlExpression") == EMPTY_FILTER_SQL_EXPRESSION + for p in predicates + ) + + +def _validate_snapshot_value(conf: dict[str, Any], value: Any) -> None: # noqa: C901 + """Reject malformed known native selection shapes before any option query.""" + if value is None: + return + kind = conf.get("filterType") + if kind in ("filter_select", "filter_range", "filter_timegrain") and not isinstance( + value, list + ): + raise _DiscoveryUnavailableError("state_invalid") + if kind == "filter_range": + if len(value) != 2 or any( + item is not None and type(item) not in (int, float) for item in value + ): + raise _DiscoveryUnavailableError("state_invalid") + if all(item is not None for item in value) and value[0] > value[1]: + raise _DiscoveryUnavailableError("state_invalid") + elif kind == "filter_timegrain": + if len(value) > 1 or any(not isinstance(item, str) for item in value): + raise _DiscoveryUnavailableError("state_invalid") + elif kind == "filter_time" and not isinstance(value, str): + raise _DiscoveryUnavailableError("state_invalid") + + +def _normalize_select_mask( # noqa: C901 + context: _Discovery, + conf: dict[str, Any], + mask: dict[str, Any], +) -> dict[str, Any]: + """Validate effective selections and recover lossless browser numeric strings.""" + invalid_reason = context.mask_error_reasons[conf["id"]] + state = mask.get("filterState") + controls = conf.get("controlValues") or {} + if not isinstance(state, dict) or not isinstance(controls, dict): + return {"invalid_reason": invalid_reason} + values = state.get("value") + if conf.get("filterType") != "filter_select": + try: + _validate_snapshot_value(conf, values) + except _DiscoveryUnavailableError: + return {"invalid_reason": invalid_reason} + return mask + # Empty selections need no parent column metadata. Predicate reconstruction + # still applies required-clear and unresolved-state semantics downstream. + if values is None or values == []: + return mask + try: + target = _select_target(conf, context) + except SupersetSecurityException: + return {"invalid_reason": "permission_denied"} + except _DiscoveryUnavailableError as ex: + return ( + mask + if ex.reason.startswith("unsupported_") + else {"invalid_reason": ex.reason} + ) + if not isinstance(values, list): + return {"invalid_reason": invalid_reason} + if controls.get("multiSelect") is False and len(values) > 1: + return {"invalid_reason": invalid_reason} + normalized = [] + try: + for value in values: + if ( + target.column.type_generic == GenericDataType.NUMERIC + and isinstance(value, str) + and re.fullmatch( + r"-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?", value + ) + ): + value = _scalar(Decimal(value)) + if ( + target.column.type_generic == GenericDataType.BOOLEAN + and type(value) in (int, float) + and value in (0, 1) + ): + value = bool(value) + if not _compatible(value, target): + return {"invalid_reason": invalid_reason} + if not _safe_number(value): + return {"invalid_reason": "invalid_scalar"} + normalized.append(value) + except _DiscoveryUnavailableError as ex: + return {"invalid_reason": ex.reason} + except ArithmeticError: + return {"invalid_reason": "invalid_scalar"} + state["value"] = normalized + return mask + + +def _url_params(pairs: Any) -> dict[str, str]: + """Rebuild the dashboard's query url_params from snapshot key-value pairs. + + The dashboard sends every non-reserved URL parameter with each dropdown + query, so ``url_param`` macros in virtual dataset SQL render against them. + """ + if pairs is None: + return {} + if not isinstance(pairs, list): + raise _DiscoveryUnavailableError("state_invalid") + params: dict[str, str] = {} + for pair in pairs: + if pair is None: + # The permalink schema permits null entries; they carry no parameter. + continue + if ( + not isinstance(pair, (list, tuple)) + or len(pair) != 2 + or any(part is not None and not isinstance(part, str) for part in pair) + ): + raise _DiscoveryUnavailableError("state_invalid") + key, value = pair + if key is None or value is None or key in _RESERVED_URL_PARAMS: + continue + # URLSearchParams keeps the last value for a repeated key. + params[key] = value + return params + + +def _hydrate_masks( # noqa: C901 + context: _Discovery, + snapshot_state: dict[str, Any] | None, +) -> dict[str, dict[str, Any]]: + """Shallow-merge initial/default/loaded masks and restore required defaults.""" + configs = context.configs + loaded_masks: dict[str, Any] = {} + if snapshot_state is not None: + if not isinstance(snapshot_state, dict): + raise _DiscoveryUnavailableError("state_invalid") + supplied_masks = snapshot_state.get("dataMask") + if isinstance(supplied_masks, dict): + # Explicit masks are authoritative even with extra permalink metadata. + loaded_masks = supplied_masks + elif supplied_masks is not None: + raise _DiscoveryUnavailableError("state_invalid") + else: + try: + state = DashboardPermalinkStateSchema().load(snapshot_state) + except ValidationError as ex: + raise _DiscoveryUnavailableError("state_invalid") from ex + if any( + pair and pair[0] in ("native_filters", "native_filters_key") + for pair in state.get("urlParams") or [] + ): + # URL-encoded selections are not native dataMask state. + raise _DiscoveryUnavailableError("state_invalid") + context.url_params = _url_params(snapshot_state.get("urlParams")) + for filter_id, mask in loaded_masks.items(): + if not isinstance(filter_id, str) or not isinstance(mask, dict): + raise _DiscoveryUnavailableError("state_invalid") + if filter_id not in configs: + continue + if any( + key in mask and not isinstance(mask[key], dict) + for key in ("filterState", "extraFormData", "ownState") + ): + raise _DiscoveryUnavailableError("state_invalid") + value = mask.get("filterState", {}).get("value") + values = value if isinstance(value, list) else [value] + for scalar in values: + if scalar is not None and ( + type(scalar) not in (str, int, float, bool) + or (isinstance(scalar, float) and not math.isfinite(scalar)) + ): + raise _DiscoveryUnavailableError("state_invalid") + hydrated: dict[str, dict[str, Any]] = {} + for filter_id, conf in configs.items(): + default = conf.get("defaultDataMask") or {} + loaded = loaded_masks.get(filter_id, {}) + context.mask_error_reasons[filter_id] = ( + "state_invalid" if "filterState" in loaded else "parent_unresolved" + ) + if not isinstance(default, dict): + hydrated[filter_id] = {"invalid": True} + continue + mask = { + "filterState": {}, + "extraFormData": {}, + **deepcopy(default), + **deepcopy(loaded), + } + controls = conf.get("controlValues") or {} + loaded_state = loaded.get("filterState") or {} + if not isinstance(loaded_state, dict) or not isinstance(controls, dict): + hydrated[filter_id] = {"invalid": True} + continue + has_value = _real_value(loaded_state.get("value")) + explicit_clear = ( + conf.get("filterType") == "filter_select" + and not has_value + and _explicit_clear(loaded) + ) + if ( + controls.get("enableEmptyFilter") + and default + and not explicit_clear + and ( + not has_value + or (not loaded.get("extraFormData") and default.get("extraFormData")) + ) + ): + mask["filterState"] = deepcopy(default.get("filterState") or {}) + mask["extraFormData"] = deepcopy(default.get("extraFormData") or {}) + context.mask_error_reasons[filter_id] = "parent_unresolved" + hydrated[filter_id] = mask + return hydrated + + +def _selected_extra( # noqa: C901 + context: _Discovery, conf: dict[str, Any], mask: dict[str, Any] +) -> dict[str, Any]: + """Rebuild parent predicates from typed values, never executing snapshot SQL.""" + from superset.mcp_service.dashboard.tool.apply_dashboard_filters import ( + _range_data_mask, + _time_data_mask, + ) + + state = mask.get("filterState") + if mask.get("invalid_reason"): + raise _DiscoveryUnavailableError(mask["invalid_reason"]) + if mask.get("invalid") or not isinstance(state, dict): + raise _DiscoveryUnavailableError("parent_unresolved") + controls = conf.get("controlValues") or {} + value = state.get("value") + if controls.get("defaultToFirstItem") and "value" not in state: + raise _DiscoveryUnavailableError("parent_unresolved") + kind = conf.get("filterType") + if kind != "filter_select" and value is None and mask.get("extraFormData"): + # As with Select, predicates without a typed value cannot be rebuilt, + # and reading them as unfiltered would overstate the child's choices. + raise _DiscoveryUnavailableError("parent_unresolved") + try: + if kind == "filter_select": + values = [] if value is None else value + if values == [] and not ( + value is None + and mask.get("extraFormData") + and not _explicit_clear(mask) + ): + return cast( + dict[str, Any], _empty_select_data_mask(conf)["extraFormData"] + ) + target = _select_target(conf, context) + if not isinstance(values, list) or any( + not _compatible(v, target) for v in values + ): + raise _DiscoveryUnavailableError("parent_unresolved") + if any(not _safe_number(v) for v in values): + raise _DiscoveryUnavailableError("invalid_scalar") + # SQL-only state is not a selected value. A canonical clear is the + # sole SQL-shaped evidence we recognize and regenerate ourselves. + if ( + value is None + and mask.get("extraFormData") + and not _explicit_clear(mask) + ): + raise _DiscoveryUnavailableError("parent_unresolved") + return cast( + dict[str, Any], _select_data_mask(conf, values)["extraFormData"] + ) + if kind == "filter_time": + if value is not None and not isinstance(value, str): + raise _DiscoveryUnavailableError("parent_unresolved") + return cast( + dict[str, Any], + _time_data_mask(conf, value or NO_TIME_RANGE)["extraFormData"], + ) + if kind == "filter_range": + values = [None, None] if value is None else value + if ( + not isinstance(values, list) + or len(values) != 2 + or any( + v is not None + and (type(v) not in (int, float) or not math.isfinite(v)) + for v in values + ) + ): + raise _DiscoveryUnavailableError("parent_unresolved") + if all(v is not None for v in values) and values[0] > values[1]: + raise _DiscoveryUnavailableError("parent_unresolved") + return cast(dict[str, Any], _range_data_mask(conf, values)["extraFormData"]) + if kind == "filter_timegrain": + values = [] if value is None else value + if ( + not isinstance(values, list) + or len(values) > 1 + or any(not isinstance(v, str) for v in values) + ): + raise _DiscoveryUnavailableError("parent_unresolved") + if not values: + if controls.get("enableEmptyFilter"): + raise _DiscoveryUnavailableError("parent_unresolved") + return {} + targets = conf.get("targets") + if ( + not isinstance(targets, list) + or len(targets) != 1 + or not isinstance(targets[0], dict) + ): + raise _DiscoveryUnavailableError("parent_unresolved") + grain_target = targets[0] + if ( + grain_target.get("datasourceType") not in (None, "table") + or type(grain_target.get("datasetId")) is not int + ): + raise _DiscoveryUnavailableError("parent_unresolved") + dataset = DatasetDAO.find_by_id( + grain_target["datasetId"], skip_base_filter=True + ) + allowed = conf.get("time_grains") + if ( + dataset is None + or (allowed and values[0] not in allowed) + or values[0] + not in [duration for duration, _ in dataset.time_grain_sqla] + ): + raise _DiscoveryUnavailableError("parent_unresolved") + return {"time_grain_sqla": values[0]} + except _DiscoveryUnavailableError as ex: + if ex.reason in ("invalid_scalar", "query_budget", "scheduling_budget"): + raise + raise _DiscoveryUnavailableError("parent_unresolved") from ex + except ArithmeticError as ex: + raise _DiscoveryUnavailableError("invalid_scalar") from ex + except (TypeError, ValueError, _FilterValidationError) as ex: + raise _DiscoveryUnavailableError("parent_unresolved") from ex + raise _DiscoveryUnavailableError("parent_unresolved") + + +def _parent_extra(context: _Discovery, filter_id: str) -> dict[str, Any]: + """Merge transitive parents in native order, excluding self and nonparents.""" + extra: dict[str, Any] = {} + for parent in _ancestors(filter_id, context.configs): + if parent in context.proposal_errors: + raise _DiscoveryUnavailableError(context.proposal_errors[parent]) + if parent not in context.normalized_masks: + context.masks[parent] = _normalize_select_mask( + context, context.configs[parent], context.masks[parent] + ) + context.normalized_masks.add(parent) + extra = merge_extra_form_data( + extra, + _selected_extra(context, context.configs[parent], context.masks[parent]), + ) + return extra + + +def _native_query( # noqa: C901 + context: _Discovery, + conf: dict[str, Any], + target: _Target, + item: FilterValueRequest, + proposal: ProposedFilterValues | None = None, +) -> tuple[ChartDataCommand, int]: + """Build the Select dropdown QueryContext and authorize it before cache use.""" + column = target.column.column_name + controls = conf.get("controlValues") or {} + metric = conf.get("sortMetric") or controls.get("sortMetric") + if metric is not None and not isinstance(metric, str): + raise _DiscoveryUnavailableError("unsupported_target") + configured_limit = current_app.config.get("NATIVE_FILTER_DEFAULT_ROW_LIMIT", 1000) + server_limit = apply_max_row_limit(item.limit + 1) + if type(configured_limit) is not int or configured_limit <= 0 or server_limit <= 0: + raise _DiscoveryUnavailableError("query_budget") + limit = min(item.limit + 1, server_limit, configured_limit) + parents = _parent_extra(context, conf["id"]) + parent_filters = parents.pop("filters", []) + form_data = { + **deepcopy(controls), + "datasource": f"{target.dataset.id}__table", + "groupby": [column], + "metrics": [], + "row_limit": limit, + "adhoc_filters": deepcopy(conf.get("adhoc_filters") or []), + "extra_filters": [], + "extra_form_data": parents, + "time_range": conf.get("time_range"), + "granularity_sqla": conf.get("granularity_sqla"), + "url_params": deepcopy(context.url_params), + "viz_type": "filter_select", + "type": "NATIVE_FILTER", + "dashboardId": context.dashboard.id, + "native_filter_id": conf["id"], + } + query = build_query_dicts_from_form_data( + form_data, target.dataset.id, "table", row_limit=limit + )[0] + # Cross-dataset parents retain native coercion when column families differ. + # Compatible strings/numbers use exact scalars, preserving whitespace/sentinels. + for predicate in parent_filters: + parent_column = next( + ( + col + for col in target.dataset.columns + if col.column_name == predicate["col"] + ), + None, + ) + values = predicate.get("val") + values = values if isinstance(values, list) else [values] + literal = parent_column is not None and all( + _compatible(value, _Target(target.dataset, parent_column)) + for value in values + ) + query["filters"].append({**predicate, "is_literal": literal}) + query["columns"] = [column] + query.setdefault("extras", {})["preserve_literal_values"] = True + query["metrics"] = [metric] if metric else [] + if target.column.type_generic == GenericDataType.NUMERIC: + # Pandas represents both SQL NULL and database NaN as NaN in float + # columns. COUNT(target) distinguishes those groups without another query. + count_label = "__mcp_native_nonnull_count" + while count_label in (column, metric): + count_label += "_" + query["metrics"].append( + { + "expressionType": "SIMPLE", + "aggregate": "COUNT", + "column": {"column_name": column}, + "label": count_label, + } + ) + query["orderby"] = ( + [[metric or column, controls.get("sortAscending", True)]] + if metric or not item.search + else [] + ) + if proposal is not None: + query["filters"].append( + { + "col": column, + "op": FilterOperator.IN.value, + "val": proposal.values, + "is_literal": True, + } + ) + elif "exact_value" in item.model_fields_set: + query["filters"].append( + { + "col": column, + "op": FilterOperator.IS_NULL.value + if item.exact_value is None + else FilterOperator.EQUALS.value, + "val": item.exact_value, + "is_literal": True, + } + ) + elif item.search: + if ( + target.column.type_generic != GenericDataType.STRING + or not item.search.isascii() + ): + raise _DiscoveryUnavailableError("unsupported_search") + query["filters"].append( + { + "col": column, + "op": FilterOperator.ILIKE.value, Review Comment: Fixed in 00ebb77fa3. When a search runs and the query's `applied_template_filters` include the target column, discovery reports `unsupported_search` with no values instead of claiming `complete`. Since 7cd346d914 the same guard also covers exact and proposal probes. ########## superset/mcp_service/dashboard/filter_values.py: ########## @@ -0,0 +1,1256 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Authorized, bounded native dropdown discovery, independent of chart scope.""" + +from __future__ import annotations + +import logging +import math +import re +import time +from collections.abc import Iterator +from contextlib import contextmanager +from copy import deepcopy +from dataclasses import dataclass, field +from decimal import Decimal +from numbers import Integral, Real +from typing import Any, cast, Literal, TYPE_CHECKING + +from fastmcp.exceptions import ToolError +from flask import current_app, g +from marshmallow import ValidationError +from pandas import DataFrame, NA +from sqlalchemy.exc import CompileError, UnsupportedCompilationError + +from superset.charts.data.form_data import set_query_context_form_data +from superset.charts.schemas import ChartDataQueryContextSchema +from superset.commands.chart.data.get_data_command import ChartDataCommand +from superset.constants import EMPTY_FILTER_SQL_EXPRESSION, NO_TIME_RANGE +from superset.daos.dataset import DatasetDAO +from superset.dashboards.filter_scope import _is_divider +from superset.dashboards.permalink.schemas import DashboardPermalinkStateSchema +from superset.exceptions import ( + OAuth2Error, + OAuth2RedirectError, + SupersetSecurityException, +) +from superset.mcp_service.chart.chart_helpers import ( + build_query_dicts_from_form_data, + merge_extra_form_data, +) +from superset.mcp_service.dashboard.schemas import ( + _extract_native_filters, + FilterValueDiscovery, + FilterValueReason, + FilterValueRequest, + FilterValueScalar, + GetDashboardInfoRequest, + NativeFilterAvailableValues, + NativeFilterValueSummary, + ProposedFilterValues, +) +from superset.mcp_service.dashboard.tool.manage_native_filters import ( + _empty_select_data_mask, + _FilterValidationError, + _select_data_mask, + current_native_filter_config, +) +from superset.mcp_service.privacy import user_can_view_data_model_metadata +from superset.utils import json +from superset.utils.core import apply_max_row_limit, FilterOperator, GenericDataType + +if TYPE_CHECKING: + from superset.connectors.sqla.models import SqlaTable, TableColumn + from superset.models.dashboard import Dashboard + +_MAX_QUERIES = 20 +_SCHEDULING_SECONDS = 10 +_PAGE_SIZE = 20 +_MAX_SAFE_INTEGER = 2**53 - 1 +# Mirrors the dashboard's extractUrlParams('regular') exclusions. +_RESERVED_URL_PARAMS = frozenset({"standalone", "edit"}) +logger = logging.getLogger(__name__) + + +class _DiscoveryUnavailableError(Exception): + """A bounded, non-sensitive failure for a single dropdown.""" + + def __init__(self, reason: FilterValueReason) -> None: + super().__init__(reason) + self.reason = reason + + +@dataclass +class _Target: + """A configured table target; this is metadata, not an access grant.""" + + dataset: SqlaTable + column: TableColumn + + +@dataclass +class _Discovery: + """Call-local configuration, hydrated state, and execution accounting.""" + + dashboard: Dashboard + configs: dict[str, dict[str, Any]] + masks: dict[str, dict[str, Any]] + started: float = field(default_factory=time.monotonic) + query_count: int = 0 + proposal_errors: dict[str, FilterValueReason] = field(default_factory=dict) + targets: dict[ + tuple[int, str], + _Target | _DiscoveryUnavailableError | SupersetSecurityException, + ] = field(default_factory=dict) + mask_error_reasons: dict[str, FilterValueReason] = field(default_factory=dict) + normalized_masks: set[str] = field(default_factory=set) + url_params: dict[str, str] = field(default_factory=dict) + + +def _check_budget(context: _Discovery) -> None: + """Stop preparation and authorization after a call reaches either budget.""" + if context.query_count >= _MAX_QUERIES: + raise _DiscoveryUnavailableError("query_budget") + if time.monotonic() - context.started >= _SCHEDULING_SECONDS: + raise _DiscoveryUnavailableError("scheduling_budget") + + +def _configured_filters(dashboard: Dashboard) -> list[dict[str, Any]]: + """Read saved definitions in order, excluding visual dividers.""" + return [c for c in current_native_filter_config(dashboard) if not _is_divider(c)] + + +def _select_target(conf: dict[str, Any], context: _Discovery) -> _Target: + """Memoize a native target's metadata preflight, never its final query gate.""" + _check_budget(context) + controls = conf.get("controlValues") or {} + if ( + conf.get("filterType") != "filter_select" + or not isinstance(controls, dict) + or controls.get("inverseSelection") + or controls.get("operatorType", "exact") != "exact" + ): + raise _DiscoveryUnavailableError("unsupported_filter") + targets = conf.get("targets") + if not isinstance(targets, list) or len(targets) != 1: + raise _DiscoveryUnavailableError("unsupported_target") + target = targets[0] + if ( + not isinstance(target, dict) + or target.get("datasourceType") not in (None, "table") + or target.get("semantic_selection_version") + or type(target.get("datasetId")) is not int + or target["datasetId"] <= 0 + or not isinstance(target.get("column"), dict) + or not isinstance(target["column"].get("name"), str) + or not target["column"]["name"] + ): + raise _DiscoveryUnavailableError("unsupported_target") + key = (target["datasetId"], conf["id"]) + cached = context.targets.get(key) + if isinstance(cached, Exception): + raise cached + if cached is not None: + return cached + dataset = DatasetDAO.find_by_id(target["datasetId"], skip_base_filter=True) + if dataset is None: + raise _DiscoveryUnavailableError("unsupported_target") + column = next( + (col for col in dataset.columns if col.column_name == target["column"]["name"]), + None, + ) + if column is None: + raise _DiscoveryUnavailableError("unsupported_target") + # Native authorization preserves dashboard-only grants and guest limits. + # This memoized context is never executed. Every final query is authorized + # separately with its complete predicates before dataframe cache access. + access_context = ChartDataQueryContextSchema().load( + { + "datasource": {"id": dataset.id, "type": "table"}, + "queries": [ + {"columns": [column.column_name], "metrics": [], "row_limit": 1} + ], + "form_data": { + "type": "NATIVE_FILTER", + "dashboardId": context.dashboard.id, + "native_filter_id": conf["id"], + }, + "result_type": "full", + "result_format": "json", + } + ) + _check_budget(context) + try: + ChartDataCommand(access_context).validate() + except SupersetSecurityException as ex: + context.targets[key] = ex + raise + if column.type_generic not in ( + GenericDataType.STRING, + GenericDataType.NUMERIC, + GenericDataType.BOOLEAN, + ): + error = _DiscoveryUnavailableError("unsupported_target") + context.targets[key] = error + raise error + resolved = _Target(dataset, column) + context.targets[key] = resolved + return resolved + + +def _compatible(value: FilterValueScalar, target: _Target) -> bool: + """Compare JSON scalar families, keeping booleans distinct from numbers.""" + if value is None: + return True + if type(value) not in (str, bool, int, float) or ( + isinstance(value, float) and not math.isfinite(value) + ): + return False + if isinstance(value, str): + return target.column.type_generic == GenericDataType.STRING + if type(value) is bool: + return target.column.type_generic == GenericDataType.BOOLEAN + return target.column.type_generic == GenericDataType.NUMERIC + + +def _ancestors(filter_id: str, configs: dict[str, dict[str, Any]]) -> list[str]: + """Resolve ancestors furthest first and reject missing or cyclic graphs.""" + ordered: list[str] = [] + active: set[str] = set() + visited: set[str] = set() + + def visit(current: str) -> None: + """Walk one configuration, retaining stable parent ordering.""" + if current in active or current not in configs: + raise _DiscoveryUnavailableError("parent_unresolved") + if current in visited: + return + active.add(current) + parents = configs[current].get("cascadeParentIds") or [] + if not isinstance(parents, list) or any( + not isinstance(p, str) for p in parents + ): + raise _DiscoveryUnavailableError("parent_unresolved") + for parent in parents: + visit(parent) + active.remove(current) + visited.add(current) + if current != filter_id: + ordered.append(current) + + visit(filter_id) + return ordered + + +def _requests( + configs: list[dict[str, Any]], + request: GetDashboardInfoRequest, +) -> list[FilterValueRequest]: + """Choose explicit IDs or a bounded page before querying any datasource.""" + if request.filter_value_requests is not None: + return request.filter_value_requests + return [ + FilterValueRequest(filter_id=c["id"]) + for c in configs[ + request.filter_values_offset : request.filter_values_offset + _PAGE_SIZE + ] + ] + + +def validate_discovery_configuration( # noqa: C901 + dashboard: Dashboard, + request: GetDashboardInfoRequest, + *, + started_at: float | None = None, +) -> _Discovery: + """Reject invalid IDs, proposed ancestors, and scalar types before querying.""" + configured = _configured_filters(dashboard) + configs: dict[str, dict[str, Any]] = { + c["id"]: c for c in configured if isinstance(c.get("id"), str) + } + if any(not isinstance(c.get("id"), str) or not c["id"] for c in configured) or len( + configs + ) != len(configured): + raise ValueError("Native filter configuration has invalid or duplicate IDs") + context = _Discovery( + dashboard, + configs, + {}, + started=time.monotonic() if started_at is None else started_at, + ) + requests = _requests(configured, request) + for item in requests: + if item.filter_id not in configs: + raise ValueError(f"Unknown native filter ID: {item.filter_id}") + if "exact_value" in item.model_fields_set: + try: + target = _select_target(configs[item.filter_id], context) + except (_DiscoveryUnavailableError, SupersetSecurityException): + continue + if not _compatible(item.exact_value, target): + raise ValueError( + f"Incompatible exact value for native filter {item.filter_id}" + ) + allowed: set[str] = set() + for item in requests: + try: + allowed.update(_ancestors(item.filter_id, configs)) + except _DiscoveryUnavailableError: + pending = [item.filter_id] + visited: set[str] = set() + while pending: + current = pending.pop() + if current in visited or current not in configs: + continue + visited.add(current) + parents = configs[current].get("cascadeParentIds") + if isinstance(parents, list): + pending.extend(p for p in parents if isinstance(p, str)) + allowed.update(visited - {item.filter_id}) + for proposal in request.proposed_filter_values or []: + if proposal.filter_id not in configs or proposal.filter_id not in allowed: + raise ValueError( + "Proposed filters must be configured ancestors of requested filters" + ) + conf = configs[proposal.filter_id] + controls = conf.get("controlValues") or {} + if ( + isinstance(controls, dict) + and controls.get("multiSelect") is False + and len(proposal.values) > 1 + ): + raise ValueError("A single-select proposal accepts at most one value") + try: + target = _select_target(conf, context) + except (_DiscoveryUnavailableError, SupersetSecurityException): + continue + if any(not _compatible(value, target) for value in proposal.values): + raise ValueError( + f"Incompatible proposed value for native filter {proposal.filter_id}" + ) + return context + + +def _real_value(value: Any) -> bool: + """Mirror fillNativeFilters' empty/all-null-array restoration test.""" + return value is not None and not ( + isinstance(value, list) and all(v is None for v in value) + ) + + +def _explicit_clear(mask: dict[str, Any]) -> bool: + """Recognize only the frontend's canonical match-nothing clear evidence.""" + extra = mask.get("extraFormData") or {} + if not isinstance(extra, dict): + return False + predicates = extra.get("adhoc_filters") + return isinstance(predicates, list) and any( + isinstance(p, dict) + and p.get("expressionType") == "SQL" + and p.get("clause") == "WHERE" + and p.get("sqlExpression") == EMPTY_FILTER_SQL_EXPRESSION + for p in predicates + ) + + +def _validate_snapshot_value(conf: dict[str, Any], value: Any) -> None: # noqa: C901 + """Reject malformed known native selection shapes before any option query.""" + if value is None: + return + kind = conf.get("filterType") + if kind in ("filter_select", "filter_range", "filter_timegrain") and not isinstance( + value, list + ): + raise _DiscoveryUnavailableError("state_invalid") + if kind == "filter_range": + if len(value) != 2 or any( + item is not None and type(item) not in (int, float) for item in value + ): + raise _DiscoveryUnavailableError("state_invalid") + if all(item is not None for item in value) and value[0] > value[1]: + raise _DiscoveryUnavailableError("state_invalid") + elif kind == "filter_timegrain": + if len(value) > 1 or any(not isinstance(item, str) for item in value): + raise _DiscoveryUnavailableError("state_invalid") + elif kind == "filter_time" and not isinstance(value, str): + raise _DiscoveryUnavailableError("state_invalid") + + +def _normalize_select_mask( # noqa: C901 + context: _Discovery, + conf: dict[str, Any], + mask: dict[str, Any], +) -> dict[str, Any]: + """Validate effective selections and recover lossless browser numeric strings.""" + invalid_reason = context.mask_error_reasons[conf["id"]] + state = mask.get("filterState") + controls = conf.get("controlValues") or {} + if not isinstance(state, dict) or not isinstance(controls, dict): + return {"invalid_reason": invalid_reason} + values = state.get("value") + if conf.get("filterType") != "filter_select": + try: + _validate_snapshot_value(conf, values) + except _DiscoveryUnavailableError: + return {"invalid_reason": invalid_reason} + return mask + # Empty selections need no parent column metadata. Predicate reconstruction + # still applies required-clear and unresolved-state semantics downstream. + if values is None or values == []: + return mask + try: + target = _select_target(conf, context) + except SupersetSecurityException: + return {"invalid_reason": "permission_denied"} + except _DiscoveryUnavailableError as ex: + return ( + mask + if ex.reason.startswith("unsupported_") + else {"invalid_reason": ex.reason} + ) + if not isinstance(values, list): + return {"invalid_reason": invalid_reason} + if controls.get("multiSelect") is False and len(values) > 1: + return {"invalid_reason": invalid_reason} + normalized = [] + try: + for value in values: + if ( + target.column.type_generic == GenericDataType.NUMERIC + and isinstance(value, str) + and re.fullmatch( + r"-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?", value + ) + ): + value = _scalar(Decimal(value)) + if ( + target.column.type_generic == GenericDataType.BOOLEAN + and type(value) in (int, float) + and value in (0, 1) + ): + value = bool(value) + if not _compatible(value, target): + return {"invalid_reason": invalid_reason} + if not _safe_number(value): + return {"invalid_reason": "invalid_scalar"} + normalized.append(value) + except _DiscoveryUnavailableError as ex: + return {"invalid_reason": ex.reason} + except ArithmeticError: + return {"invalid_reason": "invalid_scalar"} + state["value"] = normalized + return mask + + +def _url_params(pairs: Any) -> dict[str, str]: + """Rebuild the dashboard's query url_params from snapshot key-value pairs. + + The dashboard sends every non-reserved URL parameter with each dropdown + query, so ``url_param`` macros in virtual dataset SQL render against them. + """ + if pairs is None: + return {} + if not isinstance(pairs, list): + raise _DiscoveryUnavailableError("state_invalid") + params: dict[str, str] = {} + for pair in pairs: + if pair is None: + # The permalink schema permits null entries; they carry no parameter. + continue + if ( + not isinstance(pair, (list, tuple)) + or len(pair) != 2 + or any(part is not None and not isinstance(part, str) for part in pair) + ): + raise _DiscoveryUnavailableError("state_invalid") + key, value = pair + if key is None or value is None or key in _RESERVED_URL_PARAMS: + continue + # URLSearchParams keeps the last value for a repeated key. + params[key] = value + return params + + +def _hydrate_masks( # noqa: C901 + context: _Discovery, + snapshot_state: dict[str, Any] | None, +) -> dict[str, dict[str, Any]]: + """Shallow-merge initial/default/loaded masks and restore required defaults.""" + configs = context.configs + loaded_masks: dict[str, Any] = {} + if snapshot_state is not None: + if not isinstance(snapshot_state, dict): + raise _DiscoveryUnavailableError("state_invalid") + supplied_masks = snapshot_state.get("dataMask") + if isinstance(supplied_masks, dict): + # Explicit masks are authoritative even with extra permalink metadata. + loaded_masks = supplied_masks + elif supplied_masks is not None: + raise _DiscoveryUnavailableError("state_invalid") + else: + try: + state = DashboardPermalinkStateSchema().load(snapshot_state) + except ValidationError as ex: + raise _DiscoveryUnavailableError("state_invalid") from ex + if any( + pair and pair[0] in ("native_filters", "native_filters_key") + for pair in state.get("urlParams") or [] + ): + # URL-encoded selections are not native dataMask state. + raise _DiscoveryUnavailableError("state_invalid") + context.url_params = _url_params(snapshot_state.get("urlParams")) + for filter_id, mask in loaded_masks.items(): + if not isinstance(filter_id, str) or not isinstance(mask, dict): + raise _DiscoveryUnavailableError("state_invalid") + if filter_id not in configs: + continue + if any( + key in mask and not isinstance(mask[key], dict) + for key in ("filterState", "extraFormData", "ownState") + ): + raise _DiscoveryUnavailableError("state_invalid") + value = mask.get("filterState", {}).get("value") + values = value if isinstance(value, list) else [value] + for scalar in values: + if scalar is not None and ( + type(scalar) not in (str, int, float, bool) + or (isinstance(scalar, float) and not math.isfinite(scalar)) + ): + raise _DiscoveryUnavailableError("state_invalid") + hydrated: dict[str, dict[str, Any]] = {} + for filter_id, conf in configs.items(): + default = conf.get("defaultDataMask") or {} + loaded = loaded_masks.get(filter_id, {}) + context.mask_error_reasons[filter_id] = ( + "state_invalid" if "filterState" in loaded else "parent_unresolved" + ) + if not isinstance(default, dict): + hydrated[filter_id] = {"invalid": True} + continue + mask = { + "filterState": {}, + "extraFormData": {}, + **deepcopy(default), + **deepcopy(loaded), + } + controls = conf.get("controlValues") or {} + loaded_state = loaded.get("filterState") or {} + if not isinstance(loaded_state, dict) or not isinstance(controls, dict): + hydrated[filter_id] = {"invalid": True} + continue + has_value = _real_value(loaded_state.get("value")) + explicit_clear = ( + conf.get("filterType") == "filter_select" + and not has_value + and _explicit_clear(loaded) + ) + if ( + controls.get("enableEmptyFilter") + and default + and not explicit_clear + and ( + not has_value + or (not loaded.get("extraFormData") and default.get("extraFormData")) + ) + ): + mask["filterState"] = deepcopy(default.get("filterState") or {}) + mask["extraFormData"] = deepcopy(default.get("extraFormData") or {}) + context.mask_error_reasons[filter_id] = "parent_unresolved" + hydrated[filter_id] = mask + return hydrated + + +def _selected_extra( # noqa: C901 + context: _Discovery, conf: dict[str, Any], mask: dict[str, Any] +) -> dict[str, Any]: + """Rebuild parent predicates from typed values, never executing snapshot SQL.""" + from superset.mcp_service.dashboard.tool.apply_dashboard_filters import ( + _range_data_mask, + _time_data_mask, + ) + + state = mask.get("filterState") + if mask.get("invalid_reason"): + raise _DiscoveryUnavailableError(mask["invalid_reason"]) + if mask.get("invalid") or not isinstance(state, dict): + raise _DiscoveryUnavailableError("parent_unresolved") + controls = conf.get("controlValues") or {} + value = state.get("value") + if controls.get("defaultToFirstItem") and "value" not in state: + raise _DiscoveryUnavailableError("parent_unresolved") + kind = conf.get("filterType") + if kind != "filter_select" and value is None and mask.get("extraFormData"): + # As with Select, predicates without a typed value cannot be rebuilt, + # and reading them as unfiltered would overstate the child's choices. + raise _DiscoveryUnavailableError("parent_unresolved") + try: + if kind == "filter_select": + values = [] if value is None else value + if values == [] and not ( + value is None + and mask.get("extraFormData") + and not _explicit_clear(mask) + ): + return cast( + dict[str, Any], _empty_select_data_mask(conf)["extraFormData"] + ) + target = _select_target(conf, context) + if not isinstance(values, list) or any( + not _compatible(v, target) for v in values + ): + raise _DiscoveryUnavailableError("parent_unresolved") + if any(not _safe_number(v) for v in values): + raise _DiscoveryUnavailableError("invalid_scalar") + # SQL-only state is not a selected value. A canonical clear is the + # sole SQL-shaped evidence we recognize and regenerate ourselves. + if ( + value is None + and mask.get("extraFormData") + and not _explicit_clear(mask) + ): + raise _DiscoveryUnavailableError("parent_unresolved") + return cast( + dict[str, Any], _select_data_mask(conf, values)["extraFormData"] + ) + if kind == "filter_time": + if value is not None and not isinstance(value, str): + raise _DiscoveryUnavailableError("parent_unresolved") + return cast( + dict[str, Any], + _time_data_mask(conf, value or NO_TIME_RANGE)["extraFormData"], + ) + if kind == "filter_range": + values = [None, None] if value is None else value + if ( + not isinstance(values, list) + or len(values) != 2 + or any( + v is not None + and (type(v) not in (int, float) or not math.isfinite(v)) + for v in values + ) + ): + raise _DiscoveryUnavailableError("parent_unresolved") + if all(v is not None for v in values) and values[0] > values[1]: + raise _DiscoveryUnavailableError("parent_unresolved") + return cast(dict[str, Any], _range_data_mask(conf, values)["extraFormData"]) + if kind == "filter_timegrain": + values = [] if value is None else value + if ( + not isinstance(values, list) + or len(values) > 1 + or any(not isinstance(v, str) for v in values) + ): + raise _DiscoveryUnavailableError("parent_unresolved") + if not values: + if controls.get("enableEmptyFilter"): + raise _DiscoveryUnavailableError("parent_unresolved") + return {} + targets = conf.get("targets") + if ( + not isinstance(targets, list) + or len(targets) != 1 + or not isinstance(targets[0], dict) + ): + raise _DiscoveryUnavailableError("parent_unresolved") + grain_target = targets[0] + if ( + grain_target.get("datasourceType") not in (None, "table") + or type(grain_target.get("datasetId")) is not int + ): + raise _DiscoveryUnavailableError("parent_unresolved") + dataset = DatasetDAO.find_by_id( + grain_target["datasetId"], skip_base_filter=True + ) + allowed = conf.get("time_grains") + if ( + dataset is None + or (allowed and values[0] not in allowed) + or values[0] + not in [duration for duration, _ in dataset.time_grain_sqla] + ): + raise _DiscoveryUnavailableError("parent_unresolved") + return {"time_grain_sqla": values[0]} + except _DiscoveryUnavailableError as ex: + if ex.reason in ("invalid_scalar", "query_budget", "scheduling_budget"): + raise + raise _DiscoveryUnavailableError("parent_unresolved") from ex + except ArithmeticError as ex: + raise _DiscoveryUnavailableError("invalid_scalar") from ex + except (TypeError, ValueError, _FilterValidationError) as ex: + raise _DiscoveryUnavailableError("parent_unresolved") from ex + raise _DiscoveryUnavailableError("parent_unresolved") + + +def _parent_extra(context: _Discovery, filter_id: str) -> dict[str, Any]: + """Merge transitive parents in native order, excluding self and nonparents.""" + extra: dict[str, Any] = {} + for parent in _ancestors(filter_id, context.configs): + if parent in context.proposal_errors: + raise _DiscoveryUnavailableError(context.proposal_errors[parent]) + if parent not in context.normalized_masks: + context.masks[parent] = _normalize_select_mask( + context, context.configs[parent], context.masks[parent] + ) + context.normalized_masks.add(parent) + extra = merge_extra_form_data( + extra, + _selected_extra(context, context.configs[parent], context.masks[parent]), + ) + return extra + + +def _native_query( # noqa: C901 + context: _Discovery, + conf: dict[str, Any], + target: _Target, + item: FilterValueRequest, + proposal: ProposedFilterValues | None = None, +) -> tuple[ChartDataCommand, int]: + """Build the Select dropdown QueryContext and authorize it before cache use.""" + column = target.column.column_name + controls = conf.get("controlValues") or {} + metric = conf.get("sortMetric") or controls.get("sortMetric") + if metric is not None and not isinstance(metric, str): + raise _DiscoveryUnavailableError("unsupported_target") + configured_limit = current_app.config.get("NATIVE_FILTER_DEFAULT_ROW_LIMIT", 1000) + server_limit = apply_max_row_limit(item.limit + 1) + if type(configured_limit) is not int or configured_limit <= 0 or server_limit <= 0: + raise _DiscoveryUnavailableError("query_budget") + limit = min(item.limit + 1, server_limit, configured_limit) + parents = _parent_extra(context, conf["id"]) + parent_filters = parents.pop("filters", []) + form_data = { + **deepcopy(controls), + "datasource": f"{target.dataset.id}__table", + "groupby": [column], + "metrics": [], + "row_limit": limit, + "adhoc_filters": deepcopy(conf.get("adhoc_filters") or []), + "extra_filters": [], + "extra_form_data": parents, + "time_range": conf.get("time_range"), + "granularity_sqla": conf.get("granularity_sqla"), + "url_params": deepcopy(context.url_params), + "viz_type": "filter_select", + "type": "NATIVE_FILTER", + "dashboardId": context.dashboard.id, + "native_filter_id": conf["id"], + } + query = build_query_dicts_from_form_data( + form_data, target.dataset.id, "table", row_limit=limit + )[0] + # Cross-dataset parents retain native coercion when column families differ. + # Compatible strings/numbers use exact scalars, preserving whitespace/sentinels. + for predicate in parent_filters: + parent_column = next( + ( + col + for col in target.dataset.columns + if col.column_name == predicate["col"] + ), + None, + ) + values = predicate.get("val") + values = values if isinstance(values, list) else [values] + literal = parent_column is not None and all( + _compatible(value, _Target(target.dataset, parent_column)) + for value in values + ) + query["filters"].append({**predicate, "is_literal": literal}) + query["columns"] = [column] + query.setdefault("extras", {})["preserve_literal_values"] = True + query["metrics"] = [metric] if metric else [] + if target.column.type_generic == GenericDataType.NUMERIC: + # Pandas represents both SQL NULL and database NaN as NaN in float + # columns. COUNT(target) distinguishes those groups without another query. + count_label = "__mcp_native_nonnull_count" + while count_label in (column, metric): + count_label += "_" + query["metrics"].append( + { + "expressionType": "SIMPLE", + "aggregate": "COUNT", + "column": {"column_name": column}, + "label": count_label, + } + ) + query["orderby"] = ( + [[metric or column, controls.get("sortAscending", True)]] + if metric or not item.search + else [] + ) + if proposal is not None: + query["filters"].append( + { + "col": column, + "op": FilterOperator.IN.value, + "val": proposal.values, + "is_literal": True, + } + ) + elif "exact_value" in item.model_fields_set: Review Comment: Fixed in 7cd346d914. The `applied_template_filters` guard now covers every probe discovery adds on the target column (search, exact, and proposals), not only search. When the template consumes the column, an exact probe reports `unsupported_target` and a proposal is not validated. Your `exact_value: "Atlantis"` case is in `test_exact_probe_consumed_by_jinja_filter_macro_is_unsupported`. ########## superset/mcp_service/dashboard/filter_values.py: ########## @@ -0,0 +1,1256 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Authorized, bounded native dropdown discovery, independent of chart scope.""" + +from __future__ import annotations + +import logging +import math +import re +import time +from collections.abc import Iterator +from contextlib import contextmanager +from copy import deepcopy +from dataclasses import dataclass, field +from decimal import Decimal +from numbers import Integral, Real +from typing import Any, cast, Literal, TYPE_CHECKING + +from fastmcp.exceptions import ToolError +from flask import current_app, g +from marshmallow import ValidationError +from pandas import DataFrame, NA +from sqlalchemy.exc import CompileError, UnsupportedCompilationError + +from superset.charts.data.form_data import set_query_context_form_data +from superset.charts.schemas import ChartDataQueryContextSchema +from superset.commands.chart.data.get_data_command import ChartDataCommand +from superset.constants import EMPTY_FILTER_SQL_EXPRESSION, NO_TIME_RANGE +from superset.daos.dataset import DatasetDAO +from superset.dashboards.filter_scope import _is_divider +from superset.dashboards.permalink.schemas import DashboardPermalinkStateSchema +from superset.exceptions import ( + OAuth2Error, + OAuth2RedirectError, + SupersetSecurityException, +) +from superset.mcp_service.chart.chart_helpers import ( + build_query_dicts_from_form_data, + merge_extra_form_data, +) +from superset.mcp_service.dashboard.schemas import ( + _extract_native_filters, + FilterValueDiscovery, + FilterValueReason, + FilterValueRequest, + FilterValueScalar, + GetDashboardInfoRequest, + NativeFilterAvailableValues, + NativeFilterValueSummary, + ProposedFilterValues, +) +from superset.mcp_service.dashboard.tool.manage_native_filters import ( + _empty_select_data_mask, + _FilterValidationError, + _select_data_mask, + current_native_filter_config, +) +from superset.mcp_service.privacy import user_can_view_data_model_metadata +from superset.utils import json +from superset.utils.core import apply_max_row_limit, FilterOperator, GenericDataType + +if TYPE_CHECKING: + from superset.connectors.sqla.models import SqlaTable, TableColumn + from superset.models.dashboard import Dashboard + +_MAX_QUERIES = 20 +_SCHEDULING_SECONDS = 10 +_PAGE_SIZE = 20 +_MAX_SAFE_INTEGER = 2**53 - 1 +# Mirrors the dashboard's extractUrlParams('regular') exclusions. +_RESERVED_URL_PARAMS = frozenset({"standalone", "edit"}) +logger = logging.getLogger(__name__) + + +class _DiscoveryUnavailableError(Exception): + """A bounded, non-sensitive failure for a single dropdown.""" + + def __init__(self, reason: FilterValueReason) -> None: + super().__init__(reason) + self.reason = reason + + +@dataclass +class _Target: + """A configured table target; this is metadata, not an access grant.""" + + dataset: SqlaTable + column: TableColumn + + +@dataclass +class _Discovery: + """Call-local configuration, hydrated state, and execution accounting.""" + + dashboard: Dashboard + configs: dict[str, dict[str, Any]] + masks: dict[str, dict[str, Any]] + started: float = field(default_factory=time.monotonic) + query_count: int = 0 + proposal_errors: dict[str, FilterValueReason] = field(default_factory=dict) + targets: dict[ + tuple[int, str], + _Target | _DiscoveryUnavailableError | SupersetSecurityException, + ] = field(default_factory=dict) + mask_error_reasons: dict[str, FilterValueReason] = field(default_factory=dict) + normalized_masks: set[str] = field(default_factory=set) + url_params: dict[str, str] = field(default_factory=dict) + + +def _check_budget(context: _Discovery) -> None: + """Stop preparation and authorization after a call reaches either budget.""" + if context.query_count >= _MAX_QUERIES: + raise _DiscoveryUnavailableError("query_budget") + if time.monotonic() - context.started >= _SCHEDULING_SECONDS: + raise _DiscoveryUnavailableError("scheduling_budget") + + +def _configured_filters(dashboard: Dashboard) -> list[dict[str, Any]]: + """Read saved definitions in order, excluding visual dividers.""" + return [c for c in current_native_filter_config(dashboard) if not _is_divider(c)] + + +def _select_target(conf: dict[str, Any], context: _Discovery) -> _Target: + """Memoize a native target's metadata preflight, never its final query gate.""" + _check_budget(context) + controls = conf.get("controlValues") or {} + if ( + conf.get("filterType") != "filter_select" + or not isinstance(controls, dict) + or controls.get("inverseSelection") + or controls.get("operatorType", "exact") != "exact" + ): + raise _DiscoveryUnavailableError("unsupported_filter") + targets = conf.get("targets") + if not isinstance(targets, list) or len(targets) != 1: + raise _DiscoveryUnavailableError("unsupported_target") + target = targets[0] + if ( + not isinstance(target, dict) + or target.get("datasourceType") not in (None, "table") + or target.get("semantic_selection_version") + or type(target.get("datasetId")) is not int + or target["datasetId"] <= 0 + or not isinstance(target.get("column"), dict) + or not isinstance(target["column"].get("name"), str) + or not target["column"]["name"] + ): + raise _DiscoveryUnavailableError("unsupported_target") + key = (target["datasetId"], conf["id"]) + cached = context.targets.get(key) + if isinstance(cached, Exception): + raise cached + if cached is not None: + return cached + dataset = DatasetDAO.find_by_id(target["datasetId"], skip_base_filter=True) + if dataset is None: + raise _DiscoveryUnavailableError("unsupported_target") + column = next( + (col for col in dataset.columns if col.column_name == target["column"]["name"]), + None, + ) + if column is None: + raise _DiscoveryUnavailableError("unsupported_target") + # Native authorization preserves dashboard-only grants and guest limits. + # This memoized context is never executed. Every final query is authorized + # separately with its complete predicates before dataframe cache access. + access_context = ChartDataQueryContextSchema().load( + { + "datasource": {"id": dataset.id, "type": "table"}, + "queries": [ + {"columns": [column.column_name], "metrics": [], "row_limit": 1} + ], + "form_data": { + "type": "NATIVE_FILTER", + "dashboardId": context.dashboard.id, + "native_filter_id": conf["id"], + }, + "result_type": "full", + "result_format": "json", + } + ) + _check_budget(context) + try: + ChartDataCommand(access_context).validate() + except SupersetSecurityException as ex: + context.targets[key] = ex + raise + if column.type_generic not in ( + GenericDataType.STRING, + GenericDataType.NUMERIC, + GenericDataType.BOOLEAN, + ): + error = _DiscoveryUnavailableError("unsupported_target") + context.targets[key] = error + raise error + resolved = _Target(dataset, column) + context.targets[key] = resolved + return resolved + + +def _compatible(value: FilterValueScalar, target: _Target) -> bool: + """Compare JSON scalar families, keeping booleans distinct from numbers.""" + if value is None: + return True + if type(value) not in (str, bool, int, float) or ( + isinstance(value, float) and not math.isfinite(value) + ): + return False + if isinstance(value, str): + return target.column.type_generic == GenericDataType.STRING + if type(value) is bool: + return target.column.type_generic == GenericDataType.BOOLEAN + return target.column.type_generic == GenericDataType.NUMERIC + + +def _ancestors(filter_id: str, configs: dict[str, dict[str, Any]]) -> list[str]: + """Resolve ancestors furthest first and reject missing or cyclic graphs.""" + ordered: list[str] = [] + active: set[str] = set() + visited: set[str] = set() + + def visit(current: str) -> None: + """Walk one configuration, retaining stable parent ordering.""" + if current in active or current not in configs: + raise _DiscoveryUnavailableError("parent_unresolved") + if current in visited: + return + active.add(current) + parents = configs[current].get("cascadeParentIds") or [] + if not isinstance(parents, list) or any( + not isinstance(p, str) for p in parents + ): + raise _DiscoveryUnavailableError("parent_unresolved") + for parent in parents: + visit(parent) + active.remove(current) + visited.add(current) + if current != filter_id: + ordered.append(current) + + visit(filter_id) + return ordered + + +def _requests( + configs: list[dict[str, Any]], + request: GetDashboardInfoRequest, +) -> list[FilterValueRequest]: + """Choose explicit IDs or a bounded page before querying any datasource.""" + if request.filter_value_requests is not None: + return request.filter_value_requests + return [ + FilterValueRequest(filter_id=c["id"]) + for c in configs[ + request.filter_values_offset : request.filter_values_offset + _PAGE_SIZE + ] + ] + + +def validate_discovery_configuration( # noqa: C901 + dashboard: Dashboard, + request: GetDashboardInfoRequest, + *, + started_at: float | None = None, +) -> _Discovery: + """Reject invalid IDs, proposed ancestors, and scalar types before querying.""" + configured = _configured_filters(dashboard) + configs: dict[str, dict[str, Any]] = { + c["id"]: c for c in configured if isinstance(c.get("id"), str) + } + if any(not isinstance(c.get("id"), str) or not c["id"] for c in configured) or len( + configs + ) != len(configured): + raise ValueError("Native filter configuration has invalid or duplicate IDs") + context = _Discovery( + dashboard, + configs, + {}, + started=time.monotonic() if started_at is None else started_at, + ) + requests = _requests(configured, request) + for item in requests: + if item.filter_id not in configs: + raise ValueError(f"Unknown native filter ID: {item.filter_id}") + if "exact_value" in item.model_fields_set: + try: + target = _select_target(configs[item.filter_id], context) + except (_DiscoveryUnavailableError, SupersetSecurityException): + continue + if not _compatible(item.exact_value, target): + raise ValueError( + f"Incompatible exact value for native filter {item.filter_id}" + ) + allowed: set[str] = set() + for item in requests: + try: + allowed.update(_ancestors(item.filter_id, configs)) + except _DiscoveryUnavailableError: + pending = [item.filter_id] + visited: set[str] = set() + while pending: + current = pending.pop() + if current in visited or current not in configs: + continue + visited.add(current) + parents = configs[current].get("cascadeParentIds") + if isinstance(parents, list): + pending.extend(p for p in parents if isinstance(p, str)) + allowed.update(visited - {item.filter_id}) + for proposal in request.proposed_filter_values or []: + if proposal.filter_id not in configs or proposal.filter_id not in allowed: + raise ValueError( + "Proposed filters must be configured ancestors of requested filters" + ) + conf = configs[proposal.filter_id] + controls = conf.get("controlValues") or {} + if ( + isinstance(controls, dict) + and controls.get("multiSelect") is False + and len(proposal.values) > 1 + ): + raise ValueError("A single-select proposal accepts at most one value") + try: + target = _select_target(conf, context) + except (_DiscoveryUnavailableError, SupersetSecurityException): + continue + if any(not _compatible(value, target) for value in proposal.values): + raise ValueError( + f"Incompatible proposed value for native filter {proposal.filter_id}" + ) + return context + + +def _real_value(value: Any) -> bool: + """Mirror fillNativeFilters' empty/all-null-array restoration test.""" + return value is not None and not ( + isinstance(value, list) and all(v is None for v in value) + ) + + +def _explicit_clear(mask: dict[str, Any]) -> bool: + """Recognize only the frontend's canonical match-nothing clear evidence.""" + extra = mask.get("extraFormData") or {} + if not isinstance(extra, dict): + return False + predicates = extra.get("adhoc_filters") + return isinstance(predicates, list) and any( + isinstance(p, dict) + and p.get("expressionType") == "SQL" + and p.get("clause") == "WHERE" + and p.get("sqlExpression") == EMPTY_FILTER_SQL_EXPRESSION + for p in predicates + ) + + +def _validate_snapshot_value(conf: dict[str, Any], value: Any) -> None: # noqa: C901 + """Reject malformed known native selection shapes before any option query.""" + if value is None: + return + kind = conf.get("filterType") + if kind in ("filter_select", "filter_range", "filter_timegrain") and not isinstance( + value, list + ): + raise _DiscoveryUnavailableError("state_invalid") + if kind == "filter_range": + if len(value) != 2 or any( + item is not None and type(item) not in (int, float) for item in value + ): + raise _DiscoveryUnavailableError("state_invalid") + if all(item is not None for item in value) and value[0] > value[1]: + raise _DiscoveryUnavailableError("state_invalid") + elif kind == "filter_timegrain": + if len(value) > 1 or any(not isinstance(item, str) for item in value): + raise _DiscoveryUnavailableError("state_invalid") + elif kind == "filter_time" and not isinstance(value, str): + raise _DiscoveryUnavailableError("state_invalid") + + +def _normalize_select_mask( # noqa: C901 + context: _Discovery, + conf: dict[str, Any], + mask: dict[str, Any], +) -> dict[str, Any]: + """Validate effective selections and recover lossless browser numeric strings.""" + invalid_reason = context.mask_error_reasons[conf["id"]] + state = mask.get("filterState") + controls = conf.get("controlValues") or {} + if not isinstance(state, dict) or not isinstance(controls, dict): + return {"invalid_reason": invalid_reason} + values = state.get("value") + if conf.get("filterType") != "filter_select": + try: + _validate_snapshot_value(conf, values) + except _DiscoveryUnavailableError: + return {"invalid_reason": invalid_reason} + return mask + # Empty selections need no parent column metadata. Predicate reconstruction + # still applies required-clear and unresolved-state semantics downstream. + if values is None or values == []: + return mask + try: + target = _select_target(conf, context) + except SupersetSecurityException: + return {"invalid_reason": "permission_denied"} + except _DiscoveryUnavailableError as ex: + return ( + mask + if ex.reason.startswith("unsupported_") + else {"invalid_reason": ex.reason} + ) + if not isinstance(values, list): + return {"invalid_reason": invalid_reason} + if controls.get("multiSelect") is False and len(values) > 1: + return {"invalid_reason": invalid_reason} + normalized = [] + try: + for value in values: + if ( + target.column.type_generic == GenericDataType.NUMERIC + and isinstance(value, str) + and re.fullmatch( + r"-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?", value + ) + ): + value = _scalar(Decimal(value)) + if ( + target.column.type_generic == GenericDataType.BOOLEAN + and type(value) in (int, float) + and value in (0, 1) + ): + value = bool(value) + if not _compatible(value, target): + return {"invalid_reason": invalid_reason} + if not _safe_number(value): + return {"invalid_reason": "invalid_scalar"} + normalized.append(value) + except _DiscoveryUnavailableError as ex: + return {"invalid_reason": ex.reason} + except ArithmeticError: + return {"invalid_reason": "invalid_scalar"} + state["value"] = normalized + return mask + + +def _url_params(pairs: Any) -> dict[str, str]: + """Rebuild the dashboard's query url_params from snapshot key-value pairs. + + The dashboard sends every non-reserved URL parameter with each dropdown + query, so ``url_param`` macros in virtual dataset SQL render against them. + """ + if pairs is None: + return {} + if not isinstance(pairs, list): + raise _DiscoveryUnavailableError("state_invalid") + params: dict[str, str] = {} + for pair in pairs: + if pair is None: + # The permalink schema permits null entries; they carry no parameter. + continue + if ( + not isinstance(pair, (list, tuple)) + or len(pair) != 2 + or any(part is not None and not isinstance(part, str) for part in pair) + ): + raise _DiscoveryUnavailableError("state_invalid") + key, value = pair + if key is None or value is None or key in _RESERVED_URL_PARAMS: + continue + # URLSearchParams keeps the last value for a repeated key. + params[key] = value + return params + + +def _hydrate_masks( # noqa: C901 + context: _Discovery, + snapshot_state: dict[str, Any] | None, +) -> dict[str, dict[str, Any]]: + """Shallow-merge initial/default/loaded masks and restore required defaults.""" + configs = context.configs + loaded_masks: dict[str, Any] = {} + if snapshot_state is not None: + if not isinstance(snapshot_state, dict): + raise _DiscoveryUnavailableError("state_invalid") + supplied_masks = snapshot_state.get("dataMask") + if isinstance(supplied_masks, dict): + # Explicit masks are authoritative even with extra permalink metadata. + loaded_masks = supplied_masks + elif supplied_masks is not None: + raise _DiscoveryUnavailableError("state_invalid") + else: + try: + state = DashboardPermalinkStateSchema().load(snapshot_state) + except ValidationError as ex: + raise _DiscoveryUnavailableError("state_invalid") from ex + if any( + pair and pair[0] in ("native_filters", "native_filters_key") + for pair in state.get("urlParams") or [] + ): + # URL-encoded selections are not native dataMask state. + raise _DiscoveryUnavailableError("state_invalid") + context.url_params = _url_params(snapshot_state.get("urlParams")) + for filter_id, mask in loaded_masks.items(): + if not isinstance(filter_id, str) or not isinstance(mask, dict): + raise _DiscoveryUnavailableError("state_invalid") + if filter_id not in configs: + continue + if any( + key in mask and not isinstance(mask[key], dict) + for key in ("filterState", "extraFormData", "ownState") + ): + raise _DiscoveryUnavailableError("state_invalid") + value = mask.get("filterState", {}).get("value") + values = value if isinstance(value, list) else [value] + for scalar in values: + if scalar is not None and ( + type(scalar) not in (str, int, float, bool) + or (isinstance(scalar, float) and not math.isfinite(scalar)) + ): + raise _DiscoveryUnavailableError("state_invalid") + hydrated: dict[str, dict[str, Any]] = {} + for filter_id, conf in configs.items(): + default = conf.get("defaultDataMask") or {} + loaded = loaded_masks.get(filter_id, {}) + context.mask_error_reasons[filter_id] = ( + "state_invalid" if "filterState" in loaded else "parent_unresolved" + ) + if not isinstance(default, dict): + hydrated[filter_id] = {"invalid": True} + continue + mask = { + "filterState": {}, + "extraFormData": {}, + **deepcopy(default), + **deepcopy(loaded), + } + controls = conf.get("controlValues") or {} + loaded_state = loaded.get("filterState") or {} + if not isinstance(loaded_state, dict) or not isinstance(controls, dict): + hydrated[filter_id] = {"invalid": True} + continue + has_value = _real_value(loaded_state.get("value")) + explicit_clear = ( + conf.get("filterType") == "filter_select" + and not has_value + and _explicit_clear(loaded) + ) + if ( + controls.get("enableEmptyFilter") + and default + and not explicit_clear + and ( + not has_value + or (not loaded.get("extraFormData") and default.get("extraFormData")) + ) + ): + mask["filterState"] = deepcopy(default.get("filterState") or {}) + mask["extraFormData"] = deepcopy(default.get("extraFormData") or {}) + context.mask_error_reasons[filter_id] = "parent_unresolved" + hydrated[filter_id] = mask + return hydrated + + +def _selected_extra( # noqa: C901 + context: _Discovery, conf: dict[str, Any], mask: dict[str, Any] +) -> dict[str, Any]: + """Rebuild parent predicates from typed values, never executing snapshot SQL.""" + from superset.mcp_service.dashboard.tool.apply_dashboard_filters import ( + _range_data_mask, + _time_data_mask, + ) + + state = mask.get("filterState") + if mask.get("invalid_reason"): + raise _DiscoveryUnavailableError(mask["invalid_reason"]) + if mask.get("invalid") or not isinstance(state, dict): + raise _DiscoveryUnavailableError("parent_unresolved") + controls = conf.get("controlValues") or {} + value = state.get("value") + if controls.get("defaultToFirstItem") and "value" not in state: + raise _DiscoveryUnavailableError("parent_unresolved") + kind = conf.get("filterType") + if kind != "filter_select" and value is None and mask.get("extraFormData"): + # As with Select, predicates without a typed value cannot be rebuilt, + # and reading them as unfiltered would overstate the child's choices. + raise _DiscoveryUnavailableError("parent_unresolved") + try: + if kind == "filter_select": + values = [] if value is None else value + if values == [] and not ( + value is None + and mask.get("extraFormData") + and not _explicit_clear(mask) + ): + return cast( + dict[str, Any], _empty_select_data_mask(conf)["extraFormData"] + ) + target = _select_target(conf, context) + if not isinstance(values, list) or any( + not _compatible(v, target) for v in values + ): + raise _DiscoveryUnavailableError("parent_unresolved") + if any(not _safe_number(v) for v in values): + raise _DiscoveryUnavailableError("invalid_scalar") + # SQL-only state is not a selected value. A canonical clear is the + # sole SQL-shaped evidence we recognize and regenerate ourselves. + if ( + value is None + and mask.get("extraFormData") + and not _explicit_clear(mask) + ): + raise _DiscoveryUnavailableError("parent_unresolved") + return cast( + dict[str, Any], _select_data_mask(conf, values)["extraFormData"] + ) + if kind == "filter_time": + if value is not None and not isinstance(value, str): + raise _DiscoveryUnavailableError("parent_unresolved") + return cast( + dict[str, Any], + _time_data_mask(conf, value or NO_TIME_RANGE)["extraFormData"], + ) + if kind == "filter_range": + values = [None, None] if value is None else value + if ( + not isinstance(values, list) + or len(values) != 2 + or any( + v is not None + and (type(v) not in (int, float) or not math.isfinite(v)) + for v in values + ) + ): + raise _DiscoveryUnavailableError("parent_unresolved") + if all(v is not None for v in values) and values[0] > values[1]: + raise _DiscoveryUnavailableError("parent_unresolved") + return cast(dict[str, Any], _range_data_mask(conf, values)["extraFormData"]) + if kind == "filter_timegrain": + values = [] if value is None else value + if ( + not isinstance(values, list) + or len(values) > 1 + or any(not isinstance(v, str) for v in values) + ): + raise _DiscoveryUnavailableError("parent_unresolved") + if not values: + if controls.get("enableEmptyFilter"): + raise _DiscoveryUnavailableError("parent_unresolved") + return {} + targets = conf.get("targets") + if ( + not isinstance(targets, list) + or len(targets) != 1 + or not isinstance(targets[0], dict) + ): + raise _DiscoveryUnavailableError("parent_unresolved") + grain_target = targets[0] + if ( + grain_target.get("datasourceType") not in (None, "table") + or type(grain_target.get("datasetId")) is not int + ): + raise _DiscoveryUnavailableError("parent_unresolved") + dataset = DatasetDAO.find_by_id( + grain_target["datasetId"], skip_base_filter=True + ) + allowed = conf.get("time_grains") + if ( + dataset is None + or (allowed and values[0] not in allowed) + or values[0] + not in [duration for duration, _ in dataset.time_grain_sqla] + ): + raise _DiscoveryUnavailableError("parent_unresolved") + return {"time_grain_sqla": values[0]} + except _DiscoveryUnavailableError as ex: + if ex.reason in ("invalid_scalar", "query_budget", "scheduling_budget"): + raise + raise _DiscoveryUnavailableError("parent_unresolved") from ex + except ArithmeticError as ex: + raise _DiscoveryUnavailableError("invalid_scalar") from ex + except (TypeError, ValueError, _FilterValidationError) as ex: + raise _DiscoveryUnavailableError("parent_unresolved") from ex + raise _DiscoveryUnavailableError("parent_unresolved") + + +def _parent_extra(context: _Discovery, filter_id: str) -> dict[str, Any]: + """Merge transitive parents in native order, excluding self and nonparents.""" + extra: dict[str, Any] = {} + for parent in _ancestors(filter_id, context.configs): + if parent in context.proposal_errors: + raise _DiscoveryUnavailableError(context.proposal_errors[parent]) + if parent not in context.normalized_masks: + context.masks[parent] = _normalize_select_mask( + context, context.configs[parent], context.masks[parent] + ) + context.normalized_masks.add(parent) + extra = merge_extra_form_data( + extra, + _selected_extra(context, context.configs[parent], context.masks[parent]), + ) + return extra + + +def _native_query( # noqa: C901 + context: _Discovery, + conf: dict[str, Any], + target: _Target, + item: FilterValueRequest, + proposal: ProposedFilterValues | None = None, +) -> tuple[ChartDataCommand, int]: + """Build the Select dropdown QueryContext and authorize it before cache use.""" + column = target.column.column_name + controls = conf.get("controlValues") or {} + metric = conf.get("sortMetric") or controls.get("sortMetric") + if metric is not None and not isinstance(metric, str): + raise _DiscoveryUnavailableError("unsupported_target") + configured_limit = current_app.config.get("NATIVE_FILTER_DEFAULT_ROW_LIMIT", 1000) + server_limit = apply_max_row_limit(item.limit + 1) + if type(configured_limit) is not int or configured_limit <= 0 or server_limit <= 0: + raise _DiscoveryUnavailableError("query_budget") + limit = min(item.limit + 1, server_limit, configured_limit) + parents = _parent_extra(context, conf["id"]) + parent_filters = parents.pop("filters", []) + form_data = { + **deepcopy(controls), + "datasource": f"{target.dataset.id}__table", + "groupby": [column], + "metrics": [], + "row_limit": limit, + "adhoc_filters": deepcopy(conf.get("adhoc_filters") or []), + "extra_filters": [], + "extra_form_data": parents, + "time_range": conf.get("time_range"), + "granularity_sqla": conf.get("granularity_sqla"), + "url_params": deepcopy(context.url_params), + "viz_type": "filter_select", + "type": "NATIVE_FILTER", + "dashboardId": context.dashboard.id, + "native_filter_id": conf["id"], + } + query = build_query_dicts_from_form_data( + form_data, target.dataset.id, "table", row_limit=limit + )[0] + # Cross-dataset parents retain native coercion when column families differ. + # Compatible strings/numbers use exact scalars, preserving whitespace/sentinels. + for predicate in parent_filters: + parent_column = next( + ( + col + for col in target.dataset.columns + if col.column_name == predicate["col"] + ), + None, + ) + values = predicate.get("val") + values = values if isinstance(values, list) else [values] + literal = parent_column is not None and all( + _compatible(value, _Target(target.dataset, parent_column)) + for value in values + ) + query["filters"].append({**predicate, "is_literal": literal}) Review Comment: Agreed, fixed in 7cd346d914. Parent selections now go through native coercion like the dashboard's child dropdown query, and only discovery's own target predicates stay literal. Your NULL vs `'<NULL>'` rows are covered by `test_parent_sentinel_selection_matches_dashboard_null_coercion`. ########## superset/mcp_service/dashboard/schemas.py: ########## @@ -286,6 +291,145 @@ class ListDashboardsRequest( ] +def _validate_filter_value_text(value: str) -> str: + """Reject unpaired surrogates that cannot be represented in the UTF-8 response.""" + try: + value.encode("utf-8") + except UnicodeEncodeError as ex: + raise ValueError("Filter discovery strings must be valid UTF-8") from ex + return value + + +FilterValueText = Annotated[StrictStr, AfterValidator(_validate_filter_value_text)] + + +def _validate_filter_value_scalar(value: object) -> object: + """Reject non-JSON scalars without coercing their type or content.""" + if value is not None and type(value) not in (str, int, float, bool): + raise ValueError("Filter values must be JSON scalars") + if isinstance(value, float) and not math.isfinite(value): + raise ValueError("Filter values must be finite") + return value + + +FilterValueScalar = Annotated[ + FilterValueText | StrictBool | StrictInt | StrictFloat | None, + BeforeValidator(_validate_filter_value_scalar), +] + + +class FilterValueRequest(BaseModel): + """One bounded dropdown listing, literal search, or exact typed probe.""" + + model_config = ConfigDict(extra="forbid") + + filter_id: FilterValueText = Field( + min_length=1, description="Configured native filter ID" + ) + limit: StrictInt = Field(default=25, ge=1, le=100) + search: FilterValueText | None = Field(default=None, max_length=200) Review Comment: Fixed in 7cd346d914. Neither `search` nor `exact_value` advertises a default anymore, and `search` is a non-null string, so clients that fill in defaults keep a plain listing. An explicit `exact_value: null` is still the IS NULL probe. Covered by `test_discovery_mode_fields_advertise_no_default`. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
