aminghadersohi commented on code in PR #43771:
URL: https://github.com/apache/superset/pull/43771#discussion_r4202059805
##########
superset/common/form_data_query_context.py:
##########
@@ -288,71 +461,1305 @@ def _pie_contribution_post_processing(metrics:
list[Any]) -> list[dict[str, Any]
]
-def build_query_context_from_form_data(
- form_data: dict[str, Any],
- datasource: dict[str, Any],
- viz_type: str | None = None,
+def _as_list(value: Any) -> list[Any]:
+ """Return the frontend ``ensureIsArray`` representation of a value."""
+ if value is None:
+ return []
+ return list(value) if isinstance(value, (list, tuple)) else [value]
+
+
+def _label(value: Any, *, metric: bool = False) -> str:
+ """Resolve a frontend-compatible query-field label."""
+ try:
+ return get_metric_name(value) if metric else get_column_name(value)
+ except (AttributeError, KeyError, TypeError, ValueError):
+ if isinstance(value, Mapping):
+ return str(
+ value.get("label")
+ or value.get("column_name")
+ or value.get("sqlExpression")
+ or value
+ )
+ return str(value)
+
+
+def _deduplicate_fields(values: list[Any], *, metric: bool = False) ->
list[Any]:
+ """Deduplicate query fields by their frontend-visible label."""
+ result: list[Any] = []
+ labels: set[str] = set()
+ for value in values:
+ if value is None or value == "":
+ continue
+ label = _label(value, metric=metric)
+ if label in labels:
+ continue
+ labels.add(label)
+ result.append(value)
+ return result
+
+
+def retain_mixed_timeseries_secondary_form_data(
+ form_data: Mapping[str, Any],
) -> dict[str, Any]:
- """
- Build a query-context payload (the JSON shape
``ChartDataQueryContextSchema``
- loads) from a chart's form data and datasource reference.
+ """Mirror ``retainFormDataSuffix(formData, '_b')`` exactly.
- :param form_data: The chart's saved ``params`` parsed to a dict.
- :param datasource: ``{"id": <int>, "type": "table"}`` datasource reference.
- :param viz_type: The chart's viz type, used for viz-specific handling.
- :returns: A single-query query-context dict.
+ Suffixed values are installed first, including falsey values, and shared
+ unsuffixed controls fill only keys that query B did not explicitly set.
"""
- columns, metrics = _columns_and_metrics(form_data, viz_type)
-
- # SIMPLE adhoc filters (+ legacy top-level ``filters``) become query
filters;
- # free-form SQL predicates go into ``extras``. Only ``WHERE``-clause SIMPLE
- # filters are applied (matching the chart), so the export never filters on
a
- # ``HAVING`` clause the chart itself ignores.
- filters = adhoc_filters_to_query_filters(
- form_data.get("adhoc_filters", []), where_only=True
- )
- for flt in form_data.get("filters") or []:
- if isinstance(flt, dict) and flt.get("col") is not None:
- filters.append(flt)
+ secondary: dict[str, Any] = {}
+ for key, value in form_data.items():
+ if key.endswith("_b"):
+ secondary[key[:-2]] = value
+ for key, value in form_data.items():
+ if not key.endswith("_b") and key not in secondary:
+ secondary[key] = value
+ secondary_filter_keys = {
+ "adhoc_filters": "adhoc_filters_b",
+ "extra_filters": "extra_filters_b",
+ "filters": "filters_b",
+ "having": "having_b",
+ "where": "where_b",
+ }
+ if any(suffixed in form_data for suffixed in
secondary_filter_keys.values()):
+ # The frontend exposes adhoc_filters_b, while saved/server payloads can
+ # carry equivalent legacy aliases. Treat the family atomically: an
+ # explicit clear in any B alias must not be repopulated by query A's
+ # differently named filter representation.
+ for primary, suffixed in secondary_filter_keys.items():
+ if suffixed not in form_data:
+ secondary.pop(primary, None)
+ return secondary
- extras = freeform_where_having(form_data)
- if form_data.get("time_grain_sqla"):
- extras["time_grain_sqla"] = form_data["time_grain_sqla"]
- # Prefer the modern ``time_range``; fall back to the legacy
``since``/``until``
- # pair (older charts store the range that way) before defaulting to no
filter.
- time_range = form_data.get("time_range")
- if not time_range and (form_data.get("since") or form_data.get("until")):
- time_range = f"{form_data.get('since') or ''} :
{form_data.get('until') or ''}"
- time_range = time_range or "No filter"
+def _base_query_object( # noqa: C901
+ form_data: dict[str, Any],
+ *,
+ row_limit: int | None,
+ order_desc: bool | None,
+ filters_prepared: bool,
+) -> dict[str, Any]:
+ """Build the shared frontend-equivalent portion of a QueryObject."""
+ columns, metrics, orderby = query_fields_from_form_data(form_data)
query: dict[str, Any] = {
"columns": columns,
"metrics": metrics,
- "orderby": orderby_from_form_data(form_data, metrics, viz_type),
- "filters": filters,
- "time_range": time_range,
}
+ if orderby:
+ query["orderby"] = orderby
+
+ if filters_prepared:
+ query["filters"] = list(form_data.get("filters") or [])
+ for clause in ("where", "having"):
+ if form_data.get(clause):
+ query[clause] = form_data[clause]
+ if form_data.get("extras"):
+ query["extras"] = dict(form_data["extras"])
+ else:
+ filters = adhoc_filters_to_query_filters(
+ form_data.get("adhoc_filters", []), where_only=True
+ )
+ filters.extend(
+ filter_
+ for filter_ in form_data.get("filters") or []
+ if isinstance(filter_, dict) and filter_.get("col") is not None
+ )
+ query["filters"] = filters
+ if extras := freeform_where_having(form_data):
+ query["extras"] = extras
+
+ extras = dict(query.get("extras") or {})
+ if form_data.get("time_grain_sqla") is not None:
+ extras["time_grain_sqla"] = form_data["time_grain_sqla"]
if extras:
query["extras"] = extras
- if viz_type == "pie" and (
- post_processing := _pie_contribution_post_processing(metrics)
+
+ effective_limit = row_limit if row_limit is not None else
form_data.get("row_limit")
+ if effective_limit is not None:
+ query["row_limit"] = effective_limit
+ if form_data.get("row_offset") is not None:
+ query["row_offset"] = form_data["row_offset"]
+ if order_desc is not None:
+ query["order_desc"] = order_desc
+ elif "order_desc" in form_data and form_data["order_desc"] is not None:
+ query["order_desc"] = form_data["order_desc"]
+
+ time_range = form_data.get("time_range")
+ if not time_range and (form_data.get("since") or form_data.get("until")):
+ time_range = f"{form_data.get('since') or ''} :
{form_data.get('until') or ''}"
+ if time_range:
+ query["time_range"] = time_range
+ for key in ("since", "until", "annotation_layers", "url_params",
"custom_params"):
+ if form_data.get(key) is not None:
+ query[key] = form_data[key]
+
+ granularity = form_data.get("granularity") or
form_data.get("granularity_sqla")
+ if granularity:
+ query["granularity"] = granularity
+ series_limit = form_data.get("series_limit", form_data.get("limit"))
+ if series_limit is not None:
+ query["series_limit"] = series_limit
+ series_limit_metric = form_data.get("series_limit_metric")
+ if series_limit_metric is None:
+ series_limit_metric = form_data.get("timeseries_limit_metric")
+ if series_limit_metric is not None:
+ query["series_limit_metric"] = series_limit_metric
+ if form_data.get("group_others_when_limit_reached") is not None:
+ query["group_others_when_limit_reached"] = form_data[
+ "group_others_when_limit_reached"
+ ]
+ return query
+
+
+def _temporalized_columns(form_data: dict[str, Any], columns: list[Any]) ->
list[Any]:
+ """Apply the pivot BASE_AXIS temporal-column contract."""
+ time_grain = form_data.get("time_grain_sqla")
+ temporal_lookup = form_data.get("temporal_columns_lookup") or {}
+ result: list[Any] = []
+ for column in columns:
+ if (
+ isinstance(column, str)
+ and time_grain
+ and (
+ temporal_lookup.get(column)
+ or form_data.get("granularity_sqla") == column
+ )
+ ):
+ result.append(
+ {
+ "timeGrain": time_grain,
+ "columnType": "BASE_AXIS",
+ "sqlExpression": column,
+ "label": column,
+ "expressionType": "SQL",
+ }
+ )
+ else:
+ result.append(column)
+ return result
+
+
+def _box_temporalized_columns(
+ form_data: dict[str, Any], columns: list[Any]
+) -> list[Any]:
+ """Convert only physical columns confirmed temporal by Box Plot
metadata."""
+ time_grain = form_data.get("time_grain_sqla")
+ temporal_lookup = form_data.get("temporal_columns_lookup")
+ if not time_grain or not isinstance(temporal_lookup, Mapping):
+ return columns
+ return [
+ {
+ "timeGrain": time_grain,
+ "columnType": "BASE_AXIS",
+ "sqlExpression": column,
+ "label": column,
+ "expressionType": "SQL",
+ }
+ if isinstance(column, str) and temporal_lookup.get(column) is True
+ else column
+ for column in columns
+ ]
+
+
+def _table_temporalized_columns(
+ form_data: dict[str, Any], columns: list[Any]
+) -> list[Any]:
+ """Promote the first temporal table group-by to the frontend BASE_AXIS.
+
+ Table's builder treats only physical columns named in
+ ``temporal_columns_lookup`` as temporal and moves the first match to the
+ front. Later temporal dimensions remain ordinary group-bys.
+ """
+ time_grain = form_data.get("time_grain_sqla")
+ temporal_lookup = form_data.get("temporal_columns_lookup") or {}
+ if not time_grain or not isinstance(temporal_lookup, Mapping):
+ return columns
+
+ temporal_column: dict[str, Any] | None = None
+ remaining: list[Any] = []
+ for column in columns:
+ if (
+ temporal_column is None
+ and isinstance(column, str)
+ and temporal_lookup.get(column)
+ ):
+ temporal_column = {
+ "timeGrain": time_grain,
+ "columnType": "BASE_AXIS",
+ "sqlExpression": column,
+ "label": column,
+ "expressionType": "SQL",
+ "isColumnReference": True,
+ }
+ else:
+ remaining.append(column)
+ return [temporal_column, *remaining] if temporal_column else columns
+
+
+def _histogram_query(form_data: dict[str, Any], query: dict[str, Any]) -> None:
+ groupby = _as_list(form_data.get("groupby"))
+ column = form_data.get("column")
+ query["columns"] = [*groupby, *([column] if column is not None else [])]
+ query["post_processing"] = [
+ {
+ "operation": "histogram",
+ "options": {
+ "column": _label(column),
+ "groupby": [_label(value) for value in groupby],
+ "bins": int(form_data.get("bins", 5)),
+ "cumulative": form_data.get("cumulative", False),
+ "normalize": form_data.get("normalize", False),
+ },
+ }
+ ]
+ if any(
+ isinstance(filter_, dict) and filter_.get("clause") == "HAVING"
+ for filter_ in form_data.get("adhoc_filters") or []
):
+ query["metrics"] = [
+ {
+ "expressionType": "SQL",
+ "sqlExpression": "COUNT(*)",
+ "label": "COUNT(*)",
+ }
+ ]
+ else:
+ query["metrics"] = []
+
+
+def _box_plot_query(form_data: dict[str, Any], query: dict[str, Any]) -> None:
+ distributed = _as_list(form_data.get("columns"))
+ if not distributed and form_data.get("granularity_sqla"):
+ distributed = [form_data["granularity_sqla"]]
+ groupby = _as_list(form_data.get("groupby"))
+ query["columns"] = [*_box_temporalized_columns(form_data, distributed),
*groupby]
+ query["series_columns"] = groupby
+ whisker = form_data.get("whiskerOptions")
+ if not whisker:
+ query["post_processing"] = []
+ return
+ whisker_type = "tukey"
+ percentiles: list[int] | None = None
+ if whisker == "Min/max (no outliers)":
+ whisker_type = "min/max"
+ elif isinstance(whisker, str) and whisker.endswith(" percentiles"):
+ low, high = whisker.removesuffix(" percentiles").split("/", 1)
+ whisker_type = "percentile"
+ percentiles = [int(low), int(high)]
+ query["post_processing"] = [
+ {
+ "operation": "boxplot",
+ "options": {
+ "whisker_type": whisker_type,
+ "percentiles": percentiles,
+ "groupby": [_label(value) for value in groupby],
+ "metrics": [_label(value, metric=True) for value in
query["metrics"]],
+ },
+ }
+ ]
+
+
+_PIVOT_ADDITIVE_AGGREGATES = frozenset({"SUM", "COUNT", "MIN", "MAX"})
+
+
+def _all_metrics_additive(metrics: list[Any]) -> bool:
+ """Mirror Pivot's conservative additive-metric fast-path."""
+ return bool(metrics) and all(
+ isinstance(metric, Mapping)
+ and metric.get("expressionType") == "SIMPLE"
+ and metric.get("aggregate") in _PIVOT_ADDITIVE_AGGREGATES
+ for metric in metrics
+ )
+
+
+def _pivot_grouping_sets(
+ form_data: dict[str, Any], rows: list[Any], columns: list[Any]
+) -> list[list[str]]:
+ """Enumerate the rollup levels requested by Pivot's frontend builder."""
+ row_prefixes = [[], *(rows[: index + 1] for index in range(len(rows)))]
+ column_prefixes = [
+ [],
+ *(columns[: index + 1] for index in range(len(columns))),
+ ]
+ show_values_as = form_data.get("showValuesAs")
+ needs_rows_collapsed = show_values_as in {"percent_col", "percent_total"}
+ needs_columns_collapsed = show_values_as in {"percent_row",
"percent_total"}
+
+ def row_prefix_needed(prefix: list[Any]) -> bool:
+ if len(prefix) == len(rows):
+ return True
+ if not prefix:
+ return bool(form_data.get("colTotals")) or needs_rows_collapsed
+ return bool(form_data.get("rowSubTotals"))
+
+ def column_prefix_needed(prefix: list[Any]) -> bool:
+ if len(prefix) == len(columns):
+ return True
+ if not prefix:
+ return bool(form_data.get("rowTotals")) or needs_columns_collapsed
+ return bool(form_data.get("colSubTotals"))
+
+ levels = [
+ (row_prefix, column_prefix)
+ for row_prefix in row_prefixes
+ if row_prefix_needed(row_prefix)
+ for column_prefix in column_prefixes
+ if column_prefix_needed(column_prefix)
+ ]
+ if form_data.get("combineMetric"):
+ metrics_layout = form_data.get("metricsLayout")
+
+ def forced_denominator(level: tuple[list[Any], list[Any]]) -> bool:
+ row_prefix, column_prefix = level
+ return (needs_rows_collapsed and not row_prefix) or (
+ needs_columns_collapsed and not column_prefix
+ )
+
+ if metrics_layout == "ROWS":
+ levels = [
+ level
+ for level in levels
+ if len(level[0]) == len(rows) or forced_denominator(level)
+ ]
+ else:
+ levels = [
+ level
+ for level in levels
+ if len(level[1]) == len(columns) or forced_denominator(level)
+ ]
+
+ return [
+ [_label(value) for value in _deduplicate_fields([*row_prefix,
*column_prefix])]
+ for row_prefix, column_prefix in levels
+ ]
+
+
+def _pivot_query(form_data: dict[str, Any], query: dict[str, Any]) -> None:
+ rows = _as_list(form_data.get("groupbyRows"))
+ columns = _as_list(form_data.get("groupbyColumns"))
+ if form_data.get("transposePivot"):
+ rows, columns = columns, rows
+ query["columns"] = _temporalized_columns(
+ form_data, _deduplicate_fields([*rows, *columns])
+ )
+ metric = query.get("series_limit_metric") or next(
+ iter(query.get("metrics") or []), None
+ )
+ query["orderby"] = (
+ [[metric, not bool(query.get("order_desc", True))]]
+ if metric is not None
+ else []
+ )
+ if not _all_metrics_additive(query.get("metrics") or []):
+ query["grouping_sets"] = _pivot_grouping_sets(form_data, rows, columns)
+
+
+def _waterfall_query(form_data: dict[str, Any], query: dict[str, Any]) -> None:
+ x_axis = form_data.get("x_axis") or form_data.get("granularity_sqla")
+ columns = [*_as_list(x_axis), *_as_list(form_data.get("groupby"))]
+ query["columns"] = _deduplicate_fields(columns)
+ query["orderby"] = [[column, True] for column in query["columns"]]
+
+
+def _gantt_query(form_data: dict[str, Any], query: dict[str, Any]) -> None:
+ groupby = _as_list(form_data.get("series"))
+ orderby = query_fields_from_form_data(form_data)[2]
+ columns = [
+ form_data.get("start_time"),
+ form_data.get("end_time"),
+ form_data.get("y_axis"),
+ *groupby,
+ *_as_list(form_data.get("tooltip_columns")),
+ *(entry[0] for entry in orderby if entry),
+ ]
+ query["columns"] = _deduplicate_fields(columns)
+ query["metrics"] = _as_list(form_data.get("tooltip_metrics"))
+ query["orderby"] = orderby
+ query["series_columns"] = groupby
+
+
+def _normalize_query_orderby(query: dict[str, Any]) -> None:
+ """Mirror ``normalizeOrderBy`` while retaining limit-direction controls."""
+ orderby = query.get("orderby")
+ if (
+ isinstance(orderby, list)
+ and orderby
+ and isinstance(orderby[0], (list, tuple))
+ and len(orderby[0]) == 2
+ and orderby[0][0]
+ and isinstance(orderby[0][1], bool)
+ ):
+ return
+ metric = (
+ query.get("series_limit_metric")
+ or query.get("legacy_order_by")
+ or next(iter(query.get("metrics") or []), None)
+ )
+ if metric is None:
+ query.pop("orderby", None)
+ return
+ query["orderby"] = [[metric, not bool(query.get("order_desc", True))]]
+
+
+_TIME_COMPARISON_TYPES = frozenset({"values", "difference", "percentage",
"ratio"})
+
+
+def _metric_offset_map(
+ form_data: dict[str, Any],
+ metric_labels: list[str],
+ offsets: list[Any] | None = None,
+) -> dict[str, str]:
+ """Return the frontend time-comparison metric label map."""
+ if form_data.get("comparison_type") not in _TIME_COMPARISON_TYPES:
+ return {}
+ return {
+ f"{metric}__{offset}": metric
+ for metric in metric_labels
+ for offset in (
+ offsets if offsets is not None else
_as_list(form_data.get("time_compare"))
+ )
+ }
+
+
+def _table_time_offsets(form_data: dict[str, Any]) -> list[Any]:
+ """Resolve Table custom/inherited shifts like its frontend query
adapter."""
+ raw_offsets = _as_list(form_data.get("time_compare"))
+ offsets = [offset for offset in raw_offsets if offset not in {"custom",
"inherit"}]
+ if "custom" in raw_offsets and form_data.get("start_date_offset") is not
None:
+ offsets.append(form_data["start_date_offset"])
+ extra_form_data = form_data.get("extra_form_data")
+ if isinstance(extra_form_data, Mapping) and
extra_form_data.get("time_compare"):
+ inherited = extra_form_data["time_compare"]
+ if inherited not in offsets:
+ offsets = [inherited]
+ return offsets
+
+
+def _x_axis_column(form_data: Mapping[str, Any]) -> Any | None:
+ """Return a supported x-axis column, excluding legacy granularity.
+
+ ``column_name`` mappings are retained for old server/native payloads. Big
+ Number uses the stricter frontend predicate below.
+ """
+ x_axis = form_data.get("x_axis")
+ if isinstance(x_axis, str):
+ return x_axis if x_axis else None
+ if isinstance(x_axis, Mapping):
+ if isinstance(column_name := x_axis.get("column_name"), str) and
column_name:
+ return column_name
+ # Frontend SQL adhoc columns remain objects in the QueryObject.
+ return x_axis if x_axis else None
+ return None
+
+
+def _frontend_x_axis_column(form_data: Mapping[str, Any]) -> Any | None:
+ """Mirror ``isQueryFormColumn`` for physical and SQL adhoc columns."""
+ x_axis = form_data.get("x_axis")
+ if isinstance(x_axis, str):
+ return x_axis if x_axis else None
+ if (
+ isinstance(x_axis, Mapping)
+ and "sqlExpression" in x_axis
+ and "label" in x_axis
+ and x_axis.get("expressionType") in {None, "SQL"}
+ ):
+ return x_axis
+ return None
+
+
+def normalize_time_column(
+ form_data: Mapping[str, Any], query: dict[str, Any]
+) -> dict[str, Any]:
+ """Apply the final shared frontend ``normalizeTimeColumn`` mutator."""
+ x_axis = _frontend_x_axis_column(form_data)
+ columns = query.get("columns")
+ if x_axis is None or not isinstance(columns, list):
+ return query
+
+ axis_index: int | None = None
+ for index, column in enumerate(columns):
+ if isinstance(x_axis, str) and isinstance(column, str) and column ==
x_axis:
+ axis_index = index
+ break
+ if (
+ isinstance(x_axis, Mapping)
+ and isinstance(column, Mapping)
+ and column.get("sqlExpression") == x_axis.get("sqlExpression")
+ ):
+ axis_index = index
+ break
+ if axis_index is None:
+ return query
+
+ normalized = dict(query)
+ normalized_columns = list(columns)
+ grain = (query.get("extras") or {}).get("time_grain_sqla")
+ if isinstance(columns[axis_index], Mapping):
+ normalized_axis = {
+ "columnType": "BASE_AXIS",
+ **({"timeGrain": grain} if grain is not None else {}),
+ **columns[axis_index],
+ }
+ else:
+ normalized_axis = {
+ "columnType": "BASE_AXIS",
+ "sqlExpression": x_axis,
+ "label": x_axis,
+ "expressionType": "SQL",
+ "isColumnReference": True,
+ **({"timeGrain": grain} if grain is not None else {}),
+ }
+ normalized_columns[axis_index] = normalized_axis
+ normalized["columns"] = normalized_columns
+ normalized.pop("is_timeseries", None)
+ return normalized
+
+
+def _finalize_query_objects(
+ form_data: Mapping[str, Any], queries: list[dict[str, Any]]
+) -> list[dict[str, Any]]:
+ """Run shared query-context mutators after every visualization adapter."""
+ return [normalize_time_column(form_data, query) for query in queries]
+
+
+def _x_axis_label(
+ form_data: Mapping[str, Any], *, frontend_strict: bool = False
+) -> str | None:
+ """Mirror getXAxisColumn/getXAxisLabel for explicit and legacy axes."""
+ explicit = (
+ _frontend_x_axis_column(form_data)
+ if frontend_strict
+ else _x_axis_column(form_data)
+ )
+ if explicit:
+ return _label(explicit)
+ if form_data.get("granularity_sqla"):
+ return DTTM_ALIAS
+ return None
+
+
+def _rename_operator(
+ form_data: dict[str, Any],
+ query: dict[str, Any],
+ *,
+ x_axis_label: str | None,
+) -> dict[str, Any] | None:
+ """Mirror the ECharts ``renameOperator`` for Timeseries and Mixed
charts."""
+ metrics = list(query.get("metrics") or [])
+ metric_labels = [_label(metric, metric=True) for metric in metrics]
+ series_columns = query.get("series_columns")
+ columns = _as_list(
+ series_columns if series_columns is not None else query.get("columns")
+ )
+ time_offsets = _as_list(form_data.get("time_compare"))
+ offset_map = _metric_offset_map(form_data, metric_labels)
+ is_time_comparison = bool(offset_map)
+ truncate_metric = form_data.get("truncate_metric")
+
+ should_rename = (
+ bool(metrics)
+ and bool(x_axis_label)
+ and (
+ is_time_comparison
+ or (
+ (bool(columns) or len(time_offsets) > 1)
+ and "truncate_metric" in form_data
+ and bool(truncate_metric)
+ )
+ )
+ )
+ if not should_rename:
+ return None
+
+ renamed: dict[str, str | None] = {}
+ comparison_type = form_data.get("comparison_type")
+ if is_time_comparison:
+ for metric_with_offset, metric_only in offset_map.items():
+ offset_label = next(
+ (
+ str(offset)
+ for offset in time_offsets
+ if metric_with_offset.endswith(f"__{offset}")
+ ),
+ None,
+ )
+ source = (
+ metric_with_offset
+ if comparison_type == "values"
+ else f"{comparison_type}__{metric_only}__{metric_with_offset}"
+ )
+ renamed[source] = (
+ f"{metric_only}, {offset_label}" if len(metrics) > 1 else
offset_label
+ )
+
+ if (
+ comparison_type not in {"difference", "percentage", "ratio"}
+ and len(metrics) == 1
+ and not renamed
+ ):
+ renamed[metric_labels[0]] = None
+ if not renamed:
+ return None
+ return {
+ "operation": "rename",
+ "options": {"columns": renamed, "level": 0, "inplace": True},
+ }
+
+
+def _timeseries_post_processing( # noqa: C901
+ form_data: dict[str, Any],
+ query: dict[str, Any],
+ *,
+ x_axis_label: str | None,
+ groupby: list[Any],
+ mixed: bool,
+ sort_metric: Any = None,
+) -> tuple[list[dict[str, Any]], list[Any]]:
+ """Build the Timeseries/Mixed operator pipeline in frontend order."""
+ metric_labels = [_label(value, metric=True) for value in
query.get("metrics") or []]
+ sort_metric_label = (
+ _label(sort_metric, metric=True) if sort_metric is not None else None
+ )
+ offset_map = _metric_offset_map(form_data, metric_labels)
+ time_offsets = _as_list(form_data.get("time_compare")) if offset_map else
[]
+ post_processing: list[dict[str, Any]] = []
+
+ if x_axis_label and metric_labels:
+ aggregate_labels = (
+ [*offset_map.values(), *offset_map] if offset_map else
list(metric_labels)
+ )
+ if not offset_map and sort_metric_label is not None:
+ aggregate_labels.append(sort_metric_label)
+ post_processing.append(
+ {
+ "operation": "pivot",
+ "options": {
+ "index": [x_axis_label],
+ "columns": [_label(value) for value in groupby],
+ "aggregates": {
+ label: {"operator": "mean"} for label in
aggregate_labels
+ },
+ "drop_missing_columns": not form_data.get(
+ "show_empty_columns", False
+ ),
+ },
+ }
+ )
+
+ if form_data.get("resample_method") and form_data.get("resample_rule"):
+ zero_fill = form_data["resample_method"] == "zerofill"
+ post_processing.append(
+ {
+ "operation": "resample",
+ "options": {
+ "method": "asfreq" if zero_fill else
form_data["resample_method"],
+ "rule": form_data["resample_rule"],
+ "fill_value": 0 if zero_fill else None,
+ **(
+ {"fill_time_range": True}
+ if form_data.get("resample_fill_time_range")
+ else {}
+ ),
+ },
+ }
+ )
+
+ rolling_labels = (
+ [*offset_map.values(), *offset_map] if offset_map else metric_labels
+ )
+ columns_map = {label: label for label in rolling_labels}
+ rolling_type = form_data.get("rolling_type")
+ if rolling_type == "cumsum":
+ post_processing.append(
+ {
+ "operation": "cum",
+ "options": {"operator": "sum", "columns": columns_map},
+ }
+ )
+ elif rolling_type in {"sum", "mean", "std"}:
+ post_processing.append(
+ {
+ "operation": "rolling",
+ "options": {
+ "rolling_type": rolling_type,
+ "window": int(form_data.get("rolling_periods") or 1),
+ "min_periods": int(form_data.get("min_periods") or 0),
+ "columns": columns_map,
+ },
+ }
+ )
+
+ comparison_type = form_data.get("comparison_type")
+ if offset_map and comparison_type != "values":
+ post_processing.append(
+ {
+ "operation": "compare",
+ "options": {
+ "source_columns": list(offset_map.values()),
+ "compare_columns": list(offset_map),
+ "compare_type": comparison_type,
+ "drop_original_columns": True,
+ },
+ }
+ )
+
+ if not mixed and form_data.get("contributionMode"):
+ post_processing.append(
+ {
+ "operation": "contribution",
+ "options": {
+ "orientation": form_data["contributionMode"],
+ "time_shifts": time_offsets,
+ },
+ }
+ )
+
+ if rename := _rename_operator(form_data, query, x_axis_label=x_axis_label):
+ post_processing.append(rename)
+
+ if not mixed:
+ sortable = {
+ x_axis_label or "",
+ *metric_labels,
+ sort_metric_label or "",
+ }
+ if (
+ "x_axis_sort" in form_data
+ and "x_axis_sort_asc" in form_data
+ and form_data.get("x_axis_sort") in sortable
+ and not groupby
+ ):
+ options: dict[str, Any] = {"ascending":
form_data.get("x_axis_sort_asc")}
+ if form_data.get("x_axis_sort") == x_axis_label:
+ options["is_sort_index"] = True
+ else:
+ options["by"] = form_data.get("x_axis_sort")
+ post_processing.append({"operation": "sort", "options": options})
+
+ post_processing.append({"operation": "flatten"})
+ if not mixed and form_data.get("forecastEnabled") and x_axis_label:
+ x_axis = _x_axis_column(form_data)
+ axis_grain = x_axis.get("timeGrain") if isinstance(x_axis, Mapping)
else None
+ time_grain = (
+ axis_grain
+ or (query.get("extras") or {}).get("time_grain_sqla")
+ or form_data.get("time_grain_sqla")
+ or "P1D"
+ )
+ post_processing.append(
+ {
+ "operation": "prophet",
+ "options": {
+ "time_grain": time_grain,
+ "periods": int(form_data.get("forecastPeriods") or 0),
+ "confidence_interval": float(
+ form_data.get("forecastInterval") or 0
+ ),
+ "yearly_seasonality":
form_data.get("forecastSeasonalityYearly"),
+ "weekly_seasonality":
form_data.get("forecastSeasonalityWeekly"),
+ "daily_seasonality":
form_data.get("forecastSeasonalityDaily"),
+ "index": x_axis_label,
+ },
+ }
+ )
+ return post_processing, time_offsets
+
+
+def _timeseries_query(form_data: dict[str, Any], query: dict[str, Any]) ->
None:
+ groupby = _as_list(form_data.get("groupby"))
+ x_axis = _x_axis_column(form_data)
+ x_axis_label = _x_axis_label(form_data)
+ query["columns"] = _deduplicate_fields([*_as_list(x_axis), *groupby])
+ query["series_columns"] = groupby
+ if not x_axis:
+ query["is_timeseries"] = True
+
+ # Timeseries includes its sort-only metric in the SELECT when no series is
+ # present. This lets the post-processing sort operator use a metric not
+ # otherwise displayed.
+ sort_metric = form_data.get("timeseries_limit_metric")
+ if isinstance(sort_metric, list):
+ sort_metric = next(iter(sort_metric), None)
+ if (
+ not groupby
+ and sort_metric is not None
+ and _label(sort_metric, metric=True) == form_data.get("x_axis_sort")
+ and _label(sort_metric, metric=True)
+ not in {_label(metric, metric=True) for metric in query.get("metrics")
or []}
+ ):
+ extra_metric = sort_metric
+ else:
+ extra_metric = None
+ _normalize_query_orderby(query)
+ post_processing, time_offsets = _timeseries_post_processing(
+ form_data,
+ query,
+ x_axis_label=x_axis_label,
+ groupby=groupby,
+ mixed=form_data.get("viz_type") == "mixed_timeseries",
+ sort_metric=extra_metric,
+ )
+ if extra_metric is not None:
+ query.setdefault("metrics", []).append(extra_metric)
+ query["post_processing"] = post_processing
+ query["time_offsets"] = time_offsets
+ if form_data.get("viz_type") != "mixed_timeseries":
+ query["time_compare_full_range"] = bool(
+ time_offsets and form_data.get("time_compare_full_range")
+ )
+
+
+def _big_number_queries(
+ form_data: dict[str, Any], query: dict[str, Any]
+) -> list[dict[str, Any]]:
+ """Mirror Big Number with Trendline's one/two-query contract."""
+ # Saved/native Big Number payloads can carry the temporal binding as a
+ # ``{"column_name": ...}`` mapping, which the strict frontend predicate
does
+ # not recognize; keep grouping by it rather than falling back to a total.
+ frontend_x_axis = _frontend_x_axis_column(form_data)
+ explicit_x_axis = frontend_x_axis or _x_axis_column(form_data)
+ time_column = _as_list(explicit_x_axis)
+ x_axis_label = _x_axis_label(form_data, frontend_strict=True)
+ query["columns"] = time_column
+ if time_column and frontend_x_axis is None:
+ # A native ``{"column_name": ...}`` axis groups by its temporal column
+ # but is not rewritten by normalize_time_column, so drop the legacy
+ # granularity binding here rather than bucketing the same dimension
+ # twice.
+ query.pop("granularity", None)
+ extras = query.get("extras")
+ if isinstance(extras, dict):
+ extras.pop("time_grain_sqla", None)
+ if not extras:
+ query.pop("extras", None)
+ elif not time_column:
+ query["is_timeseries"] = True
+ metric_labels = [_label(value, metric=True) for value in
query.get("metrics") or []]
+ post_processing: list[dict[str, Any]] = []
+ if x_axis_label and metric_labels:
+ post_processing.append(
+ {
+ "operation": "pivot",
+ "options": {
+ "index": [x_axis_label],
+ "columns": [],
+ "aggregates": {
+ label: {"operator": "mean"} for label in metric_labels
+ },
+ "drop_missing_columns": not form_data.get(
+ "show_empty_columns", False
+ ),
+ },
+ }
+ )
+ if form_data.get("resample_method") and form_data.get("resample_rule"):
+ zero_fill = form_data["resample_method"] == "zerofill"
+ post_processing.append(
+ {
+ "operation": "resample",
+ "options": {
+ "method": "asfreq" if zero_fill else
form_data["resample_method"],
+ "rule": form_data["resample_rule"],
+ "fill_value": 0 if zero_fill else None,
+ **(
+ {"fill_time_range": True}
+ if form_data.get("resample_fill_time_range")
+ else {}
+ ),
+ },
+ }
+ )
+ rolling_type = form_data.get("rolling_type")
+ columns_map = {label: label for label in metric_labels}
+ if rolling_type == "cumsum":
+ post_processing.append(
+ {"operation": "cum", "options": {"operator": "sum", "columns":
columns_map}}
+ )
+ elif rolling_type in {"sum", "mean", "std"}:
+ post_processing.append(
+ {
+ "operation": "rolling",
+ "options": {
+ "rolling_type": rolling_type,
+ "window": int(form_data.get("rolling_periods") or 1),
+ "min_periods": int(form_data.get("min_periods") or 0),
+ "columns": columns_map,
+ },
+ }
+ )
+ post_processing.append({"operation": "flatten"})
+ query["post_processing"] = post_processing
+ queries = [query]
+ if form_data.get("aggregation") == "raw":
+ overall = dict(query)
+ overall.update(
+ {
+ "columns": [],
+ "is_timeseries": False,
+ "post_processing": [],
+ }
+ )
+ queries.append(overall)
+ return queries
+
+
+def _table_queries( # noqa: C901
+ form_data: dict[str, Any], query: dict[str, Any]
+) -> list[dict[str, Any]]:
+ if is_raw_query_mode(form_data):
+ # The extractor already applies the raw-mode contract, including native
+ # ``order_by_cols`` parsing. Do not synthesize metric ordering.
+ query["columns"] = list(
+ form_data.get("all_columns") or form_data.get("columns") or []
+ )
+ query["metrics"] = []
+ if raw_orderby := orderby_from_form_data(form_data, [], "table"):
+ query["orderby"] = raw_orderby
+ else:
+ query.pop("orderby", None)
+ return [query]
+
+ metrics = list(query.get("metrics") or [])
+ query["columns"] = _table_temporalized_columns(
+ form_data, list(query.get("columns") or [])
+ )
+ percent_metrics = _as_list(form_data.get("percent_metrics"))
+ for metric in percent_metrics:
+ if _label(metric, metric=True) not in {
+ _label(existing, metric=True) for existing in metrics
+ }:
+ metrics.append(metric)
+ query["metrics"] = metrics
+ query["orderby"] = orderby_from_form_data(form_data, metrics, "table")
+ post_processing: list[dict[str, Any]] = []
+ resolved_offsets = _table_time_offsets(form_data)
+ comparison_enabled = (
+ form_data.get("comparison_type") in _TIME_COMPARISON_TYPES
+ and bool(metrics)
+ and bool(_as_list(form_data.get("time_compare")))
+ )
+ contribution: dict[str, Any] | None = None
+ if percent_metrics:
+ base_labels = [_label(metric, metric=True) for metric in
percent_metrics]
+ labels = [
+ label
+ for metric_label in base_labels
+ for label in (
+ [
+ metric_label,
+ *(f"{metric_label}__{offset}" for offset in
resolved_offsets),
+ ]
+ if comparison_enabled
+ else [metric_label]
+ )
+ ]
+ labels = list(dict.fromkeys(labels))
+ contribution = {
+ "operation": "contribution",
+ "options": {
+ "columns": labels,
+ "rename_columns": [f"%{label}" for label in labels],
+ },
+ }
+ post_processing.append(contribution)
+
+ metric_labels = [_label(metric, metric=True) for metric in metrics]
+ offset_map = _metric_offset_map(form_data, metric_labels, resolved_offsets)
+ time_offsets = resolved_offsets if offset_map else []
+ if offset_map and form_data.get("comparison_type") != "values":
+ post_processing.append(
+ {
+ "operation": "compare",
+ "options": {
+ "source_columns": list(offset_map.values()),
+ "compare_columns": list(offset_map),
+ "compare_type": form_data.get("comparison_type"),
+ "drop_original_columns": True,
+ },
+ }
+ )
+ if post_processing:
query["post_processing"] = post_processing
- # ``granularity`` does two jobs downstream: it names the temporal column
the
- # time range filters on, and it is the column ``time_grain_sqla`` buckets
- # (``models/helpers.py`` swaps a selected column for its timestamp
expression
- # when that column equals ``granularity``). Only the first job depends on
- # there being an active range, so set it whenever form data carries one —
- # matching ``extractExtras.ts``, which sets it unconditionally. Gating it
on
- # ``time_range`` dropped the bucketing, so an ordinary "all-time totals by
- # month" chart exported one row per raw timestamp instead of one per month.
- if granularity := form_data.get("granularity") or
form_data.get("granularity_sqla"):
+ else:
+ query.pop("post_processing", None)
+ query["time_offsets"] = time_offsets
+
+ is_download = form_data.get("result_format") in {"csv", "xlsx"} or (
+ form_data.get("result_format") == "json"
+ and form_data.get("result_type") == "results"
+ )
+ if is_download:
+ if form_data.get("row_limit") is not None:
+ query["row_limit"] = int(form_data["row_limit"])
+ query["row_offset"] = 0
+ elif form_data.get("server_pagination"):
+ page_size = int(form_data.get("server_page_length") or 0)
+ configured_limit = int(form_data.get("row_limit") or 0)
+ query["row_limit"] = (
+ min(page_size, configured_limit) if configured_limit else page_size
+ )
+ query["row_offset"] = 0
+
+ queries = [query]
+ if form_data.get("percent_metric_calculation") == "all_records" and
percent_metrics:
+ all_records = dict(query)
+ all_records.update(
+ {
+ "columns": [],
+ "metrics": percent_metrics,
+ "post_processing": [],
+ "row_limit": 0,
+ "row_offset": 0,
+ "orderby": [],
+ "is_timeseries": False,
+ }
+ )
+ queries.append(all_records)
+ if form_data.get("show_totals") and metrics:
+ totals_aggregate = form_data.get("totals_aggregate")
+ if totals_aggregate not in {"SUM", "AVG"}:
+ totals_aggregate = None
+ totals_metrics = [
+ {
+ **metric,
+ "aggregate": totals_aggregate,
+ }
+ if totals_aggregate
+ and isinstance(metric, dict)
+ and metric.get("expressionType") == "SIMPLE"
+ else metric
+ for metric in metrics
+ ]
+ totals = dict(query)
+ totals.update(
+ {
+ "columns": [],
+ "metrics": totals_metrics,
+ "row_limit": 0,
+ "row_offset": 0,
+ "post_processing": [contribution] if contribution else [],
+ }
+ )
+ totals.pop("order_desc", None)
+ totals.pop("orderby", None)
+ queries.append(totals)
+ if form_data.get("server_pagination") and not is_download:
+ rowcount = dict(query)
+ rowcount.update(
+ {
+ "time_offsets": [],
+ "row_limit": int(form_data.get("row_limit") or 0),
+ "row_offset": 0,
+ "post_processing": [],
+ "is_rowcount": True,
+ }
+ )
+ queries.insert(1, rowcount)
+ return queries
+
+
+def _treemap_query(form_data: dict[str, Any], query: dict[str, Any]) -> None:
+ """Apply Treemap's bounded metric and hierarchy ordering contract."""
+ hierarchy = _as_list(form_data.get("groupby"))
+ metric = form_data.get("metric")
+ query["columns"] = hierarchy
+ query["metrics"] = [metric] if metric is not None else []
+ granularity = form_data.get("granularity",
form_data.get("granularity_sqla"))
+ if granularity is not None:
query["granularity"] = granularity
- if form_data.get("row_limit"):
- query["row_limit"] = form_data["row_limit"]
+ ordering = [[metric, False]] if form_data.get("sort_by_metric") and metric
else []
+ ordering.extend(
+ [column, True] for column in hierarchy if isinstance(column, str) and
column
+ )
+ try:
+ bounded = float(query.get("row_limit") or 0) != 0
+ except (ValueError, TypeError):
+ bounded = True
+ query.pop("orderby", None)
+ if bounded and ordering:
+ query["orderby"] = ordering
+
+
+def _bubble_query(form_data: dict[str, Any], query: dict[str, Any]) -> None:
+ """Preserve Bubble's entity/series roles and explicit sort-metric
direction."""
+ query["columns"] = _deduplicate_fields(
+ [*_as_list(form_data.get("entity")),
*_as_list(form_data.get("series"))]
+ )
+ query.pop("orderby", None)
+ if sort_metric := next(iter(_as_list(form_data.get("orderby"))), None):
+ query["orderby"] = [[sort_metric, not query.get("order_desc", True)]]
+
+
+def _entity_map_query(
+ form_data: dict[str, Any], query: dict[str, Any], *, world_map: bool
+) -> None:
+ """Group World/Country Map queries by their ``entity`` role.
+
+ Mirrors the World Map and Country Map ``buildQuery`` transforms: the
+ ``entity`` control is the only grouping column, Country Map selects one
+ ``metric``, and World Map adds a distinct ``secondary_metric`` and orders
by
+ the primary metric only when ``sort_by_metric`` is enabled.
+ """
+ entity = form_data.get("entity")
+ query["columns"] = [entity] if entity else []
+ metric = form_data.get("metric")
+ metrics = [metric] if metric else []
+ if world_map:
+ secondary_metric = form_data.get("secondary_metric")
+ if secondary_metric and (
+ not metric
+ or _label(secondary_metric, metric=True) != _label(metric,
metric=True)
+ ):
+ metrics.append(secondary_metric)
+ query.pop("orderby", None)
+ if form_data.get("sort_by_metric") and metric:
+ query["orderby"] = [[metric, False]]
+ query["metrics"] = metrics
+
+
+def build_query_objects_from_form_data( # noqa: C901
+ form_data: dict[str, Any],
+ *,
+ viz_type: str | None = None,
+ row_limit: int | None = None,
+ order_desc: bool | None = None,
+ filters_prepared: bool = False,
+ secondary_form_data: dict[str, Any] | None = None,
+) -> list[dict[str, Any]]:
+ """Build frontend-equivalent QueryObject dictionaries from form data.
+
+ Every real MCP product path and the legacy common fallback share this
+ extractor and the chart adapters below. This prevents compile/SQL/preview
+ and saved/fallback data paths from silently selecting different fields.
+ """
+ effective_viz = viz_type or str(form_data.get("viz_type") or "")
+ extraction_form_data = form_data
+ if is_raw_query_mode(form_data):
+ # ``extractQueryFields`` is strict, while the legacy server fallback
has
+ # always kept old saved tables executable by dropping malformed order
+ # entries. Normalize that bounded raw-table input before invoking the
+ # shared extractor; valid entries still use the centralized contract.
+ extraction_form_data = {
+ **form_data,
+ "query_mode": "raw",
+ "order_by_cols": orderby_from_form_data(form_data, [],
effective_viz),
+ }
+ query = _base_query_object(
+ extraction_form_data,
+ row_limit=row_limit,
+ order_desc=order_desc,
+ filters_prepared=filters_prepared,
+ )
+
+ if effective_viz in {"table", "ag-grid-table"}:
+ return _finalize_query_objects(form_data, _table_queries(form_data,
query))
+ if effective_viz == "histogram_v2":
+ _histogram_query(form_data, query)
+ elif effective_viz == "box_plot":
+ _box_plot_query(form_data, query)
+ elif effective_viz == "pivot_table_v2":
+ _pivot_query(form_data, query)
+ elif effective_viz == "waterfall":
+ _waterfall_query(form_data, query)
+ elif effective_viz in {"gantt", "gantt_chart"}:
+ _gantt_query(form_data, query)
+ elif effective_viz.startswith("echarts_timeseries") or effective_viz in {
+ "echarts_area",
+ "mixed_timeseries",
+ }:
+ _timeseries_query(form_data, query)
+ elif effective_viz == "big_number":
+ return _finalize_query_objects(form_data,
_big_number_queries(form_data, query))
+ elif effective_viz == "big_number_total":
+ query["columns"] = []
+ elif effective_viz == "handlebars":
+ _normalize_query_orderby(query)
+ elif effective_viz == "pie":
+ if form_data.get("sort_by_metric") and form_data.get("metric") is not
None:
+ query["orderby"] = [[form_data["metric"], False]]
+ query["post_processing"] = _pie_contribution_post_processing(
+ list(query.get("metrics") or [])
+ )
+ elif effective_viz == "gauge_chart":
+ if form_data.get("sort_by_metric") and form_data.get("metric") is not
None:
+ query["orderby"] = [[form_data["metric"], False]]
+ elif effective_viz == "treemap_v2":
+ _treemap_query(form_data, query)
+ elif effective_viz == "sunburst_v2":
+ if not form_data.get("sort_by_metric"):
+ query["orderby"] = []
+ elif form_data.get("metric") is not None:
+ query["orderby"] = [[form_data["metric"], False]]
+ elif effective_viz in {"bubble", "bubble_v2"}:
+ _bubble_query(form_data, query)
+ elif effective_viz in {"world_map", "country_map"}:
+ _entity_map_query(form_data, query, world_map=effective_viz ==
"world_map")
+ elif effective_viz == "ag-grid-pivot-table":
+ query["columns"] = _temporalized_columns(
+ form_data, _as_list(form_data.get("groupby"))
+ )
+
+ elif not query.get("orderby"):
+ # Unadapted visualizations retain the legacy top-N ordering fallback.
+ # Handled adapters may deliberately leave ordering empty.
+ query["orderby"] = orderby_from_form_data(
+ form_data, list(query.get("metrics") or []), effective_viz
+ )
+
+ if effective_viz != "mixed_timeseries":
+ return _finalize_query_objects(form_data, [query])
+
+ secondary = secondary_form_data or
retain_mixed_timeseries_secondary_form_data(
+ form_data
+ )
+ # Query A's saved ordering references metrics query B may not select, so
the
+ # secondary series orders by ``orderby_b`` or not at all.
+ if (orderby_b := form_data.get("orderby_b")) is not None:
+ secondary["orderby"] = orderby_b
+ elif "orderby_b" not in form_data:
+ secondary.pop("orderby", None)
+ query_b = _base_query_object(
+ secondary,
+ row_limit=row_limit,
+ order_desc=order_desc,
Review Comment:
Fixed in f2acaab6defc45cd7680662feced6b74dae95d02.
Query B no longer receives Query A's order_desc override; its own
order_desc_b wins. Regression:
test_mixed_secondary_sort_direction_ignores_primary_override checks opposite
directions with series_limit_b=1.
Validation: 851 tests passed across the 11 touched/related pytest modules;
pre-commit passed for all changed files. The regression coverage fails against
the previous implementation and passes with this fix.
##########
superset/common/form_data_query_context.py:
##########
@@ -288,71 +461,1305 @@ def _pie_contribution_post_processing(metrics:
list[Any]) -> list[dict[str, Any]
]
-def build_query_context_from_form_data(
- form_data: dict[str, Any],
- datasource: dict[str, Any],
- viz_type: str | None = None,
+def _as_list(value: Any) -> list[Any]:
+ """Return the frontend ``ensureIsArray`` representation of a value."""
+ if value is None:
+ return []
+ return list(value) if isinstance(value, (list, tuple)) else [value]
+
+
+def _label(value: Any, *, metric: bool = False) -> str:
+ """Resolve a frontend-compatible query-field label."""
+ try:
+ return get_metric_name(value) if metric else get_column_name(value)
+ except (AttributeError, KeyError, TypeError, ValueError):
+ if isinstance(value, Mapping):
+ return str(
+ value.get("label")
+ or value.get("column_name")
+ or value.get("sqlExpression")
+ or value
+ )
+ return str(value)
+
+
+def _deduplicate_fields(values: list[Any], *, metric: bool = False) ->
list[Any]:
+ """Deduplicate query fields by their frontend-visible label."""
+ result: list[Any] = []
+ labels: set[str] = set()
+ for value in values:
+ if value is None or value == "":
+ continue
+ label = _label(value, metric=metric)
+ if label in labels:
+ continue
+ labels.add(label)
+ result.append(value)
+ return result
+
+
+def retain_mixed_timeseries_secondary_form_data(
+ form_data: Mapping[str, Any],
) -> dict[str, Any]:
- """
- Build a query-context payload (the JSON shape
``ChartDataQueryContextSchema``
- loads) from a chart's form data and datasource reference.
+ """Mirror ``retainFormDataSuffix(formData, '_b')`` exactly.
- :param form_data: The chart's saved ``params`` parsed to a dict.
- :param datasource: ``{"id": <int>, "type": "table"}`` datasource reference.
- :param viz_type: The chart's viz type, used for viz-specific handling.
- :returns: A single-query query-context dict.
+ Suffixed values are installed first, including falsey values, and shared
+ unsuffixed controls fill only keys that query B did not explicitly set.
"""
- columns, metrics = _columns_and_metrics(form_data, viz_type)
-
- # SIMPLE adhoc filters (+ legacy top-level ``filters``) become query
filters;
- # free-form SQL predicates go into ``extras``. Only ``WHERE``-clause SIMPLE
- # filters are applied (matching the chart), so the export never filters on
a
- # ``HAVING`` clause the chart itself ignores.
- filters = adhoc_filters_to_query_filters(
- form_data.get("adhoc_filters", []), where_only=True
- )
- for flt in form_data.get("filters") or []:
- if isinstance(flt, dict) and flt.get("col") is not None:
- filters.append(flt)
+ secondary: dict[str, Any] = {}
+ for key, value in form_data.items():
+ if key.endswith("_b"):
+ secondary[key[:-2]] = value
+ for key, value in form_data.items():
+ if not key.endswith("_b") and key not in secondary:
+ secondary[key] = value
+ secondary_filter_keys = {
+ "adhoc_filters": "adhoc_filters_b",
+ "extra_filters": "extra_filters_b",
+ "filters": "filters_b",
+ "having": "having_b",
+ "where": "where_b",
+ }
+ if any(suffixed in form_data for suffixed in
secondary_filter_keys.values()):
+ # The frontend exposes adhoc_filters_b, while saved/server payloads can
+ # carry equivalent legacy aliases. Treat the family atomically: an
+ # explicit clear in any B alias must not be repopulated by query A's
+ # differently named filter representation.
+ for primary, suffixed in secondary_filter_keys.items():
+ if suffixed not in form_data:
+ secondary.pop(primary, None)
+ return secondary
- extras = freeform_where_having(form_data)
- if form_data.get("time_grain_sqla"):
- extras["time_grain_sqla"] = form_data["time_grain_sqla"]
- # Prefer the modern ``time_range``; fall back to the legacy
``since``/``until``
- # pair (older charts store the range that way) before defaulting to no
filter.
- time_range = form_data.get("time_range")
- if not time_range and (form_data.get("since") or form_data.get("until")):
- time_range = f"{form_data.get('since') or ''} :
{form_data.get('until') or ''}"
- time_range = time_range or "No filter"
+def _base_query_object( # noqa: C901
+ form_data: dict[str, Any],
+ *,
+ row_limit: int | None,
+ order_desc: bool | None,
+ filters_prepared: bool,
+) -> dict[str, Any]:
+ """Build the shared frontend-equivalent portion of a QueryObject."""
+ columns, metrics, orderby = query_fields_from_form_data(form_data)
query: dict[str, Any] = {
"columns": columns,
"metrics": metrics,
- "orderby": orderby_from_form_data(form_data, metrics, viz_type),
- "filters": filters,
- "time_range": time_range,
}
+ if orderby:
+ query["orderby"] = orderby
+
+ if filters_prepared:
+ query["filters"] = list(form_data.get("filters") or [])
+ for clause in ("where", "having"):
+ if form_data.get(clause):
+ query[clause] = form_data[clause]
+ if form_data.get("extras"):
+ query["extras"] = dict(form_data["extras"])
+ else:
+ filters = adhoc_filters_to_query_filters(
+ form_data.get("adhoc_filters", []), where_only=True
+ )
+ filters.extend(
+ filter_
+ for filter_ in form_data.get("filters") or []
+ if isinstance(filter_, dict) and filter_.get("col") is not None
+ )
+ query["filters"] = filters
+ if extras := freeform_where_having(form_data):
+ query["extras"] = extras
+
+ extras = dict(query.get("extras") or {})
+ if form_data.get("time_grain_sqla") is not None:
+ extras["time_grain_sqla"] = form_data["time_grain_sqla"]
if extras:
query["extras"] = extras
- if viz_type == "pie" and (
- post_processing := _pie_contribution_post_processing(metrics)
+
+ effective_limit = row_limit if row_limit is not None else
form_data.get("row_limit")
+ if effective_limit is not None:
+ query["row_limit"] = effective_limit
+ if form_data.get("row_offset") is not None:
+ query["row_offset"] = form_data["row_offset"]
+ if order_desc is not None:
+ query["order_desc"] = order_desc
+ elif "order_desc" in form_data and form_data["order_desc"] is not None:
+ query["order_desc"] = form_data["order_desc"]
+
+ time_range = form_data.get("time_range")
+ if not time_range and (form_data.get("since") or form_data.get("until")):
+ time_range = f"{form_data.get('since') or ''} :
{form_data.get('until') or ''}"
+ if time_range:
+ query["time_range"] = time_range
+ for key in ("since", "until", "annotation_layers", "url_params",
"custom_params"):
+ if form_data.get(key) is not None:
+ query[key] = form_data[key]
+
+ granularity = form_data.get("granularity") or
form_data.get("granularity_sqla")
+ if granularity:
+ query["granularity"] = granularity
+ series_limit = form_data.get("series_limit", form_data.get("limit"))
+ if series_limit is not None:
+ query["series_limit"] = series_limit
+ series_limit_metric = form_data.get("series_limit_metric")
+ if series_limit_metric is None:
+ series_limit_metric = form_data.get("timeseries_limit_metric")
+ if series_limit_metric is not None:
+ query["series_limit_metric"] = series_limit_metric
+ if form_data.get("group_others_when_limit_reached") is not None:
+ query["group_others_when_limit_reached"] = form_data[
+ "group_others_when_limit_reached"
+ ]
+ return query
+
+
+def _temporalized_columns(form_data: dict[str, Any], columns: list[Any]) ->
list[Any]:
+ """Apply the pivot BASE_AXIS temporal-column contract."""
+ time_grain = form_data.get("time_grain_sqla")
+ temporal_lookup = form_data.get("temporal_columns_lookup") or {}
+ result: list[Any] = []
+ for column in columns:
+ if (
+ isinstance(column, str)
+ and time_grain
+ and (
+ temporal_lookup.get(column)
+ or form_data.get("granularity_sqla") == column
+ )
+ ):
+ result.append(
+ {
+ "timeGrain": time_grain,
+ "columnType": "BASE_AXIS",
+ "sqlExpression": column,
+ "label": column,
+ "expressionType": "SQL",
+ }
+ )
+ else:
+ result.append(column)
+ return result
+
+
+def _box_temporalized_columns(
+ form_data: dict[str, Any], columns: list[Any]
+) -> list[Any]:
+ """Convert only physical columns confirmed temporal by Box Plot
metadata."""
+ time_grain = form_data.get("time_grain_sqla")
+ temporal_lookup = form_data.get("temporal_columns_lookup")
+ if not time_grain or not isinstance(temporal_lookup, Mapping):
+ return columns
+ return [
+ {
+ "timeGrain": time_grain,
+ "columnType": "BASE_AXIS",
+ "sqlExpression": column,
+ "label": column,
+ "expressionType": "SQL",
+ }
+ if isinstance(column, str) and temporal_lookup.get(column) is True
+ else column
+ for column in columns
+ ]
+
+
+def _table_temporalized_columns(
+ form_data: dict[str, Any], columns: list[Any]
+) -> list[Any]:
+ """Promote the first temporal table group-by to the frontend BASE_AXIS.
+
+ Table's builder treats only physical columns named in
+ ``temporal_columns_lookup`` as temporal and moves the first match to the
+ front. Later temporal dimensions remain ordinary group-bys.
+ """
+ time_grain = form_data.get("time_grain_sqla")
+ temporal_lookup = form_data.get("temporal_columns_lookup") or {}
+ if not time_grain or not isinstance(temporal_lookup, Mapping):
+ return columns
+
+ temporal_column: dict[str, Any] | None = None
+ remaining: list[Any] = []
+ for column in columns:
+ if (
+ temporal_column is None
+ and isinstance(column, str)
+ and temporal_lookup.get(column)
+ ):
+ temporal_column = {
+ "timeGrain": time_grain,
+ "columnType": "BASE_AXIS",
+ "sqlExpression": column,
+ "label": column,
+ "expressionType": "SQL",
+ "isColumnReference": True,
+ }
+ else:
+ remaining.append(column)
+ return [temporal_column, *remaining] if temporal_column else columns
+
+
+def _histogram_query(form_data: dict[str, Any], query: dict[str, Any]) -> None:
+ groupby = _as_list(form_data.get("groupby"))
+ column = form_data.get("column")
+ query["columns"] = [*groupby, *([column] if column is not None else [])]
+ query["post_processing"] = [
+ {
+ "operation": "histogram",
+ "options": {
+ "column": _label(column),
+ "groupby": [_label(value) for value in groupby],
+ "bins": int(form_data.get("bins", 5)),
+ "cumulative": form_data.get("cumulative", False),
+ "normalize": form_data.get("normalize", False),
+ },
+ }
+ ]
+ if any(
+ isinstance(filter_, dict) and filter_.get("clause") == "HAVING"
+ for filter_ in form_data.get("adhoc_filters") or []
):
+ query["metrics"] = [
+ {
+ "expressionType": "SQL",
+ "sqlExpression": "COUNT(*)",
+ "label": "COUNT(*)",
+ }
+ ]
+ else:
+ query["metrics"] = []
+
+
+def _box_plot_query(form_data: dict[str, Any], query: dict[str, Any]) -> None:
+ distributed = _as_list(form_data.get("columns"))
+ if not distributed and form_data.get("granularity_sqla"):
+ distributed = [form_data["granularity_sqla"]]
+ groupby = _as_list(form_data.get("groupby"))
+ query["columns"] = [*_box_temporalized_columns(form_data, distributed),
*groupby]
+ query["series_columns"] = groupby
+ whisker = form_data.get("whiskerOptions")
+ if not whisker:
+ query["post_processing"] = []
+ return
+ whisker_type = "tukey"
+ percentiles: list[int] | None = None
+ if whisker == "Min/max (no outliers)":
+ whisker_type = "min/max"
+ elif isinstance(whisker, str) and whisker.endswith(" percentiles"):
+ low, high = whisker.removesuffix(" percentiles").split("/", 1)
+ whisker_type = "percentile"
+ percentiles = [int(low), int(high)]
+ query["post_processing"] = [
+ {
+ "operation": "boxplot",
+ "options": {
+ "whisker_type": whisker_type,
+ "percentiles": percentiles,
+ "groupby": [_label(value) for value in groupby],
+ "metrics": [_label(value, metric=True) for value in
query["metrics"]],
+ },
+ }
+ ]
+
+
+_PIVOT_ADDITIVE_AGGREGATES = frozenset({"SUM", "COUNT", "MIN", "MAX"})
+
+
+def _all_metrics_additive(metrics: list[Any]) -> bool:
+ """Mirror Pivot's conservative additive-metric fast-path."""
+ return bool(metrics) and all(
+ isinstance(metric, Mapping)
+ and metric.get("expressionType") == "SIMPLE"
+ and metric.get("aggregate") in _PIVOT_ADDITIVE_AGGREGATES
+ for metric in metrics
+ )
+
+
+def _pivot_grouping_sets(
+ form_data: dict[str, Any], rows: list[Any], columns: list[Any]
+) -> list[list[str]]:
+ """Enumerate the rollup levels requested by Pivot's frontend builder."""
+ row_prefixes = [[], *(rows[: index + 1] for index in range(len(rows)))]
+ column_prefixes = [
+ [],
+ *(columns[: index + 1] for index in range(len(columns))),
+ ]
+ show_values_as = form_data.get("showValuesAs")
+ needs_rows_collapsed = show_values_as in {"percent_col", "percent_total"}
+ needs_columns_collapsed = show_values_as in {"percent_row",
"percent_total"}
+
+ def row_prefix_needed(prefix: list[Any]) -> bool:
+ if len(prefix) == len(rows):
+ return True
+ if not prefix:
+ return bool(form_data.get("colTotals")) or needs_rows_collapsed
+ return bool(form_data.get("rowSubTotals"))
+
+ def column_prefix_needed(prefix: list[Any]) -> bool:
+ if len(prefix) == len(columns):
+ return True
+ if not prefix:
+ return bool(form_data.get("rowTotals")) or needs_columns_collapsed
+ return bool(form_data.get("colSubTotals"))
+
+ levels = [
+ (row_prefix, column_prefix)
+ for row_prefix in row_prefixes
+ if row_prefix_needed(row_prefix)
+ for column_prefix in column_prefixes
+ if column_prefix_needed(column_prefix)
+ ]
+ if form_data.get("combineMetric"):
+ metrics_layout = form_data.get("metricsLayout")
+
+ def forced_denominator(level: tuple[list[Any], list[Any]]) -> bool:
+ row_prefix, column_prefix = level
+ return (needs_rows_collapsed and not row_prefix) or (
+ needs_columns_collapsed and not column_prefix
+ )
+
+ if metrics_layout == "ROWS":
+ levels = [
+ level
+ for level in levels
+ if len(level[0]) == len(rows) or forced_denominator(level)
+ ]
+ else:
+ levels = [
+ level
+ for level in levels
+ if len(level[1]) == len(columns) or forced_denominator(level)
+ ]
+
+ return [
+ [_label(value) for value in _deduplicate_fields([*row_prefix,
*column_prefix])]
+ for row_prefix, column_prefix in levels
+ ]
+
+
+def _pivot_query(form_data: dict[str, Any], query: dict[str, Any]) -> None:
+ rows = _as_list(form_data.get("groupbyRows"))
+ columns = _as_list(form_data.get("groupbyColumns"))
+ if form_data.get("transposePivot"):
+ rows, columns = columns, rows
+ query["columns"] = _temporalized_columns(
+ form_data, _deduplicate_fields([*rows, *columns])
+ )
+ metric = query.get("series_limit_metric") or next(
+ iter(query.get("metrics") or []), None
+ )
+ query["orderby"] = (
+ [[metric, not bool(query.get("order_desc", True))]]
+ if metric is not None
+ else []
+ )
+ if not _all_metrics_additive(query.get("metrics") or []):
+ query["grouping_sets"] = _pivot_grouping_sets(form_data, rows, columns)
+
+
+def _waterfall_query(form_data: dict[str, Any], query: dict[str, Any]) -> None:
+ x_axis = form_data.get("x_axis") or form_data.get("granularity_sqla")
+ columns = [*_as_list(x_axis), *_as_list(form_data.get("groupby"))]
+ query["columns"] = _deduplicate_fields(columns)
+ query["orderby"] = [[column, True] for column in query["columns"]]
+
+
+def _gantt_query(form_data: dict[str, Any], query: dict[str, Any]) -> None:
+ groupby = _as_list(form_data.get("series"))
+ orderby = query_fields_from_form_data(form_data)[2]
+ columns = [
+ form_data.get("start_time"),
+ form_data.get("end_time"),
+ form_data.get("y_axis"),
+ *groupby,
+ *_as_list(form_data.get("tooltip_columns")),
+ *(entry[0] for entry in orderby if entry),
+ ]
+ query["columns"] = _deduplicate_fields(columns)
+ query["metrics"] = _as_list(form_data.get("tooltip_metrics"))
+ query["orderby"] = orderby
+ query["series_columns"] = groupby
+
+
+def _normalize_query_orderby(query: dict[str, Any]) -> None:
+ """Mirror ``normalizeOrderBy`` while retaining limit-direction controls."""
+ orderby = query.get("orderby")
+ if (
+ isinstance(orderby, list)
+ and orderby
+ and isinstance(orderby[0], (list, tuple))
+ and len(orderby[0]) == 2
+ and orderby[0][0]
+ and isinstance(orderby[0][1], bool)
+ ):
+ return
+ metric = (
+ query.get("series_limit_metric")
+ or query.get("legacy_order_by")
+ or next(iter(query.get("metrics") or []), None)
+ )
+ if metric is None:
+ query.pop("orderby", None)
+ return
+ query["orderby"] = [[metric, not bool(query.get("order_desc", True))]]
+
+
+_TIME_COMPARISON_TYPES = frozenset({"values", "difference", "percentage",
"ratio"})
+
+
+def _metric_offset_map(
+ form_data: dict[str, Any],
+ metric_labels: list[str],
+ offsets: list[Any] | None = None,
+) -> dict[str, str]:
+ """Return the frontend time-comparison metric label map."""
+ if form_data.get("comparison_type") not in _TIME_COMPARISON_TYPES:
+ return {}
+ return {
+ f"{metric}__{offset}": metric
+ for metric in metric_labels
+ for offset in (
+ offsets if offsets is not None else
_as_list(form_data.get("time_compare"))
+ )
+ }
+
+
+def _table_time_offsets(form_data: dict[str, Any]) -> list[Any]:
+ """Resolve Table custom/inherited shifts like its frontend query
adapter."""
+ raw_offsets = _as_list(form_data.get("time_compare"))
+ offsets = [offset for offset in raw_offsets if offset not in {"custom",
"inherit"}]
+ if "custom" in raw_offsets and form_data.get("start_date_offset") is not
None:
+ offsets.append(form_data["start_date_offset"])
+ extra_form_data = form_data.get("extra_form_data")
+ if isinstance(extra_form_data, Mapping) and
extra_form_data.get("time_compare"):
+ inherited = extra_form_data["time_compare"]
+ if inherited not in offsets:
+ offsets = [inherited]
+ return offsets
+
+
+def _x_axis_column(form_data: Mapping[str, Any]) -> Any | None:
+ """Return a supported x-axis column, excluding legacy granularity.
+
+ ``column_name`` mappings are retained for old server/native payloads. Big
+ Number uses the stricter frontend predicate below.
+ """
+ x_axis = form_data.get("x_axis")
+ if isinstance(x_axis, str):
+ return x_axis if x_axis else None
+ if isinstance(x_axis, Mapping):
+ if isinstance(column_name := x_axis.get("column_name"), str) and
column_name:
+ return column_name
+ # Frontend SQL adhoc columns remain objects in the QueryObject.
+ return x_axis if x_axis else None
+ return None
+
+
+def _frontend_x_axis_column(form_data: Mapping[str, Any]) -> Any | None:
+ """Mirror ``isQueryFormColumn`` for physical and SQL adhoc columns."""
+ x_axis = form_data.get("x_axis")
+ if isinstance(x_axis, str):
+ return x_axis if x_axis else None
+ if (
+ isinstance(x_axis, Mapping)
+ and "sqlExpression" in x_axis
+ and "label" in x_axis
+ and x_axis.get("expressionType") in {None, "SQL"}
+ ):
+ return x_axis
+ return None
+
+
+def normalize_time_column(
+ form_data: Mapping[str, Any], query: dict[str, Any]
+) -> dict[str, Any]:
+ """Apply the final shared frontend ``normalizeTimeColumn`` mutator."""
+ x_axis = _frontend_x_axis_column(form_data)
+ columns = query.get("columns")
+ if x_axis is None or not isinstance(columns, list):
+ return query
+
+ axis_index: int | None = None
+ for index, column in enumerate(columns):
+ if isinstance(x_axis, str) and isinstance(column, str) and column ==
x_axis:
+ axis_index = index
+ break
+ if (
+ isinstance(x_axis, Mapping)
+ and isinstance(column, Mapping)
+ and column.get("sqlExpression") == x_axis.get("sqlExpression")
+ ):
+ axis_index = index
+ break
+ if axis_index is None:
+ return query
+
+ normalized = dict(query)
+ normalized_columns = list(columns)
+ grain = (query.get("extras") or {}).get("time_grain_sqla")
+ if isinstance(columns[axis_index], Mapping):
+ normalized_axis = {
+ "columnType": "BASE_AXIS",
+ **({"timeGrain": grain} if grain is not None else {}),
+ **columns[axis_index],
+ }
+ else:
+ normalized_axis = {
+ "columnType": "BASE_AXIS",
+ "sqlExpression": x_axis,
+ "label": x_axis,
+ "expressionType": "SQL",
+ "isColumnReference": True,
+ **({"timeGrain": grain} if grain is not None else {}),
+ }
+ normalized_columns[axis_index] = normalized_axis
+ normalized["columns"] = normalized_columns
+ normalized.pop("is_timeseries", None)
+ return normalized
+
+
+def _finalize_query_objects(
+ form_data: Mapping[str, Any], queries: list[dict[str, Any]]
+) -> list[dict[str, Any]]:
+ """Run shared query-context mutators after every visualization adapter."""
+ return [normalize_time_column(form_data, query) for query in queries]
+
+
+def _x_axis_label(
+ form_data: Mapping[str, Any], *, frontend_strict: bool = False
+) -> str | None:
+ """Mirror getXAxisColumn/getXAxisLabel for explicit and legacy axes."""
+ explicit = (
+ _frontend_x_axis_column(form_data)
+ if frontend_strict
+ else _x_axis_column(form_data)
+ )
+ if explicit:
+ return _label(explicit)
+ if form_data.get("granularity_sqla"):
+ return DTTM_ALIAS
+ return None
+
+
+def _rename_operator(
+ form_data: dict[str, Any],
+ query: dict[str, Any],
+ *,
+ x_axis_label: str | None,
+) -> dict[str, Any] | None:
+ """Mirror the ECharts ``renameOperator`` for Timeseries and Mixed
charts."""
+ metrics = list(query.get("metrics") or [])
+ metric_labels = [_label(metric, metric=True) for metric in metrics]
+ series_columns = query.get("series_columns")
+ columns = _as_list(
+ series_columns if series_columns is not None else query.get("columns")
+ )
+ time_offsets = _as_list(form_data.get("time_compare"))
+ offset_map = _metric_offset_map(form_data, metric_labels)
+ is_time_comparison = bool(offset_map)
+ truncate_metric = form_data.get("truncate_metric")
+
+ should_rename = (
+ bool(metrics)
+ and bool(x_axis_label)
+ and (
+ is_time_comparison
+ or (
+ (bool(columns) or len(time_offsets) > 1)
+ and "truncate_metric" in form_data
+ and bool(truncate_metric)
+ )
+ )
+ )
+ if not should_rename:
+ return None
+
+ renamed: dict[str, str | None] = {}
+ comparison_type = form_data.get("comparison_type")
+ if is_time_comparison:
+ for metric_with_offset, metric_only in offset_map.items():
+ offset_label = next(
+ (
+ str(offset)
+ for offset in time_offsets
+ if metric_with_offset.endswith(f"__{offset}")
+ ),
+ None,
+ )
+ source = (
+ metric_with_offset
+ if comparison_type == "values"
+ else f"{comparison_type}__{metric_only}__{metric_with_offset}"
+ )
+ renamed[source] = (
+ f"{metric_only}, {offset_label}" if len(metrics) > 1 else
offset_label
+ )
+
+ if (
+ comparison_type not in {"difference", "percentage", "ratio"}
+ and len(metrics) == 1
+ and not renamed
+ ):
+ renamed[metric_labels[0]] = None
+ if not renamed:
+ return None
+ return {
+ "operation": "rename",
+ "options": {"columns": renamed, "level": 0, "inplace": True},
+ }
+
+
+def _timeseries_post_processing( # noqa: C901
+ form_data: dict[str, Any],
+ query: dict[str, Any],
+ *,
+ x_axis_label: str | None,
+ groupby: list[Any],
+ mixed: bool,
+ sort_metric: Any = None,
+) -> tuple[list[dict[str, Any]], list[Any]]:
+ """Build the Timeseries/Mixed operator pipeline in frontend order."""
+ metric_labels = [_label(value, metric=True) for value in
query.get("metrics") or []]
+ sort_metric_label = (
+ _label(sort_metric, metric=True) if sort_metric is not None else None
+ )
+ offset_map = _metric_offset_map(form_data, metric_labels)
+ time_offsets = _as_list(form_data.get("time_compare")) if offset_map else
[]
+ post_processing: list[dict[str, Any]] = []
+
+ if x_axis_label and metric_labels:
+ aggregate_labels = (
+ [*offset_map.values(), *offset_map] if offset_map else
list(metric_labels)
+ )
+ if not offset_map and sort_metric_label is not None:
+ aggregate_labels.append(sort_metric_label)
+ post_processing.append(
+ {
+ "operation": "pivot",
+ "options": {
+ "index": [x_axis_label],
+ "columns": [_label(value) for value in groupby],
+ "aggregates": {
+ label: {"operator": "mean"} for label in
aggregate_labels
+ },
+ "drop_missing_columns": not form_data.get(
+ "show_empty_columns", False
+ ),
+ },
+ }
+ )
+
+ if form_data.get("resample_method") and form_data.get("resample_rule"):
+ zero_fill = form_data["resample_method"] == "zerofill"
+ post_processing.append(
+ {
+ "operation": "resample",
+ "options": {
+ "method": "asfreq" if zero_fill else
form_data["resample_method"],
+ "rule": form_data["resample_rule"],
+ "fill_value": 0 if zero_fill else None,
+ **(
+ {"fill_time_range": True}
+ if form_data.get("resample_fill_time_range")
+ else {}
+ ),
+ },
+ }
+ )
+
+ rolling_labels = (
+ [*offset_map.values(), *offset_map] if offset_map else metric_labels
+ )
+ columns_map = {label: label for label in rolling_labels}
+ rolling_type = form_data.get("rolling_type")
+ if rolling_type == "cumsum":
+ post_processing.append(
+ {
+ "operation": "cum",
+ "options": {"operator": "sum", "columns": columns_map},
+ }
+ )
+ elif rolling_type in {"sum", "mean", "std"}:
+ post_processing.append(
+ {
+ "operation": "rolling",
+ "options": {
+ "rolling_type": rolling_type,
+ "window": int(form_data.get("rolling_periods") or 1),
+ "min_periods": int(form_data.get("min_periods") or 0),
+ "columns": columns_map,
+ },
+ }
+ )
+
+ comparison_type = form_data.get("comparison_type")
+ if offset_map and comparison_type != "values":
+ post_processing.append(
+ {
+ "operation": "compare",
+ "options": {
+ "source_columns": list(offset_map.values()),
+ "compare_columns": list(offset_map),
+ "compare_type": comparison_type,
+ "drop_original_columns": True,
+ },
+ }
+ )
+
+ if not mixed and form_data.get("contributionMode"):
+ post_processing.append(
+ {
+ "operation": "contribution",
+ "options": {
+ "orientation": form_data["contributionMode"],
+ "time_shifts": time_offsets,
+ },
+ }
+ )
+
+ if rename := _rename_operator(form_data, query, x_axis_label=x_axis_label):
+ post_processing.append(rename)
+
+ if not mixed:
+ sortable = {
+ x_axis_label or "",
+ *metric_labels,
+ sort_metric_label or "",
+ }
+ if (
+ "x_axis_sort" in form_data
+ and "x_axis_sort_asc" in form_data
+ and form_data.get("x_axis_sort") in sortable
+ and not groupby
+ ):
+ options: dict[str, Any] = {"ascending":
form_data.get("x_axis_sort_asc")}
+ if form_data.get("x_axis_sort") == x_axis_label:
+ options["is_sort_index"] = True
+ else:
+ options["by"] = form_data.get("x_axis_sort")
+ post_processing.append({"operation": "sort", "options": options})
+
+ post_processing.append({"operation": "flatten"})
+ if not mixed and form_data.get("forecastEnabled") and x_axis_label:
+ x_axis = _x_axis_column(form_data)
+ axis_grain = x_axis.get("timeGrain") if isinstance(x_axis, Mapping)
else None
+ time_grain = (
+ axis_grain
+ or (query.get("extras") or {}).get("time_grain_sqla")
+ or form_data.get("time_grain_sqla")
+ or "P1D"
+ )
+ post_processing.append(
+ {
+ "operation": "prophet",
+ "options": {
+ "time_grain": time_grain,
+ "periods": int(form_data.get("forecastPeriods") or 0),
+ "confidence_interval": float(
+ form_data.get("forecastInterval") or 0
+ ),
+ "yearly_seasonality":
form_data.get("forecastSeasonalityYearly"),
+ "weekly_seasonality":
form_data.get("forecastSeasonalityWeekly"),
+ "daily_seasonality":
form_data.get("forecastSeasonalityDaily"),
+ "index": x_axis_label,
+ },
+ }
+ )
+ return post_processing, time_offsets
+
+
+def _timeseries_query(form_data: dict[str, Any], query: dict[str, Any]) ->
None:
+ groupby = _as_list(form_data.get("groupby"))
+ x_axis = _x_axis_column(form_data)
+ x_axis_label = _x_axis_label(form_data)
+ query["columns"] = _deduplicate_fields([*_as_list(x_axis), *groupby])
+ query["series_columns"] = groupby
+ if not x_axis:
+ query["is_timeseries"] = True
+
+ # Timeseries includes its sort-only metric in the SELECT when no series is
+ # present. This lets the post-processing sort operator use a metric not
+ # otherwise displayed.
+ sort_metric = form_data.get("timeseries_limit_metric")
+ if isinstance(sort_metric, list):
+ sort_metric = next(iter(sort_metric), None)
+ if (
+ not groupby
+ and sort_metric is not None
+ and _label(sort_metric, metric=True) == form_data.get("x_axis_sort")
+ and _label(sort_metric, metric=True)
+ not in {_label(metric, metric=True) for metric in query.get("metrics")
or []}
+ ):
+ extra_metric = sort_metric
+ else:
+ extra_metric = None
+ _normalize_query_orderby(query)
+ post_processing, time_offsets = _timeseries_post_processing(
+ form_data,
+ query,
+ x_axis_label=x_axis_label,
+ groupby=groupby,
+ mixed=form_data.get("viz_type") == "mixed_timeseries",
+ sort_metric=extra_metric,
+ )
+ if extra_metric is not None:
+ query.setdefault("metrics", []).append(extra_metric)
+ query["post_processing"] = post_processing
+ query["time_offsets"] = time_offsets
+ if form_data.get("viz_type") != "mixed_timeseries":
+ query["time_compare_full_range"] = bool(
+ time_offsets and form_data.get("time_compare_full_range")
+ )
+
+
+def _big_number_queries(
+ form_data: dict[str, Any], query: dict[str, Any]
+) -> list[dict[str, Any]]:
+ """Mirror Big Number with Trendline's one/two-query contract."""
+ # Saved/native Big Number payloads can carry the temporal binding as a
+ # ``{"column_name": ...}`` mapping, which the strict frontend predicate
does
+ # not recognize; keep grouping by it rather than falling back to a total.
+ frontend_x_axis = _frontend_x_axis_column(form_data)
+ explicit_x_axis = frontend_x_axis or _x_axis_column(form_data)
+ time_column = _as_list(explicit_x_axis)
+ x_axis_label = _x_axis_label(form_data, frontend_strict=True)
Review Comment:
Fixed in f2acaab6defc45cd7680662feced6b74dae95d02.
The Big Number pivot label is derived from the same explicit_x_axis used to
select the column. Regression:
test_big_number_native_axis_pivot_references_selected_column covers the native
column_name mapping with granularity_sqla and verifies the pivot index is ds,
not __timestamp.
Validation: 851 tests passed across the 11 touched/related pytest modules;
pre-commit passed for all changed files. The regression coverage fails against
the previous implementation and passes with this fix.
##########
superset/mcp_service/chart/tool/update_chart.py:
##########
@@ -206,209 +219,66 @@ def _append_table_columns(
def _merge_replacement_config(
existing_form_data: dict[str, Any],
new_form_data: dict[str, Any],
- parsed_config: ChartConfig,
+ parsed_config: Any,
*,
dataset_rebind: bool = False,
) -> dict[str, Any]:
- """Delegate update semantics to the shared form-data merge helper."""
- return merge_chart_form_data(
+ """Merge a replacement config through the plugin-aware update merge.
+
+ An explicit empty filter list clears saved filters; plugins that generate
+ their own time binding (Gantt) keep it through that clear.
+ """
+ return merge_form_data_for_update(
existing_form_data,
new_form_data,
parsed_config,
dataset_rebind=dataset_rebind,
)
-def _valid_dataset_reference(
- value: Any,
- columns: set[str],
- metrics: set[str],
- *,
- allow_metric: bool = False,
-) -> bool:
- """Return whether a form_data reference resolves against a dataset."""
- if not isinstance(value, str):
- return True
- normalized = value.casefold()
- return normalized in columns or (allow_metric and normalized in metrics)
-
-
-def _inherited_metrics_match_dataset(
- existing_form_data: dict[str, Any],
- columns: set[str],
- metrics: set[str],
-) -> bool:
- for metric in existing_form_data.get("metrics") or []:
- if isinstance(metric, str) and not _valid_dataset_reference(
- metric, columns, metrics, allow_metric=True
- ):
- return False
- if isinstance(metric, dict):
- column = metric.get("column")
- if isinstance(column, dict) and not _valid_dataset_reference(
- column.get("column_name"), columns, metrics
- ):
- return False
- return True
-
-
-def _inherited_sort_matches_dataset(
- order_by_cols: Any, columns: set[str], metrics: set[str]
-) -> bool:
- for order_by in order_by_cols or []:
- try:
- column = json.loads(order_by)[0]
- except (TypeError, ValueError, IndexError):
- return False
- if not _valid_dataset_reference(column, columns, metrics,
allow_metric=True):
- return False
- return True
-
-
-def _inherited_filters_match_dataset(
- filters: Any, columns: set[str], metrics: set[str]
-) -> bool:
- for filter_ in filters or []:
- if not isinstance(filter_, dict):
- return False
- if filter_.get("expressionType") not in (None, "SIMPLE"):
- return False
- subject = filter_.get("subject") or filter_.get("col")
- allow_metric = str(filter_.get("clause", "WHERE")).upper() == "HAVING"
- if not _valid_dataset_reference(
- subject, columns, metrics, allow_metric=allow_metric
- ):
- return False
- return True
-
-
-#: form_data keys carrying query roles, mapped to the config field that
-#: sets them explicitly. An explicit field is the caller's stated intent,
-#: so it is never treated as inherited state.
-_INHERITED_QUERY_ROLE_FIELDS = {
- "groupby": "group_by",
- "groupby_b": "group_by_secondary",
- "all_columns": None,
- "columns": None,
- "x_axis": None,
- "granularity_sqla": None,
- "metrics": None,
- "order_by_cols": "sort_by",
- "adhoc_filters": "filters",
-}
-
-_INHERITED_COLUMN_LIST_KEYS = frozenset(
- {"groupby", "groupby_b", "all_columns", "columns"}
-)
-_INHERITED_COLUMN_SCALAR_KEYS = frozenset({"x_axis", "granularity_sqla"})
+def _is_dataset_rebind(request: UpdateChartRequest, chart: Any) -> bool:
+ """Return whether a request changes the chart's datasource identity."""
+ return request.dataset_id is not None and str(request.dataset_id) != str(
+ getattr(chart, "datasource_id", None)
+ )
-def _inherited_state_invalid_keys(
- existing_form_data: dict[str, Any],
- new_form_data: dict[str, Any],
- parsed_config: ChartConfig,
- dataset_id: int,
-) -> set[str]:
- """Return inherited query fields that are invalid for a new dataset.
-
- A dataset rebind only has to discard the state that cannot resolve
- against the replacement dataset; everything else stays valid and is
- preserved so the update does not silently reset the chart.
- """
- fields_set = parsed_config.model_fields_set
- inherited_keys = {
- key
- for key, config_field in _INHERITED_QUERY_ROLE_FIELDS.items()
- if key not in new_form_data
- and config_field not in fields_set
- and existing_form_data.get(key)
- }
- if not inherited_keys:
- return set()
-
- from superset.daos.dataset import DatasetDAO
- from superset.mcp_service.chart.validation.dataset_validator import (
- build_dataset_context_from_orm,
+def _add_columns_rebind_error() -> GenerateChartResponse:
+ """Require a complete replacement config when changing table datasets."""
+ return _validation_error_response(
+ message="Cannot combine 'add_columns' with a dataset rebind.",
+ details=(
+ "Dataset-bound roles from the previous dataset cannot be reused. "
+ "Provide 'config' with the complete table configuration for the "
+ "target dataset."
+ ),
)
- context = build_dataset_context_from_orm(DatasetDAO.find_by_id(dataset_id))
- if context is None:
- # The replacement dataset cannot be inspected, so no inherited
- # reference can be shown to be safe.
- return inherited_keys
- columns = {column["name"].casefold() for column in
context.available_columns}
- metrics = {metric["name"].casefold() for metric in
context.available_metrics}
-
- invalid_keys: set[str] = set()
- for key in inherited_keys & _INHERITED_COLUMN_LIST_KEYS:
- values = existing_form_data.get(key)
- if isinstance(values, list) and not all(
- _valid_dataset_reference(value, columns, metrics) for value in
values
- ):
- invalid_keys.add(key)
- for key in inherited_keys & _INHERITED_COLUMN_SCALAR_KEYS:
- if not _valid_dataset_reference(existing_form_data.get(key), columns,
metrics):
- invalid_keys.add(key)
- if "metrics" in inherited_keys and not _inherited_metrics_match_dataset(
- existing_form_data, columns, metrics
- ):
- invalid_keys.add("metrics")
- if "order_by_cols" in inherited_keys and not
_inherited_sort_matches_dataset(
- existing_form_data.get("order_by_cols"), columns, metrics
- ):
- invalid_keys.add("order_by_cols")
- if "adhoc_filters" in inherited_keys and not
_inherited_filters_match_dataset(
- existing_form_data.get("adhoc_filters"), columns, metrics
- ):
- invalid_keys.add("adhoc_filters")
- return invalid_keys
-
-
-def _build_replacement_form_data(
- existing_form_data: dict[str, Any],
- parsed_config: ChartConfig,
- effective_dataset_id: int | None,
- replacement_dataset_id: int | None = None,
+
+def _build_dataset_rebind_payload(
+ request: UpdateChartRequest, chart: Any
) -> dict[str, Any]:
- """Map and merge a replacement config for preview and save paths."""
- new_form_data = map_config_to_form_data(
- parsed_config, dataset_id=effective_dataset_id, include_disabled=True
- )
- new_form_data.pop("_mcp_warnings", None)
- dataset_rebind = replacement_dataset_id is not None
- config_plugin = get_registry().get(parsed_config.chart_type,
include_disabled=True)
- if replacement_dataset_id is not None and not (
- config_plugin is not None and config_plugin.strict_dataset_rebind
- ):
- # Drop only the inherited state the replacement dataset cannot
- # resolve, then merge as a same-dataset update. Plugins with a strict
- # rebind contract handle the rebind in merge_update_form_data.
- invalid_keys = _inherited_state_invalid_keys(
- existing_form_data,
- new_form_data,
- parsed_config,
- replacement_dataset_id,
- )
- existing_form_data = {
- key: value
- for key, value in existing_form_data.items()
- if key not in invalid_keys
- }
- dataset_rebind = False
- merge_table_column_config(existing_form_data, new_form_data)
- merge_interactive_pivot_ui_config(existing_form_data, new_form_data)
- merged = _merge_replacement_config(
- existing_form_data,
- new_form_data,
- parsed_config,
- dataset_rebind=dataset_rebind,
+ """Rebind a chart and keep operation-owned params aligned with the
target."""
+ assert request.dataset_id is not None
+ payload: dict[str, Any] = {
+ "datasource_id": request.dataset_id,
+ "datasource_type": "table",
+ }
+ existing_form_data = _get_existing_form_data(chart)
+ canonical_form_data = canonicalize_operation_form_data(
+ scrub_dataset_bound_form_data(existing_form_data),
Review Comment:
Fixed in f2acaab6defc45cd7680662feced6b74dae95d02.
Dataset changes now require a complete config for every viz type, in both
preview and save paths, rather than scrubbing all roles into an unusable chart.
Same-dataset requests remain idempotent. Updated the dataset_id description,
MCP docs, and migration notes. Regressions:
test_dataset_only_rebind_rejects_populated_roles_in_preview_and_save and
test_dataset_only_rebind_requires_complete_config; complete-config forwarding
and target-access tests also pass.
Validation: 851 tests passed across the 11 touched/related pytest modules;
pre-commit passed for all changed files. The regression coverage fails against
the previous implementation and passes with this fix.
##########
superset/mcp_service/chart/tool/update_chart_preview.py:
##########
@@ -441,55 +486,60 @@ def update_chart_preview( # noqa: C901
"schema_version": "2.0",
"api_version": "v1",
}
- return result
+ return _finalize_response(result)
except OAuth2RedirectError as ex:
logger.warning(
"Chart preview update requires OAuth authentication:
form_data_key=%s",
request.form_data_key,
)
- return {
- "chart": None,
- "error": build_oauth2_redirect_message(ex),
- "success": False,
- "schema_version": "2.0",
- "api_version": "v1",
- }
+ return _finalize_response(
+ {
+ "chart": None,
+ "error": build_oauth2_redirect_message(ex),
+ "success": False,
+ "schema_version": "2.0",
+ "api_version": "v1",
+ }
+ )
except OAuth2Error:
logger.warning(
"OAuth2 configuration error: form_data_key=%s",
request.form_data_key
)
- return {
- "chart": None,
- "error": OAUTH2_CONFIG_ERROR_MESSAGE,
- "success": False,
- "schema_version": "2.0",
- "api_version": "v1",
- }
+ return _finalize_response(
+ {
+ "chart": None,
+ "error": OAUTH2_CONFIG_ERROR_MESSAGE,
+ "success": False,
+ "schema_version": "2.0",
+ "api_version": "v1",
+ }
+ )
except GanttSemanticNormalizationError as ex:
execution_time = int((time.time() - start_time) * 1000)
- return {
- "chart": None,
- "error": {
- "error_type": "gantt_semantic_validation_error",
- "message": "Gantt chart column roles are invalid",
- "details": str(ex),
- "suggestions": [
- "Use different physical columns for start_time and
end_time",
- "Use different physical columns for category and series",
- "Use exact dataset column casing when names differ only by
case",
- ],
- "error_code": "GANTT_SEMANTIC_VALIDATION_ERROR",
- },
- "performance": {
- "query_duration_ms": execution_time,
- "cache_status": "error",
- "optimization_suggestions": [],
- },
- "success": False,
- "schema_version": "2.0",
- "api_version": "v1",
- }
+ return _finalize_response(
+ {
+ "chart": None,
+ "error": {
+ "error_type": "gantt_semantic_validation_error",
+ "message": "Gantt chart column roles are invalid",
+ "details": str(ex),
+ "suggestions": [
+ "Use different physical columns for start_time and
end_time",
+ "Use different physical columns for category and
series",
+ "Use exact dataset column casing when names differ
only "
+ "by case",
+ ],
+ "error_code": "GANTT_SEMANTIC_VALIDATION_ERROR",
+ },
+ "performance": {
+ "query_duration_ms": execution_time,
+ "cache_status": "error",
+ "optimization_suggestions": [],
+ },
+ "success": False,
Review Comment:
Fixed in f2acaab6defc45cd7680662feced6b74dae95d02.
Restored schema_version=2.0 and api_version=v1 on the Gantt semantic
validation-error branch. Regression:
test_gantt_validation_error_includes_response_versions exercises that exception
response.
Validation: 851 tests passed across the 11 touched/related pytest modules;
pre-commit passed for all changed files. The regression coverage fails against
the previous implementation and passes with this fix.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
