sadpandajoe commented on code in PR #43771:
URL: https://github.com/apache/superset/pull/43771#discussion_r4191870218
##########
superset/common/form_data_query_context.py:
##########
@@ -288,71 +461,1274 @@ def _pie_contribution_post_processing(metrics:
list[Any]) -> list[dict[str, Any]
]
-def build_query_context_from_form_data(
- form_data: dict[str, Any],
- datasource: dict[str, Any],
- viz_type: str | None = None,
+def _as_list(value: Any) -> list[Any]:
+ """Return the frontend ``ensureIsArray`` representation of a value."""
+ if value is None:
+ return []
+ return list(value) if isinstance(value, (list, tuple)) else [value]
+
+
+def _label(value: Any, *, metric: bool = False) -> str:
+ """Resolve a frontend-compatible query-field label."""
+ try:
+ return get_metric_name(value) if metric else get_column_name(value)
+ except (AttributeError, KeyError, TypeError, ValueError):
+ if isinstance(value, Mapping):
+ return str(
+ value.get("label")
+ or value.get("column_name")
+ or value.get("sqlExpression")
+ or value
+ )
+ return str(value)
+
+
+def _deduplicate_fields(values: list[Any], *, metric: bool = False) ->
list[Any]:
+ """Deduplicate query fields by their frontend-visible label."""
+ result: list[Any] = []
+ labels: set[str] = set()
+ for value in values:
+ if value is None or value == "":
+ continue
+ label = _label(value, metric=metric)
+ if label in labels:
+ continue
+ labels.add(label)
+ result.append(value)
+ return result
+
+
+def retain_mixed_timeseries_secondary_form_data(
+ form_data: Mapping[str, Any],
) -> dict[str, Any]:
- """
- Build a query-context payload (the JSON shape
``ChartDataQueryContextSchema``
- loads) from a chart's form data and datasource reference.
+ """Mirror ``retainFormDataSuffix(formData, '_b')`` exactly.
- :param form_data: The chart's saved ``params`` parsed to a dict.
- :param datasource: ``{"id": <int>, "type": "table"}`` datasource reference.
- :param viz_type: The chart's viz type, used for viz-specific handling.
- :returns: A single-query query-context dict.
+ Suffixed values are installed first, including falsey values, and shared
+ unsuffixed controls fill only keys that query B did not explicitly set.
"""
- columns, metrics = _columns_and_metrics(form_data, viz_type)
-
- # SIMPLE adhoc filters (+ legacy top-level ``filters``) become query
filters;
- # free-form SQL predicates go into ``extras``. Only ``WHERE``-clause SIMPLE
- # filters are applied (matching the chart), so the export never filters on
a
- # ``HAVING`` clause the chart itself ignores.
- filters = adhoc_filters_to_query_filters(
- form_data.get("adhoc_filters", []), where_only=True
- )
- for flt in form_data.get("filters") or []:
- if isinstance(flt, dict) and flt.get("col") is not None:
- filters.append(flt)
+ secondary: dict[str, Any] = {}
+ for key, value in form_data.items():
+ if key.endswith("_b"):
+ secondary[key[:-2]] = value
+ for key, value in form_data.items():
+ if not key.endswith("_b") and key not in secondary:
+ secondary[key] = value
+ secondary_filter_keys = {
+ "adhoc_filters": "adhoc_filters_b",
+ "extra_filters": "extra_filters_b",
+ "filters": "filters_b",
+ "having": "having_b",
+ "where": "where_b",
+ }
+ if any(suffixed in form_data for suffixed in
secondary_filter_keys.values()):
+ # The frontend exposes adhoc_filters_b, while saved/server payloads can
+ # carry equivalent legacy aliases. Treat the family atomically: an
+ # explicit clear in any B alias must not be repopulated by query A's
+ # differently named filter representation.
+ for primary, suffixed in secondary_filter_keys.items():
+ if suffixed not in form_data:
+ secondary.pop(primary, None)
+ return secondary
- extras = freeform_where_having(form_data)
- if form_data.get("time_grain_sqla"):
- extras["time_grain_sqla"] = form_data["time_grain_sqla"]
- # Prefer the modern ``time_range``; fall back to the legacy
``since``/``until``
- # pair (older charts store the range that way) before defaulting to no
filter.
- time_range = form_data.get("time_range")
- if not time_range and (form_data.get("since") or form_data.get("until")):
- time_range = f"{form_data.get('since') or ''} :
{form_data.get('until') or ''}"
- time_range = time_range or "No filter"
+def _base_query_object( # noqa: C901
+ form_data: dict[str, Any],
+ *,
+ row_limit: int | None,
+ order_desc: bool | None,
+ filters_prepared: bool,
+) -> dict[str, Any]:
+ """Build the shared frontend-equivalent portion of a QueryObject."""
+ columns, metrics, orderby = query_fields_from_form_data(form_data)
query: dict[str, Any] = {
"columns": columns,
"metrics": metrics,
- "orderby": orderby_from_form_data(form_data, metrics, viz_type),
- "filters": filters,
- "time_range": time_range,
}
+ if orderby:
+ query["orderby"] = orderby
+
+ if filters_prepared:
+ query["filters"] = list(form_data.get("filters") or [])
+ for clause in ("where", "having"):
+ if form_data.get(clause):
+ query[clause] = form_data[clause]
+ if form_data.get("extras"):
+ query["extras"] = dict(form_data["extras"])
+ else:
+ filters = adhoc_filters_to_query_filters(
+ form_data.get("adhoc_filters", []), where_only=True
+ )
+ filters.extend(
+ filter_
+ for filter_ in form_data.get("filters") or []
+ if isinstance(filter_, dict) and filter_.get("col") is not None
+ )
+ query["filters"] = filters
+ if extras := freeform_where_having(form_data):
+ query["extras"] = extras
+
+ extras = dict(query.get("extras") or {})
+ if form_data.get("time_grain_sqla") is not None:
+ extras["time_grain_sqla"] = form_data["time_grain_sqla"]
if extras:
query["extras"] = extras
- if viz_type == "pie" and (
- post_processing := _pie_contribution_post_processing(metrics)
+
+ effective_limit = row_limit if row_limit is not None else
form_data.get("row_limit")
+ if effective_limit is not None:
+ query["row_limit"] = effective_limit
+ if form_data.get("row_offset") is not None:
+ query["row_offset"] = form_data["row_offset"]
+ if order_desc is not None:
+ query["order_desc"] = order_desc
+ elif "order_desc" in form_data and form_data["order_desc"] is not None:
+ query["order_desc"] = form_data["order_desc"]
+
+ time_range = form_data.get("time_range")
+ if not time_range and (form_data.get("since") or form_data.get("until")):
+ time_range = f"{form_data.get('since') or ''} :
{form_data.get('until') or ''}"
+ if time_range:
+ query["time_range"] = time_range
+ for key in ("since", "until", "annotation_layers", "url_params",
"custom_params"):
+ if form_data.get(key) is not None:
+ query[key] = form_data[key]
+
+ granularity = form_data.get("granularity") or
form_data.get("granularity_sqla")
+ if granularity:
+ query["granularity"] = granularity
+ series_limit = form_data.get("series_limit", form_data.get("limit"))
+ if series_limit is not None:
+ query["series_limit"] = series_limit
+ series_limit_metric = form_data.get("series_limit_metric")
+ if series_limit_metric is None:
+ series_limit_metric = form_data.get("timeseries_limit_metric")
+ if series_limit_metric is not None:
+ query["series_limit_metric"] = series_limit_metric
+ if form_data.get("group_others_when_limit_reached") is not None:
+ query["group_others_when_limit_reached"] = form_data[
+ "group_others_when_limit_reached"
+ ]
+ return query
+
+
+def _temporalized_columns(form_data: dict[str, Any], columns: list[Any]) ->
list[Any]:
+ """Apply the pivot BASE_AXIS temporal-column contract."""
+ time_grain = form_data.get("time_grain_sqla")
+ temporal_lookup = form_data.get("temporal_columns_lookup") or {}
+ result: list[Any] = []
+ for column in columns:
+ if (
+ isinstance(column, str)
+ and time_grain
+ and (
+ temporal_lookup.get(column)
+ or form_data.get("granularity_sqla") == column
+ )
+ ):
+ result.append(
+ {
+ "timeGrain": time_grain,
+ "columnType": "BASE_AXIS",
+ "sqlExpression": column,
+ "label": column,
+ "expressionType": "SQL",
+ }
+ )
+ else:
+ result.append(column)
+ return result
+
+
+def _box_temporalized_columns(
+ form_data: dict[str, Any], columns: list[Any]
+) -> list[Any]:
+ """Convert only physical columns confirmed temporal by Box Plot
metadata."""
+ time_grain = form_data.get("time_grain_sqla")
+ temporal_lookup = form_data.get("temporal_columns_lookup")
+ if not time_grain or not isinstance(temporal_lookup, Mapping):
+ return columns
+ return [
+ {
+ "timeGrain": time_grain,
+ "columnType": "BASE_AXIS",
+ "sqlExpression": column,
+ "label": column,
+ "expressionType": "SQL",
+ }
+ if isinstance(column, str) and temporal_lookup.get(column) is True
+ else column
+ for column in columns
+ ]
+
+
+def _table_temporalized_columns(
+ form_data: dict[str, Any], columns: list[Any]
+) -> list[Any]:
+ """Promote the first temporal table group-by to the frontend BASE_AXIS.
+
+ Table's builder treats only physical columns named in
+ ``temporal_columns_lookup`` as temporal and moves the first match to the
+ front. Later temporal dimensions remain ordinary group-bys.
+ """
+ time_grain = form_data.get("time_grain_sqla")
+ temporal_lookup = form_data.get("temporal_columns_lookup") or {}
+ if not time_grain or not isinstance(temporal_lookup, Mapping):
+ return columns
+
+ temporal_column: dict[str, Any] | None = None
+ remaining: list[Any] = []
+ for column in columns:
+ if (
+ temporal_column is None
+ and isinstance(column, str)
+ and temporal_lookup.get(column)
+ ):
+ temporal_column = {
+ "timeGrain": time_grain,
+ "columnType": "BASE_AXIS",
+ "sqlExpression": column,
+ "label": column,
+ "expressionType": "SQL",
+ }
+ else:
+ remaining.append(column)
+ return [temporal_column, *remaining] if temporal_column else columns
+
+
+def _histogram_query(form_data: dict[str, Any], query: dict[str, Any]) -> None:
+ groupby = _as_list(form_data.get("groupby"))
+ column = form_data.get("column")
+ query["columns"] = [*groupby, *([column] if column is not None else [])]
+ query["post_processing"] = [
+ {
+ "operation": "histogram",
+ "options": {
+ "column": _label(column),
+ "groupby": [_label(value) for value in groupby],
+ "bins": int(form_data.get("bins", 5)),
+ "cumulative": form_data.get("cumulative", False),
+ "normalize": form_data.get("normalize", False),
+ },
+ }
+ ]
+ if any(
+ isinstance(filter_, dict) and filter_.get("clause") == "HAVING"
+ for filter_ in form_data.get("adhoc_filters") or []
+ ):
+ query["metrics"] = [
+ {
+ "expressionType": "SQL",
+ "sqlExpression": "COUNT(*)",
+ "label": "COUNT(*)",
+ }
+ ]
+ else:
+ query["metrics"] = []
+
+
+def _box_plot_query(form_data: dict[str, Any], query: dict[str, Any]) -> None:
+ distributed = _as_list(form_data.get("columns"))
+ if not distributed and form_data.get("granularity_sqla"):
+ distributed = [form_data["granularity_sqla"]]
+ groupby = _as_list(form_data.get("groupby"))
+ query["columns"] = [*_box_temporalized_columns(form_data, distributed),
*groupby]
+ query["series_columns"] = groupby
+ whisker = form_data.get("whiskerOptions")
+ if not whisker:
+ query["post_processing"] = []
+ return
+ whisker_type = "tukey"
+ percentiles: list[int] | None = None
+ if whisker == "Min/max (no outliers)":
+ whisker_type = "min/max"
+ elif isinstance(whisker, str) and whisker.endswith(" percentiles"):
+ low, high = whisker.removesuffix(" percentiles").split("/", 1)
+ whisker_type = "percentile"
+ percentiles = [int(low), int(high)]
+ query["post_processing"] = [
+ {
+ "operation": "boxplot",
+ "options": {
+ "whisker_type": whisker_type,
+ "percentiles": percentiles,
+ "groupby": [_label(value) for value in groupby],
+ "metrics": [_label(value, metric=True) for value in
query["metrics"]],
+ },
+ }
+ ]
+
+
+_PIVOT_ADDITIVE_AGGREGATES = frozenset({"SUM", "COUNT", "MIN", "MAX"})
+
+
+def _all_metrics_additive(metrics: list[Any]) -> bool:
+ """Mirror Pivot's conservative additive-metric fast-path."""
+ return bool(metrics) and all(
+ isinstance(metric, Mapping)
+ and metric.get("expressionType") == "SIMPLE"
+ and metric.get("aggregate") in _PIVOT_ADDITIVE_AGGREGATES
+ for metric in metrics
+ )
+
+
+def _pivot_grouping_sets(
+ form_data: dict[str, Any], rows: list[Any], columns: list[Any]
+) -> list[list[str]]:
+ """Enumerate the rollup levels requested by Pivot's frontend builder."""
+ row_prefixes = [[], *(rows[: index + 1] for index in range(len(rows)))]
+ column_prefixes = [
+ [],
+ *(columns[: index + 1] for index in range(len(columns))),
+ ]
+ show_values_as = form_data.get("showValuesAs")
+ needs_rows_collapsed = show_values_as in {"percent_col", "percent_total"}
+ needs_columns_collapsed = show_values_as in {"percent_row",
"percent_total"}
+
+ def row_prefix_needed(prefix: list[Any]) -> bool:
+ if len(prefix) == len(rows):
+ return True
+ if not prefix:
+ return bool(form_data.get("colTotals")) or needs_rows_collapsed
+ return bool(form_data.get("rowSubTotals"))
+
+ def column_prefix_needed(prefix: list[Any]) -> bool:
+ if len(prefix) == len(columns):
+ return True
+ if not prefix:
+ return bool(form_data.get("rowTotals")) or needs_columns_collapsed
+ return bool(form_data.get("colSubTotals"))
+
+ levels = [
+ (row_prefix, column_prefix)
+ for row_prefix in row_prefixes
+ if row_prefix_needed(row_prefix)
+ for column_prefix in column_prefixes
+ if column_prefix_needed(column_prefix)
+ ]
+ if form_data.get("combineMetric"):
+ metrics_layout = form_data.get("metricsLayout")
+
+ def forced_denominator(level: tuple[list[Any], list[Any]]) -> bool:
+ row_prefix, column_prefix = level
+ return (needs_rows_collapsed and not row_prefix) or (
+ needs_columns_collapsed and not column_prefix
+ )
+
+ if metrics_layout == "ROWS":
+ levels = [
+ level
+ for level in levels
+ if len(level[0]) == len(rows) or forced_denominator(level)
+ ]
+ else:
+ levels = [
+ level
+ for level in levels
+ if len(level[1]) == len(columns) or forced_denominator(level)
+ ]
+
+ return [
+ [_label(value) for value in _deduplicate_fields([*row_prefix,
*column_prefix])]
+ for row_prefix, column_prefix in levels
+ ]
+
+
+def _pivot_query(form_data: dict[str, Any], query: dict[str, Any]) -> None:
+ rows = _as_list(form_data.get("groupbyRows"))
+ columns = _as_list(form_data.get("groupbyColumns"))
+ if form_data.get("transposePivot"):
+ rows, columns = columns, rows
+ query["columns"] = _temporalized_columns(
+ form_data, _deduplicate_fields([*rows, *columns])
+ )
+ metric = query.get("series_limit_metric") or next(
+ iter(query.get("metrics") or []), None
+ )
+ query["orderby"] = (
+ [[metric, not bool(query.get("order_desc", True))]]
+ if metric is not None
+ else []
+ )
+ if not _all_metrics_additive(query.get("metrics") or []):
+ query["grouping_sets"] = _pivot_grouping_sets(form_data, rows, columns)
+
+
+def _waterfall_query(form_data: dict[str, Any], query: dict[str, Any]) -> None:
+ x_axis = form_data.get("x_axis") or form_data.get("granularity_sqla")
+ columns = [*_as_list(x_axis), *_as_list(form_data.get("groupby"))]
+ query["columns"] = _deduplicate_fields(columns)
+ query["orderby"] = [[column, True] for column in query["columns"]]
+
+
+def _gantt_query(form_data: dict[str, Any], query: dict[str, Any]) -> None:
+ groupby = _as_list(form_data.get("series"))
+ orderby = query_fields_from_form_data(form_data)[2]
+ columns = [
+ form_data.get("start_time"),
+ form_data.get("end_time"),
+ form_data.get("y_axis"),
+ *groupby,
+ *_as_list(form_data.get("tooltip_columns")),
+ *(entry[0] for entry in orderby if entry),
+ ]
+ query["columns"] = _deduplicate_fields(columns)
+ query["metrics"] = _as_list(form_data.get("tooltip_metrics"))
+ query["orderby"] = orderby
+ query["series_columns"] = groupby
+
+
+def _normalize_query_orderby(query: dict[str, Any]) -> None:
+ """Mirror ``normalizeOrderBy`` while retaining limit-direction controls."""
+ orderby = query.get("orderby")
+ if (
+ isinstance(orderby, list)
+ and orderby
+ and isinstance(orderby[0], (list, tuple))
+ and len(orderby[0]) == 2
+ and orderby[0][0]
+ and isinstance(orderby[0][1], bool)
):
+ return
+ metric = (
+ query.get("series_limit_metric")
+ or query.get("legacy_order_by")
+ or next(iter(query.get("metrics") or []), None)
+ )
+ if metric is None:
+ query.pop("orderby", None)
+ return
+ query["orderby"] = [[metric, not bool(query.get("order_desc", True))]]
+
+
+_TIME_COMPARISON_TYPES = frozenset({"values", "difference", "percentage",
"ratio"})
+
+
+def _metric_offset_map(
+ form_data: dict[str, Any],
+ metric_labels: list[str],
+ offsets: list[Any] | None = None,
+) -> dict[str, str]:
+ """Return the frontend time-comparison metric label map."""
+ if form_data.get("comparison_type") not in _TIME_COMPARISON_TYPES:
+ return {}
+ return {
+ f"{metric}__{offset}": metric
+ for metric in metric_labels
+ for offset in (
+ offsets if offsets is not None else
_as_list(form_data.get("time_compare"))
+ )
+ }
+
+
+def _table_time_offsets(form_data: dict[str, Any]) -> list[Any]:
+ """Resolve Table custom/inherited shifts like its frontend query
adapter."""
+ raw_offsets = _as_list(form_data.get("time_compare"))
+ offsets = [offset for offset in raw_offsets if offset not in {"custom",
"inherit"}]
+ if "custom" in raw_offsets and form_data.get("start_date_offset") is not
None:
+ offsets.append(form_data["start_date_offset"])
+ extra_form_data = form_data.get("extra_form_data")
+ if isinstance(extra_form_data, Mapping) and
extra_form_data.get("time_compare"):
+ inherited = extra_form_data["time_compare"]
Review Comment:
`build_query_dicts_from_form_data` runs `prepare_form_data_for_query` first,
and its `merge_extra_filters` → `merge_extra_form_data` pops `extra_form_data`
from the form data. By the time this runs, a saved table with
`time_compare=["inherit"]` and `extra_form_data={"time_compare": "1 year ago"}`
has no `extra_form_data` left, so `time_offsets` comes out empty and the
prior-year column is missing. The new inheritance test calls the common builder
directly and skips that preparation step. Should the inherited offset be
captured before the merge, with a test through
`build_query_dicts_from_form_data`?
##########
superset/common/form_data_query_context.py:
##########
@@ -288,71 +461,1274 @@ def _pie_contribution_post_processing(metrics:
list[Any]) -> list[dict[str, Any]
]
-def build_query_context_from_form_data(
- form_data: dict[str, Any],
- datasource: dict[str, Any],
- viz_type: str | None = None,
+def _as_list(value: Any) -> list[Any]:
+ """Return the frontend ``ensureIsArray`` representation of a value."""
+ if value is None:
+ return []
+ return list(value) if isinstance(value, (list, tuple)) else [value]
+
+
+def _label(value: Any, *, metric: bool = False) -> str:
+ """Resolve a frontend-compatible query-field label."""
+ try:
+ return get_metric_name(value) if metric else get_column_name(value)
+ except (AttributeError, KeyError, TypeError, ValueError):
+ if isinstance(value, Mapping):
+ return str(
+ value.get("label")
+ or value.get("column_name")
+ or value.get("sqlExpression")
+ or value
+ )
+ return str(value)
+
+
+def _deduplicate_fields(values: list[Any], *, metric: bool = False) ->
list[Any]:
+ """Deduplicate query fields by their frontend-visible label."""
+ result: list[Any] = []
+ labels: set[str] = set()
+ for value in values:
+ if value is None or value == "":
+ continue
+ label = _label(value, metric=metric)
+ if label in labels:
+ continue
+ labels.add(label)
+ result.append(value)
+ return result
+
+
+def retain_mixed_timeseries_secondary_form_data(
+ form_data: Mapping[str, Any],
) -> dict[str, Any]:
- """
- Build a query-context payload (the JSON shape
``ChartDataQueryContextSchema``
- loads) from a chart's form data and datasource reference.
+ """Mirror ``retainFormDataSuffix(formData, '_b')`` exactly.
- :param form_data: The chart's saved ``params`` parsed to a dict.
- :param datasource: ``{"id": <int>, "type": "table"}`` datasource reference.
- :param viz_type: The chart's viz type, used for viz-specific handling.
- :returns: A single-query query-context dict.
+ Suffixed values are installed first, including falsey values, and shared
+ unsuffixed controls fill only keys that query B did not explicitly set.
"""
- columns, metrics = _columns_and_metrics(form_data, viz_type)
-
- # SIMPLE adhoc filters (+ legacy top-level ``filters``) become query
filters;
- # free-form SQL predicates go into ``extras``. Only ``WHERE``-clause SIMPLE
- # filters are applied (matching the chart), so the export never filters on
a
- # ``HAVING`` clause the chart itself ignores.
- filters = adhoc_filters_to_query_filters(
- form_data.get("adhoc_filters", []), where_only=True
- )
- for flt in form_data.get("filters") or []:
- if isinstance(flt, dict) and flt.get("col") is not None:
- filters.append(flt)
+ secondary: dict[str, Any] = {}
+ for key, value in form_data.items():
+ if key.endswith("_b"):
+ secondary[key[:-2]] = value
+ for key, value in form_data.items():
+ if not key.endswith("_b") and key not in secondary:
+ secondary[key] = value
+ secondary_filter_keys = {
+ "adhoc_filters": "adhoc_filters_b",
+ "extra_filters": "extra_filters_b",
+ "filters": "filters_b",
+ "having": "having_b",
+ "where": "where_b",
+ }
+ if any(suffixed in form_data for suffixed in
secondary_filter_keys.values()):
+ # The frontend exposes adhoc_filters_b, while saved/server payloads can
+ # carry equivalent legacy aliases. Treat the family atomically: an
+ # explicit clear in any B alias must not be repopulated by query A's
+ # differently named filter representation.
+ for primary, suffixed in secondary_filter_keys.items():
+ if suffixed not in form_data:
+ secondary.pop(primary, None)
+ return secondary
- extras = freeform_where_having(form_data)
- if form_data.get("time_grain_sqla"):
- extras["time_grain_sqla"] = form_data["time_grain_sqla"]
- # Prefer the modern ``time_range``; fall back to the legacy
``since``/``until``
- # pair (older charts store the range that way) before defaulting to no
filter.
- time_range = form_data.get("time_range")
- if not time_range and (form_data.get("since") or form_data.get("until")):
- time_range = f"{form_data.get('since') or ''} :
{form_data.get('until') or ''}"
- time_range = time_range or "No filter"
+def _base_query_object( # noqa: C901
+ form_data: dict[str, Any],
+ *,
+ row_limit: int | None,
+ order_desc: bool | None,
+ filters_prepared: bool,
+) -> dict[str, Any]:
+ """Build the shared frontend-equivalent portion of a QueryObject."""
+ columns, metrics, orderby = query_fields_from_form_data(form_data)
query: dict[str, Any] = {
"columns": columns,
"metrics": metrics,
- "orderby": orderby_from_form_data(form_data, metrics, viz_type),
- "filters": filters,
- "time_range": time_range,
}
+ if orderby:
+ query["orderby"] = orderby
+
+ if filters_prepared:
+ query["filters"] = list(form_data.get("filters") or [])
+ for clause in ("where", "having"):
+ if form_data.get(clause):
+ query[clause] = form_data[clause]
+ if form_data.get("extras"):
+ query["extras"] = dict(form_data["extras"])
+ else:
+ filters = adhoc_filters_to_query_filters(
+ form_data.get("adhoc_filters", []), where_only=True
+ )
+ filters.extend(
+ filter_
+ for filter_ in form_data.get("filters") or []
+ if isinstance(filter_, dict) and filter_.get("col") is not None
+ )
+ query["filters"] = filters
+ if extras := freeform_where_having(form_data):
+ query["extras"] = extras
+
+ extras = dict(query.get("extras") or {})
+ if form_data.get("time_grain_sqla") is not None:
+ extras["time_grain_sqla"] = form_data["time_grain_sqla"]
if extras:
query["extras"] = extras
- if viz_type == "pie" and (
- post_processing := _pie_contribution_post_processing(metrics)
+
+ effective_limit = row_limit if row_limit is not None else
form_data.get("row_limit")
+ if effective_limit is not None:
+ query["row_limit"] = effective_limit
+ if form_data.get("row_offset") is not None:
+ query["row_offset"] = form_data["row_offset"]
+ if order_desc is not None:
+ query["order_desc"] = order_desc
+ elif "order_desc" in form_data and form_data["order_desc"] is not None:
+ query["order_desc"] = form_data["order_desc"]
+
+ time_range = form_data.get("time_range")
+ if not time_range and (form_data.get("since") or form_data.get("until")):
+ time_range = f"{form_data.get('since') or ''} :
{form_data.get('until') or ''}"
+ if time_range:
+ query["time_range"] = time_range
+ for key in ("since", "until", "annotation_layers", "url_params",
"custom_params"):
+ if form_data.get(key) is not None:
+ query[key] = form_data[key]
+
+ granularity = form_data.get("granularity") or
form_data.get("granularity_sqla")
+ if granularity:
+ query["granularity"] = granularity
+ series_limit = form_data.get("series_limit", form_data.get("limit"))
+ if series_limit is not None:
+ query["series_limit"] = series_limit
+ series_limit_metric = form_data.get("series_limit_metric")
+ if series_limit_metric is None:
+ series_limit_metric = form_data.get("timeseries_limit_metric")
+ if series_limit_metric is not None:
+ query["series_limit_metric"] = series_limit_metric
+ if form_data.get("group_others_when_limit_reached") is not None:
+ query["group_others_when_limit_reached"] = form_data[
+ "group_others_when_limit_reached"
+ ]
+ return query
+
+
+def _temporalized_columns(form_data: dict[str, Any], columns: list[Any]) ->
list[Any]:
+ """Apply the pivot BASE_AXIS temporal-column contract."""
+ time_grain = form_data.get("time_grain_sqla")
+ temporal_lookup = form_data.get("temporal_columns_lookup") or {}
+ result: list[Any] = []
+ for column in columns:
+ if (
+ isinstance(column, str)
+ and time_grain
+ and (
+ temporal_lookup.get(column)
+ or form_data.get("granularity_sqla") == column
+ )
+ ):
+ result.append(
+ {
+ "timeGrain": time_grain,
+ "columnType": "BASE_AXIS",
+ "sqlExpression": column,
+ "label": column,
+ "expressionType": "SQL",
+ }
+ )
+ else:
+ result.append(column)
+ return result
+
+
+def _box_temporalized_columns(
+ form_data: dict[str, Any], columns: list[Any]
+) -> list[Any]:
+ """Convert only physical columns confirmed temporal by Box Plot
metadata."""
+ time_grain = form_data.get("time_grain_sqla")
+ temporal_lookup = form_data.get("temporal_columns_lookup")
+ if not time_grain or not isinstance(temporal_lookup, Mapping):
+ return columns
+ return [
+ {
+ "timeGrain": time_grain,
+ "columnType": "BASE_AXIS",
+ "sqlExpression": column,
+ "label": column,
+ "expressionType": "SQL",
+ }
+ if isinstance(column, str) and temporal_lookup.get(column) is True
+ else column
+ for column in columns
+ ]
+
+
+def _table_temporalized_columns(
+ form_data: dict[str, Any], columns: list[Any]
+) -> list[Any]:
+ """Promote the first temporal table group-by to the frontend BASE_AXIS.
+
+ Table's builder treats only physical columns named in
+ ``temporal_columns_lookup`` as temporal and moves the first match to the
+ front. Later temporal dimensions remain ordinary group-bys.
+ """
+ time_grain = form_data.get("time_grain_sqla")
+ temporal_lookup = form_data.get("temporal_columns_lookup") or {}
+ if not time_grain or not isinstance(temporal_lookup, Mapping):
+ return columns
+
+ temporal_column: dict[str, Any] | None = None
+ remaining: list[Any] = []
+ for column in columns:
+ if (
+ temporal_column is None
+ and isinstance(column, str)
+ and temporal_lookup.get(column)
+ ):
+ temporal_column = {
+ "timeGrain": time_grain,
+ "columnType": "BASE_AXIS",
+ "sqlExpression": column,
+ "label": column,
+ "expressionType": "SQL",
+ }
+ else:
+ remaining.append(column)
+ return [temporal_column, *remaining] if temporal_column else columns
+
+
+def _histogram_query(form_data: dict[str, Any], query: dict[str, Any]) -> None:
+ groupby = _as_list(form_data.get("groupby"))
+ column = form_data.get("column")
+ query["columns"] = [*groupby, *([column] if column is not None else [])]
+ query["post_processing"] = [
+ {
+ "operation": "histogram",
+ "options": {
+ "column": _label(column),
+ "groupby": [_label(value) for value in groupby],
+ "bins": int(form_data.get("bins", 5)),
+ "cumulative": form_data.get("cumulative", False),
+ "normalize": form_data.get("normalize", False),
+ },
+ }
+ ]
+ if any(
+ isinstance(filter_, dict) and filter_.get("clause") == "HAVING"
+ for filter_ in form_data.get("adhoc_filters") or []
+ ):
+ query["metrics"] = [
+ {
+ "expressionType": "SQL",
+ "sqlExpression": "COUNT(*)",
+ "label": "COUNT(*)",
+ }
+ ]
+ else:
+ query["metrics"] = []
+
+
+def _box_plot_query(form_data: dict[str, Any], query: dict[str, Any]) -> None:
+ distributed = _as_list(form_data.get("columns"))
+ if not distributed and form_data.get("granularity_sqla"):
+ distributed = [form_data["granularity_sqla"]]
+ groupby = _as_list(form_data.get("groupby"))
+ query["columns"] = [*_box_temporalized_columns(form_data, distributed),
*groupby]
+ query["series_columns"] = groupby
+ whisker = form_data.get("whiskerOptions")
+ if not whisker:
+ query["post_processing"] = []
+ return
+ whisker_type = "tukey"
+ percentiles: list[int] | None = None
+ if whisker == "Min/max (no outliers)":
+ whisker_type = "min/max"
+ elif isinstance(whisker, str) and whisker.endswith(" percentiles"):
+ low, high = whisker.removesuffix(" percentiles").split("/", 1)
+ whisker_type = "percentile"
+ percentiles = [int(low), int(high)]
+ query["post_processing"] = [
+ {
+ "operation": "boxplot",
+ "options": {
+ "whisker_type": whisker_type,
+ "percentiles": percentiles,
+ "groupby": [_label(value) for value in groupby],
+ "metrics": [_label(value, metric=True) for value in
query["metrics"]],
+ },
+ }
+ ]
+
+
+_PIVOT_ADDITIVE_AGGREGATES = frozenset({"SUM", "COUNT", "MIN", "MAX"})
+
+
+def _all_metrics_additive(metrics: list[Any]) -> bool:
+ """Mirror Pivot's conservative additive-metric fast-path."""
+ return bool(metrics) and all(
+ isinstance(metric, Mapping)
+ and metric.get("expressionType") == "SIMPLE"
+ and metric.get("aggregate") in _PIVOT_ADDITIVE_AGGREGATES
+ for metric in metrics
+ )
+
+
+def _pivot_grouping_sets(
+ form_data: dict[str, Any], rows: list[Any], columns: list[Any]
+) -> list[list[str]]:
+ """Enumerate the rollup levels requested by Pivot's frontend builder."""
+ row_prefixes = [[], *(rows[: index + 1] for index in range(len(rows)))]
+ column_prefixes = [
+ [],
+ *(columns[: index + 1] for index in range(len(columns))),
+ ]
+ show_values_as = form_data.get("showValuesAs")
+ needs_rows_collapsed = show_values_as in {"percent_col", "percent_total"}
+ needs_columns_collapsed = show_values_as in {"percent_row",
"percent_total"}
+
+ def row_prefix_needed(prefix: list[Any]) -> bool:
+ if len(prefix) == len(rows):
+ return True
+ if not prefix:
+ return bool(form_data.get("colTotals")) or needs_rows_collapsed
+ return bool(form_data.get("rowSubTotals"))
+
+ def column_prefix_needed(prefix: list[Any]) -> bool:
+ if len(prefix) == len(columns):
+ return True
+ if not prefix:
+ return bool(form_data.get("rowTotals")) or needs_columns_collapsed
+ return bool(form_data.get("colSubTotals"))
+
+ levels = [
+ (row_prefix, column_prefix)
+ for row_prefix in row_prefixes
+ if row_prefix_needed(row_prefix)
+ for column_prefix in column_prefixes
+ if column_prefix_needed(column_prefix)
+ ]
+ if form_data.get("combineMetric"):
+ metrics_layout = form_data.get("metricsLayout")
+
+ def forced_denominator(level: tuple[list[Any], list[Any]]) -> bool:
+ row_prefix, column_prefix = level
+ return (needs_rows_collapsed and not row_prefix) or (
+ needs_columns_collapsed and not column_prefix
+ )
+
+ if metrics_layout == "ROWS":
+ levels = [
+ level
+ for level in levels
+ if len(level[0]) == len(rows) or forced_denominator(level)
+ ]
+ else:
+ levels = [
+ level
+ for level in levels
+ if len(level[1]) == len(columns) or forced_denominator(level)
+ ]
+
+ return [
+ [_label(value) for value in _deduplicate_fields([*row_prefix,
*column_prefix])]
+ for row_prefix, column_prefix in levels
+ ]
+
+
+def _pivot_query(form_data: dict[str, Any], query: dict[str, Any]) -> None:
+ rows = _as_list(form_data.get("groupbyRows"))
+ columns = _as_list(form_data.get("groupbyColumns"))
+ if form_data.get("transposePivot"):
+ rows, columns = columns, rows
+ query["columns"] = _temporalized_columns(
+ form_data, _deduplicate_fields([*rows, *columns])
+ )
+ metric = query.get("series_limit_metric") or next(
+ iter(query.get("metrics") or []), None
+ )
+ query["orderby"] = (
+ [[metric, not bool(query.get("order_desc", True))]]
+ if metric is not None
+ else []
+ )
+ if not _all_metrics_additive(query.get("metrics") or []):
+ query["grouping_sets"] = _pivot_grouping_sets(form_data, rows, columns)
+
+
+def _waterfall_query(form_data: dict[str, Any], query: dict[str, Any]) -> None:
+ x_axis = form_data.get("x_axis") or form_data.get("granularity_sqla")
+ columns = [*_as_list(x_axis), *_as_list(form_data.get("groupby"))]
+ query["columns"] = _deduplicate_fields(columns)
+ query["orderby"] = [[column, True] for column in query["columns"]]
+
+
+def _gantt_query(form_data: dict[str, Any], query: dict[str, Any]) -> None:
+ groupby = _as_list(form_data.get("series"))
+ orderby = query_fields_from_form_data(form_data)[2]
+ columns = [
+ form_data.get("start_time"),
+ form_data.get("end_time"),
+ form_data.get("y_axis"),
+ *groupby,
+ *_as_list(form_data.get("tooltip_columns")),
+ *(entry[0] for entry in orderby if entry),
+ ]
+ query["columns"] = _deduplicate_fields(columns)
+ query["metrics"] = _as_list(form_data.get("tooltip_metrics"))
+ query["orderby"] = orderby
+ query["series_columns"] = groupby
+
+
+def _normalize_query_orderby(query: dict[str, Any]) -> None:
+ """Mirror ``normalizeOrderBy`` while retaining limit-direction controls."""
+ orderby = query.get("orderby")
+ if (
+ isinstance(orderby, list)
+ and orderby
+ and isinstance(orderby[0], (list, tuple))
+ and len(orderby[0]) == 2
+ and orderby[0][0]
+ and isinstance(orderby[0][1], bool)
):
+ return
+ metric = (
+ query.get("series_limit_metric")
+ or query.get("legacy_order_by")
+ or next(iter(query.get("metrics") or []), None)
+ )
+ if metric is None:
+ query.pop("orderby", None)
+ return
+ query["orderby"] = [[metric, not bool(query.get("order_desc", True))]]
+
+
+_TIME_COMPARISON_TYPES = frozenset({"values", "difference", "percentage",
"ratio"})
+
+
+def _metric_offset_map(
+ form_data: dict[str, Any],
+ metric_labels: list[str],
+ offsets: list[Any] | None = None,
+) -> dict[str, str]:
+ """Return the frontend time-comparison metric label map."""
+ if form_data.get("comparison_type") not in _TIME_COMPARISON_TYPES:
+ return {}
+ return {
+ f"{metric}__{offset}": metric
+ for metric in metric_labels
+ for offset in (
+ offsets if offsets is not None else
_as_list(form_data.get("time_compare"))
+ )
+ }
+
+
+def _table_time_offsets(form_data: dict[str, Any]) -> list[Any]:
+ """Resolve Table custom/inherited shifts like its frontend query
adapter."""
+ raw_offsets = _as_list(form_data.get("time_compare"))
+ offsets = [offset for offset in raw_offsets if offset not in {"custom",
"inherit"}]
+ if "custom" in raw_offsets and form_data.get("start_date_offset") is not
None:
+ offsets.append(form_data["start_date_offset"])
+ extra_form_data = form_data.get("extra_form_data")
+ if isinstance(extra_form_data, Mapping) and
extra_form_data.get("time_compare"):
+ inherited = extra_form_data["time_compare"]
+ if inherited not in offsets:
+ offsets = [inherited]
+ return offsets
+
+
+def _x_axis_column(form_data: Mapping[str, Any]) -> Any | None:
+ """Return a supported x-axis column, excluding legacy granularity.
+
+ ``column_name`` mappings are retained for old server/native payloads. Big
+ Number uses the stricter frontend predicate below.
+ """
+ x_axis = form_data.get("x_axis")
+ if isinstance(x_axis, str):
+ return x_axis if x_axis else None
+ if isinstance(x_axis, Mapping):
+ if isinstance(column_name := x_axis.get("column_name"), str) and
column_name:
+ return column_name
+ # Frontend SQL adhoc columns remain objects in the QueryObject.
+ return x_axis if x_axis else None
+ return None
+
+
+def _frontend_x_axis_column(form_data: Mapping[str, Any]) -> Any | None:
+ """Mirror ``isQueryFormColumn`` for physical and SQL adhoc columns."""
+ x_axis = form_data.get("x_axis")
+ if isinstance(x_axis, str):
+ return x_axis if x_axis else None
+ if (
+ isinstance(x_axis, Mapping)
+ and "sqlExpression" in x_axis
+ and "label" in x_axis
+ and x_axis.get("expressionType") in {None, "SQL"}
+ ):
+ return x_axis
+ return None
+
+
+def normalize_time_column(
+ form_data: Mapping[str, Any], query: dict[str, Any]
+) -> dict[str, Any]:
+ """Apply the final shared frontend ``normalizeTimeColumn`` mutator."""
+ x_axis = _frontend_x_axis_column(form_data)
+ columns = query.get("columns")
+ if x_axis is None or not isinstance(columns, list):
+ return query
+
+ axis_index: int | None = None
+ for index, column in enumerate(columns):
+ if isinstance(x_axis, str) and isinstance(column, str) and column ==
x_axis:
+ axis_index = index
+ break
+ if (
+ isinstance(x_axis, Mapping)
+ and isinstance(column, Mapping)
+ and column.get("sqlExpression") == x_axis.get("sqlExpression")
+ ):
+ axis_index = index
+ break
+ if axis_index is None:
+ return query
+
+ normalized = dict(query)
+ normalized_columns = list(columns)
+ grain = (query.get("extras") or {}).get("time_grain_sqla")
+ if isinstance(columns[axis_index], Mapping):
+ normalized_axis = {
+ "columnType": "BASE_AXIS",
+ **({"timeGrain": grain} if grain is not None else {}),
+ **columns[axis_index],
+ }
+ else:
+ normalized_axis = {
+ "columnType": "BASE_AXIS",
+ "sqlExpression": x_axis,
+ "label": x_axis,
+ "expressionType": "SQL",
+ "isColumnReference": True,
+ **({"timeGrain": grain} if grain is not None else {}),
+ }
+ normalized_columns[axis_index] = normalized_axis
+ normalized["columns"] = normalized_columns
+ normalized.pop("is_timeseries", None)
+ return normalized
+
+
+def _finalize_query_objects(
+ form_data: Mapping[str, Any], queries: list[dict[str, Any]]
+) -> list[dict[str, Any]]:
+ """Run shared query-context mutators after every visualization adapter."""
+ return [normalize_time_column(form_data, query) for query in queries]
+
+
+def _x_axis_label(
+ form_data: Mapping[str, Any], *, frontend_strict: bool = False
+) -> str | None:
+ """Mirror getXAxisColumn/getXAxisLabel for explicit and legacy axes."""
+ explicit = (
+ _frontend_x_axis_column(form_data)
+ if frontend_strict
+ else _x_axis_column(form_data)
+ )
+ if explicit:
+ return _label(explicit)
+ if form_data.get("granularity_sqla"):
+ return DTTM_ALIAS
+ return None
+
+
+def _rename_operator(
+ form_data: dict[str, Any],
+ query: dict[str, Any],
+ *,
+ x_axis_label: str | None,
+) -> dict[str, Any] | None:
+ """Mirror the ECharts ``renameOperator`` for Timeseries and Mixed
charts."""
+ metrics = list(query.get("metrics") or [])
+ metric_labels = [_label(metric, metric=True) for metric in metrics]
+ series_columns = query.get("series_columns")
+ columns = _as_list(
+ series_columns if series_columns is not None else query.get("columns")
+ )
+ time_offsets = _as_list(form_data.get("time_compare"))
+ offset_map = _metric_offset_map(form_data, metric_labels)
+ is_time_comparison = bool(offset_map)
+ truncate_metric = form_data.get("truncate_metric")
+
+ should_rename = (
+ bool(metrics)
+ and bool(x_axis_label)
+ and (
+ is_time_comparison
+ or (
+ (bool(columns) or len(time_offsets) > 1)
+ and "truncate_metric" in form_data
+ and bool(truncate_metric)
+ )
+ )
+ )
+ if not should_rename:
+ return None
+
+ renamed: dict[str, str | None] = {}
+ comparison_type = form_data.get("comparison_type")
+ if is_time_comparison:
+ for metric_with_offset, metric_only in offset_map.items():
+ offset_label = next(
+ (
+ str(offset)
+ for offset in time_offsets
+ if metric_with_offset.endswith(f"__{offset}")
+ ),
+ None,
+ )
+ source = (
+ metric_with_offset
+ if comparison_type == "values"
+ else f"{comparison_type}__{metric_only}__{metric_with_offset}"
+ )
+ renamed[source] = (
+ f"{metric_only}, {offset_label}" if len(metrics) > 1 else
offset_label
+ )
+
+ if (
+ comparison_type not in {"difference", "percentage", "ratio"}
+ and len(metrics) == 1
+ and not renamed
+ ):
+ renamed[metric_labels[0]] = None
+ if not renamed:
+ return None
+ return {
+ "operation": "rename",
+ "options": {"columns": renamed, "level": 0, "inplace": True},
+ }
+
+
+def _timeseries_post_processing( # noqa: C901
+ form_data: dict[str, Any],
+ query: dict[str, Any],
+ *,
+ x_axis_label: str | None,
+ groupby: list[Any],
+ mixed: bool,
+) -> tuple[list[dict[str, Any]], list[Any]]:
+ """Build the Timeseries/Mixed operator pipeline in frontend order."""
+ metric_labels = [_label(value, metric=True) for value in
query.get("metrics") or []]
+ offset_map = _metric_offset_map(form_data, metric_labels)
+ time_offsets = _as_list(form_data.get("time_compare")) if offset_map else
[]
+ post_processing: list[dict[str, Any]] = []
+
+ if x_axis_label and metric_labels:
+ aggregate_labels = (
+ [*offset_map.values(), *offset_map] if offset_map else
metric_labels
+ )
+ post_processing.append(
+ {
+ "operation": "pivot",
+ "options": {
+ "index": [x_axis_label],
+ "columns": [_label(value) for value in groupby],
+ "aggregates": {
+ label: {"operator": "mean"} for label in
aggregate_labels
+ },
+ "drop_missing_columns": not form_data.get(
+ "show_empty_columns", False
+ ),
+ },
+ }
+ )
+
+ if form_data.get("resample_method") and form_data.get("resample_rule"):
+ zero_fill = form_data["resample_method"] == "zerofill"
+ post_processing.append(
+ {
+ "operation": "resample",
+ "options": {
+ "method": "asfreq" if zero_fill else
form_data["resample_method"],
+ "rule": form_data["resample_rule"],
+ "fill_value": 0 if zero_fill else None,
Review Comment:
This resample operator never passes `fill_time_range`, which the frontend
emits when `resample_fill_time_range` is true and which `QueryObject` turns
into the range boundaries. For a Jan 1–8 range with daily zero-fill and data
only on Jan 3–5, Explore returns Jan 1–7 with zero edge buckets while this
rebuild returns only Jan 3–5; the Big Number copy at line 1336 has the same
gap, so previews and fallback data disagree with the saved chart. Should both
adapters forward `fill_time_range` when the form data sets it?
##########
superset/common/form_data_query_context.py:
##########
@@ -288,71 +461,1274 @@ def _pie_contribution_post_processing(metrics:
list[Any]) -> list[dict[str, Any]
]
-def build_query_context_from_form_data(
- form_data: dict[str, Any],
- datasource: dict[str, Any],
- viz_type: str | None = None,
+def _as_list(value: Any) -> list[Any]:
+ """Return the frontend ``ensureIsArray`` representation of a value."""
+ if value is None:
+ return []
+ return list(value) if isinstance(value, (list, tuple)) else [value]
+
+
+def _label(value: Any, *, metric: bool = False) -> str:
+ """Resolve a frontend-compatible query-field label."""
+ try:
+ return get_metric_name(value) if metric else get_column_name(value)
+ except (AttributeError, KeyError, TypeError, ValueError):
+ if isinstance(value, Mapping):
+ return str(
+ value.get("label")
+ or value.get("column_name")
+ or value.get("sqlExpression")
+ or value
+ )
+ return str(value)
+
+
+def _deduplicate_fields(values: list[Any], *, metric: bool = False) ->
list[Any]:
+ """Deduplicate query fields by their frontend-visible label."""
+ result: list[Any] = []
+ labels: set[str] = set()
+ for value in values:
+ if value is None or value == "":
+ continue
+ label = _label(value, metric=metric)
+ if label in labels:
+ continue
+ labels.add(label)
+ result.append(value)
+ return result
+
+
+def retain_mixed_timeseries_secondary_form_data(
+ form_data: Mapping[str, Any],
) -> dict[str, Any]:
- """
- Build a query-context payload (the JSON shape
``ChartDataQueryContextSchema``
- loads) from a chart's form data and datasource reference.
+ """Mirror ``retainFormDataSuffix(formData, '_b')`` exactly.
- :param form_data: The chart's saved ``params`` parsed to a dict.
- :param datasource: ``{"id": <int>, "type": "table"}`` datasource reference.
- :param viz_type: The chart's viz type, used for viz-specific handling.
- :returns: A single-query query-context dict.
+ Suffixed values are installed first, including falsey values, and shared
+ unsuffixed controls fill only keys that query B did not explicitly set.
"""
- columns, metrics = _columns_and_metrics(form_data, viz_type)
-
- # SIMPLE adhoc filters (+ legacy top-level ``filters``) become query
filters;
- # free-form SQL predicates go into ``extras``. Only ``WHERE``-clause SIMPLE
- # filters are applied (matching the chart), so the export never filters on
a
- # ``HAVING`` clause the chart itself ignores.
- filters = adhoc_filters_to_query_filters(
- form_data.get("adhoc_filters", []), where_only=True
- )
- for flt in form_data.get("filters") or []:
- if isinstance(flt, dict) and flt.get("col") is not None:
- filters.append(flt)
+ secondary: dict[str, Any] = {}
+ for key, value in form_data.items():
+ if key.endswith("_b"):
+ secondary[key[:-2]] = value
+ for key, value in form_data.items():
+ if not key.endswith("_b") and key not in secondary:
+ secondary[key] = value
+ secondary_filter_keys = {
+ "adhoc_filters": "adhoc_filters_b",
+ "extra_filters": "extra_filters_b",
+ "filters": "filters_b",
+ "having": "having_b",
+ "where": "where_b",
+ }
+ if any(suffixed in form_data for suffixed in
secondary_filter_keys.values()):
+ # The frontend exposes adhoc_filters_b, while saved/server payloads can
+ # carry equivalent legacy aliases. Treat the family atomically: an
+ # explicit clear in any B alias must not be repopulated by query A's
+ # differently named filter representation.
+ for primary, suffixed in secondary_filter_keys.items():
+ if suffixed not in form_data:
+ secondary.pop(primary, None)
+ return secondary
- extras = freeform_where_having(form_data)
- if form_data.get("time_grain_sqla"):
- extras["time_grain_sqla"] = form_data["time_grain_sqla"]
- # Prefer the modern ``time_range``; fall back to the legacy
``since``/``until``
- # pair (older charts store the range that way) before defaulting to no
filter.
- time_range = form_data.get("time_range")
- if not time_range and (form_data.get("since") or form_data.get("until")):
- time_range = f"{form_data.get('since') or ''} :
{form_data.get('until') or ''}"
- time_range = time_range or "No filter"
+def _base_query_object( # noqa: C901
+ form_data: dict[str, Any],
+ *,
+ row_limit: int | None,
+ order_desc: bool | None,
+ filters_prepared: bool,
+) -> dict[str, Any]:
+ """Build the shared frontend-equivalent portion of a QueryObject."""
+ columns, metrics, orderby = query_fields_from_form_data(form_data)
query: dict[str, Any] = {
"columns": columns,
"metrics": metrics,
- "orderby": orderby_from_form_data(form_data, metrics, viz_type),
- "filters": filters,
- "time_range": time_range,
}
+ if orderby:
+ query["orderby"] = orderby
+
+ if filters_prepared:
+ query["filters"] = list(form_data.get("filters") or [])
+ for clause in ("where", "having"):
+ if form_data.get(clause):
+ query[clause] = form_data[clause]
+ if form_data.get("extras"):
+ query["extras"] = dict(form_data["extras"])
+ else:
+ filters = adhoc_filters_to_query_filters(
+ form_data.get("adhoc_filters", []), where_only=True
+ )
+ filters.extend(
+ filter_
+ for filter_ in form_data.get("filters") or []
+ if isinstance(filter_, dict) and filter_.get("col") is not None
+ )
+ query["filters"] = filters
+ if extras := freeform_where_having(form_data):
+ query["extras"] = extras
+
+ extras = dict(query.get("extras") or {})
+ if form_data.get("time_grain_sqla") is not None:
+ extras["time_grain_sqla"] = form_data["time_grain_sqla"]
if extras:
query["extras"] = extras
- if viz_type == "pie" and (
- post_processing := _pie_contribution_post_processing(metrics)
+
+ effective_limit = row_limit if row_limit is not None else
form_data.get("row_limit")
+ if effective_limit is not None:
+ query["row_limit"] = effective_limit
+ if form_data.get("row_offset") is not None:
+ query["row_offset"] = form_data["row_offset"]
+ if order_desc is not None:
+ query["order_desc"] = order_desc
+ elif "order_desc" in form_data and form_data["order_desc"] is not None:
+ query["order_desc"] = form_data["order_desc"]
+
+ time_range = form_data.get("time_range")
+ if not time_range and (form_data.get("since") or form_data.get("until")):
+ time_range = f"{form_data.get('since') or ''} :
{form_data.get('until') or ''}"
+ if time_range:
+ query["time_range"] = time_range
+ for key in ("since", "until", "annotation_layers", "url_params",
"custom_params"):
+ if form_data.get(key) is not None:
+ query[key] = form_data[key]
+
+ granularity = form_data.get("granularity") or
form_data.get("granularity_sqla")
+ if granularity:
+ query["granularity"] = granularity
+ series_limit = form_data.get("series_limit", form_data.get("limit"))
+ if series_limit is not None:
+ query["series_limit"] = series_limit
+ series_limit_metric = form_data.get("series_limit_metric")
+ if series_limit_metric is None:
+ series_limit_metric = form_data.get("timeseries_limit_metric")
+ if series_limit_metric is not None:
+ query["series_limit_metric"] = series_limit_metric
+ if form_data.get("group_others_when_limit_reached") is not None:
+ query["group_others_when_limit_reached"] = form_data[
+ "group_others_when_limit_reached"
+ ]
+ return query
+
+
+def _temporalized_columns(form_data: dict[str, Any], columns: list[Any]) ->
list[Any]:
+ """Apply the pivot BASE_AXIS temporal-column contract."""
+ time_grain = form_data.get("time_grain_sqla")
+ temporal_lookup = form_data.get("temporal_columns_lookup") or {}
+ result: list[Any] = []
+ for column in columns:
+ if (
+ isinstance(column, str)
+ and time_grain
+ and (
+ temporal_lookup.get(column)
+ or form_data.get("granularity_sqla") == column
+ )
+ ):
+ result.append(
+ {
+ "timeGrain": time_grain,
+ "columnType": "BASE_AXIS",
+ "sqlExpression": column,
+ "label": column,
+ "expressionType": "SQL",
+ }
+ )
+ else:
+ result.append(column)
+ return result
+
+
+def _box_temporalized_columns(
+ form_data: dict[str, Any], columns: list[Any]
+) -> list[Any]:
+ """Convert only physical columns confirmed temporal by Box Plot
metadata."""
+ time_grain = form_data.get("time_grain_sqla")
+ temporal_lookup = form_data.get("temporal_columns_lookup")
+ if not time_grain or not isinstance(temporal_lookup, Mapping):
+ return columns
+ return [
+ {
+ "timeGrain": time_grain,
+ "columnType": "BASE_AXIS",
+ "sqlExpression": column,
+ "label": column,
+ "expressionType": "SQL",
+ }
+ if isinstance(column, str) and temporal_lookup.get(column) is True
+ else column
+ for column in columns
+ ]
+
+
+def _table_temporalized_columns(
+ form_data: dict[str, Any], columns: list[Any]
+) -> list[Any]:
+ """Promote the first temporal table group-by to the frontend BASE_AXIS.
+
+ Table's builder treats only physical columns named in
+ ``temporal_columns_lookup`` as temporal and moves the first match to the
+ front. Later temporal dimensions remain ordinary group-bys.
+ """
+ time_grain = form_data.get("time_grain_sqla")
+ temporal_lookup = form_data.get("temporal_columns_lookup") or {}
+ if not time_grain or not isinstance(temporal_lookup, Mapping):
+ return columns
+
+ temporal_column: dict[str, Any] | None = None
+ remaining: list[Any] = []
+ for column in columns:
+ if (
+ temporal_column is None
+ and isinstance(column, str)
+ and temporal_lookup.get(column)
+ ):
+ temporal_column = {
+ "timeGrain": time_grain,
+ "columnType": "BASE_AXIS",
+ "sqlExpression": column,
+ "label": column,
+ "expressionType": "SQL",
+ }
+ else:
+ remaining.append(column)
+ return [temporal_column, *remaining] if temporal_column else columns
+
+
+def _histogram_query(form_data: dict[str, Any], query: dict[str, Any]) -> None:
+ groupby = _as_list(form_data.get("groupby"))
+ column = form_data.get("column")
+ query["columns"] = [*groupby, *([column] if column is not None else [])]
+ query["post_processing"] = [
+ {
+ "operation": "histogram",
+ "options": {
+ "column": _label(column),
+ "groupby": [_label(value) for value in groupby],
+ "bins": int(form_data.get("bins", 5)),
+ "cumulative": form_data.get("cumulative", False),
+ "normalize": form_data.get("normalize", False),
+ },
+ }
+ ]
+ if any(
+ isinstance(filter_, dict) and filter_.get("clause") == "HAVING"
+ for filter_ in form_data.get("adhoc_filters") or []
+ ):
+ query["metrics"] = [
+ {
+ "expressionType": "SQL",
+ "sqlExpression": "COUNT(*)",
+ "label": "COUNT(*)",
+ }
+ ]
+ else:
+ query["metrics"] = []
+
+
+def _box_plot_query(form_data: dict[str, Any], query: dict[str, Any]) -> None:
+ distributed = _as_list(form_data.get("columns"))
+ if not distributed and form_data.get("granularity_sqla"):
+ distributed = [form_data["granularity_sqla"]]
+ groupby = _as_list(form_data.get("groupby"))
+ query["columns"] = [*_box_temporalized_columns(form_data, distributed),
*groupby]
+ query["series_columns"] = groupby
+ whisker = form_data.get("whiskerOptions")
+ if not whisker:
+ query["post_processing"] = []
+ return
+ whisker_type = "tukey"
+ percentiles: list[int] | None = None
+ if whisker == "Min/max (no outliers)":
+ whisker_type = "min/max"
+ elif isinstance(whisker, str) and whisker.endswith(" percentiles"):
+ low, high = whisker.removesuffix(" percentiles").split("/", 1)
+ whisker_type = "percentile"
+ percentiles = [int(low), int(high)]
+ query["post_processing"] = [
+ {
+ "operation": "boxplot",
+ "options": {
+ "whisker_type": whisker_type,
+ "percentiles": percentiles,
+ "groupby": [_label(value) for value in groupby],
+ "metrics": [_label(value, metric=True) for value in
query["metrics"]],
+ },
+ }
+ ]
+
+
+_PIVOT_ADDITIVE_AGGREGATES = frozenset({"SUM", "COUNT", "MIN", "MAX"})
+
+
+def _all_metrics_additive(metrics: list[Any]) -> bool:
+ """Mirror Pivot's conservative additive-metric fast-path."""
+ return bool(metrics) and all(
+ isinstance(metric, Mapping)
+ and metric.get("expressionType") == "SIMPLE"
+ and metric.get("aggregate") in _PIVOT_ADDITIVE_AGGREGATES
+ for metric in metrics
+ )
+
+
+def _pivot_grouping_sets(
+ form_data: dict[str, Any], rows: list[Any], columns: list[Any]
+) -> list[list[str]]:
+ """Enumerate the rollup levels requested by Pivot's frontend builder."""
+ row_prefixes = [[], *(rows[: index + 1] for index in range(len(rows)))]
+ column_prefixes = [
+ [],
+ *(columns[: index + 1] for index in range(len(columns))),
+ ]
+ show_values_as = form_data.get("showValuesAs")
+ needs_rows_collapsed = show_values_as in {"percent_col", "percent_total"}
+ needs_columns_collapsed = show_values_as in {"percent_row",
"percent_total"}
+
+ def row_prefix_needed(prefix: list[Any]) -> bool:
+ if len(prefix) == len(rows):
+ return True
+ if not prefix:
+ return bool(form_data.get("colTotals")) or needs_rows_collapsed
+ return bool(form_data.get("rowSubTotals"))
+
+ def column_prefix_needed(prefix: list[Any]) -> bool:
+ if len(prefix) == len(columns):
+ return True
+ if not prefix:
+ return bool(form_data.get("rowTotals")) or needs_columns_collapsed
+ return bool(form_data.get("colSubTotals"))
+
+ levels = [
+ (row_prefix, column_prefix)
+ for row_prefix in row_prefixes
+ if row_prefix_needed(row_prefix)
+ for column_prefix in column_prefixes
+ if column_prefix_needed(column_prefix)
+ ]
+ if form_data.get("combineMetric"):
+ metrics_layout = form_data.get("metricsLayout")
+
+ def forced_denominator(level: tuple[list[Any], list[Any]]) -> bool:
+ row_prefix, column_prefix = level
+ return (needs_rows_collapsed and not row_prefix) or (
+ needs_columns_collapsed and not column_prefix
+ )
+
+ if metrics_layout == "ROWS":
+ levels = [
+ level
+ for level in levels
+ if len(level[0]) == len(rows) or forced_denominator(level)
+ ]
+ else:
+ levels = [
+ level
+ for level in levels
+ if len(level[1]) == len(columns) or forced_denominator(level)
+ ]
+
+ return [
+ [_label(value) for value in _deduplicate_fields([*row_prefix,
*column_prefix])]
+ for row_prefix, column_prefix in levels
+ ]
+
+
+def _pivot_query(form_data: dict[str, Any], query: dict[str, Any]) -> None:
+ rows = _as_list(form_data.get("groupbyRows"))
+ columns = _as_list(form_data.get("groupbyColumns"))
+ if form_data.get("transposePivot"):
+ rows, columns = columns, rows
+ query["columns"] = _temporalized_columns(
+ form_data, _deduplicate_fields([*rows, *columns])
+ )
+ metric = query.get("series_limit_metric") or next(
+ iter(query.get("metrics") or []), None
+ )
+ query["orderby"] = (
+ [[metric, not bool(query.get("order_desc", True))]]
+ if metric is not None
+ else []
+ )
+ if not _all_metrics_additive(query.get("metrics") or []):
+ query["grouping_sets"] = _pivot_grouping_sets(form_data, rows, columns)
+
+
+def _waterfall_query(form_data: dict[str, Any], query: dict[str, Any]) -> None:
+ x_axis = form_data.get("x_axis") or form_data.get("granularity_sqla")
+ columns = [*_as_list(x_axis), *_as_list(form_data.get("groupby"))]
+ query["columns"] = _deduplicate_fields(columns)
+ query["orderby"] = [[column, True] for column in query["columns"]]
+
+
+def _gantt_query(form_data: dict[str, Any], query: dict[str, Any]) -> None:
+ groupby = _as_list(form_data.get("series"))
+ orderby = query_fields_from_form_data(form_data)[2]
+ columns = [
+ form_data.get("start_time"),
+ form_data.get("end_time"),
+ form_data.get("y_axis"),
+ *groupby,
+ *_as_list(form_data.get("tooltip_columns")),
+ *(entry[0] for entry in orderby if entry),
+ ]
+ query["columns"] = _deduplicate_fields(columns)
+ query["metrics"] = _as_list(form_data.get("tooltip_metrics"))
+ query["orderby"] = orderby
+ query["series_columns"] = groupby
+
+
+def _normalize_query_orderby(query: dict[str, Any]) -> None:
+ """Mirror ``normalizeOrderBy`` while retaining limit-direction controls."""
+ orderby = query.get("orderby")
+ if (
+ isinstance(orderby, list)
+ and orderby
+ and isinstance(orderby[0], (list, tuple))
+ and len(orderby[0]) == 2
+ and orderby[0][0]
+ and isinstance(orderby[0][1], bool)
):
+ return
+ metric = (
+ query.get("series_limit_metric")
+ or query.get("legacy_order_by")
+ or next(iter(query.get("metrics") or []), None)
+ )
+ if metric is None:
+ query.pop("orderby", None)
+ return
+ query["orderby"] = [[metric, not bool(query.get("order_desc", True))]]
+
+
+_TIME_COMPARISON_TYPES = frozenset({"values", "difference", "percentage",
"ratio"})
+
+
+def _metric_offset_map(
+ form_data: dict[str, Any],
+ metric_labels: list[str],
+ offsets: list[Any] | None = None,
+) -> dict[str, str]:
+ """Return the frontend time-comparison metric label map."""
+ if form_data.get("comparison_type") not in _TIME_COMPARISON_TYPES:
+ return {}
+ return {
+ f"{metric}__{offset}": metric
+ for metric in metric_labels
+ for offset in (
+ offsets if offsets is not None else
_as_list(form_data.get("time_compare"))
+ )
+ }
+
+
+def _table_time_offsets(form_data: dict[str, Any]) -> list[Any]:
+ """Resolve Table custom/inherited shifts like its frontend query
adapter."""
+ raw_offsets = _as_list(form_data.get("time_compare"))
+ offsets = [offset for offset in raw_offsets if offset not in {"custom",
"inherit"}]
+ if "custom" in raw_offsets and form_data.get("start_date_offset") is not
None:
+ offsets.append(form_data["start_date_offset"])
+ extra_form_data = form_data.get("extra_form_data")
+ if isinstance(extra_form_data, Mapping) and
extra_form_data.get("time_compare"):
+ inherited = extra_form_data["time_compare"]
+ if inherited not in offsets:
+ offsets = [inherited]
+ return offsets
+
+
+def _x_axis_column(form_data: Mapping[str, Any]) -> Any | None:
+ """Return a supported x-axis column, excluding legacy granularity.
+
+ ``column_name`` mappings are retained for old server/native payloads. Big
+ Number uses the stricter frontend predicate below.
+ """
+ x_axis = form_data.get("x_axis")
+ if isinstance(x_axis, str):
+ return x_axis if x_axis else None
+ if isinstance(x_axis, Mapping):
+ if isinstance(column_name := x_axis.get("column_name"), str) and
column_name:
+ return column_name
+ # Frontend SQL adhoc columns remain objects in the QueryObject.
+ return x_axis if x_axis else None
+ return None
+
+
+def _frontend_x_axis_column(form_data: Mapping[str, Any]) -> Any | None:
+ """Mirror ``isQueryFormColumn`` for physical and SQL adhoc columns."""
+ x_axis = form_data.get("x_axis")
+ if isinstance(x_axis, str):
+ return x_axis if x_axis else None
+ if (
+ isinstance(x_axis, Mapping)
+ and "sqlExpression" in x_axis
+ and "label" in x_axis
+ and x_axis.get("expressionType") in {None, "SQL"}
+ ):
+ return x_axis
+ return None
+
+
+def normalize_time_column(
+ form_data: Mapping[str, Any], query: dict[str, Any]
+) -> dict[str, Any]:
+ """Apply the final shared frontend ``normalizeTimeColumn`` mutator."""
+ x_axis = _frontend_x_axis_column(form_data)
+ columns = query.get("columns")
+ if x_axis is None or not isinstance(columns, list):
+ return query
+
+ axis_index: int | None = None
+ for index, column in enumerate(columns):
+ if isinstance(x_axis, str) and isinstance(column, str) and column ==
x_axis:
+ axis_index = index
+ break
+ if (
+ isinstance(x_axis, Mapping)
+ and isinstance(column, Mapping)
+ and column.get("sqlExpression") == x_axis.get("sqlExpression")
+ ):
+ axis_index = index
+ break
+ if axis_index is None:
+ return query
+
+ normalized = dict(query)
+ normalized_columns = list(columns)
+ grain = (query.get("extras") or {}).get("time_grain_sqla")
+ if isinstance(columns[axis_index], Mapping):
+ normalized_axis = {
+ "columnType": "BASE_AXIS",
+ **({"timeGrain": grain} if grain is not None else {}),
+ **columns[axis_index],
+ }
+ else:
+ normalized_axis = {
+ "columnType": "BASE_AXIS",
+ "sqlExpression": x_axis,
+ "label": x_axis,
+ "expressionType": "SQL",
+ "isColumnReference": True,
+ **({"timeGrain": grain} if grain is not None else {}),
+ }
+ normalized_columns[axis_index] = normalized_axis
+ normalized["columns"] = normalized_columns
+ normalized.pop("is_timeseries", None)
+ return normalized
+
+
+def _finalize_query_objects(
+ form_data: Mapping[str, Any], queries: list[dict[str, Any]]
+) -> list[dict[str, Any]]:
+ """Run shared query-context mutators after every visualization adapter."""
+ return [normalize_time_column(form_data, query) for query in queries]
+
+
+def _x_axis_label(
+ form_data: Mapping[str, Any], *, frontend_strict: bool = False
+) -> str | None:
+ """Mirror getXAxisColumn/getXAxisLabel for explicit and legacy axes."""
+ explicit = (
+ _frontend_x_axis_column(form_data)
+ if frontend_strict
+ else _x_axis_column(form_data)
+ )
+ if explicit:
+ return _label(explicit)
+ if form_data.get("granularity_sqla"):
+ return DTTM_ALIAS
+ return None
+
+
+def _rename_operator(
+ form_data: dict[str, Any],
+ query: dict[str, Any],
+ *,
+ x_axis_label: str | None,
+) -> dict[str, Any] | None:
+ """Mirror the ECharts ``renameOperator`` for Timeseries and Mixed
charts."""
+ metrics = list(query.get("metrics") or [])
+ metric_labels = [_label(metric, metric=True) for metric in metrics]
+ series_columns = query.get("series_columns")
+ columns = _as_list(
+ series_columns if series_columns is not None else query.get("columns")
+ )
+ time_offsets = _as_list(form_data.get("time_compare"))
+ offset_map = _metric_offset_map(form_data, metric_labels)
+ is_time_comparison = bool(offset_map)
+ truncate_metric = form_data.get("truncate_metric")
+
+ should_rename = (
+ bool(metrics)
+ and bool(x_axis_label)
+ and (
+ is_time_comparison
+ or (
+ (bool(columns) or len(time_offsets) > 1)
+ and "truncate_metric" in form_data
+ and bool(truncate_metric)
+ )
+ )
+ )
+ if not should_rename:
+ return None
+
+ renamed: dict[str, str | None] = {}
+ comparison_type = form_data.get("comparison_type")
+ if is_time_comparison:
+ for metric_with_offset, metric_only in offset_map.items():
+ offset_label = next(
+ (
+ str(offset)
+ for offset in time_offsets
+ if metric_with_offset.endswith(f"__{offset}")
+ ),
+ None,
+ )
+ source = (
+ metric_with_offset
+ if comparison_type == "values"
+ else f"{comparison_type}__{metric_only}__{metric_with_offset}"
+ )
+ renamed[source] = (
+ f"{metric_only}, {offset_label}" if len(metrics) > 1 else
offset_label
+ )
+
+ if (
+ comparison_type not in {"difference", "percentage", "ratio"}
+ and len(metrics) == 1
+ and not renamed
+ ):
+ renamed[metric_labels[0]] = None
+ if not renamed:
+ return None
+ return {
+ "operation": "rename",
+ "options": {"columns": renamed, "level": 0, "inplace": True},
+ }
+
+
+def _timeseries_post_processing( # noqa: C901
+ form_data: dict[str, Any],
+ query: dict[str, Any],
+ *,
+ x_axis_label: str | None,
+ groupby: list[Any],
+ mixed: bool,
+) -> tuple[list[dict[str, Any]], list[Any]]:
+ """Build the Timeseries/Mixed operator pipeline in frontend order."""
+ metric_labels = [_label(value, metric=True) for value in
query.get("metrics") or []]
+ offset_map = _metric_offset_map(form_data, metric_labels)
+ time_offsets = _as_list(form_data.get("time_compare")) if offset_map else
[]
+ post_processing: list[dict[str, Any]] = []
+
+ if x_axis_label and metric_labels:
+ aggregate_labels = (
+ [*offset_map.values(), *offset_map] if offset_map else
metric_labels
+ )
+ post_processing.append(
+ {
+ "operation": "pivot",
+ "options": {
+ "index": [x_axis_label],
+ "columns": [_label(value) for value in groupby],
+ "aggregates": {
+ label: {"operator": "mean"} for label in
aggregate_labels
+ },
+ "drop_missing_columns": not form_data.get(
+ "show_empty_columns", False
+ ),
+ },
+ }
+ )
+
+ if form_data.get("resample_method") and form_data.get("resample_rule"):
+ zero_fill = form_data["resample_method"] == "zerofill"
+ post_processing.append(
+ {
+ "operation": "resample",
+ "options": {
+ "method": "asfreq" if zero_fill else
form_data["resample_method"],
+ "rule": form_data["resample_rule"],
+ "fill_value": 0 if zero_fill else None,
+ },
+ }
+ )
+
+ rolling_labels = (
+ [*offset_map.values(), *offset_map] if offset_map else metric_labels
+ )
+ columns_map = {label: label for label in rolling_labels}
+ rolling_type = form_data.get("rolling_type")
+ if rolling_type == "cumsum":
+ post_processing.append(
+ {
+ "operation": "cum",
+ "options": {"operator": "sum", "columns": columns_map},
+ }
+ )
+ elif rolling_type in {"sum", "mean", "std"}:
+ post_processing.append(
+ {
+ "operation": "rolling",
+ "options": {
+ "rolling_type": rolling_type,
+ "window": int(form_data.get("rolling_periods") or 1),
+ "min_periods": int(form_data.get("min_periods") or 0),
+ "columns": columns_map,
+ },
+ }
+ )
+
+ comparison_type = form_data.get("comparison_type")
+ if offset_map and comparison_type != "values":
+ post_processing.append(
+ {
+ "operation": "compare",
+ "options": {
+ "source_columns": list(offset_map.values()),
+ "compare_columns": list(offset_map),
+ "compare_type": comparison_type,
+ "drop_original_columns": True,
+ },
+ }
+ )
+
+ if not mixed and form_data.get("contributionMode"):
+ post_processing.append(
+ {
+ "operation": "contribution",
+ "options": {
+ "orientation": form_data["contributionMode"],
+ "time_shifts": time_offsets,
+ },
+ }
+ )
+
+ if rename := _rename_operator(form_data, query, x_axis_label=x_axis_label):
+ post_processing.append(rename)
+
+ if not mixed:
+ sortable = {
+ x_axis_label or "",
+ *metric_labels,
+ }
+ if (
+ "x_axis_sort" in form_data
+ and "x_axis_sort_asc" in form_data
+ and form_data.get("x_axis_sort") in sortable
+ and not groupby
+ ):
+ options: dict[str, Any] = {"ascending":
form_data.get("x_axis_sort_asc")}
+ if form_data.get("x_axis_sort") == x_axis_label:
+ options["is_sort_index"] = True
+ else:
+ options["by"] = form_data.get("x_axis_sort")
+ post_processing.append({"operation": "sort", "options": options})
+
+ post_processing.append({"operation": "flatten"})
+ if not mixed and form_data.get("forecastEnabled") and x_axis_label:
+ post_processing.append(
+ {
+ "operation": "prophet",
+ "options": {
+ "time_grain": form_data.get("time_grain_sqla"),
Review Comment:
This takes the Prophet `time_grain` only from `time_grain_sqla`. A saved
line chart with `forecastEnabled=True` whose adhoc x-axis carries
`timeGrain="P1M"` and no panel-level grain gets `time_grain=None`, so the
forecast step fails with "Time grain missing" where Explore uses the axis
grain. Should this fall back to the x-axis grain, and then to the frontend's
daily default, with a forecast test?
##########
superset/common/form_data_query_context.py:
##########
@@ -288,71 +461,1274 @@ def _pie_contribution_post_processing(metrics:
list[Any]) -> list[dict[str, Any]
]
-def build_query_context_from_form_data(
- form_data: dict[str, Any],
- datasource: dict[str, Any],
- viz_type: str | None = None,
+def _as_list(value: Any) -> list[Any]:
+ """Return the frontend ``ensureIsArray`` representation of a value."""
+ if value is None:
+ return []
+ return list(value) if isinstance(value, (list, tuple)) else [value]
+
+
+def _label(value: Any, *, metric: bool = False) -> str:
+ """Resolve a frontend-compatible query-field label."""
+ try:
+ return get_metric_name(value) if metric else get_column_name(value)
+ except (AttributeError, KeyError, TypeError, ValueError):
+ if isinstance(value, Mapping):
+ return str(
+ value.get("label")
+ or value.get("column_name")
+ or value.get("sqlExpression")
+ or value
+ )
+ return str(value)
+
+
+def _deduplicate_fields(values: list[Any], *, metric: bool = False) ->
list[Any]:
+ """Deduplicate query fields by their frontend-visible label."""
+ result: list[Any] = []
+ labels: set[str] = set()
+ for value in values:
+ if value is None or value == "":
+ continue
+ label = _label(value, metric=metric)
+ if label in labels:
+ continue
+ labels.add(label)
+ result.append(value)
+ return result
+
+
+def retain_mixed_timeseries_secondary_form_data(
+ form_data: Mapping[str, Any],
) -> dict[str, Any]:
- """
- Build a query-context payload (the JSON shape
``ChartDataQueryContextSchema``
- loads) from a chart's form data and datasource reference.
+ """Mirror ``retainFormDataSuffix(formData, '_b')`` exactly.
- :param form_data: The chart's saved ``params`` parsed to a dict.
- :param datasource: ``{"id": <int>, "type": "table"}`` datasource reference.
- :param viz_type: The chart's viz type, used for viz-specific handling.
- :returns: A single-query query-context dict.
+ Suffixed values are installed first, including falsey values, and shared
+ unsuffixed controls fill only keys that query B did not explicitly set.
"""
- columns, metrics = _columns_and_metrics(form_data, viz_type)
-
- # SIMPLE adhoc filters (+ legacy top-level ``filters``) become query
filters;
- # free-form SQL predicates go into ``extras``. Only ``WHERE``-clause SIMPLE
- # filters are applied (matching the chart), so the export never filters on
a
- # ``HAVING`` clause the chart itself ignores.
- filters = adhoc_filters_to_query_filters(
- form_data.get("adhoc_filters", []), where_only=True
- )
- for flt in form_data.get("filters") or []:
- if isinstance(flt, dict) and flt.get("col") is not None:
- filters.append(flt)
+ secondary: dict[str, Any] = {}
+ for key, value in form_data.items():
+ if key.endswith("_b"):
+ secondary[key[:-2]] = value
+ for key, value in form_data.items():
+ if not key.endswith("_b") and key not in secondary:
+ secondary[key] = value
+ secondary_filter_keys = {
+ "adhoc_filters": "adhoc_filters_b",
+ "extra_filters": "extra_filters_b",
+ "filters": "filters_b",
+ "having": "having_b",
+ "where": "where_b",
+ }
+ if any(suffixed in form_data for suffixed in
secondary_filter_keys.values()):
+ # The frontend exposes adhoc_filters_b, while saved/server payloads can
+ # carry equivalent legacy aliases. Treat the family atomically: an
+ # explicit clear in any B alias must not be repopulated by query A's
+ # differently named filter representation.
+ for primary, suffixed in secondary_filter_keys.items():
+ if suffixed not in form_data:
+ secondary.pop(primary, None)
+ return secondary
- extras = freeform_where_having(form_data)
- if form_data.get("time_grain_sqla"):
- extras["time_grain_sqla"] = form_data["time_grain_sqla"]
- # Prefer the modern ``time_range``; fall back to the legacy
``since``/``until``
- # pair (older charts store the range that way) before defaulting to no
filter.
- time_range = form_data.get("time_range")
- if not time_range and (form_data.get("since") or form_data.get("until")):
- time_range = f"{form_data.get('since') or ''} :
{form_data.get('until') or ''}"
- time_range = time_range or "No filter"
+def _base_query_object( # noqa: C901
+ form_data: dict[str, Any],
+ *,
+ row_limit: int | None,
+ order_desc: bool | None,
+ filters_prepared: bool,
+) -> dict[str, Any]:
+ """Build the shared frontend-equivalent portion of a QueryObject."""
+ columns, metrics, orderby = query_fields_from_form_data(form_data)
query: dict[str, Any] = {
"columns": columns,
"metrics": metrics,
- "orderby": orderby_from_form_data(form_data, metrics, viz_type),
- "filters": filters,
- "time_range": time_range,
}
+ if orderby:
+ query["orderby"] = orderby
+
+ if filters_prepared:
+ query["filters"] = list(form_data.get("filters") or [])
+ for clause in ("where", "having"):
+ if form_data.get(clause):
+ query[clause] = form_data[clause]
+ if form_data.get("extras"):
+ query["extras"] = dict(form_data["extras"])
+ else:
+ filters = adhoc_filters_to_query_filters(
+ form_data.get("adhoc_filters", []), where_only=True
+ )
+ filters.extend(
+ filter_
+ for filter_ in form_data.get("filters") or []
+ if isinstance(filter_, dict) and filter_.get("col") is not None
+ )
+ query["filters"] = filters
+ if extras := freeform_where_having(form_data):
+ query["extras"] = extras
+
+ extras = dict(query.get("extras") or {})
+ if form_data.get("time_grain_sqla") is not None:
+ extras["time_grain_sqla"] = form_data["time_grain_sqla"]
if extras:
query["extras"] = extras
- if viz_type == "pie" and (
- post_processing := _pie_contribution_post_processing(metrics)
+
+ effective_limit = row_limit if row_limit is not None else
form_data.get("row_limit")
+ if effective_limit is not None:
+ query["row_limit"] = effective_limit
+ if form_data.get("row_offset") is not None:
+ query["row_offset"] = form_data["row_offset"]
+ if order_desc is not None:
+ query["order_desc"] = order_desc
+ elif "order_desc" in form_data and form_data["order_desc"] is not None:
+ query["order_desc"] = form_data["order_desc"]
+
+ time_range = form_data.get("time_range")
+ if not time_range and (form_data.get("since") or form_data.get("until")):
+ time_range = f"{form_data.get('since') or ''} :
{form_data.get('until') or ''}"
+ if time_range:
+ query["time_range"] = time_range
+ for key in ("since", "until", "annotation_layers", "url_params",
"custom_params"):
+ if form_data.get(key) is not None:
+ query[key] = form_data[key]
+
+ granularity = form_data.get("granularity") or
form_data.get("granularity_sqla")
+ if granularity:
+ query["granularity"] = granularity
+ series_limit = form_data.get("series_limit", form_data.get("limit"))
+ if series_limit is not None:
+ query["series_limit"] = series_limit
+ series_limit_metric = form_data.get("series_limit_metric")
+ if series_limit_metric is None:
+ series_limit_metric = form_data.get("timeseries_limit_metric")
+ if series_limit_metric is not None:
+ query["series_limit_metric"] = series_limit_metric
+ if form_data.get("group_others_when_limit_reached") is not None:
+ query["group_others_when_limit_reached"] = form_data[
+ "group_others_when_limit_reached"
+ ]
+ return query
+
+
+def _temporalized_columns(form_data: dict[str, Any], columns: list[Any]) ->
list[Any]:
+ """Apply the pivot BASE_AXIS temporal-column contract."""
+ time_grain = form_data.get("time_grain_sqla")
+ temporal_lookup = form_data.get("temporal_columns_lookup") or {}
+ result: list[Any] = []
+ for column in columns:
+ if (
+ isinstance(column, str)
+ and time_grain
+ and (
+ temporal_lookup.get(column)
+ or form_data.get("granularity_sqla") == column
+ )
+ ):
+ result.append(
+ {
+ "timeGrain": time_grain,
+ "columnType": "BASE_AXIS",
+ "sqlExpression": column,
+ "label": column,
+ "expressionType": "SQL",
+ }
+ )
+ else:
+ result.append(column)
+ return result
+
+
+def _box_temporalized_columns(
+ form_data: dict[str, Any], columns: list[Any]
+) -> list[Any]:
+ """Convert only physical columns confirmed temporal by Box Plot
metadata."""
+ time_grain = form_data.get("time_grain_sqla")
+ temporal_lookup = form_data.get("temporal_columns_lookup")
+ if not time_grain or not isinstance(temporal_lookup, Mapping):
+ return columns
+ return [
+ {
+ "timeGrain": time_grain,
+ "columnType": "BASE_AXIS",
+ "sqlExpression": column,
+ "label": column,
+ "expressionType": "SQL",
+ }
+ if isinstance(column, str) and temporal_lookup.get(column) is True
+ else column
+ for column in columns
+ ]
+
+
+def _table_temporalized_columns(
+ form_data: dict[str, Any], columns: list[Any]
+) -> list[Any]:
+ """Promote the first temporal table group-by to the frontend BASE_AXIS.
+
+ Table's builder treats only physical columns named in
+ ``temporal_columns_lookup`` as temporal and moves the first match to the
+ front. Later temporal dimensions remain ordinary group-bys.
+ """
+ time_grain = form_data.get("time_grain_sqla")
+ temporal_lookup = form_data.get("temporal_columns_lookup") or {}
+ if not time_grain or not isinstance(temporal_lookup, Mapping):
+ return columns
+
+ temporal_column: dict[str, Any] | None = None
+ remaining: list[Any] = []
+ for column in columns:
+ if (
+ temporal_column is None
+ and isinstance(column, str)
+ and temporal_lookup.get(column)
+ ):
+ temporal_column = {
+ "timeGrain": time_grain,
+ "columnType": "BASE_AXIS",
Review Comment:
The synthesized `BASE_AXIS` temporal column omits `isColumnReference: True`,
which the native Table query supplies for semantic-layer datasources (the
Waterfall path at line 1007 sets it). The semantic mapper then treats it as an
adhoc dimension and rejects it, so a saved Table grouped by a temporal column
with a grain fails in MCP preview/data fallback even though it works in
Explore. Should this set `isColumnReference` like the other `BASE_AXIS`
builders?
##########
superset/common/form_data_query_context.py:
##########
@@ -288,71 +461,1274 @@ def _pie_contribution_post_processing(metrics:
list[Any]) -> list[dict[str, Any]
]
-def build_query_context_from_form_data(
- form_data: dict[str, Any],
- datasource: dict[str, Any],
- viz_type: str | None = None,
+def _as_list(value: Any) -> list[Any]:
+ """Return the frontend ``ensureIsArray`` representation of a value."""
+ if value is None:
+ return []
+ return list(value) if isinstance(value, (list, tuple)) else [value]
+
+
+def _label(value: Any, *, metric: bool = False) -> str:
+ """Resolve a frontend-compatible query-field label."""
+ try:
+ return get_metric_name(value) if metric else get_column_name(value)
+ except (AttributeError, KeyError, TypeError, ValueError):
+ if isinstance(value, Mapping):
+ return str(
+ value.get("label")
+ or value.get("column_name")
+ or value.get("sqlExpression")
+ or value
+ )
+ return str(value)
+
+
+def _deduplicate_fields(values: list[Any], *, metric: bool = False) ->
list[Any]:
+ """Deduplicate query fields by their frontend-visible label."""
+ result: list[Any] = []
+ labels: set[str] = set()
+ for value in values:
+ if value is None or value == "":
+ continue
+ label = _label(value, metric=metric)
+ if label in labels:
+ continue
+ labels.add(label)
+ result.append(value)
+ return result
+
+
+def retain_mixed_timeseries_secondary_form_data(
+ form_data: Mapping[str, Any],
) -> dict[str, Any]:
- """
- Build a query-context payload (the JSON shape
``ChartDataQueryContextSchema``
- loads) from a chart's form data and datasource reference.
+ """Mirror ``retainFormDataSuffix(formData, '_b')`` exactly.
- :param form_data: The chart's saved ``params`` parsed to a dict.
- :param datasource: ``{"id": <int>, "type": "table"}`` datasource reference.
- :param viz_type: The chart's viz type, used for viz-specific handling.
- :returns: A single-query query-context dict.
+ Suffixed values are installed first, including falsey values, and shared
+ unsuffixed controls fill only keys that query B did not explicitly set.
"""
- columns, metrics = _columns_and_metrics(form_data, viz_type)
-
- # SIMPLE adhoc filters (+ legacy top-level ``filters``) become query
filters;
- # free-form SQL predicates go into ``extras``. Only ``WHERE``-clause SIMPLE
- # filters are applied (matching the chart), so the export never filters on
a
- # ``HAVING`` clause the chart itself ignores.
- filters = adhoc_filters_to_query_filters(
- form_data.get("adhoc_filters", []), where_only=True
- )
- for flt in form_data.get("filters") or []:
- if isinstance(flt, dict) and flt.get("col") is not None:
- filters.append(flt)
+ secondary: dict[str, Any] = {}
+ for key, value in form_data.items():
+ if key.endswith("_b"):
+ secondary[key[:-2]] = value
+ for key, value in form_data.items():
+ if not key.endswith("_b") and key not in secondary:
+ secondary[key] = value
+ secondary_filter_keys = {
+ "adhoc_filters": "adhoc_filters_b",
+ "extra_filters": "extra_filters_b",
+ "filters": "filters_b",
+ "having": "having_b",
+ "where": "where_b",
+ }
+ if any(suffixed in form_data for suffixed in
secondary_filter_keys.values()):
+ # The frontend exposes adhoc_filters_b, while saved/server payloads can
+ # carry equivalent legacy aliases. Treat the family atomically: an
+ # explicit clear in any B alias must not be repopulated by query A's
+ # differently named filter representation.
+ for primary, suffixed in secondary_filter_keys.items():
+ if suffixed not in form_data:
+ secondary.pop(primary, None)
+ return secondary
- extras = freeform_where_having(form_data)
- if form_data.get("time_grain_sqla"):
- extras["time_grain_sqla"] = form_data["time_grain_sqla"]
- # Prefer the modern ``time_range``; fall back to the legacy
``since``/``until``
- # pair (older charts store the range that way) before defaulting to no
filter.
- time_range = form_data.get("time_range")
- if not time_range and (form_data.get("since") or form_data.get("until")):
- time_range = f"{form_data.get('since') or ''} :
{form_data.get('until') or ''}"
- time_range = time_range or "No filter"
+def _base_query_object( # noqa: C901
+ form_data: dict[str, Any],
+ *,
+ row_limit: int | None,
+ order_desc: bool | None,
+ filters_prepared: bool,
+) -> dict[str, Any]:
+ """Build the shared frontend-equivalent portion of a QueryObject."""
+ columns, metrics, orderby = query_fields_from_form_data(form_data)
query: dict[str, Any] = {
"columns": columns,
"metrics": metrics,
- "orderby": orderby_from_form_data(form_data, metrics, viz_type),
- "filters": filters,
- "time_range": time_range,
}
+ if orderby:
+ query["orderby"] = orderby
+
+ if filters_prepared:
+ query["filters"] = list(form_data.get("filters") or [])
+ for clause in ("where", "having"):
+ if form_data.get(clause):
+ query[clause] = form_data[clause]
+ if form_data.get("extras"):
+ query["extras"] = dict(form_data["extras"])
+ else:
+ filters = adhoc_filters_to_query_filters(
+ form_data.get("adhoc_filters", []), where_only=True
+ )
+ filters.extend(
+ filter_
+ for filter_ in form_data.get("filters") or []
+ if isinstance(filter_, dict) and filter_.get("col") is not None
+ )
+ query["filters"] = filters
+ if extras := freeform_where_having(form_data):
+ query["extras"] = extras
+
+ extras = dict(query.get("extras") or {})
+ if form_data.get("time_grain_sqla") is not None:
+ extras["time_grain_sqla"] = form_data["time_grain_sqla"]
if extras:
query["extras"] = extras
- if viz_type == "pie" and (
- post_processing := _pie_contribution_post_processing(metrics)
+
+ effective_limit = row_limit if row_limit is not None else
form_data.get("row_limit")
+ if effective_limit is not None:
+ query["row_limit"] = effective_limit
+ if form_data.get("row_offset") is not None:
+ query["row_offset"] = form_data["row_offset"]
+ if order_desc is not None:
+ query["order_desc"] = order_desc
+ elif "order_desc" in form_data and form_data["order_desc"] is not None:
+ query["order_desc"] = form_data["order_desc"]
+
+ time_range = form_data.get("time_range")
+ if not time_range and (form_data.get("since") or form_data.get("until")):
+ time_range = f"{form_data.get('since') or ''} :
{form_data.get('until') or ''}"
+ if time_range:
+ query["time_range"] = time_range
+ for key in ("since", "until", "annotation_layers", "url_params",
"custom_params"):
+ if form_data.get(key) is not None:
+ query[key] = form_data[key]
+
+ granularity = form_data.get("granularity") or
form_data.get("granularity_sqla")
+ if granularity:
+ query["granularity"] = granularity
+ series_limit = form_data.get("series_limit", form_data.get("limit"))
+ if series_limit is not None:
+ query["series_limit"] = series_limit
+ series_limit_metric = form_data.get("series_limit_metric")
+ if series_limit_metric is None:
+ series_limit_metric = form_data.get("timeseries_limit_metric")
+ if series_limit_metric is not None:
+ query["series_limit_metric"] = series_limit_metric
+ if form_data.get("group_others_when_limit_reached") is not None:
+ query["group_others_when_limit_reached"] = form_data[
+ "group_others_when_limit_reached"
+ ]
+ return query
+
+
+def _temporalized_columns(form_data: dict[str, Any], columns: list[Any]) ->
list[Any]:
+ """Apply the pivot BASE_AXIS temporal-column contract."""
+ time_grain = form_data.get("time_grain_sqla")
+ temporal_lookup = form_data.get("temporal_columns_lookup") or {}
+ result: list[Any] = []
+ for column in columns:
+ if (
+ isinstance(column, str)
+ and time_grain
+ and (
+ temporal_lookup.get(column)
+ or form_data.get("granularity_sqla") == column
+ )
+ ):
+ result.append(
+ {
+ "timeGrain": time_grain,
+ "columnType": "BASE_AXIS",
+ "sqlExpression": column,
+ "label": column,
+ "expressionType": "SQL",
+ }
+ )
+ else:
+ result.append(column)
+ return result
+
+
+def _box_temporalized_columns(
+ form_data: dict[str, Any], columns: list[Any]
+) -> list[Any]:
+ """Convert only physical columns confirmed temporal by Box Plot
metadata."""
+ time_grain = form_data.get("time_grain_sqla")
+ temporal_lookup = form_data.get("temporal_columns_lookup")
+ if not time_grain or not isinstance(temporal_lookup, Mapping):
+ return columns
+ return [
+ {
+ "timeGrain": time_grain,
+ "columnType": "BASE_AXIS",
+ "sqlExpression": column,
+ "label": column,
+ "expressionType": "SQL",
+ }
+ if isinstance(column, str) and temporal_lookup.get(column) is True
+ else column
+ for column in columns
+ ]
+
+
+def _table_temporalized_columns(
+ form_data: dict[str, Any], columns: list[Any]
+) -> list[Any]:
+ """Promote the first temporal table group-by to the frontend BASE_AXIS.
+
+ Table's builder treats only physical columns named in
+ ``temporal_columns_lookup`` as temporal and moves the first match to the
+ front. Later temporal dimensions remain ordinary group-bys.
+ """
+ time_grain = form_data.get("time_grain_sqla")
+ temporal_lookup = form_data.get("temporal_columns_lookup") or {}
+ if not time_grain or not isinstance(temporal_lookup, Mapping):
+ return columns
+
+ temporal_column: dict[str, Any] | None = None
+ remaining: list[Any] = []
+ for column in columns:
+ if (
+ temporal_column is None
+ and isinstance(column, str)
+ and temporal_lookup.get(column)
+ ):
+ temporal_column = {
+ "timeGrain": time_grain,
+ "columnType": "BASE_AXIS",
+ "sqlExpression": column,
+ "label": column,
+ "expressionType": "SQL",
+ }
+ else:
+ remaining.append(column)
+ return [temporal_column, *remaining] if temporal_column else columns
+
+
+def _histogram_query(form_data: dict[str, Any], query: dict[str, Any]) -> None:
+ groupby = _as_list(form_data.get("groupby"))
+ column = form_data.get("column")
+ query["columns"] = [*groupby, *([column] if column is not None else [])]
+ query["post_processing"] = [
+ {
+ "operation": "histogram",
+ "options": {
+ "column": _label(column),
+ "groupby": [_label(value) for value in groupby],
+ "bins": int(form_data.get("bins", 5)),
+ "cumulative": form_data.get("cumulative", False),
+ "normalize": form_data.get("normalize", False),
+ },
+ }
+ ]
+ if any(
+ isinstance(filter_, dict) and filter_.get("clause") == "HAVING"
+ for filter_ in form_data.get("adhoc_filters") or []
+ ):
+ query["metrics"] = [
+ {
+ "expressionType": "SQL",
+ "sqlExpression": "COUNT(*)",
+ "label": "COUNT(*)",
+ }
+ ]
+ else:
+ query["metrics"] = []
+
+
+def _box_plot_query(form_data: dict[str, Any], query: dict[str, Any]) -> None:
+ distributed = _as_list(form_data.get("columns"))
+ if not distributed and form_data.get("granularity_sqla"):
+ distributed = [form_data["granularity_sqla"]]
+ groupby = _as_list(form_data.get("groupby"))
+ query["columns"] = [*_box_temporalized_columns(form_data, distributed),
*groupby]
+ query["series_columns"] = groupby
+ whisker = form_data.get("whiskerOptions")
+ if not whisker:
+ query["post_processing"] = []
+ return
+ whisker_type = "tukey"
+ percentiles: list[int] | None = None
+ if whisker == "Min/max (no outliers)":
+ whisker_type = "min/max"
+ elif isinstance(whisker, str) and whisker.endswith(" percentiles"):
+ low, high = whisker.removesuffix(" percentiles").split("/", 1)
+ whisker_type = "percentile"
+ percentiles = [int(low), int(high)]
+ query["post_processing"] = [
+ {
+ "operation": "boxplot",
+ "options": {
+ "whisker_type": whisker_type,
+ "percentiles": percentiles,
+ "groupby": [_label(value) for value in groupby],
+ "metrics": [_label(value, metric=True) for value in
query["metrics"]],
+ },
+ }
+ ]
+
+
+_PIVOT_ADDITIVE_AGGREGATES = frozenset({"SUM", "COUNT", "MIN", "MAX"})
+
+
+def _all_metrics_additive(metrics: list[Any]) -> bool:
+ """Mirror Pivot's conservative additive-metric fast-path."""
+ return bool(metrics) and all(
+ isinstance(metric, Mapping)
+ and metric.get("expressionType") == "SIMPLE"
+ and metric.get("aggregate") in _PIVOT_ADDITIVE_AGGREGATES
+ for metric in metrics
+ )
+
+
+def _pivot_grouping_sets(
+ form_data: dict[str, Any], rows: list[Any], columns: list[Any]
+) -> list[list[str]]:
+ """Enumerate the rollup levels requested by Pivot's frontend builder."""
+ row_prefixes = [[], *(rows[: index + 1] for index in range(len(rows)))]
+ column_prefixes = [
+ [],
+ *(columns[: index + 1] for index in range(len(columns))),
+ ]
+ show_values_as = form_data.get("showValuesAs")
+ needs_rows_collapsed = show_values_as in {"percent_col", "percent_total"}
+ needs_columns_collapsed = show_values_as in {"percent_row",
"percent_total"}
+
+ def row_prefix_needed(prefix: list[Any]) -> bool:
+ if len(prefix) == len(rows):
+ return True
+ if not prefix:
+ return bool(form_data.get("colTotals")) or needs_rows_collapsed
+ return bool(form_data.get("rowSubTotals"))
+
+ def column_prefix_needed(prefix: list[Any]) -> bool:
+ if len(prefix) == len(columns):
+ return True
+ if not prefix:
+ return bool(form_data.get("rowTotals")) or needs_columns_collapsed
+ return bool(form_data.get("colSubTotals"))
+
+ levels = [
+ (row_prefix, column_prefix)
+ for row_prefix in row_prefixes
+ if row_prefix_needed(row_prefix)
+ for column_prefix in column_prefixes
+ if column_prefix_needed(column_prefix)
+ ]
+ if form_data.get("combineMetric"):
+ metrics_layout = form_data.get("metricsLayout")
+
+ def forced_denominator(level: tuple[list[Any], list[Any]]) -> bool:
+ row_prefix, column_prefix = level
+ return (needs_rows_collapsed and not row_prefix) or (
+ needs_columns_collapsed and not column_prefix
+ )
+
+ if metrics_layout == "ROWS":
+ levels = [
+ level
+ for level in levels
+ if len(level[0]) == len(rows) or forced_denominator(level)
+ ]
+ else:
+ levels = [
+ level
+ for level in levels
+ if len(level[1]) == len(columns) or forced_denominator(level)
+ ]
+
+ return [
+ [_label(value) for value in _deduplicate_fields([*row_prefix,
*column_prefix])]
+ for row_prefix, column_prefix in levels
+ ]
+
+
+def _pivot_query(form_data: dict[str, Any], query: dict[str, Any]) -> None:
+ rows = _as_list(form_data.get("groupbyRows"))
+ columns = _as_list(form_data.get("groupbyColumns"))
+ if form_data.get("transposePivot"):
+ rows, columns = columns, rows
+ query["columns"] = _temporalized_columns(
+ form_data, _deduplicate_fields([*rows, *columns])
+ )
+ metric = query.get("series_limit_metric") or next(
+ iter(query.get("metrics") or []), None
+ )
+ query["orderby"] = (
+ [[metric, not bool(query.get("order_desc", True))]]
+ if metric is not None
+ else []
+ )
+ if not _all_metrics_additive(query.get("metrics") or []):
+ query["grouping_sets"] = _pivot_grouping_sets(form_data, rows, columns)
+
+
+def _waterfall_query(form_data: dict[str, Any], query: dict[str, Any]) -> None:
+ x_axis = form_data.get("x_axis") or form_data.get("granularity_sqla")
+ columns = [*_as_list(x_axis), *_as_list(form_data.get("groupby"))]
+ query["columns"] = _deduplicate_fields(columns)
+ query["orderby"] = [[column, True] for column in query["columns"]]
+
+
+def _gantt_query(form_data: dict[str, Any], query: dict[str, Any]) -> None:
+ groupby = _as_list(form_data.get("series"))
+ orderby = query_fields_from_form_data(form_data)[2]
+ columns = [
+ form_data.get("start_time"),
+ form_data.get("end_time"),
+ form_data.get("y_axis"),
+ *groupby,
+ *_as_list(form_data.get("tooltip_columns")),
+ *(entry[0] for entry in orderby if entry),
+ ]
+ query["columns"] = _deduplicate_fields(columns)
+ query["metrics"] = _as_list(form_data.get("tooltip_metrics"))
+ query["orderby"] = orderby
+ query["series_columns"] = groupby
+
+
+def _normalize_query_orderby(query: dict[str, Any]) -> None:
+ """Mirror ``normalizeOrderBy`` while retaining limit-direction controls."""
+ orderby = query.get("orderby")
+ if (
+ isinstance(orderby, list)
+ and orderby
+ and isinstance(orderby[0], (list, tuple))
+ and len(orderby[0]) == 2
+ and orderby[0][0]
+ and isinstance(orderby[0][1], bool)
):
+ return
+ metric = (
+ query.get("series_limit_metric")
+ or query.get("legacy_order_by")
+ or next(iter(query.get("metrics") or []), None)
+ )
+ if metric is None:
+ query.pop("orderby", None)
+ return
+ query["orderby"] = [[metric, not bool(query.get("order_desc", True))]]
+
+
+_TIME_COMPARISON_TYPES = frozenset({"values", "difference", "percentage",
"ratio"})
+
+
+def _metric_offset_map(
+ form_data: dict[str, Any],
+ metric_labels: list[str],
+ offsets: list[Any] | None = None,
+) -> dict[str, str]:
+ """Return the frontend time-comparison metric label map."""
+ if form_data.get("comparison_type") not in _TIME_COMPARISON_TYPES:
+ return {}
+ return {
+ f"{metric}__{offset}": metric
+ for metric in metric_labels
+ for offset in (
+ offsets if offsets is not None else
_as_list(form_data.get("time_compare"))
+ )
+ }
+
+
+def _table_time_offsets(form_data: dict[str, Any]) -> list[Any]:
+ """Resolve Table custom/inherited shifts like its frontend query
adapter."""
+ raw_offsets = _as_list(form_data.get("time_compare"))
+ offsets = [offset for offset in raw_offsets if offset not in {"custom",
"inherit"}]
+ if "custom" in raw_offsets and form_data.get("start_date_offset") is not
None:
+ offsets.append(form_data["start_date_offset"])
+ extra_form_data = form_data.get("extra_form_data")
+ if isinstance(extra_form_data, Mapping) and
extra_form_data.get("time_compare"):
+ inherited = extra_form_data["time_compare"]
+ if inherited not in offsets:
+ offsets = [inherited]
+ return offsets
+
+
+def _x_axis_column(form_data: Mapping[str, Any]) -> Any | None:
+ """Return a supported x-axis column, excluding legacy granularity.
+
+ ``column_name`` mappings are retained for old server/native payloads. Big
+ Number uses the stricter frontend predicate below.
+ """
+ x_axis = form_data.get("x_axis")
+ if isinstance(x_axis, str):
+ return x_axis if x_axis else None
+ if isinstance(x_axis, Mapping):
+ if isinstance(column_name := x_axis.get("column_name"), str) and
column_name:
+ return column_name
+ # Frontend SQL adhoc columns remain objects in the QueryObject.
+ return x_axis if x_axis else None
+ return None
+
+
+def _frontend_x_axis_column(form_data: Mapping[str, Any]) -> Any | None:
+ """Mirror ``isQueryFormColumn`` for physical and SQL adhoc columns."""
+ x_axis = form_data.get("x_axis")
+ if isinstance(x_axis, str):
+ return x_axis if x_axis else None
+ if (
+ isinstance(x_axis, Mapping)
+ and "sqlExpression" in x_axis
+ and "label" in x_axis
+ and x_axis.get("expressionType") in {None, "SQL"}
+ ):
+ return x_axis
+ return None
+
+
+def normalize_time_column(
+ form_data: Mapping[str, Any], query: dict[str, Any]
+) -> dict[str, Any]:
+ """Apply the final shared frontend ``normalizeTimeColumn`` mutator."""
+ x_axis = _frontend_x_axis_column(form_data)
+ columns = query.get("columns")
+ if x_axis is None or not isinstance(columns, list):
+ return query
+
+ axis_index: int | None = None
+ for index, column in enumerate(columns):
+ if isinstance(x_axis, str) and isinstance(column, str) and column ==
x_axis:
+ axis_index = index
+ break
+ if (
+ isinstance(x_axis, Mapping)
+ and isinstance(column, Mapping)
+ and column.get("sqlExpression") == x_axis.get("sqlExpression")
+ ):
+ axis_index = index
+ break
+ if axis_index is None:
+ return query
+
+ normalized = dict(query)
+ normalized_columns = list(columns)
+ grain = (query.get("extras") or {}).get("time_grain_sqla")
+ if isinstance(columns[axis_index], Mapping):
+ normalized_axis = {
+ "columnType": "BASE_AXIS",
+ **({"timeGrain": grain} if grain is not None else {}),
+ **columns[axis_index],
+ }
+ else:
+ normalized_axis = {
+ "columnType": "BASE_AXIS",
+ "sqlExpression": x_axis,
+ "label": x_axis,
+ "expressionType": "SQL",
+ "isColumnReference": True,
+ **({"timeGrain": grain} if grain is not None else {}),
+ }
+ normalized_columns[axis_index] = normalized_axis
+ normalized["columns"] = normalized_columns
+ normalized.pop("is_timeseries", None)
+ return normalized
+
+
+def _finalize_query_objects(
+ form_data: Mapping[str, Any], queries: list[dict[str, Any]]
+) -> list[dict[str, Any]]:
+ """Run shared query-context mutators after every visualization adapter."""
+ return [normalize_time_column(form_data, query) for query in queries]
+
+
+def _x_axis_label(
+ form_data: Mapping[str, Any], *, frontend_strict: bool = False
+) -> str | None:
+ """Mirror getXAxisColumn/getXAxisLabel for explicit and legacy axes."""
+ explicit = (
+ _frontend_x_axis_column(form_data)
+ if frontend_strict
+ else _x_axis_column(form_data)
+ )
+ if explicit:
+ return _label(explicit)
+ if form_data.get("granularity_sqla"):
+ return DTTM_ALIAS
+ return None
+
+
+def _rename_operator(
+ form_data: dict[str, Any],
+ query: dict[str, Any],
+ *,
+ x_axis_label: str | None,
+) -> dict[str, Any] | None:
+ """Mirror the ECharts ``renameOperator`` for Timeseries and Mixed
charts."""
+ metrics = list(query.get("metrics") or [])
+ metric_labels = [_label(metric, metric=True) for metric in metrics]
+ series_columns = query.get("series_columns")
+ columns = _as_list(
+ series_columns if series_columns is not None else query.get("columns")
+ )
+ time_offsets = _as_list(form_data.get("time_compare"))
+ offset_map = _metric_offset_map(form_data, metric_labels)
+ is_time_comparison = bool(offset_map)
+ truncate_metric = form_data.get("truncate_metric")
+
+ should_rename = (
+ bool(metrics)
+ and bool(x_axis_label)
+ and (
+ is_time_comparison
+ or (
+ (bool(columns) or len(time_offsets) > 1)
+ and "truncate_metric" in form_data
+ and bool(truncate_metric)
+ )
+ )
+ )
+ if not should_rename:
+ return None
+
+ renamed: dict[str, str | None] = {}
+ comparison_type = form_data.get("comparison_type")
+ if is_time_comparison:
+ for metric_with_offset, metric_only in offset_map.items():
+ offset_label = next(
+ (
+ str(offset)
+ for offset in time_offsets
+ if metric_with_offset.endswith(f"__{offset}")
+ ),
+ None,
+ )
+ source = (
+ metric_with_offset
+ if comparison_type == "values"
+ else f"{comparison_type}__{metric_only}__{metric_with_offset}"
+ )
+ renamed[source] = (
+ f"{metric_only}, {offset_label}" if len(metrics) > 1 else
offset_label
+ )
+
+ if (
+ comparison_type not in {"difference", "percentage", "ratio"}
+ and len(metrics) == 1
+ and not renamed
+ ):
+ renamed[metric_labels[0]] = None
+ if not renamed:
+ return None
+ return {
+ "operation": "rename",
+ "options": {"columns": renamed, "level": 0, "inplace": True},
+ }
+
+
+def _timeseries_post_processing( # noqa: C901
+ form_data: dict[str, Any],
+ query: dict[str, Any],
+ *,
+ x_axis_label: str | None,
+ groupby: list[Any],
+ mixed: bool,
+) -> tuple[list[dict[str, Any]], list[Any]]:
+ """Build the Timeseries/Mixed operator pipeline in frontend order."""
+ metric_labels = [_label(value, metric=True) for value in
query.get("metrics") or []]
+ offset_map = _metric_offset_map(form_data, metric_labels)
+ time_offsets = _as_list(form_data.get("time_compare")) if offset_map else
[]
+ post_processing: list[dict[str, Any]] = []
+
+ if x_axis_label and metric_labels:
+ aggregate_labels = (
+ [*offset_map.values(), *offset_map] if offset_map else
metric_labels
+ )
+ post_processing.append(
+ {
+ "operation": "pivot",
Review Comment:
The shared builder now pivots grouped timeseries and flattens them to wide
columns like `revenue, East` / `revenue, West`, but the Vega-Lite consumer
still expects long-form rows (`preview_utils.py` around line 1711 picks one
numeric column and only adds color when `groupby[0]` is a column). For
`x_axis="ds"`, `metrics=["revenue"]`, `groupby=["region"]` the XY preview draws
one region's line and drops the others without any error. Should the preview
adapt to the wide shape (or melt it back) in this PR?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
