aminghadersohi commented on code in PR #44948: URL: https://github.com/apache/superset/pull/44948#discussion_r4187811073
########## superset/mcp_service/chart/big_number_headline.py: ########## @@ -0,0 +1,365 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Headline value of Big Number charts. + +A Big Number chart renders one number. For the trendline variant (``big_number``) +that number is derived client-side from the time series; for ``big_number_total`` +it is the single metric value. Neither is any of the first sample rows, so this +module reproduces the frontend computation: + +- ``aggregationChoices`` in ``superset-ui-chart-controls`` (``customControls.tsx``) +- ``BigNumberWithTrendline/transformProps.ts`` and ``BigNumberTotal/transformProps.ts`` + +A headline is only returned when it is exact. Otherwise ``value`` is null with a +``reason``: a wrong number is worse than none. +""" + +from __future__ import annotations + +import math +import statistics +from collections.abc import Callable, Mapping, Sequence +from datetime import date, datetime, timezone +from decimal import Decimal +from typing import Any, cast + +from superset.mcp_service.chart.schemas import BigNumberHeadline +from superset.superset_typing import Metric +from superset.utils.core import DTTM_ALIAS, get_metric_name + +BIG_NUMBER_TRENDLINE_VIZ_TYPE = "big_number" +BIG_NUMBER_TOTAL_VIZ_TYPE = "big_number_total" + +DEFAULT_AGGREGATION = "LAST_VALUE" +RAW_AGGREGATION = "raw" + +# Metric-value transforms for the trend series. Keys and order mirror the +# frontend's `aggregationChoices`. `LAST_VALUE` and `raw` receive values ordered +# newest first and take the first, so they need no entry beyond that. +_AGGREGATIONS: dict[str, Callable[[list[float]], float | None]] = { + "raw": lambda values: values[0] if values else None, + "LAST_VALUE": lambda values: values[0] if values else None, + "sum": lambda values: sum(values) if values else None, + "mean": lambda values: sum(values) / len(values) if values else None, + "min": lambda values: min(values) if values else None, + "max": lambda values: max(values) if values else None, + "median": lambda values: statistics.median(values) if values else None, +} + +# Rolling types that make the query add a `rolling` (sum, mean, std) or `cum` +# (cumsum) post-processing step, as in the frontend's `rollingWindowOperator`. +_ROLLING_OPERATIONS = { + "cumsum": "cum", + "sum": "rolling", + "mean": "rolling", + "std": "rolling", +} + + +def is_big_number_viz_type(viz_type: str | None) -> bool: + """Whether the visualization displays a Big Number headline.""" + return viz_type in (BIG_NUMBER_TRENDLINE_VIZ_TYPE, BIG_NUMBER_TOTAL_VIZ_TYPE) + + +def executed_query_facts(query_context: Any) -> tuple[int | None, list[str]]: + """Row limit and post-processing operations of the first executed query. + + Read from the query that actually ran, so it reflects any row-limit override + and shows whether the chart's advanced analytics were part of the query. + """ + queries = getattr(query_context, "queries", None) or [] + if not queries: + return None, [] + first = queries[0] + row_limit = getattr(first, "row_limit", None) + operations = [ + str(step.get("operation")) + for step in getattr(first, "post_processing", None) or [] + if isinstance(step, Mapping) and step.get("operation") + ] + return (row_limit if isinstance(row_limit, int) else None), operations + + +def _unavailable(aggregation: str | None, reason: str) -> BigNumberHeadline: + """Return an unavailable headline with its explanation.""" + return BigNumberHeadline(value=None, aggregation=aggregation, reason=reason) + + +def _parse_date_ms(value: str) -> int | None: + """Epoch milliseconds for an ISO-8601 string, else None (frontend: + strict `dayjs.utc` parse in `parseMetricValue`).""" + try: + parsed = datetime.fromisoformat(value) + except ValueError: + return None + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return int(parsed.timestamp() * 1000) + + +def _parse_metric_value(value: Any) -> int | float | None: + """Mirror of the frontend's `parseMetricValue`, plus JSON-safety: numbers + pass through, date strings become epoch ms, anything else (including NaN and + infinities, which serialize as null) is null.""" + if isinstance(value, bool) or value is None: + return None + if isinstance(value, Decimal): + value = float(value) + if isinstance(value, (int, float)): + try: + return value if math.isfinite(value) else None + except OverflowError: + return None + if isinstance(value, datetime): + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + return int(value.timestamp() * 1000) + if isinstance(value, str): + return _parse_date_ms(value) + return None + + +def _timestamp_ms(value: Any) -> float | None: + """Sortable time value for an x-axis cell (the API serializes these as epoch + ms; direct query results may carry datetimes).""" + if isinstance(value, date) and not isinstance(value, datetime): + value = datetime(value.year, value.month, value.day, tzinfo=timezone.utc) + return _parse_metric_value(value) + + +def _metric_label(form_data: Mapping[str, Any]) -> str | None: + """Resolve the metric label using the frontend column-name aliases.""" + metric = form_data.get("metric") + if not metric: + return None + if ( + isinstance(metric, Mapping) + and not metric.get("label") + and metric.get("expressionType") == "SIMPLE" + ): + column = metric.get("column") + if isinstance(column, Mapping) and column.get("columnName"): + return f"{metric.get('aggregate')}({column['columnName']})" + try: + return get_metric_name(cast(Metric, metric)) or None + except (ValueError, TypeError, AttributeError): + return None + + +def _x_axis_labels(form_data: Mapping[str, Any]) -> list[str]: + """Candidate time-column labels: the configured x-axis, then `__timestamp`.""" + labels: list[str] = [] + x_axis = form_data.get("x_axis") + if isinstance(x_axis, str) and x_axis: + labels.append(x_axis) + elif isinstance(x_axis, Mapping): + label = x_axis.get("label") or x_axis.get("column_name") + if isinstance(label, str) and label: + labels.append(label) + labels.append(DTTM_ALIAS) + return labels + + +def _total_headline( + form_data: Mapping[str, Any], rows: Sequence[Mapping[str, Any]] +) -> BigNumberHeadline: + """`big_number_total`: the single metric value of the first row.""" + label = _metric_label(form_data) + if not rows: + return _unavailable("total", "The query returned no rows.") + if label is None or label not in rows[0]: + return _unavailable("total", "The metric column was not in the result.") + raw = rows[0][label] + parsed = _parse_metric_value(raw) + if parsed is None and isinstance(raw, str) and raw.strip(): + return BigNumberHeadline(value=raw, aggregation="total", rows_used=1) + if parsed is None: + return _unavailable("total", "The metric value is null.") + return BigNumberHeadline(value=parsed, aggregation="total", rows_used=1) + + +def _overall_value( + label: str, x_labels: Sequence[str], rows: Sequence[Mapping[str, Any]] +) -> int | float | str | None: + """Value of the `raw` ("Overall value") query layer, as the frontend reads + it: the metric column, else the first other numeric column.""" + row = rows[0] + value = row.get(label) + if value is None: + value = next( + ( + cell + for key, cell in row.items() + if key not in x_labels + and isinstance(cell, (int, float, Decimal)) + and not isinstance(cell, bool) + ), + None, + ) + if isinstance(value, Decimal): + value = float(value) + if isinstance(value, float) and not math.isfinite(value): + return None + return value if isinstance(value, (int, float, str)) else None + + +def _raw_headline( + label: str, + x_labels: Sequence[str], + queries: Sequence[Mapping[str, Any]], +) -> BigNumberHeadline: + """`raw` ("Overall value"): read from the second, un-trended query layer.""" + overall_rows = queries[1].get("data") if len(queries) > 1 else None + if not overall_rows: + return _unavailable( + RAW_AGGREGATION, + "The overall-value (raw) query layer did not return a value.", + ) + value = _overall_value(label, x_labels, overall_rows) + if value is None: + return _unavailable( + RAW_AGGREGATION, "The overall-value (raw) query returned null." + ) + return BigNumberHeadline( + value=value, aggregation=RAW_AGGREGATION, rows_used=len(overall_rows) + ) + + +def _series_problem( + form_data: Mapping[str, Any], + first: Mapping[str, Any], + row_limit: int | None, + post_processing_operations: Sequence[str], +) -> str | None: + """Why the returned trend rows cannot give an exact headline, if they can't.""" + rolling_type = form_data.get("rolling_type") + expected_operation = _ROLLING_OPERATIONS.get(str(rolling_type)) + if expected_operation and expected_operation not in post_processing_operations: Review Comment: Fixed in 1c3f0effc34d53f34f78ef8a6670d2ec20e48fee. Thanks for running it. `_series_problem` has the twin guard now. It mirrors `resampleOperator`: when `resample_rule` and `resample_method` are both set and the executed query has no `resample` step, `value` is null and the reason names the rule and method. If either control is unset, the frontend adds no step, so no guard applies. Tests: `test_resample_not_in_the_rows_has_no_headline` covers zerofill/asfreq/ffill across LAST_VALUE, sum, mean, min and median, with and without a `resample` operation. `test_incomplete_resample_needs_no_post_processing` covers the half-set cases. The tool-level twin `test_headline_is_null_when_resample_is_not_in_the_executed_query` uses your four rows of 70 with mean, 1D and zerofill. These fail on 5a930fd and pass on this commit. On the Overall value note: agreed. The `get_chart_data` description now says `raw` needs the saved query context and has no headline for unsaved state or a `form_data_key`. The new docs section says the same. ########## superset/mcp_service/chart/big_number_headline.py: ########## @@ -0,0 +1,365 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Headline value of Big Number charts. + +A Big Number chart renders one number. For the trendline variant (``big_number``) +that number is derived client-side from the time series; for ``big_number_total`` +it is the single metric value. Neither is any of the first sample rows, so this +module reproduces the frontend computation: + +- ``aggregationChoices`` in ``superset-ui-chart-controls`` (``customControls.tsx``) +- ``BigNumberWithTrendline/transformProps.ts`` and ``BigNumberTotal/transformProps.ts`` + +A headline is only returned when it is exact. Otherwise ``value`` is null with a +``reason``: a wrong number is worse than none. +""" + +from __future__ import annotations + +import math +import statistics +from collections.abc import Callable, Mapping, Sequence +from datetime import date, datetime, timezone +from decimal import Decimal +from typing import Any, cast + +from superset.mcp_service.chart.schemas import BigNumberHeadline +from superset.superset_typing import Metric +from superset.utils.core import DTTM_ALIAS, get_metric_name + +BIG_NUMBER_TRENDLINE_VIZ_TYPE = "big_number" +BIG_NUMBER_TOTAL_VIZ_TYPE = "big_number_total" + +DEFAULT_AGGREGATION = "LAST_VALUE" +RAW_AGGREGATION = "raw" + +# Metric-value transforms for the trend series. Keys and order mirror the +# frontend's `aggregationChoices`. `LAST_VALUE` and `raw` receive values ordered +# newest first and take the first, so they need no entry beyond that. +_AGGREGATIONS: dict[str, Callable[[list[float]], float | None]] = { + "raw": lambda values: values[0] if values else None, + "LAST_VALUE": lambda values: values[0] if values else None, + "sum": lambda values: sum(values) if values else None, + "mean": lambda values: sum(values) / len(values) if values else None, + "min": lambda values: min(values) if values else None, + "max": lambda values: max(values) if values else None, + "median": lambda values: statistics.median(values) if values else None, +} + +# Rolling types that make the query add a `rolling` (sum, mean, std) or `cum` +# (cumsum) post-processing step, as in the frontend's `rollingWindowOperator`. +_ROLLING_OPERATIONS = { + "cumsum": "cum", + "sum": "rolling", + "mean": "rolling", + "std": "rolling", +} + + +def is_big_number_viz_type(viz_type: str | None) -> bool: + """Whether the visualization displays a Big Number headline.""" + return viz_type in (BIG_NUMBER_TRENDLINE_VIZ_TYPE, BIG_NUMBER_TOTAL_VIZ_TYPE) + + +def executed_query_facts(query_context: Any) -> tuple[int | None, list[str]]: + """Row limit and post-processing operations of the first executed query. + + Read from the query that actually ran, so it reflects any row-limit override + and shows whether the chart's advanced analytics were part of the query. + """ + queries = getattr(query_context, "queries", None) or [] + if not queries: + return None, [] + first = queries[0] + row_limit = getattr(first, "row_limit", None) + operations = [ + str(step.get("operation")) + for step in getattr(first, "post_processing", None) or [] + if isinstance(step, Mapping) and step.get("operation") + ] + return (row_limit if isinstance(row_limit, int) else None), operations + + +def _unavailable(aggregation: str | None, reason: str) -> BigNumberHeadline: + """Return an unavailable headline with its explanation.""" + return BigNumberHeadline(value=None, aggregation=aggregation, reason=reason) + + +def _parse_date_ms(value: str) -> int | None: + """Epoch milliseconds for an ISO-8601 string, else None (frontend: + strict `dayjs.utc` parse in `parseMetricValue`).""" + try: + parsed = datetime.fromisoformat(value) + except ValueError: + return None + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return int(parsed.timestamp() * 1000) + + +def _parse_metric_value(value: Any) -> int | float | None: + """Mirror of the frontend's `parseMetricValue`, plus JSON-safety: numbers + pass through, date strings become epoch ms, anything else (including NaN and + infinities, which serialize as null) is null.""" + if isinstance(value, bool) or value is None: + return None + if isinstance(value, Decimal): + value = float(value) + if isinstance(value, (int, float)): + try: + return value if math.isfinite(value) else None + except OverflowError: + return None + if isinstance(value, datetime): + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + return int(value.timestamp() * 1000) + if isinstance(value, str): + return _parse_date_ms(value) + return None + + +def _timestamp_ms(value: Any) -> float | None: + """Sortable time value for an x-axis cell (the API serializes these as epoch + ms; direct query results may carry datetimes).""" + if isinstance(value, date) and not isinstance(value, datetime): + value = datetime(value.year, value.month, value.day, tzinfo=timezone.utc) + return _parse_metric_value(value) + + +def _metric_label(form_data: Mapping[str, Any]) -> str | None: + """Resolve the metric label using the frontend column-name aliases.""" + metric = form_data.get("metric") + if not metric: + return None + if ( + isinstance(metric, Mapping) + and not metric.get("label") + and metric.get("expressionType") == "SIMPLE" + ): + column = metric.get("column") + if isinstance(column, Mapping) and column.get("columnName"): + return f"{metric.get('aggregate')}({column['columnName']})" + try: + return get_metric_name(cast(Metric, metric)) or None + except (ValueError, TypeError, AttributeError): + return None + + +def _x_axis_labels(form_data: Mapping[str, Any]) -> list[str]: + """Candidate time-column labels: the configured x-axis, then `__timestamp`.""" + labels: list[str] = [] + x_axis = form_data.get("x_axis") + if isinstance(x_axis, str) and x_axis: + labels.append(x_axis) + elif isinstance(x_axis, Mapping): + label = x_axis.get("label") or x_axis.get("column_name") + if isinstance(label, str) and label: + labels.append(label) + labels.append(DTTM_ALIAS) + return labels + + +def _total_headline( + form_data: Mapping[str, Any], rows: Sequence[Mapping[str, Any]] +) -> BigNumberHeadline: + """`big_number_total`: the single metric value of the first row.""" + label = _metric_label(form_data) + if not rows: + return _unavailable("total", "The query returned no rows.") + if label is None or label not in rows[0]: + return _unavailable("total", "The metric column was not in the result.") + raw = rows[0][label] + parsed = _parse_metric_value(raw) + if parsed is None and isinstance(raw, str) and raw.strip(): + return BigNumberHeadline(value=raw, aggregation="total", rows_used=1) + if parsed is None: + return _unavailable("total", "The metric value is null.") + return BigNumberHeadline(value=parsed, aggregation="total", rows_used=1) + + +def _overall_value( + label: str, x_labels: Sequence[str], rows: Sequence[Mapping[str, Any]] +) -> int | float | str | None: + """Value of the `raw` ("Overall value") query layer, as the frontend reads + it: the metric column, else the first other numeric column.""" + row = rows[0] + value = row.get(label) + if value is None: + value = next( + ( + cell + for key, cell in row.items() + if key not in x_labels + and isinstance(cell, (int, float, Decimal)) + and not isinstance(cell, bool) + ), + None, + ) + if isinstance(value, Decimal): + value = float(value) + if isinstance(value, float) and not math.isfinite(value): + return None + return value if isinstance(value, (int, float, str)) else None + + +def _raw_headline( + label: str, + x_labels: Sequence[str], + queries: Sequence[Mapping[str, Any]], +) -> BigNumberHeadline: + """`raw` ("Overall value"): read from the second, un-trended query layer.""" + overall_rows = queries[1].get("data") if len(queries) > 1 else None + if not overall_rows: + return _unavailable( + RAW_AGGREGATION, + "The overall-value (raw) query layer did not return a value.", + ) + value = _overall_value(label, x_labels, overall_rows) + if value is None: + return _unavailable( + RAW_AGGREGATION, "The overall-value (raw) query returned null." + ) + return BigNumberHeadline( + value=value, aggregation=RAW_AGGREGATION, rows_used=len(overall_rows) + ) + + +def _series_problem( + form_data: Mapping[str, Any], + first: Mapping[str, Any], + row_limit: int | None, + post_processing_operations: Sequence[str], +) -> str | None: + """Why the returned trend rows cannot give an exact headline, if they can't.""" + rolling_type = form_data.get("rolling_type") + expected_operation = _ROLLING_OPERATIONS.get(str(rolling_type)) + if expected_operation and expected_operation not in post_processing_operations: + return ( + f"The chart uses a rolling window ({rolling_type}) that the returned " + "rows do not reflect." + ) + rows = first.get("data") or [] + returned_total = first.get("rowcount") + if (row_limit and len(rows) >= row_limit) or ( + isinstance(returned_total, int) and returned_total > len(rows) + ): + return ( + "The fetch was truncated at the row limit, so the series is " + "incomplete; the headline cannot be computed exactly." + ) + return None + + +def _trend_values( + label: str, + x_labels: Sequence[str], + rows: Sequence[Mapping[str, Any]], + *, + newest_first: bool, +) -> list[int | float]: + """Non-null metrics, ordered by usable timestamps for latest-value picks.""" + if not newest_first: + return [ + value + for row in rows + if (value := _parse_metric_value(row.get(label))) is not None + ] + x_label = next( + (name for name in x_labels if any(name in row for row in rows)), None + ) + dated = [ + ( + _timestamp_ms(row.get(x_label)) if x_label else None, + _parse_metric_value(row.get(label)), + ) + for row in rows + ] + + # Dated rows come first, newest first; ties retain their input order. + dated.sort(key=lambda item: (item[0] is None, -(item[0] or 0))) + # An undated metric cannot establish a latest value, even if dated metrics + # are all null. Order-independent aggregations above still include it. + return [ + value + for timestamp, value in dated + if timestamp is not None and value is not None Review Comment: You're right, and I went back to declining. This is fixed in 1c3f0effc34d53f34f78ef8a6670d2ec20e48fee. The new behaviour: when the aggregation picks the latest value (LAST_VALUE, or a non-exact `raw`) and any row lacks a usable timestamp, `value` is null with a reason. The frontend comparator returns 0 for an undated row, so it is not a total order. What the chart renders then depends on the input order and on the browser's sort, which the spec leaves implementation-defined for an inconsistent comparator. So mirroring it exactly is not possible. Declining is the only behaviour that is exact. I ran the comparator in Node across all six orders of your rows `[[1000,10],[null,99],[3000,30]]` and got 10, 30 or 99 depending on the order. An undated row with a *null* metric is enough to change the result: `[[1000,2],[null,null],[2000,3]]` renders 2, while the other orders render 3. So the guard applies to any undated row, not just undated non-null ones. This also fixes the earlier `skips_null_metric` case, which assumed otherwise. One exact case stays: if every non-null metric has the same value, no order can change it, so that value is returned. Order-independent aggregations (sum, mean, min, max, median) are unaffected because they never look at timestamps. Tests: - `test_latest_value_with_an_undated_row_has_no_headline`: every permutation × null/invalid/NaN/inf timestamps × null and non-null undated metric × LAST_VALUE/RAW. - `test_latest_value_does_not_diverge_from_the_rendered_chart`: your exact rows. - `test_latest_value_with_an_undated_row_and_one_distinct_value`: the exact case above. They fail on 5a930fd and pass on this commit. `tests/unit_tests/mcp_service/chart` + `dashboard` give 3650 passed, 3 skipped. ########## docs/admin_docs/configuration/mcp-server.mdx: ########## @@ -1519,6 +1519,16 @@ rather than being interpreted as one or zero rows. ## Chart data column counts +Big Number headlines in `get_chart_data` use the unsaved visualization type when Review Comment: Fixed in 1c3f0effc34d53f34f78ef8a6670d2ec20e48fee. The paragraph is gone from under "Chart data column counts", so that heading leads straight into its own `unique_count`/`null_count` paragraph again. A new "## Big Number headlines" section follows it. It explains that `get_chart_data` and `get_dashboard_data` return `headline` and that agents should report `headline.value` instead of the rows. It also has a field table and lists when `value` is null: no rows, missing metric or all-null values, rolling window/resample missing from the executed query or truncation, LAST_VALUE with undated rows, and `raw` without a saved query context. The changelog-style edge cases are dropped. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
