sadpandajoe commented on code in PR #45817:
URL: https://github.com/apache/superset/pull/45817#discussion_r4417411026
##########
superset/charts/client_processing.py:
##########
@@ -1222,6 +1228,171 @@ def merge_currency_formats(
return currency_formats
+def _fallback_export_value(value: Any) -> Any:
+ """Render a finite numeric cell as exact fixed-point text.
+
+ An unconfigured numeric column must never serialize as a raw float
+ (``1e-07``/``19.7999992``) or lose precision to float64 re-inference, so
+ finite ``float`` and ``int`` cells become exact text here while every other
+ value -- ``Decimal`` (handled by ``csv.format_decimal``), ``None``,
``NaN``,
+ infinity and non-numeric cells -- passes through unchanged.
+ """
+ if isinstance(value, bool):
+ return value
+ if isinstance(value, float) and math.isfinite(value):
+ # Route through ``Decimal(str())`` so the exact value the user sees is
+ # emitted as fixed-point text rather than a float repr or scientific
+ # notation, matching the Decimal branch below.
+ return format(Decimal(str(value)), "f")
+ if isinstance(value, int):
+ # Keep large integers exact: ``.map`` would otherwise re-infer an
object
+ # column of ints with nulls as float64 and round them.
+ return str(value)
+ return csv.format_decimal(value)
+
+
+def apply_export_number_formats(
+ df: pd.DataFrame,
+ coltypes: list[GenericDataType],
+ form_data: dict[str, Any],
+ datasource: Optional[Union["BaseDatasource", "Query"]] = None,
+) -> pd.DataFrame:
+ """Format numeric columns as display text for a chart-data export.
+
+ Each numeric column (by ``coltypes``, aligned with ``df.columns``) is
+ resolved to its configured d3 format, currency and small-number format the
+ same way the live Table plugin does -- ``column_config`` overrides first,
+ then the datasource's saved per-metric formats -- and handed to
+ ``format_column``. Percent metric columns default to ``PERCENT_3_POINT``
and
+ a numeric column with nothing configured takes the fixed-point fallback so
+ its cells are never serialized as raw floats. Non-numeric columns, nulls,
+ NaN and infinity are left to the existing helpers and pass through
+ unchanged. The passed frame is mutated in place and returned; the helper
+ runs no query and performs no I/O.
+ """
+ saved_formats, verbose_map = get_datasource_column_formats(datasource)
+ saved_currency_formats, _ = get_datasource_currency_formats(datasource)
+ column_config = form_data.get("column_config") or {}
+
+ def label_of(name: str) -> str:
+ """Return the column label as it appears in ``df``
(verbose-renamed)."""
+ return name if name in df.columns else verbose_map.get(name, name)
+
+ # Index the per-column overrides by the label present in ``df`` so the
+ # lookups line up with the (possibly verbose-renamed) frame columns.
+ number_format_by_label = {
+ label_of(name): fmt for name, fmt in saved_formats.items()
+ }
+ currency_by_label = {
+ label_of(name): currency for name, currency in
saved_currency_formats.items()
+ }
+ config_by_label = {label_of(name): config for name, config in
column_config.items()}
+
+ # Percent metric columns are emitted with a leading ``%`` and are not
+ # verbose-renamed, so match them by that prefixed label; they default to
+ # PERCENT_3_POINT the way the Table plugin's percent formatter does.
+ percent_metric_labels = {
+ f"%{name}"
+ for name in get_metric_names(form_data.get("percent_metrics"))
+ if f"%{name}" in df.columns
+ }
+
+ # The AG Grid table paints its cells through ``valueFormatter``, which
never
+ # builds the small-number formatter, so an export leaves that format alone
+ # for it -- mirroring ``table()``'s own ``applies_small_number_format``.
+ applies_small_number_format = form_data.get("viz_type") != AG_GRID_TABLE
+
+ # Resolve an AUTO currency the way ``table()`` does: a per-row currency
+ # context drawn from the datasource's currency-code column drives each
+ # cell's symbol, with the query-wide single currency as the fallback.
+ # Without these an AUTO currency would export no symbol while the live
+ # chart shows one (e.g. ``1,234.50`` instead of ``$1,234.50``).
+ currency_column = get_datasource_currency_column(datasource, df)
+ row_currency_context = (
+ df[currency_column].map(currency_context_value) if currency_column
else None
+ )
+ # The query-wide single currency backs an AUTO format whose per-row context
+ # is empty. It is detected from the frame already in hand (no extra query
is
+ # issued here), and only when an AUTO currency is configured, mirroring
+ # ``_detect_currency``'s own gating.
+ top_level_currency = form_data.get("currency_format") or {}
+ top_level_auto = (
+ isinstance(top_level_currency, dict)
+ and top_level_currency.get("symbol") == AUTO_CURRENCY
+ )
+ auto_currency_configured = top_level_auto or
has_auto_currency_in_column_config(
+ form_data
+ )
+ detected_currency = (
+ detect_currency_from_df(df, currency_column)
+ if auto_currency_configured and currency_column
+ else None
+ )
+
+ numeric_columns = {
+ column
+ for column, coltype in zip(df.columns, coltypes, strict=True)
+ if coltype == GenericDataType.NUMERIC
+ }
+
+ for column in df.columns:
Review Comment:
If two numeric metrics share a verbose name (say both display as `Total`),
`get_data()` renames both columns before this helper runs, so `df[column]` here
is a two-column DataFrame. `format_column` then either hands whole Series to
the per-cell formatter (the configured `,.2f` is silently skipped and raw
values ship) or, when a currency-code column is selected, zips the two columns
against the per-row contexts and the `strict=True` zip raises `ValueError`,
failing the whole CSV/XLSX export. Could the loop go by column position (or
skip and dedupe duplicate labels) so a repeated verbose name can't change or
abort the export?
##########
tests/unit_tests/charts/test_export_number_format.py:
##########
@@ -0,0 +1,285 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Unit tests for the export number-format helper and its feature flag."""
+
+from decimal import Decimal
+from typing import Any, cast
+from unittest.mock import MagicMock
+
+import numpy as np
+import pandas as pd
+
+from superset.charts.client_processing import apply_export_number_formats
+from superset.utils import csv
+from superset.utils.core import GenericDataType
+
+
+# @AC-FR001-01
+def test_applies_configured_d3_format_to_numeric_column() -> None:
+ df = pd.DataFrame({"amount": [19.7999992]})
+ form_data = {"column_config": {"amount": {"d3NumberFormat": ",.1f"}}}
+
+ result = apply_export_number_formats(df, [GenericDataType.NUMERIC],
form_data, None)
+
+ assert result["amount"].tolist() == ["19.8"]
+
+
+# @AC-FR001-02
+def test_configured_format_respects_de_locale(mocker) -> None:
+ mocker.patch("superset.utils.number_format.get_locale",
return_value="de_DE")
+ df = pd.DataFrame({"amount": [1234.5]})
+ form_data = {
+ "column_config": {
+ "amount": {
+ "d3NumberFormat": ",.2f",
+ "currencyFormat": {"symbol": "EUR", "symbolPosition": None},
+ }
+ }
+ }
+
+ result = apply_export_number_formats(df, [GenericDataType.NUMERIC],
form_data, None)
+
+ value = result["amount"].tolist()[0]
+ # Locale-aware currency text, not a raw float and not a coerced date.
+ assert isinstance(value, str)
+ assert "€" in value
+ assert value != "1234.5"
+
+
+# @AC-FR003-01
+def test_unformatted_numeric_column_takes_fixed_point_fallback(mocker) -> None:
+ fallback = mocker.spy(csv, "format_decimal")
+ df = pd.DataFrame({"amount": [Decimal("19.7999992")]})
+
+ result = apply_export_number_formats(df, [GenericDataType.NUMERIC], {},
None)
+
+ fallback.assert_called()
+ value = result["amount"].tolist()[0]
+ assert value == "19.7999992"
+ assert not isinstance(value, float)
+
+
+# @AC-FR003-02
+def test_non_numeric_and_null_cells_left_intact() -> None:
+ df = pd.DataFrame(
+ {
+ "label": ["a", None, "c"],
+ "amount": [1.0, None, 3.0],
+ }
+ )
+ form_data = {"column_config": {"amount": {"d3NumberFormat": ",.1f"}}}
+
+ result = apply_export_number_formats(
+ df, [GenericDataType.STRING, GenericDataType.NUMERIC], form_data, None
+ )
+
+ # Non-numeric column is untouched: non-null cells keep their value and the
+ # null stays null. Its null is not pinned to None vs NaN, since the dtype
+ # pandas infers for a string column (and thus its null marker) depends on
+ # the ``future.infer_string`` option rather than on this helper.
+ label = result["label"].tolist()
+ assert [label[0], label[2]] == ["a", "c"]
+ assert pd.isna(label[1])
+ # Null numeric cell is not turned into a raw float string.
+ assert result["amount"].tolist()[0] == "1.0"
+ assert result["amount"].tolist()[2] == "3.0"
+
+
+# @AC-FR006-01
+def test_export_formatted_numbers_flag_defaults_to_false() -> None:
+ from superset.config import DEFAULT_FEATURE_FLAGS
+
+ assert DEFAULT_FEATURE_FLAGS["EXPORT_FORMATTED_NUMBERS"] is False
+
+
+# NFR-REL-01
+def test_edge_values_do_not_raise() -> None:
+ df = pd.DataFrame(
+ {
+ "configured": [np.nan, np.inf, -np.inf, 10**18],
+ "fallback": [
+ Decimal("NaN"),
+ Decimal("Infinity"),
+ None,
+ Decimal(10**18),
+ ],
+ }
+ )
+ form_data = {"column_config": {"configured": {"d3NumberFormat": ",.2f"}}}
+
+ result = apply_export_number_formats(
+ df,
+ [GenericDataType.NUMERIC, GenericDataType.NUMERIC],
+ form_data,
+ None,
+ )
+
+ # The call completes without raising and returns a frame of the same shape.
+ assert list(result.columns) == ["configured", "fallback"]
+ assert len(result) == 4
+
+
+# @AC-FR002-02
+def test_configured_format_output_is_plain_text_not_date(mocker) -> None:
+ # A localized currency value is emitted as plain display text; nothing in
+ # the helper turns a formatted number into a datetime/date object.
+ mocker.patch("superset.utils.number_format.get_locale",
return_value="de_DE")
+ df = pd.DataFrame({"amount": [3.5]})
+ form_data = {
+ "column_config": {
+ "amount": {
+ "d3NumberFormat": ",.1f",
+ "currencyFormat": {"symbol": "EUR", "symbolPosition": None},
+ }
+ }
+ }
+
+ result = apply_export_number_formats(df, [GenericDataType.NUMERIC],
form_data, None)
+
+ value = result["amount"].tolist()[0]
+ import datetime
+
+ assert isinstance(value, str)
+ assert not isinstance(value, (datetime.date, datetime.datetime,
pd.Timestamp))
+ assert "€" in value
+
+
+# NFR-PERF-01
+def test_each_numeric_column_formatted_in_a_single_pass(mocker) -> None:
+ # Guard against an asymptotic regression: the fixed-point fallback maps
each
+ # cell of a numeric column exactly once (one linear pass per column), never
+ # re-scanning the frame per cell.
+ fallback = mocker.spy(csv, "format_decimal")
+ rows = 50
+ df = pd.DataFrame({"amount": [Decimal(str(i)) for i in range(rows)]})
+
+ apply_export_number_formats(df, [GenericDataType.NUMERIC], {}, None)
+
+ # Exactly one call per cell -- linear, not quadratic.
+ assert fallback.call_count == rows
+
+
+# @AC-FR005-02
+def test_helper_does_not_reference_sql_lab_export_path() -> None:
+ # Scope isolation: the chart-export helper does not import or invoke the
SQL
+ # Lab decimal formatter, keeping the two export paths independent.
+ import inspect
+
+ from superset.charts import client_processing
+
+ source = inspect.getsource(client_processing.apply_export_number_formats)
+ assert "_format_decimal_columns" not in source
+ assert "sql_lab" not in source
+
+
+# RISK-T06
+def test_verbose_map_renamed_column_gets_configured_format() -> None:
+ # The DataFrame carries the verbose label, while the saved format and the
+ # column config are keyed by the underlying metric name.
+ df = pd.DataFrame({"Total Amount": [1234.5]})
+ datasource = MagicMock()
+ datasource.data = {
+ "column_formats": {"amount": ",.2f"},
+ "verbose_map": {"amount": "Total Amount"},
+ }
+
+ result = apply_export_number_formats(df, [GenericDataType.NUMERIC], {},
datasource)
+
+ assert result["Total Amount"].tolist() == ["1,234.50"]
+
+
+# @AC-FR003-01
+def test_unconfigured_float_column_takes_fixed_point_fallback() -> None:
+ # An unconfigured numeric column holding floats must serialize as exact
+ # fixed-point text, never a float repr or scientific notation.
+ df = pd.DataFrame({"amount": [19.7999992, 1e-07]})
+
+ result = apply_export_number_formats(df, [GenericDataType.NUMERIC], {},
None)
+
+ values = result["amount"].tolist()
+ assert values == ["19.7999992", "0.0000001"]
+ assert all(isinstance(value, str) for value in values)
+
+
+# @AC-FR003-01
+def test_unconfigured_large_int_column_stays_exact() -> None:
+ # Large integers must not be rounded by a float64 re-inference of the
column.
+ df = pd.DataFrame({"n": [9007199254740993, None]}, dtype=object)
+
+ result = apply_export_number_formats(df, [GenericDataType.NUMERIC], {},
None)
+
+ assert result["n"].tolist()[0] == "9007199254740993"
+
+
+# @AC-FR001-01
+def test_percent_metric_defaults_to_percent_format() -> None:
+ # A percent metric column (leading "%") with nothing configured defaults to
+ # PERCENT_3_POINT, matching the Table plugin, rather than the raw fallback.
+ df = pd.DataFrame({"%share": [0.3]})
+ form_data = {"percent_metrics": ["share"]}
+
+ result = apply_export_number_formats(df, [GenericDataType.NUMERIC],
form_data, None)
+
+ assert result["%share"].tolist() == ["30.000%"]
+
+
+# @AC-FR001-01
+def test_ag_grid_table_skips_small_number_format() -> None:
+ # The AG Grid table never builds the small-number formatter, so an export
+ # leaves that format alone for it and uses only the column's normal format.
+ df = pd.DataFrame({"amount": [0.125]})
+ form_data = {
+ "viz_type": "ag-grid-table",
+ "column_config": {
+ "amount": {"d3NumberFormat": ",.0f", "d3SmallNumberFormat": ",.1%"}
+ },
+ }
+
+ result = apply_export_number_formats(df, [GenericDataType.NUMERIC],
form_data, None)
+
+ # The small-number format (",.1%" -> "12.5%") is ignored; the main format
+ # (",.0f" -> "0") applies to the sub-one value instead.
+ assert result["amount"].tolist() == ["0"]
+
+
+# @AC-FR002-01
+def test_auto_currency_resolves_from_row_context() -> None:
+ # An AUTO currency resolves to the row's currency symbol the way table()
+ # does (via the datasource currency-code column), instead of exporting no
+ # symbol. Without the row currency context "1,234.50" would ship bare.
+ datasource = MagicMock()
+ datasource.data = {"currency_code_column": "currency"}
+ df = pd.DataFrame({"amount": [1234.5], "currency": ["USD"]})
+ form_data = {
+ "viz_type": "table",
+ "column_config": {
+ "amount": {"d3NumberFormat": ",.2f", "currencyFormat": {"symbol":
"AUTO"}}
+ },
+ }
+
+ result = apply_export_number_formats(
+ df,
+ [GenericDataType.NUMERIC, GenericDataType.STRING],
+ form_data,
+ cast(Any, datasource),
+ )
+
+ value = result["amount"].tolist()[0]
+ assert isinstance(value, str)
+ # The USD symbol is resolved from the row context, not dropped.
+ assert "$" in value
Review Comment:
This single-row frame can't tell the per-row currency context from the
query-wide detection: `detect_currency_from_df` also returns USD for it, so if
`row_currency_context` were dropped from the `format_column` call,
`resolve_auto_currency` would still resolve to USD and the `$` assertion
passes. Could the frame carry a USD and an EUR row and assert each exact output
(`$1,234.50` / `€...`), so the test fails when the row context is lost?
##########
tests/unit_tests/common/test_query_context_processor_export_format.py:
##########
@@ -0,0 +1,515 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Integration tests for number-formatted CSV/XLSX export in get_data()."""
+
+import io
+from decimal import Decimal
+from types import SimpleNamespace
+from typing import Any, cast
+
+import pandas as pd
+
+from superset.common.chart_data import ChartDataResultFormat,
ChartDataResultType
+from superset.common.query_context import QueryContext
+from superset.common.query_context_processor import QueryContextProcessor
+from superset.utils.core import GenericDataType
+from tests.unit_tests.conftest import with_feature_flags
+
+
+def _make_processor(
+ result_format: ChartDataResultFormat,
+ form_data: dict[str, Any],
+ datasource_data: dict[str, Any] | None = None,
+ result_type: ChartDataResultType = ChartDataResultType.FULL,
+) -> QueryContextProcessor:
+ datasource = SimpleNamespace(data=datasource_data or {})
+ context = SimpleNamespace(
+ datasource=datasource,
+ result_format=result_format,
+ result_type=result_type,
+ form_data=form_data,
+ )
+ return QueryContextProcessor(cast(QueryContext, context))
+
+
+def _csv_text(payload: str | bytes | list[dict[str, Any]]) -> str:
+ # get_data() is typed as a union over export formats; the CSV branch always
+ # returns bytes, so narrow before decoding.
+ assert isinstance(payload, bytes)
+ return payload.decode("utf-8-sig")
+
+
+def _read_xlsx_column(
+ payload: str | bytes | list[dict[str, Any]], column: str
+) -> list[Any]:
+ # The XLSX branch of get_data() always returns bytes; narrow the union.
+ assert isinstance(payload, bytes)
+ frame = pd.read_excel(io.BytesIO(payload))
+ return frame[column].tolist()
+
+
+# @AC-FR001-01
+@with_feature_flags(EXPORT_FORMATTED_NUMBERS=True)
+def test_csv_export_applies_configured_number_format(app_context: None) ->
None:
+ processor = _make_processor(
+ ChartDataResultFormat.CSV,
+ {"column_config": {"amount": {"d3NumberFormat": ",.1f"}}},
+ )
+ df = pd.DataFrame({"amount": [19.7999992]})
+
+ result = processor.get_data(df, [GenericDataType.NUMERIC])
+
+ text = _csv_text(result)
+ assert "19.8" in text
+ assert "19.7999992" not in text
+
+
+# @AC-FR001-03
+@with_feature_flags(EXPORT_FORMATTED_NUMBERS=True)
+def test_csv_export_unformattable_value_takes_fixed_point_fallback(
+ app_context: None,
+) -> None:
+ processor = _make_processor(ChartDataResultFormat.CSV, {})
+ df = pd.DataFrame({"amount": [Decimal("19.7999992")]})
+
+ result = processor.get_data(df, [GenericDataType.NUMERIC])
+
+ text = _csv_text(result)
+ # Exact fixed-point text, never a float repr / scientific notation.
+ assert "19.7999992" in text
+ assert "e" not in text.lower().split("amount")[-1]
+
+
+# @AC-FR002-01
+@with_feature_flags(EXPORT_FORMATTED_NUMBERS=True)
+def test_xlsx_export_matches_csv_for_shared_column(app_context: None) -> None:
+ form_data = {"column_config": {"amount": {"d3NumberFormat": ",.1f"}}}
+
+ csv_processor = _make_processor(ChartDataResultFormat.CSV, form_data)
+ csv_result = csv_processor.get_data(
+ pd.DataFrame({"amount": [19.7999992]}), [GenericDataType.NUMERIC]
+ )
+ csv_value = _csv_text(csv_result).splitlines()[1].strip()
+
+ xlsx_processor = _make_processor(ChartDataResultFormat.XLSX, form_data)
+ xlsx_result = xlsx_processor.get_data(
+ pd.DataFrame({"amount": [19.7999992]}), [GenericDataType.NUMERIC]
+ )
+ xlsx_value = _read_xlsx_column(xlsx_result, "amount")[0]
+
+ # Both formats display 19.8 for the same source value; the formatted text
is
+ # now kept as a string in XLSX too, so the configured decimals survive.
+ assert str(xlsx_value) == csv_value == "19.8"
+
+
+# @AC-FR004-01
+@with_feature_flags(EXPORT_FORMATTED_NUMBERS=True)
+def test_csv_escapes_formatted_formula_prefixed_value(app_context: None) ->
None:
+ processor = _make_processor(
+ ChartDataResultFormat.CSV,
+ {"column_config": {"amount": {"d3NumberFormat": "+,.1f"}}},
+ )
+ df = pd.DataFrame({"amount": [19.7999992]})
+
+ result = processor.get_data(df, [GenericDataType.NUMERIC])
+
+ text = _csv_text(result)
+ # The formatter emits "+19.8"; escaping (which runs after formatting)
+ # must prefix the formula-like value with a single quote.
+ assert "'+19.8" in text
+
+
+# @AC-FR004-02
+@with_feature_flags(EXPORT_FORMATTED_NUMBERS=True)
+def test_xlsx_quotes_formatted_formula_prefixed_value(app_context: None) ->
None:
+ processor = _make_processor(
+ ChartDataResultFormat.XLSX,
+ {"column_config": {"amount": {"d3NumberFormat": "+,.2f"}}},
+ )
+ df = pd.DataFrame({"amount": [1234567.891]})
+
+ result = processor.get_data(df, [GenericDataType.NUMERIC])
+
+ # The formatter emits "+1,234,567.89"; formula-quoting (which runs after
+ # formatting) must prefix the formula-like value with a single quote.
+ value = _read_xlsx_column(result, "amount")[0]
+ assert value == "'+1,234,567.89"
+
+
+# @AC-FR006-02
+def test_flag_off_csv_and_xlsx_byte_identical(app_context: None) -> None:
+ form_data = {"column_config": {"amount": {"d3NumberFormat": ",.1f"}}}
+
+ csv_processor = _make_processor(ChartDataResultFormat.CSV, form_data)
+ csv_result = csv_processor.get_data(
+ pd.DataFrame({"amount": [19.7999992]}), [GenericDataType.NUMERIC]
+ )
+ # Flag defaults to False: the raw float survives unformatted.
+ assert b"19.7999992" in csv_result
+ assert b"19.8" not in csv_result
+
+ xlsx_processor = _make_processor(ChartDataResultFormat.XLSX, form_data)
+ xlsx_result = xlsx_processor.get_data(
+ pd.DataFrame({"amount": [19.7999992]}), [GenericDataType.NUMERIC]
+ )
+ assert _read_xlsx_column(xlsx_result, "amount")[0] == 19.7999992
+
+
+# @AC-FR005-01
+@with_feature_flags(EXPORT_FORMATTED_NUMBERS=True)
+def test_json_result_unchanged_by_formatting_stage(app_context: None) -> None:
+ processor = _make_processor(
+ ChartDataResultFormat.JSON,
+ {"column_config": {"amount": {"d3NumberFormat": ",.1f"}}},
+ )
+ df = pd.DataFrame({"amount": [19.7999992]})
+
+ result = processor.get_data(df, [GenericDataType.NUMERIC])
+
+ # JSON path returns records before the table_like formatting stage.
+ assert result == [{"amount": 19.7999992}]
+
+
+# @AC-FR001-02
+@with_feature_flags(EXPORT_FORMATTED_NUMBERS=True)
+def test_csv_export_de_locale_formats_consistently(app_context: None, mocker)
-> None:
+ # A ``de`` request locale drives the currency symbol position; the d3
number
+ # grammar itself is locale-independent, matching the frontend d3-format.
+ mocker.patch("superset.utils.number_format.get_locale",
return_value="de_DE")
+ processor = _make_processor(
+ ChartDataResultFormat.CSV,
+ {
+ "column_config": {
+ "amount": {
+ "d3NumberFormat": ",.2f",
+ "currencyFormat": {"symbol": "EUR", "symbolPosition":
None},
+ }
+ }
+ },
+ )
+ df = pd.DataFrame({"amount": [1234.5]})
+
+ result = processor.get_data(df, [GenericDataType.NUMERIC])
+
+ text = _csv_text(result)
+ # Consistent localized currency text: no raw float survives, and the value
+ # is not coerced to a date.
+ assert "€" in text
+ assert "1234.5" not in text
+ assert "1,234.50" in text
+ assert "Mai" not in text
+
+
+# @AC-FR002-02
+@with_feature_flags(EXPORT_FORMATTED_NUMBERS=True)
+def test_xlsx_export_de_locale_value_not_date_coerced(
+ app_context: None, mocker
+) -> None:
+ # A localized currency value stays a string through the XLSX text path;
+ # ``apply_column_types`` cannot re-coerce it to a number (or a date).
+ mocker.patch("superset.utils.number_format.get_locale",
return_value="de_DE")
+ processor = _make_processor(
+ ChartDataResultFormat.XLSX,
+ {
+ "column_config": {
+ "amount": {
+ "d3NumberFormat": ",.2f",
+ "currencyFormat": {"symbol": "EUR", "symbolPosition":
None},
+ }
+ }
+ },
+ )
+ df = pd.DataFrame({"amount": [1234.5]})
+
+ result = processor.get_data(df, [GenericDataType.NUMERIC])
+
+ value = _read_xlsx_column(result, "amount")[0]
+ # The cell is the formatted string, never a datetime and never "03. Mai".
+ assert isinstance(value, str)
+ assert "€" in value
+ assert "1,234.50" in value
+ assert not isinstance(value, pd.Timestamp)
+ assert "Mai" not in value
+
+
+# @AC-FR005-01
+@with_feature_flags(EXPORT_FORMATTED_NUMBERS=True)
+def test_arrow_result_byte_identical_with_flag_on(app_context: None) -> None:
+ form_data = {"column_config": {"amount": {"d3NumberFormat": ",.1f"}}}
+ source = pd.DataFrame({"amount": [19.7999992]})
+
+ on_processor = _make_processor(ChartDataResultFormat.ARROW, form_data)
+ on_bytes = on_processor.get_data(source.copy(), [GenericDataType.NUMERIC])
+
+ # Re-serialize the untouched frame independently: the formatting stage runs
+ # only on the table_like branch, so Arrow must be byte-for-byte identical.
+ reference = QueryContextProcessor._to_arrow_ipc(source.copy())
+
+ assert isinstance(on_bytes, bytes)
+ assert on_bytes == reference
+
+
+# @AC-FR005-01
+def test_arrow_result_identical_regardless_of_flag(app_context: None) -> None:
+ # The flag-off Arrow bytes equal the independently serialized reference
too,
+ # confirming the stage never touches the Arrow branch in either state.
+ form_data = {"column_config": {"amount": {"d3NumberFormat": ",.1f"}}}
+ source = pd.DataFrame({"amount": [19.7999992]})
+
+ off_processor = _make_processor(ChartDataResultFormat.ARROW, form_data)
+ off_bytes = off_processor.get_data(source.copy(),
[GenericDataType.NUMERIC])
+ reference = QueryContextProcessor._to_arrow_ipc(source.copy())
+
+ assert off_bytes == reference
+
+
+# @AC-FR005-02
+def test_sql_lab_decimal_formatting_path_is_untouched() -> None:
+ # Scope guard: the SQL Lab export helper keeps its own decimal handling and
+ # never routes through the new chart-export helper.
+ import inspect
+
+ from superset.commands.sql_lab import export as sql_lab_export
+
+ source = inspect.getsource(sql_lab_export)
+ assert "apply_export_number_formats" not in source
+ assert "EXPORT_FORMATTED_NUMBERS" not in source
+
+ # And its behavior is unchanged: a finite decimal becomes fixed-point text
+ # while floats/ints/None keep their types, with no d3 formatting applied.
+ df = pd.DataFrame(
+ {"amount": [Decimal("19.7999992"), 3.0, None]},
+ dtype=object,
+ )
+
+ sql_lab_export._format_decimal_columns(df)
+
+ values = df["amount"].tolist()
+ assert values[0] == "19.7999992"
+ assert values[1] == 3.0
+ assert values[2] is None
+
+
+# @AC-FR003-01
+@with_feature_flags(EXPORT_FORMATTED_NUMBERS=True)
+def test_csv_export_null_nan_inf_pass_through(app_context: None) -> None:
+ # Edge values through the full export path: no raise, nulls stay blank, and
+ # a configured column still renders its finite cells.
+ processor = _make_processor(
+ ChartDataResultFormat.CSV,
+ {"column_config": {"amount": {"d3NumberFormat": ",.1f"}}},
+ )
+ df = pd.DataFrame(
+ {
+ "label": ["ok", None, "z"],
+ "amount": [19.75, float("nan"), float("inf")],
+ }
+ )
+
+ result = processor.get_data(df, [GenericDataType.STRING,
GenericDataType.NUMERIC])
+
+ text = _csv_text(result)
+ # The finite cell is formatted; the frame still has three data rows.
+ assert "19.8" in text
+ assert len(text.strip().splitlines()) == 1 + 3
+
+
+# @AC-FR003-02
+@with_feature_flags(EXPORT_FORMATTED_NUMBERS=True)
+def test_csv_export_unformattable_decimal_edge_values(app_context: None) ->
None:
+ # Fixed-point fallback over a column with no config: finite decimals become
+ # exact text, non-finite decimals and None pass through without raising.
+ processor = _make_processor(ChartDataResultFormat.CSV, {})
+ df = pd.DataFrame(
+ {"amount": [Decimal("19.7999992"), Decimal("NaN"), None]},
+ dtype=object,
+ )
+
+ result = processor.get_data(df, [GenericDataType.NUMERIC])
+
+ text = _csv_text(result)
+ assert "19.7999992" in text
+ # No scientific notation leaked into the serialized decimal.
+ assert "e+" not in text.lower()
+
+
+# @AC-FR002-01
+@with_feature_flags(EXPORT_FORMATTED_NUMBERS=True)
+def test_xlsx_export_keeps_configured_trailing_decimals(app_context: None) ->
None:
+ # A ",.2f" format renders 3.5 as "3.50"; the XLSX cell must stay text so
the
+ # trailing zero is not coerced back to the number 3.5 by
apply_column_types.
+ # (pd.read_excel would re-infer a numeric-looking string, so read the raw
+ # cell via openpyxl to observe what was actually stored.)
+ from openpyxl import load_workbook
+
+ processor = _make_processor(
+ ChartDataResultFormat.XLSX,
+ {"column_config": {"amount": {"d3NumberFormat": ",.2f"}}},
+ )
+ df = pd.DataFrame({"amount": [3.5]})
+
+ result = processor.get_data(df, [GenericDataType.NUMERIC])
+
+ assert isinstance(result, bytes)
+ cell = load_workbook(io.BytesIO(result)).active["A2"]
+ assert cell.data_type == "s"
+ assert cell.value == "3.50"
+
+
+# @AC-FR001-01
+@with_feature_flags(EXPORT_FORMATTED_NUMBERS=True)
+def test_chart_config_format_wins_over_saved_datasource_format(
+ app_context: None,
+) -> None:
+ # A chart's column_config override must win over the datasource's saved
+ # per-metric format and currency, pinning the config.get(...) precedence in
+ # apply_export_number_formats (a saved ",.2f"/USD would otherwise leak).
+ processor = _make_processor(
+ ChartDataResultFormat.CSV,
+ {
+ "column_config": {
+ "amount": {
+ "d3NumberFormat": ",.1f",
+ "currencyFormat": {"symbol": "EUR", "symbolPosition":
None},
+ }
+ }
+ },
+ datasource_data={
+ "column_formats": {"amount": ",.2f"},
+ "currency_formats": {"amount": {"symbol": "USD"}},
+ },
+ )
+ df = pd.DataFrame({"amount": [1234.5]})
+
+ result = processor.get_data(df, [GenericDataType.NUMERIC])
+
+ text = _csv_text(result)
+ # The chart override (one decimal, EUR) wins over the saved format.
+ assert "€" in text
+ assert "1,234.5" in text
+ assert "1,234.50" not in text
+ assert "$" not in text
+
+
+# @AC-FR001-01
+@with_feature_flags(EXPORT_FORMATTED_NUMBERS=True)
+def test_small_number_format_forwarded_to_export(app_context: None) -> None:
+ # The column's d3SmallNumberFormat is applied per cell: a sub-one value
takes
+ # it while larger values keep the column's normal format, matching the
chart.
+ processor = _make_processor(
+ ChartDataResultFormat.CSV,
+ {
+ "column_config": {
+ "amount": {
+ "d3NumberFormat": ",.0f",
+ "d3SmallNumberFormat": ",.1%",
+ }
+ }
+ },
+ )
+ df = pd.DataFrame({"amount": [0.125, 5.0]})
+
+ result = processor.get_data(df, [GenericDataType.NUMERIC])
+
+ text = _csv_text(result)
+ # Sub-one value rendered by the small-number format, the other by the main.
+ assert "12.5%" in text
+ assert "5" in text
+
+
+# @AC-FR001-01
+@with_feature_flags(EXPORT_FORMATTED_NUMBERS=True)
+def test_post_processed_export_skips_pre_pivot_formatting(app_context: None)
-> None:
+ # A POST_PROCESSED export for a post-processor viz_type
(table/pivot_table_v2)
+ # is re-parsed and pivoted by apply_client_processing, so get_data must
leave
+ # the raw numbers here to avoid turning a formatted thousands string into
NaN;
+ # formatting happens after post-processing.
+ processor = _make_processor(
+ ChartDataResultFormat.CSV,
+ {
+ "viz_type": "pivot_table_v2",
+ "column_config": {"amount": {"d3NumberFormat": ",.1f"}},
+ },
+ result_type=ChartDataResultType.POST_PROCESSED,
+ )
+ df = pd.DataFrame({"amount": [19.7999992]})
+
+ result = processor.get_data(df, [GenericDataType.NUMERIC])
+
+ text = _csv_text(result)
+ # Raw value survives: the pre-pivot formatting stage was skipped.
+ assert "19.7999992" in text
+ assert "19.8" not in text
+
+
+# @AC-FR001-01
+@with_feature_flags(EXPORT_FORMATTED_NUMBERS=True)
+def test_post_processed_export_without_post_processor_is_formatted(
+ app_context: None,
+) -> None:
+ # A POST_PROCESSED export for a viz_type with no post-processor (e.g.
+ # ag-grid-table, as scheduled reports/alerts request) is returned unchanged
+ # by apply_client_processing, so get_data must still format it here.
+ processor = _make_processor(
+ ChartDataResultFormat.CSV,
+ {
+ "viz_type": "ag-grid-table",
+ "column_config": {"amount": {"d3NumberFormat": ",.1f"}},
+ },
+ result_type=ChartDataResultType.POST_PROCESSED,
+ )
+ df = pd.DataFrame({"amount": [19.7999992]})
+
+ result = processor.get_data(df, [GenericDataType.NUMERIC])
+
+ text = _csv_text(result)
+ assert "19.8" in text
+ assert "19.7999992" not in text
+
+
+# @AC-FR001-01
+@with_feature_flags(EXPORT_FORMATTED_NUMBERS=True)
+def test_post_processed_export_formatted_by_client_processing(
+ app_context: None,
+) -> None:
+ # Round trip: get_data skips the pre-pivot stage for a post-processor
+ # viz_type, so the numbers must be formatted by apply_client_processing
after
+ # the pivot/table arithmetic. This fails if that post-processing formatting
+ # branch regresses -- get_data alone would ship an unformatted aggregate.
+ from superset.charts.client_processing import apply_client_processing
+
+ form_data = {
+ "viz_type": "table",
+ "column_config": {"amount": {"d3NumberFormat": ",.2f"}},
+ }
+ processor = _make_processor(
+ ChartDataResultFormat.CSV,
+ form_data,
+ result_type=ChartDataResultType.POST_PROCESSED,
+ )
+ df = pd.DataFrame({"amount": [2234.5]})
+
+ raw = processor.get_data(df, [GenericDataType.NUMERIC])
+ # Pre-pivot formatting was skipped: the aggregate is still unformatted
here.
+ assert "2,234.50" not in _csv_text(raw)
+
+ result = {
+ "queries": [{"result_format": ChartDataResultFormat.CSV, "data": raw}],
+ }
+ processed = apply_client_processing(result, form_data, None)
+
+ assert "2,234.50" in processed["queries"][0]["data"]
Review Comment:
The round trip through `apply_client_processing` only covers CSV. The new
gate (`result_format in table_like()`) also fires for XLSX, but nothing feeds
an XLSX `table` export with a `,.2f` column (`3.5`) through it and checks the
final workbook cell is the text `3.50`. Narrowing the gate to CSV, or the XLSX
re-serialization turning the text back into a number, would pass every current
test. Could you add an XLSX variant that reads the raw cell value/type?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
