This is an automated email from the ASF dual-hosted git repository.
sadpandajoe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/superset.git
The following commit(s) were added to refs/heads/master by this push:
new 3b7001836c7 fix(charts): keep large nullable-integer dimensions exact
(#45308)
3b7001836c7 is described below
commit 3b7001836c786a5a668e9424471150d091f7752f
Author: Joe Li <[email protected]>
AuthorDate: Thu Oct 22 15:14:38 2026 -0700
fix(charts): keep large nullable-integer dimensions exact (#45308)
Co-authored-by: Claude <[email protected]>
---
UPDATING.md | 9 +
superset/common/utils/dataframe_utils.py | 64 ++
superset/models/helpers.py | 21 +
superset/utils/excel.py | 41 +-
tests/unit_tests/charts/test_client_processing.py | 41 +
.../charts/test_exact_integer_chart_data.py | 824 +++++++++++++++++++++
tests/unit_tests/common/test_dataframe_utils.py | 53 ++
.../unit_tests/common/test_query_cache_manager.py | 38 +
.../common/test_query_context_processor.py | 50 ++
tests/unit_tests/datasource/test_query_api.py | 24 +
.../models/normalize_df_large_integer_test.py | 307 ++++++++
tests/unit_tests/utils/csv_tests.py | 14 +
tests/unit_tests/utils/excel_tests.py | 38 +
13 files changed, 1513 insertions(+), 11 deletions(-)
diff --git a/UPDATING.md b/UPDATING.md
index d9819c4720c..5342a2faaa1 100644
--- a/UPDATING.md
+++ b/UPDATING.md
@@ -24,6 +24,15 @@ assists people when migrating to a new version.
## Next
+- **Large nullable-integer dimensions stay exact in chart data.** In a plain
chart
+ query (no post-processing or time offsets, and not a report's
client-processed
+ result), a dimension column of nullable integers that holds a value beyond
+ `2**53 - 1` is no longer rounded to a float. JSON responses carry such
values as
+ strings (the existing contract for large integers), and CSV, Excel and Arrow
+ exports carry the exact integer. Consumers that parsed these values as JSON
+ numbers receive a string for them. Metric columns are unchanged, and results
+ cached before the upgrade keep the old values until they expire.
+
- **Bulk semantic view creation is capped and time-bounded.** With
`SEMANTIC_LAYERS` enabled, `POST /api/v1/semantic_view/` refuses requests
with
more than `SEMANTIC_VIEW_BULK_CREATE_MAX_VIEWS` views (default 100) with a
400,
diff --git a/superset/common/utils/dataframe_utils.py
b/superset/common/utils/dataframe_utils.py
index 062b727cf01..3a4fb7d0ef4 100644
--- a/superset/common/utils/dataframe_utils.py
+++ b/superset/common/utils/dataframe_utils.py
@@ -22,6 +22,10 @@ from typing import Any, Literal, TYPE_CHECKING
import numpy as np
import pandas as pd
+from pandas.api.types import infer_dtype
+
+from superset.common.chart_data import ChartDataResultType
+from superset.utils.core import get_metric_names, JS_MAX_INTEGER
if TYPE_CHECKING:
from superset.common.query_object import QueryObject
@@ -78,6 +82,66 @@ def df_columns_to_num(df: pd.DataFrame, columns:
Collection[str]) -> None:
df[col] = converted
+INT64_RANGE = (-(2**63), 2**63 - 1)
+
+
+def can_keep_exact_integers(query_object: QueryObject) -> bool:
+ """
+ Whether the result of ``query_object`` may carry exact large integers.
+
+ Only plain chart queries qualify. Queries with Python-side post-processing
or
+ time offsets keep the float64 normalization (a precaution: that machinery
is
+ written for numeric columns), and so do requests that go through report
+ client processing (``POST_PROCESSED``), which formats the rebuilt frame on
+ numeric dtypes only. The decision reads only fields that are part of the
query
+ cache key (``post_processing``, ``time_offsets`` and the per-query
+ ``result_type``), so a cached frame always has the shape its key implies.
+ """
+ return not (
+ query_object.post_processing
+ or query_object.time_offsets
+ or query_object.result_type == ChartDataResultType.POST_PROCESSED
+ )
+
+
+def lossy_integer_columns(df: pd.DataFrame, query_object: QueryObject) ->
list[str]:
+ """
+ Dimension columns that ``infer_objects`` would round to float64.
+
+ These are object columns of Python integers and nulls (how
+ ``SupersetResultSet`` hands back a nullable BIGINT) holding a value beyond
+ ``JS_MAX_INTEGER`` where every value fits int64. Without nulls pandas
+ already yields an exact int64, and values outside int64 stay exact objects
+ or are left to the float64 normalization (the uint64-only range).
+
+ Only columns selected as ``columns`` and not named as a metric qualify,
using
+ the labels the dataframe carries (no verbose-name mapping): a saved metric
+ comes back under its raw ``metric_name``, and metrics-only queries have
none.
+ """
+ if not df.columns.is_unique:
+ return []
+ candidates = set(query_object.column_names) - set(
+ get_metric_names(query_object.metrics)
+ )
+ columns: list[str] = []
+ for col in df.columns:
+ series = df[col]
+ if (
+ col not in candidates
+ or series.dtype != object
+ or infer_dtype(series, skipna=True) != "integer"
+ or not series.isna().any()
+ ):
+ continue
+ present = [int(value) for value in series.dropna()]
+ low, high = min(present), max(present)
+ if (low < -JS_MAX_INTEGER or high > JS_MAX_INTEGER) and (
+ INT64_RANGE[0] <= low and high <= INT64_RANGE[1]
+ ):
+ columns.append(col)
+ return columns
+
+
def is_datetime_series(series: Any) -> bool:
if series is None or not isinstance(series, pd.Series):
return False
diff --git a/superset/models/helpers.py b/superset/models/helpers.py
index 1aac9ef742c..0613eb934d1 100644
--- a/superset/models/helpers.py
+++ b/superset/models/helpers.py
@@ -2721,6 +2721,8 @@ class ExploreMixin: # pylint:
disable=too-many-public-methods
df: pd.DataFrame,
query_object: QueryObject,
sql_shifted_temporal_labels: set[str] | None = None,
+ *,
+ keep_exact_integers: bool = False,
) -> pd.DataFrame:
"""
Normalize the dataframe by converting datetime columns and ensuring
@@ -2729,6 +2731,9 @@ class ExploreMixin: # pylint:
disable=too-many-public-methods
:param df: The dataframe to normalize
:param query_object: The query object with metadata about columns
:param sql_shifted_temporal_labels: labels already shifted in
generated SQL
+ :param keep_exact_integers: leave nullable integer dimension columns
holding
+ values beyond ``JS_MAX_INTEGER`` as exact Python integers instead
of
+ rounding them to float64; only for frames that are not processed
further
:return: Normalized dataframe
"""
sql_shifted_temporal_labels = sql_shifted_temporal_labels or set()
@@ -2789,9 +2794,22 @@ class ExploreMixin: # pylint:
disable=too-many-public-methods
if getattr(self, "enforce_numerical_metrics", True):
dataframe_utils.df_metrics_to_num(df, query_object)
+ # Nullable integer dimensions beyond JS_MAX_INTEGER are set aside and
put
+ # back below: ``infer_objects`` would round them to float64.
+ exact_columns = (
+ {
+ col: df[col]
+ for col in dataframe_utils.lossy_integer_columns(df,
query_object)
+ }
+ if keep_exact_integers
+ else {}
+ )
+
# ``mask`` + ``infer_objects`` rather than ``replace``, which emits a
# FutureWarning about deprecated silent downcasting on object columns.
df = df.mask(df.isin([np.inf, -np.inf])).infer_objects()
+ for col, series in exact_columns.items():
+ df[col] = series
return df
@@ -2820,6 +2838,9 @@ class ExploreMixin: # pylint:
disable=too-many-public-methods
df,
query_object,
result.sql_shifted_temporal_labels,
+ keep_exact_integers=dataframe_utils.can_keep_exact_integers(
+ query_object
+ ),
)
# Process time offsets if requested
diff --git a/superset/utils/excel.py b/superset/utils/excel.py
index 437cd6739dd..5270e458106 100644
--- a/superset/utils/excel.py
+++ b/superset/utils/excel.py
@@ -18,6 +18,7 @@ import io
from datetime import datetime
from typing import Any, Optional
+import numpy as np
import pandas as pd
from superset.utils.core import GenericDataType
@@ -196,6 +197,20 @@ def df_to_excel(
return output.getvalue()
+def _large_number_as_text(value: Any) -> Any:
+ """
+ Return numbers Excel cannot hold exactly as text, everything else
unchanged.
+
+ NumPy integers are boxed to Python integers first: ``abs`` of the NumPy
+ minimum would overflow and ``str`` of a rounded float would lose digits.
+ """
+ if isinstance(value, np.integer):
+ value = int(value)
+ if isinstance(value, (int, float)) and abs(value) > 10**15:
+ return str(value)
+ return value
+
+
def apply_column_types(
df: pd.DataFrame, column_types: list[GenericDataType]
) -> pd.DataFrame:
@@ -219,19 +234,23 @@ def apply_column_types(
series = df.iloc[:, idx]
if column_type == GenericDataType.NUMERIC:
try:
- # Nullable integers retain exact values alongside blank totals.
- # Convert to object before apply, which would otherwise coerce
- # nullable integers through float64 during scalar mapping.
- series = pd.to_numeric(series,
dtype_backend="numpy_nullable").astype(
- object
- )
+ # Object columns of integers already hold exact Python (or
NumPy)
+ # integers, including values beyond int64 that ``to_numeric``
+ # would round to float64 when the column has nulls: leave them.
+ if not (
+ series.dtype == object
+ and pd.api.types.infer_dtype(series, skipna=True) ==
"integer"
+ ):
+ # Nullable integers retain exact values alongside blank
+ # totals. Convert to object before apply, which would
+ # otherwise coerce nullable integers through float64 during
+ # scalar mapping.
+ series = pd.to_numeric(
+ series, dtype_backend="numpy_nullable"
+ ).astype(object)
# if the number is too large, convert it to a string
# Excel does not support numbers larger than 10^15
- series = series.apply(
- lambda x: (
- str(x) if isinstance(x, (int, float)) and abs(x) >
10**15 else x
- )
- )
+ series = series.apply(_large_number_as_text)
except ValueError:
series = series.astype(str)
elif isinstance(series.dtype, pd.DatetimeTZDtype):
diff --git a/tests/unit_tests/charts/test_client_processing.py
b/tests/unit_tests/charts/test_client_processing.py
index e3bf7965748..b00b151866d 100644
--- a/tests/unit_tests/charts/test_client_processing.py
+++ b/tests/unit_tests/charts/test_client_processing.py
@@ -5338,3 +5338,44 @@ def
test_pivot_mixed_totals_preserve_integer_xlsx(combine: bool, transpose: bool
assert any(cell.value == str(large) and cell.data_type == "s" for cell in
cells)
assert any(cell.value == 3 and cell.data_type == "n" for cell in cells)
assert not any(str(cell.value).startswith(str(large - 1)) for cell in
cells)
+
+
[email protected]("apply_number_format", [False, True])
+def test_pivot_mixed_str_int_big_integer_dimension(apply_number_format: bool)
-> None:
+ """A dimension that mixes exact big-integer strings, numbers and nulls (the
+ shape the chart-data API uses for integers beyond 2**53 - 1) pivots without
+ error and keeps every label exact."""
+ wire = [
+ {"big": "4611686018427387905", "metric": 1},
+ {"big": None, "metric": 2},
+ {"big": 7, "metric": 3},
+ {"big": "4611686018427387905", "metric": 4},
+ {"big": "-4611686018427387905", "metric": 5},
+ ]
+ df = pd.DataFrame.from_dict(wire)
+ form_data = {
+ "groupbyRows": ["big"],
+ "groupbyColumns": [],
+ "metrics": ["metric"],
+ "aggregateFunction": "Sum",
+ "rowTotals": True,
+ "colTotals": True,
+ }
+
+ pivoted = pivot_table_v2(
+ df.copy(), form_data, apply_number_format=apply_number_format
+ )
+ unformatted = table(df.copy(), {"all_columns": ["big", "metric"]}, None,
True)
+
+ sums = {
+ label: str(value)
+ for label, value, *_ in pivoted.reset_index().itertuples(index=False)
+ if isinstance(label, str)
+ }
+ assert sums == {
+ "7": "3",
+ "-4611686018427387905": "5",
+ "4611686018427387905": "5",
+ "Total (Sum)": "15",
+ }
+ assert unformatted["big"].tolist() == df["big"].tolist()
diff --git a/tests/unit_tests/charts/test_exact_integer_chart_data.py
b/tests/unit_tests/charts/test_exact_integer_chart_data.py
new file mode 100644
index 00000000000..afacc368702
--- /dev/null
+++ b/tests/unit_tests/charts/test_exact_integer_chart_data.py
@@ -0,0 +1,824 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""
+Chart data over a real database: nullable large integers stay exact in plain
+queries and keep today's float behaviour wherever Python-side processing runs.
+"""
+
+from __future__ import annotations
+
+import copy
+import io
+import sqlite3
+from collections.abc import Iterator
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+from unittest.mock import patch
+
+import pandas as pd
+import pyarrow as pa
+import pytest
+from flask_caching import Cache
+from openpyxl import load_workbook
+from sqlalchemy.orm.session import Session
+
+from superset.app import SupersetApp
+from superset.charts.client_processing import apply_client_processing
+from superset.charts.schemas import ChartDataQueryContextSchema
+from superset.common.chart_data import ChartDataResultFormat,
ChartDataResultType
+from superset.common.query_context import QueryContext
+from superset.common.query_context_factory import QueryContextFactory
+from superset.common.query_serialization import load_serialized_query,
serialize_query
+from superset.connectors.sqla.models import SqlaTable, SqlMetric, TableColumn
+from superset.constants import CacheRegion
+from superset.models.core import Database
+from superset.utils import json
+
+LARGE = 333466480986292423
+
+
[email protected]
+def table(session: Session, tmp_path: Path) -> SqlaTable:
+ """A table whose ``big`` column is a nullable BIGINT holding a large
value."""
+ path = tmp_path / "chart_data.db"
+ connection = sqlite3.connect(path)
+ connection.execute(
+ "CREATE TABLE t (id INTEGER, grp TEXT, ts DATETIME, big BIGINT, val
INTEGER)"
+ )
+ connection.executemany(
+ "INSERT INTO t VALUES (?, ?, ?, ?, ?)",
+ [
+ (1, "a", "2024-01-01 00:00:00", LARGE, 10),
+ (2, "a", "2024-01-02 00:00:00", None, 20),
+ (3, "b", "2024-01-03 00:00:00", 1000, 30),
+ (4, "b", "2024-01-04 00:00:00", LARGE, 40),
+ (5, "c", "2024-01-05 00:00:00", None, 50),
+ ],
+ )
+ connection.commit()
+ connection.close()
+
+ SqlaTable.metadata.create_all(session.get_bind()) # pylint:
disable=no-member
+ database = Database(database_name="db", sqlalchemy_uri=f"sqlite:///{path}")
+ sqla_table = SqlaTable(
+ table_name="t",
+ database=database,
+ columns=[
+ TableColumn(column_name="id", type="INTEGER"),
+ TableColumn(column_name="grp", type="TEXT", groupby=True),
+ TableColumn(column_name="ts", type="DATETIME", is_dttm=True),
+ TableColumn(column_name="big", type="BIGINT", verbose_name="Big
Value"),
+ TableColumn(column_name="val", type="INTEGER"),
+ ],
+ metrics=[
+ SqlMetric(
+ metric_name="sum_val", expression="SUM(val)",
verbose_name="Sum Val"
+ ),
+ SqlMetric(
+ metric_name="max_big", expression="MAX(big)",
verbose_name="Max Big"
+ ),
+ ],
+ )
+ session.add(database)
+ session.add(sqla_table)
+ session.flush()
+ # ``changed_on`` is part of every cache key; settle it so keys are stable
+ sqla_table.changed_on = datetime(2024, 1, 1)
+ session.flush()
+ return sqla_table
+
+
+def create_context(
+ table: SqlaTable,
+ queries: list[dict[str, Any]],
+ result_type: ChartDataResultType = ChartDataResultType.FULL,
+ result_format: ChartDataResultFormat = ChartDataResultFormat.JSON,
+ form_data: dict[str, Any] | None = None,
+ force: bool = True,
+) -> QueryContext:
+ return QueryContextFactory().create(
+ datasource={"type": "table", "id": table.id},
+ queries=queries,
+ form_data=form_data or {},
+ result_type=result_type,
+ result_format=result_format,
+ force=force,
+ )
+
+
+def run(table: SqlaTable, queries: list[dict[str, Any]], **kwargs: Any) -> Any:
+ """Return the first query's payload for a freshly built context."""
+ payload = create_context(table, queries, **kwargs).get_payload()
+ return payload["queries"][0]
+
+
+def rows(payload: dict[str, Any]) -> list[dict[str, Any]]:
+ return sorted(payload["data"], key=lambda row: json.dumps(row,
default=str))
+
+
+def cells(data: bytes) -> list[list[Any]]:
+ sheet = load_workbook(io.BytesIO(data)).active
+ return [[cell.value for cell in row] for row in sheet.iter_rows()]
+
+
+def schema_context(
+ table: SqlaTable,
+ queries: list[dict[str, Any]],
+ result_type: ChartDataResultType,
+ form_data: dict[str, Any] | None = None,
+ result_format: ChartDataResultFormat = ChartDataResultFormat.JSON,
+) -> QueryContext:
+ """Build a context the way the chart-data API does for a saved chart."""
+ body = {
+ "datasource": {"id": table.id, "type": "table"},
+ "queries": copy.deepcopy(queries),
+ "form_data": form_data or {},
+ "result_format": result_format.value,
+ "result_type": result_type.value,
+ }
+ schema = ChartDataQueryContextSchema()
+ schema.query_context_factory = QueryContextFactory()
+ return schema.load(body)
+
+
+def client_processed(
+ table: SqlaTable,
+ queries: list[dict[str, Any]],
+ form_data: dict[str, Any],
+ result_format: ChartDataResultFormat = ChartDataResultFormat.JSON,
+) -> Any:
+ """What a report receives: the client-processed result of a saved chart."""
+ query_context = schema_context(
+ table, queries, ChartDataResultType.POST_PROCESSED, form_data,
result_format
+ )
+ processed = apply_client_processing(
+ query_context.get_payload(), form_data, query_context.datasource
+ )
+ data = processed["queries"][0]["data"]
+ return cells(data) if result_format == ChartDataResultFormat.XLSX else data
+
+
+QUERY = {"row_limit": 100}
+
+
+def test_plain_dimension_queries_keep_nullable_big_integers_exact(
+ app_context: None, table: SqlaTable
+) -> None:
+ raw = rows(run(table, [{**QUERY, "columns": ["big"]}]))
+ assert raw == [
+ {"big": "333466480986292423"},
+ {"big": "333466480986292423"},
+ {"big": 1000},
+ {"big": None},
+ {"big": None},
+ ]
+
+ aggregate = rows(
+ run(table, [{**QUERY, "columns": ["big"], "metrics": ["sum_val",
"max_big"]}])
+ )
+ # the dimension is exact; the saved metrics (one of them nullable) are
+ # normalized as before, whatever their verbose names
+ assert aggregate == [
+ {"big": "333466480986292423", "sum_val": 50, "max_big":
3.334664809862924e17},
+ {"big": 1000, "sum_val": 30, "max_big": 1000.0},
+ {"big": None, "sum_val": 70, "max_big": None},
+ ]
+
+
[email protected](
+ "result_type, queries",
+ [
+ pytest.param(
+ ChartDataResultType.SAMPLES, [{**QUERY, "columns": ["big"]}],
id="samples"
+ ),
+ pytest.param(
+ ChartDataResultType.DRILL_DETAIL, [{**QUERY}], id="drill-to-detail"
+ ),
+ ],
+)
+def test_sample_queries_keep_nullable_big_integers_exact(
+ app_context: None,
+ table: SqlaTable,
+ result_type: ChartDataResultType,
+ queries: list[dict[str, Any]],
+) -> None:
+ payload = run(table, queries, result_type=result_type)
+
+ assert sorted((row["big"] for row in payload["data"]), key=repr) == sorted(
+ ["333466480986292423", "333466480986292423", 1000, None, None],
key=repr
+ )
+
+
+def test_grouping_sets_levels_keep_nullable_big_integers_exact(
+ app_context: None, table: SqlaTable
+) -> None:
+ """SQLite has no ``GROUPING SETS``, so this runs the per-level fallback."""
+ query = {
+ **QUERY,
+ "columns": ["grp", "big"],
+ "metrics": ["sum_val"],
+ "grouping_sets": [["grp", "big"], ["grp"]],
+ }
+
+ data = run(table, [query])["data"]
+
+ assert {row["big"] for row in data} == {"333466480986292423", 1000, None}
+
+
+def test_exports_keep_nullable_big_integers_exact(
+ app_context: None, table: SqlaTable
+) -> None:
+ query = {**QUERY, "columns": ["big", "grp"]}
+
+ csv_data = run(table, [query],
result_format=ChartDataResultFormat.CSV)["data"]
+ xlsx_data = run(table, [query],
result_format=ChartDataResultFormat.XLSX)["data"]
+ arrow_data = run(table, [query],
result_format=ChartDataResultFormat.ARROW)["data"]
+
+ assert csv_data.decode("utf-8-sig") == (
+ "Big
Value,grp\n333466480986292423,a\n,a\n1000,b\n333466480986292423,b\n,c\n"
+ )
+ assert cells(xlsx_data) == [
+ ["Big Value", "grp"],
+ ["333466480986292423", "a"],
+ [None, "a"],
+ [1000, "b"],
+ ["333466480986292423", "b"],
+ [None, "c"],
+ ]
+ arrow = pa.ipc.open_stream(arrow_data).read_all()
+ assert arrow.schema.field("big").type == pa.int64()
+ assert arrow.column("big").to_pylist() == [LARGE, None, 1000, LARGE, None]
+
+
+def test_metrics_stay_float(app_context: None, table: SqlaTable) -> None:
+ by_group = rows(run(table, [{**QUERY, "columns": ["grp"], "metrics":
["max_big"]}]))
+
+ assert by_group == [
+ {"grp": "a", "max_big": 3.334664809862924e17},
+ {"grp": "b", "max_big": 3.334664809862924e17},
+ {"grp": "c", "max_big": None},
+ ]
+
+
+def test_post_processed_dimension_stays_float(
+ app_context: None, table: SqlaTable
+) -> None:
+ cumulative = [
+ {
+ "operation": "cum",
+ "options": {"operator": "sum", "columns": {"sum_val": "sum_val"}},
+ }
+ ]
+
+ post_processing = rows(
+ run(
+ table,
+ [
+ {
+ **QUERY,
+ "columns": ["grp", "big"],
+ "metrics": ["sum_val"],
+ "post_processing": cumulative,
+ }
+ ],
+ )
+ )
+ post_processed_context = rows(
+ run(
+ table,
+ [{**QUERY, "columns": ["big"]}],
+ result_type=ChartDataResultType.POST_PROCESSED,
+ )
+ )
+
+ assert post_processing == [
+ {"grp": "a", "big": 3.334664809862924e17, "sum_val": 30},
+ {"grp": "a", "big": None, "sum_val": 20},
+ {"grp": "b", "big": 1000.0, "sum_val": 60},
+ {"grp": "b", "big": 3.334664809862924e17, "sum_val": 100},
+ {"grp": "c", "big": None, "sum_val": 150},
+ ]
+ assert post_processed_context == [
+ {"big": 1000.0},
+ {"big": 3.334664809862924e17},
+ {"big": 3.334664809862924e17},
+ {"big": None},
+ {"big": None},
+ ]
+
+
+def test_contribution_totals_unchanged(app_context: None, table: SqlaTable) ->
None:
+ contribution = [
+ {
+ "operation": "contribution",
+ "options": {
+ "orientation": "column",
+ "columns": ["sum_val"],
+ "rename_columns": ["%sum_val"],
+ },
+ }
+ ]
+ queries = [
+ {
+ **QUERY,
+ "columns": ["big"],
+ "metrics": ["sum_val"],
+ "post_processing": contribution,
+ },
+ {**QUERY, "metrics": ["sum_val"]},
+ ]
+
+ payload = create_context(table, queries).get_payload()
+
+ assert [rows(query) for query in payload["queries"]] == [
+ [
+ {"big": 1000.0, "sum_val": 30, "%sum_val": 0.2},
+ {"big": 3.334664809862924e17, "sum_val": 50, "%sum_val": 1 / 3},
+ {"big": None, "sum_val": 70, "%sum_val": 0.4666666666666667},
+ ],
+ [{"sum_val": 150}],
+ ]
+
+
+def test_time_offset_query_stays_float_and_null_keys_join(
+ app_context: None, table: SqlaTable
+) -> None:
+ """The row whose dimension is NULL still finds its offset value."""
+ query = {
+ **QUERY,
+ "columns": ["big"],
+ "metrics": ["sum_val"],
+ "time_offsets": ["3 days ago"],
+ "granularity": "ts",
+ "time_range": "2024-01-04 : 2024-01-06",
+ "is_timeseries": True,
+ "extras": {"time_grain_sqla": "P1D"},
+ }
+
+ assert rows(run(table, [query])) == [
+ {
+ "__timestamp": pd.Timestamp("2024-01-04"),
+ "big": 3.334664809862924e17,
+ "sum_val": 40,
+ "sum_val__3 days ago": 10,
+ },
+ {
+ "__timestamp": pd.Timestamp("2024-01-05"),
+ "big": None,
+ "sum_val": 50,
+ "sum_val__3 days ago": 20,
+ },
+ ]
+
+
+TABLE_RAW = {
+ "viz_type": "table",
+ "query_mode": "raw",
+ "all_columns": ["big"],
+ "metrics": [],
+}
+TABLE_AGGREGATE = {
+ "viz_type": "table",
+ "query_mode": "aggregate",
+ "groupby": ["big"],
+ "metrics": ["sum_val"],
+}
+PIVOT = {
+ "viz_type": "pivot_table_v2",
+ "groupbyRows": ["big"],
+ "groupbyColumns": [],
+ "metrics": ["sum_val"],
+ "aggregateFunction": "Sum",
+ "rowTotals": True,
+ "colTotals": True,
+ "transposePivot": False,
+ "combineMetric": False,
+}
+RAW_QUERY = [{**QUERY, "columns": ["big"]}]
+AGGREGATE_QUERY = [{**QUERY, "columns": ["big"], "metrics": ["sum_val"]}]
+
+
[email protected](
+ "form_data, queries, result_format, expected",
+ [
+ pytest.param(
+ {**TABLE_RAW, "column_config": {"big": {"d3NumberFormat":
",.2f"}}},
+ RAW_QUERY,
+ ChartDataResultFormat.JSON,
+ {
+ "Big Value": {
+ 0: "333,466,480,986,292,416.00",
+ 1: "",
+ 2: "1,000.00",
+ 3: "333,466,480,986,292,416.00",
+ 4: "",
+ }
+ },
+ id="raw-table-number-format",
+ ),
+ pytest.param(
+ {
+ **TABLE_RAW,
+ "column_config": {
+ "big": {
+ "currencyFormat": {"symbol": "USD", "symbolPosition":
"prefix"}
+ }
+ },
+ },
+ RAW_QUERY,
+ ChartDataResultFormat.JSON,
+ {"Big Value": {0: "$ 333P", 1: "", 2: "$ 1k", 3: "$ 333P", 4: ""}},
+ id="raw-table-currency",
+ ),
+ pytest.param(
+ {**TABLE_AGGREGATE, "column_config": {"big": {"d3NumberFormat":
",.2f"}}},
+ AGGREGATE_QUERY,
+ ChartDataResultFormat.JSON,
+ {
+ "Big Value": {0: "", 1: "1,000.00", 2:
"333,466,480,986,292,416.00"},
+ "Sum Val": {0: "70", 1: "30", 2: "50"},
+ },
+ id="aggregate-table-number-format",
+ ),
+ pytest.param(
+ PIVOT,
+ AGGREGATE_QUERY,
+ ChartDataResultFormat.JSON,
+ {
+ "Sum Val": {
+ "1000.0": "30",
+ "3.334664809862924e+17": "50",
+ "nan": "70",
+ "Total (Sum)": "150",
+ },
+ "Total (Sum)": {
+ "1000.0": "30",
+ "3.334664809862924e+17": "50",
+ "nan": "70",
+ "Total (Sum)": "150",
+ },
+ },
+ id="pivot-table",
+ ),
+ pytest.param(
+ TABLE_RAW,
+ RAW_QUERY,
+ ChartDataResultFormat.CSV,
+ 'Big
Value\n3.334664809862924e+17\n""\n1000.0\n3.334664809862924e+17\n""\n',
+ id="raw-table-csv",
+ ),
+ pytest.param(
+ TABLE_RAW,
+ RAW_QUERY,
+ ChartDataResultFormat.XLSX,
+ [
+ ["Big Value"],
+ ["3.334664809862924e+17"],
+ [None],
+ [1000],
+ ["3.334664809862924e+17"],
+ ],
+ id="raw-table-xlsx",
+ ),
+ pytest.param(
+ PIVOT,
+ AGGREGATE_QUERY,
+ ChartDataResultFormat.CSV,
+ ",Sum Val,Total (Sum)\n1000.0,30,30\n3.334664809862924e+17,50,50\n"
+ "nan,70,70\nTotal (Sum),150,150\n",
+ id="pivot-table-csv",
+ ),
+ pytest.param(
+ PIVOT,
+ AGGREGATE_QUERY,
+ ChartDataResultFormat.XLSX,
+ [
+ [None, "Sum Val", "Total (Sum)"],
+ ["1000.0", 30, 30],
+ ["3.334664809862924e+17", 50, 50],
+ ["nan", 70, 70],
+ ["Total (Sum)", 150, 150],
+ ],
+ id="pivot-table-xlsx",
+ ),
+ ],
+)
+def test_post_processed_table_formatting_matches_float_baseline(
+ app_context: None,
+ table: SqlaTable,
+ form_data: dict[str, Any],
+ queries: list[dict[str, Any]],
+ result_format: ChartDataResultFormat,
+ expected: Any,
+) -> None:
+ """Report (client-processed) Tables and Pivot Tables are formatted from
float
+ columns exactly as before: a large nullable dimension must not reach client
+ processing as an unformattable object column."""
+ assert client_processed(table, queries, form_data, result_format) ==
expected
+
+
[email protected](
+ "context_type", [ChartDataResultType.FULL,
ChartDataResultType.POST_PROCESSED]
+)
+def test_queries_inherit_the_context_result_type(
+ app_context: None, table: SqlaTable, context_type: ChartDataResultType
+) -> None:
+ """Saved charts, report/POST bodies and async rebuilds all hand each query
the
+ context's result type, so the per-query value that gates exact integers
(and
+ is part of the cache key) always agrees with the context."""
+ queries = [{**QUERY, "columns": ["big"]}]
+ form_data = {"viz_type": "table", "result_type": context_type.value}
+ contexts = {
+ "saved-chart": schema_context(table, queries, context_type),
+ "report-body": schema_context(table, queries, context_type, form_data),
+ }
+ contexts["async-rebuild"] = load_serialized_query(
+ serialize_query(contexts["saved-chart"], 0)
+ )
+
+ for context in contexts.values():
+ assert context.result_type == context_type
+ assert context.queries[0].result_type == context_type
+ keys = {
+ name: context.queries[0].cache_key(datasource=table.uid)
+ for name, context in contexts.items()
+ }
+ assert len(set(keys.values())) == 1
+
+
+def test_full_and_post_processed_requests_use_different_cache_keys(
+ app_context: None, table: SqlaTable
+) -> None:
+ queries = [{**QUERY, "columns": ["big"]}]
+
+ full = schema_context(table, queries, ChartDataResultType.FULL)
+ post_processed = schema_context(table, queries,
ChartDataResultType.POST_PROCESSED)
+
+ assert full.queries[0].cache_key(datasource=table.uid) !=
post_processed.queries[
+ 0
+ ].cache_key(datasource=table.uid)
+
+
+def test_mixed_result_type_query_shares_the_plain_cache_key(
+ app_context: None, table: SqlaTable
+) -> None:
+ """Documented residual: a query that sets its own ``result_type`` keeps it
+ under a differently typed context. Its cache key is the plain request's, so
+ it is gated (and cached) as that plain request: exact and deterministic."""
+ mixed = [{**QUERY, "columns": ["big"], "result_type": "full"}]
+ plain = [{**QUERY, "columns": ["big"]}]
+
+ mixed_context = schema_context(table, mixed,
ChartDataResultType.POST_PROCESSED)
+ plain_context = schema_context(table, plain, ChartDataResultType.FULL)
+
+ assert mixed_context.result_type == ChartDataResultType.POST_PROCESSED
+ assert mixed_context.queries[0].result_type == ChartDataResultType.FULL
+ assert mixed_context.queries[0].cache_key(
+ datasource=table.uid
+ ) == plain_context.queries[0].cache_key(datasource=table.uid)
+ assert mixed_context.get_payload()["queries"][0]["data"][0] == {
+ "big": "333466480986292423"
+ }
+
+
+MIXED_TABLE_RAW = {
+ "viz_type": "table",
+ "query_mode": "raw",
+ "all_columns": ["big", "val"],
+ "metrics": [],
+ "column_config": {
+ "big": {"d3NumberFormat": ",.2f"},
+ "val": {"currencyFormat": {"symbol": "USD", "symbolPosition":
"prefix"}},
+ },
+}
+MIXED_TABLE_AGGREGATE = {
+ **TABLE_AGGREGATE,
+ "column_config": {
+ "big": {"d3NumberFormat": ",.2f"},
+ "sum_val": {"d3NumberFormat": ",.2f"},
+ },
+}
+MIXED_PIVOT = {**PIVOT, "valueFormat": ",.2f"}
+PIVOT_TOTALS = {"a": "30.00", "b": "70.00", "c": "50.00", "Total (Sum)":
"150.00"}
+
+
[email protected](
+ "form_data, query, control, mixed",
+ [
+ pytest.param(
+ MIXED_TABLE_RAW,
+ {**QUERY, "columns": ["big", "val"]},
+ {
+ "Big Value": {
+ 0: "333,466,480,986,292,416.00",
+ 1: "",
+ 2: "1,000.00",
+ 3: "333,466,480,986,292,416.00",
+ 4: "",
+ },
+ "val": {0: "$ 10", 1: "$ 20", 2: "$ 30", 3: "$ 40", 4: "$ 50"},
+ },
+ {
+ "Big Value": {
+ 0: "333466480986292423",
+ 1: None,
+ 2: 1000,
+ 3: "333466480986292423",
+ 4: None,
+ },
+ "val": {0: "$ 10", 1: "$ 20", 2: "$ 30", 3: "$ 40", 4: "$ 50"},
+ },
+ id="raw-table",
+ ),
+ pytest.param(
+ MIXED_TABLE_AGGREGATE,
+ {**QUERY, "columns": ["big"], "metrics": ["sum_val"]},
+ {
+ "Big Value": {0: "", 1: "1,000.00", 2:
"333,466,480,986,292,416.00"},
+ "Sum Val": {0: "70.00", 1: "30.00", 2: "50.00"},
+ },
+ {
+ "Big Value": {0: None, 1: 1000, 2: "333466480986292423"},
+ "Sum Val": {0: "70.00", 1: "30.00", 2: "50.00"},
+ },
+ id="aggregate-table",
+ ),
+ pytest.param(
+ MIXED_PIVOT,
+ {**QUERY, "columns": ["big"], "metrics": ["sum_val"]},
+ {
+ "Sum Val": {
+ "1000.0": "30.00",
+ "3.334664809862924e+17": "50.00",
+ "nan": "70.00",
+ "Total (Sum)": "150.00",
+ },
+ "Total (Sum)": {
+ "1000.0": "30.00",
+ "3.334664809862924e+17": "50.00",
+ "nan": "70.00",
+ "Total (Sum)": "150.00",
+ },
+ },
+ {
+ "Sum Val": {
+ "1000": "30.00",
+ "333466480986292423": "50.00",
+ "nan": "70.00",
+ "Total (Sum)": "150.00",
+ },
+ "Total (Sum)": {
+ "1000": "30.00",
+ "333466480986292423": "50.00",
+ "nan": "70.00",
+ "Total (Sum)": "150.00",
+ },
+ },
+ id="pivot-table-rows",
+ ),
+ pytest.param(
+ {**MIXED_PIVOT, "groupbyRows": ["grp"], "groupbyColumns": ["big"]},
+ {**QUERY, "columns": ["grp", "big"], "metrics": ["sum_val"]},
+ {
+ "Sum Val 1000.0": {
+ "a": "",
+ "b": "30.00",
+ "c": "",
+ "Total (Sum)": "30.00",
+ },
+ "Sum Val 3.334664809862924e+17": {
+ "a": "10.00",
+ "b": "40.00",
+ "c": "",
+ "Total (Sum)": "50.00",
+ },
+ "Sum Val nan": {
+ "a": "20.00",
+ "b": "",
+ "c": "50.00",
+ "Total (Sum)": "70.00",
+ },
+ "Sum Val Subtotal": PIVOT_TOTALS,
+ "Total (Sum)": PIVOT_TOTALS,
+ },
+ {
+ "Sum Val 1000": {
+ "a": "",
+ "b": "30.00",
+ "c": "",
+ "Total (Sum)": "30.00",
+ },
+ "Sum Val 333466480986292423": {
+ "a": "10.00",
+ "b": "40.00",
+ "c": "",
+ "Total (Sum)": "50.00",
+ },
+ "Sum Val nan": {
+ "a": "20.00",
+ "b": "",
+ "c": "50.00",
+ "Total (Sum)": "70.00",
+ },
+ "Sum Val Subtotal": PIVOT_TOTALS,
+ "Total (Sum)": PIVOT_TOTALS,
+ },
+ id="pivot-table-columns",
+ ),
+ ],
+)
+def test_full_query_inside_post_processed_context_through_client_processing(
+ app_context: None,
+ table: SqlaTable,
+ form_data: dict[str, Any],
+ query: dict[str, Any],
+ control: dict[str, Any],
+ mixed: dict[str, Any],
+) -> None:
+ """A query that sets ``result_type="full"`` inside a POST_PROCESSED context
+ (the mixed construction) goes through the real client-processing path for a
+ Table and a Pivot Table without error, next to a control request whose
query
+ inherits the context's type.
+
+ Documented residual, asserted as it is: the control request normalizes the
+ nullable large integer to a float, so its column is formatted from the
+ rounded value (``333,466,480,986,292,416.00``). The mixed query is gated
as a
+ plain request, so the column reaches client processing exact but as an
object
+ column: its digits are intact and its configured number format is skipped
+ (a Table prints ``"333466480986292423"``, ``1000`` and ``None``
unformatted;
+ Pivot Table labels carry the exact digits). Everything else is identical to
+ the control: formatting of the other columns (currency, number format,
+ ``valueFormat``) and every pivot value and total. Changing this is a
+ deliberate decision, for example normalizing an explicit per-query
+ ``result_type`` to the context's in ``QueryContextFactory.create``.
+ """
+ mixed_query = {**query, "result_type": "full"}
+ mixed_context = schema_context(
+ table, [mixed_query], ChartDataResultType.POST_PROCESSED, form_data
+ )
+ assert mixed_context.result_type == ChartDataResultType.POST_PROCESSED
+ assert mixed_context.queries[0].result_type == ChartDataResultType.FULL
+
+ assert client_processed(table, [query], form_data) == control
+ assert client_processed(table, [mixed_query], form_data) == mixed
+
+
[email protected]
+def result_cache(app: SupersetApp, tmp_path: Path) -> Iterator[Cache]:
+ cache = Cache(
+ config={
+ "CACHE_TYPE": "FileSystemCache",
+ "CACHE_DIR": str(tmp_path / "result_cache"),
+ "CACHE_DEFAULT_TIMEOUT": 300,
+ }
+ )
+ cache.init_app(app)
+ regions = {CacheRegion.DEFAULT: cache, CacheRegion.DATA: cache}
+ with patch("superset.common.utils.query_cache_manager._cache", regions):
+ yield cache
+
+
+def test_warm_cache_and_force_cached_keep_each_requests_shape(
+ app_context: None, table: SqlaTable, result_cache: Cache
+) -> None:
+ queries = [{**QUERY, "columns": ["big"]}]
+
+ def df_payload(result_type: ChartDataResultType, force_cached: bool =
False) -> Any:
+ context = create_context(table, queries, result_type=result_type,
force=False)
+ return context.get_df_payload(context.queries[0],
force_cached=force_cached)
+
+ first = df_payload(ChartDataResultType.FULL)
+ warm = df_payload(ChartDataResultType.FULL)
+ force_cached = df_payload(ChartDataResultType.FULL, force_cached=True)
+
+ assert [bool(p["is_cached"]) for p in (first, warm, force_cached)] == [
+ False,
+ True,
+ True,
+ ]
+ for payload in (first, warm, force_cached):
+ assert payload["df"]["big"].dtype == object
+ assert payload["df"]["big"].tolist() == [LARGE, None, 1000, LARGE,
None]
+
+ # the POST_PROCESSED request has its own entry and keeps the float frame
+ post_processed = df_payload(ChartDataResultType.POST_PROCESSED)
+ post_processed_warm = df_payload(ChartDataResultType.POST_PROCESSED)
+ assert not post_processed["is_cached"]
+ assert post_processed_warm["is_cached"]
+ for payload in (post_processed, post_processed_warm):
+ assert payload["df"]["big"].dtype == "float64"
diff --git a/tests/unit_tests/common/test_dataframe_utils.py
b/tests/unit_tests/common/test_dataframe_utils.py
index 51d17f401ed..353072d2847 100644
--- a/tests/unit_tests/common/test_dataframe_utils.py
+++ b/tests/unit_tests/common/test_dataframe_utils.py
@@ -15,10 +15,14 @@
# specific language governing permissions and limitations
# under the License.
import datetime
+from typing import Any
from unittest.mock import MagicMock
import pandas as pd
+import pytest
+from superset.common.chart_data import ChartDataResultType
+from superset.common.query_object import QueryObject
from superset.common.utils import dataframe_utils
@@ -73,3 +77,52 @@ def test_df_metrics_to_num_converts_string_numerics():
assert df["text_col"].dtype == object, "text_col should remain object"
# dim_col: not in metric_names -> should NOT be touched
assert df["dim_col"].dtype == object, "dim_col should not be touched"
+
+
[email protected](
+ "fields, expected",
+ [
+ pytest.param({}, True, id="plain"),
+ pytest.param({"result_type": ChartDataResultType.FULL}, True,
id="full"),
+ pytest.param({"result_type": ChartDataResultType.RESULTS}, True,
id="results"),
+ pytest.param({"result_type": ChartDataResultType.SAMPLES}, True,
id="samples"),
+ pytest.param(
+ {"result_type": ChartDataResultType.DRILL_DETAIL}, True,
id="drill-detail"
+ ),
+ pytest.param({"result_type": None}, True, id="no-result-type"),
+ pytest.param({"grouping_sets": [["a"], []]}, True, id="grouping-sets"),
+ pytest.param(
+ {"post_processing": [{"operation": "cum", "options": {}}]},
+ False,
+ id="post-processing",
+ ),
+ pytest.param({"time_offsets": ["1 week ago"]}, False,
id="time-offsets"),
+ pytest.param(
+ {"result_type": ChartDataResultType.POST_PROCESSED},
+ False,
+ id="post-processed",
+ ),
+ ],
+)
+def test_can_keep_exact_integers(fields: dict[str, Any], expected: bool) ->
None:
+ query_object = QueryObject(datasource=MagicMock(), columns=["a"], **fields)
+
+ assert dataframe_utils.can_keep_exact_integers(query_object) is expected
+
+
+def test_lossy_integer_columns_lists_only_selected_dimensions() -> None:
+ large = 2**60 + 1
+ lossy = pd.Series([large, None], dtype=object)
+ query_object = QueryObject(
+ datasource=MagicMock(), columns=["dim", "small"], metrics=["metric"]
+ )
+ df = pd.DataFrame(
+ {
+ "dim": lossy,
+ "small": pd.Series([7, None], dtype=object),
+ "metric": lossy.copy(),
+ "unselected": lossy.copy(),
+ }
+ )
+
+ assert dataframe_utils.lossy_integer_columns(df, query_object) == ["dim"]
diff --git a/tests/unit_tests/common/test_query_cache_manager.py
b/tests/unit_tests/common/test_query_cache_manager.py
index 05f1fbee09f..46167feb543 100644
--- a/tests/unit_tests/common/test_query_cache_manager.py
+++ b/tests/unit_tests/common/test_query_cache_manager.py
@@ -14,10 +14,18 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from datetime import timedelta
+from pathlib import Path
from unittest.mock import MagicMock, patch
+import numpy as np
+import pandas as pd
+from flask_caching import Cache
+
+from superset.app import SupersetApp
from superset.common.utils.query_cache_manager import QueryCacheManager
from superset.constants import CacheRegion
+from superset.models.helpers import QueryResult
class TestQueryCacheManagerGet:
@@ -73,3 +81,33 @@ class TestQueryCacheManagerGet:
assert result.is_cached is True
assert result.semantic_cache_status == "MISS"
+
+
+def test_kept_big_integer_column_survives_cache_round_trip(
+ app: SupersetApp, tmp_path: Path
+) -> None:
+ """Exact large integers (Python and NumPy scalars, nulls and NaN) in an
object
+ column come back from a real cache backend unchanged."""
+ cache = Cache(
+ config={"CACHE_TYPE": "FileSystemCache", "CACHE_DIR": str(tmp_path /
"cache")}
+ )
+ cache.init_app(app)
+ values = [2**62 + 1, None, np.int64(-(2**63)), float("nan"),
np.int64(2**63 - 1)]
+ df = pd.DataFrame({"value": pd.Series(values, dtype=object)})
+
+ with patch(
+ "superset.common.utils.query_cache_manager._cache",
+ {CacheRegion.DEFAULT: cache, CacheRegion.DATA: cache},
+ ):
+ QueryCacheManager().set_query_result(
+ key="exact-integers",
+ query_result=QueryResult(df=df, query="", duration=timedelta(0)),
+ timeout=300,
+ region=CacheRegion.DATA,
+ )
+ loaded = QueryCacheManager.get(key="exact-integers",
region=CacheRegion.DATA)
+
+ assert loaded.is_cached
+ assert loaded.df["value"].dtype == object
+ assert [type(value) for value in loaded.df["value"]] == [type(v) for v in
values]
+ pd.testing.assert_frame_equal(loaded.df, df)
diff --git a/tests/unit_tests/common/test_query_context_processor.py
b/tests/unit_tests/common/test_query_context_processor.py
index 15c29779946..752d1f2b1c1 100644
--- a/tests/unit_tests/common/test_query_context_processor.py
+++ b/tests/unit_tests/common/test_query_context_processor.py
@@ -2448,6 +2448,56 @@ def
test_grouping_sets_fallback_applies_row_offset_once_globally() -> None:
assert len(result.df) == 2
+def test_grouping_sets_fallback_mixed_levels_exact() -> None:
+ """
+ Concatenating the per-level results must keep an exact large integer exact:
+ one level carries the nullable-integer dimension as an object column,
another
+ has no such column, a third carries it as float64. ``object`` + ``float64``
+ stays an exact ``object`` column (a nullable ``Int64`` would be widened to
+ ``Float64`` and round the value again).
+ """
+ from datetime import timedelta
+
+ import numpy as np
+
+ from superset.common.query_object import QueryObject
+ from superset.models.helpers import QueryResult
+
+ large = 2**62 + 1
+ levels = {
+ 2: pd.DataFrame(
+ {
+ "state": ["CA", "NY"],
+ "big": pd.Series([large, None], dtype=object),
+ "value": [1, 2],
+ }
+ ),
+ 1: pd.DataFrame({"state": ["CA", "NY"], "value": [3, 4]}),
+ 0: pd.DataFrame(
+ {"big": pd.Series([7.0, np.nan], dtype="float64"), "value": [5, 6]}
+ ),
+ }
+ mock_datasource = MagicMock()
+ query_obj = QueryObject(
+ datasource=mock_datasource,
+ columns=["state", "big"],
+ grouping_sets=[["state", "big"], ["state"], []],
+ )
+ processor = QueryContextProcessor(MagicMock())
+ processor._qc_datasource = mock_datasource
+ mock_datasource.get_query_result.side_effect = lambda sub_query:
QueryResult(
+ df=levels[len(sub_query.columns)],
+ query="SELECT 1",
+ duration=timedelta(seconds=0),
+ )
+
+ result = processor._grouping_sets_fallback(query_obj)
+
+ assert result.df["big"].dtype == object
+ assert result.df["big"].iloc[0] == large
+ assert type(result.df["big"].iloc[0]) is int
+
+
def test_relative_offset_preserves_inner_bounds(
processor: QueryContextProcessor,
) -> None:
diff --git a/tests/unit_tests/datasource/test_query_api.py
b/tests/unit_tests/datasource/test_query_api.py
index a20510eda69..6ff7c596901 100644
--- a/tests/unit_tests/datasource/test_query_api.py
+++ b/tests/unit_tests/datasource/test_query_api.py
@@ -19,6 +19,7 @@
from typing import Any
+import numpy as np
import pandas as pd
import pytest
from marshmallow import ValidationError
@@ -173,6 +174,29 @@ def test_arrow_serializer_round_trips() -> None:
pd.testing.assert_frame_equal(restored, df)
[email protected]("scalar", [int, np.int64], ids=["python-int",
"numpy-int64"])
+def test_to_arrow_ipc_object_big_integers_int64(scalar: type) -> None:
+ """Nullable integers kept as exact objects serialize as int64 across the
+ signed range, with nulls preserved."""
+ import pyarrow as pa
+
+ from superset.common.query_context_processor import QueryContextProcessor
+
+ values = [2**63 - 1, None, -(2**63), 2**53 + 1, 7]
+ frame = pd.DataFrame(
+ {
+ "value": pd.Series(
+ [None if v is None else scalar(v) for v in values],
dtype=object
+ )
+ }
+ )
+
+ table =
pa.ipc.open_stream(QueryContextProcessor._to_arrow_ipc(frame)).read_all()
+
+ assert table.schema.field("value").type == pa.int64()
+ assert table.column("value").to_pylist() == values
+
+
def test_chart_data_schema_rejects_arrow() -> None:
"""`ARROW` lives on the shared enum so `get_data` can serialize it, but
`_send_chart_response` has no Arrow branch — accepting it on chart/data
diff --git a/tests/unit_tests/models/normalize_df_large_integer_test.py
b/tests/unit_tests/models/normalize_df_large_integer_test.py
new file mode 100644
index 00000000000..f31df031d57
--- /dev/null
+++ b/tests/unit_tests/models/normalize_df_large_integer_test.py
@@ -0,0 +1,307 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Chart-data results must not round integers beyond 2**53 - 1."""
+
+from datetime import timedelta
+from decimal import Decimal
+from typing import Any
+from unittest.mock import MagicMock
+
+import numpy as np
+import pandas as pd
+import pytest
+
+from superset.common.chart_data import ChartDataResultType
+from superset.common.query_object import QueryObject
+from superset.dataframe import df_to_records
+from superset.models.helpers import ExploreMixin, QueryResult
+from superset.utils import json
+from superset.utils.core import JS_MAX_INTEGER
+
+LARGE_INTEGER = 333466480986292423
+
+
+def _datasource(
+ df: pd.DataFrame | None = None, verbose_map: dict[str, str] | None = None
+) -> MagicMock:
+ """A datasource double with the real ``ExploreMixin`` normalization
bound."""
+ datasource = MagicMock()
+ datasource.offset = 0
+ datasource.enforce_numerical_metrics = True
+ datasource.columns = []
+ datasource.verbose_map = verbose_map or {}
+ datasource.get_column = lambda name: None
+ datasource.query = lambda query_obj: QueryResult(
+ df=df if df is not None else pd.DataFrame(),
+ query="",
+ duration=timedelta(0),
+ )
+ datasource.processing_time_offsets = lambda frame, *args, **kwargs: {
+ "df": frame,
+ "queries": [],
+ }
+ for method in (
+ "_python_date_format",
+ "_collect_dttm_labels",
+ "_offset_only_dttm_cols",
+ "normalize_df",
+ "get_query_result",
+ ):
+ setattr(datasource, method, getattr(ExploreMixin,
method).__get__(datasource))
+ return datasource
+
+
+def _run_query_and_serialize(df: pd.DataFrame, **query_fields: Any) -> str:
+ """Run ``ExploreMixin.get_query_result`` and the chart-data JSON
encoding."""
+ datasource = _datasource(df)
+
+ # a plain query: one selected column, no metrics, no Python-side processing
+ query_object = MagicMock()
+ query_object.columns = ["value"]
+ query_object.column_names = ["value"]
+ query_object.metrics = []
+ query_object.metric_names = []
+ query_object.granularity = None
+ query_object.time_shift = None
+ query_object.post_processing = []
+ query_object.time_offsets = []
+ query_object.result_type = ChartDataResultType.FULL
+ query_object.exec_post_processing = lambda frame: frame
+ for name, value in query_fields.items():
+ setattr(query_object, name, value)
+
+ result = datasource.get_query_result(query_object)
+ return json.dumps(
+ df_to_records(result.df),
+ default=json.json_int_dttm_ser,
+ ignore_nan=True,
+ )
+
+
+def _normalize(
+ df: pd.DataFrame,
+ columns: list[Any],
+ metrics: list[Any] | None = None,
+ verbose_map: dict[str, str] | None = None,
+ **kwargs: Any,
+) -> pd.DataFrame:
+ """Run ``normalize_df`` for a real ``QueryObject`` selecting
``columns``."""
+ datasource = _datasource(verbose_map=verbose_map)
+ query_object = QueryObject(
+ datasource=datasource, columns=columns, metrics=metrics or []
+ )
+ return datasource.normalize_df(df, query_object, **kwargs)
+
+
+def _unchanged_normalization(df: pd.DataFrame) -> pd.DataFrame:
+ """What ``normalize_df`` returned for a column before the keyword
existed."""
+ return df.mask(df.isin([np.inf, -np.inf])).infer_objects()
+
+
+def test_nullable_bigint_column_keeps_every_digit() -> None:
+ # a nullable BIGINT column arrives as an object column of ints and None
+ df = pd.DataFrame({"value": pd.Series([LARGE_INTEGER, None],
dtype=object)})
+
+ payload = _run_query_and_serialize(df)
+
+ assert str(LARGE_INTEGER) in payload
+
+
[email protected]("null", [None, float("nan"), pd.NA], ids=["none",
"nan", "na"])
[email protected]("scalar", [int, np.int64], ids=["python-int",
"numpy-int64"])
+def test_kept_column_wire_contract(scalar: type, null: Any) -> None:
+ """Exact strings beyond 2**53 - 1, ordinary numerics and JSON nulls."""
+ values = [
+ LARGE_INTEGER,
+ null,
+ 7,
+ -LARGE_INTEGER,
+ 2**63 - 1,
+ -(2**63),
+ JS_MAX_INTEGER,
+ JS_MAX_INTEGER + 1,
+ ]
+ column = pd.Series([v if v is null else scalar(v) for v in values],
dtype=object)
+
+ payload = _run_query_and_serialize(pd.DataFrame({"value": column}))
+
+ assert payload == (
+ '[{"value": "333466480986292423"}, {"value": null}, {"value": 7}, '
+ '{"value": "-333466480986292423"}, {"value": "9223372036854775807"}, '
+ '{"value": "-9223372036854775808"}, {"value": 9007199254740991}, '
+ '{"value": "9007199254740992"}]'
+ )
+
+
[email protected](
+ "query_fields, exact",
+ [
+ pytest.param({}, True, id="plain-full"),
+ pytest.param({"result_type": None}, True, id="no-result-type"),
+ pytest.param({"result_type": ChartDataResultType.RESULTS}, True,
id="results"),
+ pytest.param({"result_type": ChartDataResultType.SAMPLES}, True,
id="samples"),
+ pytest.param(
+ {"result_type": ChartDataResultType.DRILL_DETAIL}, True,
id="drill-detail"
+ ),
+ pytest.param({"grouping_sets": [["value"], []]}, True,
id="grouping-sets"),
+ pytest.param(
+ {"post_processing": [{"operation": "cum"}]}, False,
id="post-processing"
+ ),
+ pytest.param({"time_offsets": ["1 week ago"]}, False,
id="time-offsets"),
+ pytest.param(
+ {"result_type": ChartDataResultType.POST_PROCESSED},
+ False,
+ id="post-processed",
+ ),
+ ],
+)
+def test_exact_integers_follow_the_query_gate(
+ query_fields: dict[str, Any], exact: bool
+) -> None:
+ df = pd.DataFrame({"value": pd.Series([LARGE_INTEGER, None],
dtype=object)})
+
+ payload = _run_query_and_serialize(df, **query_fields)
+
+ expected = (
+ '[{"value": "333466480986292423"}, {"value": null}]'
+ if exact
+ else '[{"value": 3.334664809862924e+17}, {"value": null}]'
+ )
+ assert payload == expected
+
+
+def test_normalize_df_defaults_to_float_normalization() -> None:
+ """Callers that do not opt in (e.g. the time-offset subquery) keep
float64."""
+ df = pd.DataFrame({"value": pd.Series([LARGE_INTEGER, None],
dtype=object)})
+
+ result = _normalize(df, ["value"])
+
+ assert result["value"].dtype == np.float64
+ assert result["value"].iloc[0] == float(LARGE_INTEGER)
+
+
+KEPT_VALUES = {
+ "just-beyond-safe": [JS_MAX_INTEGER + 1, None],
+ "beyond-safe-plus-one": [JS_MAX_INTEGER + 2, None],
+ "negative-beyond-safe": [-(JS_MAX_INTEGER + 2), None],
+ "int64-minimum": [-(2**63), None],
+ "int64-maximum": [2**63 - 1, None],
+ "mixed-small-large-negative": [1, JS_MAX_INTEGER + 2, None, -5,
-LARGE_INTEGER],
+ "numpy-scalars": [np.int64(JS_MAX_INTEGER + 2), None, np.int64(-(2**63))],
+ "nan-null": [JS_MAX_INTEGER + 2, float("nan")],
+ "pandas-na-null": [JS_MAX_INTEGER + 2, pd.NA],
+}
+
+NOT_KEPT_VALUES = {
+ "safe-boundary": [JS_MAX_INTEGER, None],
+ "small-only": [7, None],
+ "above-int64": [2**63, None],
+ "uint64-only": [2**64 - 1, None],
+ "uint64-range-with-small": [2**63 + 5, 7, None],
+ "negative-and-above-int64": [-1, 2**63 + 1, None],
+ "beyond-64-bits": [2**64, None],
+ "no-nulls": [JS_MAX_INTEGER + 2, 3],
+ "bool-mixed": [True, JS_MAX_INTEGER + 2, None],
+ "float-mixed": [JS_MAX_INTEGER + 2, 1.5, None],
+ "all-null": [None, None],
+ "decimal": [Decimal(JS_MAX_INTEGER + 2), None],
+ "string": ["a", None],
+}
+
+
[email protected]("values", KEPT_VALUES.values(),
ids=KEPT_VALUES.keys())
+def test_kept_columns_stay_exact_python_integers(values: list[Any]) -> None:
+ df = pd.DataFrame({"value": pd.Series(values, dtype=object)})
+
+ result = _normalize(df, ["value"], keep_exact_integers=True)
+
+ assert result["value"].dtype == object
+ assert [None if pd.isna(v) else v for v in result["value"]] == [
+ None if pd.isna(v) else v for v in values
+ ]
+
+
[email protected]("values", NOT_KEPT_VALUES.values(),
ids=NOT_KEPT_VALUES.keys())
+def test_kept_columns_only_lossy_int64_range(values: list[Any]) -> None:
+ """Every other shape is normalized exactly as before the keyword
existed."""
+ df = pd.DataFrame({"value": pd.Series(values, dtype=object)})
+
+ result = _normalize(df, ["value"], keep_exact_integers=True)
+
+ pd.testing.assert_frame_equal(result, _unchanged_normalization(df))
+
+
+def test_value_above_int64_maximum_stays_float() -> None:
+ df = pd.DataFrame({"value": pd.Series([2**63, None], dtype=object)})
+
+ result = _normalize(df, ["value"], keep_exact_integers=True)
+
+ assert result["value"].dtype == np.float64
+
+
+def test_duplicate_labels_keep_float_normalization() -> None:
+ df = pd.DataFrame(
+ [[JS_MAX_INTEGER + 2, JS_MAX_INTEGER + 3], [None, None]],
+ columns=["value", "value"],
+ dtype=object,
+ )
+
+ result = _normalize(df, ["value"], keep_exact_integers=True)
+
+ pd.testing.assert_frame_equal(result, _unchanged_normalization(df))
+
+
+def test_saved_metric_with_verbose_name_stays_float() -> None:
+ """The frame carries the saved metric's raw name while ``metric_names`` is
+ mapped to the verbose name, so metrics must be told apart by the raw
label."""
+ verbose_map = {"max_big": "Max Big Verbose"}
+ datasource = _datasource(verbose_map=verbose_map)
+ query_object = QueryObject(
+ datasource=datasource, columns=["big"], metrics=["max_big"]
+ )
+ assert query_object.metric_names == ["Max Big Verbose"]
+ nullable = pd.Series([LARGE_INTEGER, None], dtype=object)
+ df = pd.DataFrame({"big": nullable, "max_big": nullable.copy()})
+
+ result = datasource.normalize_df(df, query_object,
keep_exact_integers=True)
+
+ assert result["big"].dtype == object
+ assert result["big"].tolist() == [LARGE_INTEGER, None]
+ assert result["max_big"].dtype == np.float64
+
+
+def test_metrics_only_totals_query_converts_nothing() -> None:
+ """A metrics-only query selects no columns, so nothing qualifies."""
+ df = pd.DataFrame({"sum_big": pd.Series([LARGE_INTEGER, None],
dtype=object)})
+
+ result = _normalize(df, [], metrics=["sum_big"], keep_exact_integers=True)
+
+ assert result["sum_big"].dtype == np.float64
+
+
+def test_columns_not_selected_by_the_query_are_not_kept() -> None:
+ df = pd.DataFrame(
+ {
+ "selected": pd.Series([LARGE_INTEGER, None], dtype=object),
+ "extra": pd.Series([LARGE_INTEGER, None], dtype=object),
+ }
+ )
+
+ result = _normalize(df, ["selected"], keep_exact_integers=True)
+
+ assert result["selected"].dtype == object
+ assert result["extra"].dtype == np.float64
diff --git a/tests/unit_tests/utils/csv_tests.py
b/tests/unit_tests/utils/csv_tests.py
index 3a3210a5171..706f015d22c 100644
--- a/tests/unit_tests/utils/csv_tests.py
+++ b/tests/unit_tests/utils/csv_tests.py
@@ -19,6 +19,7 @@
from typing import Any
from unittest import mock
+import numpy as np
import pandas as pd
import pyarrow as pa
import pytest
@@ -255,6 +256,19 @@ def test_df_to_escaped_csv_preserves_numeric_columns():
assert rows[2] == ["safe", "-20"]
[email protected]("scalar", [int, np.int64], ids=["python-int",
"numpy-int64"])
+def test_df_to_escaped_csv_object_big_integers(scalar: type) -> None:
+ """Nullable big integers keep every digit; null and NaN rows are blank."""
+ values = [scalar(2**53 + 1), None, scalar(7), float("nan"),
scalar(-(2**63))]
+ df = pd.DataFrame(
+ {"value": pd.Series(values, dtype=object), "label": list("abcde")}
+ )
+
+ assert df_to_escaped_csv(df, encoding="utf8", index=False) == (
+
"value,label\n9007199254740993,a\n,b\n7,c\n,d\n-9223372036854775808,e\n"
+ )
+
+
def test_df_to_escaped_csv_non_default_index():
"""
String cells are escaped against the correct rows even when the DataFrame
diff --git a/tests/unit_tests/utils/excel_tests.py
b/tests/unit_tests/utils/excel_tests.py
index 719d07dbd68..ff1718119df 100644
--- a/tests/unit_tests/utils/excel_tests.py
+++ b/tests/unit_tests/utils/excel_tests.py
@@ -18,7 +18,9 @@
import io
from datetime import datetime, timedelta, timezone
+import numpy as np
import pandas as pd
+import pytest
from openpyxl import load_workbook
from pandas.api.types import is_numeric_dtype
@@ -310,3 +312,39 @@ def
test_numeric_column_with_blanks_preserves_large_integers() -> None:
assert sheet["A3"].value is None
assert sheet["A4"].value == 3
assert sheet["A4"].data_type == "n"
+
+
[email protected](
+ "values, expected",
+ [
+ pytest.param(
+ [2**53 + 1, None, 7, -(2**53 + 3)],
+ ["9007199254740993", None, 7, "'-9007199254740995"],
+ id="python-integers",
+ ),
+ pytest.param(
+ [np.int64(2**53 + 1), None, np.int64(7), np.int64(-(2**63))],
+ ["9007199254740993", None, 7, "'-9223372036854775808"],
+ id="numpy-integers",
+ ),
+ pytest.param(
+ [2**63 - 1, np.int64(2**63 - 1), None, np.int64(-7)],
+ ["9223372036854775807", "9223372036854775807", None, -7],
+ id="python-and-numpy-integers",
+ ),
+ ],
+)
+def test_apply_column_types_object_big_integers_exact(
+ values: list[object], expected: list[object]
+) -> None:
+ """Nullable integers (an object column) keep every digit in the workbook.
+
+ Values beyond 10**15 become text, smaller values stay numeric cells, nulls
+ stay blank, and negative text keeps the formula-quoting prefix.
+ """
+ df = pd.DataFrame({"value": pd.Series(values, dtype=object)})
+
+ apply_column_types(df, [GenericDataType.NUMERIC])
+ sheet = load_workbook(io.BytesIO(df_to_excel(df, index=False))).active
+
+ assert [row[0].value for row in sheet.iter_rows(min_row=2)] == expected
