This is an automated email from the ASF dual-hosted git repository.
aminghadersohi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/superset.git
The following commit(s) were added to refs/heads/master by this push:
new 76b62d34fba fix(semantic-layer): preserve nullable integer precision
in chart data (#45126)
76b62d34fba is described below
commit 76b62d34fba99ced85e1ae9949062757e9fa7e4e
Author: Mike Bridge <[email protected]>
AuthorDate: Sat Oct 10 16:21:30 2026 -0600
fix(semantic-layer): preserve nullable integer precision in chart data
(#45126)
---
.../creating-your-first-dashboard.mdx | 13 +
superset/common/utils/dataframe_utils.py | 8 +-
superset/semantic_layers/mapper.py | 20 +-
superset/semantic_layers/models.py | 17 +-
.../utils/pandas_postprocessing/contribution.py | 24 +-
.../semantic_layers/integer_precision_test.py | 366 +++++++++++++++++++++
6 files changed, 441 insertions(+), 7 deletions(-)
diff --git a/docs/docs/using-superset/creating-your-first-dashboard.mdx
b/docs/docs/using-superset/creating-your-first-dashboard.mdx
index 0f0e3449202..3000021ca18 100644
--- a/docs/docs/using-superset/creating-your-first-dashboard.mdx
+++ b/docs/docs/using-superset/creating-your-first-dashboard.mdx
@@ -165,6 +165,19 @@ You can also certify metrics if you'd like for your team
in this view.
- [Blog: Unlocking the Power of Virtual
Datasets](https://preset.io/blog/unlocking-the-power-of-virtual-datasets-in-apache-superset/)
:::
+### Integer precision in semantic views
+
+Semantic-view chart-data responses preserve signed 64-bit integer values even
when the same
+column contains nulls, including time-comparison results. Integers outside
+JavaScript's safe integer range are returned as exact strings; nulls remain
null.
+This applies to values supplied as integers by the semantic provider, not
values
+already rounded to floating point.
+
+Chart calculations such as contribution, rolling windows or differences convert
+metric columns to floating-point numbers first, as they do for SQL datasets, so
+their results follow normal floating-point rounding. Previously cached chart
+results can retain rounded values until the query is refreshed.
+
### Native filters on semantic views
When the `SEMANTIC_LAYERS` feature flag is enabled, Superset can connect to
external semantic layers
diff --git a/superset/common/utils/dataframe_utils.py
b/superset/common/utils/dataframe_utils.py
index d85fabc5ded..062b727cf01 100644
--- a/superset/common/utils/dataframe_utils.py
+++ b/superset/common/utils/dataframe_utils.py
@@ -17,6 +17,7 @@
from __future__ import annotations
import datetime
+from collections.abc import Collection
from typing import Any, Literal, TYPE_CHECKING
import numpy as np
@@ -61,8 +62,13 @@ def full_outer_join_df(
def df_metrics_to_num(df: pd.DataFrame, query_object: QueryObject) -> None:
"""Converting metrics to numeric when pandas.read_sql cannot"""
+ df_columns_to_num(df, query_object.metric_names)
+
+
+def df_columns_to_num(df: pd.DataFrame, columns: Collection[str]) -> None:
+ """Soft-convert the named object columns to numeric, in place."""
for col, dtype in df.dtypes.items():
- if dtype.type == np.object_ and col in query_object.metric_names:
+ if dtype.type == np.object_ and col in columns:
# soft-convert a metric column to numeric only if all
# non-null values look numeric (e.g. ClickHouse returns
# SUM() results as strings). Leaves truly non-numeric
diff --git a/superset/semantic_layers/mapper.py
b/superset/semantic_layers/mapper.py
index 1c055630bb8..c99c8e392f0 100644
--- a/superset/semantic_layers/mapper.py
+++ b/superset/semantic_layers/mapper.py
@@ -172,7 +172,9 @@ def get_results(query_object: QueryObject) -> QueryResult:
main_result = dispatcher(main_query)
main_result = _coerce_empty_result(main_result, main_query)
- main_df = stringify_extension_columns(main_result.results).to_pandas()
+ main_df = stringify_extension_columns(main_result.results).to_pandas(
+ integer_object_nulls=True,
+ )
# Collect all requests (SQL queries, HTTP requests, etc.) for
troubleshooting
all_requests = list(main_result.requests)
@@ -208,7 +210,17 @@ def get_results(query_object: QueryObject) -> QueryResult:
# Add this query's requests to the collection
all_requests.extend(result.requests)
- offset_df = stringify_extension_columns(result.results).to_pandas()
+ offset_df = stringify_extension_columns(result.results).to_pandas(
+ integer_object_nulls=True,
+ )
+ # A missing join match would upcast int64 offset metrics to float64.
+ offset_df = offset_df.astype(
+ {
+ metric: object
+ for metric in metric_names
+ if metric in offset_df and offset_df[metric].dtype.kind == "i"
+ }
+ )
# Handle empty results - add NaN columns directly instead of merging
# This avoids dtype mismatch issues with empty DataFrames
@@ -320,7 +332,9 @@ def map_semantic_result_to_query_result(
return QueryResult(
# Core data
- df=stringify_extension_columns(semantic_result.results).to_pandas(),
+ df=stringify_extension_columns(semantic_result.results).to_pandas(
+ integer_object_nulls=True,
+ ),
query=query_str,
duration=duration,
# Template filters - not applicable to semantic layers
diff --git a/superset/semantic_layers/models.py
b/superset/semantic_layers/models.py
index d5afe5c4b80..15cc74c06f4 100644
--- a/superset/semantic_layers/models.py
+++ b/superset/semantic_layers/models.py
@@ -52,6 +52,7 @@ from superset_core.semantic_layers.view import (
)
from superset.common.query_object import QueryObject
+from superset.common.utils.dataframe_utils import df_columns_to_num
from superset.exceptions import (
InvalidPostProcessingError,
QueryObjectValidationError,
@@ -65,7 +66,7 @@ from superset.semantic_layers.completeness import
provider_completeness
from superset.semantic_layers.mapper import get_results
from superset.semantic_layers.registry import registry
from superset.utils import json
-from superset.utils.core import GenericDataType
+from superset.utils.core import GenericDataType, TIME_COMPARISON
if TYPE_CHECKING:
from superset.superset_typing import ExplorableData, QueryObjectDict
@@ -389,6 +390,20 @@ class SemanticView(AuditMixinNullable, Model):
security_manager.raise_for_unsupported_guest_rls(self)
result: QueryResult = get_results(query_object)
if query_object.post_processing and not result.df.empty:
+ # Nullable integer metrics arrive as exact Python ints;
calculations
+ # need numbers, as for SQL datasets. Plain chart data stays exact.
+ metric_names: list[str] = query_object.metric_names
+ df_columns_to_num(
+ result.df,
+ {
+ *metric_names,
+ *(
+ TIME_COMPARISON.join([metric, offset])
+ for metric in metric_names
+ for offset in query_object.time_offsets
+ ),
+ },
+ )
try:
result.df = query_object.exec_post_processing(result.df)
except InvalidPostProcessingError as ex:
diff --git a/superset/utils/pandas_postprocessing/contribution.py
b/superset/utils/pandas_postprocessing/contribution.py
index fc4a7425343..c4dbc56ec9f 100644
--- a/superset/utils/pandas_postprocessing/contribution.py
+++ b/superset/utils/pandas_postprocessing/contribution.py
@@ -17,10 +17,11 @@
from __future__ import annotations
+from decimal import Decimal
from typing import Any
from flask_babel import gettext as _
-from pandas import DataFrame, MultiIndex
+from pandas import DataFrame, MultiIndex, Series
from pandas.api.types import infer_dtype, is_bool_dtype, is_numeric_dtype
from superset.exceptions import InvalidPostProcessingError
@@ -68,6 +69,23 @@ def _select_arithmetic_columns(df: DataFrame) -> DataFrame:
]
+def _align_total(total: Any, column: Series) -> Any:
+ """
+ Match a supplied contribution total to the column it divides.
+
+ Totals come from a separate query and can stay ``Decimal`` while the
+ column was converted to a numeric dtype; ``float`` and ``Decimal`` don't
+ divide, so such a total is converted to ``float``.
+
+ :param total: Total for the column, as supplied by the totals query.
+ :param column: Column the total divides.
+ :return: The total, converted to ``float`` when needed.
+ """
+ if isinstance(total, Decimal) and is_numeric_dtype(column.dtype):
+ return float(total)
+ return total
+
+
@validate_column_args("columns")
def contribution(
df: DataFrame,
@@ -132,7 +150,9 @@ def contribution(
if total is None or total == 0:
contribution_df[rename_col] = 0
else:
- contribution_df[rename_col] = numeric_df_view[col] / total
+ contribution_df[rename_col] = numeric_df_view[col] /
_align_total(
+ total, numeric_df_view[col]
+ )
else:
numeric_df_view = numeric_df_view / numeric_df_view.values.sum(
axis=0, keepdims=True
diff --git a/tests/unit_tests/semantic_layers/integer_precision_test.py
b/tests/unit_tests/semantic_layers/integer_precision_test.py
new file mode 100644
index 00000000000..0402aa8959b
--- /dev/null
+++ b/tests/unit_tests/semantic_layers/integer_precision_test.py
@@ -0,0 +1,366 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Preserve nullable integer values through semantic chart-data
serialization."""
+
+from decimal import Decimal
+from unittest.mock import MagicMock
+
+import pyarrow as pa
+import pytest
+from flask.testing import FlaskClient
+from pytest_mock import MockerFixture
+from superset_core.semantic_layers.types import (
+ Dimension,
+ Metric,
+ SemanticQuery,
+ SemanticResult,
+)
+from superset_core.semantic_layers.view import SemanticView as ProviderView
+from werkzeug.test import TestResponse
+
+from superset.semantic_layers.models import SemanticLayer, SemanticView
+
+
[email protected]
+def semantic_layer(mocker: MockerFixture) -> SemanticLayer:
+ """Give chart-data views the registered layer required by cache keys."""
+ mocker.patch.dict(
+ "superset.semantic_layers.models.registry",
+ {"fixture": MagicMock(result_cache_version=None)},
+ )
+ return SemanticLayer(type="fixture")
+
+
[email protected]("value", [9007199254740993, -9007199254740993, 2**63
- 1, 7])
[email protected](
+ ("with_offset", "missing_prior_category"),
+ [(False, False), (True, False), (True, True)],
+)
+def test_chart_data_preserves_nullable_integer_precision(
+ client: FlaskClient,
+ full_api_access: None,
+ mocker: MockerFixture,
+ semantic_layer: SemanticLayer,
+ value: int,
+ with_offset: bool,
+ missing_prior_category: bool,
+) -> None:
+ """Serialize exact main/offset integers and nulls through the real host
path."""
+ provider: MagicMock = MagicMock(spec=ProviderView)
+ provider.features = frozenset()
+ provider.selection_identity_version = None
+ provider.uid.return_value = "integer-precision-view"
+ provider.get_dimensions.return_value = {
+ Dimension("category", "category", pa.string()),
+ }
+ provider.get_metrics.return_value = {
+ Metric("amount", "amount", pa.int64(), "SUM(amount)"),
+ }
+ offset_values: list[int | None] = (
+ [value] if missing_prior_category else [None, -value]
+ )
+ provider.get_table.side_effect = [
+ SemanticResult(
+ results=pa.table(
+ {
+ "category": ["A", "B"],
+ "amount": pa.array([value, None], type=pa.int64()),
+ }
+ ),
+ requests=[],
+ ),
+ SemanticResult(
+ results=pa.table(
+ {
+ "category": ["A"] if missing_prior_category else ["A",
"B"],
+ "amount": pa.array(offset_values, type=pa.int64()),
+ }
+ ),
+ requests=[],
+ ),
+ ]
+ view: SemanticView = SemanticView(
+ id=7, name="Amounts", configuration="{}", semantic_layer=semantic_layer
+ )
+ view.__dict__["implementation"] = provider
+ mocker.patch(
+ "superset.common.query_context_factory.DatasourceDAO.get_datasource",
+ return_value=view,
+ )
+ mocker.patch("superset.common.query_context.QueryContext.raise_for_access")
+ mocker.patch("superset.security_manager.raise_for_unsupported_guest_rls")
+ mocker.patch(
+
"superset.common.query_context_processor.QueryContextProcessor.get_cache_timeout",
+ return_value=-1,
+ )
+
+ response: TestResponse = client.post(
+ "/api/v1/chart/data",
+ json={
+ "datasource": {"id": 7, "type": "semantic_view"},
+ "queries": [
+ {
+ "columns": ["category"],
+ "metrics": ["amount"],
+ "time_offsets": ["1 week ago"] if with_offset else [],
+ }
+ ],
+ "result_format": "json",
+ "result_type": "full",
+ "force": True,
+ },
+ )
+
+ assert response.status_code == 200, response.get_data(as_text=True)
+ expected: list[dict[str, str | int | None]] = [
+ {"category": "A", "amount": str(value) if abs(value) > 2**53 - 1 else
value},
+ {"category": "B", "amount": None},
+ ]
+ if with_offset:
+ if missing_prior_category:
+ expected[0]["amount__1 week ago"] = (
+ str(value) if abs(value) > 2**53 - 1 else value
+ )
+ expected[1]["amount__1 week ago"] = None
+ else:
+ expected[0]["amount__1 week ago"] = None
+ expected[1]["amount__1 week ago"] = (
+ str(-value) if abs(value) > 2**53 - 1 else -value
+ )
+ assert response.get_json()["result"][0]["data"] == expected
+ assert provider.get_table.call_count == (2 if with_offset else 1)
+ if abs(value) > 2**53 - 1:
+ # Check exact quoted wire values, not a numpy float/int equality
comparison.
+ assert f'"{value}"' in response.get_data(as_text=True)
+ if with_offset:
+ offset_value: int = value if missing_prior_category else -value
+ assert f'"{offset_value}"' in response.get_data(as_text=True)
+
+
[email protected](
+ ("post_processing", "time_offsets", "expected"),
+ [
+ (
+ [
+ {
+ "operation": "contribution",
+ "options": {"orientation": "column", "columns":
["amount"]},
+ }
+ ],
+ [],
+ {"amount": [0.25, 0.0, 0.75]},
+ ),
+ (
+ [
+ {
+ "operation": "diff",
+ "options": {"columns": {"amount": "amount"}, "periods": 1},
+ }
+ ],
+ [],
+ {"amount": [None, None, None]},
+ ),
+ (
+ [
+ {
+ "operation": "contribution",
+ "options": {
+ "orientation": "column",
+ "columns": ["amount__1 week ago"],
+ },
+ }
+ ],
+ ["1 week ago"],
+ {"amount__1 week ago": [0.2, 0.0, 0.8]},
+ ),
+ ],
+ ids=["contribution", "diff", "contribution-offset"],
+)
+def test_post_processing_receives_numeric_nullable_integer_metrics(
+ client: FlaskClient,
+ full_api_access: None,
+ mocker: MockerFixture,
+ semantic_layer: SemanticLayer,
+ post_processing: list[dict[str, object]],
+ time_offsets: list[str],
+ expected: dict[str, list[float | None]],
+) -> None:
+ """Metric calculations see numbers, as for SQL datasets, not object
integers."""
+ provider: MagicMock = MagicMock(spec=ProviderView)
+ provider.features = frozenset()
+ provider.selection_identity_version = None
+ provider.uid.return_value = "integer-precision-view"
+ provider.get_dimensions.return_value = {
+ Dimension("category", "category", pa.string()),
+ }
+ provider.get_metrics.return_value = {
+ Metric("amount", "amount", pa.int64(), "SUM(amount)"),
+ }
+ provider.get_table.side_effect = [
+ SemanticResult(
+ results=pa.table(
+ {
+ "category": ["A", "B", "C"],
+ "amount": pa.array([5, None, 15], type=pa.int64()),
+ }
+ ),
+ requests=[],
+ ),
+ SemanticResult(
+ results=pa.table(
+ {
+ "category": ["A", "B", "C"],
+ "amount": pa.array([2, None, 8], type=pa.int64()),
+ }
+ ),
+ requests=[],
+ ),
+ ]
+ view: SemanticView = SemanticView(
+ id=7, name="Amounts", configuration="{}", semantic_layer=semantic_layer
+ )
+ view.__dict__["implementation"] = provider
+ mocker.patch(
+ "superset.common.query_context_factory.DatasourceDAO.get_datasource",
+ return_value=view,
+ )
+ mocker.patch("superset.common.query_context.QueryContext.raise_for_access")
+ mocker.patch("superset.security_manager.raise_for_unsupported_guest_rls")
+ mocker.patch(
+
"superset.common.query_context_processor.QueryContextProcessor.get_cache_timeout",
+ return_value=-1,
+ )
+
+ response: TestResponse = client.post(
+ "/api/v1/chart/data",
+ json={
+ "datasource": {"id": 7, "type": "semantic_view"},
+ "queries": [
+ {
+ "columns": ["category"],
+ "metrics": ["amount"],
+ "time_offsets": time_offsets,
+ "post_processing": post_processing,
+ }
+ ],
+ "result_format": "json",
+ "result_type": "full",
+ "force": True,
+ },
+ )
+
+ assert response.status_code == 200, response.get_data(as_text=True)
+ data: list[dict[str, object]] = response.get_json()["result"][0]["data"]
+ for column, values in expected.items():
+ assert [row[column] for row in data] == values
+
+
[email protected](
+ ("dtype", "values"),
+ [
+ (
+ pa.decimal128(12, 2),
+ [Decimal("5.25"), None, Decimal("15.75"), Decimal("26.25")],
+ ),
+ (pa.int64(), [5, None, 15, 27]),
+ (pa.float64(), [5.25, None, 15.75, 26.25]),
+ ],
+ ids=["decimal", "int", "float"],
+)
+def test_contribution_with_separate_totals_query(
+ client: FlaskClient,
+ full_api_access: None,
+ mocker: MockerFixture,
+ semantic_layer: SemanticLayer,
+ dtype: pa.DataType,
+ values: list[Decimal | int | float | None],
+) -> None:
+ """Contribution divides by totals from a second query of the same
metric."""
+ provider: MagicMock = MagicMock(spec=ProviderView)
+ provider.features = frozenset()
+ provider.selection_identity_version = None
+ provider.uid.return_value = "contribution-totals-view"
+ provider.get_dimensions.return_value = {
+ Dimension("category", "category", pa.string()),
+ }
+ provider.get_metrics.return_value = {
+ Metric("amount", "amount", dtype, "SUM(amount)"),
+ }
+ total: Decimal | int | float = sum(value for value in values if value is
not None)
+ main_result: SemanticResult = SemanticResult(
+ results=pa.table(
+ {
+ "category": ["A", "B", "C", "D"],
+ "amount": pa.array(values, type=dtype),
+ }
+ ),
+ requests=[],
+ )
+ totals_result: SemanticResult = SemanticResult(
+ results=pa.table({"amount": pa.array([total], type=dtype)}),
+ requests=[],
+ )
+
+ def get_table(query: SemanticQuery) -> SemanticResult:
+ return main_result if query.dimensions else totals_result
+
+ provider.get_table.side_effect = get_table
+ view: SemanticView = SemanticView(
+ id=7, name="Amounts", configuration="{}", semantic_layer=semantic_layer
+ )
+ view.__dict__["implementation"] = provider
+ mocker.patch(
+ "superset.common.query_context_factory.DatasourceDAO.get_datasource",
+ return_value=view,
+ )
+ mocker.patch("superset.common.query_context.QueryContext.raise_for_access")
+ mocker.patch("superset.security_manager.raise_for_unsupported_guest_rls")
+ mocker.patch(
+
"superset.common.query_context_processor.QueryContextProcessor.get_cache_timeout",
+ return_value=-1,
+ )
+
+ response: TestResponse = client.post(
+ "/api/v1/chart/data",
+ json={
+ "datasource": {"id": 7, "type": "semantic_view"},
+ "queries": [
+ {
+ "columns": ["category"],
+ "metrics": ["amount"],
+ "post_processing": [
+ {
+ "operation": "contribution",
+ "options": {"orientation": "column", "columns":
["amount"]},
+ }
+ ],
+ },
+ {"columns": [], "metrics": ["amount"], "post_processing": []},
+ ],
+ "result_format": "json",
+ "result_type": "full",
+ "force": True,
+ },
+ )
+
+ assert response.status_code == 200, response.get_data(as_text=True)
+ data: list[dict[str, object]] = response.get_json()["result"][0]["data"]
+ expected: list[float] = [
+ 0.0 if value is None else float(value) / float(total) for value in
values
+ ]
+ assert [row["amount"] for row in data] == pytest.approx(expected)
