This is an automated email from the ASF dual-hosted git repository.
zhengruifeng pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new c3cee12b03dc [SPARK-58553][PS][FOLLOWUP] Do not cast NumPy fmax/fmin
results to double
c3cee12b03dc is described below
commit c3cee12b03dcc605d06c59447f300fce6d371a50
Author: Spenser Sun <[email protected]>
AuthorDate: Fri Aug 28 13:42:56 2026 +0800
[SPARK-58553][PS][FOLLOWUP] Do not cast NumPy fmax/fmin results to double
### What changes were proposed in this pull request?
`_fmax_func` and `_fmin_func` no longer cast their result to double. Every
branch of both expressions returns one of the operands, so the result now keeps
the operands' type instead of being widened.
### Why are the changes needed?
The cast rounds integral values above 2^53 and returns a dtype NumPy does
not use:
```python
np.fmax(psdf.x1, psdf.x2) # 9007199254740992.0, float64
np.fmax(pdf.x1, pdf.x2) # 9007199254740993, int64
```
### Does this PR introduce _any_ user-facing change?
Yes. `np.fmax` / `np.fmin` on a non-double column now return the operands'
type rather than double, and integral values above 2^53 are exact.
Floating-point results, including the signed-zero tie, are unchanged.
### How was this patch tested?
Two new tests in `test_numpy_compat.py`, inherited by the Spark Connect
parity suite; both fail without the change. Neither is gated by
`_skip_if_numpy_differs`, since nothing they compare varies by platform or
NumPy version (checked on the minimum dependencies and on pandas 3).
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Code (Claude Opus 5)
Closes #58366 from Spenserrrr/numpy-trunc-fmax-fmin-precision.
Authored-by: Spenser Sun <[email protected]>
Signed-off-by: Ruifeng Zheng <[email protected]>
(cherry picked from commit 77a4781e18d5ec50bf23caf52de4143adeb7675f)
Signed-off-by: Ruifeng Zheng <[email protected]>
---
python/pyspark/pandas/numpy_compat.py | 5 ++-
python/pyspark/pandas/tests/test_numpy_compat.py | 54 ++++++++++++++++++++++++
2 files changed, 57 insertions(+), 2 deletions(-)
diff --git a/python/pyspark/pandas/numpy_compat.py
b/python/pyspark/pandas/numpy_compat.py
index 05dbc7c1c7b4..865213080d77 100644
--- a/python/pyspark/pandas/numpy_compat.py
+++ b/python/pyspark/pandas/numpy_compat.py
@@ -243,6 +243,8 @@ def _floor_divide_func(c1: Column, c2: Column) -> Column:
_tie_returns_first_operand = LooseVersion(np.__version__) >=
LooseVersion("2.3.0")
+# Every branch returns one of the operands, so without a cast the result keeps
their
+# common type: an integral pair stays exact past 2^53 and the dtype matches
NumPy's.
def _fmax_func(c1: Column, c2: Column) -> Column:
tie = c1 if _tie_returns_first_operand else c2
return (
@@ -250,13 +252,12 @@ def _fmax_func(c1: Column, c2: Column) -> Column:
.when(F.isnan(c2.cast("double")), c1)
.when(c1 == c2, tie)
.otherwise(F.greatest(c1, c2))
- .cast("double")
)
def _fmin_func(c1: Column, c2: Column) -> Column:
tie = c1 if _tie_returns_first_operand else c2
- return F.when(c1 == c2, tie).otherwise(F.least(c1, c2)).cast("double")
+ return F.when(c1 == c2, tie).otherwise(F.least(c1, c2))
binary_np_spark_mappings = {
diff --git a/python/pyspark/pandas/tests/test_numpy_compat.py
b/python/pyspark/pandas/tests/test_numpy_compat.py
index b211d5d91aaa..4b3f24427103 100644
--- a/python/pyspark/pandas/tests/test_numpy_compat.py
+++ b/python/pyspark/pandas/tests/test_numpy_compat.py
@@ -502,6 +502,60 @@ class NumPyCompatTestsMixin:
)
self.assert_eq(np.signbit(result.to_pandas()),
expected_signbit)
+ def test_np_fmax_fmin_integer_precision(self):
+ # The result keeps the operands' type, so an integral pair stays exact
past 2^53,
+ # where a double result rounds to the nearest even value. The last row
also pins the
+ # tie branch, which returns an operand rather than a comparison; it is
an equal
+ # non-zero pair, so no signed-zero tie arises and the test needs no
skip.
+ pdf = pd.DataFrame(
+ {
+ "x1": [2**53 + 1, -(2**53 + 1), 2**53 + 1],
+ "x2": [2, -2, 2**53 + 1],
+ }
+ )
+ psdf = ps.from_pandas(pdf)
+
+ for np_func in (np.fmax, np.fmin):
+ self.assert_eq(np_func(psdf.x1, psdf.x2), np_func(pdf.x1, pdf.x2))
+
+ def test_np_fmax_fmin_non_default_dtypes(self):
+ # Both helpers select an operand, so the result keeps the operands'
type for every
+ # dtype NumPy also preserves. Tie rows are equal non-zero pairs, since
the signed-zero
+ # tie is the one case where NumPy's own answer varies; keep them that
way so this test
+ # needs no skip either.
+ for dtype in ("int8", "int16", "int32", "float32"):
+ with self.subTest(dtype=dtype):
+ pdf = pd.DataFrame(
+ {
+ "x1": np.array([-2, 1, 3], dtype=dtype),
+ "x2": np.array([2, -1, 3], dtype=dtype),
+ }
+ )
+ psdf = ps.from_pandas(pdf)
+
+ for np_func in (np.fmax, np.fmin):
+ self.assert_eq(np_func(psdf.x1, psdf.x2), np_func(pdf.x1,
pdf.x2))
+
+ for pdf in (
+ pd.DataFrame({"x1": [True, False, True], "x2": [False, False,
True]}),
+ pd.DataFrame(
+ {
+ "x1": [Decimal("7.5"), Decimal("-2.5"), Decimal("3.0")],
+ "x2": [Decimal("2.0"), Decimal("-9.0"), Decimal("3.0")],
+ }
+ ),
+ pd.DataFrame(
+ {
+ "x1": pd.to_datetime(["2020-01-01", "2021-06-01"]),
+ "x2": pd.to_datetime(["2020-06-01", "2021-01-01"]),
+ }
+ ),
+ ):
+ psdf = ps.from_pandas(pdf)
+
+ for np_func in (np.fmax, np.fmin):
+ self.assert_eq(np_func(psdf.x1, psdf.x2), np_func(pdf.x1,
pdf.x2))
+
def test_np_copysign(self):
for pdf in (
pd.DataFrame(
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
