This is an automated email from the ASF dual-hosted git repository.
zhengruifeng pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 312414569333 [SPARK-58582][PS][FOLLOWUP] Keep integral operands in
integer space for NumPy fmod
312414569333 is described below
commit 31241456933350b17d578ffcdaba3aae27f4129d
Author: Spenser Sun <[email protected]>
AuthorDate: Tue Sep 1 09:06:26 2026 +0800
[SPARK-58582][PS][FOLLOWUP] Keep integral operands in integer space for
NumPy fmod
### What changes were proposed in this pull request?
`_fmod_func` cast both operands to double before taking the remainder,
including on the branch reached only when neither operand is floating. Integral
operands now take the remainder as longs, and only the result is cast back to
double.
### Why are the changes needed?
A double carries 53 bits of mantissa, so an integral operand above 2**53
was rounded before the remainder was taken: `np.fmod(9007199254740993, 2)` gave
`0.0` where pandas gives `1`. The same cast also returned `-0.0` for `-64 % 2`,
a sign of zero an integral remainder cannot have. The `pandas_udf` this mapping
replaced called NumPy directly and was exact.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
New rows in `test_np_fmod` for integral operands above 2**53, compared
exactly since a relative tolerance accepts an off-by-one at those magnitudes,
plus an `np.signbit` assertion for the sign of zero. Both fail without the
change.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Code (Claude Opus 5)
Closes #58319 from Spenserrrr/numpy-fmod-int-precision.
Authored-by: Spenser Sun <[email protected]>
Signed-off-by: Ruifeng Zheng <[email protected]>
(cherry picked from commit 6374c2f83df8b83abade73b80008156972ee602a)
Signed-off-by: Ruifeng Zheng <[email protected]>
---
python/pyspark/pandas/numpy_compat.py | 28 ++++++++++++++++--------
python/pyspark/pandas/tests/test_numpy_compat.py | 21 ++++++++++++++++++
2 files changed, 40 insertions(+), 9 deletions(-)
diff --git a/python/pyspark/pandas/numpy_compat.py
b/python/pyspark/pandas/numpy_compat.py
index aaa7d843cd75..5775e46b49a9 100644
--- a/python/pyspark/pandas/numpy_compat.py
+++ b/python/pyspark/pandas/numpy_compat.py
@@ -152,18 +152,28 @@ def _copysign_func(c1: Column, c2: Column) -> Column:
def _fmod_func(c1: Column, c2: Column) -> Column:
c1_double = c1.cast("double")
c2_double = c2.cast("double")
+ integral_types = ["tinyint", "smallint", "int", "bigint"]
- return F.when(
- F.typeof(c1).isin("float", "double") | F.typeof(c2).isin("float",
"double"),
- F.when(c1.isNull() | F.isnan(c1), c1_double)
- .when(c2.isNull() | F.isnan(c2), c2_double)
- .when(c2_double == 0, F.lit(float("nan")))
- .otherwise(F.try_mod(c1_double, c2_double)),
- ).otherwise(
+ return (
+ # A null or a nan propagates for every operand type.
F.when(c1.isNull() | F.isnan(c1), c1_double)
.when(c2.isNull() | F.isnan(c2), c2_double)
- .when(c2_double == 0, F.lit(0.0))
- .otherwise(F.try_mod(c1_double, c2_double))
+ # Integral operands take the remainder as longs: a double cannot hold
9007199254740993.
+ .when(
+ F.typeof(c1).isin(integral_types) &
F.typeof(c2).isin(integral_types),
+ F.when(c2_double == 0, F.lit(0.0)).otherwise(
+ F.try_mod(c1.cast("long"), c2.cast("long")).cast("double")
+ ),
+ )
+ # Floating operands, where a zero divisor is nan rather than the 0
above.
+ .when(
+ F.typeof(c1).isin("float", "double") | F.typeof(c2).isin("float",
"double"),
+ F.when(c2_double == 0,
F.lit(float("nan"))).otherwise(F.try_mod(c1_double, c2_double)),
+ )
+ # A decimal falls through here, since typeof carries its precision, as
in decimal(10,2).
+ # np.fmod raises on Decimal objects, so there is no NumPy behavior to
match: a zero
+ # divisor returns 0 and any other divisor takes the remainder in
double.
+ .otherwise(F.when(c2_double == 0,
F.lit(0.0)).otherwise(F.try_mod(c1_double, c2_double)))
)
diff --git a/python/pyspark/pandas/tests/test_numpy_compat.py
b/python/pyspark/pandas/tests/test_numpy_compat.py
index a733155b7199..04dc60181020 100644
--- a/python/pyspark/pandas/tests/test_numpy_compat.py
+++ b/python/pyspark/pandas/tests/test_numpy_compat.py
@@ -313,6 +313,27 @@ class NumPyCompatTestsMixin:
self.assert_eq(np.fmod(psdf.x1, psdf.x2), np.fmod(pdf.x1, pdf.x2),
almost=True)
+ # Integral operands above 2**53, where casting an operand to double
would drop its low
+ # bits: fmod(9007199254740993, 2) is 1, not 0. Compared exactly, since
almost=True would
+ # accept an off-by-one at these magnitudes.
+ pdf = pd.DataFrame(
+ {
+ "x1": [9007199254740993, 9007199254740995, -9007199254740993,
4611686018427387905],
+ "x2": [2, 4, 2, 1000000000],
+ }
+ )
+ psdf = ps.from_pandas(pdf)
+ self.assert_eq(np.fmod(psdf.x1, psdf.x2), np.fmod(pdf.x1,
pdf.x2).astype("float64"))
+
+ # almost=True treats -0.0 and 0.0 as equal, so check the sign of zero
explicitly:
+ # an integral remainder is never negative zero, unlike a double one.
+ pdf = pd.DataFrame({"x1": [-64, -2, 0, 64], "x2": [2, 2, 3, 2]})
+ psdf = ps.from_pandas(pdf)
+
+ result = np.fmod(psdf.x1, psdf.x2)
+ expected = np.fmod(pdf.x1, pdf.x2)
+ self.assert_eq(np.signbit(result.to_pandas()), np.signbit(expected))
+
def test_np_modf(self):
# np.modf(x) returns a tuple (fractional part, integral part).
for pdf in (
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
