This is an automated email from the ASF dual-hosted git repository.
zhengruifeng pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 423bb9d8c85d [SPARK-60656][PS] Fix NumPy maximum/minimum NaN
propagation
423bb9d8c85d is described below
commit 423bb9d8c85d14dc1b2dde147a99335369e82e83
Author: FNU Aditya Ranjan <[email protected]>
AuthorDate: Thu Nov 5 21:36:38 2026 +0800
[SPARK-60656][PS] Fix NumPy maximum/minimum NaN propagation
### What changes were proposed in this pull request?
Fix pandas-on-Spark `np.maximum` and `np.minimum` to propagate missing
values consistently with NumPy.
Spark's `greatest` and `least` can skip SQL NULL values, and `least` can
return a finite value when another operand is a runtime-generated NaN.
The fix introduces dedicated functions that propagate NULL and NaN by
returning the original operand, preserving integer precision and result types.
### Why are the changes needed?
Previously, `np.maximum` could return `-1.0` instead of NaN for imported
missing values. Similarly, `np.minimum` could return `-1.0` instead of NaN
generated by `np.cos(np.inf)`.
### How was this patch tested?
- Added regression tests for imported NaN and runtime-generated NaN.
- Verified integer precision beyond 2^53.
- Focused unittest passed.
- `git diff --check` passed.
Closes SPARK-60656.
Closes #60015 from adiranjan25/fix/spark-60656-runtime-nan-maximum.
Authored-by: FNU Aditya Ranjan <[email protected]>
Signed-off-by: Ruifeng Zheng <[email protected]>
(cherry picked from commit 81a93ab2b3a05598177d745a4604e908a7d9f68e)
Signed-off-by: Ruifeng Zheng <[email protected]>
skip-checks: true
---
python/pyspark/pandas/numpy_compat.py | 20 +++++++++--
python/pyspark/pandas/tests/test_numpy_compat.py | 45 ++++++++++++++++++++++++
2 files changed, 63 insertions(+), 2 deletions(-)
diff --git a/python/pyspark/pandas/numpy_compat.py
b/python/pyspark/pandas/numpy_compat.py
index 9295856ba964..ce2c2ced5b2b 100644
--- a/python/pyspark/pandas/numpy_compat.py
+++ b/python/pyspark/pandas/numpy_compat.py
@@ -222,6 +222,22 @@ _tie_returns_first_operand = LooseVersion(np.__version__)
>= LooseVersion("2.3.0
# Every branch returns one of the operands, so without a cast the result keeps
their
# common type: an integral pair stays exact past 2^53 and the dtype matches
NumPy's.
+def _maximum_func(c1: Column, c2: Column) -> Column:
+ return (
+ F.when(c1.isNull() | F.isnan(c1.cast("double")), c1)
+ .when(c2.isNull() | F.isnan(c2.cast("double")), c2)
+ .otherwise(F.greatest(c1, c2))
+ )
+
+
+def _minimum_func(c1: Column, c2: Column) -> Column:
+ return (
+ F.when(c1.isNull() | F.isnan(c1.cast("double")), c1)
+ .when(c2.isNull() | F.isnan(c2.cast("double")), c2)
+ .otherwise(F.least(c1, c2))
+ )
+
+
def _fmax_func(c1: Column, c2: Column) -> Column:
tie = c1 if _tie_returns_first_operand else c2
return (
@@ -279,8 +295,8 @@ binary_np_spark_mappings = {
(c1.cast(BooleanType()) | c2.cast(BooleanType()))
& (~(c1.cast(BooleanType())) | ~(c2.cast(BooleanType())))
),
- "maximum": F.greatest,
- "minimum": F.least,
+ "maximum": _maximum_func,
+ "minimum": _minimum_func,
"nextafter": pandas_udf( # type: ignore[call-overload]
lambda s1, s2: np.nextafter(s1, s2), DoubleType()
),
diff --git a/python/pyspark/pandas/tests/test_numpy_compat.py
b/python/pyspark/pandas/tests/test_numpy_compat.py
index 8b26fbebda4a..f1fde3b4e3c9 100644
--- a/python/pyspark/pandas/tests/test_numpy_compat.py
+++ b/python/pyspark/pandas/tests/test_numpy_compat.py
@@ -805,6 +805,51 @@ class NumPyCompatTestsMixin:
)
self.assert_eq(np.signbit(result.to_pandas()),
expected_signbit)
+ def test_np_maximum_minimum_null_propagation(self):
+ for pdf in (
+ pd.DataFrame(
+ {
+ "x1": [np.nan, -1.0, 2.0, np.nan],
+ "x2": [-1.0, np.nan, 3.0, np.nan],
+ }
+ ),
+ pd.DataFrame(
+ {
+ "x1": [1, 2, 3],
+ "x2": [3, 2, 1],
+ }
+ ),
+ ):
+ psdf = ps.from_pandas(pdf)
+ for np_func in (np.maximum, np.minimum):
+ self.assert_eq(
+ np_func(psdf.x1, psdf.x2),
+ np_func(pdf.x1, pdf.x2),
+ )
+
+ # NaN generated by Spark expressions must also propagate.
+ pser = pd.Series([np.inf, 1.0])
+ psser = ps.from_pandas(pser)
+ for np_func in (np.maximum, np.minimum):
+ self.assert_eq(
+ np_func(np.cos(psser), -1.0),
+ np_func(np.cos(pser), -1.0),
+ )
+
+ # Preserve integer precision beyond the exact range of float64.
+ pdf = pd.DataFrame(
+ {
+ "x1": [2**53 + 1, -(2**53 + 1)],
+ "x2": [2, -2],
+ }
+ )
+ psdf = ps.from_pandas(pdf)
+ for np_func in (np.maximum, np.minimum):
+ self.assert_eq(
+ np_func(psdf.x1, psdf.x2),
+ np_func(pdf.x1, pdf.x2),
+ )
+
def test_np_fmax_fmin_integer_precision(self):
# The result keeps the operands' type, so an integral pair stays exact
past 2^53,
# where a double result rounds to the nearest even value. The last row
also pins the
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]