This is an automated email from the ASF dual-hosted git repository.

zhengruifeng pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-4.x by this push:
     new 423bb9d8c85d [SPARK-60656][PS] Fix NumPy maximum/minimum NaN 
propagation
423bb9d8c85d is described below

commit 423bb9d8c85d14dc1b2dde147a99335369e82e83
Author: FNU Aditya Ranjan <[email protected]>
AuthorDate: Thu Nov 5 21:36:38 2026 +0800

    [SPARK-60656][PS] Fix NumPy maximum/minimum NaN propagation
    
    ### What changes were proposed in this pull request?
    
    Fix pandas-on-Spark `np.maximum` and `np.minimum` to propagate missing 
values consistently with NumPy.
    
    Spark's `greatest` and `least` can skip SQL NULL values, and `least` can 
return a finite value when another operand is a runtime-generated NaN.
    
    The fix introduces dedicated functions that propagate NULL and NaN by 
returning the original operand, preserving integer precision and result types.
    
    ### Why are the changes needed?
    
    Previously, `np.maximum` could return `-1.0` instead of NaN for imported 
missing values. Similarly, `np.minimum` could return `-1.0` instead of NaN 
generated by `np.cos(np.inf)`.
    
    ### How was this patch tested?
    
    - Added regression tests for imported NaN and runtime-generated NaN.
    - Verified integer precision beyond 2^53.
    - Focused unittest passed.
    - `git diff --check` passed.
    
    Closes SPARK-60656.
    
    Closes #60015 from adiranjan25/fix/spark-60656-runtime-nan-maximum.
    
    Authored-by: FNU Aditya Ranjan <[email protected]>
    Signed-off-by: Ruifeng Zheng <[email protected]>
    
    (cherry picked from commit 81a93ab2b3a05598177d745a4604e908a7d9f68e)
    Signed-off-by: Ruifeng Zheng <[email protected]>
    
    
    skip-checks: true
---
 python/pyspark/pandas/numpy_compat.py            | 20 +++++++++--
 python/pyspark/pandas/tests/test_numpy_compat.py | 45 ++++++++++++++++++++++++
 2 files changed, 63 insertions(+), 2 deletions(-)

diff --git a/python/pyspark/pandas/numpy_compat.py 
b/python/pyspark/pandas/numpy_compat.py
index 9295856ba964..ce2c2ced5b2b 100644
--- a/python/pyspark/pandas/numpy_compat.py
+++ b/python/pyspark/pandas/numpy_compat.py
@@ -222,6 +222,22 @@ _tie_returns_first_operand = LooseVersion(np.__version__) 
>= LooseVersion("2.3.0
 
 # Every branch returns one of the operands, so without a cast the result keeps 
their
 # common type: an integral pair stays exact past 2^53 and the dtype matches 
NumPy's.
+def _maximum_func(c1: Column, c2: Column) -> Column:
+    return (
+        F.when(c1.isNull() | F.isnan(c1.cast("double")), c1)
+        .when(c2.isNull() | F.isnan(c2.cast("double")), c2)
+        .otherwise(F.greatest(c1, c2))
+    )
+
+
+def _minimum_func(c1: Column, c2: Column) -> Column:
+    return (
+        F.when(c1.isNull() | F.isnan(c1.cast("double")), c1)
+        .when(c2.isNull() | F.isnan(c2.cast("double")), c2)
+        .otherwise(F.least(c1, c2))
+    )
+
+
 def _fmax_func(c1: Column, c2: Column) -> Column:
     tie = c1 if _tie_returns_first_operand else c2
     return (
@@ -279,8 +295,8 @@ binary_np_spark_mappings = {
         (c1.cast(BooleanType()) | c2.cast(BooleanType()))
         & (~(c1.cast(BooleanType())) | ~(c2.cast(BooleanType())))
     ),
-    "maximum": F.greatest,
-    "minimum": F.least,
+    "maximum": _maximum_func,
+    "minimum": _minimum_func,
     "nextafter": pandas_udf(  # type: ignore[call-overload]
         lambda s1, s2: np.nextafter(s1, s2), DoubleType()
     ),
diff --git a/python/pyspark/pandas/tests/test_numpy_compat.py 
b/python/pyspark/pandas/tests/test_numpy_compat.py
index 8b26fbebda4a..f1fde3b4e3c9 100644
--- a/python/pyspark/pandas/tests/test_numpy_compat.py
+++ b/python/pyspark/pandas/tests/test_numpy_compat.py
@@ -805,6 +805,51 @@ class NumPyCompatTestsMixin:
                 )
                 self.assert_eq(np.signbit(result.to_pandas()), 
expected_signbit)
 
+    def test_np_maximum_minimum_null_propagation(self):
+        for pdf in (
+            pd.DataFrame(
+                {
+                    "x1": [np.nan, -1.0, 2.0, np.nan],
+                    "x2": [-1.0, np.nan, 3.0, np.nan],
+                }
+            ),
+            pd.DataFrame(
+                {
+                    "x1": [1, 2, 3],
+                    "x2": [3, 2, 1],
+                }
+            ),
+        ):
+            psdf = ps.from_pandas(pdf)
+            for np_func in (np.maximum, np.minimum):
+                self.assert_eq(
+                    np_func(psdf.x1, psdf.x2),
+                    np_func(pdf.x1, pdf.x2),
+                )
+
+        # NaN generated by Spark expressions must also propagate.
+        pser = pd.Series([np.inf, 1.0])
+        psser = ps.from_pandas(pser)
+        for np_func in (np.maximum, np.minimum):
+            self.assert_eq(
+                np_func(np.cos(psser), -1.0),
+                np_func(np.cos(pser), -1.0),
+            )
+
+        # Preserve integer precision beyond the exact range of float64.
+        pdf = pd.DataFrame(
+            {
+                "x1": [2**53 + 1, -(2**53 + 1)],
+                "x2": [2, -2],
+            }
+        )
+        psdf = ps.from_pandas(pdf)
+        for np_func in (np.maximum, np.minimum):
+            self.assert_eq(
+                np_func(psdf.x1, psdf.x2),
+                np_func(pdf.x1, pdf.x2),
+            )
+
     def test_np_fmax_fmin_integer_precision(self):
         # The result keeps the operands' type, so an integral pair stays exact 
past 2^53,
         # where a double result rounds to the nearest even value. The last row 
also pins the


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to