This is an automated email from the ASF dual-hosted git repository.

zhengruifeng pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-4.x by this push:
     new 854f1cdaeaed [SPARK-58464][PS] Use native functions for NumPy bitwise 
shifts
854f1cdaeaed is described below

commit 854f1cdaeaed13e9a8f84030361eb47b3a2b7a99
Author: Ruifeng Zheng <[email protected]>
AuthorDate: Mon Aug 3 16:26:29 2026 +0800

    [SPARK-58464][PS] Use native functions for NumPy bitwise shifts
    
    ### What changes were proposed in this pull request?
    
    Replace the scalar pandas UDF mappings for NumPy left_shift and right_shift 
on pandas-on-Spark objects with native Spark SQL shiftleft and shiftright 
functions through call_function. The public PySpark helpers accept only literal 
shift counts, while NumPy ufunc dispatch can supply column-valued counts.
    
    Native conditional expressions preserve NumPy behavior for negative and 
out-of-range int64 shift counts. Add pandas-on-Spark parity coverage for signed 
int64 boundary values and shift counts below, at, and above the bit width.
    
    ### Why are the changes needed?
    
    Spark provides native bit-shift functions, so these mappings no longer need 
to cross the Python worker boundary.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. NumPy bitwise shifts now execute natively and preserve the Spark 
integral type of the left operand instead of always returning a long result 
from the pandas UDF. Their values remain NumPy-compatible, including 
out-of-range shift counts.
    
    ### How was this patch tested?
    
    - Added pandas-on-Spark parity coverage for NumPy bitwise shifts with int64 
boundary values and out-of-range counts.
    - Ran JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64 SPARK_TESTING=1 
SPARK_PREPEND_CLASSES=1 python/run-tests --testnames 
pyspark.pandas.tests.test_numpy_compat.
    - Ran git diff --check.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Codex (GPT-5)
    
    Closes #57667 from zhengruifeng/pandas-native-shifts-dev2.
    
    Authored-by: Ruifeng Zheng <[email protected]>
    Signed-off-by: Ruifeng Zheng <[email protected]>
    (cherry picked from commit b6c0808a205b98b3f0fd018f923a9804f514804a)
    Signed-off-by: Ruifeng Zheng <[email protected]>
---
 python/pyspark/pandas/numpy_compat.py            | 16 ++++++++++------
 python/pyspark/pandas/tests/test_numpy_compat.py | 15 +++++++++++++++
 2 files changed, 25 insertions(+), 6 deletions(-)

diff --git a/python/pyspark/pandas/numpy_compat.py 
b/python/pyspark/pandas/numpy_compat.py
index febba91883bd..316e2e9e3afd 100644
--- a/python/pyspark/pandas/numpy_compat.py
+++ b/python/pyspark/pandas/numpy_compat.py
@@ -20,7 +20,7 @@ import numpy as np
 
 from pyspark.sql import functions as F
 from pyspark.sql.pandas.functions import pandas_udf
-from pyspark.sql.types import DoubleType, LongType, BooleanType
+from pyspark.sql.types import DoubleType, BooleanType
 from pyspark.pandas.base import IndexOpsMixin
 
 unary_np_spark_mappings = {
@@ -107,8 +107,10 @@ binary_np_spark_mappings = {
     "ldexp": pandas_udf(  # type: ignore[call-overload]
         lambda s1, s2: np.ldexp(s1, s2), DoubleType()
     ),
-    "left_shift": pandas_udf(  # type: ignore[call-overload]
-        lambda s1, s2: np.left_shift(s1, s2), LongType()
+    # F.shiftleft accepts literal counts only; call_function also accepts a 
column.
+    # NumPy returns zero for counts outside an int64's bit width, unlike JVM 
shifts.
+    "left_shift": lambda c1, c2: F.when((c2 < 0) | (c2 >= 64), 
F.lit(0)).otherwise(
+        F.call_function("shiftleft", c1, c2)
     ),
     "logaddexp": pandas_udf(  # type: ignore[call-overload]
         lambda s1, s2: np.logaddexp(s1, s2), DoubleType()
@@ -129,9 +131,11 @@ binary_np_spark_mappings = {
     "nextafter": pandas_udf(  # type: ignore[call-overload]
         lambda s1, s2: np.nextafter(s1, s2), DoubleType()
     ),
-    "right_shift": pandas_udf(  # type: ignore[call-overload]
-        lambda s1, s2: np.right_shift(s1, s2), LongType()
-    ),
+    # F.shiftright accepts literal counts only; call_function also accepts a 
column.
+    # NumPy sign-extends counts outside an int64's bit width, unlike JVM 
shifts.
+    "right_shift": lambda c1, c2: F.when(
+        (c2 < 0) | (c2 >= 64), F.call_function("shiftright", c1, F.lit(63))
+    ).otherwise(F.call_function("shiftright", c1, c2)),
 }
 
 
diff --git a/python/pyspark/pandas/tests/test_numpy_compat.py 
b/python/pyspark/pandas/tests/test_numpy_compat.py
index 432283cb5fe8..bd7ba4c3e210 100644
--- a/python/pyspark/pandas/tests/test_numpy_compat.py
+++ b/python/pyspark/pandas/tests/test_numpy_compat.py
@@ -115,6 +115,21 @@ class NumPyCompatTestsMixin:
 
                 self.assert_eq(np_func(psdf.a), np_func(pdf.a), almost=True)
 
+    def test_np_bitwise_shift_functions(self):
+        pdf = pd.DataFrame(
+            {
+                "value": [np.iinfo(np.int64).min, -2, -1, 0, 1, 2, 
np.iinfo(np.int64).max],
+                "bits": [-1, 0, 1, 63, 64, 65, 2],
+            }
+        )
+        psdf = ps.from_pandas(pdf)
+
+        for np_func in (np.left_shift, np.right_shift):
+            with self.subTest(name=np_func.__name__):
+                self.assert_eq(
+                    np_func(psdf.value, psdf.bits), np_func(pdf.value, 
pdf.bits), almost=True
+                )
+
     def test_np_spark_compat_series(self):
         from pyspark.pandas.numpy_compat import unary_np_spark_mappings, 
binary_np_spark_mappings
 


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to