This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new cf2aaddb5020 [SPARK-55980][PS] Always apply _cast_back_float in
numeric arithmetic
cf2aaddb5020 is described below
commit cf2aaddb5020bcbe391b9cd4e8e46e2025a8c29f
Author: Devin Petersohn <[email protected]>
AuthorDate: Fri Mar 13 09:44:36 2026 +0900
[SPARK-55980][PS] Always apply _cast_back_float in numeric arithmetic
### What changes were proposed in this pull request?
Remove ANSI mode guard from `_cast_back_float` calls in `num_ops.py`.
### Why are the changes needed?
Simplify the code without changing behavior.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
CI
### Was this patch authored or co-authored using generative AI tooling?
Co-authored-by: Claude Opus 4
Closes #54779 from devin-petersohn/devin/always-cast-back-float.
Authored-by: Devin Petersohn <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
python/pyspark/pandas/data_type_ops/num_ops.py | 43 ++++++--------------------
1 file changed, 9 insertions(+), 34 deletions(-)
diff --git a/python/pyspark/pandas/data_type_ops/num_ops.py
b/python/pyspark/pandas/data_type_ops/num_ops.py
index 8c3b9ab66bc3..e2a3180331eb 100644
--- a/python/pyspark/pandas/data_type_ops/num_ops.py
+++ b/python/pyspark/pandas/data_type_ops/num_ops.py
@@ -125,14 +125,10 @@ class NumericOps(DataTypeOps):
_sanitize_list_like(right)
if not is_valid_operand_for_numeric_arithmetic(right):
raise TypeError("Addition can not be applied to given types.")
- spark_session = left._internal.spark_frame.sparkSession
new_right = transform_boolean_operand_to_numeric(right,
spark_type=left.spark.data_type)
def wrapped_add(lc: PySparkColumn, rc: Any) -> PySparkColumn:
- expr = PySparkColumn.__add__(lc, rc)
- if is_ansi_mode_enabled(spark_session):
- expr = _cast_back_float(expr, left.dtype, right)
- return expr
+ return _cast_back_float(PySparkColumn.__add__(lc, rc), left.dtype,
right)
return column_op(wrapped_add)(left, new_right)
@@ -140,14 +136,10 @@ class NumericOps(DataTypeOps):
_sanitize_list_like(right)
if not is_valid_operand_for_numeric_arithmetic(right):
raise TypeError("Subtraction can not be applied to given types.")
- spark_session = left._internal.spark_frame.sparkSession
new_right = transform_boolean_operand_to_numeric(right,
spark_type=left.spark.data_type)
def wrapped_sub(lc: PySparkColumn, rc: Any) -> PySparkColumn:
- expr = PySparkColumn.__sub__(lc, rc)
- if is_ansi_mode_enabled(spark_session):
- expr = _cast_back_float(expr, left.dtype, right)
- return expr
+ return _cast_back_float(PySparkColumn.__sub__(lc, rc), left.dtype,
right)
return column_op(wrapped_sub)(left, new_right)
@@ -162,10 +154,9 @@ class NumericOps(DataTypeOps):
expr = F.when(F.lit(right_op == 0), F.lit(None)).otherwise(
((left_op % right_op) + right_op) % right_op
)
- expr = _cast_back_float(expr, left.dtype, right)
else:
expr = ((left_op % right_op) + right_op) % right_op
- return expr
+ return _cast_back_float(expr, left.dtype, right)
new_right = transform_boolean_operand_to_numeric(right,
spark_type=left.spark.data_type)
@@ -190,14 +181,10 @@ class NumericOps(DataTypeOps):
_sanitize_list_like(right)
if not isinstance(right, numbers.Number):
raise TypeError("Addition can not be applied to given types.")
- spark_session = left._internal.spark_frame.sparkSession
new_right = transform_boolean_operand_to_numeric(right)
def wrapped_radd(lc: PySparkColumn, rc: Any) -> PySparkColumn:
- expr = PySparkColumn.__radd__(lc, rc)
- if is_ansi_mode_enabled(spark_session):
- expr = _cast_back_float(expr, left.dtype, right)
- return expr
+ return _cast_back_float(PySparkColumn.__radd__(lc, rc),
left.dtype, right)
return column_op(wrapped_radd)(left, new_right)
@@ -205,14 +192,10 @@ class NumericOps(DataTypeOps):
_sanitize_list_like(right)
if not isinstance(right, numbers.Number):
raise TypeError("Subtraction can not be applied to given types.")
- spark_session = left._internal.spark_frame.sparkSession
new_right = transform_boolean_operand_to_numeric(right)
def wrapped_rsub(lc: PySparkColumn, rc: Any) -> PySparkColumn:
- expr = PySparkColumn.__rsub__(lc, rc)
- if is_ansi_mode_enabled(spark_session):
- expr = _cast_back_float(expr, left.dtype, right)
- return expr
+ return _cast_back_float(PySparkColumn.__rsub__(lc, rc),
left.dtype, right)
return column_op(wrapped_rsub)(left, new_right)
@@ -220,14 +203,10 @@ class NumericOps(DataTypeOps):
_sanitize_list_like(right)
if not isinstance(right, numbers.Number):
raise TypeError("Multiplication can not be applied to given
types.")
- spark_session = left._internal.spark_frame.sparkSession
new_right = transform_boolean_operand_to_numeric(right)
def wrapped_rmul(lc: PySparkColumn, rc: Any) -> PySparkColumn:
- expr = PySparkColumn.__mul__(lc, rc)
- if is_ansi_mode_enabled(spark_session):
- expr = _cast_back_float(expr, left.dtype, right)
- return expr
+ return _cast_back_float(PySparkColumn.__mul__(lc, rc), left.dtype,
right)
return column_op(wrapped_rmul)(left, new_right)
@@ -256,10 +235,9 @@ class NumericOps(DataTypeOps):
result = F.when(
left_op != 0, ((F.lit(right_op) % left_op) + left_op) %
left_op
).otherwise(F.lit(None))
- result = _cast_back_float(result, left.dtype, right)
- return result
else:
- return ((right_op % left_op) + left_op) % left_op
+ result = ((right_op % left_op) + left_op) % left_op
+ return _cast_back_float(result, left.dtype, right)
return column_op(safe_rmod)(left, new_right)
@@ -472,10 +450,7 @@ class FractionalOps(NumericOps):
new_right = transform_boolean_operand_to_numeric(right,
spark_type=left.spark.data_type)
def wrapped_mul(lc: PySparkColumn, rc: Any) -> PySparkColumn:
- expr = PySparkColumn.__mul__(lc, rc)
- if is_ansi:
- expr = _cast_back_float(expr, left.dtype, right)
- return expr
+ return _cast_back_float(PySparkColumn.__mul__(lc, rc), left.dtype,
right)
return column_op(wrapped_mul)(left, new_right)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
