ueshin commented on a change in pull request #32611:
URL: https://github.com/apache/spark/pull/32611#discussion_r637228133
##########
File path: python/pyspark/pandas/tests/data_type_ops/test_date_ops.py
##########
@@ -60,7 +61,9 @@ def test_sub(self):
with option_context("compute.ops_on_diff_frames", True):
for pser, psser in self.pser_psser_pairs:
if isinstance(psser.spark.data_type, DateType):
- self.assert_eq((self.pser - pser).dt.days, (self.psser -
psser).sort_index())
+ if LooseVersion(pd.__version__) >= LooseVersion("0.24.2"):
+ self.assert_eq(
+ (self.pser - pser).dt.days, (self.psser -
psser).sort_index())
Review comment:
What happens in `else` case?
##########
File path: python/pyspark/pandas/data_type_ops/boolean_ops.py
##########
@@ -26,3 +41,183 @@ class BooleanOps(DataTypeOps):
@property
def pretty_name(self) -> str:
return 'booleans'
+
+ def add(self, left, right):
+ if isinstance(right, numbers.Number):
+ left = left.spark.apply(lambda scol:
scol.cast(as_spark_type(type(right))))
+ return column_op(Column.__add__)(left, right)
+ elif isinstance(right, IndexOpsMixin) and is_numeric_index_ops(right):
+ left = transform_boolean_operand_to_numeric(left,
right.spark.data_type)
+ return column_op(Column.__add__)(left, right)
+ else:
+ raise TypeError(
+ "Addition can not be applied to %s and the given type." %
self.pretty_name)
+
+ def sub(self, left, right):
+ if isinstance(right, numbers.Number):
+ left = left.spark.apply(lambda scol:
scol.cast(as_spark_type(type(right))))
+ return column_op(Column.__sub__)(left, right)
+ elif isinstance(right, IndexOpsMixin) and is_numeric_index_ops(right):
+ left = transform_boolean_operand_to_numeric(left,
right.spark.data_type)
+ return column_op(Column.__sub__)(left, right)
+ else:
+ raise TypeError(
+ "Subtraction can not be applied to %s and the given type." %
self.pretty_name)
+
+ def mul(self, left, right):
+ if isinstance(right, numbers.Number):
+ left = left.spark.apply(lambda scol:
scol.cast(as_spark_type(type(right))))
+ return column_op(Column.__mul__)(left, right)
+ elif isinstance(right, IndexOpsMixin) and is_numeric_index_ops(right):
+ left = transform_boolean_operand_to_numeric(left,
right.spark.data_type)
+ return column_op(Column.__mul__)(left, right)
+ else:
+ raise TypeError(
+ "Multiplication can not be applied to %s and the given type."
% self.pretty_name)
+
+ def truediv(self, left, right):
+ def truediv(left, right):
+ return F.when(F.lit(right != 0) | F.lit(right).isNull(),
left.__div__(right)).otherwise(
+ F.when(F.lit(left == np.inf) | F.lit(left == -np.inf),
left).otherwise(
+ F.lit(np.inf).__div__(left)
+ )
+ )
+
+ if isinstance(right, numbers.Number):
+ left = left.spark.apply(lambda scol:
scol.cast(as_spark_type(type(right))))
+ return numpy_column_op(truediv)(left, right)
+ elif isinstance(right, IndexOpsMixin) and is_numeric_index_ops(right):
+ left = transform_boolean_operand_to_numeric(left,
right.spark.data_type)
+ return numpy_column_op(truediv)(left, right)
+ else:
+ raise TypeError(
+ "True division can not be applied to %s and the given type." %
self.pretty_name)
+
+ def floordiv(self, left, right):
+ def floordiv(left, right):
+ return F.when(F.lit(right is np.nan), np.nan).otherwise(
+ F.when(
+ F.lit(right != 0) | F.lit(right).isNull(),
F.floor(left.__div__(right))
+ ).otherwise(
+ F.when(F.lit(left == np.inf) | F.lit(left == -np.inf),
left).otherwise(
+ F.lit(np.inf).__div__(left)
+ )
+ )
+ )
+
+ if isinstance(right, numbers.Number):
+ left = left.spark.apply(lambda scol:
scol.cast(as_spark_type(type(right))))
+ return numpy_column_op(floordiv)(left, right)
+ elif isinstance(right, IndexOpsMixin) and is_numeric_index_ops(right):
+ left = transform_boolean_operand_to_numeric(left,
right.spark.data_type)
+ return numpy_column_op(floordiv)(left, right)
+ else:
+ raise TypeError(
+ "Floor division can not be applied to %s and the given type."
% self.pretty_name)
+
+ def mod(self, left, right):
+ def mod(left, right):
+ return ((left % right) + right) % right
+
+ if isinstance(right, numbers.Number):
+ left = left.spark.apply(lambda scol:
scol.cast(as_spark_type(type(right))))
+ return column_op(mod)(left, right)
+ elif isinstance(right, IndexOpsMixin) and is_numeric_index_ops(right):
+ left = transform_boolean_operand_to_numeric(left,
right.spark.data_type)
+ return column_op(mod)(left, right)
+ else:
+ raise TypeError(
+ "Modulo can not be applied to %s and the given type." %
self.pretty_name)
+
+ def pow(self, left, right):
+ def pow_func(left, right):
+ return F.when(left == 1, left).otherwise(Column.__pow__(left,
right))
+
+ if isinstance(right, numbers.Number):
+ left = left.spark.apply(lambda scol:
scol.cast(as_spark_type(type(right))))
+ return column_op(pow_func)(left, right)
+ elif isinstance(right, IndexOpsMixin) and is_numeric_index_ops(right):
+ left = transform_boolean_operand_to_numeric(left,
right.spark.data_type)
+ return column_op(pow_func)(left, right)
+
+ else:
+ raise TypeError(
+ "Exponentiation can not be applied to %s and the given type."
% self.pretty_name)
+
+ def radd(self, left, right=None):
+ if isinstance(right, numbers.Number):
+ left = left.spark.apply(lambda scol:
scol.cast(as_spark_type(type(right))))
+ return column_op(Column.__radd__)(left, right)
+ else:
+ raise TypeError(
+ "Addition can not be applied to %s and the given type." %
self.pretty_name)
+
+ def rsub(self, left, right=None):
Review comment:
We don't need `=None`?
##########
File path: python/pyspark/pandas/tests/data_type_ops/test_boolean_ops.py
##########
@@ -33,107 +35,148 @@ def pser(self):
def psser(self):
return ps.from_pandas(self.pser)
+ @property
+ def float_pser(self):
+ return pd.Series([1, 2, 3], dtype=float)
+
+ @property
+ def float_psser(self):
+ return ps.from_pandas(self.float_pser)
+
def test_add(self):
- self.assertRaises(TypeError, lambda: self.psser + 1)
- self.assertRaises(TypeError, lambda: self.psser + 0.1)
+ self.assert_eq(self.pser + 1, self.psser + 1)
+ self.assert_eq(self.pser + 0.1, self.psser + 0.1)
Review comment:
What if:
```py
psser = self.psser
psser + psser.astype(int)
psser + psser
```
##########
File path: python/pyspark/pandas/tests/data_type_ops/test_num_ops.py
##########
@@ -30,6 +34,14 @@ class NumOpsTest(PandasOnSparkTestCase, TestCasesUtils):
returns float32.
The underlying reason is the respective Spark operations return DoubleType
always.
"""
+ @property
+ def float_pser(self):
+ return pd.Series([1, 2, 3], dtype=float)
+
+ @property
+ def float_psser(self):
+ return ps.from_pandas(self.float_pser)
+
def test_add(self):
for pser, psser in self.numeric_pser_psser_pairs:
self.assert_eq(pser + pser, psser + psser)
Review comment:
What if:
```py
psser + True
psser + psser.astype(bool)
```
##########
File path: python/pyspark/pandas/tests/data_type_ops/test_date_ops.py
##########
@@ -60,7 +61,9 @@ def test_sub(self):
with option_context("compute.ops_on_diff_frames", True):
for pser, psser in self.pser_psser_pairs:
if isinstance(psser.spark.data_type, DateType):
- self.assert_eq((self.pser - pser).dt.days, (self.psser -
psser).sort_index())
+ if LooseVersion(pd.__version__) >= LooseVersion("0.24.2"):
+ self.assert_eq(
+ (self.pser - pser).dt.days, (self.psser -
psser).sort_index())
Review comment:
btw, this is not related to this PR?
##########
File path: python/pyspark/pandas/data_type_ops/boolean_ops.py
##########
@@ -26,3 +41,183 @@ class BooleanOps(DataTypeOps):
@property
def pretty_name(self) -> str:
return 'booleans'
+
+ def add(self, left, right):
+ if isinstance(right, numbers.Number):
+ left = left.spark.apply(lambda scol:
scol.cast(as_spark_type(type(right))))
Review comment:
`left.spark.transform` instead of `apply`?
##########
File path: python/pyspark/pandas/data_type_ops/boolean_ops.py
##########
@@ -26,3 +41,183 @@ class BooleanOps(DataTypeOps):
@property
def pretty_name(self) -> str:
return 'booleans'
+
+ def add(self, left, right):
Review comment:
Could you add type annotations?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
