This is an automated email from the ASF dual-hosted git repository.
ruifengz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 373975b8ef3 [SPARK-43676][SPARK-43677][SPARK-43678][SPARK-43679][PS]
Fix `DatetimeOps` for Spark Connect
373975b8ef3 is described below
commit 373975b8ef35afc80d396e69081173875022a970
Author: itholic <[email protected]>
AuthorDate: Mon May 29 19:36:38 2023 +0800
[SPARK-43676][SPARK-43677][SPARK-43678][SPARK-43679][PS] Fix `DatetimeOps`
for Spark Connect
### What changes were proposed in this pull request?
This PR proposes to fix `DatetimeOps` test for pandas API on Spark with
Spark Connect.
This includes SPARK-43676, SPARK-43677, SPARK-43678, SPARK-43679 at once,
because they are all related similar modifications in single file.
### Why are the changes needed?
To support all features for pandas API on Spark with Spark Connect.
### Does this PR introduce _any_ user-facing change?
Yes, `DatetimeOps.lt`, `DatetimeOps.le`, `DatetimeOps.ge`,
`DatetimeOps.gt` are now working as expected on Spark Connect.
### How was this patch tested?
Uncomment the UTs, and tested manually.
Closes #41306 from itholic/SPARK-43676-9.
Authored-by: itholic <[email protected]>
Signed-off-by: Ruifeng Zheng <[email protected]>
---
python/pyspark/pandas/data_type_ops/datetime_ops.py | 17 +++++------------
.../connect/data_type_ops/test_parity_datetime_ops.py | 16 ----------------
2 files changed, 5 insertions(+), 28 deletions(-)
diff --git a/python/pyspark/pandas/data_type_ops/datetime_ops.py
b/python/pyspark/pandas/data_type_ops/datetime_ops.py
index 506ffff9598..c5f4df96bde 100644
--- a/python/pyspark/pandas/data_type_ops/datetime_ops.py
+++ b/python/pyspark/pandas/data_type_ops/datetime_ops.py
@@ -33,6 +33,7 @@ from pyspark.sql.types import (
TimestampNTZType,
NumericType,
)
+from pyspark.sql.utils import pyspark_column_op
from pyspark.pandas._typing import Dtype, IndexOpsLike, SeriesOrIndex
from pyspark.pandas.base import IndexOpsMixin
@@ -109,28 +110,20 @@ class DatetimeOps(DataTypeOps):
raise TypeError("Datetime subtraction can only be applied to
datetime series.")
def lt(self, left: IndexOpsLike, right: Any) -> SeriesOrIndex:
- from pyspark.pandas.base import column_op
-
_sanitize_list_like(right)
- return column_op(Column.__lt__)(left, right)
+ return pyspark_column_op("__lt__")(left, right)
def le(self, left: IndexOpsLike, right: Any) -> SeriesOrIndex:
- from pyspark.pandas.base import column_op
-
_sanitize_list_like(right)
- return column_op(Column.__le__)(left, right)
+ return pyspark_column_op("__le__")(left, right)
def ge(self, left: IndexOpsLike, right: Any) -> SeriesOrIndex:
- from pyspark.pandas.base import column_op
-
_sanitize_list_like(right)
- return column_op(Column.__ge__)(left, right)
+ return pyspark_column_op("__ge__")(left, right)
def gt(self, left: IndexOpsLike, right: Any) -> SeriesOrIndex:
- from pyspark.pandas.base import column_op
-
_sanitize_list_like(right)
- return column_op(Column.__gt__)(left, right)
+ return pyspark_column_op("__gt__")(left, right)
def prepare(self, col: pd.Series) -> pd.Series:
"""Prepare column when from_pandas."""
diff --git
a/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_datetime_ops.py
b/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_datetime_ops.py
index 697c191b743..6d081b10aba 100644
---
a/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_datetime_ops.py
+++
b/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_datetime_ops.py
@@ -34,22 +34,6 @@ class DatetimeOpsParityTests(
def test_astype(self):
super().test_astype()
- @unittest.skip("TODO(SPARK-43676): Fix DatetimeOps.ge to work with Spark
Connect Column.")
- def test_ge(self):
- super().test_ge()
-
- @unittest.skip("TODO(SPARK-43677): Fix DatetimeOps.gt to work with Spark
Connect Column.")
- def test_gt(self):
- super().test_gt()
-
- @unittest.skip("TODO(SPARK-43678): Fix DatetimeOps.le to work with Spark
Connect Column.")
- def test_le(self):
- super().test_le()
-
- @unittest.skip("TODO(SPARK-43679): Fix DatetimeOps.lt to work with Spark
Connect Column.")
- def test_lt(self):
- super().test_lt()
-
if __name__ == "__main__":
from pyspark.pandas.tests.connect.data_type_ops.test_parity_datetime_ops
import * # noqa: F401
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
