This is an automated email from the ASF dual-hosted git repository.
ruifengz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 20b6a32f4980 Revert "[SPARK-46593][PS][TESTS] Refactor `data_type_ops`
tests"
20b6a32f4980 is described below
commit 20b6a32f498093405a5ab758fcae2829ce4a17b3
Author: Ruifeng Zheng <ruife...@apache.org>
AuthorDate: Fri Jan 5 14:08:15 2024 +0800
Revert "[SPARK-46593][PS][TESTS] Refactor `data_type_ops` tests"
### What changes were proposed in this pull request?
revert https://github.com/apache/spark/pull/44592 which happened to disable
the parity tests
they should be run on connect mode, but actually run on vanilla mode
Closes #44605 from zhengruifeng/revert_46593.
Authored-by: Ruifeng Zheng <ruife...@apache.org>
Signed-off-by: Ruifeng Zheng <ruife...@apache.org>
---
.../tests/connect/data_type_ops/test_parity_as_type.py | 12 +++++++-----
.../pandas/tests/connect/data_type_ops/test_parity_base.py | 5 +----
.../tests/connect/data_type_ops/test_parity_binary_ops.py | 7 +++----
.../tests/connect/data_type_ops/test_parity_boolean_ops.py | 12 +++++++-----
.../connect/data_type_ops/test_parity_categorical_ops.py | 12 +++++++-----
.../tests/connect/data_type_ops/test_parity_complex_ops.py | 7 +++----
.../tests/connect/data_type_ops/test_parity_date_ops.py | 12 +++++++-----
.../tests/connect/data_type_ops/test_parity_datetime_ops.py | 12 +++++++-----
.../tests/connect/data_type_ops/test_parity_null_ops.py | 7 +++----
.../connect/data_type_ops/test_parity_num_arithmetic.py | 12 +++++++-----
.../tests/connect/data_type_ops/test_parity_num_ops.py | 12 +++++++-----
.../tests/connect/data_type_ops/test_parity_num_reverse.py | 12 +++++++-----
.../tests/connect/data_type_ops/test_parity_string_ops.py | 12 +++++++-----
.../tests/connect/data_type_ops/test_parity_timedelta_ops.py | 12 +++++++-----
.../tests/connect/data_type_ops/test_parity_udt_ops.py | 7 +++----
.../tests/{ => connect}/data_type_ops/testing_utils.py | 7 +------
python/pyspark/pandas/tests/data_type_ops/testing_utils.py | 8 ++------
17 files changed, 86 insertions(+), 82 deletions(-)
diff --git
a/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_as_type.py
b/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_as_type.py
index 7feecf3db76e..a2a9e28a5ab5 100644
--- a/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_as_type.py
+++ b/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_as_type.py
@@ -16,17 +16,19 @@
#
import unittest
+from pyspark import pandas as ps
from pyspark.pandas.tests.data_type_ops.test_as_type import AsTypeTestsMixin
-from pyspark.pandas.tests.data_type_ops.testing_utils import OpsTestBase
+from pyspark.pandas.tests.connect.data_type_ops.testing_utils import
OpsTestBase
+from pyspark.testing.pandasutils import PandasOnSparkTestUtils
from pyspark.testing.connectutils import ReusedConnectTestCase
class AsTypeParityTests(
- AsTypeTestsMixin,
- OpsTestBase,
- ReusedConnectTestCase,
+ AsTypeTestsMixin, PandasOnSparkTestUtils, OpsTestBase,
ReusedConnectTestCase
):
- pass
+ @property
+ def psdf(self):
+ return ps.from_pandas(self.pdf)
if __name__ == "__main__":
diff --git
a/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_base.py
b/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_base.py
index 1623db58af38..c277f5ce0664 100644
--- a/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_base.py
+++ b/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_base.py
@@ -20,10 +20,7 @@ from pyspark.pandas.tests.data_type_ops.test_base import
BaseTestsMixin
from pyspark.testing.connectutils import ReusedConnectTestCase
-class BaseParityTests(
- BaseTestsMixin,
- ReusedConnectTestCase,
-):
+class BaseParityTests(BaseTestsMixin, ReusedConnectTestCase):
pass
diff --git
a/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_binary_ops.py
b/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_binary_ops.py
index 9af064218965..29b13868e03f 100644
---
a/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_binary_ops.py
+++
b/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_binary_ops.py
@@ -17,14 +17,13 @@
import unittest
from pyspark.pandas.tests.data_type_ops.test_binary_ops import
BinaryOpsTestsMixin
-from pyspark.pandas.tests.data_type_ops.testing_utils import OpsTestBase
+from pyspark.pandas.tests.connect.data_type_ops.testing_utils import
OpsTestBase
+from pyspark.testing.pandasutils import PandasOnSparkTestUtils
from pyspark.testing.connectutils import ReusedConnectTestCase
class BinaryOpsParityTests(
- BinaryOpsTestsMixin,
- OpsTestBase,
- ReusedConnectTestCase,
+ BinaryOpsTestsMixin, PandasOnSparkTestUtils, OpsTestBase,
ReusedConnectTestCase
):
pass
diff --git
a/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_boolean_ops.py
b/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_boolean_ops.py
index 6804768585ae..9ad2aa0ad17a 100644
---
a/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_boolean_ops.py
+++
b/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_boolean_ops.py
@@ -16,17 +16,19 @@
#
import unittest
+from pyspark import pandas as ps
from pyspark.pandas.tests.data_type_ops.test_boolean_ops import
BooleanOpsTestsMixin
-from pyspark.pandas.tests.data_type_ops.testing_utils import OpsTestBase
+from pyspark.pandas.tests.connect.data_type_ops.testing_utils import
OpsTestBase
+from pyspark.testing.pandasutils import PandasOnSparkTestUtils
from pyspark.testing.connectutils import ReusedConnectTestCase
class BooleanOpsParityTests(
- BooleanOpsTestsMixin,
- OpsTestBase,
- ReusedConnectTestCase,
+ BooleanOpsTestsMixin, PandasOnSparkTestUtils, OpsTestBase,
ReusedConnectTestCase
):
- pass
+ @property
+ def psdf(self):
+ return ps.from_pandas(self.pdf)
if __name__ == "__main__":
diff --git
a/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_categorical_ops.py
b/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_categorical_ops.py
index 851aaaa171b8..1b4dabdb0453 100644
---
a/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_categorical_ops.py
+++
b/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_categorical_ops.py
@@ -16,17 +16,19 @@
#
import unittest
+from pyspark import pandas as ps
from pyspark.pandas.tests.data_type_ops.test_categorical_ops import
CategoricalOpsTestsMixin
-from pyspark.pandas.tests.data_type_ops.testing_utils import OpsTestBase
+from pyspark.pandas.tests.connect.data_type_ops.testing_utils import
OpsTestBase
+from pyspark.testing.pandasutils import PandasOnSparkTestUtils
from pyspark.testing.connectutils import ReusedConnectTestCase
class CategoricalOpsParityTests(
- CategoricalOpsTestsMixin,
- OpsTestBase,
- ReusedConnectTestCase,
+ CategoricalOpsTestsMixin, PandasOnSparkTestUtils, OpsTestBase,
ReusedConnectTestCase
):
- pass
+ @property
+ def psdf(self):
+ return ps.from_pandas(self.pdf)
if __name__ == "__main__":
diff --git
a/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_complex_ops.py
b/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_complex_ops.py
index 1877be058305..ef587578f4ae 100644
---
a/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_complex_ops.py
+++
b/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_complex_ops.py
@@ -17,14 +17,13 @@
import unittest
from pyspark.pandas.tests.data_type_ops.test_complex_ops import
ComplexOpsTestsMixin
-from pyspark.pandas.tests.data_type_ops.testing_utils import OpsTestBase
+from pyspark.pandas.tests.connect.data_type_ops.testing_utils import
OpsTestBase
+from pyspark.testing.pandasutils import PandasOnSparkTestUtils
from pyspark.testing.connectutils import ReusedConnectTestCase
class ComplexOpsParityTests(
- ComplexOpsTestsMixin,
- OpsTestBase,
- ReusedConnectTestCase,
+ ComplexOpsTestsMixin, PandasOnSparkTestUtils, OpsTestBase,
ReusedConnectTestCase
):
pass
diff --git
a/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_date_ops.py
b/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_date_ops.py
index ddc1545590ef..baa3180baaa7 100644
--- a/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_date_ops.py
+++ b/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_date_ops.py
@@ -16,17 +16,19 @@
#
import unittest
+from pyspark import pandas as ps
from pyspark.pandas.tests.data_type_ops.test_date_ops import DateOpsTestsMixin
-from pyspark.pandas.tests.data_type_ops.testing_utils import OpsTestBase
+from pyspark.pandas.tests.connect.data_type_ops.testing_utils import
OpsTestBase
+from pyspark.testing.pandasutils import PandasOnSparkTestUtils
from pyspark.testing.connectutils import ReusedConnectTestCase
class DateOpsParityTests(
- DateOpsTestsMixin,
- OpsTestBase,
- ReusedConnectTestCase,
+ DateOpsTestsMixin, PandasOnSparkTestUtils, OpsTestBase,
ReusedConnectTestCase
):
- pass
+ @property
+ def psdf(self):
+ return ps.from_pandas(self.pdf)
if __name__ == "__main__":
diff --git
a/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_datetime_ops.py
b/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_datetime_ops.py
index 2a7395248567..2641e3a32dcd 100644
---
a/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_datetime_ops.py
+++
b/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_datetime_ops.py
@@ -16,17 +16,19 @@
#
import unittest
+from pyspark import pandas as ps
from pyspark.pandas.tests.data_type_ops.test_datetime_ops import
DatetimeOpsTestsMixin
-from pyspark.pandas.tests.data_type_ops.testing_utils import OpsTestBase
+from pyspark.pandas.tests.connect.data_type_ops.testing_utils import
OpsTestBase
+from pyspark.testing.pandasutils import PandasOnSparkTestUtils
from pyspark.testing.connectutils import ReusedConnectTestCase
class DatetimeOpsParityTests(
- DatetimeOpsTestsMixin,
- OpsTestBase,
- ReusedConnectTestCase,
+ DatetimeOpsTestsMixin, PandasOnSparkTestUtils, OpsTestBase,
ReusedConnectTestCase
):
- pass
+ @property
+ def psdf(self):
+ return ps.from_pandas(self.pdf)
if __name__ == "__main__":
diff --git
a/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_null_ops.py
b/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_null_ops.py
index 5c27fa6d3e4e..5df4c791c98b 100644
--- a/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_null_ops.py
+++ b/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_null_ops.py
@@ -17,14 +17,13 @@
import unittest
from pyspark.pandas.tests.data_type_ops.test_null_ops import NullOpsTestsMixin
-from pyspark.pandas.tests.data_type_ops.testing_utils import OpsTestBase
+from pyspark.pandas.tests.connect.data_type_ops.testing_utils import
OpsTestBase
+from pyspark.testing.pandasutils import PandasOnSparkTestUtils
from pyspark.testing.connectutils import ReusedConnectTestCase
class NullOpsParityTests(
- NullOpsTestsMixin,
- OpsTestBase,
- ReusedConnectTestCase,
+ NullOpsTestsMixin, PandasOnSparkTestUtils, OpsTestBase,
ReusedConnectTestCase
):
pass
diff --git
a/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_num_arithmetic.py
b/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_num_arithmetic.py
index aa98a3009dc4..6f5c294e4ad5 100644
---
a/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_num_arithmetic.py
+++
b/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_num_arithmetic.py
@@ -16,17 +16,19 @@
#
import unittest
+from pyspark import pandas as ps
from pyspark.pandas.tests.data_type_ops.test_num_arithmetic import
ArithmeticTestsMixin
-from pyspark.pandas.tests.data_type_ops.testing_utils import OpsTestBase
+from pyspark.pandas.tests.connect.data_type_ops.testing_utils import
OpsTestBase
+from pyspark.testing.pandasutils import PandasOnSparkTestUtils
from pyspark.testing.connectutils import ReusedConnectTestCase
class ArithmeticParityTests(
- ArithmeticTestsMixin,
- OpsTestBase,
- ReusedConnectTestCase,
+ ArithmeticTestsMixin, PandasOnSparkTestUtils, OpsTestBase,
ReusedConnectTestCase
):
- pass
+ @property
+ def psdf(self):
+ return ps.from_pandas(self.pdf)
if __name__ == "__main__":
diff --git
a/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_num_ops.py
b/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_num_ops.py
index d66a5f155f81..56eba708c945 100644
--- a/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_num_ops.py
+++ b/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_num_ops.py
@@ -16,17 +16,19 @@
#
import unittest
+from pyspark import pandas as ps
from pyspark.pandas.tests.data_type_ops.test_num_ops import NumOpsTestsMixin
-from pyspark.pandas.tests.data_type_ops.testing_utils import OpsTestBase
+from pyspark.pandas.tests.connect.data_type_ops.testing_utils import
OpsTestBase
+from pyspark.testing.pandasutils import PandasOnSparkTestUtils
from pyspark.testing.connectutils import ReusedConnectTestCase
class NumOpsParityTests(
- NumOpsTestsMixin,
- OpsTestBase,
- ReusedConnectTestCase,
+ NumOpsTestsMixin, PandasOnSparkTestUtils, OpsTestBase,
ReusedConnectTestCase
):
- pass
+ @property
+ def psdf(self):
+ return ps.from_pandas(self.pdf)
if __name__ == "__main__":
diff --git
a/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_num_reverse.py
b/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_num_reverse.py
index 5983c9a2c4f5..4d322d8b9b06 100644
---
a/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_num_reverse.py
+++
b/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_num_reverse.py
@@ -16,17 +16,19 @@
#
import unittest
+from pyspark import pandas as ps
from pyspark.pandas.tests.data_type_ops.test_num_reverse import
ReverseTestsMixin
-from pyspark.pandas.tests.data_type_ops.testing_utils import OpsTestBase
+from pyspark.pandas.tests.connect.data_type_ops.testing_utils import
OpsTestBase
+from pyspark.testing.pandasutils import PandasOnSparkTestUtils
from pyspark.testing.connectutils import ReusedConnectTestCase
class ReverseParityTests(
- ReverseTestsMixin,
- OpsTestBase,
- ReusedConnectTestCase,
+ ReverseTestsMixin, PandasOnSparkTestUtils, OpsTestBase,
ReusedConnectTestCase
):
- pass
+ @property
+ def psdf(self):
+ return ps.from_pandas(self.pdf)
if __name__ == "__main__":
diff --git
a/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_string_ops.py
b/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_string_ops.py
index e80971e12121..f507756a7a48 100644
---
a/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_string_ops.py
+++
b/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_string_ops.py
@@ -16,17 +16,19 @@
#
import unittest
+from pyspark import pandas as ps
from pyspark.pandas.tests.data_type_ops.test_string_ops import
StringOpsTestsMixin
-from pyspark.pandas.tests.data_type_ops.testing_utils import OpsTestBase
+from pyspark.pandas.tests.connect.data_type_ops.testing_utils import
OpsTestBase
+from pyspark.testing.pandasutils import PandasOnSparkTestUtils
from pyspark.testing.connectutils import ReusedConnectTestCase
class StringOpsParityTests(
- StringOpsTestsMixin,
- OpsTestBase,
- ReusedConnectTestCase,
+ StringOpsTestsMixin, PandasOnSparkTestUtils, OpsTestBase,
ReusedConnectTestCase
):
- pass
+ @property
+ def psdf(self):
+ return ps.from_pandas(self.pdf)
if __name__ == "__main__":
diff --git
a/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_timedelta_ops.py
b/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_timedelta_ops.py
index 3c5a3b217352..edd29fa1ed28 100644
---
a/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_timedelta_ops.py
+++
b/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_timedelta_ops.py
@@ -16,17 +16,19 @@
#
import unittest
+import pyspark.pandas as ps
from pyspark.pandas.tests.data_type_ops.test_timedelta_ops import
TimedeltaOpsTestsMixin
-from pyspark.pandas.tests.data_type_ops.testing_utils import OpsTestBase
+from pyspark.pandas.tests.connect.data_type_ops.testing_utils import
OpsTestBase
+from pyspark.testing.pandasutils import PandasOnSparkTestUtils
from pyspark.testing.connectutils import ReusedConnectTestCase
class TimedeltaOpsParityTests(
- TimedeltaOpsTestsMixin,
- OpsTestBase,
- ReusedConnectTestCase,
+ TimedeltaOpsTestsMixin, PandasOnSparkTestUtils, OpsTestBase,
ReusedConnectTestCase
):
- pass
+ @property
+ def psdf(self):
+ return ps.from_pandas(self.pdf)
if __name__ == "__main__":
diff --git
a/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_udt_ops.py
b/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_udt_ops.py
index 237079922222..70a79e4cd3f9 100644
--- a/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_udt_ops.py
+++ b/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_udt_ops.py
@@ -17,14 +17,13 @@
import unittest
from pyspark.pandas.tests.data_type_ops.test_udt_ops import UDTOpsTestsMixin
-from pyspark.pandas.tests.data_type_ops.testing_utils import OpsTestBase
+from pyspark.pandas.tests.connect.data_type_ops.testing_utils import
OpsTestBase
+from pyspark.testing.pandasutils import PandasOnSparkTestUtils
from pyspark.testing.connectutils import ReusedConnectTestCase
class UDTOpsParityTests(
- UDTOpsTestsMixin,
- OpsTestBase,
- ReusedConnectTestCase,
+ UDTOpsTestsMixin, PandasOnSparkTestUtils, OpsTestBase,
ReusedConnectTestCase
):
pass
diff --git a/python/pyspark/pandas/tests/data_type_ops/testing_utils.py
b/python/pyspark/pandas/tests/connect/data_type_ops/testing_utils.py
similarity index 97%
copy from python/pyspark/pandas/tests/data_type_ops/testing_utils.py
copy to python/pyspark/pandas/tests/connect/data_type_ops/testing_utils.py
index 25eb97328d47..f1e36aecd194 100644
--- a/python/pyspark/pandas/tests/data_type_ops/testing_utils.py
+++ b/python/pyspark/pandas/tests/connect/data_type_ops/testing_utils.py
@@ -27,7 +27,6 @@ from pyspark.pandas.typedef.typehints import (
extension_float_dtypes_available,
extension_object_dtypes_available,
)
-from pyspark.testing.pandasutils import PandasOnSparkTestCase
if extension_dtypes_available:
from pandas import Int8Dtype, Int16Dtype, Int32Dtype, Int64Dtype
@@ -39,7 +38,7 @@ if extension_object_dtypes_available:
from pandas import BooleanDtype, StringDtype
-class OpsTestBase(PandasOnSparkTestCase):
+class OpsTestBase:
"""The test base for arithmetic operations of different data types."""
@property
@@ -101,10 +100,6 @@ class OpsTestBase(PandasOnSparkTestCase):
def pdf(self):
return pd.concat([self.numeric_pdf, self.non_numeric_pdf], axis=1)
- @property
- def psdf(self):
- return ps.from_pandas(self.pdf)
-
@property
def df_cols(self):
return self.pdf.columns
diff --git a/python/pyspark/pandas/tests/data_type_ops/testing_utils.py
b/python/pyspark/pandas/tests/data_type_ops/testing_utils.py
index 25eb97328d47..37a708948a80 100644
--- a/python/pyspark/pandas/tests/data_type_ops/testing_utils.py
+++ b/python/pyspark/pandas/tests/data_type_ops/testing_utils.py
@@ -27,7 +27,7 @@ from pyspark.pandas.typedef.typehints import (
extension_float_dtypes_available,
extension_object_dtypes_available,
)
-from pyspark.testing.pandasutils import PandasOnSparkTestCase
+from pyspark.testing.pandasutils import ComparisonTestBase
if extension_dtypes_available:
from pandas import Int8Dtype, Int16Dtype, Int32Dtype, Int64Dtype
@@ -39,7 +39,7 @@ if extension_object_dtypes_available:
from pandas import BooleanDtype, StringDtype
-class OpsTestBase(PandasOnSparkTestCase):
+class OpsTestBase(ComparisonTestBase):
"""The test base for arithmetic operations of different data types."""
@property
@@ -101,10 +101,6 @@ class OpsTestBase(PandasOnSparkTestCase):
def pdf(self):
return pd.concat([self.numeric_pdf, self.non_numeric_pdf], axis=1)
- @property
- def psdf(self):
- return ps.from_pandas(self.pdf)
-
@property
def df_cols(self):
return self.pdf.columns
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
