This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new b1c118f48ea4 [SPARK-50507][PYTHON][TESTS] Group pandas function
related tests
b1c118f48ea4 is described below
commit b1c118f48ea4dfe20eab0c269d670f3e0ddff07d
Author: Ruifeng Zheng <[email protected]>
AuthorDate: Thu Dec 5 17:50:00 2024 -0800
[SPARK-50507][PYTHON][TESTS] Group pandas function related tests
### What changes were proposed in this pull request?
Group pandas function related tests
### Why are the changes needed?
these tests were grouped in classic side, also group the parity tests for
consistency
### Does this PR introduce _any_ user-facing change?
no, test-only
### How was this patch tested?
updated ci
### Was this patch authored or co-authored using generative AI tooling?
no
Closes #49074 from zhengruifeng/group_connect_test_pandas.
Authored-by: Ruifeng Zheng <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
dev/sparktestsupport/modules.py | 18 +++++++++---------
.../__init__.py} | 20 --------------------
.../{ => pandas}/test_parity_pandas_cogrouped_map.py | 2 +-
.../{ => pandas}/test_parity_pandas_grouped_map.py | 2 +-
.../test_parity_pandas_grouped_map_with_state.py | 2 +-
.../connect/{ => pandas}/test_parity_pandas_map.py | 2 +-
.../connect/{ => pandas}/test_parity_pandas_udf.py | 2 +-
.../test_parity_pandas_udf_grouped_agg.py | 2 +-
.../{ => pandas}/test_parity_pandas_udf_scalar.py | 2 +-
.../{ => pandas}/test_parity_pandas_udf_window.py | 2 +-
.../sql/tests/{ => pandas}/test_pandas_sqlmetrics.py | 2 +-
11 files changed, 18 insertions(+), 38 deletions(-)
diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py
index b8702113a26c..e21a0c85de06 100644
--- a/dev/sparktestsupport/modules.py
+++ b/dev/sparktestsupport/modules.py
@@ -534,8 +534,8 @@ pyspark_sql = Module(
"pyspark.sql.tests.pandas.test_pandas_udf_typehints",
"pyspark.sql.tests.pandas.test_pandas_udf_typehints_with_future_annotations",
"pyspark.sql.tests.pandas.test_pandas_udf_window",
+ "pyspark.sql.tests.pandas.test_pandas_sqlmetrics",
"pyspark.sql.tests.pandas.test_converter",
- "pyspark.sql.tests.test_pandas_sqlmetrics",
"pyspark.sql.tests.test_python_datasource",
"pyspark.sql.tests.test_python_streaming_datasource",
"pyspark.sql.tests.test_readwriter",
@@ -1054,11 +1054,7 @@ pyspark_connect = Module(
"pyspark.sql.tests.connect.test_parity_memory_profiler",
"pyspark.sql.tests.connect.test_parity_udtf",
"pyspark.sql.tests.connect.test_parity_tvf",
- "pyspark.sql.tests.connect.test_parity_pandas_udf",
- "pyspark.sql.tests.connect.test_parity_pandas_map",
"pyspark.sql.tests.connect.test_parity_arrow_map",
- "pyspark.sql.tests.connect.test_parity_pandas_grouped_map",
- "pyspark.sql.tests.connect.test_parity_pandas_cogrouped_map",
"pyspark.sql.tests.connect.test_parity_arrow_grouped_map",
"pyspark.sql.tests.connect.test_parity_arrow_cogrouped_map",
"pyspark.sql.tests.connect.test_parity_python_datasource",
@@ -1074,13 +1070,17 @@ pyspark_connect = Module(
"pyspark.sql.tests.connect.streaming.test_parity_listener",
"pyspark.sql.tests.connect.streaming.test_parity_foreach",
"pyspark.sql.tests.connect.streaming.test_parity_foreach_batch",
- "pyspark.sql.tests.connect.test_parity_pandas_grouped_map_with_state",
- "pyspark.sql.tests.connect.test_parity_pandas_udf_scalar",
- "pyspark.sql.tests.connect.test_parity_pandas_udf_grouped_agg",
- "pyspark.sql.tests.connect.test_parity_pandas_udf_window",
"pyspark.sql.tests.connect.test_resources",
"pyspark.sql.tests.connect.shell.test_progress",
"pyspark.sql.tests.connect.test_df_debug",
+ "pyspark.sql.tests.connect.pandas.test_parity_pandas_map",
+ "pyspark.sql.tests.connect.pandas.test_parity_pandas_grouped_map",
+
"pyspark.sql.tests.connect.pandas.test_parity_pandas_grouped_map_with_state",
+ "pyspark.sql.tests.connect.pandas.test_parity_pandas_cogrouped_map",
+ "pyspark.sql.tests.connect.pandas.test_parity_pandas_udf",
+ "pyspark.sql.tests.connect.pandas.test_parity_pandas_udf_scalar",
+ "pyspark.sql.tests.connect.pandas.test_parity_pandas_udf_grouped_agg",
+ "pyspark.sql.tests.connect.pandas.test_parity_pandas_udf_window",
],
excluded_python_implementations=[
"PyPy" # Skip these tests under PyPy since they require numpy,
pandas, and pyarrow and
diff --git a/python/pyspark/sql/tests/connect/test_parity_pandas_udf.py
b/python/pyspark/sql/tests/connect/pandas/__init__.py
similarity index 56%
copy from python/pyspark/sql/tests/connect/test_parity_pandas_udf.py
copy to python/pyspark/sql/tests/connect/pandas/__init__.py
index 364e41716474..cce3acad34a4 100644
--- a/python/pyspark/sql/tests/connect/test_parity_pandas_udf.py
+++ b/python/pyspark/sql/tests/connect/pandas/__init__.py
@@ -14,23 +14,3 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-
-from pyspark.sql.tests.pandas.test_pandas_udf import PandasUDFTestsMixin
-from pyspark.testing.connectutils import ReusedConnectTestCase
-
-
-class PandasUDFParityTests(PandasUDFTestsMixin, ReusedConnectTestCase):
- pass
-
-
-if __name__ == "__main__":
- import unittest
- from pyspark.sql.tests.connect.test_parity_pandas_udf import * # noqa:
F401
-
- try:
- import xmlrunner # type: ignore[import]
-
- testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
- except ImportError:
- testRunner = None
- unittest.main(testRunner=testRunner, verbosity=2)
diff --git
a/python/pyspark/sql/tests/connect/test_parity_pandas_cogrouped_map.py
b/python/pyspark/sql/tests/connect/pandas/test_parity_pandas_cogrouped_map.py
similarity index 93%
rename from python/pyspark/sql/tests/connect/test_parity_pandas_cogrouped_map.py
rename to
python/pyspark/sql/tests/connect/pandas/test_parity_pandas_cogrouped_map.py
index 00d71bda2d93..a71e6369f5e2 100644
--- a/python/pyspark/sql/tests/connect/test_parity_pandas_cogrouped_map.py
+++
b/python/pyspark/sql/tests/connect/pandas/test_parity_pandas_cogrouped_map.py
@@ -28,7 +28,7 @@ class CogroupedApplyInPandasTests(
if __name__ == "__main__":
- from pyspark.sql.tests.connect.test_parity_pandas_cogrouped_map import *
# noqa: F401
+ from pyspark.sql.tests.connect.pandas.test_parity_pandas_cogrouped_map
import * # noqa: F401
try:
import xmlrunner
diff --git a/python/pyspark/sql/tests/connect/test_parity_pandas_grouped_map.py
b/python/pyspark/sql/tests/connect/pandas/test_parity_pandas_grouped_map.py
similarity index 94%
rename from python/pyspark/sql/tests/connect/test_parity_pandas_grouped_map.py
rename to
python/pyspark/sql/tests/connect/pandas/test_parity_pandas_grouped_map.py
index 8c76313c5c96..52110718808b 100644
--- a/python/pyspark/sql/tests/connect/test_parity_pandas_grouped_map.py
+++ b/python/pyspark/sql/tests/connect/pandas/test_parity_pandas_grouped_map.py
@@ -28,7 +28,7 @@ class
GroupedApplyInPandasTests(GroupedApplyInPandasTestsMixin, ReusedConnectTes
if __name__ == "__main__":
- from pyspark.sql.tests.connect.test_parity_pandas_grouped_map import * #
noqa: F401
+ from pyspark.sql.tests.connect.pandas.test_parity_pandas_grouped_map
import * # noqa: F401
try:
import xmlrunner
diff --git
a/python/pyspark/sql/tests/connect/test_parity_pandas_grouped_map_with_state.py
b/python/pyspark/sql/tests/connect/pandas/test_parity_pandas_grouped_map_with_state.py
similarity index 92%
rename from
python/pyspark/sql/tests/connect/test_parity_pandas_grouped_map_with_state.py
rename to
python/pyspark/sql/tests/connect/pandas/test_parity_pandas_grouped_map_with_state.py
index 67d42a7c8613..2da8b4aa3be8 100644
---
a/python/pyspark/sql/tests/connect/test_parity_pandas_grouped_map_with_state.py
+++
b/python/pyspark/sql/tests/connect/pandas/test_parity_pandas_grouped_map_with_state.py
@@ -29,7 +29,7 @@ class GroupedApplyInPandasWithStateTests(
if __name__ == "__main__":
- from pyspark.sql.tests.connect.test_parity_pandas_grouped_map_with_state
import * # noqa: F401
+ from
pyspark.sql.tests.connect.pandas.test_parity_pandas_grouped_map_with_state
import * # noqa: F401,E501
try:
import xmlrunner
diff --git a/python/pyspark/sql/tests/connect/test_parity_pandas_map.py
b/python/pyspark/sql/tests/connect/pandas/test_parity_pandas_map.py
similarity index 93%
rename from python/pyspark/sql/tests/connect/test_parity_pandas_map.py
rename to python/pyspark/sql/tests/connect/pandas/test_parity_pandas_map.py
index 999afd24c652..965ef5dcf894 100644
--- a/python/pyspark/sql/tests/connect/test_parity_pandas_map.py
+++ b/python/pyspark/sql/tests/connect/pandas/test_parity_pandas_map.py
@@ -28,7 +28,7 @@ class MapInPandasParityTests(
if __name__ == "__main__":
import unittest
- from pyspark.sql.tests.connect.test_parity_pandas_map import * # noqa:
F401
+ from pyspark.sql.tests.connect.pandas.test_parity_pandas_map import * #
noqa: F401
try:
import xmlrunner # type: ignore[import]
diff --git a/python/pyspark/sql/tests/connect/test_parity_pandas_udf.py
b/python/pyspark/sql/tests/connect/pandas/test_parity_pandas_udf.py
similarity index 93%
rename from python/pyspark/sql/tests/connect/test_parity_pandas_udf.py
rename to python/pyspark/sql/tests/connect/pandas/test_parity_pandas_udf.py
index 364e41716474..aa2b4748ff3f 100644
--- a/python/pyspark/sql/tests/connect/test_parity_pandas_udf.py
+++ b/python/pyspark/sql/tests/connect/pandas/test_parity_pandas_udf.py
@@ -25,7 +25,7 @@ class PandasUDFParityTests(PandasUDFTestsMixin,
ReusedConnectTestCase):
if __name__ == "__main__":
import unittest
- from pyspark.sql.tests.connect.test_parity_pandas_udf import * # noqa:
F401
+ from pyspark.sql.tests.connect.pandas.test_parity_pandas_udf import * #
noqa: F401
try:
import xmlrunner # type: ignore[import]
diff --git
a/python/pyspark/sql/tests/connect/test_parity_pandas_udf_grouped_agg.py
b/python/pyspark/sql/tests/connect/pandas/test_parity_pandas_udf_grouped_agg.py
similarity index 93%
rename from
python/pyspark/sql/tests/connect/test_parity_pandas_udf_grouped_agg.py
rename to
python/pyspark/sql/tests/connect/pandas/test_parity_pandas_udf_grouped_agg.py
index fdb81bffbce1..dfcb2b94c1bc 100644
--- a/python/pyspark/sql/tests/connect/test_parity_pandas_udf_grouped_agg.py
+++
b/python/pyspark/sql/tests/connect/pandas/test_parity_pandas_udf_grouped_agg.py
@@ -28,7 +28,7 @@ class PandasUDFGroupedAggParityTests(
if __name__ == "__main__":
- from pyspark.sql.tests.connect.test_parity_pandas_udf_grouped_agg import *
# noqa: F401
+ from pyspark.sql.tests.connect.pandas.test_parity_pandas_udf_grouped_agg
import * # noqa: F401
try:
import xmlrunner # type: ignore[import]
diff --git a/python/pyspark/sql/tests/connect/test_parity_pandas_udf_scalar.py
b/python/pyspark/sql/tests/connect/pandas/test_parity_pandas_udf_scalar.py
similarity index 93%
rename from python/pyspark/sql/tests/connect/test_parity_pandas_udf_scalar.py
rename to
python/pyspark/sql/tests/connect/pandas/test_parity_pandas_udf_scalar.py
index 451f0f68d6ee..9cab05f569d4 100644
--- a/python/pyspark/sql/tests/connect/test_parity_pandas_udf_scalar.py
+++ b/python/pyspark/sql/tests/connect/pandas/test_parity_pandas_udf_scalar.py
@@ -24,7 +24,7 @@ class PandasUDFScalarParityTests(ScalarPandasUDFTestsMixin,
ReusedConnectTestCas
if __name__ == "__main__":
- from pyspark.sql.tests.connect.test_parity_pandas_udf_scalar import * #
noqa: F401
+ from pyspark.sql.tests.connect.pandas.test_parity_pandas_udf_scalar import
* # noqa: F401
try:
import xmlrunner # type: ignore[import]
diff --git a/python/pyspark/sql/tests/connect/test_parity_pandas_udf_window.py
b/python/pyspark/sql/tests/connect/pandas/test_parity_pandas_udf_window.py
similarity index 93%
rename from python/pyspark/sql/tests/connect/test_parity_pandas_udf_window.py
rename to
python/pyspark/sql/tests/connect/pandas/test_parity_pandas_udf_window.py
index b2288c9d949e..08da1b4648b2 100644
--- a/python/pyspark/sql/tests/connect/test_parity_pandas_udf_window.py
+++ b/python/pyspark/sql/tests/connect/pandas/test_parity_pandas_udf_window.py
@@ -28,7 +28,7 @@ class PandasUDFWindowParityTests(
if __name__ == "__main__":
- from pyspark.sql.tests.connect.test_parity_pandas_udf_window import * #
noqa: F401
+ from pyspark.sql.tests.connect.pandas.test_parity_pandas_udf_window import
* # noqa: F401
try:
import xmlrunner # type: ignore[import]
diff --git a/python/pyspark/sql/tests/test_pandas_sqlmetrics.py
b/python/pyspark/sql/tests/pandas/test_pandas_sqlmetrics.py
similarity index 96%
rename from python/pyspark/sql/tests/test_pandas_sqlmetrics.py
rename to python/pyspark/sql/tests/pandas/test_pandas_sqlmetrics.py
index 22a0e92e818d..cb2f2ff28568 100644
--- a/python/pyspark/sql/tests/test_pandas_sqlmetrics.py
+++ b/python/pyspark/sql/tests/pandas/test_pandas_sqlmetrics.py
@@ -57,7 +57,7 @@ class PandasSQLMetrics(ReusedSQLTestCase):
if __name__ == "__main__":
- from pyspark.sql.tests.test_pandas_sqlmetrics import * # noqa: F401
+ from pyspark.sql.tests.pandas.test_pandas_sqlmetrics import * # noqa: F401
try:
import xmlrunner
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
