This is an automated email from the ASF dual-hosted git repository.
gaogaotiantian pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new e80e483ddd14 [SPARK-58768][PYTHON][TEST] Fix monkey-patch in
test_parity_udf and test_parity_udtf
e80e483ddd14 is described below
commit e80e483ddd14b1260d8dd233cbb16231959d19e8
Author: Tian Gao <[email protected]>
AuthorDate: Fri Aug 14 18:46:00 2026 -0700
[SPARK-58768][PYTHON][TEST] Fix monkey-patch in test_parity_udf and
test_parity_udtf
### What changes were proposed in this pull request?
* For `test_parity_udtf`:
* Removed the unnecessary monkeypatch for
`sql.udtf.UserDefinedTableFunction`
* Use the general version builtin function because it should be
automatically redirected
* For `test_parity_udf`:
* Move the monkey patch to `setUpClass`
* Patch the test module, instead of `pyspark.sql`
### Why are the changes needed?
The original monkey-patch has a large blast radius and is sensitive to
import order. We should minimum monkey patch impact.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
CI.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #58001 from gaogaotiantian/fix-monkey-patch.
Authored-by: Tian Gao <[email protected]>
Signed-off-by: Tian Gao <[email protected]>
(cherry picked from commit 76735aaf5dd6a794259c63d91e25dc150210157c)
Signed-off-by: Tian Gao <[email protected]>
---
python/pyspark/sql/tests/connect/test_parity_udf.py | 18 ++++++++++--------
python/pyspark/sql/tests/connect/test_parity_udtf.py | 6 +-----
2 files changed, 11 insertions(+), 13 deletions(-)
diff --git a/python/pyspark/sql/tests/connect/test_parity_udf.py
b/python/pyspark/sql/tests/connect/test_parity_udf.py
index 33ec11adcf74..e3dffaf483f7 100644
--- a/python/pyspark/sql/tests/connect/test_parity_udf.py
+++ b/python/pyspark/sql/tests/connect/test_parity_udf.py
@@ -17,19 +17,21 @@
import unittest
-from pyspark.testing.connectutils import should_test_connect
-
-if should_test_connect:
- from pyspark import sql
- from pyspark.sql.connect.udf import UserDefinedFunction
-
- sql.udf.UserDefinedFunction = UserDefinedFunction
-
from pyspark.sql.tests.test_udf import BaseUDFTestsMixin
from pyspark.testing.connectutils import ReusedConnectTestCase
class UDFParityTests(BaseUDFTestsMixin, ReusedConnectTestCase):
+ @classmethod
+ def setUpClass(cls):
+ # test_udf uses UserDefinedFunction so we need to monkeypatch it
+ from pyspark.sql.connect.udf import UserDefinedFunction
+ import pyspark.sql.tests.test_udf
+
+ pyspark.sql.tests.test_udf.UserDefinedFunction = UserDefinedFunction
+
+ super().setUpClass()
+
@unittest.skip("Spark Connect does not support mapPartitions() but the
test depends on it.")
def test_worker_original_stdin_closed(self):
super().test_worker_original_stdin_closed()
diff --git a/python/pyspark/sql/tests/connect/test_parity_udtf.py
b/python/pyspark/sql/tests/connect/test_parity_udtf.py
index 2ab0e733a4be..9fce00983308 100644
--- a/python/pyspark/sql/tests/connect/test_parity_udtf.py
+++ b/python/pyspark/sql/tests/connect/test_parity_udtf.py
@@ -18,6 +18,7 @@ import os
import unittest
from pyspark.testing.connectutils import should_test_connect
+from pyspark.sql.functions import lit, udtf
from pyspark.sql.tests.test_udtf import (
BaseUDTFTestsMixin,
UDTFArrowTestsMixin,
@@ -26,11 +27,6 @@ from pyspark.sql.tests.test_udtf import (
from pyspark.testing.connectutils import ReusedConnectTestCase
if should_test_connect:
- from pyspark import sql
- from pyspark.sql.connect.udtf import UserDefinedTableFunction
-
- sql.udtf.UserDefinedTableFunction = UserDefinedTableFunction
- from pyspark.sql.connect.functions import lit, udtf
from pyspark.errors.exceptions.connect import (
PickleException,
PythonException,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
