This is an automated email from the ASF dual-hosted git repository.
xinrong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 79918028b142 [SPARK-46867][PYTHON][CONNECT][TESTS] Remove unnecessary
dependency from test_mixed_udf_and_sql.py
79918028b142 is described below
commit 79918028b142685fe1c3871a3593e91100ab6bbf
Author: Xinrong Meng <[email protected]>
AuthorDate: Thu Jan 25 14:16:12 2024 -0800
[SPARK-46867][PYTHON][CONNECT][TESTS] Remove unnecessary dependency from
test_mixed_udf_and_sql.py
### What changes were proposed in this pull request?
Remove unnecessary dependency from test_mixed_udf_and_sql.py.
### Why are the changes needed?
Otherwise, test_mixed_udf_and_sql.py depends on Spark Connect's dependency
"grpc", possibly leading to conflicts or compatibility issues.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Test change only.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #44886 from xinrong-meng/fix_dep.
Authored-by: Xinrong Meng <[email protected]>
Signed-off-by: Xinrong Meng <[email protected]>
---
python/pyspark/sql/tests/connect/test_parity_pandas_udf_scalar.py | 4 ++++
python/pyspark/sql/tests/pandas/test_pandas_udf_scalar.py | 5 +++--
2 files changed, 7 insertions(+), 2 deletions(-)
diff --git a/python/pyspark/sql/tests/connect/test_parity_pandas_udf_scalar.py
b/python/pyspark/sql/tests/connect/test_parity_pandas_udf_scalar.py
index c950ca2e17c3..6a3d03246549 100644
--- a/python/pyspark/sql/tests/connect/test_parity_pandas_udf_scalar.py
+++ b/python/pyspark/sql/tests/connect/test_parity_pandas_udf_scalar.py
@@ -15,6 +15,7 @@
# limitations under the License.
#
import unittest
+from pyspark.sql.connect.column import Column
from pyspark.sql.tests.pandas.test_pandas_udf_scalar import
ScalarPandasUDFTestsMixin
from pyspark.testing.connectutils import ReusedConnectTestCase
@@ -51,6 +52,9 @@ class PandasUDFScalarParityTests(ScalarPandasUDFTestsMixin,
ReusedConnectTestCas
def test_vectorized_udf_invalid_length(self):
self.check_vectorized_udf_invalid_length()
+ def test_mixed_udf_and_sql(self):
+ self._test_mixed_udf_and_sql(Column)
+
if __name__ == "__main__":
from pyspark.sql.tests.connect.test_parity_pandas_udf_scalar import * #
noqa: F401
diff --git a/python/pyspark/sql/tests/pandas/test_pandas_udf_scalar.py
b/python/pyspark/sql/tests/pandas/test_pandas_udf_scalar.py
index dfbab5c8b3cd..9f6bdb83caf7 100644
--- a/python/pyspark/sql/tests/pandas/test_pandas_udf_scalar.py
+++ b/python/pyspark/sql/tests/pandas/test_pandas_udf_scalar.py
@@ -1321,8 +1321,9 @@ class ScalarPandasUDFTestsMixin:
self.assertEqual(expected_multi, df_multi_2.collect())
def test_mixed_udf_and_sql(self):
- from pyspark.sql.connect.column import Column as ConnectColumn
+ self._test_mixed_udf_and_sql(Column)
+ def _test_mixed_udf_and_sql(self, col_type):
df = self.spark.range(0, 1).toDF("v")
# Test mixture of UDFs, Pandas UDFs and SQL expression.
@@ -1333,7 +1334,7 @@ class ScalarPandasUDFTestsMixin:
return x + 1
def f2(x):
- assert type(x) in (Column, ConnectColumn)
+ assert type(x) == col_type
return x + 10
@pandas_udf("int")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
