This is an automated email from the ASF dual-hosted git repository.
dbtsai pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new 9c7363a16e17 [SPARK-57397][PYTHON] Raise an exception when parsed UDT
is the wrong type
9c7363a16e17 is described below
commit 9c7363a16e17a640b4f524035813b2d8a5c55daa
Author: Tian Gao <[email protected]>
AuthorDate: Fri Jun 12 10:09:17 2026 -0700
[SPARK-57397][PYTHON] Raise an exception when parsed UDT is the wrong type
### What changes were proposed in this pull request?
Check whether the imported class is a real UserDefinedType before calling
it.
### Why are the changes needed?
If it's not a UDT, it could run wrong code.
### Does this PR introduce _any_ user-facing change?
Not for normal use.
### How was this patch tested?
Regression test added.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #56460 from gaogaotiantian/add-import-type-check.
Authored-by: Tian Gao <[email protected]>
Signed-off-by: DB Tsai <[email protected]>
(cherry picked from commit 1f5367459d790e868640ed2f9fe1bf0c6b776b63)
Signed-off-by: DB Tsai <[email protected]>
---
python/pyspark/sql/tests/test_types.py | 11 +++++++++++
python/pyspark/sql/types.py | 5 +++++
2 files changed, 16 insertions(+)
diff --git a/python/pyspark/sql/tests/test_types.py
b/python/pyspark/sql/tests/test_types.py
index f1c6ad11707e..754cd256e99a 100644
--- a/python/pyspark/sql/tests/test_types.py
+++ b/python/pyspark/sql/tests/test_types.py
@@ -60,6 +60,7 @@ from pyspark.sql.types import (
BinaryType,
BooleanType,
NullType,
+ UserDefinedType,
VariantType,
VariantVal,
)
@@ -930,6 +931,16 @@ class TypesTestsMixin:
_make_type_verifier(PythonOnlyUDT())(PythonOnlyPoint(1.0, 2.0))
self.assertRaises(ValueError, lambda:
_make_type_verifier(PythonOnlyUDT())([1.0, 2.0]))
+ def test_udt_from_json_import_type_mismatch(self):
+ json = {
+ "type": "udt",
+ "pyClass": "random.random",
+ "serializedClass": "",
+ "sqlType": StringType().jsonValue(),
+ }
+ with self.assertRaises(PySparkTypeError):
+ UserDefinedType.fromJson(json)
+
def test_simple_udt_in_df(self):
schema = StructType().add("key", LongType()).add("val",
PythonOnlyUDT())
df = self.spark.createDataFrame(
diff --git a/python/pyspark/sql/types.py b/python/pyspark/sql/types.py
index 1cf5ac7acd71..315bbce517f0 100644
--- a/python/pyspark/sql/types.py
+++ b/python/pyspark/sql/types.py
@@ -1731,6 +1731,11 @@ class UserDefinedType(DataType):
)
else:
UDT = getattr(m, pyClass)
+ if not (isinstance(UDT, type) and issubclass(UDT,
UserDefinedType)):
+ raise PySparkTypeError(
+ errorClass="FIELD_TYPE_MISMATCH",
+ messageParameters={"obj": str(UDT), "data_type":
"UserDefinedType"},
+ )
return UDT()
def __eq__(self, other: Any) -> bool:
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
