This is an automated email from the ASF dual-hosted git repository.
dbtsai pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 882d4a4fe150 [SPARK-57397][PYTHON] Raise an exception when parsed UDT
is the wrong type
882d4a4fe150 is described below
commit 882d4a4fe150ac6b3c0bf4dd94ff2af1151395df
Author: Tian Gao <[email protected]>
AuthorDate: Fri Jun 12 10:09:17 2026 -0700
[SPARK-57397][PYTHON] Raise an exception when parsed UDT is the wrong type
### What changes were proposed in this pull request?
Check whether the imported class is a real UserDefinedType before calling
it.
### Why are the changes needed?
If it's not a UDT, it could run wrong code.
### Does this PR introduce _any_ user-facing change?
Not for normal use.
### How was this patch tested?
Regression test added.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #56460 from gaogaotiantian/add-import-type-check.
Authored-by: Tian Gao <[email protected]>
Signed-off-by: DB Tsai <[email protected]>
(cherry picked from commit 1f5367459d790e868640ed2f9fe1bf0c6b776b63)
Signed-off-by: DB Tsai <[email protected]>
---
python/pyspark/sql/tests/test_types.py | 11 +++++++++++
python/pyspark/sql/types.py | 5 +++++
2 files changed, 16 insertions(+)
diff --git a/python/pyspark/sql/tests/test_types.py
b/python/pyspark/sql/tests/test_types.py
index 0a5219202a3a..fd5249e1ac7a 100644
--- a/python/pyspark/sql/tests/test_types.py
+++ b/python/pyspark/sql/tests/test_types.py
@@ -67,6 +67,7 @@ from pyspark.sql.types import (
GeographyType,
GeometryType,
NullType,
+ UserDefinedType,
VariantType,
VariantVal,
)
@@ -1055,6 +1056,16 @@ class TypesTestsMixin:
_make_type_verifier(PythonOnlyUDT())(PythonOnlyPoint(1.0, 2.0))
self.assertRaises(ValueError, lambda:
_make_type_verifier(PythonOnlyUDT())([1.0, 2.0]))
+ def test_udt_from_json_import_type_mismatch(self):
+ json = {
+ "type": "udt",
+ "pyClass": "random.random",
+ "serializedClass": "",
+ "sqlType": StringType().jsonValue(),
+ }
+ with self.assertRaises(PySparkTypeError):
+ UserDefinedType.fromJson(json)
+
def test_simple_udt_in_df(self):
schema = StructType().add("key", LongType()).add("val",
PythonOnlyUDT())
df = self.spark.createDataFrame(
diff --git a/python/pyspark/sql/types.py b/python/pyspark/sql/types.py
index 41631fca0ac2..0b4fc95b4a80 100644
--- a/python/pyspark/sql/types.py
+++ b/python/pyspark/sql/types.py
@@ -1996,6 +1996,11 @@ class UserDefinedType(DataType):
)
else:
UDT = getattr(m, pyClass)
+ if not (isinstance(UDT, type) and issubclass(UDT,
UserDefinedType)):
+ raise PySparkTypeError(
+ errorClass="FIELD_TYPE_MISMATCH",
+ messageParameters={"obj": str(UDT), "data_type":
"UserDefinedType"},
+ )
return UDT()
def __eq__(self, other: Any) -> bool:
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
