This is an automated email from the ASF dual-hosted git repository.
gaogaotiantian pushed a commit to branch branch-3.5
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.5 by this push:
new 0fff94ef4cd2 [SPARK-57397][3.5][PYTHON] Raise an exception when parsed
UDT is the wrong type
0fff94ef4cd2 is described below
commit 0fff94ef4cd21f7dcf020e3500e1a35e0556171f
Author: Tian Gao <[email protected]>
AuthorDate: Mon Jun 15 12:14:24 2026 -0700
[SPARK-57397][3.5][PYTHON] Raise an exception when parsed UDT is the wrong
type
### What changes were proposed in this pull request?
Check whether the imported class is a real UserDefinedType before calling
it.
### Why are the changes needed?
If it's not a UDT, it could run wrong code.
### Does this PR introduce _any_ user-facing change?
Not for normal use.
### How was this patch tested?
Regression test added.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #56476 from gaogaotiantian/udt-import-backport.
Authored-by: Tian Gao <[email protected]>
Signed-off-by: Tian Gao <[email protected]>
---
python/pyspark/errors/error_classes.py | 5 +++++
python/pyspark/sql/tests/test_types.py | 11 +++++++++++
python/pyspark/sql/types.py | 5 +++++
3 files changed, 21 insertions(+)
diff --git a/python/pyspark/errors/error_classes.py
b/python/pyspark/errors/error_classes.py
index 0fbe489f623c..aaa9fb705d2f 100644
--- a/python/pyspark/errors/error_classes.py
+++ b/python/pyspark/errors/error_classes.py
@@ -231,6 +231,11 @@ ERROR_CLASSES_JSON = """
"Duplicated field names in Arrow Struct are not allowed, got
<field_names>"
]
},
+ "FIELD_TYPE_MISMATCH": {
+ "message": [
+ "<obj> is not an instance of type <data_type>."
+ ]
+ },
"HIGHER_ORDER_FUNCTION_SHOULD_RETURN_COLUMN" : {
"message" : [
"Function `<func_name>` should return Column, got <return_type>."
diff --git a/python/pyspark/sql/tests/test_types.py
b/python/pyspark/sql/tests/test_types.py
index 85dd9ed2dd10..9821499ca920 100644
--- a/python/pyspark/sql/tests/test_types.py
+++ b/python/pyspark/sql/tests/test_types.py
@@ -49,6 +49,7 @@ from pyspark.sql.types import (
BinaryType,
BooleanType,
NullType,
+ UserDefinedType,
)
from pyspark.sql.types import (
_array_signed_int_typecode_ctype_mappings,
@@ -541,6 +542,16 @@ class TypesTestsMixin:
_make_type_verifier(PythonOnlyUDT())(PythonOnlyPoint(1.0, 2.0))
self.assertRaises(ValueError, lambda:
_make_type_verifier(PythonOnlyUDT())([1.0, 2.0]))
+ def test_udt_from_json_import_type_mismatch(self):
+ json = {
+ "type": "udt",
+ "pyClass": "random.random",
+ "serializedClass": "",
+ "sqlType": StringType().jsonValue(),
+ }
+ with self.assertRaises(PySparkTypeError):
+ UserDefinedType.fromJson(json)
+
def test_simple_udt_in_df(self):
schema = StructType().add("key", LongType()).add("val",
PythonOnlyUDT())
df = self.spark.createDataFrame(
diff --git a/python/pyspark/sql/types.py b/python/pyspark/sql/types.py
index bdeb21951e2e..10cc4d1c957a 100644
--- a/python/pyspark/sql/types.py
+++ b/python/pyspark/sql/types.py
@@ -1210,6 +1210,11 @@ class UserDefinedType(DataType):
)
else:
UDT = getattr(m, pyClass)
+ if not (isinstance(UDT, type) and issubclass(UDT,
UserDefinedType)):
+ raise PySparkTypeError(
+ error_class="FIELD_TYPE_MISMATCH",
+ message_parameters={"obj": str(UDT), "data_type":
"UserDefinedType"},
+ )
return UDT()
def __eq__(self, other: Any) -> bool:
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
