This is an automated email from the ASF dual-hosted git repository.
cloud-fan pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new abb6a8731561 [SPARK-58103][SQL] Assign a name to the error condition
_LEGACY_ERROR_TEMP_2228
abb6a8731561 is described below
commit abb6a8731561699a8a8fcd532919303d231a6f32
Author: Wenchen Fan <[email protected]>
AuthorDate: Wed Jul 15 09:30:02 2026 +0800
[SPARK-58103][SQL] Assign a name to the error condition
_LEGACY_ERROR_TEMP_2228
### What changes were proposed in this pull request?
Replace the legacy error condition `_LEGACY_ERROR_TEMP_2228` with a
descriptive condition `UDT_CLASS_NOT_FOUND` (SQLSTATE `38000`), part of the
error-class migration (umbrella
[SPARK-37935](https://issues.apache.org/jira/browse/SPARK-37935)).
- Add `UDT_CLASS_NOT_FOUND` to `error-conditions.json` with two subclasses
and remove `_LEGACY_ERROR_TEMP_2228`:
- `FOR_USER_CLASS` — the UDT class registered for a user class via
`UDTRegistration` cannot be loaded (replaces `_LEGACY_ERROR_TEMP_2228`).
- `WITHOUT_USER_CLASS` — the UDT class named directly in a schema string
cannot be loaded.
- `DataTypeErrors.cannotLoadUserDefinedTypeError` now raises
`UDT_CLASS_NOT_FOUND.FOR_USER_CLASS`.
- `DataType.fromJson`'s Scala/Java UDT branch now catches
`ClassNotFoundException` from `SparkClassUtils.classForName` and rewraps it as
`UDT_CLASS_NOT_FOUND.WITHOUT_USER_CLASS`. Previously a raw
`java.lang.ClassNotFoundException` (no error class, no SQLSTATE) propagated
when a schema string (for example the schema stored in Parquet/ORC file
metadata, or a Delta table's stored schema) referenced a UDT class that was not
on the classpath.
- Add tests in `DataTypeSuite` (`WITHOUT_USER_CLASS`) and
`UDTRegistrationSuite` (`FOR_USER_CLASS`).
### Why are the changes needed?
Part of the error-class migration
([SPARK-37935](https://issues.apache.org/jira/browse/SPARK-37935)). The legacy
condition used free-form `name`/`userClass` parameters and, for the
`DataType.fromJson` path, a missing UDT class was not wrapped at all: a raw
`ClassNotFoundException` propagated without any error class or SQLSTATE, so it
could not be identified or handled programmatically. Assigning a proper name
and SQLSTATE, and wrapping the `fromJson` path, makes both failures diagnosable.
### Does this PR introduce _any_ user-facing change?
Yes, within the unreleased `master` branch only. The error condition name
and SQLSTATE change (`_LEGACY_ERROR_TEMP_2228` ->
`UDT_CLASS_NOT_FOUND.FOR_USER_CLASS`), and loading a non-existent UDT class
named in a schema string now raises `UDT_CLASS_NOT_FOUND.WITHOUT_USER_CLASS`
(SQLSTATE `38000`) instead of a raw `ClassNotFoundException`.
### How was this patch tested?
New unit tests:
- `DataTypeSuite`: "UDT_CLASS_NOT_FOUND.WITHOUT_USER_CLASS when UDT class
is not on classpath".
- `UDTRegistrationSuite`: "UDT_CLASS_NOT_FOUND.FOR_USER_CLASS when UDT
class is not loadable".
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 4.8
Closes #57228 from cloud-fan/SPARK-58103.
Authored-by: Wenchen Fan <[email protected]>
Signed-off-by: Wenchen Fan <[email protected]>
(cherry picked from commit 014aaf8648ef45a027bcfd554268fd9a64f7c10c)
Signed-off-by: Wenchen Fan <[email protected]>
---
.../src/main/resources/error/error-conditions.json | 24 +++++++++++++++++-----
.../apache/spark/sql/errors/DataTypeErrors.scala | 4 ++--
.../org/apache/spark/sql/types/DataType.scala | 13 ++++++++++--
.../org/apache/spark/sql/types/DataTypeSuite.scala | 16 ++++++++++++++-
.../apache/spark/sql/UDTRegistrationSuite.scala | 13 ++++++++++++
5 files changed, 60 insertions(+), 10 deletions(-)
diff --git a/common/utils/src/main/resources/error/error-conditions.json
b/common/utils/src/main/resources/error/error-conditions.json
index fecf19dd51c0..3e6345d490c1 100644
--- a/common/utils/src/main/resources/error/error-conditions.json
+++ b/common/utils/src/main/resources/error/error-conditions.json
@@ -7712,6 +7712,25 @@
],
"sqlState" : "2203G"
},
+ "UDT_CLASS_NOT_FOUND" : {
+ "message" : [
+ "Could not load UserDefinedType class <udtClass>.",
+ "Ensure the JAR containing this class is on the classpath."
+ ],
+ "subClass" : {
+ "FOR_USER_CLASS" : {
+ "message" : [
+ "The class was registered as the UserDefinedType for user class
<userClass>."
+ ]
+ },
+ "WITHOUT_USER_CLASS" : {
+ "message" : [
+ ""
+ ]
+ }
+ },
+ "sqlState" : "38000"
+ },
"UDT_CLASS_NOT_USER_DEFINED_TYPE" : {
"message" : [
"The class <udtClass> cannot be loaded as a user-defined type because it
is not a subtype of UserDefinedType."
@@ -10871,11 +10890,6 @@
"<name> is not an UserDefinedType. Please make sure registering an
UserDefinedType for <userClass>."
]
},
- "_LEGACY_ERROR_TEMP_2228" : {
- "message" : [
- "Can not load in UserDefinedType <name> for user class <userClass>."
- ]
- },
"_LEGACY_ERROR_TEMP_2229" : {
"message" : [
"<name> is not a public class. Only public classes are supported."
diff --git
a/sql/api/src/main/scala/org/apache/spark/sql/errors/DataTypeErrors.scala
b/sql/api/src/main/scala/org/apache/spark/sql/errors/DataTypeErrors.scala
index e7c5d329e404..8d65217fb4e4 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/errors/DataTypeErrors.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/errors/DataTypeErrors.scala
@@ -79,8 +79,8 @@ private[sql] object DataTypeErrors extends DataTypeErrorsBase
{
def cannotLoadUserDefinedTypeError(name: String, userClass: String):
Throwable = {
new SparkException(
- errorClass = "_LEGACY_ERROR_TEMP_2228",
- messageParameters = Map("name" -> name, "userClass" -> userClass),
+ errorClass = "UDT_CLASS_NOT_FOUND.FOR_USER_CLASS",
+ messageParameters = Map("udtClass" -> name, "userClass" -> userClass),
cause = null)
}
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/types/DataType.scala
b/sql/api/src/main/scala/org/apache/spark/sql/types/DataType.scala
index 8bce7b50f5d8..c8621ca4da17 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/types/DataType.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/types/DataType.scala
@@ -27,7 +27,7 @@ import org.json4s.JsonAST.JValue
import org.json4s.JsonDSL._
import org.json4s.jackson.JsonMethods._
-import org.apache.spark.{SparkIllegalArgumentException, SparkThrowable}
+import org.apache.spark.{SparkClassNotFoundException,
SparkIllegalArgumentException, SparkThrowable}
import org.apache.spark.annotation.Stable
import org.apache.spark.sql.catalyst.analysis.SqlApiAnalysis
import org.apache.spark.sql.catalyst.parser.DataTypeParser
@@ -345,7 +345,16 @@ object DataType {
}
// Defense in depth: resolve the class without initializing it and
verify that it really is a
// UserDefinedType subclass before constructing it.
- val clazz = SparkClassUtils.classForName[UserDefinedType[_]](udtClass,
initialize = false)
+ val clazz =
+ try {
+ SparkClassUtils.classForName[UserDefinedType[_]](udtClass,
initialize = false)
+ } catch {
+ case e: ClassNotFoundException =>
+ throw new SparkClassNotFoundException(
+ errorClass = "UDT_CLASS_NOT_FOUND.WITHOUT_USER_CLASS",
+ messageParameters = Map("udtClass" -> udtClass),
+ cause = e)
+ }
if (!classOf[UserDefinedType[_]].isAssignableFrom(clazz)) {
throw DataTypeErrors.udtClassNotUserDefinedTypeError(udtClass)
}
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeSuite.scala
index 6282832420b7..285d840eed6c 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeSuite.scala
@@ -22,7 +22,8 @@ import java.util.Locale
import com.fasterxml.jackson.core.JsonParseException
import org.json4s.jackson.JsonMethods
-import org.apache.spark.{SparkException, SparkFunSuite,
SparkIllegalArgumentException}
+import org.apache.spark.{SparkClassNotFoundException, SparkException,
SparkFunSuite}
+import org.apache.spark.SparkIllegalArgumentException
import org.apache.spark.sql.catalyst.analysis.{caseInsensitiveResolution,
caseSensitiveResolution}
import org.apache.spark.sql.catalyst.parser.{CatalystSqlParser, ParseException}
import org.apache.spark.sql.catalyst.plans.SQLHelper
@@ -255,6 +256,19 @@ class DataTypeSuite extends SparkFunSuite with SQLHelper {
}
}
+ test("UDT_CLASS_NOT_FOUND.WITHOUT_USER_CLASS when UDT class is not on
classpath") {
+ val fakeUdtClass = "org.apache.spark.sql.types.NonExistentUDT"
+ val udtJson =
+
s"""{"class":"$fakeUdtClass","pyClass":"","sqlType":"integer","type":"udt"}"""
+ checkError(
+ exception = intercept[SparkClassNotFoundException] {
+ DataType.fromJson(udtJson)
+ },
+ condition = "UDT_CLASS_NOT_FOUND.WITHOUT_USER_CLASS",
+ sqlState = Some("38000"),
+ parameters = Map("udtClass" -> fakeUdtClass))
+ }
+
def checkDataTypeFromJson(dataType: DataType): Unit = {
test(s"from Json - $dataType") {
assert(DataType.fromJson(dataType.json) === dataType)
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/UDTRegistrationSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/UDTRegistrationSuite.scala
index 5c4de5d4d963..503a6106e7a1 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/UDTRegistrationSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/UDTRegistrationSuite.scala
@@ -86,4 +86,17 @@ class UDTRegistrationSuite extends SparkFunSuite {
assert(!UDTRegistration.exists(classOf[TestUserClass3].getName))
assert(UDTRegistration.getUDTFor(classOf[TestUserClass3].getName).isEmpty)
}
+
+ test("UDT_CLASS_NOT_FOUND.FOR_USER_CLASS when UDT class is not loadable") {
+ val userClass = classOf[TestUserClass3].getName
+ val fakeUdtClass = "org.apache.spark.sql.NonExistentUDT"
+ UDTRegistration.register(userClass, fakeUdtClass)
+ checkError(
+ exception = intercept[SparkException] {
+ UDTRegistration.getUDTFor(userClass)
+ },
+ condition = "UDT_CLASS_NOT_FOUND.FOR_USER_CLASS",
+ sqlState = Some("38000"),
+ parameters = Map("udtClass" -> fakeUdtClass, "userClass" -> userClass))
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
