This is an automated email from the ASF dual-hosted git repository.
LuciferYang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 24f94f4a574d [SPARK-58238][SQL] Assign names to the error conditions
_LEGACY_ERROR_TEMP_3093-3095
24f94f4a574d is described below
commit 24f94f4a574d1b16022b129f772e6b728b05da12
Author: YangJie <[email protected]>
AuthorDate: Wed Jul 22 12:37:32 2026 +0800
[SPARK-58238][SQL] Assign names to the error conditions
_LEGACY_ERROR_TEMP_3093-3095
### What changes were proposed in this pull request?
Resolve three legacy error conditions raised in `HiveInspectors`:
- `_LEGACY_ERROR_TEMP_3093` (the `javaTypeToDataType` fallthrough for an
unsupported Hive UDF Java type) becomes a new subclass
`UNSUPPORTED_HIVE_FUNCTION_TYPE.UNKNOWN_TYPE`, alongside the existing
`RAW_LIST` / `RAW_MAP` / `WILDCARD` subclasses raised from the same method.
- `_LEGACY_ERROR_TEMP_3094` (`decimalTypeInfo`) and
`_LEGACY_ERROR_TEMP_3095` (the `toTypeInfo` fallthrough) reuse the existing
`unsupportedHiveType` helper, which throws `UNSUPPORTED_DATATYPE` — the same
condition already used by the sibling `TimeType` branches in those two methods.
The three legacy entries are removed from `error-conditions.json`.
### Why are the changes needed?
The error-conditions README disallows new `_LEGACY_ERROR_TEMP_*` entries
and asks existing ones to be resolved. This resolves three of them, reusing
existing conditions where the semantics already match (`UNSUPPORTED_DATATYPE`)
rather than minting new names.
### Does this PR introduce _any_ user-facing change?
No. The `_LEGACY_ERROR_TEMP_*` names are not part of the public API. The
`UNKNOWN_TYPE` message renders as "Unsupported Hive UDF/UDAF/UDTF Java type:
<c>."; the `_3094`/`_3095` paths now render the standard "Unsupported data type
<typeName>." consistent with the neighboring `TimeType` handling.
### How was this patch tested?
- Added a `checkError` test in `HiveUDFSuite` (`UDFRawSet`, a UDF returning
a raw `java.util.Set`) asserting `UNSUPPORTED_HIVE_FUNCTION_TYPE.UNKNOWN_TYPE`.
- Updated the `HiveScriptTransformationSuite` SPARK-32400 tests to
`checkError` on `UNSUPPORTED_DATATYPE` (previously asserted the old `_3095`
message text).
- `build/sbt "hive/testOnly *HiveUDFSuite *HiveScriptTransformationSuite"
"core/testOnly org.apache.spark.SparkThrowableSuite"` passes.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Code (Opus 4.8)
Closes #57400 from LuciferYang/assign-name-legacy-3093-3095.
Authored-by: YangJie <[email protected]>
Signed-off-by: yangjie01 <[email protected]>
---
.../src/main/resources/error/error-conditions.json | 20 ++---
.../org/apache/spark/sql/hive/HiveInspectors.scala | 10 +--
.../apache/spark/sql/hive/execution/UDFRawSet.java | 33 ++++++++
.../execution/HiveScriptTransformationSuite.scala | 88 ++++++++++++----------
.../spark/sql/hive/execution/HiveUDFSuite.scala | 16 ++++
5 files changed, 106 insertions(+), 61 deletions(-)
diff --git a/common/utils/src/main/resources/error/error-conditions.json
b/common/utils/src/main/resources/error/error-conditions.json
index 90c885780c8e..e87f913f9aeb 100644
--- a/common/utils/src/main/resources/error/error-conditions.json
+++ b/common/utils/src/main/resources/error/error-conditions.json
@@ -8805,6 +8805,11 @@
"raw java.util.Map. Spark cannot infer the key and value types."
]
},
+ "UNKNOWN_TYPE" : {
+ "message" : [
+ "<c>."
+ ]
+ },
"WILDCARD" : {
"message" : [
"wildcard type parameter. Spark cannot infer a concrete data type
for wildcard type parameters, such as List<?> or Map<?, ?>."
@@ -11538,21 +11543,6 @@
"Corrupted <typeName> in catalog: <numCols> parts expected, but part
<index> is missing."
]
},
- "_LEGACY_ERROR_TEMP_3093" : {
- "message" : [
- "Unsupported java type <c>"
- ]
- },
- "_LEGACY_ERROR_TEMP_3094" : {
- "message" : [
- "<dt> is not supported."
- ]
- },
- "_LEGACY_ERROR_TEMP_3095" : {
- "message" : [
- "<dt> cannot be converted to Hive TypeInfo"
- ]
- },
"_LEGACY_ERROR_TEMP_3096" : {
"message" : [
"Converted table has <resLen> columns,",
diff --git
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
index b45e8aaa6e63..7f76132e2572 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
@@ -256,7 +256,8 @@ private[hive] trait HiveInspectors {
messageParameters = Map.empty)
case c => throw new AnalysisException(
- errorClass = "_LEGACY_ERROR_TEMP_3093", messageParameters = Map("c" ->
c.toString))
+ errorClass = "UNSUPPORTED_HIVE_FUNCTION_TYPE.UNKNOWN_TYPE",
+ messageParameters = Map("c" -> c.toString))
}
private def isSubClassOf(t: Type, parent: Class[_]): Boolean = t match {
@@ -1267,8 +1268,7 @@ private[hive] trait HiveInspectors {
private def decimalTypeInfo(decimalType: DecimalType): TypeInfo =
decimalType match {
case DecimalType.Fixed(precision, scale) => new
DecimalTypeInfo(precision, scale)
- case dt => throw new AnalysisException(
- errorClass = "_LEGACY_ERROR_TEMP_3094", messageParameters = Map("dt"
-> toSQLType(dt)))
+ case dt => throw unsupportedHiveType(dt)
}
def toTypeInfo: TypeInfo = dt match {
@@ -1297,9 +1297,7 @@ private[hive] trait HiveInspectors {
case _: YearMonthIntervalType => intervalYearMonthTypeInfo
// Hive has no TIME type, so there is no Hive TypeInfo to map it to.
case _: TimeType => throw unsupportedHiveType(dt)
- case dt =>
- throw new AnalysisException(
- errorClass = "_LEGACY_ERROR_TEMP_3095", messageParameters = Map("dt"
-> toSQLType(dt)))
+ case dt => throw unsupportedHiveType(dt)
}
}
}
diff --git
a/sql/hive/src/test/java/org/apache/spark/sql/hive/execution/UDFRawSet.java
b/sql/hive/src/test/java/org/apache/spark/sql/hive/execution/UDFRawSet.java
new file mode 100644
index 000000000000..e67b3274cd11
--- /dev/null
+++ b/sql/hive/src/test/java/org/apache/spark/sql/hive/execution/UDFRawSet.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.hive.execution;
+
+import org.apache.hadoop.hive.ql.exec.UDF;
+
+import java.util.Collections;
+import java.util.Set;
+
+/**
+ * UDF that returns a raw (non-parameterized) java Set, a type Spark cannot map
+ * to a Catalyst data type.
+ */
+public class UDFRawSet extends UDF {
+ @SuppressWarnings("rawtypes")
+ public Set evaluate(Object o) {
+ return Collections.singleton("data1");
+ }
+}
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationSuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationSuite.scala
index 6a16b312e2b3..b1ff05b8c1b0 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationSuite.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationSuite.scala
@@ -381,29 +381,33 @@ class HiveScriptTransformationSuite extends
BaseScriptTransformationSuite with T
).toDF("a", "b", "c")
df.createTempView("v")
- val e1 = intercept[AnalysisException] {
- val plan = createScriptTransformationExec(
- script = "cat",
- output = Seq(
- AttributeReference("a", IntegerType)(),
- AttributeReference("b", CalendarIntervalType)()),
- child = df.select($"a", $"b").queryExecution.sparkPlan,
- ioschema = hiveIOSchema)
- QueryTest.executePlan(plan, hiveContext)
- }.getMessage
- assert(e1.contains("\"INTERVAL\" cannot be converted to Hive TypeInfo"))
-
- val e2 = intercept[AnalysisException] {
- val plan = createScriptTransformationExec(
- script = "cat",
- output = Seq(
- AttributeReference("a", IntegerType)(),
- AttributeReference("c", new TestUDT.MyDenseVectorUDT)()),
- child = df.select($"a", $"c").queryExecution.sparkPlan,
- ioschema = hiveIOSchema)
- QueryTest.executePlan(plan, hiveContext)
- }.getMessage
- assert(e2.contains("UDT(\"ARRAY<DOUBLE>\") cannot be converted to Hive
TypeInfo"))
+ checkError(
+ exception = intercept[AnalysisException] {
+ val plan = createScriptTransformationExec(
+ script = "cat",
+ output = Seq(
+ AttributeReference("a", IntegerType)(),
+ AttributeReference("b", CalendarIntervalType)()),
+ child = df.select($"a", $"b").queryExecution.sparkPlan,
+ ioschema = hiveIOSchema)
+ QueryTest.executePlan(plan, hiveContext)
+ },
+ condition = "UNSUPPORTED_DATATYPE",
+ parameters = Map("typeName" -> "\"INTERVAL\""))
+
+ checkError(
+ exception = intercept[AnalysisException] {
+ val plan = createScriptTransformationExec(
+ script = "cat",
+ output = Seq(
+ AttributeReference("a", IntegerType)(),
+ AttributeReference("c", new TestUDT.MyDenseVectorUDT)()),
+ child = df.select($"a", $"c").queryExecution.sparkPlan,
+ ioschema = hiveIOSchema)
+ QueryTest.executePlan(plan, hiveContext)
+ },
+ condition = "UNSUPPORTED_DATATYPE",
+ parameters = Map("typeName" -> "UDT(\"ARRAY<DOUBLE>\")"))
}
}
@@ -416,23 +420,27 @@ class HiveScriptTransformationSuite extends
BaseScriptTransformationSuite with T
(1, new CalendarInterval(7, 1, 1000), new
TestUDT.MyDenseVector(Array(1, 2, 3)))
).toDF("a", "b", "c")
df.createTempView("v")
- val e1 = intercept[AnalysisException] {
- sql(
- """
- |SELECT TRANSFORM(a, b) USING 'cat' AS (a, b)
- |FROM v
- """.stripMargin).collect()
- }.getMessage
- assert(e1.contains("\"INTERVAL\" cannot be converted to Hive TypeInfo"))
-
- val e2 = intercept[AnalysisException] {
- sql(
- """
- |SELECT TRANSFORM(a, c) USING 'cat' AS (a, c)
- |FROM v
- """.stripMargin).collect()
- }.getMessage
- assert(e2.contains("UDT(\"ARRAY<DOUBLE>\") cannot be converted to Hive
TypeInfo"))
+ checkError(
+ exception = intercept[AnalysisException] {
+ sql(
+ """
+ |SELECT TRANSFORM(a, b) USING 'cat' AS (a, b)
+ |FROM v
+ """.stripMargin).collect()
+ },
+ condition = "UNSUPPORTED_DATATYPE",
+ parameters = Map("typeName" -> "\"INTERVAL\""))
+
+ checkError(
+ exception = intercept[AnalysisException] {
+ sql(
+ """
+ |SELECT TRANSFORM(a, c) USING 'cat' AS (a, c)
+ |FROM v
+ """.stripMargin).collect()
+ },
+ condition = "UNSUPPORTED_DATATYPE",
+ parameters = Map("typeName" -> "UDT(\"ARRAY<DOUBLE>\")"))
}
}
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
index 7cf98fae8c8b..01af6b3cc570 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
@@ -332,6 +332,22 @@ class HiveUDFSuite extends QueryTest with
TestHiveSingleton {
hiveContext.reset()
}
+ test("UDFRawSet") {
+ val testData = spark.sparkContext.parallelize(StringCaseClass("") ::
Nil).toDF()
+ testData.createOrReplaceTempView("inputTable")
+
+ sql(s"CREATE TEMPORARY FUNCTION testUDFRawSet " +
+ s"AS '${classOf[UDFRawSet].getName}'")
+ val err = intercept[AnalysisException](sql("SELECT testUDFRawSet(s) FROM
inputTable"))
+ checkError(
+ exception = err.getCause.asInstanceOf[AnalysisException],
+ condition = "UNSUPPORTED_HIVE_FUNCTION_TYPE.UNKNOWN_TYPE",
+ parameters = Map("c" -> "interface java.util.Set"))
+
+ sql("DROP TEMPORARY FUNCTION IF EXISTS testUDFRawSet")
+ hiveContext.reset()
+ }
+
test("UDFListListInt") {
val testData = spark.sparkContext.parallelize(
ListListIntCaseClass(Nil) ::
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
