This is an automated email from the ASF dual-hosted git repository.
maxgekk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 15c9ec7cbbbb [SPARK-42332][SQL] Changing the require to a
SparkException in ComplexTypeMergingExpression
15c9ec7cbbbb is described below
commit 15c9ec7cbbbba3b66ec413b7964a374cb9508a80
Author: hannahkamundson <[email protected]>
AuthorDate: Sun Jan 21 23:28:17 2024 +0300
[SPARK-42332][SQL] Changing the require to a SparkException in
ComplexTypeMergingExpression
### What changes were proposed in this pull request?
- I created `SparkException.require` which is the same as the Scala
`require` precondition except it uses `SparkIllegalArgumentException`.
- I changed 2 `require`s to use this `SparkException.require` in
`ComplexTypeMergingExpression`
- The exception message was updated for these `require`s
- A SQLState error was added for the `require`s in
`ComplexTypeMergingExpression`
### Why are the changes needed?
All user facing exceptions should be `SparkException`s. The `require`s need
to be changed to match this convention.
### Does this PR introduce _any_ user-facing change?
Yes.
- `ComplexTypeMergingExpression throws a `SparkIllegalArgumentException`
instead of `IllegalArgumentException`.
- It also updates the SqlState for these
- It also updates the messages associated with the exceptions thrown
- It also updates the docs with the new error classes.
### How was this patch tested?
Unit tests
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #44336 from hannahkamundson/SPARK-42332.
Lead-authored-by: hannahkamundson <[email protected]>
Co-authored-by: Hannah <[email protected]>
Signed-off-by: Max Gekk <[email protected]>
---
.../src/main/resources/error/error-classes.json | 18 +++++++++++
.../scala/org/apache/spark/SparkException.scala | 15 +++++++++
...lex-expression-unsupported-input-error-class.md | 36 ++++++++++++++++++++++
docs/sql-error-conditions.md | 8 +++++
.../sql/catalyst/expressions/Expression.scala | 19 +++++++-----
.../sql/errors/QueryCompilationErrorsSuite.scala | 35 ++++++++++++++++++---
6 files changed, 118 insertions(+), 13 deletions(-)
diff --git a/common/utils/src/main/resources/error/error-classes.json
b/common/utils/src/main/resources/error/error-classes.json
index 8794a8632c52..072081f48448 100644
--- a/common/utils/src/main/resources/error/error-classes.json
+++ b/common/utils/src/main/resources/error/error-classes.json
@@ -506,6 +506,24 @@
],
"sqlState" : "22004"
},
+ "COMPLEX_EXPRESSION_UNSUPPORTED_INPUT" : {
+ "message" : [
+ "Cannot process input data types for the expression: <expression>."
+ ],
+ "subClass" : {
+ "MISMATCHED_TYPES" : {
+ "message" : [
+ "All input types must be the same except nullable, containsNull,
valueContainsNull flags, but found the input types <inputTypes>."
+ ]
+ },
+ "NO_INPUTS" : {
+ "message" : [
+ "The collection of input data types must not be empty."
+ ]
+ }
+ },
+ "sqlState" : "42K09"
+ },
"CONCURRENT_QUERY" : {
"message" : [
"Another instance of this query was just started by a concurrent
session."
diff --git a/common/utils/src/main/scala/org/apache/spark/SparkException.scala
b/common/utils/src/main/scala/org/apache/spark/SparkException.scala
index 67bdc23b5f08..ebb6e772249b 100644
--- a/common/utils/src/main/scala/org/apache/spark/SparkException.scala
+++ b/common/utils/src/main/scala/org/apache/spark/SparkException.scala
@@ -106,6 +106,21 @@ object SparkException {
messageParameters = Map("message" -> msg),
cause = cause)
}
+
+ /**
+ * This is like the Scala require precondition, except it uses
SparkIllegalArgumentException.
+ * @param requirement The requirement you want to check
+ * @param errorClass The error class to type if the requirement isn't passed
+ * @param messageParameters Message parameters to append to the message
+ */
+ def require(
+ requirement: Boolean,
+ errorClass: String,
+ messageParameters: Map[String, String]): Unit = {
+ if (!requirement) {
+ throw new SparkIllegalArgumentException(errorClass, messageParameters)
+ }
+ }
}
/**
diff --git
a/docs/sql-error-conditions-complex-expression-unsupported-input-error-class.md
b/docs/sql-error-conditions-complex-expression-unsupported-input-error-class.md
new file mode 100644
index 000000000000..e8b8630c19aa
--- /dev/null
+++
b/docs/sql-error-conditions-complex-expression-unsupported-input-error-class.md
@@ -0,0 +1,36 @@
+---
+layout: global
+title: COMPLEX_EXPRESSION_UNSUPPORTED_INPUT error class
+displayTitle: COMPLEX_EXPRESSION_UNSUPPORTED_INPUT error class
+license: |
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+---
+
+[SQLSTATE:
42K09](sql-error-conditions-sqlstates.html#class-42-syntax-error-or-access-rule-violation)
+
+Cannot process input data types for the expression: `<expression>`.
+
+This error class has the following derived error classes:
+
+## MISMATCHED_TYPES
+
+All input types must be the same except nullable, containsNull,
valueContainsNull flags, but found the input types `<inputTypes>`.
+
+## NO_INPUTS
+
+The collection of input data types must not be empty.
+
+
diff --git a/docs/sql-error-conditions.md b/docs/sql-error-conditions.md
index 74eaa4228612..133cfbacb5fa 100644
--- a/docs/sql-error-conditions.md
+++ b/docs/sql-error-conditions.md
@@ -419,6 +419,14 @@ The comparator has returned a NULL for a comparison
between `<firstValue>` and `
It should return a positive integer for "greater than", 0 for "equal" and a
negative integer for "less than".
To revert to deprecated behavior where NULL is treated as 0 (equal), you must
set "spark.sql.legacy.allowNullComparisonResultInArraySort" to "true".
+###
[COMPLEX_EXPRESSION_UNSUPPORTED_INPUT](sql-error-conditions-complex-expression-unsupported-input-error-class.html)
+
+[SQLSTATE:
42K09](sql-error-conditions-sqlstates.html#class-42-syntax-error-or-access-rule-violation)
+
+Cannot process input data types for the expression: `<expression>`.
+
+For more details see
[COMPLEX_EXPRESSION_UNSUPPORTED_INPUT](sql-error-conditions-complex-expression-unsupported-input-error-class.html)
+
### CONCURRENT_QUERY
[SQLSTATE:
0A000](sql-error-conditions-sqlstates.html#class-0A-feature-not-supported)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala
index 484418f5e5a7..a3432716002a 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala
@@ -23,6 +23,7 @@ import org.apache.spark.{QueryContext, SparkException}
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.analysis.{FunctionRegistry,
TypeCheckResult, TypeCoercion}
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.DataTypeMismatch
+import org.apache.spark.sql.catalyst.expressions.Cast.{toSQLExpr, toSQLType}
import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateFunction
import org.apache.spark.sql.catalyst.expressions.codegen._
import org.apache.spark.sql.catalyst.expressions.codegen.Block._
@@ -1305,14 +1306,16 @@ trait ComplexTypeMergingExpression extends Expression {
lazy val inputTypesForMerging: Seq[DataType] = children.map(_.dataType)
def dataTypeCheck: Unit = {
- require(
- inputTypesForMerging.nonEmpty,
- "The collection of input data types must not be empty.")
- require(
- TypeCoercion.haveSameType(inputTypesForMerging),
- "All input types must be the same except nullable, containsNull,
valueContainsNull flags. " +
- s"The expression is: $this. " +
- s"The input types found
are\n\t${inputTypesForMerging.mkString("\n\t")}.")
+ SparkException.require(
+ requirement = inputTypesForMerging.nonEmpty,
+ errorClass = "COMPLEX_EXPRESSION_UNSUPPORTED_INPUT.NO_INPUTS",
+ messageParameters = Map("expression" -> toSQLExpr(this)))
+ SparkException.require(
+ requirement = TypeCoercion.haveSameType(inputTypesForMerging),
+ errorClass = "COMPLEX_EXPRESSION_UNSUPPORTED_INPUT.MISMATCHED_TYPES",
+ messageParameters = Map(
+ "expression" -> toSQLExpr(this),
+ "inputTypes" -> inputTypesForMerging.map(toSQLType).mkString("[", ",
", "]")))
}
private lazy val internalDataType: DataType = {
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala
index 3c397bd6a4e1..7ea0c3843444 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala
@@ -17,17 +17,17 @@
package org.apache.spark.sql.errors
-import org.apache.spark.{SPARK_DOC_ROOT, SparkUnsupportedOperationException}
-import org.apache.spark.sql.{AnalysisException, ClassData,
IntegratedUDFTestUtils, QueryTest, Row}
+import org.apache.spark.{SPARK_DOC_ROOT, SparkIllegalArgumentException,
SparkUnsupportedOperationException}
+import org.apache.spark.sql._
import org.apache.spark.sql.api.java.{UDF1, UDF2, UDF23Test}
-import org.apache.spark.sql.catalyst.expressions.UnsafeRow
+import org.apache.spark.sql.catalyst.expressions.{Coalesce, Literal, UnsafeRow}
import org.apache.spark.sql.catalyst.parser.ParseException
import org.apache.spark.sql.execution.datasources.v2.jdbc.JDBCTableCatalog
import org.apache.spark.sql.expressions.SparkUserDefinedFunction
-import org.apache.spark.sql.functions.{array, from_json, grouping,
grouping_id, lit, struct, sum, udf}
+import org.apache.spark.sql.functions._
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.test.SharedSparkSession
-import org.apache.spark.sql.types.{BooleanType, IntegerType, MapType,
StringType, StructField, StructType}
+import org.apache.spark.sql.types._
import org.apache.spark.util.Utils
case class StringLongClass(a: String, b: Long)
@@ -920,6 +920,31 @@ class QueryCompilationErrorsSuite
}
}
+ test("ComplexTypeMergingExpression should throw exception if no children") {
+ val coalesce = Coalesce(Seq.empty)
+
+ checkError(
+ exception = intercept[SparkIllegalArgumentException] {
+ coalesce.dataType
+ },
+ errorClass = "COMPLEX_EXPRESSION_UNSUPPORTED_INPUT.NO_INPUTS",
+ parameters = Map("expression" -> "\"coalesce()\""))
+ }
+
+ test("ComplexTypeMergingExpression should throw " +
+ "exception if children have different data types") {
+ val coalesce = Coalesce(Seq(Literal(1), Literal("a"), Literal("a")))
+
+ checkError(
+ exception = intercept[SparkIllegalArgumentException] {
+ coalesce.dataType
+ },
+ errorClass = "COMPLEX_EXPRESSION_UNSUPPORTED_INPUT.MISMATCHED_TYPES",
+ parameters = Map(
+ "expression" -> "\"coalesce(1, a, a)\"",
+ "inputTypes" -> "[\"INT\", \"STRING\", \"STRING\"]"))
+ }
+
test("UNSUPPORTED_CALL: call the unsupported method update()") {
checkError(
exception = intercept[SparkUnsupportedOperationException] {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
