This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 9619ada3505 [SPARK-44477][SQL] Treat TYPE_CHECK_FAILURE_WITH_HINT as
an error subclass
9619ada3505 is described below
commit 9619ada35059faf601ebefd5c225ae1ebf86f5ef
Author: Bruce Robbins <[email protected]>
AuthorDate: Fri Jul 21 09:44:45 2023 +0900
[SPARK-44477][SQL] Treat TYPE_CHECK_FAILURE_WITH_HINT as an error subclass
### What changes were proposed in this pull request?
In `CheckAnalysis#checkAnalysis0`, qualify the error subclass
`TYPE_CHECK_FAILURE_WITH_HINT` with the error class `DATATYPE_MISMATCH`.
### Why are the changes needed?
`CheckAnalysis` treats `TYPE_CHECK_FAILURE_WITH_HINT` as an error class,
but it is actually an error subclass of `DATATYPE_MISMATCH`.
```
spark-sql (default)> select bitmap_count(12);
[INTERNAL_ERROR] Cannot find main error class 'TYPE_CHECK_FAILURE_WITH_HINT'
org.apache.spark.SparkException: [INTERNAL_ERROR] Cannot find main error
class 'TYPE_CHECK_FAILURE_WITH_HINT'
at
org.apache.spark.SparkException$.internalError(SparkException.scala:83)
at
org.apache.spark.SparkException$.internalError(SparkException.scala:87)
at
org.apache.spark.ErrorClassesJsonReader.$anonfun$getMessageTemplate$1(ErrorClassesJSONReader.scala:68)
at
scala.collection.immutable.HashMap$HashMap1.getOrElse0(HashMap.scala:361)
at
scala.collection.immutable.HashMap$HashTrieMap.getOrElse0(HashMap.scala:594)
at
scala.collection.immutable.HashMap$HashTrieMap.getOrElse0(HashMap.scala:589)
at scala.collection.immutable.HashMap.getOrElse(HashMap.scala:73)
```
This issue only occurs when an expression uses
`TypeCheckResult.TypeCheckFailure` to indicate input type check failure.
`TypeCheckResult.TypeCheckFailure` appears to be deprecated in favor of
`TypeCheckResult.DataTypeMismatch`, but recently two expressions were added
that use `TypeCheckResult.TypeCheckFailure`: `BitmapCount` and `BitmapOrAgg`.
`BitmapCount` and `BitmapOrAgg` should probably be fixed to use
`TypeCheckResult.DataTypeMismatch`. Regardless, the code in `CheckAnalysis`
that handles `TypeCheckResult.TypeCheckFailure` should either be fixed or
removed. In this PR, I chose to fix it.
### Does this PR introduce _any_ user-facing change?
No, except for the user seeing the correct error message.
### How was this patch tested?
New unit test.
Closes #42064 from bersprockets/type_check_issue.
Authored-by: Bruce Robbins <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
.../sql/catalyst/analysis/CheckAnalysis.scala | 2 +-
.../sql/catalyst/analysis/AnalysisErrorSuite.scala | 32 ++++++++++++++++++++++
2 files changed, 33 insertions(+), 1 deletion(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
index 1772c224a55..fca1b780088 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
@@ -281,7 +281,7 @@ trait CheckAnalysis extends PredicateHelper with
LookupCatalog with QueryErrorsB
e.failAnalysis(
errorClass =
"DATATYPE_MISMATCH.TYPE_CHECK_FAILURE_WITH_HINT",
messageParameters = Map(
- "expr" -> toSQLExpr(e),
+ "sqlExpr" -> toSQLExpr(e),
"msg" -> message,
"hint" -> extraHint))
case checkRes: TypeCheckResult.InvalidFormat =>
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
index 6c43f84e8d0..e2e98007330 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
@@ -94,6 +94,28 @@ case class TestFunction(
copy(children = newChildren)
}
+case class TestFunctionWithTypeCheckFailure(
+ children: Seq[Expression],
+ inputTypes: Seq[AbstractDataType])
+ extends Expression with Unevaluable {
+
+ override def checkInputDataTypes(): TypeCheckResult = {
+ for ((child, idx) <- children.zipWithIndex) {
+ val expectedDataType = inputTypes(idx)
+ if (child.dataType != expectedDataType) {
+ return TypeCheckResult.TypeCheckFailure(
+ s"Expression must be a ${expectedDataType.simpleString}")
+ }
+ }
+ TypeCheckResult.TypeCheckSuccess
+ }
+
+ override def nullable: Boolean = true
+ override def dataType: DataType = StringType
+ override protected def withNewChildrenInternal(newChildren:
IndexedSeq[Expression]): Expression =
+ copy(children = newChildren)
+}
+
case class UnresolvedTestPlan() extends UnresolvedLeafNode
class AnalysisErrorSuite extends AnalysisTest {
@@ -168,6 +190,16 @@ class AnalysisErrorSuite extends AnalysisTest {
"inputType" -> "\"DATE\"",
"requiredType" -> "\"INT\""))
+ errorClassTest(
+ "SPARK-44477: type check failure",
+ testRelation.select(
+ TestFunctionWithTypeCheckFailure(dateLit :: Nil, BinaryType ::
Nil).as("a")),
+ errorClass = "DATATYPE_MISMATCH.TYPE_CHECK_FAILURE_WITH_HINT",
+ messageParameters = Map(
+ "sqlExpr" -> "\"testfunctionwithtypecheckfailure(NULL)\"",
+ "msg" -> "Expression must be a binary",
+ "hint" -> ""))
+
errorClassTest(
"invalid window function",
testRelation2.select(
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
