This is an automated email from the ASF dual-hosted git repository.
gengliang pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new a210e39 [SPARK-38716][SQL] Provide query context in map key not
exists error
a210e39 is described below
commit a210e3929bb96086894a9a5a72f0fe5946b1659d
Author: Gengliang Wang <[email protected]>
AuthorDate: Fri Apr 1 11:40:32 2022 +0800
[SPARK-38716][SQL] Provide query context in map key not exists error
### What changes were proposed in this pull request?
Provide query context in `map key does not exist` runtime error with ANSI
SQL mode on, including
- operator `[]`
- function `element_at()`
### Why are the changes needed?
Provide SQL query context of runtime errors to users, so that they can
understand it better.
### Does this PR introduce _any_ user-facing change?
Yes, improve the runtime error message of "map key does not exist"
### How was this patch tested?
UT
Closes #36025 from gengliangwang/mapKeyContext.
Authored-by: Gengliang Wang <[email protected]>
Signed-off-by: Gengliang Wang <[email protected]>
(cherry picked from commit f0fc991f1d2e7b04b0e3967481f7e75e4322ae15)
Signed-off-by: Gengliang Wang <[email protected]>
---
core/src/main/resources/error/error-classes.json | 4 ++--
.../scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala | 5 ++++-
.../spark/sql/catalyst/expressions/complexTypeExtractors.scala | 6 ++++--
.../scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala | 9 ++++++---
sql/core/src/test/resources/sql-tests/results/ansi/map.sql.out | 9 +++++++++
5 files changed, 25 insertions(+), 8 deletions(-)
diff --git a/core/src/main/resources/error/error-classes.json
b/core/src/main/resources/error/error-classes.json
index d7d7702..c8c1841 100644
--- a/core/src/main/resources/error/error-classes.json
+++ b/core/src/main/resources/error/error-classes.json
@@ -121,10 +121,10 @@
"sqlState" : "42000"
},
"MAP_KEY_DOES_NOT_EXIST" : {
- "message" : [ "Key %s does not exist. If necessary set %s to false to
bypass this error." ]
+ "message" : [ "Key %s does not exist. If necessary set %s to false to
bypass this error.%s" ]
},
"MAP_KEY_DOES_NOT_EXIST_IN_ELEMENT_AT" : {
- "message" : [ "Key %s does not exist. To return NULL instead, use
'try_element_at'. If necessary set %s to false to bypass this error." ]
+ "message" : [ "Key %s does not exist. To return NULL instead, use
'try_element_at'. If necessary set %s to false to bypass this error.%s" ]
},
"MISSING_COLUMN" : {
"message" : [ "Column '%s' does not exist. Did you mean one of the
following? [%s]" ],
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 6d95067..6b44483 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -41,6 +41,7 @@ import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.catalyst.rules._
import org.apache.spark.sql.catalyst.streaming.StreamingRelationV2
import org.apache.spark.sql.catalyst.trees.{AlwaysProcess, CurrentOrigin}
+import org.apache.spark.sql.catalyst.trees.CurrentOrigin.withOrigin
import org.apache.spark.sql.catalyst.trees.TreePattern._
import org.apache.spark.sql.catalyst.util.{toPrettySQL, CharVarcharUtils}
import org.apache.spark.sql.connector.catalog._
@@ -1749,7 +1750,9 @@ class Analyzer(override val catalogManager:
CatalogManager)
case u @ UnresolvedExtractValue(child, fieldName) =>
val newChild = innerResolve(child, isTopLevel = false)
if (newChild.resolved) {
- ExtractValue(newChild, fieldName, resolver)
+ withOrigin(u.origin) {
+ ExtractValue(newChild, fieldName, resolver)
+ }
} else {
u.copy(child = newChild)
}
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala
index cb7e06b..3cd404a 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala
@@ -367,7 +367,7 @@ trait GetMapValueUtil extends BinaryExpression with
ImplicitCastInputTypes {
if (!found) {
if (failOnError) {
- throw QueryExecutionErrors.mapKeyNotExistError(ordinal,
isElementAtFunction)
+ throw QueryExecutionErrors.mapKeyNotExistError(ordinal,
isElementAtFunction, origin.context)
} else {
null
}
@@ -400,9 +400,11 @@ trait GetMapValueUtil extends BinaryExpression with
ImplicitCastInputTypes {
}
val keyJavaType = CodeGenerator.javaType(keyType)
+ lazy val errorContext = ctx.addReferenceObj("errCtx", origin.context)
nullSafeCodeGen(ctx, ev, (eval1, eval2) => {
val keyNotFoundBranch = if (failOnError) {
- s"throw QueryExecutionErrors.mapKeyNotExistError($eval2,
$isElementAtFunction);"
+ s"throw QueryExecutionErrors.mapKeyNotExistError(" +
+ s"$eval2, $isElementAtFunction, $errorContext);"
} else {
s"${ev.isNull} = true;"
}
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala
index 304801e..2a9b3c0 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala
@@ -165,13 +165,16 @@ object QueryExecutionErrors {
messageParameters = Array(index.toString, numElements.toString,
SQLConf.ANSI_ENABLED.key))
}
- def mapKeyNotExistError(key: Any, isElementAtFunction: Boolean):
NoSuchElementException = {
+ def mapKeyNotExistError(
+ key: Any,
+ isElementAtFunction: Boolean,
+ context: String): NoSuchElementException = {
if (isElementAtFunction) {
new SparkNoSuchElementException(errorClass =
"MAP_KEY_DOES_NOT_EXIST_IN_ELEMENT_AT",
- messageParameters = Array(key.toString, SQLConf.ANSI_ENABLED.key))
+ messageParameters = Array(key.toString, SQLConf.ANSI_ENABLED.key,
context))
} else {
new SparkNoSuchElementException(errorClass = "MAP_KEY_DOES_NOT_EXIST",
- messageParameters = Array(key.toString,
SQLConf.ANSI_STRICT_INDEX_OPERATOR.key))
+ messageParameters = Array(key.toString,
SQLConf.ANSI_STRICT_INDEX_OPERATOR.key, context))
}
}
diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/map.sql.out
b/sql/core/src/test/resources/sql-tests/results/ansi/map.sql.out
index 5f7bd9f..5ba3727 100644
--- a/sql/core/src/test/resources/sql-tests/results/ansi/map.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/ansi/map.sql.out
@@ -9,6 +9,9 @@ struct<>
-- !query output
org.apache.spark.SparkNoSuchElementException
Key 5 does not exist. To return NULL instead, use 'try_element_at'. If
necessary set spark.sql.ansi.enabled to false to bypass this error.
+== SQL(line 1, position 7) ==
+select element_at(map(1, 'a', 2, 'b'), 5)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-- !query
@@ -18,6 +21,9 @@ struct<>
-- !query output
org.apache.spark.SparkNoSuchElementException
Key 5 does not exist. If necessary set spark.sql.ansi.strictIndexOperator to
false to bypass this error.
+== SQL(line 1, position 7) ==
+select map(1, 'a', 2, 'b')[5]
+ ^^^^^^^^^^^^^^^^^^^^^^
-- !query
@@ -109,3 +115,6 @@ struct<>
-- !query output
org.apache.spark.SparkNoSuchElementException
Key 5 does not exist. To return NULL instead, use 'try_element_at'. If
necessary set spark.sql.ansi.enabled to false to bypass this error.
+== SQL(line 1, position 7) ==
+select element_at(map(1, 'a', 2, 'b'), 5)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
