This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 276bdbafe83 [SPARK-38967][SQL] Turn
"spark.sql.ansi.strictIndexOperator" into an internal configuration
276bdbafe83 is described below
commit 276bdbafe83a5c0b8425a20eb8101a630be8b752
Author: Gengliang Wang <[email protected]>
AuthorDate: Wed Apr 20 21:41:58 2022 +0800
[SPARK-38967][SQL] Turn "spark.sql.ansi.strictIndexOperator" into an
internal configuration
### What changes were proposed in this pull request?
Currently, most the ANSI error message shows the hint "If necessary set
spark.sql.ansi.enabled to false to bypass this error."
There is only one special case: "Map key not exist" or "array index out of
bound" from the `[]` operator. It shows the config
spark.sql.ansi.strictIndexOperator instead.
This one special case can confuse users. To make it simple:
- Turn "spark.sql.ansi.strictIndexOperator" into an internal configuration
- Show the configuration `spark.sql.ansi.enabled` in error messages instead
- If it is "map key not exist" error, show the hint for using
`try_element_at`. Otherwise, we don't show it. For array, `[]` operator is
using 0-based index while `try_element_at` is using 1-based index.
### Why are the changes needed?
Make the hints in ANSI runtime error message simple and consistent
### Does this PR introduce _any_ user-facing change?
No, the new configuration is not released yet.
### How was this patch tested?
Existing UT
Closes #36282 from gengliangwang/updateErrorMsg.
Authored-by: Gengliang Wang <[email protected]>
Signed-off-by: Wenchen Fan <[email protected]>
---
core/src/main/resources/error/error-classes.json | 3 ---
.../sql/catalyst/expressions/collectionOperations.scala | 2 --
.../sql/catalyst/expressions/complexTypeExtractors.scala | 7 ++-----
.../apache/spark/sql/errors/QueryExecutionErrors.scala | 16 ++++------------
.../scala/org/apache/spark/sql/internal/SQLConf.scala | 1 +
.../test/resources/sql-tests/results/ansi/array.sql.out | 4 ++--
.../test/resources/sql-tests/results/ansi/map.sql.out | 2 +-
7 files changed, 10 insertions(+), 25 deletions(-)
diff --git a/core/src/main/resources/error/error-classes.json
b/core/src/main/resources/error/error-classes.json
index 23c1cee1c72..6d070c35c74 100644
--- a/core/src/main/resources/error/error-classes.json
+++ b/core/src/main/resources/error/error-classes.json
@@ -124,9 +124,6 @@
"sqlState" : "42000"
},
"MAP_KEY_DOES_NOT_EXIST" : {
- "message" : [ "Key %s does not exist. If necessary set %s to false to
bypass this error.%s" ]
- },
- "MAP_KEY_DOES_NOT_EXIST_IN_ELEMENT_AT" : {
"message" : [ "Key %s does not exist. To return NULL instead, use
'try_element_at'. If necessary set %s to false to bypass this error.%s" ]
},
"MISSING_COLUMN" : {
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
index ca008391d1b..1b42ea5eb87 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
@@ -2116,8 +2116,6 @@ case class ElementAt(
case MapType(_, valueType, _) => valueType
}
- override val isElementAtFunction: Boolean = true
-
override def inputTypes: Seq[AbstractDataType] = {
(left.dataType, right.dataType) match {
case (arr: ArrayType, e2: IntegralType) if (e2 != LongType) =>
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala
index 3cd404a9c0d..e889d37411c 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala
@@ -341,8 +341,6 @@ trait GetArrayItemUtil {
*/
trait GetMapValueUtil extends BinaryExpression with ImplicitCastInputTypes {
- protected val isElementAtFunction: Boolean = false
-
// todo: current search is O(n), improve it.
def getValueEval(
value: Any,
@@ -367,7 +365,7 @@ trait GetMapValueUtil extends BinaryExpression with
ImplicitCastInputTypes {
if (!found) {
if (failOnError) {
- throw QueryExecutionErrors.mapKeyNotExistError(ordinal,
isElementAtFunction, origin.context)
+ throw QueryExecutionErrors.mapKeyNotExistError(ordinal, origin.context)
} else {
null
}
@@ -403,8 +401,7 @@ trait GetMapValueUtil extends BinaryExpression with
ImplicitCastInputTypes {
lazy val errorContext = ctx.addReferenceObj("errCtx", origin.context)
nullSafeCodeGen(ctx, ev, (eval1, eval2) => {
val keyNotFoundBranch = if (failOnError) {
- s"throw QueryExecutionErrors.mapKeyNotExistError(" +
- s"$eval2, $isElementAtFunction, $errorContext);"
+ s"throw QueryExecutionErrors.mapKeyNotExistError($eval2,
$errorContext);"
} else {
s"${ev.isNull} = true;"
}
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala
index afe5c26bfcc..c73b78b264c 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala
@@ -152,7 +152,7 @@ object QueryExecutionErrors extends QueryErrorsBase {
}
def invalidArrayIndexError(index: Int, numElements: Int):
ArrayIndexOutOfBoundsException = {
- invalidArrayIndexErrorInternal(index, numElements,
SQLConf.ANSI_STRICT_INDEX_OPERATOR.key)
+ invalidArrayIndexErrorInternal(index, numElements,
SQLConf.ANSI_ENABLED.key)
}
def invalidInputIndexError(index: Int, numElements: Int):
ArrayIndexOutOfBoundsException = {
@@ -176,17 +176,9 @@ object QueryExecutionErrors extends QueryErrorsBase {
Array(toSQLValue(index), toSQLValue(numElements),
SQLConf.ANSI_ENABLED.key))
}
- def mapKeyNotExistError(
- key: Any,
- isElementAtFunction: Boolean,
- context: String): NoSuchElementException = {
- if (isElementAtFunction) {
- new SparkNoSuchElementException(errorClass =
"MAP_KEY_DOES_NOT_EXIST_IN_ELEMENT_AT",
- messageParameters = Array(toSQLValue(key), SQLConf.ANSI_ENABLED.key,
context))
- } else {
- new SparkNoSuchElementException(errorClass = "MAP_KEY_DOES_NOT_EXIST",
- messageParameters = Array(toSQLValue(key),
SQLConf.ANSI_STRICT_INDEX_OPERATOR.key, context))
- }
+ def mapKeyNotExistError(key: Any, context: String): NoSuchElementException =
{
+ new SparkNoSuchElementException(errorClass = "MAP_KEY_DOES_NOT_EXIST",
+ messageParameters = Array(toSQLValue(key), SQLConf.ANSI_ENABLED.key,
context))
}
def invalidFractionOfSecondError(): DateTimeException = {
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 301d792bb3e..50d09d046bc 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -2877,6 +2877,7 @@ object SQLConf {
.createWithDefault(false)
val ANSI_STRICT_INDEX_OPERATOR =
buildConf("spark.sql.ansi.strictIndexOperator")
+ .internal()
.doc(s"When true and '${ANSI_ENABLED.key}' is true, accessing complex SQL
types via [] " +
"operator will throw an exception if array index is out of bound, or map
key does not " +
"exist. Otherwise, Spark will return a null result when accessing an
invalid index.")
diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/array.sql.out
b/sql/core/src/test/resources/sql-tests/results/ansi/array.sql.out
index 00ac2eeba7f..187630d78d2 100644
--- a/sql/core/src/test/resources/sql-tests/results/ansi/array.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/ansi/array.sql.out
@@ -254,7 +254,7 @@ select array(1, 2, 3)[5]
struct<>
-- !query output
org.apache.spark.SparkArrayIndexOutOfBoundsException
-Invalid index: 5, numElements: 3. If necessary set
spark.sql.ansi.strictIndexOperator to false to bypass this error.
+Invalid index: 5, numElements: 3. If necessary set spark.sql.ansi.enabled to
false to bypass this error.
-- !query
@@ -263,7 +263,7 @@ select array(1, 2, 3)[-1]
struct<>
-- !query output
org.apache.spark.SparkArrayIndexOutOfBoundsException
-Invalid index: -1, numElements: 3. If necessary set
spark.sql.ansi.strictIndexOperator to false to bypass this error.
+Invalid index: -1, numElements: 3. If necessary set spark.sql.ansi.enabled to
false to bypass this error.
-- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/map.sql.out
b/sql/core/src/test/resources/sql-tests/results/ansi/map.sql.out
index 5ba37278fbc..fa41865dd7b 100644
--- a/sql/core/src/test/resources/sql-tests/results/ansi/map.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/ansi/map.sql.out
@@ -20,7 +20,7 @@ select map(1, 'a', 2, 'b')[5]
struct<>
-- !query output
org.apache.spark.SparkNoSuchElementException
-Key 5 does not exist. If necessary set spark.sql.ansi.strictIndexOperator to
false to bypass this error.
+Key 5 does not exist. To return NULL instead, use 'try_element_at'. If
necessary set spark.sql.ansi.enabled to false to bypass this error.
== SQL(line 1, position 7) ==
select map(1, 'a', 2, 'b')[5]
^^^^^^^^^^^^^^^^^^^^^^
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
