This is an automated email from the ASF dual-hosted git repository.
maxgekk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 4d3bc8f5b55 [SPARK-41580][SQL] Assign name to _LEGACY_ERROR_TEMP_2137
4d3bc8f5b55 is described below
commit 4d3bc8f5b55969f7c954991239ff43f9faba1346
Author: itholic <haejoon....@databricks.com>
AuthorDate: Thu Jan 5 10:58:14 2023 +0500
[SPARK-41580][SQL] Assign name to _LEGACY_ERROR_TEMP_2137
### What changes were proposed in this pull request?
This PR proposes to assign name to _LEGACY_ERROR_TEMP_2137,
"INVALID_JSON_ROOT_FIELD".
### Why are the changes needed?
We should assign proper name to _LEGACY_ERROR_TEMP_*
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
`./build/sbt "sql/testOnly org.apache.spark.sql.SQLQueryTestSuite*`
Closes #39305 from itholic/LEGACY_2137.
Authored-by: itholic <haejoon....@databricks.com>
Signed-off-by: Max Gekk <max.g...@gmail.com>
---
core/src/main/resources/error/error-classes.json | 10 +++++-----
.../org/apache/spark/sql/errors/QueryExecutionErrors.scala | 2 +-
.../spark/sql/execution/datasources/json/JsonSuite.scala | 14 +++++++++++---
3 files changed, 17 insertions(+), 9 deletions(-)
diff --git a/core/src/main/resources/error/error-classes.json
b/core/src/main/resources/error/error-classes.json
index 12f4b0f9c37..29cafdcc1b6 100644
--- a/core/src/main/resources/error/error-classes.json
+++ b/core/src/main/resources/error/error-classes.json
@@ -760,6 +760,11 @@
"The identifier <ident> is invalid. Please, consider quoting it with
back-quotes as `<ident>`."
]
},
+ "INVALID_JSON_ROOT_FIELD" : {
+ "message" : [
+ "Cannot convert JSON root field to target Spark type."
+ ]
+ },
"INVALID_JSON_SCHEMA_MAP_TYPE" : {
"message" : [
"Input schema <jsonSchema> can only contain STRING as a key type for a
MAP."
@@ -4110,11 +4115,6 @@
"Failed to parse an empty string for data type <dataType>"
]
},
- "_LEGACY_ERROR_TEMP_2137" : {
- "message" : [
- "Root converter returned null"
- ]
- },
"_LEGACY_ERROR_TEMP_2138" : {
"message" : [
"Cannot have circular references in bean class, but got the circular
reference of class <clazz>"
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala
index 227e86994f5..0c92d56ed04 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala
@@ -1457,7 +1457,7 @@ private[sql] object QueryExecutionErrors extends
QueryErrorsBase {
def rootConverterReturnNullError(): SparkRuntimeException = {
new SparkRuntimeException(
- errorClass = "_LEGACY_ERROR_TEMP_2137",
+ errorClass = "INVALID_JSON_ROOT_FIELD",
messageParameters = Map.empty)
}
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
index 0d2c98316e7..a4b7df9af42 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
@@ -25,11 +25,12 @@ import java.time.{Duration, Instant, LocalDate,
LocalDateTime, Period, ZoneId}
import java.util.Locale
import com.fasterxml.jackson.core.JsonFactory
+import org.apache.commons.lang3.exception.ExceptionUtils
import org.apache.hadoop.fs.{Path, PathFilter}
import org.apache.hadoop.io.SequenceFile.CompressionType
import org.apache.hadoop.io.compress.GzipCodec
-import org.apache.spark.{SparkConf, SparkException, SparkUpgradeException,
TestUtils}
+import org.apache.spark.{SparkConf, SparkException, SparkRuntimeException,
SparkUpgradeException, TestUtils}
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.{functions => F, _}
import org.apache.spark.sql.catalyst.json._
@@ -3192,10 +3193,17 @@ abstract class JsonSuite
}
test("SPARK-36379: proceed parsing with root nulls in permissive mode") {
- assert(intercept[SparkException] {
+ val exception = intercept[SparkException] {
spark.read.option("mode", "failfast")
.schema("a string").json(Seq("""[{"a": "str"},
null]""").toDS).collect()
- }.getMessage.contains("Malformed records are detected"))
+ }
+ assert(exception.getMessage.contains("Malformed records are detected"))
+
+ checkError(
+ exception =
ExceptionUtils.getRootCause(exception).asInstanceOf[SparkRuntimeException],
+ errorClass = "INVALID_JSON_ROOT_FIELD",
+ parameters = Map.empty
+ )
// Permissive modes should proceed parsing malformed records (null).
// Here, since an array fails to parse in the middle, we will return one
row.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
