This is an automated email from the ASF dual-hosted git repository.
maxgekk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 5d5e7f94315 [SPARK-38687][SQL] Use error classes in the compilation
errors of generators
5d5e7f94315 is described below
commit 5d5e7f94315c233d983139fa39163a838882be89
Author: panbingkun <[email protected]>
AuthorDate: Mon May 23 17:35:33 2022 +0300
[SPARK-38687][SQL] Use error classes in the compilation errors of generators
## What changes were proposed in this pull request?
Migrate the following errors in QueryCompilationErrors onto use error
classes:
- nestedGeneratorError => UNSUPPORTED_GENERATOR.NESTED_IN_EXPRESSIONS
- moreThanOneGeneratorError => UNSUPPORTED_GENERATOR.MULTI_GENERATOR
- generatorOutsideSelectError => UNSUPPORTED_GENERATOR.OUTSIDE_SELECT
- generatorNotExpectedError => UNSUPPORTED_GENERATOR.NOT_GENERATOR
### Why are the changes needed?
Porting compilation errors of generator to new error framework, improve
test coverage, and document expected error messages in tests.
### Does this PR introduce any user-facing change?
No
### How was this patch tested?
By running new test:
```
$ build/sbt "sql/testOnly *QueryCompilationErrorsSuite*"
```
Closes #36617 from panbingkun/SPARK-38687.
Authored-by: panbingkun <[email protected]>
Signed-off-by: Max Gekk <[email protected]>
---
core/src/main/resources/error/error-classes.json | 17 ++++++
.../spark/sql/errors/QueryCompilationErrors.scala | 22 ++++----
.../apache/spark/sql/errors/QueryErrorsBase.scala | 8 ++-
.../sql/catalyst/analysis/AnalysisErrorSuite.scala | 23 ++++----
.../apache/spark/sql/GeneratorFunctionSuite.scala | 9 ++--
.../sql/errors/QueryCompilationErrorsSuite.scala | 61 ++++++++++++++++++++++
6 files changed, 111 insertions(+), 29 deletions(-)
diff --git a/core/src/main/resources/error/error-classes.json
b/core/src/main/resources/error/error-classes.json
index f6fba105872..eb328c6e20a 100644
--- a/core/src/main/resources/error/error-classes.json
+++ b/core/src/main/resources/error/error-classes.json
@@ -295,6 +295,23 @@
},
"sqlState" : "0A000"
},
+ "UNSUPPORTED_GENERATOR" : {
+ "message" : [ "The generator is not supported: " ],
+ "subClass" : {
+ "MULTI_GENERATOR" : {
+ "message" : [ "only one generator allowed per <clause> clause but
found <size>: <generators>" ]
+ },
+ "NESTED_IN_EXPRESSIONS" : {
+ "message" : [ "nested in expressions <expressions>" ]
+ },
+ "NOT_GENERATOR" : {
+ "message" : [ "<name> is expected to be a generator. However, its
class is <classCanonicalName>, which is not a generator." ]
+ },
+ "OUTSIDE_SELECT" : {
+ "message" : [ "outside the SELECT clause, found: <plan>" ]
+ }
+ }
+ },
"UNSUPPORTED_GROUPING_EXPRESSION" : {
"message" : [ "grouping()/grouping_id() can only be used with
GroupingSets/Cube/Rollup" ]
},
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala
index 3d133d6cfab..008f13961a6 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala
@@ -30,7 +30,7 @@ import org.apache.spark.sql.catalyst.expressions.{Alias,
Attribute, AttributeRef
import org.apache.spark.sql.catalyst.plans.JoinType
import org.apache.spark.sql.catalyst.plans.logical.{InsertIntoStatement, Join,
LogicalPlan, SerdeInfo, Window}
import org.apache.spark.sql.catalyst.trees.{Origin, TreeNode}
-import org.apache.spark.sql.catalyst.util.{toPrettySQL, FailFastMode,
ParseMode, PermissiveMode}
+import org.apache.spark.sql.catalyst.util.{FailFastMode, ParseMode,
PermissiveMode}
import org.apache.spark.sql.connector.catalog._
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
import org.apache.spark.sql.connector.catalog.functions.{BoundFunction,
UnboundFunction}
@@ -112,21 +112,19 @@ object QueryCompilationErrors extends QueryErrorsBase {
}
def nestedGeneratorError(trimmedNestedGenerator: Expression): Throwable = {
- new AnalysisException(
- "Generators are not supported when it's nested in " +
- "expressions, but got: " + toPrettySQL(trimmedNestedGenerator))
+ new AnalysisException(errorClass = "UNSUPPORTED_GENERATOR",
+ messageParameters = Array("NESTED_IN_EXPRESSIONS",
toSQLExpr(trimmedNestedGenerator)))
}
def moreThanOneGeneratorError(generators: Seq[Expression], clause: String):
Throwable = {
- new AnalysisException(
- s"Only one generator allowed per $clause clause but found " +
- generators.size + ": " + generators.map(toPrettySQL).mkString(", "))
+ new AnalysisException(errorClass = "UNSUPPORTED_GENERATOR",
+ messageParameters = Array("MULTI_GENERATOR",
+ clause, generators.size.toString,
generators.map(toSQLExpr).mkString(", ")))
}
def generatorOutsideSelectError(plan: LogicalPlan): Throwable = {
- new AnalysisException(
- "Generators are not supported outside the SELECT clause, but " +
- "got: " + plan.simpleString(SQLConf.get.maxToStringFields))
+ new AnalysisException(errorClass = "UNSUPPORTED_GENERATOR",
+ messageParameters = Array("OUTSIDE_SELECT",
plan.simpleString(SQLConf.get.maxToStringFields)))
}
def legacyStoreAssignmentPolicyError(): Throwable = {
@@ -324,8 +322,8 @@ object QueryCompilationErrors extends QueryErrorsBase {
}
def generatorNotExpectedError(name: FunctionIdentifier, classCanonicalName:
String): Throwable = {
- new AnalysisException(s"$name is expected to be a generator. However, " +
- s"its class is $classCanonicalName, which is not a generator.")
+ new AnalysisException(errorClass = "UNSUPPORTED_GENERATOR",
+ messageParameters = Array("NOT_GENERATOR", toSQLId(name.toString),
classCanonicalName))
}
def functionWithUnsupportedSyntaxError(prettyName: String, syntax: String):
Throwable = {
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryErrorsBase.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryErrorsBase.scala
index 81c4d0ac408..5253f0ec877 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryErrorsBase.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryErrorsBase.scala
@@ -19,8 +19,8 @@ package org.apache.spark.sql.errors
import java.util.Locale
-import org.apache.spark.sql.catalyst.expressions.Literal
-import org.apache.spark.sql.catalyst.util.quoteIdentifier
+import org.apache.spark.sql.catalyst.expressions.{Expression, Literal}
+import org.apache.spark.sql.catalyst.util.{quoteIdentifier, toPrettySQL}
import org.apache.spark.sql.types.{DataType, DoubleType, FloatType}
/**
@@ -88,4 +88,8 @@ trait QueryErrorsBase {
def toDSOption(option: String): String = {
quoteByDefault(option)
}
+
+ def toSQLExpr(e: Expression): String = {
+ quoteByDefault(toPrettySQL(e))
+ }
}
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
index 569fd57252a..58cc4418ab0 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
@@ -484,7 +484,7 @@ class AnalysisErrorSuite extends AnalysisTest {
errorTest(
"generator nested in expressions",
listRelation.select(Explode($"list") + 1),
- "Generators are not supported when it's nested in expressions, but got:
(explode(list) + 1)"
+ """The generator is not supported: nested in expressions "(explode(list) +
1)""""
:: Nil
)
@@ -495,29 +495,29 @@ class AnalysisErrorSuite extends AnalysisTest {
AttributeReference("nestedList", ArrayType(ArrayType(IntegerType)))())
nestedListRelation.select(Explode(Explode($"nestedList")))
},
- "Generators are not supported when it's nested in expressions, but got: " +
- "explode(explode(nestedList))" :: Nil
+ "The generator is not supported: nested in expressions " +
+ """"explode(explode(nestedList))"""" :: Nil
)
errorTest(
"SPARK-30998: unsupported nested inner generators for aggregates",
testRelation.select(Explode(Explode(
CreateArray(CreateArray(min($"a") :: max($"a") :: Nil) :: Nil)))),
- "Generators are not supported when it's nested in expressions, but got: " +
- "explode(explode(array(array(min(a), max(a)))))" :: Nil
+ "The generator is not supported: nested in expressions " +
+ """"explode(explode(array(array(min(a), max(a)))))"""" :: Nil
)
errorTest(
"generator nested in expressions for aggregates",
testRelation.select(Explode(CreateArray(min($"a") :: max($"a") :: Nil)) +
1),
- "Generators are not supported when it's nested in expressions, but got: " +
- "(explode(array(min(a), max(a))) + 1)" :: Nil
+ "The generator is not supported: nested in expressions " +
+ """"(explode(array(min(a), max(a))) + 1)"""" :: Nil
)
errorTest(
"generator appears in operator which is not Project",
listRelation.sortBy(Explode($"list").asc),
- "Generators are not supported outside the SELECT clause, but got: Sort" ::
Nil
+ "The generator is not supported: outside the SELECT clause, found: Sort"
:: Nil
)
errorTest(
@@ -567,15 +567,16 @@ class AnalysisErrorSuite extends AnalysisTest {
errorTest(
"more than one generators in SELECT",
listRelation.select(Explode($"list"), Explode($"list")),
- "Only one generator allowed per select clause but found 2: explode(list),
explode(list)" :: Nil
+ "The generator is not supported: only one generator allowed per select
clause but found 2: " +
+ """"explode(list)", "explode(list)"""" :: Nil
)
errorTest(
"more than one generators for aggregates in SELECT",
testRelation.select(Explode(CreateArray(min($"a") :: Nil)),
Explode(CreateArray(max($"a") :: Nil))),
- "Only one generator allowed per select clause but found 2: " +
- "explode(array(min(a))), explode(array(max(a)))" :: Nil
+ "The generator is not supported: only one generator allowed per select
clause but found 2: " +
+ """"explode(array(min(a)))", "explode(array(max(a)))"""" :: Nil
)
errorTest(
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/GeneratorFunctionSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/GeneratorFunctionSuite.scala
index 436ccb08294..b02513d7d78 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/GeneratorFunctionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/GeneratorFunctionSuite.scala
@@ -331,7 +331,7 @@ class GeneratorFunctionSuite extends QueryTest with
SharedSparkSession {
val msg1 = intercept[AnalysisException] {
sql("select 1 + explode(array(min(c2), max(c2))) from t1 group by c1")
}.getMessage
- assert(msg1.contains("Generators are not supported when it's nested in
expressions"))
+ assert(msg1.contains("The generator is not supported: nested in
expressions"))
val msg2 = intercept[AnalysisException] {
sql(
@@ -341,7 +341,8 @@ class GeneratorFunctionSuite extends QueryTest with
SharedSparkSession {
|from t1 group by c1
""".stripMargin)
}.getMessage
- assert(msg2.contains("Only one generator allowed per aggregate clause"))
+ assert(msg2.contains("The generator is not supported: " +
+ "only one generator allowed per aggregate clause"))
}
}
@@ -349,8 +350,8 @@ class GeneratorFunctionSuite extends QueryTest with
SharedSparkSession {
val errMsg = intercept[AnalysisException] {
sql("SELECT array(array(1, 2), array(3))
v").select(explode(explode($"v"))).collect
}.getMessage
- assert(errMsg.contains("Generators are not supported when it's nested in
expressions, " +
- "but got: explode(explode(v))"))
+ assert(errMsg.contains("The generator is not supported: " +
+ """nested in expressions "explode(explode(v))""""))
}
test("SPARK-30997: generators in aggregate expressions for dataframe") {
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala
index 9b255dc4d19..420911e1f30 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala
@@ -583,6 +583,67 @@ class QueryCompilationErrorsSuite
msg = "The deserializer is not supported: try to map \"STRUCT<a: STRING,
b: INT>\" " +
"to Tuple1, but failed as the number of fields does not line up.")
}
+
+ test("UNSUPPORTED_GENERATOR: " +
+ "generators are not supported when it's nested in expressions") {
+ val e = intercept[AnalysisException](
+ sql("""select explode(Array(1, 2, 3)) + 1""").collect()
+ )
+
+ checkErrorClass(
+ exception = e,
+ errorClass = "UNSUPPORTED_GENERATOR",
+ errorSubClass = Some("NESTED_IN_EXPRESSIONS"),
+ msg = """The generator is not supported: """ +
+ """nested in expressions "(explode(array(1, 2, 3)) + 1)"""")
+ }
+
+ test("UNSUPPORTED_GENERATOR: only one generator allowed") {
+ val e = intercept[AnalysisException](
+ sql("""select explode(Array(1, 2, 3)), explode(Array(1, 2,
3))""").collect()
+ )
+
+ checkErrorClass(
+ exception = e,
+ errorClass = "UNSUPPORTED_GENERATOR",
+ errorSubClass = Some("MULTI_GENERATOR"),
+ msg = "The generator is not supported: only one generator allowed per
select clause " +
+ """but found 2: "explode(array(1, 2, 3))", "explode(array(1, 2, 3))""""
+ )
+ }
+
+ test("UNSUPPORTED_GENERATOR: generators are not supported outside the SELECT
clause") {
+ val e = intercept[AnalysisException](
+ sql("""select 1 from t order by explode(Array(1, 2, 3))""").collect()
+ )
+
+ checkErrorClass(
+ exception = e,
+ errorClass = "UNSUPPORTED_GENERATOR",
+ errorSubClass = Some("OUTSIDE_SELECT"),
+ msg = "The generator is not supported: outside the SELECT clause, found:
" +
+ "'Sort [explode(array(1, 2, 3)) ASC NULLS FIRST], true"
+ )
+ }
+
+ test("UNSUPPORTED_GENERATOR: not a generator") {
+ val e = intercept[AnalysisException](
+ sql(
+ """
+ |SELECT explodedvalue.*
+ |FROM VALUES array(1, 2, 3) AS (value)
+ |LATERAL VIEW array_contains(value, 1) AS
explodedvalue""".stripMargin).collect()
+ )
+
+ checkErrorClass(
+ exception = e,
+ errorClass = "UNSUPPORTED_GENERATOR",
+ errorSubClass = Some("NOT_GENERATOR"),
+ msg = """The generator is not supported: `array_contains` is expected to
be a generator. """ +
+ "However, its class is
org.apache.spark.sql.catalyst.expressions.ArrayContains, " +
+ "which is not a generator.; line 4 pos 0"
+ )
+ }
}
class MyCastToString extends SparkUserDefinedFunction(
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
