This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-3.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.4 by this push:
new 905393118c4 [SPARK-45100][SQL][3.4] Fix an internal error from
`reflect()`on `NULL` class and method
905393118c4 is described below
commit 905393118c475d75b76a743c4c805904f12bf018
Author: Max Gekk <[email protected]>
AuthorDate: Fri Sep 8 08:08:13 2023 -0700
[SPARK-45100][SQL][3.4] Fix an internal error from `reflect()`on `NULL`
class and method
### What changes were proposed in this pull request?
In the PR, I propose to check that the `class` and `method` arguments are
not a NULL in `CallMethodViaReflection`. And if they are, throw an
`AnalysisException` with new error class `DATATYPE_MISMATCH.UNEXPECTED_NULL`.
This is a backport of https://github.com/apache/spark/pull/42849.
### Why are the changes needed?
To fix the issue demonstrated by the example:
```sql
$ spark-sql (default)> select reflect('java.util.UUID', CAST(NULL AS
STRING));
[INTERNAL_ERROR] The Spark SQL phase analysis failed with an internal
error. You hit a bug in Spark or the Spark plugins you use. Please, report this
bug to the corresponding communities or vendors, and provide the full stack
trace.
```
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
By running new test:
```
$ build/sbt "test:testOnly *.MiscFunctionsSuite"
```
### Was this patch authored or co-authored using generative AI tooling?
No.
Authored-by: Max Gekk <max.gekkgmail.com>
(cherry picked from commit fd424caf6c46e7030ac2deb2afbe3f4a5fc1095c)
Closes #42855 from MaxGekk/fix-internal-error-in-reflect-3.4.
Authored-by: Max Gekk <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../expressions/CallMethodViaReflection.scala | 8 ++++++++
.../org/apache/spark/sql/MiscFunctionsSuite.scala | 20 ++++++++++++++++++++
2 files changed, 28 insertions(+)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala
index 52b057a3276..4511b5b548d 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala
@@ -78,6 +78,10 @@ case class CallMethodViaReflection(children: Seq[Expression])
"inputExpr" -> toSQLExpr(children.head)
)
)
+ case (e, 0) if e.eval() == null =>
+ DataTypeMismatch(
+ errorSubClass = "UNEXPECTED_NULL",
+ messageParameters = Map("exprName" -> toSQLId("class")))
case (e, 1) if !(e.dataType == StringType && e.foldable) =>
DataTypeMismatch(
errorSubClass = "NON_FOLDABLE_INPUT",
@@ -87,6 +91,10 @@ case class CallMethodViaReflection(children: Seq[Expression])
"inputExpr" -> toSQLExpr(children(1))
)
)
+ case (e, 1) if e.eval() == null =>
+ DataTypeMismatch(
+ errorSubClass = "UNEXPECTED_NULL",
+ messageParameters = Map("exprName" -> toSQLId("method")))
case (e, idx) if idx > 1 &&
!CallMethodViaReflection.typeMapping.contains(e.dataType) =>
DataTypeMismatch(
errorSubClass = "UNEXPECTED_INPUT_TYPE",
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/MiscFunctionsSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/MiscFunctionsSuite.scala
index 45ae3e54977..de7de641e1d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/MiscFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/MiscFunctionsSuite.scala
@@ -34,6 +34,26 @@ class MiscFunctionsSuite extends QueryTest with
SharedSparkSession {
s"reflect('$className', 'method1', a, b)",
s"java_method('$className', 'method1', a, b)"),
Row("m1one", "m1one"))
+
+ checkError(
+ exception = intercept[AnalysisException] {
+ df.selectExpr("reflect(cast(null as string), 'fromString', a)")
+ },
+ errorClass = "DATATYPE_MISMATCH.UNEXPECTED_NULL",
+ parameters = Map(
+ "exprName" -> "`class`",
+ "sqlExpr" -> "\"reflect(CAST(NULL AS STRING), fromString, a)\""),
+ context = ExpectedContext("", "", 0, 45, "reflect(cast(null as string),
'fromString', a)"))
+ checkError(
+ exception = intercept[AnalysisException] {
+ df.selectExpr("reflect('java.util.UUID', cast(null as string), a)")
+ },
+ errorClass = "DATATYPE_MISMATCH.UNEXPECTED_NULL",
+ parameters = Map(
+ "exprName" -> "`method`",
+ "sqlExpr" -> "\"reflect(java.util.UUID, CAST(NULL AS STRING), a)\""),
+ context = ExpectedContext("", "", 0, 49,
+ "reflect('java.util.UUID', cast(null as string), a)"))
}
test("version") {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
