This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new f8263bb9fdf [SPARK-38990][SQL] Avoid `NullPointerException` when
evaluating date_trunc/trunc format as a bound reference
f8263bb9fdf is described below
commit f8263bb9fdf9cd9b3c645ddb05eaddf39a2c4335
Author: Bruce Robbins <[email protected]>
AuthorDate: Fri Apr 22 12:30:34 2022 +0900
[SPARK-38990][SQL] Avoid `NullPointerException` when evaluating
date_trunc/trunc format as a bound reference
### What changes were proposed in this pull request?
Change `TruncInstant.evalHelper` to pass the input row to `format.eval`
when `format` is a not a literal (and therefore might be a bound reference).
### Why are the changes needed?
This query fails with a `java.lang.NullPointerException`:
```
select date_trunc(col1, col2)
from values
('week', timestamp'2012-01-01')
as data(col1, col2);
```
This only happens if the data comes from an inline table. When the source
is an inline table, `ConvertToLocalRelation` attempts to evaluate the function
against the data in interpreted mode.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Update to unit tests.
Closes #36312 from bersprockets/date_trunc_issue.
Authored-by: Bruce Robbins <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
(cherry picked from commit 2e4f4abf553cedec1fa8611b9494a01d24e6238a)
Signed-off-by: Hyukjin Kwon <[email protected]>
---
.../catalyst/expressions/datetimeExpressions.scala | 2 +-
.../catalyst/expressions/DateExpressionsSuite.scala | 20 +++++++++++++++++++-
2 files changed, 20 insertions(+), 2 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
index 0e74eff7f5a..3eebeb2c5c2 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
@@ -2058,7 +2058,7 @@ trait TruncInstant extends BinaryExpression with
ImplicitCastInputTypes {
val level = if (format.foldable) {
truncLevel
} else {
- DateTimeUtils.parseTruncLevel(format.eval().asInstanceOf[UTF8String])
+
DateTimeUtils.parseTruncLevel(format.eval(input).asInstanceOf[UTF8String])
}
if (level < minLevel) {
// unknown format or too small level
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
index 5d77bcfe2d4..9ab7b65dadd 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
@@ -30,7 +30,7 @@ import scala.reflect.ClassTag
import scala.util.Random
import org.apache.spark.{SparkFunSuite, SparkUpgradeException}
-import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow}
import
org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection
import org.apache.spark.sql.catalyst.util.{DateTimeUtils, IntervalUtils,
TimestampFormatter}
import org.apache.spark.sql.catalyst.util.DateTimeConstants._
@@ -754,6 +754,15 @@ class DateExpressionsSuite extends SparkFunSuite with
ExpressionEvalHelper {
checkEvaluation(
TruncDate(Literal.create(input, DateType),
NonFoldableLiteral.create(fmt, StringType)),
expected)
+ // SPARK-38990: ensure that evaluation with input rows also works
+ val catalystInput = CatalystTypeConverters.convertToCatalyst(input)
+ val inputRow = InternalRow(catalystInput, UTF8String.fromString(fmt))
+ checkEvaluation(
+ TruncDate(
+ BoundReference(ordinal = 0, dataType = DateType, nullable = true),
+ BoundReference(ordinal = 1, dataType = StringType, nullable = true)),
+ expected,
+ inputRow)
}
test("TruncDate") {
@@ -780,6 +789,15 @@ class DateExpressionsSuite extends SparkFunSuite with
ExpressionEvalHelper {
TruncTimestamp(
NonFoldableLiteral.create(fmt, StringType), Literal.create(input,
TimestampType)),
expected)
+ // SPARK-38990: ensure that evaluation with input rows also works
+ val catalystInput = CatalystTypeConverters.convertToCatalyst(input)
+ val inputRow = InternalRow(UTF8String.fromString(fmt), catalystInput)
+ checkEvaluation(
+ TruncTimestamp(
+ BoundReference(ordinal = 0, dataType = StringType, nullable = true),
+ BoundReference(ordinal = 1, dataType = TimestampType, nullable =
true)),
+ expected,
+ inputRow)
}
test("TruncTimestamp") {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
