This is an automated email from the ASF dual-hosted git repository.
yuanzhou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 300d0b4b87 [GLUTEN-11142][VL] Add timestampadd support for Spark 4.0
(#11171)
300d0b4b87 is described below
commit 300d0b4b87560a9720a539ef439608e24694fecb
Author: Mingliang Zhu <[email protected]>
AuthorDate: Tue Nov 25 20:41:23 2025 +0800
[GLUTEN-11142][VL] Add timestampadd support for Spark 4.0 (#11171)
* [GLUTEN-11142][VL] Add timestampadd support for Spark 4.0
---
.../functions/DateFunctionsValidateSuite.scala | 3 +-
.../gluten/utils/velox/VeloxTestSettings.scala | 4 --
.../expressions/GlutenDateExpressionsSuite.scala | 50 +++++++++++-----------
.../gluten/sql/shims/spark40/Spark40Shims.scala | 3 ++
4 files changed, 30 insertions(+), 30 deletions(-)
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/functions/DateFunctionsValidateSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/functions/DateFunctionsValidateSuite.scala
index 5d2d071f46..1eee8a3e52 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/functions/DateFunctionsValidateSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/functions/DateFunctionsValidateSuite.scala
@@ -278,8 +278,7 @@ abstract class DateFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- // TODO: fix on spark-4.0
- testWithRangeSparkVersion("timestampadd", "3.3", "3.5") {
+ testWithMinSparkVersion("timestampadd", "3.3") {
withTempPath {
path =>
val ts = Timestamp.valueOf("2020-02-29 00:00:00.500")
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 4c66d77e57..c1ffb63301 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -148,10 +148,6 @@ class VeloxTestSettings extends BackendTestSettings {
// Vanilla Spark does not have a unified DST Timestamp fastTime.
1320570000000L and
// 1320566400000L both represent 2011-11-06 01:00:00.
.exclude("SPARK-42635: timestampadd near daylight saving transition")
-
- // TODO: fix in Spark-4.0
- .exclude("SPARK-50669: timestampadd with long types")
-
//
https://github.com/facebookincubator/velox/pull/10563/files#diff-140dc50e6dac735f72d29014da44b045509df0dd1737f458de1fe8cfd33d8145
.excludeGlutenTest("from_unixtime")
// Replaced by a gluten test to pass timezone through config.
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala
index 704e4f5d39..306a377ff5 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala
@@ -478,8 +478,7 @@ class GlutenDateExpressionsSuite extends
DateExpressionsSuite with GlutenTestsTr
}
}
- // TODO: fix in Spark-4.0
- ignoreGluten("SPARK-42635: timestampadd near daylight saving transition") {
+ testGluten("SPARK-42635: timestampadd near daylight saving transition") {
// In America/Los_Angeles timezone, timestamp value `skippedTime` is
2011-03-13 03:00:00.
// The next second of 2011-03-13 01:59:59 jumps to 2011-03-13 03:00:00.
val skippedTime = 1300010400000000L
@@ -492,91 +491,94 @@ class GlutenDateExpressionsSuite extends
DateExpressionsSuite with GlutenTestsTr
// Adding one day is **not** equivalent to adding <unit>_PER_DAY time
units, because not every
// day has 24 hours: 2011-03-13 has 23 hours, 2011-11-06 has 25 hours.
- // timestampadd(DAY, 1, 2011-03-12 03:00:00) = 2011-03-13 03:00:00
+ // timestampadd(DAY, 1L, 2011-03-12 03:00:00) = 2011-03-13 03:00:00
checkEvaluation(
- TimestampAdd("DAY", Literal(1), Literal(skippedTime - 23 *
MICROS_PER_HOUR, TimestampType)),
+ TimestampAdd(
+ "DAY",
+ Literal(1L),
+ Literal(skippedTime - 23 * MICROS_PER_HOUR, TimestampType)),
skippedTime)
- // timestampadd(HOUR, 24, 2011-03-12 03:00:00) = 2011-03-13 04:00:00
+ // timestampadd(HOUR, 24L, 2011-03-12 03:00:00) = 2011-03-13 04:00:00
checkEvaluation(
TimestampAdd(
"HOUR",
- Literal(24),
+ Literal(24L),
Literal(skippedTime - 23 * MICROS_PER_HOUR, TimestampType)),
skippedTime + MICROS_PER_HOUR)
- // timestampadd(HOUR, 23, 2011-03-12 03:00:00) = 2011-03-13 03:00:00
+ // timestampadd(HOUR, 23L, 2011-03-12 03:00:00) = 2011-03-13 03:00:00
checkEvaluation(
TimestampAdd(
"HOUR",
- Literal(23),
+ Literal(23L),
Literal(skippedTime - 23 * MICROS_PER_HOUR, TimestampType)),
skippedTime)
// timestampadd(SECOND, SECONDS_PER_DAY, 2011-03-12 03:00:00) =
2011-03-13 04:00:00
checkEvaluation(
TimestampAdd(
"SECOND",
- Literal(SECONDS_PER_DAY.toInt),
+ Literal(SECONDS_PER_DAY),
Literal(skippedTime - 23 * MICROS_PER_HOUR, TimestampType)),
skippedTime + MICROS_PER_HOUR)
// timestampadd(SECOND, SECONDS_PER_DAY, 2011-03-12 03:00:00) =
2011-03-13 03:59:59
checkEvaluation(
TimestampAdd(
"SECOND",
- Literal(SECONDS_PER_DAY.toInt - 1),
+ Literal(SECONDS_PER_DAY - 1),
Literal(skippedTime - 23 * MICROS_PER_HOUR, TimestampType)),
skippedTime + MICROS_PER_HOUR - MICROS_PER_SECOND
)
- // timestampadd(DAY, 1, 2011-11-05 02:00:00) = 2011-11-06 02:00:00
+ // timestampadd(DAY, 1L, 2011-11-05 02:00:00) = 2011-11-06 02:00:00
checkEvaluation(
TimestampAdd(
"DAY",
- Literal(1),
+ Literal(1L),
Literal(repeatedTime - 24 * MICROS_PER_HOUR, TimestampType)),
repeatedTime + MICROS_PER_HOUR)
- // timestampadd(DAY, 1, 2011-11-05 01:00:00) = 2011-11-06 01:00:00
(pre-transition)
+ // timestampadd(DAY, 1L, 2011-11-05 01:00:00) = 2011-11-06 01:00:00
(pre-transition)
checkEvaluation(
TimestampAdd(
"DAY",
- Literal(1),
+ Literal(1L),
Literal(repeatedTime - 25 * MICROS_PER_HOUR, TimestampType)),
repeatedTime - MICROS_PER_HOUR)
- // timestampadd(DAY, -1, 2011-11-07 01:00:00) = 2011-11-06 01:00:00
(post-transition)
+ // timestampadd(DAY, -1L, 2011-11-07 01:00:00) = 2011-11-06 01:00:00
(post-transition)
// Vanilla spark result is 1320570000000000L, velox result is
1320566400000000L, they
// are all 2011-11-06 01:00:00.
checkEvaluation(
TimestampAdd(
"DAY",
- Literal(-1),
+ Literal(-1L),
Literal(repeatedTime + 24 * MICROS_PER_HOUR, TimestampType)),
repeatedTime - MICROS_PER_HOUR)
- // timestampadd(MONTH, 1, 2011-10-06 01:00:00) = 2011-11-06 01:00:00
(pre-transition)
+ // timestampadd(MONTH, 1L, 2011-10-06 01:00:00) = 2011-11-06 01:00:00
(pre-transition)
checkEvaluation(
TimestampAdd(
"MONTH",
- Literal(1),
+ Literal(1L),
Literal(repeatedTime - MICROS_PER_HOUR - 31 * MICROS_PER_DAY,
TimestampType)),
repeatedTime - MICROS_PER_HOUR)
- // timestampadd(MONTH, -1, 2011-12-06 01:00:00) = 2011-11-06 01:00:00
(post-transition)
+ // timestampadd(MONTH, -1L, 2011-12-06 01:00:00) = 2011-11-06 01:00:00
(post-transition)
// Vanilla spark result is 1320570000000000L, velox result is
1320566400000000L, they
// are all 2011-11-06 01:00:00.
checkEvaluation(
TimestampAdd(
"MONTH",
- Literal(-1),
+ Literal(-1L),
Literal(repeatedTime + 30 * MICROS_PER_DAY, TimestampType)),
repeatedTime - MICROS_PER_HOUR)
- // timestampadd(HOUR, 23, 2011-11-05 02:00:00) = 2011-11-06 01:00:00
(pre-transition)
+ // timestampadd(HOUR, 23L, 2011-11-05 02:00:00) = 2011-11-06 01:00:00
(pre-transition)
checkEvaluation(
TimestampAdd(
"HOUR",
- Literal(23),
+ Literal(23L),
Literal(repeatedTime - 24 * MICROS_PER_HOUR, TimestampType)),
repeatedTime - MICROS_PER_HOUR)
- // timestampadd(HOUR, 24, 2011-11-05 02:00:00) = 2011-11-06 01:00:00
(post-transition)
+ // timestampadd(HOUR, 24L, 2011-11-05 02:00:00) = 2011-11-06 01:00:00
(post-transition)
checkEvaluation(
TimestampAdd(
"HOUR",
- Literal(24),
+ Literal(24L),
Literal(repeatedTime - 24 * MICROS_PER_HOUR, TimestampType)),
repeatedTime)
}
diff --git
a/shims/spark40/src/main/scala/org/apache/gluten/sql/shims/spark40/Spark40Shims.scala
b/shims/spark40/src/main/scala/org/apache/gluten/sql/shims/spark40/Spark40Shims.scala
index 21bc6c0571..70ff762e82 100644
---
a/shims/spark40/src/main/scala/org/apache/gluten/sql/shims/spark40/Spark40Shims.scala
+++
b/shims/spark40/src/main/scala/org/apache/gluten/sql/shims/spark40/Spark40Shims.scala
@@ -704,6 +704,9 @@ class Spark40Shims extends SparkShims {
override def extractExpressionTimestampAddUnit(exp: Expression):
Option[Seq[String]] = {
exp match {
+ // Velox does not support quantity larger than Int.MaxValue.
+ case TimestampAdd(_, LongLiteral(quantity), _, _) if quantity >
Integer.MAX_VALUE =>
+ Option.empty
case timestampAdd: TimestampAdd =>
Option.apply(Seq(timestampAdd.unit,
timestampAdd.timeZoneId.getOrElse("")))
case _ => Option.empty
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
