This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new bed4de55cf [GLUTEN-8639][VL] Support casting from double/float to
timestamp (#8640)
bed4de55cf is described below
commit bed4de55cf4965a8aea6353b62e3bbe0d2e0fcbf
Author: Arnav Balyan <[email protected]>
AuthorDate: Wed Mar 19 07:41:21 2025 +0530
[GLUTEN-8639][VL] Support casting from double/float to timestamp (#8640)
---
.../sql/catalyst/expressions/VeloxCastSuite.scala | 70 ++++++++++++++++++++++
.../substrait/SubstraitToVeloxPlanValidator.cc | 4 +-
.../gluten/utils/velox/VeloxTestSettings.scala | 1 +
.../gluten/utils/velox/VeloxTestSettings.scala | 1 +
4 files changed, 74 insertions(+), 2 deletions(-)
diff --git
a/backends-velox/src/test/scala/org/apache/spark/sql/catalyst/expressions/VeloxCastSuite.scala
b/backends-velox/src/test/scala/org/apache/spark/sql/catalyst/expressions/VeloxCastSuite.scala
new file mode 100644
index 0000000000..9e40832f75
--- /dev/null
+++
b/backends-velox/src/test/scala/org/apache/spark/sql/catalyst/expressions/VeloxCastSuite.scala
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.catalyst.expressions
+
+import org.apache.gluten.execution.VeloxWholeStageTransformerSuite
+
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.UTC_OPT
+import org.apache.spark.sql.types._
+
+import java.sql.Timestamp
+import java.util.TimeZone
+
+class VeloxCastSuite extends VeloxWholeStageTransformerSuite with
ExpressionEvalHelper {
+ def cast(v: Any, targetType: DataType, timeZoneId: Option[String] = None):
Cast = {
+ v match {
+ case lit: Expression =>
+ logDebug(s"Cast from: ${lit.dataType.typeName}, to:
${targetType.typeName}")
+ Cast(lit, targetType, timeZoneId)
+ case _ =>
+ val lit = Literal(v)
+ logDebug(s"Cast from: ${lit.dataType.typeName}, to:
${targetType.typeName}")
+ Cast(lit, targetType, timeZoneId)
+ }
+ }
+
+ test("cast from double to timestamp format") {
+ val originalDefaultTz = TimeZone.getDefault
+ try {
+ TimeZone.setDefault(TimeZone.getTimeZone("UTC"))
+
+ checkEvaluation(
+ cast(0.0, TimestampType, UTC_OPT),
+ Timestamp.valueOf("1970-01-01 00:00:00")
+ )
+
+ checkEvaluation(
+ cast(1.5, TimestampType, UTC_OPT),
+ Timestamp.valueOf("1970-01-01 00:00:01.5")
+ )
+
+ checkEvaluation(
+ cast(12345.6789, TimestampType, UTC_OPT),
+ Timestamp.valueOf("1970-01-01 03:25:45.6789")
+ )
+
+ checkEvaluation(
+ cast(-1.2, TimestampType, UTC_OPT),
+ Timestamp.valueOf("1969-12-31 23:59:58.8")
+ )
+ } finally {
+ TimeZone.setDefault(originalDefaultTz)
+ }
+ }
+ override protected val resourcePath: String = "N/A"
+ override protected val fileFormat: String = "N/A"
+}
diff --git a/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc
b/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc
index 899a16d0d9..3de256ff31 100644
--- a/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc
+++ b/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc
@@ -241,7 +241,6 @@ bool SubstraitToVeloxPlanValidator::isAllowedCast(const
TypePtr& fromType, const
// 2. Date to most categories except few supported types is not allowed.
// 3. Timestamp to most categories except few supported types is not allowed.
// 4. Certain complex types are not allowed.
-
// Don't support isIntervalYearMonth.
if (fromType->isIntervalYearMonth() || toType->isIntervalYearMonth()) {
LOG_VALIDATION_MSG("Casting involving INTERVAL_YEAR_MONTH is not
supported.");
@@ -274,7 +273,8 @@ bool SubstraitToVeloxPlanValidator::isAllowedCast(const
TypePtr& fromType, const
if (fromType->isVarchar()) {
return true;
}
- if (fromType->isTinyint() || fromType->isSmallint() ||
fromType->isInteger() || fromType->isBigint()) {
+ if (fromType->isTinyint() || fromType->isSmallint() ||
fromType->isInteger() || fromType->isBigint() ||
+ fromType->isDouble() || fromType->isReal()) {
return true;
}
LOG_VALIDATION_MSG("Casting from " + fromType->toString() + " to TIMESTAMP
is not supported.");
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 1b8d4b9737..e51b41b479 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -99,6 +99,7 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude(
"Process Infinity, -Infinity, NaN in case insensitive manner" // +inf
not supported in folly.
)
+ .exclude("cast from timestamp II") // Rewrite test for Gluten not
supported with ANSI mode
.exclude("ANSI mode: Throw exception on casting out-of-range value to byte
type")
.exclude("ANSI mode: Throw exception on casting out-of-range value to
short type")
.exclude("ANSI mode: Throw exception on casting out-of-range value to int
type")
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 7bae3d5ff6..1d5883bb8a 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -100,6 +100,7 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude(
"Process Infinity, -Infinity, NaN in case insensitive manner" // +inf
not supported in folly.
)
+ .exclude("cast from timestamp II") // Rewrite test for Gluten not
supported with ANSI mode
.exclude("ANSI mode: Throw exception on casting out-of-range value to byte
type")
.exclude("ANSI mode: Throw exception on casting out-of-range value to
short type")
.exclude("ANSI mode: Throw exception on casting out-of-range value to int
type")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
