This is an automated email from the ASF dual-hosted git repository.
zhli pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new d18fd7ad5 [VL] Enable unix_seconds Spark function (#5602)
d18fd7ad5 is described below
commit d18fd7ad502d0bf406f091ea4912a259bf32fac9
Author: Ankita Victor <[email protected]>
AuthorDate: Tue May 7 09:48:35 2024 +0530
[VL] Enable unix_seconds Spark function (#5602)
[VL] Enable unix_seconds Spark function.
---
.../scala/org/apache/gluten/utils/CHExpressionUtil.scala | 1 +
.../gluten/execution/ScalarFunctionsValidateSuite.scala | 15 +++++++++++++++
docs/velox-backend-support-progress.md | 9 +++++----
.../org/apache/gluten/expression/ExpressionMappings.scala | 1 +
.../org/apache/gluten/expression/ExpressionNames.scala | 1 +
5 files changed, 23 insertions(+), 4 deletions(-)
diff --git
a/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala
b/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala
index bbe65034a..0e645d039 100644
---
a/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala
+++
b/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala
@@ -178,6 +178,7 @@ object CHExpressionUtil {
ARRAYS_ZIP -> DefaultValidator(),
DATE_FROM_UNIX_DATE -> DefaultValidator(),
UNIX_DATE -> DefaultValidator(),
+ UNIX_SECONDS -> DefaultValidator(),
MONOTONICALLY_INCREASING_ID -> DefaultValidator(),
SPARK_PARTITION_ID -> DefaultValidator(),
URL_DECODE -> DefaultValidator(),
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
index 6fb451411..0a7a7d6cb 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
@@ -844,6 +844,21 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateTest {
}
}
+ test("unix_seconds") {
+ withTempPath {
+ path =>
+ val t1 = Timestamp.valueOf("2024-08-22 10:10:10.010")
+ val t2 = Timestamp.valueOf("2014-12-31 00:00:00.012")
+ val t3 = Timestamp.valueOf("1968-12-31 23:59:59.001")
+ Seq(t1, t2, t3).toDF("t").write.parquet(path.getCanonicalPath)
+
+
spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("view")
+ runQueryAndCompare("select unix_seconds(t) from view") {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ }
+ }
+
test("unix_millis") {
withTempPath {
path =>
diff --git a/docs/velox-backend-support-progress.md
b/docs/velox-backend-support-progress.md
index 90f4fda9e..e98587efb 100644
--- a/docs/velox-backend-support-progress.md
+++ b/docs/velox-backend-support-progress.md
@@ -338,8 +338,8 @@ Gluten supports 199 functions. (Drag to right to see all
data types)
| second | second |
| S | | | | | | |
| | | | | | | | |
| | | |
| session_window | |
| | | | | | | |
| | | | | | | | |
| | | |
| timestamp | |
| | | | | | | |
| | | | | | | | |
| | | |
-| timestamp_micros | | timestamp_micros
| S | | | |
| | | | | | | | | |
| | | | | |
-| timestamp_millis | |
timestamp_millis | S | | |
| | | | | | | | |
| | | | | | | |
+| timestamp_micros | | timestamp_micros
| S | | | | | | |
| | | | | | | | |
| | | |
+| timestamp_millis | | timestamp_millis
| S | | | | | | |
| | | | | | | | |
| | | |
| timestamp_seconds | |
| | | | | | | |
| | | | | | | | |
| | | |
| to_date | |
| S | | | | | | |
| | S | S | | | | | |
| | | |
| to_timestamp | |
| | | | | | | |
| | | | | | | | |
| | | |
@@ -347,8 +347,9 @@ Gluten supports 199 functions. (Drag to right to see all
data types)
| to_utc_timestamp | |
| | | | | | | |
| | | | | | | | |
| | | |
| trunc | |
| | | | | | | |
| | | | | | | | |
| | | |
| unix_timestamp | | unix_timestamp
| | | | | | | |
| | | | | | | | |
| | | |
-| unix_millis | | unix_millis | S
| | | | | | |
| | | | | | | | |
| | | |
-| unix_micros | | unix_micros |
S | | | | | | |
| | | | | | | | |
| | | |
+| unix_seconds | | unix_seconds
| S | | | | | | |
| | | | | | | | |
| | | |
+| unix_millis | | unix_millis
| S | | | | | | |
| | | | | | | | |
| | | |
+| unix_micros | | unix_micros
| S | | | | | | |
| | | | | | | | |
| | | |
| weekday | |
| S | | | | | | |
| | S | | | | | | |
| | | |
| weekofyear | week,week_of_year |
| S | | | | | | |
| | | | | | | | |
| | | |
| window | |
| | | | | | | |
| | | | | | | | |
| | | |
diff --git
a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala
b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala
index 19a77e515..920b6fab8 100644
---
a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala
+++
b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala
@@ -189,6 +189,7 @@ object ExpressionMappings {
Sig[MakeYMInterval](MAKE_YM_INTERVAL),
Sig[ToUTCTimestamp](TO_UTC_TIMESTAMP),
Sig[FromUTCTimestamp](FROM_UTC_TIMESTAMP),
+ Sig[UnixSeconds](UNIX_SECONDS),
Sig[UnixMillis](UNIX_MILLIS),
Sig[UnixMicros](UNIX_MICROS),
Sig[MillisToTimestamp](TIMESTAMP_MILLIS),
diff --git
a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
index 1e0d86a66..2ad1c6f73 100644
---
a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
+++
b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
@@ -209,6 +209,7 @@ object ExpressionNames {
final val MAKE_YM_INTERVAL = "make_ym_interval"
final val TO_UTC_TIMESTAMP = "to_utc_timestamp"
final val FROM_UTC_TIMESTAMP = "from_utc_timestamp"
+ final val UNIX_SECONDS = "unix_seconds"
final val UNIX_MILLIS = "unix_millis"
final val UNIX_MICROS = "unix_micros"
final val TIMESTAMP_MILLIS = "timestamp_millis"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
