This is an automated email from the ASF dual-hosted git repository.
zhli pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new b84c6d01d [VL] enable
unix_millis,unix_micros,timestamp_millis,timestamp_micros (#5601)
b84c6d01d is described below
commit b84c6d01d52621052f573e27057f490dee356593
Author: Zhen Li <[email protected]>
AuthorDate: Mon May 6 11:02:38 2024 +0800
[VL] enable unix_millis,unix_micros,timestamp_millis,timestamp_micros
(#5601)
[VL] Enable unix_millis,unix_micros,timestamp_millis,timestamp_micros
functions
---
.../org/apache/gluten/utils/CHExpressionUtil.scala | 6 +++-
.../execution/ScalarFunctionsValidateSuite.scala | 42 ++++++++++++++++++++++
docs/velox-backend-support-progress.md | 6 ++--
.../gluten/expression/ExpressionMappings.scala | 4 +++
.../apache/gluten/expression/ExpressionNames.scala | 4 +++
5 files changed, 59 insertions(+), 3 deletions(-)
diff --git
a/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala
b/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala
index 3663ef07a..540af65ac 100644
---
a/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala
+++
b/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala
@@ -190,6 +190,10 @@ object CHExpressionUtil {
REGR_INTERCEPT -> DefaultValidator(),
REGR_SXY -> DefaultValidator(),
TO_UTC_TIMESTAMP -> DefaultValidator(),
- FROM_UTC_TIMESTAMP -> DefaultValidator()
+ FROM_UTC_TIMESTAMP -> DefaultValidator(),
+ UNIX_MILLIS -> DefaultValidator(),
+ UNIX_MICROS -> DefaultValidator(),
+ TIMESTAMP_MILLIS -> DefaultValidator(),
+ TIMESTAMP_MICROS -> DefaultValidator()
)
}
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
index 84f580b0b..d4ef39412 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
@@ -826,4 +826,46 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateTest {
}
}
+ test("unix_millis") {
+ withTempPath {
+ path =>
+ val t1 = Timestamp.valueOf("2015-07-22 10:00:00.012")
+ val t2 = Timestamp.valueOf("2014-12-31 23:59:59.012")
+ val t3 = Timestamp.valueOf("2014-12-31 23:59:59.001")
+ Seq(t1, t2, t3).toDF("t").write.parquet(path.getCanonicalPath)
+
+
spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("time")
+ runQueryAndCompare("select unix_millis(t) from time") {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ }
+ }
+
+ test("unix_micros") {
+ withTempPath {
+ path =>
+ val t1 = Timestamp.valueOf("2015-07-22 10:00:00.012")
+ val t2 = Timestamp.valueOf("2014-12-31 23:59:59.012")
+ val t3 = Timestamp.valueOf("2014-12-31 23:59:59.001")
+ Seq(t1, t2, t3).toDF("t").write.parquet(path.getCanonicalPath)
+
+
spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("time")
+ runQueryAndCompare("select unix_micros(t) from time") {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ }
+ }
+
+ test("timestamp_millis") {
+ runQueryAndCompare("select timestamp_millis(l_orderkey) from lineitem") {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ }
+
+ test("timestamp_micros") {
+ runQueryAndCompare("select timestamp_micros(l_orderkey) from lineitem") {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ }
+
}
diff --git a/docs/velox-backend-support-progress.md
b/docs/velox-backend-support-progress.md
index 78d122147..dcd731961 100644
--- a/docs/velox-backend-support-progress.md
+++ b/docs/velox-backend-support-progress.md
@@ -338,8 +338,8 @@ Gluten supports 199 functions. (Drag to right to see all
data types)
| second | second |
| S | | | | | | |
| | | | | | | | |
| | | |
| session_window | |
| | | | | | | |
| | | | | | | | |
| | | |
| timestamp | |
| | | | | | | |
| | | | | | | | |
| | | |
-| timestamp_micros | |
| | | | | | | |
| | | | | | | | |
| | | |
-| timestamp_millis | |
| | | | | | | |
| | | | | | | | |
| | | |
+| timestamp_micros | | timestamp_micros
| S | | | |
| | | | | | | | | |
| | | | | |
+| timestamp_millis | |
timestamp_millis | S | | |
| | | | | | | | |
| | | | | | | |
| timestamp_seconds | |
| | | | | | | |
| | | | | | | | |
| | | |
| to_date | |
| S | | | | | | |
| | S | S | | | | | |
| | | |
| to_timestamp | |
| | | | | | | |
| | | | | | | | |
| | | |
@@ -347,6 +347,8 @@ Gluten supports 199 functions. (Drag to right to see all
data types)
| to_utc_timestamp | |
| | | | | | | |
| | | | | | | | |
| | | |
| trunc | |
| | | | | | | |
| | | | | | | | |
| | | |
| unix_timestamp | | unix_timestamp
| | | | | | | |
| | | | | | | | |
| | | |
+| unix_millis | | unix_millis | S
| | | | | | |
| | | | | | | | |
| | | |
+| unix_micros | | unix_micros |
S | | | | | | |
| | | | | | | | |
| | | |
| weekday | |
| S | | | | | | |
| | S | | | | | | |
| | | |
| weekofyear | week,week_of_year |
| S | | | | | | |
| | | | | | | | |
| | | |
| window | |
| | | | | | | |
| | | | | | | | |
| | | |
diff --git
a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala
b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala
index 6be5b0f9b..7b4d6cd15 100644
---
a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala
+++
b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala
@@ -189,6 +189,10 @@ object ExpressionMappings {
Sig[MakeYMInterval](MAKE_YM_INTERVAL),
Sig[ToUTCTimestamp](TO_UTC_TIMESTAMP),
Sig[FromUTCTimestamp](FROM_UTC_TIMESTAMP),
+ Sig[UnixMillis](UNIX_MILLIS),
+ Sig[UnixMicros](UNIX_MICROS),
+ Sig[MillisToTimestamp](TIMESTAMP_MILLIS),
+ Sig[MicrosToTimestamp](TIMESTAMP_MICROS),
// JSON functions
Sig[GetJsonObject](GET_JSON_OBJECT),
Sig[LengthOfJsonArray](JSON_ARRAY_LENGTH),
diff --git
a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
index 44384be72..7d516ecd0 100644
---
a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
+++
b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
@@ -209,6 +209,10 @@ object ExpressionNames {
final val MAKE_YM_INTERVAL = "make_ym_interval"
final val TO_UTC_TIMESTAMP = "to_utc_timestamp"
final val FROM_UTC_TIMESTAMP = "from_utc_timestamp"
+ final val UNIX_MILLIS = "unix_millis"
+ final val UNIX_MICROS = "unix_micros"
+ final val TIMESTAMP_MILLIS = "timestamp_millis"
+ final val TIMESTAMP_MICROS = "timestamp_micros"
// JSON functions
final val GET_JSON_OBJECT = "get_json_object"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
