This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new ce23b1445 [VL] Enable from_utc_timestamp Spark function (#5140)
ce23b1445 is described below
commit ce23b1445a8d67c545bd01c1a70e5452aaee6d4f
Author: Ankita Victor <[email protected]>
AuthorDate: Tue Apr 2 13:46:24 2024 +0530
[VL] Enable from_utc_timestamp Spark function (#5140)
---
.../org/apache/gluten/utils/CHExpressionUtil.scala | 3 +-
.../execution/ScalarFunctionsValidateSuite.scala | 21 ++++++++++++
docs/velox-backend-support-progress.md | 2 +-
.../gluten/expression/ExpressionMappings.scala | 1 +
.../utils/clickhouse/ClickHouseTestSettings.scala | 2 ++
.../gluten/utils/velox/VeloxTestSettings.scala | 4 +++
.../spark/sql/GlutenDateFunctionsSuite.scala | 38 ++++++++++++++++++++++
.../utils/clickhouse/ClickHouseTestSettings.scala | 2 ++
.../gluten/utils/velox/VeloxTestSettings.scala | 4 +++
.../spark/sql/GlutenDateFunctionsSuite.scala | 38 ++++++++++++++++++++++
.../utils/clickhouse/ClickHouseTestSettings.scala | 2 ++
.../gluten/utils/velox/VeloxTestSettings.scala | 4 +++
.../spark/sql/GlutenDateFunctionsSuite.scala | 38 ++++++++++++++++++++++
.../apache/gluten/expression/ExpressionNames.scala | 1 +
14 files changed, 158 insertions(+), 2 deletions(-)
diff --git
a/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala
b/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala
index 70aa18980..f630bc0e5 100644
---
a/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala
+++
b/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala
@@ -184,6 +184,7 @@ object CHExpressionUtil {
MAKE_YM_INTERVAL -> DefaultValidator(),
KURTOSIS -> DefaultValidator(),
REGR_R2 -> DefaultValidator(),
- TO_UTC_TIMESTAMP -> DefaultValidator()
+ TO_UTC_TIMESTAMP -> DefaultValidator(),
+ FROM_UTC_TIMESTAMP -> DefaultValidator()
)
}
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
index e026f7142..220a9d6c5 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
@@ -395,6 +395,27 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateTest {
}
}
+ test("from_utc_timestamp") {
+ withTempPath {
+ path =>
+ Seq(
+ (Timestamp.valueOf("2015-07-24 00:00:00"), "America/Los_Angeles"),
+ (Timestamp.valueOf("2015-07-25 00:00:00"), "America/Los_Angeles")
+ ).toDF("a", "b")
+ .write
+ .parquet(path.getCanonicalPath)
+
+
spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("view")
+
+ runQueryAndCompare("SELECT from_utc_timestamp(a,
\"America/Los_Angeles\") from view") {
+ checkOperatorMatch[ProjectExecTransformer]
+ }
+ runQueryAndCompare("SELECT from_utc_timestamp(a, b) from view") {
+ checkOperatorMatch[ProjectExecTransformer]
+ }
+ }
+ }
+
test("map extract - getmapvalue") {
withTempPath {
path =>
diff --git a/docs/velox-backend-support-progress.md
b/docs/velox-backend-support-progress.md
index 4b14c7df9..0f22bc20e 100644
--- a/docs/velox-backend-support-progress.md
+++ b/docs/velox-backend-support-progress.md
@@ -320,7 +320,7 @@ Gluten supports 199 functions. (Drag to right to see all
data types)
| dayofyear | day_of_year,doy |
| S | | | | | | |
| | S | S | | | | | |
| | | |
| extract | |
| | | | | | | |
| | S | S | | | | | |
| | | |
| from_unixtime | from_unixtime |
| S | | | | | | S |
| | | | | | | | |
| | | |
-| from_utc_timestamp | |
| | | | | | | |
| | | | | | | | |
| | | |
+| from_utc_timestamp | | from_utc_timestamp
| S | | | | | | |
| | | | | | | | |
| | | |
| hour | hour |
| S | | | | | | |
| | | S | | | | | |
| | | |
| last_day | | last_day
| | | | | | | |
| | | | | | | | |
| | | |
| make_date | | make_date
| S | | | | | | |
| | | | | | | | |
| | | |
diff --git
a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala
b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala
index 1bb6c5d16..ae080efed 100644
---
a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala
+++
b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala
@@ -182,6 +182,7 @@ object ExpressionMappings {
Sig[MakeTimestamp](MAKE_TIMESTAMP),
Sig[MakeYMInterval](MAKE_YM_INTERVAL),
Sig[ToUTCTimestamp](TO_UTC_TIMESTAMP),
+ Sig[FromUTCTimestamp](FROM_UTC_TIMESTAMP),
// JSON functions
Sig[GetJsonObject](GET_JSON_OBJECT),
Sig[LengthOfJsonArray](JSON_ARRAY_LENGTH),
diff --git
a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
index 4c1f8e65e..be67c244e 100644
---
a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
+++
b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
@@ -297,6 +297,8 @@ class ClickHouseTestSettings extends BackendTestSettings {
.excludeGlutenTest("to_unix_timestamp")
.exclude("to_utc_timestamp with literal zone")
.exclude("to_utc_timestamp with column zone")
+ .exclude("from_utc_timestamp with literal zone")
+ .exclude("from_utc_timestamp with column zone")
enableSuite[GlutenDeprecatedAPISuite]
enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOff].excludeGlutenTest(
"SPARK-32659: Fix the data issue when pruning DPP on non-atomic type")
diff --git
a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 924554aea..ebe7f27bd 100644
---
a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -265,6 +265,10 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("to_utc_timestamp with literal zone")
// Replaced by another test.
.exclude("to_utc_timestamp with column zone")
+ // Replaced by another test
+ .exclude("from_utc_timestamp with literal zone")
+ // Replaced by another test
+ .exclude("from_utc_timestamp with column zone")
enableSuite[GlutenDataFrameFunctionsSuite]
// blocked by Velox-5768
.exclude("aggregate function - array for primitive type containing null")
diff --git
a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
index b1d1cd43c..154e67ae3 100644
---
a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
+++
b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
@@ -286,4 +286,42 @@ class GlutenDateFunctionsSuite extends DateFunctionsSuite
with GlutenSQLTestsTra
Row(Timestamp.valueOf("2015-07-24 22:00:00")))
)
}
+
+ testGluten("from_utc_timestamp with literal zone") {
+ val df = Seq(
+ (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00"),
+ (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00")
+ ).toDF("a", "b")
+ checkAnswer(
+ df.select(from_utc_timestamp(col("a"), "America/Los_Angeles")),
+ Seq(
+ Row(Timestamp.valueOf("2015-07-23 17:00:00")),
+ Row(Timestamp.valueOf("2015-07-24 17:00:00")))
+ )
+ checkAnswer(
+ df.select(from_utc_timestamp(col("b"), "America/Los_Angeles")),
+ Seq(
+ Row(Timestamp.valueOf("2015-07-23 17:00:00")),
+ Row(Timestamp.valueOf("2015-07-24 17:00:00")))
+ )
+ }
+
+ testGluten("from_utc_timestamp with column zone") {
+ val df = Seq(
+ (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00",
"Europe/Paris"),
+ (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00",
"America/Los_Angeles")
+ ).toDF("a", "b", "c")
+ checkAnswer(
+ df.select(from_utc_timestamp(col("a"), col("c"))),
+ Seq(
+ Row(Timestamp.valueOf("2015-07-24 02:00:00")),
+ Row(Timestamp.valueOf("2015-07-24 17:00:00")))
+ )
+ checkAnswer(
+ df.select(from_utc_timestamp(col("b"), col("c"))),
+ Seq(
+ Row(Timestamp.valueOf("2015-07-24 02:00:00")),
+ Row(Timestamp.valueOf("2015-07-24 17:00:00")))
+ )
+ }
}
diff --git
a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
index be6273857..374474f87 100644
---
a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
+++
b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
@@ -323,6 +323,8 @@ class ClickHouseTestSettings extends BackendTestSettings {
.excludeGlutenTest("to_unix_timestamp")
.exclude("to_utc_timestamp with literal zone")
.exclude("to_utc_timestamp with column zone")
+ .exclude("from_utc_timestamp with literal zone")
+ .exclude("from_utc_timestamp with column zone")
enableSuite[GlutenDeprecatedAPISuite]
enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOff].excludeGlutenTest(
"SPARK-32659: Fix the data issue when pruning DPP on non-atomic type")
diff --git
a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index c95ece47f..2abf3e1b8 100644
---
a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -1028,6 +1028,10 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("to_utc_timestamp with literal zone")
// Replaced by another test.
.exclude("to_utc_timestamp with column zone")
+ // Replaced by another test
+ .exclude("from_utc_timestamp with literal zone")
+ // Replaced by another test
+ .exclude("from_utc_timestamp with column zone")
enableSuite[GlutenDeprecatedAPISuite]
enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOff]
enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOn]
diff --git
a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
index 8fdb69229..fbbfdf45d 100644
---
a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
+++
b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
@@ -284,4 +284,42 @@ class GlutenDateFunctionsSuite extends DateFunctionsSuite
with GlutenSQLTestsTra
Row(Timestamp.valueOf("2015-07-24 22:00:00")))
)
}
+
+ testGluten("from_utc_timestamp with literal zone") {
+ val df = Seq(
+ (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00"),
+ (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00")
+ ).toDF("a", "b")
+ checkAnswer(
+ df.select(from_utc_timestamp(col("a"), "America/Los_Angeles")),
+ Seq(
+ Row(Timestamp.valueOf("2015-07-23 17:00:00")),
+ Row(Timestamp.valueOf("2015-07-24 17:00:00")))
+ )
+ checkAnswer(
+ df.select(from_utc_timestamp(col("b"), "America/Los_Angeles")),
+ Seq(
+ Row(Timestamp.valueOf("2015-07-23 17:00:00")),
+ Row(Timestamp.valueOf("2015-07-24 17:00:00")))
+ )
+ }
+
+ testGluten("from_utc_timestamp with column zone") {
+ val df = Seq(
+ (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00",
"Europe/Paris"),
+ (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00",
"America/Los_Angeles")
+ ).toDF("a", "b", "c")
+ checkAnswer(
+ df.select(from_utc_timestamp(col("a"), col("c"))),
+ Seq(
+ Row(Timestamp.valueOf("2015-07-24 02:00:00")),
+ Row(Timestamp.valueOf("2015-07-24 17:00:00")))
+ )
+ checkAnswer(
+ df.select(from_utc_timestamp(col("b"), col("c"))),
+ Seq(
+ Row(Timestamp.valueOf("2015-07-24 02:00:00")),
+ Row(Timestamp.valueOf("2015-07-24 17:00:00")))
+ )
+ }
}
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
index 370df059c..d90722301 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
@@ -321,6 +321,8 @@ class ClickHouseTestSettings extends BackendTestSettings {
.excludeGlutenTest("to_unix_timestamp")
.exclude("to_utc_timestamp with literal zone")
.exclude("to_utc_timestamp with column zone")
+ .exclude("from_utc_timestamp with literal zone")
+ .exclude("from_utc_timestamp with column zone")
enableSuite[GlutenDeprecatedAPISuite]
enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOff].excludeGlutenTest(
"SPARK-32659: Fix the data issue when pruning DPP on non-atomic type")
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index f581bb2da..c63bd12cf 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -1032,6 +1032,10 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("to_utc_timestamp with literal zone")
// Replaced by another test.
.exclude("to_utc_timestamp with column zone")
+ // Replaced by another test
+ .exclude("from_utc_timestamp with literal zone")
+ // Replaced by another test
+ .exclude("from_utc_timestamp with column zone")
enableSuite[GlutenDeprecatedAPISuite]
enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOff]
enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOn]
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
index 8fdb69229..fbbfdf45d 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
@@ -284,4 +284,42 @@ class GlutenDateFunctionsSuite extends DateFunctionsSuite
with GlutenSQLTestsTra
Row(Timestamp.valueOf("2015-07-24 22:00:00")))
)
}
+
+ testGluten("from_utc_timestamp with literal zone") {
+ val df = Seq(
+ (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00"),
+ (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00")
+ ).toDF("a", "b")
+ checkAnswer(
+ df.select(from_utc_timestamp(col("a"), "America/Los_Angeles")),
+ Seq(
+ Row(Timestamp.valueOf("2015-07-23 17:00:00")),
+ Row(Timestamp.valueOf("2015-07-24 17:00:00")))
+ )
+ checkAnswer(
+ df.select(from_utc_timestamp(col("b"), "America/Los_Angeles")),
+ Seq(
+ Row(Timestamp.valueOf("2015-07-23 17:00:00")),
+ Row(Timestamp.valueOf("2015-07-24 17:00:00")))
+ )
+ }
+
+ testGluten("from_utc_timestamp with column zone") {
+ val df = Seq(
+ (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00",
"Europe/Paris"),
+ (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00",
"America/Los_Angeles")
+ ).toDF("a", "b", "c")
+ checkAnswer(
+ df.select(from_utc_timestamp(col("a"), col("c"))),
+ Seq(
+ Row(Timestamp.valueOf("2015-07-24 02:00:00")),
+ Row(Timestamp.valueOf("2015-07-24 17:00:00")))
+ )
+ checkAnswer(
+ df.select(from_utc_timestamp(col("b"), col("c"))),
+ Seq(
+ Row(Timestamp.valueOf("2015-07-24 02:00:00")),
+ Row(Timestamp.valueOf("2015-07-24 17:00:00")))
+ )
+ }
}
diff --git
a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
index 96febfde7..f800e5251 100644
---
a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
+++
b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
@@ -199,6 +199,7 @@ object ExpressionNames {
final val MAKE_TIMESTAMP = "make_timestamp"
final val MAKE_YM_INTERVAL = "make_ym_interval"
final val TO_UTC_TIMESTAMP = "to_utc_timestamp"
+ final val FROM_UTC_TIMESTAMP = "from_utc_timestamp"
// JSON functions
final val GET_JSON_OBJECT = "get_json_object"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
