This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new dc853f6af [VL] Update to_utc_timestamp and from_utc_timestamp tests
(#5358)
dc853f6af is described below
commit dc853f6af7ed569dce16d1f44f374642755c6207
Author: Ankita Victor <[email protected]>
AuthorDate: Fri Jun 7 15:23:24 2024 +0530
[VL] Update to_utc_timestamp and from_utc_timestamp tests (#5358)
---
.../gluten/utils/velox/VeloxTestSettings.scala | 8 ---
.../spark/sql/GlutenDateFunctionsSuite.scala | 76 ----------------------
.../gluten/utils/velox/VeloxTestSettings.scala | 8 ---
.../spark/sql/GlutenDateFunctionsSuite.scala | 76 ----------------------
.../gluten/utils/velox/VeloxTestSettings.scala | 8 ---
.../spark/sql/GlutenDateFunctionsSuite.scala | 76 ----------------------
.../gluten/utils/velox/VeloxTestSettings.scala | 8 ---
.../spark/sql/GlutenDateFunctionsSuite.scala | 76 ----------------------
8 files changed, 336 deletions(-)
diff --git
a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 5762855c2..5df53953e 100644
---
a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -265,14 +265,6 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("to_timestamp")
// Legacy mode is not supported, assuming this mode is not commonly used.
.exclude("SPARK-30668: use legacy timestamp parser in to_timestamp")
- // Replaced by another test.
- .exclude("to_utc_timestamp with literal zone")
- // Replaced by another test.
- .exclude("to_utc_timestamp with column zone")
- // Replaced by another test
- .exclude("from_utc_timestamp with literal zone")
- // Replaced by another test
- .exclude("from_utc_timestamp with column zone")
enableSuite[GlutenDataFrameFunctionsSuite]
// blocked by Velox-5768
.exclude("aggregate function - array for primitive type containing null")
diff --git
a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
index 154e67ae3..8d1f7320d 100644
---
a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
+++
b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
@@ -248,80 +248,4 @@ class GlutenDateFunctionsSuite extends DateFunctionsSuite
with GlutenSQLTestsTra
}
}
}
-
- testGluten("to_utc_timestamp with literal zone") {
- val df = Seq(
- (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00"),
- (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00")
- ).toDF("a", "b")
- checkAnswer(
- df.select(to_utc_timestamp(col("a"), "America/Los_Angeles")),
- Seq(
- Row(Timestamp.valueOf("2015-07-24 07:00:00")),
- Row(Timestamp.valueOf("2015-07-25 07:00:00")))
- )
- checkAnswer(
- df.select(to_utc_timestamp(col("b"), "America/Los_Angeles")),
- Seq(
- Row(Timestamp.valueOf("2015-07-24 07:00:00")),
- Row(Timestamp.valueOf("2015-07-25 07:00:00")))
- )
- }
-
- testGluten("to_utc_timestamp with column zone") {
- val df = Seq(
- (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00",
"America/Los_Angeles"),
- (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00",
"Europe/Paris")
- ).toDF("a", "b", "c")
- checkAnswer(
- df.select(to_utc_timestamp(col("a"), col("c"))),
- Seq(
- Row(Timestamp.valueOf("2015-07-24 07:00:00")),
- Row(Timestamp.valueOf("2015-07-24 22:00:00")))
- )
- checkAnswer(
- df.select(to_utc_timestamp(col("b"), col("c"))),
- Seq(
- Row(Timestamp.valueOf("2015-07-24 07:00:00")),
- Row(Timestamp.valueOf("2015-07-24 22:00:00")))
- )
- }
-
- testGluten("from_utc_timestamp with literal zone") {
- val df = Seq(
- (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00"),
- (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00")
- ).toDF("a", "b")
- checkAnswer(
- df.select(from_utc_timestamp(col("a"), "America/Los_Angeles")),
- Seq(
- Row(Timestamp.valueOf("2015-07-23 17:00:00")),
- Row(Timestamp.valueOf("2015-07-24 17:00:00")))
- )
- checkAnswer(
- df.select(from_utc_timestamp(col("b"), "America/Los_Angeles")),
- Seq(
- Row(Timestamp.valueOf("2015-07-23 17:00:00")),
- Row(Timestamp.valueOf("2015-07-24 17:00:00")))
- )
- }
-
- testGluten("from_utc_timestamp with column zone") {
- val df = Seq(
- (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00",
"Europe/Paris"),
- (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00",
"America/Los_Angeles")
- ).toDF("a", "b", "c")
- checkAnswer(
- df.select(from_utc_timestamp(col("a"), col("c"))),
- Seq(
- Row(Timestamp.valueOf("2015-07-24 02:00:00")),
- Row(Timestamp.valueOf("2015-07-24 17:00:00")))
- )
- checkAnswer(
- df.select(from_utc_timestamp(col("b"), col("c"))),
- Seq(
- Row(Timestamp.valueOf("2015-07-24 02:00:00")),
- Row(Timestamp.valueOf("2015-07-24 17:00:00")))
- )
- }
}
diff --git
a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 898fc2b39..79357e9e2 100644
---
a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -1070,14 +1070,6 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("to_timestamp")
// Legacy mode is not supported, assuming this mode is not commonly used.
.exclude("SPARK-30668: use legacy timestamp parser in to_timestamp")
- // Replaced by another test.
- .exclude("to_utc_timestamp with literal zone")
- // Replaced by another test.
- .exclude("to_utc_timestamp with column zone")
- // Replaced by another test
- .exclude("from_utc_timestamp with literal zone")
- // Replaced by another test
- .exclude("from_utc_timestamp with column zone")
enableSuite[GlutenDeprecatedAPISuite]
enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOff]
enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOn]
diff --git
a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
index fbbfdf45d..a946e6de4 100644
---
a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
+++
b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
@@ -246,80 +246,4 @@ class GlutenDateFunctionsSuite extends DateFunctionsSuite
with GlutenSQLTestsTra
}
}
}
-
- testGluten("to_utc_timestamp with literal zone") {
- val df = Seq(
- (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00"),
- (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00")
- ).toDF("a", "b")
- checkAnswer(
- df.select(to_utc_timestamp(col("a"), "America/Los_Angeles")),
- Seq(
- Row(Timestamp.valueOf("2015-07-24 07:00:00")),
- Row(Timestamp.valueOf("2015-07-25 07:00:00")))
- )
- checkAnswer(
- df.select(to_utc_timestamp(col("b"), "America/Los_Angeles")),
- Seq(
- Row(Timestamp.valueOf("2015-07-24 07:00:00")),
- Row(Timestamp.valueOf("2015-07-25 07:00:00")))
- )
- }
-
- testGluten("to_utc_timestamp with column zone") {
- val df = Seq(
- (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00",
"America/Los_Angeles"),
- (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00",
"Europe/Paris")
- ).toDF("a", "b", "c")
- checkAnswer(
- df.select(to_utc_timestamp(col("a"), col("c"))),
- Seq(
- Row(Timestamp.valueOf("2015-07-24 07:00:00")),
- Row(Timestamp.valueOf("2015-07-24 22:00:00")))
- )
- checkAnswer(
- df.select(to_utc_timestamp(col("b"), col("c"))),
- Seq(
- Row(Timestamp.valueOf("2015-07-24 07:00:00")),
- Row(Timestamp.valueOf("2015-07-24 22:00:00")))
- )
- }
-
- testGluten("from_utc_timestamp with literal zone") {
- val df = Seq(
- (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00"),
- (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00")
- ).toDF("a", "b")
- checkAnswer(
- df.select(from_utc_timestamp(col("a"), "America/Los_Angeles")),
- Seq(
- Row(Timestamp.valueOf("2015-07-23 17:00:00")),
- Row(Timestamp.valueOf("2015-07-24 17:00:00")))
- )
- checkAnswer(
- df.select(from_utc_timestamp(col("b"), "America/Los_Angeles")),
- Seq(
- Row(Timestamp.valueOf("2015-07-23 17:00:00")),
- Row(Timestamp.valueOf("2015-07-24 17:00:00")))
- )
- }
-
- testGluten("from_utc_timestamp with column zone") {
- val df = Seq(
- (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00",
"Europe/Paris"),
- (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00",
"America/Los_Angeles")
- ).toDF("a", "b", "c")
- checkAnswer(
- df.select(from_utc_timestamp(col("a"), col("c"))),
- Seq(
- Row(Timestamp.valueOf("2015-07-24 02:00:00")),
- Row(Timestamp.valueOf("2015-07-24 17:00:00")))
- )
- checkAnswer(
- df.select(from_utc_timestamp(col("b"), col("c"))),
- Seq(
- Row(Timestamp.valueOf("2015-07-24 02:00:00")),
- Row(Timestamp.valueOf("2015-07-24 17:00:00")))
- )
- }
}
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index bd437bbe8..c532355df 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -1073,14 +1073,6 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("to_timestamp")
// Legacy mode is not supported, assuming this mode is not commonly used.
.exclude("SPARK-30668: use legacy timestamp parser in to_timestamp")
- // Replaced by another test.
- .exclude("to_utc_timestamp with literal zone")
- // Replaced by another test.
- .exclude("to_utc_timestamp with column zone")
- // Replaced by another test
- .exclude("from_utc_timestamp with literal zone")
- // Replaced by another test
- .exclude("from_utc_timestamp with column zone")
enableSuite[GlutenDeprecatedAPISuite]
enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOff]
enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOn]
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
index fbbfdf45d..a946e6de4 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
@@ -246,80 +246,4 @@ class GlutenDateFunctionsSuite extends DateFunctionsSuite
with GlutenSQLTestsTra
}
}
}
-
- testGluten("to_utc_timestamp with literal zone") {
- val df = Seq(
- (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00"),
- (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00")
- ).toDF("a", "b")
- checkAnswer(
- df.select(to_utc_timestamp(col("a"), "America/Los_Angeles")),
- Seq(
- Row(Timestamp.valueOf("2015-07-24 07:00:00")),
- Row(Timestamp.valueOf("2015-07-25 07:00:00")))
- )
- checkAnswer(
- df.select(to_utc_timestamp(col("b"), "America/Los_Angeles")),
- Seq(
- Row(Timestamp.valueOf("2015-07-24 07:00:00")),
- Row(Timestamp.valueOf("2015-07-25 07:00:00")))
- )
- }
-
- testGluten("to_utc_timestamp with column zone") {
- val df = Seq(
- (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00",
"America/Los_Angeles"),
- (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00",
"Europe/Paris")
- ).toDF("a", "b", "c")
- checkAnswer(
- df.select(to_utc_timestamp(col("a"), col("c"))),
- Seq(
- Row(Timestamp.valueOf("2015-07-24 07:00:00")),
- Row(Timestamp.valueOf("2015-07-24 22:00:00")))
- )
- checkAnswer(
- df.select(to_utc_timestamp(col("b"), col("c"))),
- Seq(
- Row(Timestamp.valueOf("2015-07-24 07:00:00")),
- Row(Timestamp.valueOf("2015-07-24 22:00:00")))
- )
- }
-
- testGluten("from_utc_timestamp with literal zone") {
- val df = Seq(
- (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00"),
- (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00")
- ).toDF("a", "b")
- checkAnswer(
- df.select(from_utc_timestamp(col("a"), "America/Los_Angeles")),
- Seq(
- Row(Timestamp.valueOf("2015-07-23 17:00:00")),
- Row(Timestamp.valueOf("2015-07-24 17:00:00")))
- )
- checkAnswer(
- df.select(from_utc_timestamp(col("b"), "America/Los_Angeles")),
- Seq(
- Row(Timestamp.valueOf("2015-07-23 17:00:00")),
- Row(Timestamp.valueOf("2015-07-24 17:00:00")))
- )
- }
-
- testGluten("from_utc_timestamp with column zone") {
- val df = Seq(
- (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00",
"Europe/Paris"),
- (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00",
"America/Los_Angeles")
- ).toDF("a", "b", "c")
- checkAnswer(
- df.select(from_utc_timestamp(col("a"), col("c"))),
- Seq(
- Row(Timestamp.valueOf("2015-07-24 02:00:00")),
- Row(Timestamp.valueOf("2015-07-24 17:00:00")))
- )
- checkAnswer(
- df.select(from_utc_timestamp(col("b"), col("c"))),
- Seq(
- Row(Timestamp.valueOf("2015-07-24 02:00:00")),
- Row(Timestamp.valueOf("2015-07-24 17:00:00")))
- )
- }
}
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index af8d0dead..104c22dbe 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -1090,14 +1090,6 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("to_timestamp")
// Legacy mode is not supported, assuming this mode is not commonly used.
.exclude("SPARK-30668: use legacy timestamp parser in to_timestamp")
- // Replaced by another test.
- .exclude("to_utc_timestamp with literal zone")
- // Replaced by another test.
- .exclude("to_utc_timestamp with column zone")
- // Replaced by another test
- .exclude("from_utc_timestamp with literal zone")
- // Replaced by another test
- .exclude("from_utc_timestamp with column zone")
enableSuite[GlutenDeprecatedAPISuite]
enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOff]
enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOn]
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
index fbbfdf45d..a946e6de4 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
@@ -246,80 +246,4 @@ class GlutenDateFunctionsSuite extends DateFunctionsSuite
with GlutenSQLTestsTra
}
}
}
-
- testGluten("to_utc_timestamp with literal zone") {
- val df = Seq(
- (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00"),
- (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00")
- ).toDF("a", "b")
- checkAnswer(
- df.select(to_utc_timestamp(col("a"), "America/Los_Angeles")),
- Seq(
- Row(Timestamp.valueOf("2015-07-24 07:00:00")),
- Row(Timestamp.valueOf("2015-07-25 07:00:00")))
- )
- checkAnswer(
- df.select(to_utc_timestamp(col("b"), "America/Los_Angeles")),
- Seq(
- Row(Timestamp.valueOf("2015-07-24 07:00:00")),
- Row(Timestamp.valueOf("2015-07-25 07:00:00")))
- )
- }
-
- testGluten("to_utc_timestamp with column zone") {
- val df = Seq(
- (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00",
"America/Los_Angeles"),
- (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00",
"Europe/Paris")
- ).toDF("a", "b", "c")
- checkAnswer(
- df.select(to_utc_timestamp(col("a"), col("c"))),
- Seq(
- Row(Timestamp.valueOf("2015-07-24 07:00:00")),
- Row(Timestamp.valueOf("2015-07-24 22:00:00")))
- )
- checkAnswer(
- df.select(to_utc_timestamp(col("b"), col("c"))),
- Seq(
- Row(Timestamp.valueOf("2015-07-24 07:00:00")),
- Row(Timestamp.valueOf("2015-07-24 22:00:00")))
- )
- }
-
- testGluten("from_utc_timestamp with literal zone") {
- val df = Seq(
- (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00"),
- (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00")
- ).toDF("a", "b")
- checkAnswer(
- df.select(from_utc_timestamp(col("a"), "America/Los_Angeles")),
- Seq(
- Row(Timestamp.valueOf("2015-07-23 17:00:00")),
- Row(Timestamp.valueOf("2015-07-24 17:00:00")))
- )
- checkAnswer(
- df.select(from_utc_timestamp(col("b"), "America/Los_Angeles")),
- Seq(
- Row(Timestamp.valueOf("2015-07-23 17:00:00")),
- Row(Timestamp.valueOf("2015-07-24 17:00:00")))
- )
- }
-
- testGluten("from_utc_timestamp with column zone") {
- val df = Seq(
- (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00",
"Europe/Paris"),
- (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00",
"America/Los_Angeles")
- ).toDF("a", "b", "c")
- checkAnswer(
- df.select(from_utc_timestamp(col("a"), col("c"))),
- Seq(
- Row(Timestamp.valueOf("2015-07-24 02:00:00")),
- Row(Timestamp.valueOf("2015-07-24 17:00:00")))
- )
- checkAnswer(
- df.select(from_utc_timestamp(col("b"), col("c"))),
- Seq(
- Row(Timestamp.valueOf("2015-07-24 02:00:00")),
- Row(Timestamp.valueOf("2015-07-24 17:00:00")))
- )
- }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
