This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new b4cd0e18ff [VL] Support casting timestamp type to varchar type (#8338)
b4cd0e18ff is described below
commit b4cd0e18ff72eaa335168fb80b18827e8f268202
Author: PHILO-HE <[email protected]>
AuthorDate: Mon Jan 6 14:57:53 2025 +0800
[VL] Support casting timestamp type to varchar type (#8338)
---
.../substrait/SubstraitToVeloxPlanValidator.cc | 4 +-
.../utils/clickhouse/ClickHouseTestSettings.scala | 1 +
.../gluten/utils/velox/VeloxTestSettings.scala | 2 +
.../sql/catalyst/expressions/GlutenCastSuite.scala | 80 +++++++++++++++++++-
.../utils/clickhouse/ClickHouseTestSettings.scala | 1 +
.../gluten/utils/velox/VeloxTestSettings.scala | 2 +
.../sql/catalyst/expressions/GlutenCastSuite.scala | 80 +++++++++++++++++++-
.../utils/clickhouse/ClickHouseTestSettings.scala | 1 +
.../gluten/utils/velox/VeloxTestSettings.scala | 4 +
.../sql/catalyst/expressions/GlutenCastSuite.scala | 80 +++++++++++++++++++-
.../catalyst/expressions/GlutenTryCastSuite.scala | 83 ++++++++++++++++++++-
.../utils/clickhouse/ClickHouseTestSettings.scala | 1 +
.../gluten/utils/velox/VeloxTestSettings.scala | 4 +
.../sql/catalyst/expressions/GlutenCastSuite.scala | 80 +++++++++++++++++++-
.../catalyst/expressions/GlutenTryCastSuite.scala | 85 +++++++++++++++++++++-
15 files changed, 500 insertions(+), 8 deletions(-)
diff --git a/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc
b/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc
index 25e87699fe..cfc8498555 100644
--- a/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc
+++ b/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc
@@ -300,8 +300,8 @@ bool SubstraitToVeloxPlanValidator::validateCast(
LOG_VALIDATION_MSG("Invalid input type in casting:
ARRAY/MAP/ROW/VARBINARY.");
return false;
case TypeKind::TIMESTAMP:
- // Only support cast timestamp to date
- if (!toType->isDate()) {
+ // Only support casting timestamp to date or varchar.
+ if (!toType->isDate() && toType->kind() != TypeKind::VARCHAR) {
LOG_VALIDATION_MSG(
"Casting from TIMESTAMP to " + toType->toString() + " is not
supported or has incorrect result.");
return false;
diff --git
a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
index f43a3977a3..b985347f20 100644
---
a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
+++
b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
@@ -569,6 +569,7 @@ class ClickHouseTestSettings extends BackendTestSettings {
.exclude("cast string to timestamp")
.exclude("cast from boolean")
.exclude("data type casting")
+ .excludeGlutenTest("data type casting")
.exclude("cast between string and interval")
.exclude("SPARK-27671: cast from nested null type in struct")
.exclude("Process Infinity, -Infinity, NaN in case insensitive manner")
diff --git
a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 2c6b882850..baed98729b 100644
---
a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -71,6 +71,8 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("SPARK-35719: cast timestamp with local time zone to timestamp
without timezone")
// User defined type.
.exclude("SPARK-32828: cast from a derived user-defined type to a base
type")
+ // Set timezone through config.
+ .exclude("data type casting")
enableSuite[GlutenAnsiCastSuiteWithAnsiModeOff]
.exclude(
diff --git
a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
index d892a895e8..c87b8ddce6 100644
---
a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
+++
b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
@@ -17,9 +17,13 @@
package org.apache.spark.sql.catalyst.expressions
import org.apache.spark.sql.GlutenTestsTrait
+import
org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{withDefaultTimeZone,
ALL_TIMEZONES, UTC, UTC_OPT}
+import org.apache.spark.sql.catalyst.util.DateTimeUtils.{fromJavaTimestamp,
millisToMicros, TimeZoneUTC}
+import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types._
-import java.sql.Date
+import java.sql.{Date, Timestamp}
+import java.util.Calendar
class GlutenCastSuite extends CastSuite with GlutenTestsTrait {
override def cast(v: Any, targetType: DataType, timeZoneId: Option[String] =
None): CastBase = {
@@ -75,4 +79,78 @@ class GlutenCastSuite extends CastSuite with
GlutenTestsTrait {
val d = Date.valueOf("1970-01-01")
checkEvaluation(cast(d, DateType), d)
}
+
+ testGluten("data type casting") {
+ val sd = "1970-01-01"
+ val d = Date.valueOf(sd)
+ val zts = sd + " 00:00:00"
+ val sts = sd + " 00:00:02"
+ val nts = sts + ".1"
+ val ts = withDefaultTimeZone(UTC)(Timestamp.valueOf(nts))
+
+ for (tz <- ALL_TIMEZONES) {
+ withSQLConf(
+ SQLConf.SESSION_LOCAL_TIMEZONE.key -> tz.getId
+ ) {
+ val timeZoneId = Option(tz.getId)
+ var c = Calendar.getInstance(TimeZoneUTC)
+ c.set(2015, 2, 8, 2, 30, 0)
+ checkEvaluation(
+ cast(
+ cast(new Timestamp(c.getTimeInMillis), StringType, timeZoneId),
+ TimestampType,
+ timeZoneId),
+ millisToMicros(c.getTimeInMillis))
+ c = Calendar.getInstance(TimeZoneUTC)
+ c.set(2015, 10, 1, 2, 30, 0)
+ checkEvaluation(
+ cast(
+ cast(new Timestamp(c.getTimeInMillis), StringType, timeZoneId),
+ TimestampType,
+ timeZoneId),
+ millisToMicros(c.getTimeInMillis))
+ }
+ }
+
+ checkEvaluation(cast("abdef", StringType), "abdef")
+ checkEvaluation(cast("12.65", DecimalType.SYSTEM_DEFAULT), Decimal(12.65))
+
+ checkEvaluation(cast(cast(sd, DateType), StringType), sd)
+ checkEvaluation(cast(cast(d, StringType), DateType), 0)
+
+ withSQLConf(
+ SQLConf.SESSION_LOCAL_TIMEZONE.key -> UTC_OPT.get
+ ) {
+ checkEvaluation(cast(cast(nts, TimestampType, UTC_OPT), StringType,
UTC_OPT), nts)
+ checkEvaluation(
+ cast(cast(ts, StringType, UTC_OPT), TimestampType, UTC_OPT),
+ fromJavaTimestamp(ts))
+
+ // all convert to string type to check
+ checkEvaluation(
+ cast(cast(cast(nts, TimestampType, UTC_OPT), DateType, UTC_OPT),
StringType),
+ sd)
+ checkEvaluation(
+ cast(cast(cast(ts, DateType, UTC_OPT), TimestampType, UTC_OPT),
StringType, UTC_OPT),
+ zts)
+ }
+
+ checkEvaluation(cast(cast("abdef", BinaryType), StringType), "abdef")
+
+ checkEvaluation(
+ cast(
+ cast(cast(cast(cast(cast("5", ByteType), ShortType), IntegerType),
FloatType), DoubleType),
+ LongType),
+ 5.toLong)
+
+ checkEvaluation(cast("23", DoubleType), 23d)
+ checkEvaluation(cast("23", IntegerType), 23)
+ checkEvaluation(cast("23", FloatType), 23f)
+ checkEvaluation(cast("23", DecimalType.USER_DEFAULT), Decimal(23))
+ checkEvaluation(cast("23", ByteType), 23.toByte)
+ checkEvaluation(cast("23", ShortType), 23.toShort)
+ checkEvaluation(cast(123, IntegerType), 123)
+
+ checkEvaluation(cast(Literal.create(null, IntegerType), ShortType), null)
+ }
}
diff --git
a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
index 126749f78c..219eb0d0b9 100644
---
a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
+++
b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
@@ -588,6 +588,7 @@ class ClickHouseTestSettings extends BackendTestSettings {
.exclude("cast string to timestamp")
.exclude("cast from boolean")
.exclude("data type casting")
+ .excludeGlutenTest("data type casting")
.exclude("cast between string and interval")
.exclude("SPARK-27671: cast from nested null type in struct")
.exclude("Process Infinity, -Infinity, NaN in case insensitive manner")
diff --git
a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index f83b91ede1..d1f8b5b0c4 100644
---
a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -112,6 +112,8 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("SPARK-35711: cast timestamp without time zone to timestamp with
local time zone")
// Timezone.
.exclude("SPARK-35719: cast timestamp with local time zone to timestamp
without timezone")
+ // Set timezone through config.
+ .exclude("data type casting")
enableSuite[GlutenCollectionExpressionsSuite]
// Rewrite in Gluten to replace Seq with Array
.exclude("Shuffle")
diff --git
a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
index 70586a6985..f91f6d6be8 100644
---
a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
+++
b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
@@ -17,9 +17,13 @@
package org.apache.spark.sql.catalyst.expressions
import org.apache.spark.sql.GlutenTestsTrait
+import
org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{withDefaultTimeZone,
ALL_TIMEZONES, UTC, UTC_OPT}
+import org.apache.spark.sql.catalyst.util.DateTimeUtils.{fromJavaTimestamp,
millisToMicros, TimeZoneUTC}
+import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types._
-import java.sql.Date
+import java.sql.{Date, Timestamp}
+import java.util.Calendar
class GlutenCastSuite extends CastSuite with GlutenTestsTrait {
override def cast(v: Any, targetType: DataType, timeZoneId: Option[String] =
None): CastBase = {
@@ -74,4 +78,78 @@ class GlutenCastSuite extends CastSuite with
GlutenTestsTrait {
val d = Date.valueOf("1970-01-01")
checkEvaluation(cast(d, DateType), d)
}
+
+ testGluten("data type casting") {
+ val sd = "1970-01-01"
+ val d = Date.valueOf(sd)
+ val zts = sd + " 00:00:00"
+ val sts = sd + " 00:00:02"
+ val nts = sts + ".1"
+ val ts = withDefaultTimeZone(UTC)(Timestamp.valueOf(nts))
+
+ for (tz <- ALL_TIMEZONES) {
+ withSQLConf(
+ SQLConf.SESSION_LOCAL_TIMEZONE.key -> tz.getId
+ ) {
+ val timeZoneId = Option(tz.getId)
+ var c = Calendar.getInstance(TimeZoneUTC)
+ c.set(2015, 2, 8, 2, 30, 0)
+ checkEvaluation(
+ cast(
+ cast(new Timestamp(c.getTimeInMillis), StringType, timeZoneId),
+ TimestampType,
+ timeZoneId),
+ millisToMicros(c.getTimeInMillis))
+ c = Calendar.getInstance(TimeZoneUTC)
+ c.set(2015, 10, 1, 2, 30, 0)
+ checkEvaluation(
+ cast(
+ cast(new Timestamp(c.getTimeInMillis), StringType, timeZoneId),
+ TimestampType,
+ timeZoneId),
+ millisToMicros(c.getTimeInMillis))
+ }
+ }
+
+ checkEvaluation(cast("abdef", StringType), "abdef")
+ checkEvaluation(cast("12.65", DecimalType.SYSTEM_DEFAULT), Decimal(12.65))
+
+ checkEvaluation(cast(cast(sd, DateType), StringType), sd)
+ checkEvaluation(cast(cast(d, StringType), DateType), 0)
+
+ withSQLConf(
+ SQLConf.SESSION_LOCAL_TIMEZONE.key -> UTC_OPT.get
+ ) {
+ checkEvaluation(cast(cast(nts, TimestampType, UTC_OPT), StringType,
UTC_OPT), nts)
+ checkEvaluation(
+ cast(cast(ts, StringType, UTC_OPT), TimestampType, UTC_OPT),
+ fromJavaTimestamp(ts))
+
+ // all convert to string type to check
+ checkEvaluation(
+ cast(cast(cast(nts, TimestampType, UTC_OPT), DateType, UTC_OPT),
StringType),
+ sd)
+ checkEvaluation(
+ cast(cast(cast(ts, DateType, UTC_OPT), TimestampType, UTC_OPT),
StringType, UTC_OPT),
+ zts)
+ }
+
+ checkEvaluation(cast(cast("abdef", BinaryType), StringType), "abdef")
+
+ checkEvaluation(
+ cast(
+ cast(cast(cast(cast(cast("5", ByteType), ShortType), IntegerType),
FloatType), DoubleType),
+ LongType),
+ 5.toLong)
+
+ checkEvaluation(cast("23", DoubleType), 23d)
+ checkEvaluation(cast("23", IntegerType), 23)
+ checkEvaluation(cast("23", FloatType), 23f)
+ checkEvaluation(cast("23", DecimalType.USER_DEFAULT), Decimal(23))
+ checkEvaluation(cast("23", ByteType), 23.toByte)
+ checkEvaluation(cast("23", ShortType), 23.toShort)
+ checkEvaluation(cast(123, IntegerType), 123)
+
+ checkEvaluation(cast(Literal.create(null, IntegerType), ShortType), null)
+ }
}
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
index 829fae1cf5..56c95ae1bd 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
@@ -531,6 +531,7 @@ class ClickHouseTestSettings extends BackendTestSettings {
.exclude("cast string to timestamp")
.exclude("cast from boolean")
.exclude("data type casting")
+ .excludeGlutenTest("data type casting")
.exclude("cast between string and interval")
.exclude("SPARK-27671: cast from nested null type in struct")
.exclude("Process Infinity, -Infinity, NaN in case insensitive manner")
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 498c2a3b4e..1de6961192 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -91,6 +91,8 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude(
"Process Infinity, -Infinity, NaN in case insensitive manner" // +inf
not supported in folly.
)
+ // Set timezone through config.
+ .exclude("data type casting")
enableSuite[GlutenTryCastSuite]
.exclude(
"Process Infinity, -Infinity, NaN in case insensitive manner" // +inf
not supported in folly.
@@ -101,6 +103,8 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("ANSI mode: Throw exception on casting out-of-range value to long
type")
.exclude("cast from invalid string to numeric should throw
NumberFormatException")
.exclude("SPARK-26218: Fix the corner case of codegen when casting float
to Integer")
+ // Set timezone through config.
+ .exclude("data type casting")
enableSuite[GlutenCollectionExpressionsSuite]
// Rewrite in Gluten to replace Seq with Array
.exclude("Shuffle")
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
index f2a83bf234..a40bc6abad 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
@@ -17,9 +17,13 @@
package org.apache.spark.sql.catalyst.expressions
import org.apache.spark.sql.GlutenTestsTrait
+import
org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{withDefaultTimeZone,
ALL_TIMEZONES, UTC, UTC_OPT}
+import org.apache.spark.sql.catalyst.util.DateTimeUtils.{fromJavaTimestamp,
millisToMicros, TimeZoneUTC}
+import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types._
-import java.sql.Date
+import java.sql.{Date, Timestamp}
+import java.util.Calendar
class GlutenCastSuite extends CastSuiteBase with GlutenTestsTrait {
override def cast(v: Any, targetType: DataType, timeZoneId: Option[String] =
None): Cast = {
@@ -72,5 +76,79 @@ class GlutenCastSuite extends CastSuiteBase with
GlutenTestsTrait {
checkEvaluation(cast(d, DateType), d)
}
+ testGluten("data type casting") {
+ val sd = "1970-01-01"
+ val d = Date.valueOf(sd)
+ val zts = sd + " 00:00:00"
+ val sts = sd + " 00:00:02"
+ val nts = sts + ".1"
+ val ts = withDefaultTimeZone(UTC)(Timestamp.valueOf(nts))
+
+ for (tz <- ALL_TIMEZONES) {
+ withSQLConf(
+ SQLConf.SESSION_LOCAL_TIMEZONE.key -> tz.getId
+ ) {
+ val timeZoneId = Option(tz.getId)
+ var c = Calendar.getInstance(TimeZoneUTC)
+ c.set(2015, 2, 8, 2, 30, 0)
+ checkEvaluation(
+ cast(
+ cast(new Timestamp(c.getTimeInMillis), StringType, timeZoneId),
+ TimestampType,
+ timeZoneId),
+ millisToMicros(c.getTimeInMillis))
+ c = Calendar.getInstance(TimeZoneUTC)
+ c.set(2015, 10, 1, 2, 30, 0)
+ checkEvaluation(
+ cast(
+ cast(new Timestamp(c.getTimeInMillis), StringType, timeZoneId),
+ TimestampType,
+ timeZoneId),
+ millisToMicros(c.getTimeInMillis))
+ }
+ }
+
+ checkEvaluation(cast("abdef", StringType), "abdef")
+ checkEvaluation(cast("12.65", DecimalType.SYSTEM_DEFAULT), Decimal(12.65))
+
+ checkEvaluation(cast(cast(sd, DateType), StringType), sd)
+ checkEvaluation(cast(cast(d, StringType), DateType), 0)
+
+ withSQLConf(
+ SQLConf.SESSION_LOCAL_TIMEZONE.key -> UTC_OPT.get
+ ) {
+ checkEvaluation(cast(cast(nts, TimestampType, UTC_OPT), StringType,
UTC_OPT), nts)
+ checkEvaluation(
+ cast(cast(ts, StringType, UTC_OPT), TimestampType, UTC_OPT),
+ fromJavaTimestamp(ts))
+
+ // all convert to string type to check
+ checkEvaluation(
+ cast(cast(cast(nts, TimestampType, UTC_OPT), DateType, UTC_OPT),
StringType),
+ sd)
+ checkEvaluation(
+ cast(cast(cast(ts, DateType, UTC_OPT), TimestampType, UTC_OPT),
StringType, UTC_OPT),
+ zts)
+ }
+
+ checkEvaluation(cast(cast("abdef", BinaryType), StringType), "abdef")
+
+ checkEvaluation(
+ cast(
+ cast(cast(cast(cast(cast("5", ByteType), ShortType), IntegerType),
FloatType), DoubleType),
+ LongType),
+ 5.toLong)
+
+ checkEvaluation(cast("23", DoubleType), 23d)
+ checkEvaluation(cast("23", IntegerType), 23)
+ checkEvaluation(cast("23", FloatType), 23f)
+ checkEvaluation(cast("23", DecimalType.USER_DEFAULT), Decimal(23))
+ checkEvaluation(cast("23", ByteType), 23.toByte)
+ checkEvaluation(cast("23", ShortType), 23.toShort)
+ checkEvaluation(cast(123, IntegerType), 123)
+
+ checkEvaluation(cast(Literal.create(null, IntegerType), ShortType), null)
+ }
+
override protected def evalMode: EvalMode.Value = EvalMode.LEGACY
}
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenTryCastSuite.scala
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenTryCastSuite.scala
index b0bddf9354..e8e5ad7e00 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenTryCastSuite.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenTryCastSuite.scala
@@ -17,5 +17,86 @@
package org.apache.spark.sql.catalyst.expressions
import org.apache.spark.sql.GlutenTestsTrait
+import
org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{withDefaultTimeZone,
ALL_TIMEZONES, UTC, UTC_OPT}
+import org.apache.spark.sql.catalyst.util.DateTimeUtils.{fromJavaTimestamp,
millisToMicros, TimeZoneUTC}
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.types.{BinaryType, ByteType, DateType, Decimal,
DecimalType, DoubleType, FloatType, IntegerType, LongType, ShortType,
StringType, TimestampType}
-class GlutenTryCastSuite extends TryCastSuite with GlutenTestsTrait {}
+import java.sql.{Date, Timestamp}
+import java.util.Calendar
+
+class GlutenTryCastSuite extends TryCastSuite with GlutenTestsTrait {
+ testGluten("data type casting") {
+ val sd = "1970-01-01"
+ val d = Date.valueOf(sd)
+ val zts = sd + " 00:00:00"
+ val sts = sd + " 00:00:02"
+ val nts = sts + ".1"
+ val ts = withDefaultTimeZone(UTC)(Timestamp.valueOf(nts))
+
+ for (tz <- ALL_TIMEZONES) {
+ withSQLConf(
+ SQLConf.SESSION_LOCAL_TIMEZONE.key -> tz.getId
+ ) {
+ val timeZoneId = Option(tz.getId)
+ var c = Calendar.getInstance(TimeZoneUTC)
+ c.set(2015, 2, 8, 2, 30, 0)
+ checkEvaluation(
+ cast(
+ cast(new Timestamp(c.getTimeInMillis), StringType, timeZoneId),
+ TimestampType,
+ timeZoneId),
+ millisToMicros(c.getTimeInMillis))
+ c = Calendar.getInstance(TimeZoneUTC)
+ c.set(2015, 10, 1, 2, 30, 0)
+ checkEvaluation(
+ cast(
+ cast(new Timestamp(c.getTimeInMillis), StringType, timeZoneId),
+ TimestampType,
+ timeZoneId),
+ millisToMicros(c.getTimeInMillis))
+ }
+ }
+
+ checkEvaluation(cast("abdef", StringType), "abdef")
+ checkEvaluation(cast("12.65", DecimalType.SYSTEM_DEFAULT), Decimal(12.65))
+
+ checkEvaluation(cast(cast(sd, DateType), StringType), sd)
+ checkEvaluation(cast(cast(d, StringType), DateType), 0)
+
+ withSQLConf(
+ SQLConf.SESSION_LOCAL_TIMEZONE.key -> UTC_OPT.get
+ ) {
+ checkEvaluation(cast(cast(nts, TimestampType, UTC_OPT), StringType,
UTC_OPT), nts)
+ checkEvaluation(
+ cast(cast(ts, StringType, UTC_OPT), TimestampType, UTC_OPT),
+ fromJavaTimestamp(ts))
+
+ // all convert to string type to check
+ checkEvaluation(
+ cast(cast(cast(nts, TimestampType, UTC_OPT), DateType, UTC_OPT),
StringType),
+ sd)
+ checkEvaluation(
+ cast(cast(cast(ts, DateType, UTC_OPT), TimestampType, UTC_OPT),
StringType, UTC_OPT),
+ zts)
+ }
+
+ checkEvaluation(cast(cast("abdef", BinaryType), StringType), "abdef")
+
+ checkEvaluation(
+ cast(
+ cast(cast(cast(cast(cast("5", ByteType), ShortType), IntegerType),
FloatType), DoubleType),
+ LongType),
+ 5.toLong)
+
+ checkEvaluation(cast("23", DoubleType), 23d)
+ checkEvaluation(cast("23", IntegerType), 23)
+ checkEvaluation(cast("23", FloatType), 23f)
+ checkEvaluation(cast("23", DecimalType.USER_DEFAULT), Decimal(23))
+ checkEvaluation(cast("23", ByteType), 23.toByte)
+ checkEvaluation(cast("23", ShortType), 23.toShort)
+ checkEvaluation(cast(123, IntegerType), 123)
+
+ checkEvaluation(cast(Literal.create(null, IntegerType), ShortType), null)
+ }
+}
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
index 59e6985801..bdb160ad24 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
@@ -531,6 +531,7 @@ class ClickHouseTestSettings extends BackendTestSettings {
.exclude("cast string to timestamp")
.exclude("cast from boolean")
.exclude("data type casting")
+ .excludeGlutenTest("data type casting")
.exclude("cast between string and interval")
.exclude("SPARK-27671: cast from nested null type in struct")
.exclude("Process Infinity, -Infinity, NaN in case insensitive manner")
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 904439e83e..2cf2f8ad31 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -92,6 +92,8 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude(
"Process Infinity, -Infinity, NaN in case insensitive manner" // +inf
not supported in folly.
)
+ // Set timezone through config.
+ .exclude("data type casting")
enableSuite[GlutenTryCastSuite]
.exclude(
"Process Infinity, -Infinity, NaN in case insensitive manner" // +inf
not supported in folly.
@@ -102,6 +104,8 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("ANSI mode: Throw exception on casting out-of-range value to long
type")
.exclude("cast from invalid string to numeric should throw
NumberFormatException")
.exclude("SPARK-26218: Fix the corner case of codegen when casting float
to Integer")
+ // Set timezone through config.
+ .exclude("data type casting")
enableSuite[GlutenCollectionExpressionsSuite]
// Rewrite in Gluten to replace Seq with Array
.exclude("Shuffle")
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
index 9b959d3ba5..f842e20adc 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
@@ -17,9 +17,13 @@
package org.apache.spark.sql.catalyst.expressions
import org.apache.spark.sql.GlutenTestsTrait
+import
org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{withDefaultTimeZone,
ALL_TIMEZONES, UTC, UTC_OPT}
+import org.apache.spark.sql.catalyst.util.DateTimeUtils.{fromJavaTimestamp,
millisToMicros, TimeZoneUTC}
+import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types._
-import java.sql.Date
+import java.sql.{Date, Timestamp}
+import java.util.Calendar
class GlutenCastSuite extends CastSuiteBase with GlutenTestsTrait {
override def cast(v: Any, targetType: DataType, timeZoneId: Option[String] =
None): Cast = {
@@ -75,5 +79,79 @@ class GlutenCastSuite extends CastSuiteBase with
GlutenTestsTrait {
checkEvaluation(cast(d, DateType), d)
}
+ testGluten("data type casting") {
+ val sd = "1970-01-01"
+ val d = Date.valueOf(sd)
+ val zts = sd + " 00:00:00"
+ val sts = sd + " 00:00:02"
+ val nts = sts + ".1"
+ val ts = withDefaultTimeZone(UTC)(Timestamp.valueOf(nts))
+
+ for (tz <- ALL_TIMEZONES) {
+ withSQLConf(
+ SQLConf.SESSION_LOCAL_TIMEZONE.key -> tz.getId
+ ) {
+ val timeZoneId = Option(tz.getId)
+ var c = Calendar.getInstance(TimeZoneUTC)
+ c.set(2015, 2, 8, 2, 30, 0)
+ checkEvaluation(
+ cast(
+ cast(new Timestamp(c.getTimeInMillis), StringType, timeZoneId),
+ TimestampType,
+ timeZoneId),
+ millisToMicros(c.getTimeInMillis))
+ c = Calendar.getInstance(TimeZoneUTC)
+ c.set(2015, 10, 1, 2, 30, 0)
+ checkEvaluation(
+ cast(
+ cast(new Timestamp(c.getTimeInMillis), StringType, timeZoneId),
+ TimestampType,
+ timeZoneId),
+ millisToMicros(c.getTimeInMillis))
+ }
+ }
+
+ checkEvaluation(cast("abdef", StringType), "abdef")
+ checkEvaluation(cast("12.65", DecimalType.SYSTEM_DEFAULT), Decimal(12.65))
+
+ checkEvaluation(cast(cast(sd, DateType), StringType), sd)
+ checkEvaluation(cast(cast(d, StringType), DateType), 0)
+
+ withSQLConf(
+ SQLConf.SESSION_LOCAL_TIMEZONE.key -> UTC_OPT.get
+ ) {
+ checkEvaluation(cast(cast(nts, TimestampType, UTC_OPT), StringType,
UTC_OPT), nts)
+ checkEvaluation(
+ cast(cast(ts, StringType, UTC_OPT), TimestampType, UTC_OPT),
+ fromJavaTimestamp(ts))
+
+ // all convert to string type to check
+ checkEvaluation(
+ cast(cast(cast(nts, TimestampType, UTC_OPT), DateType, UTC_OPT),
StringType),
+ sd)
+ checkEvaluation(
+ cast(cast(cast(ts, DateType, UTC_OPT), TimestampType, UTC_OPT),
StringType, UTC_OPT),
+ zts)
+ }
+
+ checkEvaluation(cast(cast("abdef", BinaryType), StringType), "abdef")
+
+ checkEvaluation(
+ cast(
+ cast(cast(cast(cast(cast("5", ByteType), ShortType), IntegerType),
FloatType), DoubleType),
+ LongType),
+ 5.toLong)
+
+ checkEvaluation(cast("23", DoubleType), 23d)
+ checkEvaluation(cast("23", IntegerType), 23)
+ checkEvaluation(cast("23", FloatType), 23f)
+ checkEvaluation(cast("23", DecimalType.USER_DEFAULT), Decimal(23))
+ checkEvaluation(cast("23", ByteType), 23.toByte)
+ checkEvaluation(cast("23", ShortType), 23.toShort)
+ checkEvaluation(cast(123, IntegerType), 123)
+
+ checkEvaluation(cast(Literal.create(null, IntegerType), ShortType), null)
+ }
+
override protected def evalMode: EvalMode.Value = EvalMode.LEGACY
}
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenTryCastSuite.scala
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenTryCastSuite.scala
index b0bddf9354..a9617e0645 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenTryCastSuite.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenTryCastSuite.scala
@@ -17,5 +17,88 @@
package org.apache.spark.sql.catalyst.expressions
import org.apache.spark.sql.GlutenTestsTrait
+import
org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{withDefaultTimeZone,
ALL_TIMEZONES, UTC, UTC_OPT}
+import org.apache.spark.sql.catalyst.util.DateTimeUtils.{fromJavaTimestamp,
millisToMicros, TimeZoneUTC}
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.types.{BinaryType, ByteType, DateType, Decimal,
DecimalType, DoubleType, FloatType, IntegerType, LongType, ShortType,
StringType, TimestampType}
-class GlutenTryCastSuite extends TryCastSuite with GlutenTestsTrait {}
+import java.sql.{Date, Timestamp}
+import java.util.Calendar
+
+class GlutenTryCastSuite extends TryCastSuite with GlutenTestsTrait {
+
+ testGluten("data type casting") {
+ val sd = "1970-01-01"
+ val d = Date.valueOf(sd)
+ val zts = sd + " 00:00:00"
+ val sts = sd + " 00:00:02"
+ val nts = sts + ".1"
+ val ts = withDefaultTimeZone(UTC)(Timestamp.valueOf(nts))
+
+ for (tz <- ALL_TIMEZONES) {
+ withSQLConf(
+ SQLConf.SESSION_LOCAL_TIMEZONE.key -> tz.getId
+ ) {
+ val timeZoneId = Option(tz.getId)
+ var c = Calendar.getInstance(TimeZoneUTC)
+ c.set(2015, 2, 8, 2, 30, 0)
+ checkEvaluation(
+ cast(
+ cast(new Timestamp(c.getTimeInMillis), StringType, timeZoneId),
+ TimestampType,
+ timeZoneId),
+ millisToMicros(c.getTimeInMillis))
+ c = Calendar.getInstance(TimeZoneUTC)
+ c.set(2015, 10, 1, 2, 30, 0)
+ checkEvaluation(
+ cast(
+ cast(new Timestamp(c.getTimeInMillis), StringType, timeZoneId),
+ TimestampType,
+ timeZoneId),
+ millisToMicros(c.getTimeInMillis))
+ }
+ }
+
+ checkEvaluation(cast("abdef", StringType), "abdef")
+ checkEvaluation(cast("12.65", DecimalType.SYSTEM_DEFAULT), Decimal(12.65))
+
+ checkEvaluation(cast(cast(sd, DateType), StringType), sd)
+ checkEvaluation(cast(cast(d, StringType), DateType), 0)
+
+ withSQLConf(
+ SQLConf.SESSION_LOCAL_TIMEZONE.key -> UTC_OPT.get
+ ) {
+ checkEvaluation(cast(cast(nts, TimestampType, UTC_OPT), StringType,
UTC_OPT), nts)
+ checkEvaluation(
+ cast(cast(ts, StringType, UTC_OPT), TimestampType, UTC_OPT),
+ fromJavaTimestamp(ts))
+
+ // all convert to string type to check
+ checkEvaluation(
+ cast(cast(cast(nts, TimestampType, UTC_OPT), DateType, UTC_OPT),
StringType),
+ sd)
+ checkEvaluation(
+ cast(cast(cast(ts, DateType, UTC_OPT), TimestampType, UTC_OPT),
StringType, UTC_OPT),
+ zts)
+ }
+
+ checkEvaluation(cast(cast("abdef", BinaryType), StringType), "abdef")
+
+ checkEvaluation(
+ cast(
+ cast(cast(cast(cast(cast("5", ByteType), ShortType), IntegerType),
FloatType), DoubleType),
+ LongType),
+ 5.toLong)
+
+ checkEvaluation(cast("23", DoubleType), 23d)
+ checkEvaluation(cast("23", IntegerType), 23)
+ checkEvaluation(cast("23", FloatType), 23f)
+ checkEvaluation(cast("23", DecimalType.USER_DEFAULT), Decimal(23))
+ checkEvaluation(cast("23", ByteType), 23.toByte)
+ checkEvaluation(cast("23", ShortType), 23.toShort)
+ checkEvaluation(cast(123, IntegerType), 123)
+
+ checkEvaluation(cast(Literal.create(null, IntegerType), ShortType), null)
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
