This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new db86b02612 [GLUTEN-9392][VL] Support casting complex types (#10443)
db86b02612 is described below
commit db86b026127d7f2cca8b9f1b8ade8eca0195faa5
Author: kevinwilfong <[email protected]>
AuthorDate: Sun Aug 17 17:58:25 2025 -0700
[GLUTEN-9392][VL] Support casting complex types (#10443)
---
.../functions/ScalarFunctionsValidateSuite.scala | 231 ++++++++++++++
.../substrait/SubstraitToVeloxPlanValidator.cc | 63 ++--
.../gluten/utils/velox/VeloxTestSettings.scala | 7 -
.../sql/catalyst/expressions/GlutenCastSuite.scala | 59 ----
.../sql/catalyst/expressions/GlutenCastSuite.scala | 59 ----
.../sql/catalyst/expressions/GlutenCastSuite.scala | 59 ----
.../sql/catalyst/expressions/GlutenCastSuite.scala | 59 ----
.../gluten/expressions/GlutenCastSuite.scala | 353 +++++++++++++++++++++
8 files changed, 613 insertions(+), 277 deletions(-)
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
index 1d376f5e23..6bef01e96f 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
@@ -20,8 +20,10 @@ import org.apache.gluten.config.GlutenConfig
import org.apache.gluten.execution.{BatchScanExecTransformer,
FilterExecTransformer, ProjectExecTransformer}
import org.apache.spark.{SparkConf, SparkException}
+import org.apache.spark.sql.Row
import org.apache.spark.sql.catalyst.optimizer.NullPropagation
import org.apache.spark.sql.execution.ProjectExec
+import org.apache.spark.sql.types._
class ScalarFunctionsValidateSuiteRasOff extends ScalarFunctionsValidateSuite {
override protected def sparkConf: SparkConf = {
@@ -1175,6 +1177,37 @@ abstract class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
runQueryAndCompare("select cast(array(timestamp'2024-01-01 12:00:00') AS
array<string>)") {
checkGlutenOperatorMatch[ProjectExecTransformer]
}
+ // Cast Array[String] to Array
+ runQueryAndCompare("select cast(array('123', '-98', 'abc', null) AS
array<tinyint>)") {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ runQueryAndCompare("select cast(array('123', '-98', 'abc', null) AS
array<smallint>)") {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ runQueryAndCompare("select cast(array('123', '-98', 'abc', null) AS
array<int>)") {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ runQueryAndCompare("select cast(array('123', '-98', 'abc', null) AS
array<bigint>)") {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ runQueryAndCompare("select cast(array('123e-2', '-234.548', 'xyz', null)
AS array<float>)") {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ runQueryAndCompare("select cast(array('123e-2', '-234.548', 'xyz', null)
AS array<double>)") {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ runQueryAndCompare("""
+ |select
+ | cast(
+ | array('2023-01-01 12:00:00', '2023-01-02
12:00:00', 'def', null)
+ | AS array<timestamp>)
+ |""".stripMargin) {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ runQueryAndCompare(
+ "select cast(array('2024-01-01', '2024-01-02', 'uvw', null) AS
array<date>)") {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
// Cast Array as String
withTempView("cast_table") {
withTempPath {
@@ -1207,6 +1240,204 @@ abstract class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
runQueryAndCompare("select cast(array(timestamp'2024-01-01 12:00:00') AS
string)") {
checkGlutenOperatorMatch[ProjectExecTransformer]
}
+ // Cast Map
+ withTempView("byte_map_tbl") {
+ withTempPath {
+ path =>
+ Seq[Map[Byte, Byte]](
+ Map(0.toByte -> 1.toByte, 2.toByte -> 3.toByte, 4.toByte ->
0.toByte))
+ .toDF("c1")
+ .write
+ .parquet(path.getCanonicalPath)
+
+
spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("byte_map_tbl")
+
+ runQueryAndCompare("select cast(c1 as map<tinyint, double>) from
byte_map_tbl") {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ runQueryAndCompare("select cast(c1 as map<double, tinyint>) from
byte_map_tbl") {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ runQueryAndCompare("select cast(c1 as map<tinyint, string>) from
byte_map_tbl") {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ runQueryAndCompare("select cast(c1 as map<string, tinyint>) from
byte_map_tbl") {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ runQueryAndCompare("select cast(c1 as map<tinyint, boolean>) from
byte_map_tbl") {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ }
+ }
+
+ withTempView("small_int_map_tbl") {
+ withTempPath {
+ path =>
+ Seq[Map[Short, Short]](
+ Map(
+ 1000.toShort -> 1001.toShort,
+ 1002.toShort -> 1003.toShort,
+ 1004.toShort -> 0.toShort))
+ .toDF("c1")
+ .write
+ .parquet(path.getCanonicalPath)
+
+
spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("small_int_map_tbl")
+
+ runQueryAndCompare("select cast(c1 as map<smallint, double>) from
small_int_map_tbl") {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ runQueryAndCompare("select cast(c1 as map<double, smallint>) from
small_int_map_tbl") {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ runQueryAndCompare("select cast(c1 as map<smallint, string>) from
small_int_map_tbl") {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ runQueryAndCompare("select cast(c1 as map<string, smallint>) from
small_int_map_tbl") {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ runQueryAndCompare("select cast(c1 as map<smallint, boolean>) from
small_int_map_tbl") {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ }
+ }
+
+ withTempView("int_map_tbl") {
+ withTempPath {
+ path =>
+ Seq[Map[Int, String]](Map(100 -> "101", 102 -> "103", 104 -> "xyz"))
+ .toDF("c1")
+ .write
+ .parquet(path.getCanonicalPath)
+
+
spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("int_map_tbl")
+
+ runQueryAndCompare("select cast(c1 as map<double, int>) from
int_map_tbl") {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ runQueryAndCompare("select cast(c1 as map<string, string>) from
int_map_tbl") {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ }
+ }
+
+ withTempView("float_map_tbl") {
+ withTempPath {
+ path =>
+ Seq[Map[Float, String]](Map(1.0f -> "2.0", -3.0f -> "40e-1", 5.0f ->
"xyz"))
+ .toDF("c1")
+ .write
+ .parquet(path.getCanonicalPath)
+
+
spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("float_map_tbl")
+
+ runQueryAndCompare("select cast(c1 as map<float, float>) from
float_map_tbl") {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ runQueryAndCompare("select cast(c1 as map<string, string>) from
float_map_tbl") {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ }
+ }
+
+ runQueryAndCompare("""
+ |select
+ | cast(
+ | map(
+ | timestamp '2023-01-01 12:00:00', '2023-01-01
13:00:00',
+ | timestamp '2023-01-02 12:00:00', 'xyz')
+ | as map<string, timestamp>)
+ |""".stripMargin) {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ runQueryAndCompare("""
+ |select
+ | cast(
+ | map(date '2024-01-01', '2024-01-02', date
'2024-02-01', 'xyz')
+ | as map<string, date>)
+ |""".stripMargin) {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ // Cast struct
+ withTempView("struct_tbl") {
+ val structData = Seq(
+ Row(
+ Row(
+ Seq("123", "456.7", "2023-01-01 12:00:00", "2024-01-01"),
+ Map(1.toByte -> 2.toShort, 3.toByte -> 4.toShort),
+ Row(
+ 123.0,
+ 456.1f,
+ 0
+ )
+ )))
+
+ val structSchema = new StructType().add(
+ "c1",
+ new StructType()
+ .add("a", ArrayType(StringType))
+ .add("b", MapType(ByteType, ShortType))
+ .add("c", new StructType().add("x", DoubleType).add("y",
FloatType).add("z", IntegerType))
+ )
+ withTempPath {
+ path =>
+ spark
+ .createDataFrame(spark.sparkContext.parallelize(structData),
structSchema)
+ .write
+ .parquet(path.getCanonicalPath)
+
+
spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("struct_tbl")
+
+ runQueryAndCompare("""
+ |select
+ | cast(
+ | c1 as
+ | struct<
+ | a: array<bigint>,
+ | b: map<smallint, int>,
+ | c: struct<x: string, y: string,
z:boolean>>)
+ |from struct_tbl
+ |""".stripMargin) {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ runQueryAndCompare("""
+ |select
+ | cast(
+ | c1 as
+ | struct<
+ | a: array<double>,
+ | b: map<int, bigint>,
+ | c: struct<x: int, y: boolean,
z:string>>)
+ |from struct_tbl
+ |""".stripMargin) {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ runQueryAndCompare("""
+ |select
+ | cast(
+ | c1 as
+ | struct<
+ | a: array<timestamp>,
+ | b: map<bigint, boolean>,
+ | c: struct<x: tinyint, y: smallint,
z:double>>)
+ |from struct_tbl
+ |""".stripMargin) {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ runQueryAndCompare("""
+ |select
+ | cast(
+ | c1 as
+ | struct<
+ | a: array<date>,
+ | b: map<string, double>,
+ | c: struct<x: int, y: bigint, z:float>>)
+ |from struct_tbl
+ |""".stripMargin) {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ }
+ }
}
testWithMinSparkVersion("equal_null", "3.4") {
diff --git a/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc
b/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc
index 2bc083fd20..1f0a5b3361 100644
--- a/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc
+++ b/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc
@@ -235,34 +235,6 @@ bool SubstraitToVeloxPlanValidator::validateScalarFunction(
return true;
}
-bool isSupportedArrayCast(const TypePtr& fromType, const TypePtr& toType) {
- // https://github.com/apache/incubator-gluten/issues/9392
- // is currently WIP to add support for other types.
- if (toType->isVarchar()) {
- return fromType->isDouble() || fromType->isBoolean() ||
fromType->isTimestamp()
- || fromType->isInteger() || fromType->isBigint() || fromType->isDate();
- }
-
- if (toType->isDouble()) {
- if (fromType->isInteger() || fromType->isBigint() ||
fromType->isSmallint() || fromType->isTinyint()) {
- return true;
- }
- }
-
- if (toType->isBoolean()) {
- if (fromType->isDate() || fromType->isShortDecimal()) {
- return false;
- }
-
- if (fromType->isTinyint() || fromType->isSmallint() ||
fromType->isInteger() || fromType->isBigint() ||
- fromType->isReal() || fromType->isDouble()) {
- return true;
- }
- }
-
- return false;
-}
-
bool SubstraitToVeloxPlanValidator::isAllowedCast(const TypePtr& fromType,
const TypePtr& toType) {
// Currently cast is not allowed for various categories, code has a bunch of
rules
// which define the cast categories and if we should offload to velox.
Currently,
@@ -321,19 +293,42 @@ bool SubstraitToVeloxPlanValidator::isAllowedCast(const
TypePtr& fromType, const
return false;
}
+ // For complex types recursively check that their children can be cast.
if (fromType->isArray() && toType->isArray()) {
const auto& toElem = toType->asArray().elementType();
const auto& fromElem = fromType->asArray().elementType();
- if (!isAllowedCast(fromElem, toElem)) {
- return false;
- }
+ return isAllowedCast(fromElem, toElem);
+ }
+
+ if (fromType->isMap() && toType->isMap()) {
+ const auto& fromKey = fromType->asMap().keyType();
+ const auto& fromValue = fromType->asMap().valueType();
+ const auto& toKey = toType->asMap().keyType();
+ const auto& toValue = toType->asMap().valueType();
+
+ return isAllowedCast(fromKey, toKey) && isAllowedCast(fromValue,
toValue);
+ }
+
+ if (fromType->isRow() && toType->isRow()) {
+ const auto& fromChildren = fromType->asRow().children();
+ const auto& toChildren = toType->asRow().children();
- return isSupportedArrayCast(fromElem, toElem);
+ if (fromChildren.size() != toChildren.size()) {
+ return false;
+ }
+
+ for (size_t childIdx = 0; childIdx < fromChildren.size(); ++childIdx) {
+ if (!isAllowedCast(fromChildren[childIdx], toChildren[childIdx])) {
+ return false;
+ }
+ }
+
+ return true;
}
- // Limited support for Complex types.
- if (fromType->isArray() || fromType->isMap() || fromType->isRow()) {
+ // Casting a complex type to/from any other type is not allowed.
+ if (fromType->isArray() || fromType->isMap() || fromType->isRow() ||
toType->isArray() || toType->isMap() || toType->isRow()) {
return false;
}
diff --git
a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 725adb206a..b08d6ed056 100644
---
a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -107,13 +107,6 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("SPARK-35719: cast timestamp with local time zone to timestamp
without timezone")
enableSuite[GlutenTryCastSuite]
- .exclude(
- // array/map/struct not supported yet.
- "cast from invalid string array to numeric array should throw
NumberFormatException",
- "cast from array II",
- "cast from map II",
- "cast from struct II"
- )
// Timezone.
.exclude("SPARK-35711: cast timestamp without time zone to timestamp with
local time zone")
// Timezone.
diff --git
a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
index 05ef3a3686..cd4351e708 100644
---
a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
+++
b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
@@ -67,65 +67,6 @@ class GlutenCastSuite extends CastSuite with
GlutenTestsTrait {
}
}
- test("cast array of integer types to array of double") {
- val intArray = Literal.create(Seq(1, 2, 3), ArrayType(IntegerType))
- val bigintArray = Literal.create(Seq(10000000000L), ArrayType(LongType))
- val smallintArray = Literal.create(Seq(1.toShort, -1.toShort),
ArrayType(ShortType))
- val tinyintArray = Literal.create(Seq(1.toByte, -1.toByte),
ArrayType(ByteType))
-
- checkEvaluation(cast(intArray, ArrayType(DoubleType)), Seq(1.0, 2.0, 3.0))
- checkEvaluation(cast(bigintArray, ArrayType(DoubleType)), Seq(1.0e10))
- checkEvaluation(cast(smallintArray, ArrayType(DoubleType)), Seq(1.0, -1.0))
- checkEvaluation(cast(tinyintArray, ArrayType(DoubleType)), Seq(1.0, -1.0))
- }
-
- test("cast array of double to array of integer types") {
- val doubleArray = Literal.create(Seq(1.9, -2.1), ArrayType(DoubleType))
-
- checkEvaluation(cast(doubleArray, ArrayType(IntegerType)), Seq(1, -2))
- checkEvaluation(cast(doubleArray, ArrayType(LongType)), Seq(1L, -2L))
- checkEvaluation(cast(doubleArray, ArrayType(ShortType)), Seq(1.toShort,
-2.toShort))
- checkEvaluation(cast(doubleArray, ArrayType(ByteType)), Seq(1.toByte,
-2.toByte))
- }
-
- test("cast array element from allowed types to string (varchar)") {
- val doubleArray = Literal.create(Seq(1.1, null, 3.3),
ArrayType(DoubleType))
- val boolArray = Literal.create(Seq(true, false), ArrayType(BooleanType))
- val timestampArray = Literal.create(
- Seq(Timestamp.valueOf("2023-01-01 12:00:00"), null),
- ArrayType(TimestampType)
- )
- val integerArray = Literal.create(Seq(1, null, 2), ArrayType(IntegerType))
- val longArray = Literal.create(Seq(1L, null, 2L), ArrayType(LongType))
- val dateArray = Literal.create(
- Seq(Date.valueOf("2024-01-01"), null, Date.valueOf("2024-01-03")),
- ArrayType(DateType)
- )
-
- checkEvaluation(cast(doubleArray, ArrayType(StringType)), Seq("1.1", null,
"3.3"))
- checkEvaluation(cast(boolArray, ArrayType(StringType)), Seq("true",
"false"))
- checkEvaluation(cast(timestampArray, ArrayType(StringType)),
Seq("2023-01-01 12:00:00", null))
- checkEvaluation(cast(integerArray, ArrayType(StringType)), Seq("1", null,
"2"))
- checkEvaluation(cast(longArray, ArrayType(StringType)), Seq("1", null,
"2"))
- checkEvaluation(cast(dateArray, ArrayType(StringType)), Seq("2024-01-01",
null, "2024-01-03"))
- }
-
- test("cast array of numeric types to array of boolean") {
- val tinyintArray = Literal.create(Seq(0.toByte, 1.toByte, -1.toByte),
ArrayType(ByteType))
- val smallintArray = Literal.create(Seq(0.toShort, 2.toShort, -2.toShort),
ArrayType(ShortType))
- val intArray = Literal.create(Seq(0, 3, -3), ArrayType(IntegerType))
- val bigintArray = Literal.create(Seq(0L, 100L, -100L), ArrayType(LongType))
- val floatArray = Literal.create(Seq(0.0f, 1.5f, -2.3f),
ArrayType(FloatType))
- val doubleArray = Literal.create(Seq(0.0, 2.5, -3.7),
ArrayType(DoubleType))
-
- checkEvaluation(cast(tinyintArray, ArrayType(BooleanType)), Seq(false,
true, true))
- checkEvaluation(cast(smallintArray, ArrayType(BooleanType)), Seq(false,
true, true))
- checkEvaluation(cast(intArray, ArrayType(BooleanType)), Seq(false, true,
true))
- checkEvaluation(cast(bigintArray, ArrayType(BooleanType)), Seq(false,
true, true))
- checkEvaluation(cast(floatArray, ArrayType(BooleanType)), Seq(false, true,
true))
- checkEvaluation(cast(doubleArray, ArrayType(BooleanType)), Seq(false,
true, true))
- }
-
testGluten("missing cases - from short") {
DataTypeTestUtils.numericTypeWithoutDecimal.foreach {
t =>
diff --git
a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
index 407e3e5ef4..9be180930d 100644
---
a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
+++
b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
@@ -43,65 +43,6 @@ class GlutenCastSuite extends CastSuite with
GlutenTestsTrait {
UDTRegistration.register(classOf[IExampleBaseType].getName,
classOf[ExampleBaseTypeUDT].getName)
UDTRegistration.register(classOf[IExampleSubType].getName,
classOf[ExampleSubTypeUDT].getName)
- test("cast array of integer types to array of double") {
- val intArray = Literal.create(Seq(1, 2, 3), ArrayType(IntegerType))
- val bigintArray = Literal.create(Seq(10000000000L), ArrayType(LongType))
- val smallintArray = Literal.create(Seq(1.toShort, -1.toShort),
ArrayType(ShortType))
- val tinyintArray = Literal.create(Seq(1.toByte, -1.toByte),
ArrayType(ByteType))
-
- checkEvaluation(cast(intArray, ArrayType(DoubleType)), Seq(1.0, 2.0, 3.0))
- checkEvaluation(cast(bigintArray, ArrayType(DoubleType)), Seq(1.0e10))
- checkEvaluation(cast(smallintArray, ArrayType(DoubleType)), Seq(1.0, -1.0))
- checkEvaluation(cast(tinyintArray, ArrayType(DoubleType)), Seq(1.0, -1.0))
- }
-
- test("cast array of double to array of integer types") {
- val doubleArray = Literal.create(Seq(1.9, -2.1), ArrayType(DoubleType))
-
- checkEvaluation(cast(doubleArray, ArrayType(IntegerType)), Seq(1, -2))
- checkEvaluation(cast(doubleArray, ArrayType(LongType)), Seq(1L, -2L))
- checkEvaluation(cast(doubleArray, ArrayType(ShortType)), Seq(1.toShort,
-2.toShort))
- checkEvaluation(cast(doubleArray, ArrayType(ByteType)), Seq(1.toByte,
-2.toByte))
- }
-
- test("cast array element from allowed types to string (varchar)") {
- val doubleArray = Literal.create(Seq(1.1, null, 3.3),
ArrayType(DoubleType))
- val boolArray = Literal.create(Seq(true, false), ArrayType(BooleanType))
- val timestampArray = Literal.create(
- Seq(Timestamp.valueOf("2023-01-01 12:00:00"), null),
- ArrayType(TimestampType)
- )
- val integerArray = Literal.create(Seq(1, null, 2), ArrayType(IntegerType))
- val longArray = Literal.create(Seq(1L, null, 2L), ArrayType(LongType))
- val dateArray = Literal.create(
- Seq(Date.valueOf("2024-01-01"), null, Date.valueOf("2024-01-03")),
- ArrayType(DateType)
- )
-
- checkEvaluation(cast(doubleArray, ArrayType(StringType)), Seq("1.1", null,
"3.3"))
- checkEvaluation(cast(boolArray, ArrayType(StringType)), Seq("true",
"false"))
- checkEvaluation(cast(timestampArray, ArrayType(StringType)),
Seq("2023-01-01 12:00:00", null))
- checkEvaluation(cast(integerArray, ArrayType(StringType)), Seq("1", null,
"2"))
- checkEvaluation(cast(longArray, ArrayType(StringType)), Seq("1", null,
"2"))
- checkEvaluation(cast(dateArray, ArrayType(StringType)), Seq("2024-01-01",
null, "2024-01-03"))
- }
-
- test("cast array of numeric types to array of boolean") {
- val tinyintArray = Literal.create(Seq(0.toByte, 1.toByte, -1.toByte),
ArrayType(ByteType))
- val smallintArray = Literal.create(Seq(0.toShort, 2.toShort, -2.toShort),
ArrayType(ShortType))
- val intArray = Literal.create(Seq(0, 3, -3), ArrayType(IntegerType))
- val bigintArray = Literal.create(Seq(0L, 100L, -100L), ArrayType(LongType))
- val floatArray = Literal.create(Seq(0.0f, 1.5f, -2.3f),
ArrayType(FloatType))
- val doubleArray = Literal.create(Seq(0.0, 2.5, -3.7),
ArrayType(DoubleType))
-
- checkEvaluation(cast(tinyintArray, ArrayType(BooleanType)), Seq(false,
true, true))
- checkEvaluation(cast(smallintArray, ArrayType(BooleanType)), Seq(false,
true, true))
- checkEvaluation(cast(intArray, ArrayType(BooleanType)), Seq(false, true,
true))
- checkEvaluation(cast(bigintArray, ArrayType(BooleanType)), Seq(false,
true, true))
- checkEvaluation(cast(floatArray, ArrayType(BooleanType)), Seq(false, true,
true))
- checkEvaluation(cast(doubleArray, ArrayType(BooleanType)), Seq(false,
true, true))
- }
-
testGluten("missing cases - from boolean") {
(DataTypeTestUtils.numericTypeWithoutDecimal + BooleanType).foreach {
t =>
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
index 493a47c56b..64fa85d37f 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
@@ -55,65 +55,6 @@ class GlutenCastSuite extends CastWithAnsiOffSuite with
GlutenTestsTrait {
}
}
- test("cast array of integer types to array of double") {
- val intArray = Literal.create(Seq(1, 2, 3), ArrayType(IntegerType))
- val bigintArray = Literal.create(Seq(10000000000L), ArrayType(LongType))
- val smallintArray = Literal.create(Seq(1.toShort, -1.toShort),
ArrayType(ShortType))
- val tinyintArray = Literal.create(Seq(1.toByte, -1.toByte),
ArrayType(ByteType))
-
- checkEvaluation(cast(intArray, ArrayType(DoubleType)), Seq(1.0, 2.0, 3.0))
- checkEvaluation(cast(bigintArray, ArrayType(DoubleType)), Seq(1.0e10))
- checkEvaluation(cast(smallintArray, ArrayType(DoubleType)), Seq(1.0, -1.0))
- checkEvaluation(cast(tinyintArray, ArrayType(DoubleType)), Seq(1.0, -1.0))
- }
-
- test("cast array of double to array of integer types") {
- val doubleArray = Literal.create(Seq(1.9, -2.1), ArrayType(DoubleType))
-
- checkEvaluation(cast(doubleArray, ArrayType(IntegerType)), Seq(1, -2))
- checkEvaluation(cast(doubleArray, ArrayType(LongType)), Seq(1L, -2L))
- checkEvaluation(cast(doubleArray, ArrayType(ShortType)), Seq(1.toShort,
-2.toShort))
- checkEvaluation(cast(doubleArray, ArrayType(ByteType)), Seq(1.toByte,
-2.toByte))
- }
-
- test("cast array element from allowed types to string (varchar)") {
- val doubleArray = Literal.create(Seq(1.1, null, 3.3),
ArrayType(DoubleType))
- val boolArray = Literal.create(Seq(true, false), ArrayType(BooleanType))
- val timestampArray = Literal.create(
- Seq(Timestamp.valueOf("2023-01-01 12:00:00"), null),
- ArrayType(TimestampType)
- )
- val integerArray = Literal.create(Seq(1, null, 2), ArrayType(IntegerType))
- val longArray = Literal.create(Seq(1L, null, 2L), ArrayType(LongType))
- val dateArray = Literal.create(
- Seq(Date.valueOf("2024-01-01"), null, Date.valueOf("2024-01-03")),
- ArrayType(DateType)
- )
-
- checkEvaluation(cast(doubleArray, ArrayType(StringType)), Seq("1.1", null,
"3.3"))
- checkEvaluation(cast(boolArray, ArrayType(StringType)), Seq("true",
"false"))
- checkEvaluation(cast(timestampArray, ArrayType(StringType)),
Seq("2023-01-01 12:00:00", null))
- checkEvaluation(cast(integerArray, ArrayType(StringType)), Seq("1", null,
"2"))
- checkEvaluation(cast(longArray, ArrayType(StringType)), Seq("1", null,
"2"))
- checkEvaluation(cast(dateArray, ArrayType(StringType)), Seq("2024-01-01",
null, "2024-01-03"))
- }
-
- test("cast array of numeric types to array of boolean") {
- val tinyintArray = Literal.create(Seq(0.toByte, 1.toByte, -1.toByte),
ArrayType(ByteType))
- val smallintArray = Literal.create(Seq(0.toShort, 2.toShort, -2.toShort),
ArrayType(ShortType))
- val intArray = Literal.create(Seq(0, 3, -3), ArrayType(IntegerType))
- val bigintArray = Literal.create(Seq(0L, 100L, -100L), ArrayType(LongType))
- val floatArray = Literal.create(Seq(0.0f, 1.5f, -2.3f),
ArrayType(FloatType))
- val doubleArray = Literal.create(Seq(0.0, 2.5, -3.7),
ArrayType(DoubleType))
-
- checkEvaluation(cast(tinyintArray, ArrayType(BooleanType)), Seq(false,
true, true))
- checkEvaluation(cast(smallintArray, ArrayType(BooleanType)), Seq(false,
true, true))
- checkEvaluation(cast(intArray, ArrayType(BooleanType)), Seq(false, true,
true))
- checkEvaluation(cast(bigintArray, ArrayType(BooleanType)), Seq(false,
true, true))
- checkEvaluation(cast(floatArray, ArrayType(BooleanType)), Seq(false, true,
true))
- checkEvaluation(cast(doubleArray, ArrayType(BooleanType)), Seq(false,
true, true))
- }
-
testGluten("missing cases - from byte") {
DataTypeTestUtils.numericTypeWithoutDecimal.foreach {
t =>
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
index 4332f704ef..7cf89937a6 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
@@ -44,65 +44,6 @@ class GlutenCastSuite extends CastWithAnsiOffSuite with
GlutenTestsTrait {
UDTRegistration.register(classOf[IExampleBaseType].getName,
classOf[ExampleBaseTypeUDT].getName)
UDTRegistration.register(classOf[IExampleSubType].getName,
classOf[ExampleSubTypeUDT].getName)
- test("cast array of integer types to array of double") {
- val intArray = Literal.create(Seq(1, 2, 3), ArrayType(IntegerType))
- val bigintArray = Literal.create(Seq(10000000000L), ArrayType(LongType))
- val smallintArray = Literal.create(Seq(1.toShort, -1.toShort),
ArrayType(ShortType))
- val tinyintArray = Literal.create(Seq(1.toByte, -1.toByte),
ArrayType(ByteType))
-
- checkEvaluation(cast(intArray, ArrayType(DoubleType)), Seq(1.0, 2.0, 3.0))
- checkEvaluation(cast(bigintArray, ArrayType(DoubleType)), Seq(1.0e10))
- checkEvaluation(cast(smallintArray, ArrayType(DoubleType)), Seq(1.0, -1.0))
- checkEvaluation(cast(tinyintArray, ArrayType(DoubleType)), Seq(1.0, -1.0))
- }
-
- test("cast array of double to array of integer types") {
- val doubleArray = Literal.create(Seq(1.9, -2.1), ArrayType(DoubleType))
-
- checkEvaluation(cast(doubleArray, ArrayType(IntegerType)), Seq(1, -2))
- checkEvaluation(cast(doubleArray, ArrayType(LongType)), Seq(1L, -2L))
- checkEvaluation(cast(doubleArray, ArrayType(ShortType)), Seq(1.toShort,
-2.toShort))
- checkEvaluation(cast(doubleArray, ArrayType(ByteType)), Seq(1.toByte,
-2.toByte))
- }
-
- test("cast array element from allowed types to string (varchar)") {
- val doubleArray = Literal.create(Seq(1.1, null, 3.3),
ArrayType(DoubleType))
- val boolArray = Literal.create(Seq(true, false), ArrayType(BooleanType))
- val timestampArray = Literal.create(
- Seq(Timestamp.valueOf("2023-01-01 12:00:00"), null),
- ArrayType(TimestampType)
- )
- val integerArray = Literal.create(Seq(1, null, 2), ArrayType(IntegerType))
- val longArray = Literal.create(Seq(1L, null, 2L), ArrayType(LongType))
- val dateArray = Literal.create(
- Seq(Date.valueOf("2024-01-01"), null, Date.valueOf("2024-01-03")),
- ArrayType(DateType)
- )
-
- checkEvaluation(cast(doubleArray, ArrayType(StringType)), Seq("1.1", null,
"3.3"))
- checkEvaluation(cast(boolArray, ArrayType(StringType)), Seq("true",
"false"))
- checkEvaluation(cast(timestampArray, ArrayType(StringType)),
Seq("2023-01-01 12:00:00", null))
- checkEvaluation(cast(integerArray, ArrayType(StringType)), Seq("1", null,
"2"))
- checkEvaluation(cast(longArray, ArrayType(StringType)), Seq("1", null,
"2"))
- checkEvaluation(cast(dateArray, ArrayType(StringType)), Seq("2024-01-01",
null, "2024-01-03"))
- }
-
- test("cast array of numeric types to array of boolean") {
- val tinyintArray = Literal.create(Seq(0.toByte, 1.toByte, -1.toByte),
ArrayType(ByteType))
- val smallintArray = Literal.create(Seq(0.toShort, 2.toShort, -2.toShort),
ArrayType(ShortType))
- val intArray = Literal.create(Seq(0, 3, -3), ArrayType(IntegerType))
- val bigintArray = Literal.create(Seq(0L, 100L, -100L), ArrayType(LongType))
- val floatArray = Literal.create(Seq(0.0f, 1.5f, -2.3f),
ArrayType(FloatType))
- val doubleArray = Literal.create(Seq(0.0, 2.5, -3.7),
ArrayType(DoubleType))
-
- checkEvaluation(cast(tinyintArray, ArrayType(BooleanType)), Seq(false,
true, true))
- checkEvaluation(cast(smallintArray, ArrayType(BooleanType)), Seq(false,
true, true))
- checkEvaluation(cast(intArray, ArrayType(BooleanType)), Seq(false, true,
true))
- checkEvaluation(cast(bigintArray, ArrayType(BooleanType)), Seq(false,
true, true))
- checkEvaluation(cast(floatArray, ArrayType(BooleanType)), Seq(false, true,
true))
- checkEvaluation(cast(doubleArray, ArrayType(BooleanType)), Seq(false,
true, true))
- }
-
testGluten("missing cases - from boolean") {
(DataTypeTestUtils.numericTypeWithoutDecimal ++ Set(BooleanType)).foreach {
t =>
diff --git
a/gluten-ut/test/src/test/scala/org/apache/gluten/expressions/GlutenCastSuite.scala
b/gluten-ut/test/src/test/scala/org/apache/gluten/expressions/GlutenCastSuite.scala
new file mode 100644
index 0000000000..0094da623b
--- /dev/null
+++
b/gluten-ut/test/src/test/scala/org/apache/gluten/expressions/GlutenCastSuite.scala
@@ -0,0 +1,353 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.expressions
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.catalyst.expressions.{Cast, ExpressionEvalHelper,
Literal}
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.types._
+
+import java.sql.{Date, Timestamp}
+
+class GlutenCastSuite extends SparkFunSuite with ExpressionEvalHelper {
+ def checkCast(v: Literal, targetType: DataType, expected: Any): Unit = {
+ withSQLConf(SQLConf.ANSI_ENABLED.key -> "false") {
+ checkEvaluation(Cast(v, targetType), expected)
+ }
+ }
+
+ test("cast array of integer types to array of double") {
+ val intArray = Literal.create(Seq(1, 2, 3), ArrayType(IntegerType))
+ val bigintArray = Literal.create(Seq(10000000000L), ArrayType(LongType))
+ val smallintArray = Literal.create(Seq(1.toShort, -1.toShort),
ArrayType(ShortType))
+ val tinyintArray = Literal.create(Seq(1.toByte, -1.toByte),
ArrayType(ByteType))
+
+ checkCast(intArray, ArrayType(DoubleType), Seq(1.0, 2.0, 3.0))
+ checkCast(bigintArray, ArrayType(DoubleType), Seq(1.0e10))
+ checkCast(smallintArray, ArrayType(DoubleType), Seq(1.0, -1.0))
+ checkCast(tinyintArray, ArrayType(DoubleType), Seq(1.0, -1.0))
+ }
+
+ test("cast array of double to array of integer types") {
+ val doubleArray = Literal.create(Seq(1.9, -2.1), ArrayType(DoubleType))
+
+ checkCast(doubleArray, ArrayType(IntegerType), Seq(1, -2))
+ checkCast(doubleArray, ArrayType(LongType), Seq(1L, -2L))
+ checkCast(doubleArray, ArrayType(ShortType), Seq(1.toShort, -2.toShort))
+ checkCast(doubleArray, ArrayType(ByteType), Seq(1.toByte, -2.toByte))
+ }
+
+ test("cast array element from allowed types to string (varchar)") {
+ val doubleArray = Literal.create(Seq(1.1, null, 3.3),
ArrayType(DoubleType))
+ val boolArray = Literal.create(Seq(true, false), ArrayType(BooleanType))
+ val timestampArray = Literal.create(
+ Seq(Timestamp.valueOf("2023-01-01 12:00:00"), null),
+ ArrayType(TimestampType)
+ )
+ val integerArray = Literal.create(Seq(1, null, 2), ArrayType(IntegerType))
+ val longArray = Literal.create(Seq(1L, null, 2L), ArrayType(LongType))
+ val dateArray = Literal.create(
+ Seq(Date.valueOf("2024-01-01"), null, Date.valueOf("2024-01-03")),
+ ArrayType(DateType)
+ )
+
+ checkCast(doubleArray, ArrayType(StringType), Seq("1.1", null, "3.3"))
+ checkCast(boolArray, ArrayType(StringType), Seq("true", "false"))
+ checkCast(timestampArray, ArrayType(StringType), Seq("2023-01-01
12:00:00", null))
+ checkCast(integerArray, ArrayType(StringType), Seq("1", null, "2"))
+ checkCast(longArray, ArrayType(StringType), Seq("1", null, "2"))
+ checkCast(dateArray, ArrayType(StringType), Seq("2024-01-01", null,
"2024-01-03"))
+ }
+
+ test("cast array of numeric types to array of boolean") {
+ val tinyintArray = Literal.create(Seq(0.toByte, 1.toByte, -1.toByte),
ArrayType(ByteType))
+ val smallintArray = Literal.create(Seq(0.toShort, 2.toShort, -2.toShort),
ArrayType(ShortType))
+ val intArray = Literal.create(Seq(0, 3, -3), ArrayType(IntegerType))
+ val bigintArray = Literal.create(Seq(0L, 100L, -100L), ArrayType(LongType))
+ val floatArray = Literal.create(Seq(0.0f, 1.5f, -2.3f),
ArrayType(FloatType))
+ val doubleArray = Literal.create(Seq(0.0, 2.5, -3.7),
ArrayType(DoubleType))
+
+ checkCast(tinyintArray, ArrayType(BooleanType), Seq(false, true, true))
+ checkCast(smallintArray, ArrayType(BooleanType), Seq(false, true, true))
+ checkCast(intArray, ArrayType(BooleanType), Seq(false, true, true))
+ checkCast(bigintArray, ArrayType(BooleanType), Seq(false, true, true))
+ checkCast(floatArray, ArrayType(BooleanType), Seq(false, true, true))
+ checkCast(doubleArray, ArrayType(BooleanType), Seq(false, true, true))
+ }
+
+ test("cast array of string") {
+ val integerStringArray = Literal.create(Seq("123", "-98", "abc"),
ArrayType(StringType))
+ val floatingPointStringArray =
+ Literal.create(Seq("123e-2", "-234.548", "xyz"), ArrayType(StringType))
+ val timestampStringArray = Literal.create(
+ Seq("2023-01-01 12:00:00", "2023-01-02 12:00:00", "def"),
+ ArrayType(StringType))
+ val dateStringArray =
+ Literal.create(Seq("2024-01-01", "2024-01-02", "uvw"),
ArrayType(StringType))
+
+ checkCast(integerStringArray, ArrayType(ByteType), Seq(123.toByte,
-98.toByte, null))
+ checkCast(integerStringArray, ArrayType(ShortType), Seq(123.toShort,
-98.toShort, null))
+ checkCast(integerStringArray, ArrayType(IntegerType), Seq(123, -98, null))
+ checkCast(integerStringArray, ArrayType(LongType), Seq(123L, -98L, null))
+ checkCast(floatingPointStringArray, ArrayType(DoubleType), Seq(1.23,
-234.548, null))
+ checkCast(floatingPointStringArray, ArrayType(FloatType), Seq(1.23f,
-234.548f, null))
+ checkCast(
+ timestampStringArray,
+ ArrayType(TimestampType),
+ Seq(Timestamp.valueOf("2023-01-01 12:00:00"),
Timestamp.valueOf("2023-01-02 12:00:00"), null))
+ checkCast(
+ dateStringArray,
+ ArrayType(DateType),
+ Seq(Date.valueOf("2024-01-01"), Date.valueOf("2024-01-02"), null))
+ }
+
+ test("cast maps") {
+ val byteMap = Literal.create(
+ Map(0.toByte -> 1.toByte, 2.toByte -> 3.toByte, 4.toByte -> 0.toByte),
+ MapType(ByteType, ByteType))
+ checkCast(
+ byteMap,
+ MapType(ByteType, DoubleType),
+ Map(0.toByte -> 1.0, 2.toByte -> 3.0, 4.toByte -> 0.0))
+ checkCast(
+ byteMap,
+ MapType(DoubleType, ByteType),
+ Map(0.0 -> 1.toByte, 2.0 -> 3.toByte, 4.0 -> 0.toByte))
+ checkCast(
+ byteMap,
+ MapType(ByteType, StringType),
+ Map(0.toByte -> "1", 2.toByte -> "3", 4.toByte -> "0"))
+ checkCast(
+ byteMap,
+ MapType(StringType, ByteType),
+ Map("0" -> 1.toByte, "2" -> 3.toByte, "4" -> 0.toByte))
+ checkCast(
+ byteMap,
+ MapType(ByteType, BooleanType),
+ Map(0.toByte -> true, 2.toByte -> true, 4.toByte -> false))
+
+ val shortMap = Literal.create(
+ Map(1000.toShort -> 1001.toShort, 1002.toShort -> 1003.toShort,
1004.toShort -> 0.toShort),
+ MapType(ShortType, ShortType))
+ checkCast(
+ shortMap,
+ MapType(ShortType, DoubleType),
+ Map(1000.toShort -> 1001.0, 1002.toShort -> 1003.0, 1004.toShort -> 0.0))
+ checkCast(
+ shortMap,
+ MapType(DoubleType, ShortType),
+ Map(1000.0 -> 1001.toShort, 1002.0 -> 1003.toShort, 1004.0 -> 0.toShort))
+ checkCast(
+ shortMap,
+ MapType(ShortType, StringType),
+ Map(1000.toShort -> "1001", 1002.toShort -> "1003", 1004.toShort -> "0"))
+ checkCast(
+ shortMap,
+ MapType(StringType, ShortType),
+ Map("1000" -> 1001.toShort, "1002" -> 1003.toShort, "1004" -> 0.toShort))
+ checkCast(
+ shortMap,
+ MapType(ShortType, BooleanType),
+ Map(1000.toShort -> true, 1002.toShort -> true, 1004.toShort -> false))
+
+ val intMap = Literal.create(
+ Map(100 -> "101", 102 -> "103", 104 -> "xyz"),
+ MapType(IntegerType, StringType))
+ checkCast(
+ intMap,
+ MapType(DoubleType, IntegerType),
+ Map(100.0 -> 101, 102.0 -> 103, 104.0 -> null))
+ checkCast(
+ intMap,
+ MapType(StringType, StringType),
+ Map("100" -> "101", "102" -> "103", "104" -> "xyz"))
+
+ val floatMap = Literal.create(
+ Map(1.0f -> "2.0", -3.0f -> "40e-1", 5.0f -> "xyz"),
+ MapType(FloatType, StringType))
+ checkCast(
+ floatMap,
+ MapType(FloatType, FloatType),
+ Map(1.0f -> 2.0f, -3.0f -> 4.0f, 5.0f -> null))
+ checkCast(
+ floatMap,
+ MapType(StringType, StringType),
+ Map("1.0" -> "2.0", "-3.0" -> "40e-1", "5.0" -> "xyz"))
+
+ val timestampMap = Literal.create(
+ Map(
+ Timestamp.valueOf("2023-01-01 12:00:00") -> "2023-01-01 13:00:00",
+ Timestamp.valueOf("2023-01-02 12:00:00") -> "xyz"),
+ MapType(TimestampType, StringType)
+ )
+ checkCast(
+ timestampMap,
+ MapType(StringType, TimestampType),
+ Map(
+ "2023-01-01 12:00:00" -> Timestamp.valueOf("2023-01-01 13:00:00"),
+ "2023-01-02 12:00:00" -> null))
+
+ val dateMap = Literal.create(
+ Map(Date.valueOf("2024-01-01") -> "2024-01-02",
Date.valueOf("2024-02-01") -> "xyz"),
+ MapType(DateType, StringType))
+ checkCast(
+ dateMap,
+ MapType(StringType, DateType),
+ Map("2024-01-01" -> Date.valueOf("2024-01-02"), "2024-02-01" -> null))
+ }
+
+ test("cast structs") {
+ val struct = Literal.create(
+ Row(
+ Seq("123", "456.7", "2023-01-01 12:00:00", "2024-01-01"),
+ Map(1.toByte -> 2.toShort, 3.toByte -> 4.toShort),
+ Row(
+ 123.0,
+ 456.1f,
+ 0
+ )
+ ),
+ StructType(
+ Seq(
+ StructField("a", ArrayType(StringType)),
+ StructField("b", MapType(ByteType, ShortType)),
+ StructField(
+ "c",
+ StructType(
+ Seq(
+ StructField("x", DoubleType),
+ StructField("y", FloatType),
+ StructField("z", IntegerType)
+ ))
+ )
+ ))
+ )
+
+ checkCast(
+ struct,
+ StructType(
+ Seq(
+ StructField("a", ArrayType(LongType)),
+ StructField("b", MapType(ShortType, IntegerType)),
+ StructField(
+ "c",
+ StructType(
+ Seq(
+ StructField("x", StringType),
+ StructField("y", StringType),
+ StructField("z", BooleanType)
+ ))
+ )
+ )),
+ Row(
+ Seq(123L, 456L, null, null),
+ Map(1.toShort -> 2, 3.toShort -> 4),
+ Row(
+ "123.0",
+ "456.1",
+ false
+ )
+ )
+ )
+
+ checkCast(
+ struct,
+ StructType(
+ Seq(
+ StructField("a", ArrayType(DoubleType)),
+ StructField("b", MapType(IntegerType, LongType)),
+ StructField(
+ "c",
+ StructType(
+ Seq(
+ StructField("x", IntegerType),
+ StructField("y", BooleanType),
+ StructField("z", StringType)
+ ))
+ )
+ )),
+ Row(
+ Seq(123.0, 456.7, null, null),
+ Map(1 -> 2L, 3 -> 4L),
+ Row(
+ 123,
+ true,
+ "0"
+ )
+ )
+ )
+
+ checkCast(
+ struct,
+ StructType(
+ Seq(
+ StructField("a", ArrayType(TimestampType)),
+ StructField("b", MapType(LongType, BooleanType)),
+ StructField(
+ "c",
+ StructType(
+ Seq(
+ StructField("x", ByteType),
+ StructField("y", ShortType),
+ StructField("z", DoubleType)
+ ))
+ )
+ )),
+ Row(
+ Seq(
+ null,
+ null,
+ Timestamp.valueOf("2023-01-01 12:00:00"),
+ Timestamp.valueOf("2024-01-01 00:00:00")),
+ Map(1L -> true, 3L -> true),
+ Row(
+ 123.toByte,
+ 456.toShort,
+ 0.0
+ )
+ )
+ )
+
+ checkCast(
+ struct,
+ StructType(
+ Seq(
+ StructField("a", ArrayType(DateType)),
+ StructField("b", MapType(StringType, DoubleType)),
+ StructField(
+ "c",
+ StructType(
+ Seq(
+ StructField("x", IntegerType),
+ StructField("y", LongType),
+ StructField("z", FloatType)
+ ))
+ )
+ )),
+ Row(
+ Seq(null, null, Date.valueOf("2023-01-01"),
Date.valueOf("2024-01-01")),
+ Map("1" -> 2.0, "3" -> 4.0),
+ Row(
+ 123,
+ 456L,
+ 0.0f
+ )
+ )
+ )
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
