This is an automated email from the ASF dual-hosted git repository.
comphead pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new c7cee9bf0 test: enable nested array cast coverage (#4278)
c7cee9bf0 is described below
commit c7cee9bf06a3c9fb7618458cb2d1c7cac7c30eed
Author: Manu Zhang <[email protected]>
AuthorDate: Thu May 21 02:22:17 2026 +0800
test: enable nested array cast coverage (#4278)
---
.../scala/org/apache/comet/CometCastSuite.scala | 43 ++++++++++++++++++----
1 file changed, 36 insertions(+), 7 deletions(-)
diff --git a/spark/src/test/scala/org/apache/comet/CometCastSuite.scala
b/spark/src/test/scala/org/apache/comet/CometCastSuite.scala
index b1115be2d..4968809c2 100644
--- a/spark/src/test/scala/org/apache/comet/CometCastSuite.scala
+++ b/spark/src/test/scala/org/apache/comet/CometCastSuite.scala
@@ -1590,7 +1590,7 @@ class CometCastSuite extends CometTestBase with
AdaptiveSparkPlanHelper {
}
// https://github.com/apache/datafusion-comet/issues/3906
- ignore("cast nested ArrayType to nested ArrayType") {
+ test("cast nested ArrayType to nested ArrayType") {
val types = Seq(
BooleanType,
StringType,
@@ -1601,14 +1601,12 @@ class CometCastSuite extends CometTestBase with
AdaptiveSparkPlanHelper {
FloatType,
DoubleType,
DecimalType(10, 2),
- DecimalType(38, 18),
+ // DecimalType(38, 18) is excluded for the same reason as the
one-dimensional array
+ // matrix: decimal-to-float/double casts can differ by ~1 ULP from Spark.
DateType,
TimestampType,
BinaryType)
- testArrayCastMatrix(
- types,
- dt => ArrayType(ArrayType(dt)),
- dt => generateArrays(100, ArrayType(dt)))
+ testArrayCastMatrix(types, dt => ArrayType(ArrayType(dt)), dt =>
generateNestedArrays(20, dt))
}
// CAST from TimestampNTZType
@@ -1716,7 +1714,7 @@ class CometCastSuite extends CometTestBase with
AdaptiveSparkPlanHelper {
}
private def generateArrays(rowNum: Int, elementType: DataType): DataFrame = {
- import scala.collection.JavaConverters._
+ import scala.jdk.CollectionConverters._
val schema = StructType(Seq(StructField("a", ArrayType(elementType),
true)))
def buildRows(values: Seq[Any]): Seq[Row] = {
Range(0, rowNum).map { i =>
@@ -1770,6 +1768,37 @@ class CometCastSuite extends CometTestBase with
AdaptiveSparkPlanHelper {
}
}
+ private def generateNestedArrays(rowNum: Int, elementType: DataType):
DataFrame = {
+ import scala.jdk.CollectionConverters._
+ val schema = StructType(Seq(StructField("a",
ArrayType(ArrayType(elementType)), true)))
+ val innerArrays = generateArrays(rowNum, elementType)
+ .collect()
+ .map { row =>
+ if (row.isNullAt(0)) {
+ null
+ } else {
+ row.getSeq[Any](0)
+ }
+ }
+ .toSeq
+
+ def buildRows(values: Seq[Seq[Any]]): Seq[Row] = {
+ Range(0, rowNum).map { i =>
+ Row(
+ Seq[Any](
+ values(i % values.length),
+ // Keep every third row's middle nested-array element null.
+ if (i % 3 == 0) null else values((i + 1) % values.length),
+ values((i + 2) % values.length)))
+ }
+ }
+
+ val sampleValue = innerArrays.find(_ != null).orNull
+ val rows = Seq(Row(Seq(sampleValue, null, sampleValue)),
Row(Seq.empty[Any]), Row(null)) ++
+ buildRows(innerArrays)
+ spark.createDataFrame(rows.asJava, schema)
+ }
+
// https://github.com/apache/datafusion-comet/issues/2038
test("test implicit cast to dictionary with case when and dictionary type") {
withSQLConf("parquet.enable.dictionary" -> "true") {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
