This is an automated email from the ASF dual-hosted git repository.
andygrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new 20279ba514 feat: remove constraint on array of nested elements (#4714)
20279ba514 is described below
commit 20279ba51449246d48ed82af02f752531d8637ed
Author: hsiang-c <[email protected]>
AuthorDate: Thu Jul 9 13:19:39 2026 -0700
feat: remove constraint on array of nested elements (#4714)
---
spark/src/main/scala/org/apache/comet/serde/arrays.scala | 16 ++++------------
.../resources/sql-tests/expressions/array/sort_array.sql | 6 ------
.../misc/codegen_dispatch_disabled_fallback.sql | 12 ------------
3 files changed, 4 insertions(+), 30 deletions(-)
diff --git a/spark/src/main/scala/org/apache/comet/serde/arrays.scala
b/spark/src/main/scala/org/apache/comet/serde/arrays.scala
index 1b39f5cd1f..7039d3278b 100644
--- a/spark/src/main/scala/org/apache/comet/serde/arrays.scala
+++ b/spark/src/main/scala/org/apache/comet/serde/arrays.scala
@@ -120,21 +120,13 @@ object CometSortArray extends
CometExpressionSerde[SortArray] with CodegenDispat
"When `" + CometConf.COMET_EXEC_STRICT_FLOATING_POINT.key + "=true`,
sorting on" +
" floating-point types is not 100% compatible with Spark")
- override def getUnsupportedReasons(): Seq[String] = Seq(
- "Nested arrays with `Struct` or `Null` child values are not supported
natively")
-
- private def supportedSortArrayElementType(
- dt: DataType,
- nestedInArray: Boolean = false): Boolean = {
+ private def supportedSortArrayElementType(dt: DataType): Boolean = {
dt match {
- // DataFusion's array_sort compares nested arrays through Arrow's rank
kernel.
- // That kernel does not support Struct or Null child values,
- // so array<array<struct<...>>> and array<array<null>> would fail at
runtime.
- case _: NullType if !nestedInArray =>
+ case _: NullType =>
true
case ArrayType(elementType, _) =>
- supportedSortArrayElementType(elementType, nestedInArray = true)
- case StructType(fields) if !nestedInArray =>
+ supportedSortArrayElementType(elementType)
+ case StructType(fields) =>
fields.forall(f => supportedSortArrayElementType(f.dataType))
case _ =>
supportedScalarSortElementType(dt)
diff --git
a/spark/src/test/resources/sql-tests/expressions/array/sort_array.sql
b/spark/src/test/resources/sql-tests/expressions/array/sort_array.sql
index dc9e7ec85b..ceb29e5a39 100644
--- a/spark/src/test/resources/sql-tests/expressions/array/sort_array.sql
+++ b/spark/src/test/resources/sql-tests/expressions/array/sort_array.sql
@@ -295,9 +295,6 @@ INSERT INTO test_sort_array_nested_struct VALUES
(array()),
(NULL)
--- SortArray mixes in CodegenDispatchFallback, so nested arrays with Struct
children have no
--- native path but route through the JVM codegen dispatcher (Spark's own
SortArray.doGenCode
--- inside the Comet pipeline) and stay native while matching Spark exactly.
query
SELECT sort_array(arr) FROM test_sort_array_nested_struct
@@ -397,9 +394,6 @@ SELECT
sort_array(array(NULL, NULL)),
sort_array(cast(NULL as array<int>))
--- nested arrays with Struct children have no native path but route through
the JVM codegen
--- dispatcher (Spark's own SortArray.doGenCode inside the Comet pipeline) and
stay native while
--- matching Spark exactly.
query
SELECT sort_array(
array(
diff --git
a/spark/src/test/resources/sql-tests/expressions/misc/codegen_dispatch_disabled_fallback.sql
b/spark/src/test/resources/sql-tests/expressions/misc/codegen_dispatch_disabled_fallback.sql
index eebf40bff1..d6068279e8 100644
---
a/spark/src/test/resources/sql-tests/expressions/misc/codegen_dispatch_disabled_fallback.sql
+++
b/spark/src/test/resources/sql-tests/expressions/misc/codegen_dispatch_disabled_fallback.sql
@@ -44,18 +44,6 @@ INSERT INTO test_dispatch_disabled_binary VALUES
(unhex('0A0B'), unhex('0C0D')),
query expect_fallback(spark.comet.exec.scalaUDF.codegen.enabled=false)
SELECT concat(c1, c2) FROM test_dispatch_disabled_binary
-statement
-CREATE TABLE test_dispatch_disabled_nested(arr array<array<struct<a:int>>>)
USING parquet
-
-statement
-INSERT INTO test_dispatch_disabled_nested VALUES
- (array(array(named_struct('a', 2)), array(named_struct('a', 1)))),
- (NULL)
-
--- SortArray over nested arrays with Struct children
-query expect_fallback(spark.comet.exec.scalaUDF.codegen.enabled=false)
-SELECT sort_array(arr) FROM test_dispatch_disabled_nested
-
statement
CREATE TABLE test_dispatch_disabled_date(d date) USING parquet
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
