This is an automated email from the ASF dual-hosted git repository.
viirya pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new 39e030b1 fix: Unsupported types for SinglePartition should fallback to
Spark (#765)
39e030b1 is described below
commit 39e030b16ba3fbafaa84d1a0e80b0858b61938df
Author: Liang-Chi Hsieh <[email protected]>
AuthorDate: Fri Aug 2 09:23:03 2024 -0700
fix: Unsupported types for SinglePartition should fallback to Spark (#765)
---
.../main/scala/org/apache/comet/serde/QueryPlanSerde.scala | 6 +++---
.../org/apache/comet/exec/CometColumnarShuffleSuite.scala | 13 +++++++++++++
2 files changed, 16 insertions(+), 3 deletions(-)
diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala
b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala
index 948c534d..f8f53ad2 100644
--- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala
+++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala
@@ -2931,8 +2931,8 @@ object QueryPlanSerde extends Logging with
ShimQueryPlanSerde with CometExprShim
msg = s"unsupported Spark partitioning expressions: $expressions"
}
supported
- case SinglePartition => true
- case RoundRobinPartitioning(_) => true
+ case SinglePartition => inputs.forall(attr =>
supportedDataType(attr.dataType))
+ case RoundRobinPartitioning(_) => inputs.forall(attr =>
supportedDataType(attr.dataType))
case RangePartitioning(orderings, _) =>
val supported =
orderings.map(QueryPlanSerde.exprToProto(_,
inputs)).forall(_.isDefined) &&
@@ -2980,7 +2980,7 @@ object QueryPlanSerde extends Logging with
ShimQueryPlanSerde with CometExprShim
msg = s"unsupported Spark partitioning expressions: $expressions"
}
supported
- case SinglePartition => true
+ case SinglePartition => inputs.forall(attr =>
supportedDataType(attr.dataType))
case _ =>
msg = s"unsupported Spark partitioning:
${partitioning.getClass.getName}"
false
diff --git
a/spark/src/test/scala/org/apache/comet/exec/CometColumnarShuffleSuite.scala
b/spark/src/test/scala/org/apache/comet/exec/CometColumnarShuffleSuite.scala
index c38be7c4..25f2a753 100644
--- a/spark/src/test/scala/org/apache/comet/exec/CometColumnarShuffleSuite.scala
+++ b/spark/src/test/scala/org/apache/comet/exec/CometColumnarShuffleSuite.scala
@@ -66,6 +66,19 @@ abstract class CometColumnarShuffleSuite extends
CometTestBase with AdaptiveSpar
setupTestData()
+ test("Unsupported types for SinglePartition should fallback to Spark") {
+ checkSparkAnswer(spark.sql("""
+ |SELECT
+ | AVG(null),
+ | COUNT(null),
+ | FIRST(null),
+ | LAST(null),
+ | MAX(null),
+ | MIN(null),
+ | SUM(null)
+ """.stripMargin))
+ }
+
test("Disable Comet shuffle with AQE coalesce partitions enabled") {
Seq(true, false).foreach { coalescePartitionsEnabled =>
withSQLConf(
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
