This is an automated email from the ASF dual-hosted git repository.
viirya pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new befabdc7 minor: stop warning that AQEShuffleRead cannot run natively
(#842)
befabdc7 is described below
commit befabdc72ea7e9394e960b678bfbf7897ea12ee0
Author: Andy Grove <[email protected]>
AuthorDate: Tue Aug 20 12:17:44 2024 -0600
minor: stop warning that AQEShuffleRead cannot run natively (#842)
* minor: stop warning that AQEShuffleRead cannot run natively
* address feedback
* revert a change
---
.../apache/comet/CometSparkSessionExtensions.scala | 19 ++++++++++---------
.../scala/org/apache/comet/CometExpressionSuite.scala | 4 +---
2 files changed, 11 insertions(+), 12 deletions(-)
diff --git
a/spark/src/main/scala/org/apache/comet/CometSparkSessionExtensions.scala
b/spark/src/main/scala/org/apache/comet/CometSparkSessionExtensions.scala
index 19d348ec..e5b0b0cd 100644
--- a/spark/src/main/scala/org/apache/comet/CometSparkSessionExtensions.scala
+++ b/spark/src/main/scala/org/apache/comet/CometSparkSessionExtensions.scala
@@ -502,7 +502,7 @@ class CometSparkSessionExtensions
withInfo(op, "ShuffleHashJoin is not enabled")
op
- case op: ShuffledHashJoinExec if !op.children.forall(isCometNative(_))
=>
+ case op: ShuffledHashJoinExec if !op.children.forall(isCometNative) =>
withInfo(
op,
"ShuffleHashJoin disabled because the following children are not
native " +
@@ -822,15 +822,16 @@ class CometSparkSessionExtensions
s
case op =>
- // An operator that is not supported by Comet
op match {
- // Broadcast exchange exec is transformed by the parent node. We
include
- // this case specially here so we do not add a misleading 'info'
message
- case _: BroadcastExchangeExec => op
- case _: CometExec | _: CometBroadcastExchangeExec | _:
CometShuffleExchangeExec => op
- case o =>
- withInfo(o, s"${o.nodeName} is not supported")
- o
+ case _: CometExec | _: AQEShuffleReadExec | _:
BroadcastExchangeExec |
+ _: CometBroadcastExchangeExec | _: CometShuffleExchangeExec =>
+ // Some execs should never be replaced. We include
+ // these cases specially here so we do not add a misleading
'info' message
+ op
+ case _ =>
+ // An operator that is not supported by Comet
+ withInfo(op, s"${op.nodeName} is not supported")
+ op
}
}
}
diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala
b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala
index 38945bbb..09d7ca97 100644
--- a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala
+++ b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala
@@ -1731,8 +1731,7 @@ class CometExpressionSuite extends CometTestBase with
AdaptiveSparkPlanHelper {
Set(
"HashAggregate is not native because the following children are
not native (AQEShuffleRead)",
"HashAggregate is not native because the following children are
not native (Exchange)",
- "Comet shuffle is not enabled: spark.comet.exec.shuffle.enabled
is not enabled",
- "AQEShuffleRead is not supported")),
+ "Comet shuffle is not enabled: spark.comet.exec.shuffle.enabled
is not enabled")),
(
"SELECT A.c1, A.sum_c0, A.sum_c2, B.casted from "
+ s"(SELECT c1, sum(c0) as sum_c0, sum(c2) as sum_c2 from $table
group by c1) as A, "
@@ -1740,7 +1739,6 @@ class CometExpressionSuite extends CometTestBase with
AdaptiveSparkPlanHelper {
+ "where A.c1 = B.c1 ",
Set(
"Comet shuffle is not enabled: spark.comet.exec.shuffle.enabled
is not enabled",
- "AQEShuffleRead is not supported",
"make_interval is not supported",
"HashAggregate is not native because the following children are
not native (AQEShuffleRead)",
"HashAggregate is not native because the following children are
not native (Exchange)",
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
