This is an automated email from the ASF dual-hosted git repository.
ulyssesyou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 95dcdbdbd [CORE] Rename CoalesceExecTransformer to
ColumnarCoalesceExec (#6000)
95dcdbdbd is described below
commit 95dcdbdbd9f4f17de40884b74a65564bba2a9bf9
Author: Xiduo You <[email protected]>
AuthorDate: Thu Jun 6 13:01:25 2024 +0800
[CORE] Rename CoalesceExecTransformer to ColumnarCoalesceExec (#6000)
---
...esceExecTransformer.scala => ColumnarCoalesceExec.scala} | 13 +++++--------
.../gluten/extension/columnar/OffloadSingleNode.scala | 2 +-
.../gluten/extension/columnar/TransformHintRule.scala | 5 +++--
3 files changed, 9 insertions(+), 11 deletions(-)
diff --git
a/gluten-core/src/main/scala/org/apache/gluten/execution/CoalesceExecTransformer.scala
b/gluten-core/src/main/scala/org/apache/gluten/execution/ColumnarCoalesceExec.scala
similarity index 87%
rename from
gluten-core/src/main/scala/org/apache/gluten/execution/CoalesceExecTransformer.scala
rename to
gluten-core/src/main/scala/org/apache/gluten/execution/ColumnarCoalesceExec.scala
index 8f30805be..f40a7f8f0 100644
---
a/gluten-core/src/main/scala/org/apache/gluten/execution/CoalesceExecTransformer.scala
+++
b/gluten-core/src/main/scala/org/apache/gluten/execution/ColumnarCoalesceExec.scala
@@ -16,7 +16,7 @@
*/
package org.apache.gluten.execution
-import org.apache.gluten.extension.{GlutenPlan, ValidationResult}
+import org.apache.gluten.extension.GlutenPlan
import org.apache.spark.{Partition, SparkContext, TaskContext}
import org.apache.spark.rdd.RDD
@@ -26,7 +26,7 @@ import
org.apache.spark.sql.catalyst.plans.physical.{Partitioning, SinglePartiti
import org.apache.spark.sql.execution.{SparkPlan, UnaryExecNode}
import org.apache.spark.sql.vectorized.ColumnarBatch
-case class CoalesceExecTransformer(numPartitions: Int, child: SparkPlan)
+case class ColumnarCoalesceExec(numPartitions: Int, child: SparkPlan)
extends UnaryExecNode
with GlutenPlan {
@@ -38,9 +38,6 @@ case class CoalesceExecTransformer(numPartitions: Int, child:
SparkPlan)
if (numPartitions == 1) SinglePartition else
UnknownPartitioning(numPartitions)
}
- override protected def doValidateInternal(): ValidationResult =
- ValidationResult.ok
-
override protected def doExecute(): RDD[InternalRow] = {
throw new UnsupportedOperationException()
}
@@ -49,18 +46,18 @@ case class CoalesceExecTransformer(numPartitions: Int,
child: SparkPlan)
if (numPartitions == 1 && child.executeColumnar().getNumPartitions < 1) {
// Make sure we don't output an RDD with 0 partitions, when claiming
that we have a
// `SinglePartition`.
- new CoalesceExecTransformer.EmptyRDDWithPartitions(sparkContext,
numPartitions)
+ new ColumnarCoalesceExec.EmptyRDDWithPartitions(sparkContext,
numPartitions)
} else {
child.executeColumnar().coalesce(numPartitions, shuffle = false)
}
}
- override protected def withNewChildInternal(newChild: SparkPlan):
CoalesceExecTransformer =
+ override protected def withNewChildInternal(newChild: SparkPlan):
ColumnarCoalesceExec =
copy(child = newChild)
}
-object CoalesceExecTransformer {
+object ColumnarCoalesceExec {
class EmptyRDDWithPartitions(@transient private val sc: SparkContext,
numPartitions: Int)
extends RDD[ColumnarBatch](sc, Nil) {
diff --git
a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala
b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala
index b82254072..39cc8ad2e 100644
---
a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala
+++
b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala
@@ -281,7 +281,7 @@ object OffloadOthers {
applyScanTransformer(plan)
case plan: CoalesceExec =>
logDebug(s"Columnar Processing for ${plan.getClass} is currently
supported.")
- CoalesceExecTransformer(plan.numPartitions, plan.child)
+ ColumnarCoalesceExec(plan.numPartitions, plan.child)
case plan: ProjectExec =>
val columnarChild = plan.child
logDebug(s"Columnar Processing for ${plan.getClass} is currently
supported.")
diff --git
a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/TransformHintRule.scala
b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/TransformHintRule.scala
index 1da0c0db8..ca35c74f6 100644
---
a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/TransformHintRule.scala
+++
b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/TransformHintRule.scala
@@ -487,8 +487,9 @@ case class AddTransformHintRule() extends Rule[SparkPlan] {
)
transformer.doValidate().tagOnFallback(plan)
case plan: CoalesceExec =>
- val transformer = CoalesceExecTransformer(plan.numPartitions,
plan.child)
- transformer.doValidate().tagOnFallback(plan)
+ ColumnarCoalesceExec(plan.numPartitions, plan.child)
+ .doValidate()
+ .tagOnFallback(plan)
case plan: GlobalLimitExec =>
val (limit, offset) =
SparkShimLoader.getSparkShims.getLimitAndOffsetFromGlobalLimit(plan)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
