This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new b6c6e264d [CORE] Unify the transforming for shuffle expression (#5793)
b6c6e264d is described below
commit b6c6e264d90efc1ff268f35fad86d88cd3957465
Author: exmy <[email protected]>
AuthorDate: Fri May 17 20:47:11 2024 +0800
[CORE] Unify the transforming for shuffle expression (#5793)
---
.../gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala | 10 ----------
.../org/apache/gluten/backendsapi/SparkPlanExecApi.scala | 7 -------
.../org/apache/gluten/expression/ExpressionConverter.scala | 11 +++++------
3 files changed, 5 insertions(+), 23 deletions(-)
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
index f54bf9b3f..cf7d38d62 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
@@ -145,16 +145,6 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi {
original)
}
- override def genShuffleTransformer(
- substraitExprName: String,
- child: ExpressionTransformer,
- original: Shuffle): ExpressionTransformer = {
- GenericExpressionTransformer(
- substraitExprName,
- Seq(child, LiteralTransformer(Literal(original.randomSeed.get))),
- original)
- }
-
override def genTryAddTransformer(
substraitExprName: String,
left: ExpressionTransformer,
diff --git
a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala
b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala
index aa27d1ce1..c694d5804 100644
---
a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala
+++
b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala
@@ -212,13 +212,6 @@ trait SparkPlanExecApi {
GenericExpressionTransformer(substraitExprName, Seq(), original)
}
- def genShuffleTransformer(
- substraitExprName: String,
- child: ExpressionTransformer,
- original: Shuffle): ExpressionTransformer = {
- GenericExpressionTransformer(substraitExprName, Seq(child), original)
- }
-
def genTryAddTransformer(
substraitExprName: String,
left: ExpressionTransformer,
diff --git
a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala
b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala
index e692890c4..5aacbed05 100644
---
a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala
+++
b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala
@@ -627,7 +627,6 @@ object ExpressionConverter extends SQLConfHelper with
Logging {
replaceWithExpressionTransformerInternal(a.function, attributeSeq,
expressionsMap),
a
)
-
case a: ArrayExists =>
BackendsApiManager.getSparkPlanExecApiInstance.genArrayExistsTransformer(
substraitExprName,
@@ -635,13 +634,13 @@ object ExpressionConverter extends SQLConfHelper with
Logging {
replaceWithExpressionTransformerInternal(a.function, attributeSeq,
expressionsMap),
a
)
-
case s: Shuffle =>
- BackendsApiManager.getSparkPlanExecApiInstance.genShuffleTransformer(
+ GenericExpressionTransformer(
substraitExprName,
- replaceWithExpressionTransformerInternal(s.child, attributeSeq,
expressionsMap),
- s
- )
+ Seq(
+ replaceWithExpressionTransformerInternal(s.child, attributeSeq,
expressionsMap),
+ LiteralTransformer(Literal(s.randomSeed.get))),
+ s)
case expr =>
GenericExpressionTransformer(
substraitExprName,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
