(incubator-gluten) branch main updated: [Core] minor change for eliminateProjectList (#5144)

[zhli]

This is an automated email from the ASF dual-hosted git repository.

zhli pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git


The following commit(s) were added to refs/heads/main by this push:
     new 471423d7f [Core] minor change for eliminateProjectList (#5144)
471423d7f is described below

commit 471423d7f3e8f6153f61be9de0d5d10dd5bc84e5
Author: Zhen Li <10524738+zhli1142...@users.noreply.github.com>
AuthorDate: Thu Mar 28 23:46:38 2024 +0800

    [Core] minor change for eliminateProjectList (#5144)
    
    [Core] minor change for eliminateProjectList.
---
 .../glutenproject/extension/columnar/PullOutPreProject.scala   | 10 +++++-----
 .../scala/io/glutenproject/utils/PullOutProjectHelper.scala    |  7 +++----
 2 files changed, 8 insertions(+), 9 deletions(-)

diff --git 
a/gluten-core/src/main/scala/io/glutenproject/extension/columnar/PullOutPreProject.scala
 
b/gluten-core/src/main/scala/io/glutenproject/extension/columnar/PullOutPreProject.scala
index 92d6c9fab..fcc63b72f 100644
--- 
a/gluten-core/src/main/scala/io/glutenproject/extension/columnar/PullOutPreProject.scala
+++ 
b/gluten-core/src/main/scala/io/glutenproject/extension/columnar/PullOutPreProject.scala
@@ -121,7 +121,7 @@ object PullOutPreProject extends Rule[SparkPlan] with 
PullOutProjectHelper {
       // post-projection, the additional columns need to be removed, leaving 
only the original
       // output of the child.
       val preProject = ProjectExec(
-        eliminateProjectList(sort.child.outputSet, expressionMap.values.toSeq),
+        eliminateProjectList(expressionMap.values.toSeq, sort.child.outputSet),
         sort.child)
       val newSort = sort.copy(sortOrder = newSortOrder, child = preProject)
       // The pre-project and post-project of SortExec always appear together, 
so it's more
@@ -133,7 +133,7 @@ object PullOutPreProject extends Rule[SparkPlan] with 
PullOutProjectHelper {
       val expressionMap = new mutable.HashMap[Expression, NamedExpression]()
       val newSortOrder = getNewSortOrder(topK.sortOrder, expressionMap)
       val preProject = ProjectExec(
-        eliminateProjectList(topK.child.outputSet, expressionMap.values.toSeq),
+        eliminateProjectList(expressionMap.values.toSeq, topK.child.outputSet),
         topK.child)
       topK.copy(sortOrder = newSortOrder, child = preProject)
 
@@ -152,7 +152,7 @@ object PullOutPreProject extends Rule[SparkPlan] with 
PullOutProjectHelper {
         newGroupingExpressions = newGroupingExpressions,
         newAggregateExpressions = newAggregateExpressions)
       val preProject = ProjectExec(
-        eliminateProjectList(agg.child.outputSet, expressionMap.values.toSeq),
+        eliminateProjectList(expressionMap.values.toSeq, agg.child.outputSet),
         agg.child)
       newAgg.withNewChildren(Seq(preProject))
 
@@ -175,7 +175,7 @@ object PullOutPreProject extends Rule[SparkPlan] with 
PullOutProjectHelper {
         partitionSpec = newPartitionSpec,
         windowExpression = 
newWindowExpressions.asInstanceOf[Seq[NamedExpression]],
         child = ProjectExec(
-          eliminateProjectList(window.child.outputSet, 
expressionMap.values.toSeq),
+          eliminateProjectList(expressionMap.values.toSeq, 
window.child.outputSet),
           window.child)
       )
 
@@ -188,7 +188,7 @@ object PullOutPreProject extends Rule[SparkPlan] with 
PullOutProjectHelper {
       expand.copy(
         projections = newProjections,
         child = ProjectExec(
-          eliminateProjectList(expand.child.outputSet, 
expressionMap.values.toSeq),
+          eliminateProjectList(expressionMap.values.toSeq, 
expand.child.outputSet),
           expand.child))
 
     case generate: GenerateExec =>
diff --git 
a/gluten-core/src/main/scala/io/glutenproject/utils/PullOutProjectHelper.scala 
b/gluten-core/src/main/scala/io/glutenproject/utils/PullOutProjectHelper.scala
index 543a5413c..e1fb5c3e6 100644
--- 
a/gluten-core/src/main/scala/io/glutenproject/utils/PullOutProjectHelper.scala
+++ 
b/gluten-core/src/main/scala/io/glutenproject/utils/PullOutProjectHelper.scala
@@ -71,10 +71,9 @@ trait PullOutProjectHelper {
     }
 
   protected def eliminateProjectList(
-      childOutput: AttributeSet,
-      appendAttributes: Seq[NamedExpression]): Seq[NamedExpression] = {
-    childOutput.toIndexedSeq ++ appendAttributes
-      .filter(attr => !childOutput.contains(attr))
+      projectAttributes: Seq[NamedExpression],
+      childOutput: AttributeSet): Seq[NamedExpression] = {
+    (childOutput -- projectAttributes).toIndexedSeq ++ projectAttributes
       .sortWith(_.exprId.id < _.exprId.id)
   }
 


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@gluten.apache.org
For additional commands, e-mail: commits-h...@gluten.apache.org