This is an automated email from the ASF dual-hosted git repository.
hongze pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new cd77ce17b3 [GLUTEN-7600][VL] Simplify offload rules in RAS (#7646)
cd77ce17b3 is described below
commit cd77ce17b3d7a5dbe733a22da087544aab2609cf
Author: Hongze Zhang <[email protected]>
AuthorDate: Wed Oct 23 12:57:12 2024 +0800
[GLUTEN-7600][VL] Simplify offload rules in RAS (#7646)
Closes #7600
---
.../extension/columnar/OffloadSingleNode.scala | 2 +-
.../columnar/enumerated/EnumeratedTransform.scala | 16 ++++------
.../columnar/enumerated/RasOffloadFilter.scala | 35 ----------------------
.../enumerated/RasOffloadHashAggregate.scala | 34 ---------------------
.../columnar/enumerated/RasOffloadProject.scala | 33 --------------------
5 files changed, 7 insertions(+), 113 deletions(-)
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala
index b34e83af70..ce5b212069 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala
@@ -201,7 +201,7 @@ object OffloadOthers {
// Do not look-up on children on the input node in this rule. Otherwise
// it may break RAS which would group all the possible input nodes to
// search for validate candidates.
- class ReplaceSingleNode() extends LogLevelUtil with Logging {
+ private class ReplaceSingleNode() extends LogLevelUtil with Logging {
def doReplace(p: SparkPlan): SparkPlan = {
val plan = p
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedTransform.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedTransform.scala
index cf6aeb42c8..ea9078df1e 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedTransform.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedTransform.scala
@@ -29,7 +29,7 @@ import org.apache.gluten.sql.shims.SparkShimLoader
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.catalyst.rules.Rule
import org.apache.spark.sql.execution._
-import org.apache.spark.sql.execution.aggregate.{ObjectHashAggregateExec,
SortAggregateExec}
+import org.apache.spark.sql.execution.aggregate.{HashAggregateExec,
ObjectHashAggregateExec, SortAggregateExec}
import org.apache.spark.sql.execution.datasources.WriteFilesExec
import org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanExecBase
import org.apache.spark.sql.execution.exchange.Exchange
@@ -62,16 +62,14 @@ case class EnumeratedTransform(session: SparkSession,
outputsColumnar: Boolean)
Seq(
RasOffload.from[Exchange](OffloadExchange()),
RasOffload.from[BaseJoinExec](OffloadJoin()),
- RasOffloadHashAggregate,
- RasOffloadFilter,
- RasOffloadProject,
+ RasOffload.from[FilterExec](OffloadOthers()),
+ RasOffload.from[ProjectExec](OffloadOthers()),
RasOffload.from[DataSourceV2ScanExecBase](OffloadOthers()),
RasOffload.from[DataSourceScanExec](OffloadOthers()),
RasOffload
- .from(
- (node: SparkPlan) =>
HiveTableScanExecTransformer.isHiveTableScan(node),
- OffloadOthers()),
+ .from(HiveTableScanExecTransformer.isHiveTableScan, OffloadOthers()),
RasOffload.from[CoalesceExec](OffloadOthers()),
+ RasOffload.from[HashAggregateExec](OffloadOthers()),
RasOffload.from[SortAggregateExec](OffloadOthers()),
RasOffload.from[ObjectHashAggregateExec](OffloadOthers()),
RasOffload.from[UnionExec](OffloadOthers()),
@@ -81,9 +79,7 @@ case class EnumeratedTransform(session: SparkSession,
outputsColumnar: Boolean)
RasOffload.from[TakeOrderedAndProjectExec](OffloadOthers()),
RasOffload.from[WindowExec](OffloadOthers()),
RasOffload
- .from(
- (node: SparkPlan) =>
SparkShimLoader.getSparkShims.isWindowGroupLimitExec(node),
- OffloadOthers()),
+ .from(SparkShimLoader.getSparkShims.isWindowGroupLimitExec,
OffloadOthers()),
RasOffload.from[LimitExec](OffloadOthers()),
RasOffload.from[GenerateExec](OffloadOthers()),
RasOffload.from[EvalPythonExec](OffloadOthers())
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffloadFilter.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffloadFilter.scala
deleted file mode 100644
index 54ab9158be..0000000000
---
a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffloadFilter.scala
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.gluten.extension.columnar.enumerated
-
-import org.apache.gluten.backendsapi.BackendsApiManager
-
-import org.apache.spark.sql.execution.{FilterExec, SparkPlan}
-
-object RasOffloadFilter extends RasOffload {
- override def offload(node: SparkPlan): SparkPlan = node match {
- case FilterExec(condition, child) =>
- val out = BackendsApiManager.getSparkPlanExecApiInstance
- .genFilterExecTransformer(condition, child)
- out
- case other =>
- other
- }
-
- override def typeIdentifier(): RasOffload.TypeIdentifier =
- RasOffload.TypeIdentifier.of[FilterExec]
-}
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffloadHashAggregate.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffloadHashAggregate.scala
deleted file mode 100644
index 272e05ca7b..0000000000
---
a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffloadHashAggregate.scala
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.gluten.extension.columnar.enumerated
-
-import org.apache.gluten.execution.HashAggregateExecBaseTransformer
-
-import org.apache.spark.sql.execution.SparkPlan
-import org.apache.spark.sql.execution.aggregate.HashAggregateExec
-
-object RasOffloadHashAggregate extends RasOffload {
- override def offload(node: SparkPlan): SparkPlan = node match {
- case agg: HashAggregateExec =>
- val out = HashAggregateExecBaseTransformer.from(agg)
- out
- case other => other
- }
-
- override def typeIdentifier(): RasOffload.TypeIdentifier =
- RasOffload.TypeIdentifier.of[HashAggregateExec]
-}
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffloadProject.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffloadProject.scala
deleted file mode 100644
index 0bbf57499b..0000000000
---
a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffloadProject.scala
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.gluten.extension.columnar.enumerated
-
-import org.apache.gluten.execution.ProjectExecTransformer
-
-import org.apache.spark.sql.execution.{ProjectExec, SparkPlan}
-
-object RasOffloadProject extends RasOffload {
- override def offload(node: SparkPlan): SparkPlan = node match {
- case ProjectExec(projectList, child) =>
- ProjectExecTransformer(projectList, child)
- case other =>
- other
- }
-
- override def typeIdentifier(): RasOffload.TypeIdentifier =
- RasOffload.TypeIdentifier.of[ProjectExec]
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
