This is an automated email from the ASF dual-hosted git repository.
jackylee-ch pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new b1e00723b0 [MINOR][CORE] Remove residual Spark 3.2 compatibility code
in gluten-core (#12522)
b1e00723b0 is described below
commit b1e00723b0f131a16e271e31673cd0a371d8a32a
Author: YangJie <[email protected]>
AuthorDate: Thu Jul 16 09:23:58 2026 +0800
[MINOR][CORE] Remove residual Spark 3.2 compatibility code in gluten-core
(#12522)
---
.../main/scala/org/apache/gluten/execution/GlutenPlan.scala | 13 +------------
.../gluten/extension/columnar/transition/package.scala | 4 +---
.../main/scala/org/apache/spark/util/SparkPlanUtil.scala | 3 +--
.../main/scala/org/apache/spark/util/SparkVersionUtil.scala | 1 -
.../sql/execution/ColumnarCollapseTransformStages.scala | 1 -
5 files changed, 3 insertions(+), 19 deletions(-)
diff --git
a/gluten-core/src/main/scala/org/apache/gluten/execution/GlutenPlan.scala
b/gluten-core/src/main/scala/org/apache/gluten/execution/GlutenPlan.scala
index 010a5963ec..296e90a01c 100644
--- a/gluten-core/src/main/scala/org/apache/gluten/execution/GlutenPlan.scala
+++ b/gluten-core/src/main/scala/org/apache/gluten/execution/GlutenPlan.scala
@@ -16,7 +16,6 @@
*/
package org.apache.gluten.execution
-import org.apache.gluten.exception.GlutenException
import org.apache.gluten.extension.columnar.transition.{Convention,
ConventionReq}
import org.apache.spark.sql.execution.SparkPlan
@@ -26,7 +25,7 @@ import org.apache.spark.sql.execution.SparkPlan
*
* The following Spark APIs are marked final so forbidden from overriding:
* - supportsColumnar
- * - supportsRowBased (Spark version >= 3.3)
+ * - supportsRowBased
*
* Instead, subclasses are expected to implement the following APIs:
* - batchType
@@ -46,7 +45,6 @@ trait GlutenPlan
extends SparkPlan
with Convention.KnownBatchType
with Convention.KnownRowType
- with GlutenPlan.SupportsRowBasedCompatible
with ConventionReq.KnownChildConvention {
final override val supportsColumnar: Boolean = {
@@ -77,12 +75,3 @@ trait GlutenPlan
})
}
}
-
-object GlutenPlan {
- // To be compatible with Spark (version < 3.3)
- trait SupportsRowBasedCompatible {
- def supportsRowBased(): Boolean = {
- throw new GlutenException("Illegal state: The method is not expected to
be called")
- }
- }
-}
diff --git
a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/package.scala
b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/package.scala
index dbec061b15..e0eb01fc27 100644
---
a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/package.scala
+++
b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/package.scala
@@ -24,7 +24,6 @@ import org.apache.spark.sql.catalyst.expressions.Attribute
import org.apache.spark.sql.execution._
import org.apache.spark.sql.execution.adaptive.AQEShuffleReadExec
import org.apache.spark.sql.execution.debug.DebugExec
-import org.apache.spark.util.SparkVersionUtil
package object transition {
@@ -35,8 +34,7 @@ package object transition {
// Extend this list in shim layer once Spark has more.
def canPropagateConvention(plan: SparkPlan): Boolean = plan match {
case p: DebugExec => true
- case p: UnionExec if SparkVersionUtil.gteSpark33 =>
- true
+ case p: UnionExec => true
case p: AQEShuffleReadExec => true
case p: InputAdapter => true
case p: WholeStageCodegenExec => true
diff --git
a/gluten-core/src/main/scala/org/apache/spark/util/SparkPlanUtil.scala
b/gluten-core/src/main/scala/org/apache/spark/util/SparkPlanUtil.scala
index a7a6cdaf3d..5ce14f0904 100644
--- a/gluten-core/src/main/scala/org/apache/spark/util/SparkPlanUtil.scala
+++ b/gluten-core/src/main/scala/org/apache/spark/util/SparkPlanUtil.scala
@@ -23,8 +23,7 @@ import org.apache.spark.sql.internal.SQLConf
object SparkPlanUtil {
def supportsRowBased(plan: SparkPlan): Boolean = {
- val m = classOf[SparkPlan].getMethod("supportsRowBased")
- m.invoke(plan).asInstanceOf[Boolean]
+ plan.supportsRowBased
}
def isPlannedV1Write(plan: DataWritingCommandExec): Boolean = {
diff --git
a/gluten-core/src/main/scala/org/apache/spark/util/SparkVersionUtil.scala
b/gluten-core/src/main/scala/org/apache/spark/util/SparkVersionUtil.scala
index b9089363b5..97d428515c 100644
--- a/gluten-core/src/main/scala/org/apache/spark/util/SparkVersionUtil.scala
+++ b/gluten-core/src/main/scala/org/apache/spark/util/SparkVersionUtil.scala
@@ -20,7 +20,6 @@ object SparkVersionUtil {
private val comparedWithSpark33 = compareMajorMinorVersion((3, 3))
private val comparedWithSpark35 = compareMajorMinorVersion((3, 5))
val eqSpark33: Boolean = comparedWithSpark33 == 0
- val gteSpark33: Boolean = comparedWithSpark33 >= 0
val gteSpark35: Boolean = comparedWithSpark35 >= 0
val gteSpark40: Boolean = compareMajorMinorVersion((4, 0)) >= 0
val gteSpark41: Boolean = compareMajorMinorVersion((4, 1)) >= 0
diff --git
a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarCollapseTransformStages.scala
b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarCollapseTransformStages.scala
index f35aea8fdc..cf9fcdfd97 100644
---
a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarCollapseTransformStages.scala
+++
b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarCollapseTransformStages.scala
@@ -220,7 +220,6 @@ case class ColumnarInputAdapter(child: SparkPlan)
extends InputAdapterGenerateTreeStringShim
with Convention.KnownBatchType
with Convention.KnownRowType
- with GlutenPlan.SupportsRowBasedCompatible
with ConventionReq.KnownChildConvention {
override def output: Seq[Attribute] = child.output
final override val supportsColumnar: Boolean = true
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
