This is an automated email from the ASF dual-hosted git repository.
felixybw pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 492b4ebd4d [GLUTEN-10334] Share the results compare the spark version
(#10335)
492b4ebd4d is described below
commit 492b4ebd4d4501118f278d077579718d1c2fb571
Author: Jiaan Geng <[email protected]>
AuthorDate: Wed Aug 6 13:31:49 2025 +0800
[GLUTEN-10334] Share the results compare the spark version (#10335)
This PR proposes to share the results compare the spark version.
---
.../apache/gluten/extension/columnar/transition/Convention.scala | 9 +--------
.../apache/gluten/extension/columnar/transition/package.scala | 5 +----
.../spark/sql/execution/adaptive/GlutenCostEvaluator.scala | 5 +----
.../src/main/scala/org/apache/spark/util/SparkPlanUtil.scala | 6 ++----
.../src/main/scala/org/apache/spark/util/SparkTaskUtil.scala | 4 ++--
.../src/main/scala/org/apache/spark/util/SparkVersionUtil.scala | 6 ++++++
6 files changed, 13 insertions(+), 22 deletions(-)
diff --git
a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/Convention.scala
b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/Convention.scala
index 8ab66bf561..a00e2b5106 100644
---
a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/Convention.scala
+++
b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/Convention.scala
@@ -169,10 +169,9 @@ object Convention {
trait KnownRowTypeForSpark33OrLater extends KnownRowType {
this: SparkPlan =>
- import KnownRowTypeForSpark33OrLater._
final override def rowType(): RowType = {
- if (lteSpark32) {
+ if (SparkVersionUtil.lteSpark32) {
// It's known that in Spark 3.2, one Spark plan node is considered
either only having
// row-based support or only having columnar support at a time.
// Hence, if the plan supports columnar output, we'd disable its
row-based support.
@@ -190,10 +189,4 @@ object Convention {
def rowType0(): RowType
}
-
- object KnownRowTypeForSpark33OrLater {
- private val lteSpark32: Boolean = {
- SparkVersionUtil.compareMajorMinorVersion((3, 2)) <= 0
- }
- }
}
diff --git
a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/package.scala
b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/package.scala
index 56f3cc612f..a10c41cbca 100644
---
a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/package.scala
+++
b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/package.scala
@@ -27,9 +27,6 @@ import org.apache.spark.sql.execution.debug.DebugExec
import org.apache.spark.util.SparkVersionUtil
package object transition {
- private val gteSpark33: Boolean = {
- SparkVersionUtil.compareMajorMinorVersion((3, 3)) >= 0
- }
type TransitionGraph = FloydWarshallGraph[TransitionGraph.Vertex, Transition]
// These 5 plan operators (as of Spark 3.5) are operators that have the
@@ -38,7 +35,7 @@ package object transition {
// Extend this list in shim layer once Spark has more.
def canPropagateConvention(plan: SparkPlan): Boolean = plan match {
case p: DebugExec => true
- case p: UnionExec if gteSpark33 =>
+ case p: UnionExec if SparkVersionUtil.gteSpark33 =>
true
case p: AQEShuffleReadExec => true
case p: InputAdapter => true
diff --git
a/gluten-core/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala
b/gluten-core/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala
index e4204770f4..f53a80639a 100644
---
a/gluten-core/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala
+++
b/gluten-core/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala
@@ -28,12 +28,9 @@ import org.apache.spark.util.{SparkVersionUtil, Utils}
* Since https://github.com/apache/incubator-gluten/pull/6143.
*/
case class GlutenCostEvaluator() extends CostEvaluator with SQLConfHelper {
- private val ltSpark33: Boolean = {
- SparkVersionUtil.compareMajorMinorVersion((3, 3)) < 0
- }
private val vanillaCostEvaluator: CostEvaluator = {
- if (ltSpark33) {
+ if (SparkVersionUtil.lteSpark32) {
val clazz =
Utils.classForName("org.apache.spark.sql.execution.adaptive.SimpleCostEvaluator$")
clazz.getDeclaredField("MODULE$").get(null).asInstanceOf[CostEvaluator]
} else {
diff --git
a/gluten-core/src/main/scala/org/apache/spark/util/SparkPlanUtil.scala
b/gluten-core/src/main/scala/org/apache/spark/util/SparkPlanUtil.scala
index 4f5c213e5a..5ce2addafb 100644
--- a/gluten-core/src/main/scala/org/apache/spark/util/SparkPlanUtil.scala
+++ b/gluten-core/src/main/scala/org/apache/spark/util/SparkPlanUtil.scala
@@ -21,11 +21,9 @@ import
org.apache.spark.sql.execution.command.DataWritingCommandExec
import org.apache.spark.sql.internal.SQLConf
object SparkPlanUtil {
- private val lteSpark32 = SparkVersionUtil.compareMajorMinorVersion((3, 2))
<= 0
- private val lteSpark33 = SparkVersionUtil.compareMajorMinorVersion((3, 3))
<= 0
def supportsRowBased(plan: SparkPlan): Boolean = {
- if (lteSpark32) {
+ if (SparkVersionUtil.lteSpark32) {
return !plan.supportsColumnar
}
@@ -34,7 +32,7 @@ object SparkPlanUtil {
}
def isPlannedV1Write(plan: DataWritingCommandExec): Boolean = {
- if (lteSpark33) {
+ if (SparkVersionUtil.lteSpark33) {
return false
}
diff --git
a/gluten-core/src/main/scala/org/apache/spark/util/SparkTaskUtil.scala
b/gluten-core/src/main/scala/org/apache/spark/util/SparkTaskUtil.scala
index 02df5e224f..afbc03dd12 100644
--- a/gluten-core/src/main/scala/org/apache/spark/util/SparkTaskUtil.scala
+++ b/gluten-core/src/main/scala/org/apache/spark/util/SparkTaskUtil.scala
@@ -64,7 +64,7 @@ object SparkTaskUtil {
ctors.head
}
- if (SparkVersionUtil.compareMajorMinorVersion((3, 2)) <= 0) {
+ if (SparkVersionUtil.lteSpark32) {
return ctor
.newInstance(
stageId,
@@ -81,7 +81,7 @@ object SparkTaskUtil {
.asInstanceOf[TaskContext]
}
- if (SparkVersionUtil.compareMajorMinorVersion((3, 3)) == 0) {
+ if (SparkVersionUtil.eqSpark33) {
return ctor
.newInstance(
stageId,
diff --git
a/gluten-core/src/main/scala/org/apache/spark/util/SparkVersionUtil.scala
b/gluten-core/src/main/scala/org/apache/spark/util/SparkVersionUtil.scala
index d1febeaa9c..49da44f257 100644
--- a/gluten-core/src/main/scala/org/apache/spark/util/SparkVersionUtil.scala
+++ b/gluten-core/src/main/scala/org/apache/spark/util/SparkVersionUtil.scala
@@ -17,6 +17,12 @@
package org.apache.spark.util
object SparkVersionUtil {
+ val lteSpark32: Boolean = compareMajorMinorVersion((3, 2)) <= 0
+ private val comparedWithSpark33 = compareMajorMinorVersion((3, 3))
+ val eqSpark33: Boolean = comparedWithSpark33 == 0
+ val lteSpark33: Boolean = lteSpark32 || eqSpark33
+ val gteSpark33: Boolean = comparedWithSpark33 >= 0
+
// Returns X. X < 0 if one < other, x == 0 if one == other, x > 0 if one >
other.
def compareMajorMinorVersion(other: (Int, Int)): Int = {
val (major, minor) =
VersionUtils.majorMinorVersion(org.apache.spark.SPARK_VERSION)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
