This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 5c8e55a251 [CORE] Remove remaining Spark 3.2 compatibility code
(#11731)
5c8e55a251 is described below
commit 5c8e55a2510f232f4e4cc2d4fa211fbb80ee9c3a
Author: Shannon <[email protected]>
AuthorDate: Thu Mar 12 10:01:59 2026 -0700
[CORE] Remove remaining Spark 3.2 compatibility code (#11731)
---
.../execution/DeletionVectorWriteTransformer.scala | 2 +-
.../sql/perf/DeltaOptimizedWriterTransformer.scala | 2 +-
.../execution/CHColumnarToCarrierRowExec.scala | 2 +-
.../perf/GlutenDeltaOptimizedWriterExec.scala | 2 +-
.../delta/stats/GlutenDeltaJobStatsTracker.scala | 2 +-
.../execution/ColumnarPartialGenerateExec.scala | 2 +-
.../execution/ColumnarPartialProjectExec.scala | 2 +-
.../GpuResizeBufferColumnarBatchExec.scala | 2 +-
.../execution/VeloxColumnarToCarrierRowExec.scala | 2 +-
.../gluten/execution/VeloxResizeBatchesExec.scala | 2 +-
.../api/python/ColumnarArrowEvalPythonExec.scala | 2 +-
.../spark/sql/execution/BaseArrowScanExec.scala | 2 +-
.../gluten/execution/MiscOperatorSuite.scala | 18 +++-------
.../GlutenColumnarToColumnarTransition.scala | 2 +-
.../org/apache/gluten/execution/GlutenPlan.scala | 6 ++--
.../enumerated/planner/plan/GroupLeafExec.scala | 6 ++--
.../columnar/offload/OffloadSingleNode.scala | 2 +-
.../extension/columnar/transition/Convention.scala | 26 +-------------
.../execution/adaptive/GlutenCostEvaluator.scala | 17 ++++-----
.../org/apache/spark/util/SparkPlanUtil.scala | 6 +---
.../org/apache/spark/util/SparkTaskUtil.scala | 17 ---------
.../org/apache/spark/util/SparkVersionUtil.scala | 2 --
.../BasicPhysicalOperatorTransformer.scala | 2 +-
.../CartesianProductExecTransformer.scala | 2 +-
.../gluten/execution/ColumnarCoalesceExec.scala | 2 +-
.../execution/ColumnarCollectLimitBaseExec.scala | 2 +-
.../execution/ColumnarCollectTailBaseExec.scala | 2 +-
.../gluten/execution/ColumnarToRowExecBase.scala | 2 +-
.../execution/ColumnarV2TableWriteExec.scala | 2 +-
.../gluten/execution/RowToColumnarExecBase.scala | 2 +-
.../TakeOrderedAndProjectExecTransformer.scala | 2 +-
.../gluten/execution/WholeStageTransformer.scala | 2 +-
.../sql/execution/ApplyResourceProfileExec.scala | 2 +-
.../execution/ColumnarBroadcastExchangeExec.scala | 2 +-
.../ColumnarCollapseTransformStages.scala | 4 +--
.../ColumnarShuffleExchangeExecBase.scala | 2 +-
.../execution/ColumnarSubqueryBroadcastExec.scala | 2 +-
.../sql/execution/ColumnarWriteFilesExec.scala | 2 +-
.../spark/sql/execution/RDDScanTransformer.scala | 2 +-
.../sql/execution/RangeExecBaseTransformer.scala | 2 +-
.../columnar/transition/TransitionSuiteBase.scala | 18 +++++-----
.../sql/execution/FallbackStrategiesSuite.scala | 4 +--
.../sql/execution/FallbackStrategiesSuite.scala | 4 +--
.../sql/execution/FallbackStrategiesSuite.scala | 4 +--
.../sql/execution/FallbackStrategiesSuite.scala | 4 +--
.../sql/gluten/GlutenFallbackStrategiesSuite.scala | 4 +--
.../sql/gluten/GlutenFallbackStrategiesSuite.scala | 4 +--
.../parquet/ParquetFooterReaderShim.scala | 42 ----------------------
48 files changed, 72 insertions(+), 176 deletions(-)
diff --git
a/backends-clickhouse/src-delta33/main/scala/org/apache/spark/sql/execution/DeletionVectorWriteTransformer.scala
b/backends-clickhouse/src-delta33/main/scala/org/apache/spark/sql/execution/DeletionVectorWriteTransformer.scala
index a66efedc9a..82dd46df41 100644
---
a/backends-clickhouse/src-delta33/main/scala/org/apache/spark/sql/execution/DeletionVectorWriteTransformer.scala
+++
b/backends-clickhouse/src-delta33/main/scala/org/apache/spark/sql/execution/DeletionVectorWriteTransformer.scala
@@ -96,7 +96,7 @@ case class DeletionVectorWriteTransformer(
override def batchType(): Convention.BatchType =
BackendsApiManager.getSettings.primaryBatchType
- override def rowType0(): Convention.RowType = Convention.RowType.None
+ override def rowType(): Convention.RowType = Convention.RowType.None
override protected def doExecute(): RDD[InternalRow] = throw new
UnsupportedOperationException()
}
diff --git
a/backends-clickhouse/src-delta33/main/scala/org/apache/spark/sql/perf/DeltaOptimizedWriterTransformer.scala
b/backends-clickhouse/src-delta33/main/scala/org/apache/spark/sql/perf/DeltaOptimizedWriterTransformer.scala
index 800e4d1766..070a56d672 100644
---
a/backends-clickhouse/src-delta33/main/scala/org/apache/spark/sql/perf/DeltaOptimizedWriterTransformer.scala
+++
b/backends-clickhouse/src-delta33/main/scala/org/apache/spark/sql/perf/DeltaOptimizedWriterTransformer.scala
@@ -248,7 +248,7 @@ case class DeltaOptimizedWriterTransformer(
override def batchType(): Convention.BatchType =
BackendsApiManager.getSettings.primaryBatchType
- override def rowType0(): Convention.RowType = Convention.RowType.None
+ override def rowType(): Convention.RowType = Convention.RowType.None
override protected def doExecute(): RDD[InternalRow] = {
child.execute()
diff --git
a/backends-clickhouse/src/main/scala/org/apache/gluten/execution/CHColumnarToCarrierRowExec.scala
b/backends-clickhouse/src/main/scala/org/apache/gluten/execution/CHColumnarToCarrierRowExec.scala
index 95ac277ec0..9d0c209655 100644
---
a/backends-clickhouse/src/main/scala/org/apache/gluten/execution/CHColumnarToCarrierRowExec.scala
+++
b/backends-clickhouse/src/main/scala/org/apache/gluten/execution/CHColumnarToCarrierRowExec.scala
@@ -26,7 +26,7 @@ import org.apache.spark.sql.vectorized.ColumnarBatch
case class CHColumnarToCarrierRowExec(override val child: SparkPlan)
extends ColumnarToCarrierRowExecBase {
override protected def fromBatchType(): Convention.BatchType = CHBatchType
- override def rowType0(): Convention.RowType = CHCarrierRowType
+ override def rowType(): Convention.RowType = CHCarrierRowType
override protected def withNewChildInternal(newChild: SparkPlan): SparkPlan =
copy(child = newChild)
// Since https://github.com/apache/incubator-gluten/pull/1595.
diff --git
a/backends-velox/src-delta33/main/scala/org/apache/spark/sql/delta/perf/GlutenDeltaOptimizedWriterExec.scala
b/backends-velox/src-delta33/main/scala/org/apache/spark/sql/delta/perf/GlutenDeltaOptimizedWriterExec.scala
index 8e83a8af87..80f8e44e88 100644
---
a/backends-velox/src-delta33/main/scala/org/apache/spark/sql/delta/perf/GlutenDeltaOptimizedWriterExec.scala
+++
b/backends-velox/src-delta33/main/scala/org/apache/spark/sql/delta/perf/GlutenDeltaOptimizedWriterExec.scala
@@ -235,7 +235,7 @@ case class GlutenDeltaOptimizedWriterExec(
override def batchType(): Convention.BatchType = VeloxBatchType
- override def rowType0(): Convention.RowType = Convention.RowType.None
+ override def rowType(): Convention.RowType = Convention.RowType.None
}
/**
diff --git
a/backends-velox/src-delta33/main/scala/org/apache/spark/sql/delta/stats/GlutenDeltaJobStatsTracker.scala
b/backends-velox/src-delta33/main/scala/org/apache/spark/sql/delta/stats/GlutenDeltaJobStatsTracker.scala
index ae11263943..680cca1082 100644
---
a/backends-velox/src-delta33/main/scala/org/apache/spark/sql/delta/stats/GlutenDeltaJobStatsTracker.scala
+++
b/backends-velox/src-delta33/main/scala/org/apache/spark/sql/delta/stats/GlutenDeltaJobStatsTracker.scala
@@ -331,7 +331,7 @@ object GlutenDeltaJobStatsTracker extends Logging {
with LeafExecNode {
override def output: Seq[Attribute] = keySchema ++ dataSchema
override def batchType(): Convention.BatchType = VeloxBatchType
- override def rowType0(): Convention.RowType = Convention.RowType.None
+ override def rowType(): Convention.RowType = Convention.RowType.None
override protected def doExecute(): RDD[InternalRow] = throw new
UnsupportedOperationException()
override protected def doExecuteColumnar(): RDD[ColumnarBatch] =
throw new UnsupportedOperationException()
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/execution/ColumnarPartialGenerateExec.scala
b/backends-velox/src/main/scala/org/apache/gluten/execution/ColumnarPartialGenerateExec.scala
index 4e447df064..dcd6380335 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/execution/ColumnarPartialGenerateExec.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/execution/ColumnarPartialGenerateExec.scala
@@ -385,7 +385,7 @@ case class ColumnarPartialGenerateExec(generateExec:
GenerateExec, child: SparkP
override def batchType(): Convention.BatchType =
BackendsApiManager.getSettings.primaryBatchType
- override def rowType0(): Convention.RowType = Convention.RowType.None
+ override def rowType(): Convention.RowType = Convention.RowType.None
final override def doExecute(): RDD[InternalRow] = {
throw new UnsupportedOperationException(
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/execution/ColumnarPartialProjectExec.scala
b/backends-velox/src/main/scala/org/apache/gluten/execution/ColumnarPartialProjectExec.scala
index efc2a94cee..17554262ef 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/execution/ColumnarPartialProjectExec.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/execution/ColumnarPartialProjectExec.scala
@@ -87,7 +87,7 @@ case class ColumnarPartialProjectExec(projectList:
Seq[Expression], child: Spark
override def batchType(): Convention.BatchType =
BackendsApiManager.getSettings.primaryBatchType
- override def rowType0(): Convention.RowType = Convention.RowType.None
+ override def rowType(): Convention.RowType = Convention.RowType.None
final override def doExecute(): RDD[InternalRow] = {
throw new UnsupportedOperationException(
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/execution/GpuResizeBufferColumnarBatchExec.scala
b/backends-velox/src/main/scala/org/apache/gluten/execution/GpuResizeBufferColumnarBatchExec.scala
index 84768d71bc..9de62e5bb8 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/execution/GpuResizeBufferColumnarBatchExec.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/execution/GpuResizeBufferColumnarBatchExec.scala
@@ -64,5 +64,5 @@ case class GpuResizeBufferColumnarBatchExec(override val
child: SparkPlan, minOu
override def batchType(): Convention.BatchType = VeloxBatchType
- override def rowType0(): Convention.RowType = Convention.RowType.None
+ override def rowType(): Convention.RowType = Convention.RowType.None
}
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxColumnarToCarrierRowExec.scala
b/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxColumnarToCarrierRowExec.scala
index a5007bae75..65ce184861 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxColumnarToCarrierRowExec.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxColumnarToCarrierRowExec.scala
@@ -24,7 +24,7 @@ import org.apache.spark.sql.execution.SparkPlan
case class VeloxColumnarToCarrierRowExec(override val child: SparkPlan)
extends ColumnarToCarrierRowExecBase {
override protected def fromBatchType(): Convention.BatchType = VeloxBatchType
- override def rowType0(): Convention.RowType = VeloxCarrierRowType
+ override def rowType(): Convention.RowType = VeloxCarrierRowType
override protected def withNewChildInternal(newChild: SparkPlan): SparkPlan =
copy(child = newChild)
}
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxResizeBatchesExec.scala
b/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxResizeBatchesExec.scala
index 9febad0e94..3b2c9490e7 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxResizeBatchesExec.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxResizeBatchesExec.scala
@@ -61,5 +61,5 @@ case class VeloxResizeBatchesExec(
override def batchType(): Convention.BatchType = VeloxBatchType
- override def rowType0(): Convention.RowType = Convention.RowType.None
+ override def rowType(): Convention.RowType = Convention.RowType.None
}
diff --git
a/backends-velox/src/main/scala/org/apache/spark/api/python/ColumnarArrowEvalPythonExec.scala
b/backends-velox/src/main/scala/org/apache/spark/api/python/ColumnarArrowEvalPythonExec.scala
index 033f8a4284..6bb2f50e1f 100644
---
a/backends-velox/src/main/scala/org/apache/spark/api/python/ColumnarArrowEvalPythonExec.scala
+++
b/backends-velox/src/main/scala/org/apache/spark/api/python/ColumnarArrowEvalPythonExec.scala
@@ -226,7 +226,7 @@ case class ColumnarArrowEvalPythonExec(
override def batchType(): Convention.BatchType = ArrowJavaBatchType
- override def rowType0(): Convention.RowType = Convention.RowType.None
+ override def rowType(): Convention.RowType = Convention.RowType.None
override protected def doValidateInternal(): ValidationResult = {
val (_, inputs) =
udfs.map(ColumnarArrowEvalPythonExec.collectFunctions).unzip
diff --git
a/backends-velox/src/main/scala/org/apache/spark/sql/execution/BaseArrowScanExec.scala
b/backends-velox/src/main/scala/org/apache/spark/sql/execution/BaseArrowScanExec.scala
index ebccd6f5bb..13d0f5699b 100644
---
a/backends-velox/src/main/scala/org/apache/spark/sql/execution/BaseArrowScanExec.scala
+++
b/backends-velox/src/main/scala/org/apache/spark/sql/execution/BaseArrowScanExec.scala
@@ -25,5 +25,5 @@ trait BaseArrowScanExec extends GlutenPlan {
ArrowBatchTypes.ArrowJavaBatchType
}
- final override def rowType0(): Convention.RowType = Convention.RowType.None
+ final override def rowType(): Convention.RowType = Convention.RowType.None
}
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala
index e1a0fd98ee..8db8295b61 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala
@@ -18,10 +18,9 @@ package org.apache.gluten.execution
import org.apache.gluten.config.{GlutenConfig, GlutenCoreConfig, VeloxConfig}
import org.apache.gluten.expression.VeloxDummyExpression
-import org.apache.gluten.sql.shims.SparkShimLoader
import org.apache.spark.SparkConf
-import org.apache.spark.sql.{AnalysisException, DataFrame, Row}
+import org.apache.spark.sql.{DataFrame, Row}
import org.apache.spark.sql.execution._
import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanHelper,
AQEShuffleReadExec, ShuffleQueryStageExec}
import org.apache.spark.sql.execution.joins.BaseJoinExec
@@ -911,18 +910,9 @@ class MiscOperatorSuite extends
VeloxWholeStageTransformerSuite with AdaptiveSpa
test("Verify parquet field name with special character") {
withTable("t") {
-
- // https://github.com/apache/spark/pull/35229 Spark remove parquet field
name check after 3.2
- if (!SparkShimLoader.getSparkVersion.startsWith("3.2")) {
- sql("create table t using parquet as select sum(l_partkey) from
lineitem")
- runQueryAndCompare("select * from t") {
- checkGlutenPlan[FileSourceScanExecTransformer]
- }
- } else {
- val msg = intercept[AnalysisException] {
- sql("create table t using parquet as select sum(l_partkey) from
lineitem")
- }.message
- assert(msg.contains("contains invalid character"))
+ sql("create table t using parquet as select sum(l_partkey) from
lineitem")
+ runQueryAndCompare("select * from t") {
+ checkGlutenPlan[FileSourceScanExecTransformer]
}
}
}
diff --git
a/gluten-core/src/main/scala/org/apache/gluten/execution/GlutenColumnarToColumnarTransition.scala
b/gluten-core/src/main/scala/org/apache/gluten/execution/GlutenColumnarToColumnarTransition.scala
index d2f62f0bf9..c92224fcde 100644
---
a/gluten-core/src/main/scala/org/apache/gluten/execution/GlutenColumnarToColumnarTransition.scala
+++
b/gluten-core/src/main/scala/org/apache/gluten/execution/GlutenColumnarToColumnarTransition.scala
@@ -30,7 +30,7 @@ trait GlutenColumnarToColumnarTransition extends
ColumnarToColumnarTransition wi
override def batchType(): Convention.BatchType = to
- override def rowType0(): Convention.RowType = {
+ override def rowType(): Convention.RowType = {
Convention.RowType.None
}
diff --git
a/gluten-core/src/main/scala/org/apache/gluten/execution/GlutenPlan.scala
b/gluten-core/src/main/scala/org/apache/gluten/execution/GlutenPlan.scala
index 2cd408f67c..91d5ae4e72 100644
--- a/gluten-core/src/main/scala/org/apache/gluten/execution/GlutenPlan.scala
+++ b/gluten-core/src/main/scala/org/apache/gluten/execution/GlutenPlan.scala
@@ -30,7 +30,7 @@ import org.apache.spark.sql.execution.SparkPlan
*
* Instead, subclasses are expected to implement the following APIs:
* - batchType
- * - rowType0
+ * - rowType
* - requiredChildConvention (optional)
*
* With implementations of the APIs provided, Gluten query planner will be
able to find and insert
@@ -45,7 +45,7 @@ import org.apache.spark.sql.execution.SparkPlan
trait GlutenPlan
extends SparkPlan
with Convention.KnownBatchType
- with Convention.KnownRowTypeForSpark33OrLater
+ with Convention.KnownRowType
with GlutenPlan.SupportsRowBasedCompatible
with ConventionReq.KnownChildConvention {
@@ -59,7 +59,7 @@ trait GlutenPlan
override def batchType(): Convention.BatchType
- override def rowType0(): Convention.RowType
+ override def rowType(): Convention.RowType
override def requiredChildConvention(): Seq[ConventionReq] = {
// In the normal case, children's convention should follow parent node's
convention.
diff --git
a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/planner/plan/GroupLeafExec.scala
b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/planner/plan/GroupLeafExec.scala
index 02746c4534..ed5f48768a 100644
---
a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/planner/plan/GroupLeafExec.scala
+++
b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/planner/plan/GroupLeafExec.scala
@@ -35,7 +35,7 @@ import java.util.concurrent.atomic.AtomicBoolean
case class GroupLeafExec(groupId: Int, metadata: GlutenMetadata, convReq:
Conv.Req)
extends LeafExecNode
with Convention.KnownBatchType
- with Convention.KnownRowTypeForSpark33OrLater
+ with Convention.KnownRowType
with GlutenPlan.SupportsRowBasedCompatible {
private val frozen = new AtomicBoolean(false)
@@ -62,7 +62,7 @@ case class GroupLeafExec(groupId: Int, metadata:
GlutenMetadata, convReq: Conv.R
batchType != Convention.BatchType.None
}
- override val rowType0: Convention.RowType = {
+ override val rowType: Convention.RowType = {
val out = convReq.req.requiredRowType match {
case ConventionReq.RowType.Any => Convention.RowType.VanillaRowType
case ConventionReq.RowType.Is(r) => r
@@ -71,7 +71,7 @@ case class GroupLeafExec(groupId: Int, metadata:
GlutenMetadata, convReq: Conv.R
}
final override val supportsRowBased: Boolean = {
- rowType() != Convention.RowType.None
+ rowType != Convention.RowType.None
}
private def ensureNotFrozen(): Unit = {
diff --git
a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/offload/OffloadSingleNode.scala
b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/offload/OffloadSingleNode.scala
index 4cf3e1d0c9..db20e9efe2 100644
---
a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/offload/OffloadSingleNode.scala
+++
b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/offload/OffloadSingleNode.scala
@@ -99,7 +99,7 @@ object OffloadSingleNode {
private lazy val conv: Convention = Convention.get(hiddenPlan)
override def batchType(): Convention.BatchType = conv.batchType
- override def rowType0(): Convention.RowType = conv.rowType
+ override def rowType(): Convention.RowType = conv.rowType
override def output: Seq[Attribute] = hiddenPlan.output
override def outputPartitioning: Partitioning =
hiddenPlan.outputPartitioning
override def outputOrdering: Seq[SortOrder] = hiddenPlan.outputOrdering
diff --git
a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/Convention.scala
b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/Convention.scala
index a00e2b5106..116a74e53b 100644
---
a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/Convention.scala
+++
b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/Convention.scala
@@ -17,7 +17,6 @@
package org.apache.gluten.extension.columnar.transition
import org.apache.spark.sql.execution.{ColumnarToRowExec, RowToColumnarExec,
SparkPlan}
-import org.apache.spark.util.SparkVersionUtil
import scala.collection.mutable
@@ -163,30 +162,7 @@ object Convention {
def batchType(): BatchType
}
- sealed trait KnownRowType {
+ trait KnownRowType {
def rowType(): RowType
}
-
- trait KnownRowTypeForSpark33OrLater extends KnownRowType {
- this: SparkPlan =>
-
- final override def rowType(): RowType = {
- if (SparkVersionUtil.lteSpark32) {
- // It's known that in Spark 3.2, one Spark plan node is considered
either only having
- // row-based support or only having columnar support at a time.
- // Hence, if the plan supports columnar output, we'd disable its
row-based support.
- // The same for the opposite.
- if (supportsColumnar) {
- Convention.RowType.None
- } else {
- assert(rowType0() != Convention.RowType.None)
- rowType0()
- }
- } else {
- rowType0()
- }
- }
-
- def rowType0(): RowType
- }
}
diff --git
a/gluten-core/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala
b/gluten-core/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala
index f53a80639a..957230da39 100644
---
a/gluten-core/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala
+++
b/gluten-core/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala
@@ -20,7 +20,7 @@ import org.apache.gluten.config.GlutenCoreConfig
import org.apache.spark.sql.catalyst.SQLConfHelper
import org.apache.spark.sql.execution.SparkPlan
-import org.apache.spark.util.{SparkVersionUtil, Utils}
+import org.apache.spark.util.Utils
/**
* This [[CostEvaluator]] is to force use the new physical plan when cost is
equal.
@@ -30,16 +30,11 @@ import org.apache.spark.util.{SparkVersionUtil, Utils}
case class GlutenCostEvaluator() extends CostEvaluator with SQLConfHelper {
private val vanillaCostEvaluator: CostEvaluator = {
- if (SparkVersionUtil.lteSpark32) {
- val clazz =
Utils.classForName("org.apache.spark.sql.execution.adaptive.SimpleCostEvaluator$")
- clazz.getDeclaredField("MODULE$").get(null).asInstanceOf[CostEvaluator]
- } else {
- val forceOptimizeSkewedJoin =
-
conf.getConfString("spark.sql.adaptive.forceOptimizeSkewedJoin").toBoolean
- val clazz =
Utils.classForName("org.apache.spark.sql.execution.adaptive.SimpleCostEvaluator")
- val ctor = clazz.getConstructor(classOf[Boolean])
-
ctor.newInstance(forceOptimizeSkewedJoin.asInstanceOf[Object]).asInstanceOf[CostEvaluator]
- }
+ val forceOptimizeSkewedJoin =
+
conf.getConfString("spark.sql.adaptive.forceOptimizeSkewedJoin").toBoolean
+ val clazz =
Utils.classForName("org.apache.spark.sql.execution.adaptive.SimpleCostEvaluator")
+ val ctor = clazz.getConstructor(classOf[Boolean])
+
ctor.newInstance(forceOptimizeSkewedJoin.asInstanceOf[Object]).asInstanceOf[CostEvaluator]
}
override def evaluateCost(plan: SparkPlan): Cost = {
diff --git
a/gluten-core/src/main/scala/org/apache/spark/util/SparkPlanUtil.scala
b/gluten-core/src/main/scala/org/apache/spark/util/SparkPlanUtil.scala
index 5ce2addafb..a7a6cdaf3d 100644
--- a/gluten-core/src/main/scala/org/apache/spark/util/SparkPlanUtil.scala
+++ b/gluten-core/src/main/scala/org/apache/spark/util/SparkPlanUtil.scala
@@ -23,16 +23,12 @@ import org.apache.spark.sql.internal.SQLConf
object SparkPlanUtil {
def supportsRowBased(plan: SparkPlan): Boolean = {
- if (SparkVersionUtil.lteSpark32) {
- return !plan.supportsColumnar
- }
-
val m = classOf[SparkPlan].getMethod("supportsRowBased")
m.invoke(plan).asInstanceOf[Boolean]
}
def isPlannedV1Write(plan: DataWritingCommandExec): Boolean = {
- if (SparkVersionUtil.lteSpark33) {
+ if (SparkVersionUtil.eqSpark33) {
return false
}
diff --git
a/gluten-core/src/main/scala/org/apache/spark/util/SparkTaskUtil.scala
b/gluten-core/src/main/scala/org/apache/spark/util/SparkTaskUtil.scala
index afbc03dd12..a19579e433 100644
--- a/gluten-core/src/main/scala/org/apache/spark/util/SparkTaskUtil.scala
+++ b/gluten-core/src/main/scala/org/apache/spark/util/SparkTaskUtil.scala
@@ -64,23 +64,6 @@ object SparkTaskUtil {
ctors.head
}
- if (SparkVersionUtil.lteSpark32) {
- return ctor
- .newInstance(
- stageId,
- stageAttemptNumber,
- partitionId,
- taskAttemptId,
- attemptNumber,
- taskMemoryManager,
- localProperties,
- metricsSystem,
- taskMetrics,
- resources
- )
- .asInstanceOf[TaskContext]
- }
-
if (SparkVersionUtil.eqSpark33) {
return ctor
.newInstance(
diff --git
a/gluten-core/src/main/scala/org/apache/spark/util/SparkVersionUtil.scala
b/gluten-core/src/main/scala/org/apache/spark/util/SparkVersionUtil.scala
index 50114ab702..b9089363b5 100644
--- a/gluten-core/src/main/scala/org/apache/spark/util/SparkVersionUtil.scala
+++ b/gluten-core/src/main/scala/org/apache/spark/util/SparkVersionUtil.scala
@@ -17,11 +17,9 @@
package org.apache.spark.util
object SparkVersionUtil {
- val lteSpark32: Boolean = compareMajorMinorVersion((3, 2)) <= 0
private val comparedWithSpark33 = compareMajorMinorVersion((3, 3))
private val comparedWithSpark35 = compareMajorMinorVersion((3, 5))
val eqSpark33: Boolean = comparedWithSpark33 == 0
- val lteSpark33: Boolean = lteSpark32 || eqSpark33
val gteSpark33: Boolean = comparedWithSpark33 >= 0
val gteSpark35: Boolean = comparedWithSpark35 >= 0
val gteSpark40: Boolean = compareMajorMinorVersion((4, 0)) >= 0
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/BasicPhysicalOperatorTransformer.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/BasicPhysicalOperatorTransformer.scala
index 57e7373fc3..1514aca532 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/BasicPhysicalOperatorTransformer.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/BasicPhysicalOperatorTransformer.scala
@@ -248,7 +248,7 @@ case class ColumnarUnionExec(children: Seq[SparkPlan],
partitioning: Partitionin
override def batchType(): Convention.BatchType =
BackendsApiManager.getSettings.primaryBatchType
- override def rowType0(): Convention.RowType = Convention.RowType.None
+ override def rowType(): Convention.RowType = Convention.RowType.None
override def outputPartitioning: Partitioning = partitioning
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/CartesianProductExecTransformer.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/CartesianProductExecTransformer.scala
index 326bd75329..868e6f1a9e 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/CartesianProductExecTransformer.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/CartesianProductExecTransformer.scala
@@ -46,7 +46,7 @@ import java.io.{IOException, ObjectOutputStream}
case class ColumnarCartesianProductBridge(child: SparkPlan) extends
UnaryExecNode with GlutenPlan {
override def output: Seq[Attribute] = child.output
override def batchType(): Convention.BatchType =
BackendsApiManager.getSettings.primaryBatchType
- override def rowType0(): Convention.RowType = Convention.RowType.None
+ override def rowType(): Convention.RowType = Convention.RowType.None
override protected def doExecute(): RDD[InternalRow] =
throw new UnsupportedOperationException()
override protected def doExecuteColumnar(): RDD[ColumnarBatch] =
child.executeColumnar()
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/ColumnarCoalesceExec.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/ColumnarCoalesceExec.scala
index 107bf544cd..ce801d80ec 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/ColumnarCoalesceExec.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/ColumnarCoalesceExec.scala
@@ -39,7 +39,7 @@ case class ColumnarCoalesceExec(numPartitions: Int, child:
SparkPlan)
override def batchType(): Convention.BatchType =
BackendsApiManager.getSettings.primaryBatchType
- override def rowType0(): Convention.RowType = Convention.RowType.None
+ override def rowType(): Convention.RowType = Convention.RowType.None
override protected def doExecute(): RDD[InternalRow] = {
throw new UnsupportedOperationException()
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/ColumnarCollectLimitBaseExec.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/ColumnarCollectLimitBaseExec.scala
index da1999a24c..333707f469 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/ColumnarCollectLimitBaseExec.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/ColumnarCollectLimitBaseExec.scala
@@ -36,7 +36,7 @@ abstract class ColumnarCollectLimitBaseExec(
) extends LimitExec
with ValidatablePlan {
- override def rowType0(): Convention.RowType = Convention.RowType.None
+ override def rowType(): Convention.RowType = Convention.RowType.None
override def batchType(): Convention.BatchType =
BackendsApiManager.getSettings.primaryBatchType
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/ColumnarCollectTailBaseExec.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/ColumnarCollectTailBaseExec.scala
index 56f9ce69ee..4ced96ec5e 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/ColumnarCollectTailBaseExec.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/ColumnarCollectTailBaseExec.scala
@@ -58,7 +58,7 @@ abstract class ColumnarCollectTailBaseExec(
.genColumnarShuffleExchangeMetrics(sparkContext, shuffleWriterType) ++
readMetrics ++ writeMetrics
- override def rowType0(): Convention.RowType = Convention.RowType.None
+ override def rowType(): Convention.RowType = Convention.RowType.None
override def output: Seq[Attribute] = child.output
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/ColumnarToRowExecBase.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/ColumnarToRowExecBase.scala
index 15103bbd84..3d3f4445c5 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/ColumnarToRowExecBase.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/ColumnarToRowExecBase.scala
@@ -42,7 +42,7 @@ abstract class ColumnarToRowExecBase(child: SparkPlan)
override def batchType(): Convention.BatchType = Convention.BatchType.None
- override def rowType0(): Convention.RowType =
Convention.RowType.VanillaRowType
+ override def rowType(): Convention.RowType =
Convention.RowType.VanillaRowType
override def requiredChildConvention(): Seq[ConventionReq] = {
List(
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/ColumnarV2TableWriteExec.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/ColumnarV2TableWriteExec.scala
index cb9470765b..c737284b35 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/ColumnarV2TableWriteExec.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/ColumnarV2TableWriteExec.scala
@@ -56,7 +56,7 @@ trait ColumnarV2TableWriteExec extends V2TableWriteExec with
ValidatablePlan {
override def batchType(): Convention.BatchType = Convention.BatchType.None
- override def rowType0(): Convention.RowType = RowType.VanillaRowType
+ override def rowType(): Convention.RowType = RowType.VanillaRowType
override def requiredChildConvention(): Seq[ConventionReq] = Seq(
ConventionReq.ofBatch(
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/RowToColumnarExecBase.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/RowToColumnarExecBase.scala
index a500da9790..e8e5ae50ab 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/RowToColumnarExecBase.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/RowToColumnarExecBase.scala
@@ -47,7 +47,7 @@ abstract class RowToColumnarExecBase(child: SparkPlan)
override def batchType(): Convention.BatchType =
BackendsApiManager.getSettings.primaryBatchType
- override def rowType0(): Convention.RowType = Convention.RowType.None
+ override def rowType(): Convention.RowType = Convention.RowType.None
override def requiredChildConvention(): Seq[ConventionReq] = {
Seq(ConventionReq.vanillaRow)
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/TakeOrderedAndProjectExecTransformer.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/TakeOrderedAndProjectExecTransformer.scala
index 39b37a0d18..dc47dbba10 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/TakeOrderedAndProjectExecTransformer.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/TakeOrderedAndProjectExecTransformer.scala
@@ -44,7 +44,7 @@ case class TakeOrderedAndProjectExecTransformer(
override def outputPartitioning: Partitioning = SinglePartition
override def outputOrdering: Seq[SortOrder] = sortOrder
override def batchType(): Convention.BatchType =
BackendsApiManager.getSettings.primaryBatchType
- override def rowType0(): Convention.RowType = Convention.RowType.None
+ override def rowType(): Convention.RowType = Convention.RowType.None
override def output: Seq[Attribute] = {
projectList.map(_.toAttribute)
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/WholeStageTransformer.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/WholeStageTransformer.scala
index acef5d798e..959931b754 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/WholeStageTransformer.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/WholeStageTransformer.scala
@@ -60,7 +60,7 @@ trait TransformSupport extends ValidatablePlan {
BackendsApiManager.getSettings.primaryBatchType
}
- override def rowType0(): Convention.RowType = {
+ override def rowType(): Convention.RowType = {
Convention.RowType.None
}
diff --git
a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ApplyResourceProfileExec.scala
b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ApplyResourceProfileExec.scala
index 17640f4612..b175cb5a05 100644
---
a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ApplyResourceProfileExec.scala
+++
b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ApplyResourceProfileExec.scala
@@ -42,7 +42,7 @@ case class ApplyResourceProfileExec(child: SparkPlan,
resourceProfile: ResourceP
Convention.get(child).batchType
}
- override def rowType0(): Convention.RowType = {
+ override def rowType(): Convention.RowType = {
Convention.get(child).rowType
}
diff --git
a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarBroadcastExchangeExec.scala
b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarBroadcastExchangeExec.scala
index 371f9948b7..26a4ef3919 100644
---
a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarBroadcastExchangeExec.scala
+++
b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarBroadcastExchangeExec.scala
@@ -128,7 +128,7 @@ case class ColumnarBroadcastExchangeExec(mode:
BroadcastMode, child: SparkPlan)
override def batchType(): Convention.BatchType =
BackendsApiManager.getSettings.primaryBatchType
- override def rowType0(): Convention.RowType = Convention.RowType.None
+ override def rowType(): Convention.RowType = Convention.RowType.None
override def doCanonicalize(): SparkPlan = {
ColumnarBroadcastExchangeExec(mode.canonicalized, child.canonicalized)
diff --git
a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarCollapseTransformStages.scala
b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarCollapseTransformStages.scala
index 134db93e70..c81d380a78 100644
---
a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarCollapseTransformStages.scala
+++
b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarCollapseTransformStages.scala
@@ -188,13 +188,13 @@ case class ColumnarCollapseTransformStages(glutenConf:
GlutenConfig) extends Rul
case class ColumnarInputAdapter(child: SparkPlan)
extends InputAdapterGenerateTreeStringShim
with Convention.KnownBatchType
- with Convention.KnownRowTypeForSpark33OrLater
+ with Convention.KnownRowType
with GlutenPlan.SupportsRowBasedCompatible
with ConventionReq.KnownChildConvention {
override def output: Seq[Attribute] = child.output
final override val supportsColumnar: Boolean = true
final override val supportsRowBased: Boolean = false
- override def rowType0(): Convention.RowType = Convention.RowType.None
+ override def rowType(): Convention.RowType = Convention.RowType.None
override def batchType(): Convention.BatchType =
BackendsApiManager.getSettings.primaryBatchType
override def requiredChildConvention(): Seq[ConventionReq] = Seq(
diff --git
a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarShuffleExchangeExecBase.scala
b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarShuffleExchangeExecBase.scala
index 17d1ec4038..24db757720 100644
---
a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarShuffleExchangeExecBase.scala
+++
b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarShuffleExchangeExecBase.scala
@@ -137,7 +137,7 @@ abstract class ColumnarShuffleExchangeExecBase(
override def batchType(): Convention.BatchType =
BackendsApiManager.getSettings.primaryBatchType
- override def rowType0(): Convention.RowType = Convention.RowType.None
+ override def rowType(): Convention.RowType = Convention.RowType.None
override def doExecute(): RDD[InternalRow] = {
throw new UnsupportedOperationException()
diff --git
a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarSubqueryBroadcastExec.scala
b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarSubqueryBroadcastExec.scala
index 3e91a5c7e4..611433c51b 100644
---
a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarSubqueryBroadcastExec.scala
+++
b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarSubqueryBroadcastExec.scala
@@ -132,7 +132,7 @@ case class ColumnarSubqueryBroadcastExec(
override def batchType(): Convention.BatchType =
BackendsApiManager.getSettings.primaryBatchType
- override def rowType0(): Convention.RowType = Convention.RowType.None
+ override def rowType(): Convention.RowType = Convention.RowType.None
override protected def doExecute(): RDD[InternalRow] = {
throw new UnsupportedOperationException(
diff --git
a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarWriteFilesExec.scala
b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarWriteFilesExec.scala
index 998f4f86b5..2d430bfb77 100644
---
a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarWriteFilesExec.scala
+++
b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarWriteFilesExec.scala
@@ -70,7 +70,7 @@ abstract class ColumnarWriteFilesExec protected (
* Since https://github.com/apache/incubator-gluten/pull/6745.
*/
override def batchType(): Convention.BatchType =
BackendsApiManager.getSettings.primaryBatchType
- override def rowType0(): RowType = {
+ override def rowType(): RowType = {
RowType.VanillaRowType
}
diff --git
a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/RDDScanTransformer.scala
b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/RDDScanTransformer.scala
index e3fc728477..91b719cece 100644
---
a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/RDDScanTransformer.scala
+++
b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/RDDScanTransformer.scala
@@ -29,7 +29,7 @@ abstract class RDDScanTransformer(
override val outputOrdering: Seq[SortOrder]
) extends ValidatablePlan {
- override def rowType0(): Convention.RowType = Convention.RowType.None
+ override def rowType(): Convention.RowType = Convention.RowType.None
override def batchType(): Convention.BatchType =
BackendsApiManager.getSettings.primaryBatchType
override def output: Seq[Attribute] = {
outputAttributes
diff --git
a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/RangeExecBaseTransformer.scala
b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/RangeExecBaseTransformer.scala
index 653b93883a..e28344579f 100644
---
a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/RangeExecBaseTransformer.scala
+++
b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/RangeExecBaseTransformer.scala
@@ -44,7 +44,7 @@ abstract class ColumnarRangeBaseExec extends LeafExecNode
with ValidatablePlan {
s"${output.mkString("[", ", ", "]")}"
}
- override def rowType0(): Convention.RowType = Convention.RowType.None
+ override def rowType(): Convention.RowType = Convention.RowType.None
override protected def doExecute(): RDD[InternalRow] = {
throw new UnsupportedOperationException(s"This operator doesn't support
doExecute().")
diff --git
a/gluten-substrait/src/test/scala/org/apache/gluten/extension/columnar/transition/TransitionSuiteBase.scala
b/gluten-substrait/src/test/scala/org/apache/gluten/extension/columnar/transition/TransitionSuiteBase.scala
index 827881ebc9..4fddfcfbd5 100644
---
a/gluten-substrait/src/test/scala/org/apache/gluten/extension/columnar/transition/TransitionSuiteBase.scala
+++
b/gluten-substrait/src/test/scala/org/apache/gluten/extension/columnar/transition/TransitionSuiteBase.scala
@@ -30,7 +30,7 @@ object TransitionSuiteBase {
case class BatchLeaf(override val batchType: Convention.BatchType)
extends LeafExecNode
with GlutenPlan {
- override def rowType0(): Convention.RowType = Convention.RowType.None
+ override def rowType(): Convention.RowType = Convention.RowType.None
override protected def doExecute(): RDD[InternalRow] = throw new
UnsupportedOperationException()
@@ -40,7 +40,7 @@ object TransitionSuiteBase {
case class BatchUnary(override val batchType: Convention.BatchType, override
val child: SparkPlan)
extends UnaryExecNode
with GlutenPlan {
- override def rowType0(): Convention.RowType = Convention.RowType.None
+ override def rowType(): Convention.RowType = Convention.RowType.None
override protected def withNewChildInternal(newChild: SparkPlan):
SparkPlan =
copy(child = newChild)
@@ -56,7 +56,7 @@ object TransitionSuiteBase {
override val right: SparkPlan)
extends BinaryExecNode
with GlutenPlan {
- override def rowType0(): Convention.RowType = Convention.RowType.None
+ override def rowType(): Convention.RowType = Convention.RowType.None
override protected def withNewChildrenInternal(
newLeft: SparkPlan,
@@ -67,7 +67,7 @@ object TransitionSuiteBase {
override def output: Seq[Attribute] = left.output ++ right.output
}
- case class RowLeaf(override val rowType0: Convention.RowType)
+ case class RowLeaf(override val rowType: Convention.RowType)
extends LeafExecNode
with GlutenPlan {
override def batchType(): Convention.BatchType = Convention.BatchType.None
@@ -77,7 +77,7 @@ object TransitionSuiteBase {
override def output: Seq[Attribute] = List.empty
}
- case class RowUnary(override val rowType0: Convention.RowType, override val
child: SparkPlan)
+ case class RowUnary(override val rowType: Convention.RowType, override val
child: SparkPlan)
extends UnaryExecNode
with GlutenPlan {
override def batchType(): Convention.BatchType = Convention.BatchType.None
@@ -91,7 +91,7 @@ object TransitionSuiteBase {
}
case class RowBinary(
- override val rowType0: Convention.RowType,
+ override val rowType: Convention.RowType,
override val left: SparkPlan,
override val right: SparkPlan)
extends BinaryExecNode
@@ -114,7 +114,7 @@ object TransitionSuiteBase {
extends RowToColumnarTransition
with GlutenPlan {
override def batchType(): Convention.BatchType = toBatchType
- override def rowType0(): Convention.RowType = Convention.RowType.None
+ override def rowType(): Convention.RowType = Convention.RowType.None
override def requiredChildConvention(): Seq[ConventionReq] = {
List(ConventionReq.ofRow(ConventionReq.RowType.Is(fromRowType)))
}
@@ -133,7 +133,7 @@ object TransitionSuiteBase {
extends ColumnarToRowTransition
with GlutenPlan {
override def batchType(): Convention.BatchType = Convention.BatchType.None
- override def rowType0(): Convention.RowType = toRowType
+ override def rowType(): Convention.RowType = toRowType
override def requiredChildConvention(): Seq[ConventionReq] = {
List(ConventionReq.ofBatch(ConventionReq.BatchType.Is(fromBatchType)))
}
@@ -165,7 +165,7 @@ object TransitionSuiteBase {
extends UnaryExecNode
with GlutenPlan {
override def batchType(): Convention.BatchType = Convention.BatchType.None
- override def rowType0(): Convention.RowType = to
+ override def rowType(): Convention.RowType = to
override def requiredChildConvention(): Seq[ConventionReq] = {
List(ConventionReq.ofRow(ConventionReq.RowType.Is(from)))
}
diff --git
a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala
b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala
index c0e3d1a349..6c1a073f87 100644
---
a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala
+++
b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala
@@ -208,7 +208,7 @@ private object FallbackStrategiesSuite {
// For replacing LeafOp.
case class LeafOpTransformer() extends LeafExecNode with GlutenPlan {
override def batchType(): Convention.BatchType =
BackendsApiManager.getSettings.primaryBatchType
- override def rowType0(): Convention.RowType = Convention.RowType.None
+ override def rowType(): Convention.RowType = Convention.RowType.None
override protected def doExecute(): RDD[InternalRow] = throw new
UnsupportedOperationException()
override def output: Seq[Attribute] = Seq.empty
}
@@ -218,7 +218,7 @@ private object FallbackStrategiesSuite {
extends UnaryExecNode
with GlutenPlan {
override def batchType(): Convention.BatchType =
BackendsApiManager.getSettings.primaryBatchType
- override def rowType0(): Convention.RowType = Convention.RowType.None
+ override def rowType(): Convention.RowType = Convention.RowType.None
override protected def doExecute(): RDD[InternalRow] = throw new
UnsupportedOperationException()
override def output: Seq[Attribute] = child.output
override protected def withNewChildInternal(newChild: SparkPlan):
UnaryOp1Transformer =
diff --git
a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala
b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala
index 6f3894d23b..6ab2176cf7 100644
---
a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala
+++
b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala
@@ -237,7 +237,7 @@ private object FallbackStrategiesSuite {
// For replacing LeafOp.
case class LeafOpTransformer() extends LeafExecNode with GlutenPlan {
override def batchType(): Convention.BatchType =
BackendsApiManager.getSettings.primaryBatchType
- override def rowType0(): Convention.RowType = Convention.RowType.None
+ override def rowType(): Convention.RowType = Convention.RowType.None
override protected def doExecute(): RDD[InternalRow] = throw new
UnsupportedOperationException()
override def output: Seq[Attribute] = Seq.empty
}
@@ -247,7 +247,7 @@ private object FallbackStrategiesSuite {
extends UnaryExecNode
with GlutenPlan {
override def batchType(): Convention.BatchType =
BackendsApiManager.getSettings.primaryBatchType
- override def rowType0(): Convention.RowType = Convention.RowType.None
+ override def rowType(): Convention.RowType = Convention.RowType.None
override protected def doExecute(): RDD[InternalRow] = throw new
UnsupportedOperationException()
override def output: Seq[Attribute] = child.output
override protected def withNewChildInternal(newChild: SparkPlan):
UnaryOp1Transformer =
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala
index 6f3894d23b..6ab2176cf7 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala
@@ -237,7 +237,7 @@ private object FallbackStrategiesSuite {
// For replacing LeafOp.
case class LeafOpTransformer() extends LeafExecNode with GlutenPlan {
override def batchType(): Convention.BatchType =
BackendsApiManager.getSettings.primaryBatchType
- override def rowType0(): Convention.RowType = Convention.RowType.None
+ override def rowType(): Convention.RowType = Convention.RowType.None
override protected def doExecute(): RDD[InternalRow] = throw new
UnsupportedOperationException()
override def output: Seq[Attribute] = Seq.empty
}
@@ -247,7 +247,7 @@ private object FallbackStrategiesSuite {
extends UnaryExecNode
with GlutenPlan {
override def batchType(): Convention.BatchType =
BackendsApiManager.getSettings.primaryBatchType
- override def rowType0(): Convention.RowType = Convention.RowType.None
+ override def rowType(): Convention.RowType = Convention.RowType.None
override protected def doExecute(): RDD[InternalRow] = throw new
UnsupportedOperationException()
override def output: Seq[Attribute] = child.output
override protected def withNewChildInternal(newChild: SparkPlan):
UnaryOp1Transformer =
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala
index de2436a39e..d7d00bf005 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala
@@ -238,7 +238,7 @@ private object FallbackStrategiesSuite {
// For replacing LeafOp.
case class LeafOpTransformer() extends LeafExecNode with GlutenPlan {
override def batchType(): Convention.BatchType =
BackendsApiManager.getSettings.primaryBatchType
- override def rowType0(): Convention.RowType = Convention.RowType.None
+ override def rowType(): Convention.RowType = Convention.RowType.None
override protected def doExecute(): RDD[InternalRow] = throw new
UnsupportedOperationException()
override def output: Seq[Attribute] = Seq.empty
}
@@ -248,7 +248,7 @@ private object FallbackStrategiesSuite {
extends UnaryExecNode
with GlutenPlan {
override def batchType(): Convention.BatchType =
BackendsApiManager.getSettings.primaryBatchType
- override def rowType0(): Convention.RowType = Convention.RowType.None
+ override def rowType(): Convention.RowType = Convention.RowType.None
override protected def doExecute(): RDD[InternalRow] = throw new
UnsupportedOperationException()
override def output: Seq[Attribute] = child.output
override protected def withNewChildInternal(newChild: SparkPlan):
UnaryOp1Transformer =
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackStrategiesSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackStrategiesSuite.scala
index 50beded8de..57146b4abf 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackStrategiesSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackStrategiesSuite.scala
@@ -239,7 +239,7 @@ private object GlutenFallbackStrategiesSuite {
// For replacing LeafOp.
case class LeafOpTransformer() extends LeafExecNode with GlutenPlan {
override def batchType(): Convention.BatchType =
BackendsApiManager.getSettings.primaryBatchType
- override def rowType0(): Convention.RowType = Convention.RowType.None
+ override def rowType(): Convention.RowType = Convention.RowType.None
override protected def doExecute(): RDD[InternalRow] = throw new
UnsupportedOperationException()
override def output: Seq[Attribute] = Seq.empty
}
@@ -249,7 +249,7 @@ private object GlutenFallbackStrategiesSuite {
extends UnaryExecNode
with GlutenPlan {
override def batchType(): Convention.BatchType =
BackendsApiManager.getSettings.primaryBatchType
- override def rowType0(): Convention.RowType = Convention.RowType.None
+ override def rowType(): Convention.RowType = Convention.RowType.None
override protected def doExecute(): RDD[InternalRow] = throw new
UnsupportedOperationException()
override def output: Seq[Attribute] = child.output
override protected def withNewChildInternal(newChild: SparkPlan):
UnaryOp1Transformer =
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackStrategiesSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackStrategiesSuite.scala
index 50beded8de..57146b4abf 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackStrategiesSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackStrategiesSuite.scala
@@ -239,7 +239,7 @@ private object GlutenFallbackStrategiesSuite {
// For replacing LeafOp.
case class LeafOpTransformer() extends LeafExecNode with GlutenPlan {
override def batchType(): Convention.BatchType =
BackendsApiManager.getSettings.primaryBatchType
- override def rowType0(): Convention.RowType = Convention.RowType.None
+ override def rowType(): Convention.RowType = Convention.RowType.None
override protected def doExecute(): RDD[InternalRow] = throw new
UnsupportedOperationException()
override def output: Seq[Attribute] = Seq.empty
}
@@ -249,7 +249,7 @@ private object GlutenFallbackStrategiesSuite {
extends UnaryExecNode
with GlutenPlan {
override def batchType(): Convention.BatchType =
BackendsApiManager.getSettings.primaryBatchType
- override def rowType0(): Convention.RowType = Convention.RowType.None
+ override def rowType(): Convention.RowType = Convention.RowType.None
override protected def doExecute(): RDD[InternalRow] = throw new
UnsupportedOperationException()
override def output: Seq[Attribute] = child.output
override protected def withNewChildInternal(newChild: SparkPlan):
UnaryOp1Transformer =
diff --git
a/shims/spark32/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFooterReaderShim.scala
b/shims/spark32/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFooterReaderShim.scala
deleted file mode 100644
index b1419e5e62..0000000000
---
a/shims/spark32/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFooterReaderShim.scala
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.sql.execution.datasources.parquet
-
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.{FileStatus, Path}
-import org.apache.parquet.format.converter.ParquetMetadataConverter
-import org.apache.parquet.hadoop.metadata.ParquetMetadata
-
-/** Shim layer for ParquetFooterReader to maintain compatibility across
different Spark versions. */
-object ParquetFooterReaderShim {
-
- /** @since Spark 4.1 */
- def readFooter(
- configuration: Configuration,
- fileStatus: FileStatus,
- filter: ParquetMetadataConverter.MetadataFilter): ParquetMetadata = {
- ParquetFooterReader.readFooter(configuration, fileStatus, filter)
- }
-
- /** @since Spark 4.1 */
- def readFooter(
- configuration: Configuration,
- file: Path,
- filter: ParquetMetadataConverter.MetadataFilter): ParquetMetadata = {
- ParquetFooterReader.readFooter(configuration, file, filter)
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
