This is an automated email from the ASF dual-hosted git repository.
changchen pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
from b6edc89050 Revert "[GLUTEN-11145][VL] Iceberg test supports Spark3.5
and Spark4.0 (#11146)" (#11374)
new f610deefa0 Spark: Move 4.0 as 4.1
new 7ea23158f2 Spark: Copy back 4.1 as 4.0
new 9026029bc1 Spark: Initial support for 4.1.0
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/util/install-spark-resources.sh | 5 +
.github/workflows/velox_backend_x86.yml | 106 +++++++++++++++++++++
.../gluten/extension/ArrowConvertorRule.scala | 44 +++++----
.../apache/gluten/utils/ParquetMetadataUtils.scala | 4 +-
.../gluten/execution/MiscOperatorSuite.scala | 6 +-
.../gluten/execution/VeloxHashJoinSuite.scala | 7 +-
.../execution/VeloxStringFunctionsSuite.scala | 3 +-
.../python/ArrowEvalPythonExecSuite.scala | 6 +-
dev/format-scala-code.sh | 2 +-
.../gluten/extension/caller/CallerInfo.scala | 9 +-
.../org/apache/spark/util/SparkVersionUtil.scala | 1 +
.../gluten/expression/ExpressionConverter.scala | 7 --
.../gluten/expression/ExpressionMappings.scala | 1 -
.../spark/sql/execution/GlutenImplicits.scala | 8 +-
.../parquet/GlutenParquetRowIndexSuite.scala | 2 +-
pom.xml | 82 +++++++++++++++-
.../org/apache/gluten/sql/shims/SparkShims.scala | 10 ++
shims/pom.xml | 6 ++
.../gluten/sql/shims/spark32/Spark32Shims.scala | 14 ++-
.../parquet/ParquetFooterReaderShim.scala | 26 +++--
.../gluten/sql/shims/spark33/Spark33Shims.scala | 14 ++-
.../parquet/ParquetFooterReaderShim.scala | 26 +++--
.../gluten/sql/shims/spark34/Spark34Shims.scala | 12 +++
.../parquet/ParquetFooterReaderShim.scala | 26 +++--
.../gluten/sql/shims/spark35/Spark35Shims.scala | 12 +++
.../parquet/ParquetFooterReaderShim.scala | 26 +++--
.../gluten/sql/shims/spark40/Spark40Shims.scala | 13 +++
.../parquet/ParquetFooterReaderShim.scala | 26 +++--
.../sql/execution/datasources/v2/Spark35Scan.scala | 50 ----------
shims/{spark35 => spark41}/pom.xml | 4 +-
.../execution/vectorized/ColumnarArrayShim.java | 12 +++
.../vectorized/WritableColumnVectorShim.java | 0
.../org.apache.gluten.sql.shims.SparkShimProvider | 1 +
.../gluten/execution/GenerateTreeStringShim.scala | 6 +-
.../gluten/execution/PartitionedFileUtilShim.scala | 0
.../gluten/sql/shims/spark41/Spark41Shims.scala} | 16 +++-
.../sql/shims/spark41}/SparkShimProvider.scala | 4 +-
.../org/apache/gluten/utils/InternalRowUtl.scala | 0
.../main/scala/org/apache/spark/ShuffleUtils.scala | 0
.../scala/org/apache/spark/SparkContextUtils.scala | 0
.../spark/shuffle/SparkSortShuffleWriterUtil.scala | 0
.../catalyst/expressions/PromotePrecision.scala | 0
.../expressions/objects/InvokeExtractors.scala | 0
.../catalyst/optimizer/CollapseProjectShim.scala | 0
.../apache/spark/sql/classic/ClassicColumn.scala | 0
.../apache/spark/sql/classic/ClassicDataset.scala | 0
.../apache/spark/sql/classic/ClassicTypes.scala | 0
.../org/apache/spark/sql/classic/conversions.scala | 0
.../sql/execution/AbstractFileSourceScanExec.scala | 0
.../execution/ExpandOutputPartitioningShim.scala | 0
.../sql/execution/FileSourceScanExecShim.scala | 1 -
.../sql/execution/GlutenFileFormatWriter.scala | 0
...PartitioningAndOrderingPreservingNodeShim.scala | 0
.../parquet/ParquetFooterReaderShim.scala | 43 +++++++++
.../datasources/v2/AbstractBatchScanExec.scala | 1 +
.../datasources/v2/BatchScanExec.scala.deprecated | 0
.../datasources/v2/BatchScanExecShim.scala | 1 +
.../datasources/v2/utils/CatalogUtil.scala | 0
.../execution/python/BasePythonRunnerShim.scala | 0
.../sql/execution/python/EvalPythonExecBase.scala | 0
.../apache/spark/sql/execution/ui/TypeAlias.scala | 0
.../hive/execution/AbstractHiveTableScanExec.scala | 0
62 files changed, 501 insertions(+), 142 deletions(-)
copy
gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetIOSuite.scala
=>
shims/spark32/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFooterReaderShim.scala
(50%)
copy
gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetIOSuite.scala
=>
shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFooterReaderShim.scala
(50%)
copy
gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetIOSuite.scala
=>
shims/spark34/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFooterReaderShim.scala
(50%)
copy
gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetIOSuite.scala
=>
shims/spark35/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFooterReaderShim.scala
(50%)
copy
gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetIOSuite.scala
=>
shims/spark40/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFooterReaderShim.scala
(50%)
delete mode 100644
shims/spark40/src/main/scala/org/apache/spark/sql/execution/datasources/v2/Spark35Scan.scala
copy shims/{spark35 => spark41}/pom.xml (97%)
copy shims/{spark40 =>
spark41}/src/main/java/org/apache/spark/sql/execution/vectorized/ColumnarArrayShim.java
(95%)
copy shims/{spark40 =>
spark41}/src/main/java/org/apache/spark/sql/execution/vectorized/WritableColumnVectorShim.java
(100%)
create mode 100644
shims/spark41/src/main/resources/META-INF/services/org.apache.gluten.sql.shims.SparkShimProvider
copy shims/{spark40 =>
spark41}/src/main/scala/org/apache/gluten/execution/GenerateTreeStringShim.scala
(94%)
copy shims/{spark40 =>
spark41}/src/main/scala/org/apache/gluten/execution/PartitionedFileUtilShim.scala
(100%)
copy
shims/{spark40/src/main/scala/org/apache/gluten/sql/shims/spark40/Spark40Shims.scala
=>
spark41/src/main/scala/org/apache/gluten/sql/shims/spark41/Spark41Shims.scala}
(98%)
copy shims/{spark32/src/main/scala/org/apache/gluten/sql/shims/spark32 =>
spark41/src/main/scala/org/apache/gluten/sql/shims/spark41}/SparkShimProvider.scala
(93%)
copy shims/{spark35 =>
spark41}/src/main/scala/org/apache/gluten/utils/InternalRowUtl.scala (100%)
copy shims/{spark34 =>
spark41}/src/main/scala/org/apache/spark/ShuffleUtils.scala (100%)
copy shims/{spark34 =>
spark41}/src/main/scala/org/apache/spark/SparkContextUtils.scala (100%)
copy shims/{spark34 =>
spark41}/src/main/scala/org/apache/spark/shuffle/SparkSortShuffleWriterUtil.scala
(100%)
copy shims/{spark34 =>
spark41}/src/main/scala/org/apache/spark/sql/catalyst/expressions/PromotePrecision.scala
(100%)
copy shims/{spark40 =>
spark41}/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/InvokeExtractors.scala
(100%)
copy shims/{spark33 =>
spark41}/src/main/scala/org/apache/spark/sql/catalyst/optimizer/CollapseProjectShim.scala
(100%)
copy shims/{spark40 =>
spark41}/src/main/scala/org/apache/spark/sql/classic/ClassicColumn.scala (100%)
copy shims/{spark40 =>
spark41}/src/main/scala/org/apache/spark/sql/classic/ClassicDataset.scala (100%)
copy shims/{spark40 =>
spark41}/src/main/scala/org/apache/spark/sql/classic/ClassicTypes.scala (100%)
copy shims/{spark40 =>
spark41}/src/main/scala/org/apache/spark/sql/classic/conversions.scala (100%)
copy shims/{spark40 =>
spark41}/src/main/scala/org/apache/spark/sql/execution/AbstractFileSourceScanExec.scala
(100%)
copy shims/{spark34 =>
spark41}/src/main/scala/org/apache/spark/sql/execution/ExpandOutputPartitioningShim.scala
(100%)
copy shims/{spark40 =>
spark41}/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala
(99%)
copy shims/{spark40 =>
spark41}/src/main/scala/org/apache/spark/sql/execution/GlutenFileFormatWriter.scala
(100%)
copy shims/{spark34 =>
spark41}/src/main/scala/org/apache/spark/sql/execution/PartitioningAndOrderingPreservingNodeShim.scala
(100%)
create mode 100644
shims/spark41/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFooterReaderShim.scala
copy shims/{spark40 =>
spark41}/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AbstractBatchScanExec.scala
(99%)
copy shims/{spark34 =>
spark41}/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExec.scala.deprecated
(100%)
copy shims/{spark40 =>
spark41}/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala
(99%)
copy shims/{spark40 =>
spark41}/src/main/scala/org/apache/spark/sql/execution/datasources/v2/utils/CatalogUtil.scala
(100%)
copy shims/{spark40 =>
spark41}/src/main/scala/org/apache/spark/sql/execution/python/BasePythonRunnerShim.scala
(100%)
copy shims/{spark40 =>
spark41}/src/main/scala/org/apache/spark/sql/execution/python/EvalPythonExecBase.scala
(100%)
copy shims/{spark40 =>
spark41}/src/main/scala/org/apache/spark/sql/execution/ui/TypeAlias.scala (100%)
copy shims/{spark34 =>
spark41}/src/main/scala/org/apache/spark/sql/hive/execution/AbstractHiveTableScanExec.scala
(100%)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
