This is an automated email from the ASF dual-hosted git repository.
rui pushed a change to branch wip_data_col
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
omit 5f06cb4754 test
omit d80c627cb2 Use empty data columns
add 3db705794b [VL] Manually register velox conf on executor start (#9721)
add d1f48d7e94 [VL] Add missing doc for
spark.gluten.sql.columnar.backend.velox.showTaskMetricsWhenFinished (#9738)
add 6c9fc228e2 [GLUTEN-9728][CH] Fix: Inconsistent result from
JoinAggregateToAggregateUnion (#9729)
add 93e101e2d9 [GLUTEN-6887][VL] Daily Update Velox Version (2025_05_24)
(#9742)
add ac97e4f863 [FLINK] Fix error package name (#9747)
add 9bbe1f0970 [GLUTEN-9736][VL] Fix store ID mismatch (#9746)
add 08353b9faa [GLUTEN-9561][CH] Upgrade delta version to 3.3.1 (#9562)
add 4946bfd086 [GLUTEN-9666][CORE][VL][CH] V1 write: Move from FakeRow to
BatchCarrierRow to simplify code (#9731)
add 61ab633d21 [GLUTEN-9682][Flink] Support double in vector-to-row
conversion (#9683)
add 117ea1e04e [GLUTEN-6609][CH] Remove unnecessary readFile override in
GlutenDiskS3. (#9649)
add 1a6ef61351 Add a Shim layer for PartitionedFileUtil of Spark 3.5
(#9748)
add fec8708ef8 Use column names
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (5f06cb4754)
\
N -- N -- N refs/heads/wip_data_col (fec8708ef8)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.../org.apache.gluten.sql.shims.DeltaShimProvider | 1 -
.../spark/sql/delta/commands/DeleteCommand.scala | 554 ------------------
.../sql/delta/commands/OptimizeTableCommand.scala | 566 -------------------
.../commands/OptimizeTableCommandOverwrites.scala | 335 -----------
.../spark/sql/delta/commands/UpdateCommand.scala | 553 ------------------
.../commands/merge/ClassicMergeExecutor.scala | 569 -------------------
.../gluten/vectorized/DeltaWriterJNIWrapper.java | 2 +-
.../org.apache.gluten.sql.shims.DeltaShimProvider | 1 +
.../scala/io/delta/tables/ClickhouseTable.scala | 0
.../DeltaExpressionExtensionTransformer.scala | 0
.../gluten/parser/GlutenCacheFilesSqlParser.scala | 0
.../gluten/parser/GlutenClickhouseSqlParser.scala | 0
.../sql/shims/delta33/Delta33ShimProvider.scala} | 6 +-
.../gluten/sql/shims/delta33/Delta33Shims.scala} | 6 +-
.../delta/ClickhouseOptimisticTransaction.scala | 0
.../org/apache/spark/sql/delta/DeltaAdapter.scala | 0
.../org/apache/spark/sql/delta/DeltaLog.scala | 250 +++++----
.../spark/sql/delta/PreprocessTableWithDVs.scala | 0
.../org/apache/spark/sql/delta/Snapshot.scala | 380 ++++++++++---
.../sql/delta/catalog/ClickHouseTableV2.scala | 0
.../commands/DMLWithDeletionVectorsHelper.scala | 63 ++-
.../spark/sql/delta/commands/VacuumCommand.scala | 607 +++++++++++++++-----
.../merge/MergeIntoMaterializeSource.scala | 254 +++++----
.../files/MergeTreeDelayedCommitProtocol.scala | 6 +-
.../rules/CHOptimizeMetadataOnlyDeltaQuery.scala | 0
.../execution/DeletionVectorWriteTransformer.scala | 4 +-
.../sql/execution/FileDeltaColumnarWrite.scala | 31 +-
.../execution/MergeTreeDeltaColumnarWrite.scala | 2 +-
.../sql/execution/datasources/DeltaV1Writes.scala | 0
.../clickhouse/MergeTreeFileFormatDataWriter.scala | 61 +-
.../v1/clickhouse/MergeTreeFileFormatWriter.scala | 8 +-
.../v2/clickhouse/ClickHouseDataSource.scala | 0
.../v2/clickhouse/ClickHouseSparkCatalog.scala | 0
.../v2/clickhouse/metadata/AddFileTags.scala | 0
.../source/DeltaMergeTreeFileFormat.scala | 0
.../spark/gluten/delta/DeltaStatsUtils.scala | 0
.../GlutenDeltaMergeTreeDeletionVectorSuite.scala | 6 +-
.../GlutenDeltaParquetDeletionVectorSuite.scala | 0
.../sql/execution/GlutenDeltaExpressionSuite.scala | 0
.../apache/spark/sql/execution/RDDScanSuite.scala | 0
.../execution/datasources/DeltaV1WritesSuite.scala | 0
.../columnar/KafkaMiscColumnarRules.scala | 6 +-
.../gluten/backendsapi/clickhouse/CHBackend.scala | 1 -
.../clickhouse}/CHBatchType.scala | 2 +-
.../backendsapi/clickhouse/CHCarrierRowType.scala | 9 +-
.../backendsapi/clickhouse/CHListenerApi.scala | 2 +-
.../gluten/backendsapi/clickhouse/CHRuleApi.scala | 1 -
.../clickhouse/CHSparkPlanExecApi.scala | 3 +
.../execution/CHColumnarToCarrierRowExec.scala | 29 +-
.../extension/CHRemoveTopmostColumnarToRow.scala | 4 +-
.../extension/CoalesceAggregationUnion.scala | 4 +-
.../extension/JoinAggregateToAggregateUnion.scala | 128 +++--
.../spark/sql/execution/CHColumnarWrite.scala | 16 +-
.../sql/execution/CHColumnarWriteFilesExec.scala | 5 +-
.../spark/sql/execution/CHRDDScanTransformer.scala | 21 +-
.../datasources/v1/CHFormatWriterInjects.scala | 5 +-
.../datasources/v1/CHMergeTreeWriterInjects.scala | 12 +-
.../datasources/v1/FakeRowOutputWriter.scala | 17 +-
.../GlutenClickHouseDeltaParquetWriteSuite.scala | 1 +
.../GlutenCoalesceAggregationUnionSuite.scala | 6 +-
.../execution/GlutenEliminateJoinSuite.scala | 42 ++
.../GlutenClickHouseMergeTreeOptimizeSuite.scala | 8 +-
.../GlutenClickHouseMergeTreeWriteOnS3Suite.scala | 12 +-
.../org/apache/gluten/utils/HDFSTestHelper.scala | 6 +-
.../apache/spark/gluten/NativeWriteChecker.scala | 5 +-
.../gluten/backendsapi/velox/VeloxBatchType.scala | 3 +-
.../backendsapi/velox/VeloxCarrierRowType.scala | 5 +-
.../backendsapi/velox/VeloxListenerApi.scala | 25 +-
.../backendsapi/velox/VeloxSparkPlanExecApi.scala | 3 +
.../execution/VeloxColumnarToCarrierRowExec.scala | 10 +-
.../velox/VeloxFormatWriterInjects.scala | 28 +-
.../VeloxListenerApiTest.java} | 30 +-
.../apache/gluten/execution/VeloxScanSuite.scala | 2 +-
.../execution/VeloxParquetWriteForHiveSuite.scala | 5 +-
cpp-ch/local-engine/Common/Base85Codec.h | 1 +
.../Disks/ObjectStorages/GlutenDiskS3.cpp | 12 -
.../Disks/ObjectStorages/GlutenDiskS3.h | 4 -
.../Delta/Bitmap/DeltaDVRoaringBitmapArray.cpp | 11 +-
.../Storages/SubstraitSource/Delta/DeltaWriter.cpp | 110 ++--
.../Storages/SubstraitSource/Delta/DeltaWriter.h | 36 +-
cpp-ch/local-engine/local_engine_jni.cpp | 5 +-
cpp/core/utils/ObjectStore.cc | 11 +
cpp/core/utils/ObjectStore.h | 10 +-
.../serializer/VeloxColumnarToRowConverter.cc | 1 -
.../serializer/VeloxRowToColumnarConverter.cc | 8 +-
cpp/velox/substrait/SubstraitToVeloxPlan.cc | 27 +-
docs/Configuration.md | 2 +
docs/get-started/Velox.md | 6 +-
ep/build-velox/src/get_velox.sh | 2 +-
.../execution/DeltaFilterExecTransformer.scala | 0
.../execution/DeltaProjectExecTransformer.scala | 0
.../sql/perf/DeltaOptimizedWriterTransformer.scala | 0
.../apache/gluten/rexnode/RexNodeConverter.java | 4 +-
.../rexnode/functions/BaseRexCallConverters.java | 3 +-
.../rexnode/functions/ModRexCallConverter.java | 6 +-
.../gluten/rexnode/functions/RexCallConverter.java | 12 +-
.../rexnode/functions/RexCallConverterFactory.java | 2 +-
.../apache/gluten/util/LogicalTypeConverter.java | 3 +
.../gluten/vectorized/ArrowVectorAccessor.java | 142 +++++
.../gluten/vectorized/ArrowVectorWriter.java | 26 +-
.../vectorized/FlinkRowToVLVectorConvertor.java | 37 +-
.../table/runtime/stream/custom/ScanTest.java | 33 +-
.../gluten/backendsapi/SparkPlanExecApi.scala | 2 +
.../execution/ColumnarToCarrierRowExecBase.scala | 34 +-
.../execution/datasources/FakeRowEnhancement.scala | 44 --
.../GlutenFormatWriterInjectsBase.scala | 7 +-
.../datasources/GlutenWriterColumnarRules.scala | 69 +--
.../ClickHouseAdaptiveQueryExecSuite.scala | 11 +-
.../velox/VeloxAdaptiveQueryExecSuite.scala | 11 +-
.../ClickHouseAdaptiveQueryExecSuite.scala | 11 +-
.../velox/VeloxAdaptiveQueryExecSuite.scala | 11 +-
.../ClickHouseAdaptiveQueryExecSuite.scala | 11 +-
.../velox/VeloxAdaptiveQueryExecSuite.scala | 11 +-
.../ClickHouseAdaptiveQueryExecSuite.scala | 11 +-
.../velox/VeloxAdaptiveQueryExecSuite.scala | 11 +-
.../datasources/GlutenNoopWriterRuleSuite.scala | 9 +-
pom.xml | 4 +-
.../apache/gluten/execution}/BatchCarrierRow.scala | 27 +-
.../datasource/GlutenFormatWriterInjects.scala | 7 +-
.../spark/sql/execution/datasources/FakeRow.scala | 74 ---
.../datasources/BasicWriteStatsTracker.scala | 260 ---------
.../datasources/FileFormatDataWriter.scala | 601 +-------------------
.../execution/datasources/FileFormatWriter.scala | 8 +-
.../datasources/BasicWriteStatsTracker.scala | 252 ---------
.../datasources/FileFormatDataWriter.scala | 619 +--------------------
.../execution/datasources/FileFormatWriter.scala | 6 +-
.../gluten/execution/PartitionedFileUtilShim.scala | 149 +++++
.../gluten/sql/shims/spark35/Spark35Shims.scala | 5 +-
.../sql/execution/AbstractFileSourceScanExec.scala | 6 +-
129 files changed, 2191 insertions(+), 5315 deletions(-)
delete mode 100644
backends-clickhouse/src-delta-32/main/resources/META-INF/services/org.apache.gluten.sql.shims.DeltaShimProvider
delete mode 100644
backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/commands/DeleteCommand.scala
delete mode 100644
backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala
delete mode 100644
backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala
delete mode 100644
backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/commands/UpdateCommand.scala
delete mode 100644
backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/commands/merge/ClassicMergeExecutor.scala
rename backends-clickhouse/{src-delta-32 =>
src-delta-33}/main/java/org/apache/gluten/vectorized/DeltaWriterJNIWrapper.java
(97%)
create mode 100644
backends-clickhouse/src-delta-33/main/resources/META-INF/services/org.apache.gluten.sql.shims.DeltaShimProvider
rename backends-clickhouse/{src-delta-32 =>
src-delta-33}/main/scala/io/delta/tables/ClickhouseTable.scala (100%)
rename backends-clickhouse/{src-delta-32 =>
src-delta-33}/main/scala/org/apache/gluten/extension/DeltaExpressionExtensionTransformer.scala
(100%)
rename backends-clickhouse/{src-delta-32 =>
src-delta-33}/main/scala/org/apache/gluten/parser/GlutenCacheFilesSqlParser.scala
(100%)
rename backends-clickhouse/{src-delta-32 =>
src-delta-33}/main/scala/org/apache/gluten/parser/GlutenClickhouseSqlParser.scala
(100%)
rename
backends-clickhouse/{src-delta-32/main/scala/org/apache/gluten/sql/shims/delta32/Delta32ShimProvider.scala
=>
src-delta-33/main/scala/org/apache/gluten/sql/shims/delta33/Delta33ShimProvider.scala}
(88%)
rename
backends-clickhouse/{src-delta-32/main/scala/org/apache/gluten/sql/shims/delta32/Delta32Shims.scala
=>
src-delta-33/main/scala/org/apache/gluten/sql/shims/delta33/Delta33Shims.scala}
(96%)
rename backends-clickhouse/{src-delta-32 =>
src-delta-33}/main/scala/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala
(100%)
rename backends-clickhouse/{src-delta-32 =>
src-delta-33}/main/scala/org/apache/spark/sql/delta/DeltaAdapter.scala (100%)
rename backends-clickhouse/{src-delta-32 =>
src-delta-33}/main/scala/org/apache/spark/sql/delta/DeltaLog.scala (88%)
rename backends-clickhouse/{src-delta-32 =>
src-delta-33}/main/scala/org/apache/spark/sql/delta/PreprocessTableWithDVs.scala
(100%)
rename backends-clickhouse/{src-delta-32 =>
src-delta-33}/main/scala/org/apache/spark/sql/delta/Snapshot.scala (59%)
rename backends-clickhouse/{src-delta-32 =>
src-delta-33}/main/scala/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala
(100%)
rename backends-clickhouse/{src-delta-32 =>
src-delta-33}/main/scala/org/apache/spark/sql/delta/commands/DMLWithDeletionVectorsHelper.scala
(96%)
rename backends-clickhouse/{src-delta-32 =>
src-delta-33}/main/scala/org/apache/spark/sql/delta/commands/VacuumCommand.scala
(53%)
rename backends-clickhouse/{src-delta-32 =>
src-delta-33}/main/scala/org/apache/spark/sql/delta/commands/merge/MergeIntoMaterializeSource.scala
(71%)
rename backends-clickhouse/{src-delta-32 =>
src-delta-33}/main/scala/org/apache/spark/sql/delta/files/MergeTreeDelayedCommitProtocol.scala
(92%)
rename backends-clickhouse/{src-delta-32 =>
src-delta-33}/main/scala/org/apache/spark/sql/delta/rules/CHOptimizeMetadataOnlyDeltaQuery.scala
(100%)
rename backends-clickhouse/{src-delta-32 =>
src-delta-33}/main/scala/org/apache/spark/sql/execution/DeletionVectorWriteTransformer.scala
(97%)
rename backends-clickhouse/{src-delta-32 =>
src-delta-33}/main/scala/org/apache/spark/sql/execution/FileDeltaColumnarWrite.scala
(87%)
rename backends-clickhouse/{src-delta-32 =>
src-delta-33}/main/scala/org/apache/spark/sql/execution/MergeTreeDeltaColumnarWrite.scala
(99%)
rename backends-clickhouse/{src-delta-32 =>
src-delta-33}/main/scala/org/apache/spark/sql/execution/datasources/DeltaV1Writes.scala
(100%)
rename backends-clickhouse/{src-delta-32 =>
src-delta-33}/main/scala/org/apache/spark/sql/execution/datasources/v1/clickhouse/MergeTreeFileFormatDataWriter.scala
(93%)
rename backends-clickhouse/{src-delta-32 =>
src-delta-33}/main/scala/org/apache/spark/sql/execution/datasources/v1/clickhouse/MergeTreeFileFormatWriter.scala
(97%)
rename backends-clickhouse/{src-delta-32 =>
src-delta-33}/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala
(100%)
rename backends-clickhouse/{src-delta-32 =>
src-delta-33}/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala
(100%)
rename backends-clickhouse/{src-delta-32 =>
src-delta-33}/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/metadata/AddFileTags.scala
(100%)
rename backends-clickhouse/{src-delta-32 =>
src-delta-33}/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala
(100%)
rename backends-clickhouse/{src-delta-32 =>
src-delta-33}/test/scala/org/apache/spark/gluten/delta/DeltaStatsUtils.scala
(100%)
rename backends-clickhouse/{src-delta-32 =>
src-delta-33}/test/scala/org/apache/spark/gluten/delta/GlutenDeltaMergeTreeDeletionVectorSuite.scala
(97%)
rename backends-clickhouse/{src-delta-32 =>
src-delta-33}/test/scala/org/apache/spark/gluten/delta/GlutenDeltaParquetDeletionVectorSuite.scala
(100%)
rename backends-clickhouse/{src-delta-32 =>
src-delta-33}/test/scala/org/apache/spark/sql/execution/GlutenDeltaExpressionSuite.scala
(100%)
rename backends-clickhouse/{src-delta-32 =>
src-delta-33}/test/scala/org/apache/spark/sql/execution/RDDScanSuite.scala
(100%)
rename backends-clickhouse/{src-delta-32 =>
src-delta-33}/test/scala/org/apache/spark/sql/execution/datasources/DeltaV1WritesSuite.scala
(100%)
rename backends-clickhouse/src/main/scala/org/apache/gluten/{columnarbatch =>
backendsapi/clickhouse}/CHBatchType.scala (97%)
copy
backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxCarrierRowType.scala
=>
backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHCarrierRowType.scala
(74%)
copy
backends-velox/src/main/scala/org/apache/gluten/execution/VeloxColumnarToCarrierRowExec.scala
=>
backends-clickhouse/src/main/scala/org/apache/gluten/execution/CHColumnarToCarrierRowExec.scala
(53%)
copy
backends-velox/src/test/java/org/apache/gluten/{test/VeloxBackendTestBase.java
=> backendsapi/VeloxListenerApiTest.java} (56%)
rename gluten-delta/{src-delta-32 =>
src-delta-33}/main/scala/org/apache/gluten/execution/DeltaFilterExecTransformer.scala
(100%)
rename gluten-delta/{src-delta-32 =>
src-delta-33}/main/scala/org/apache/gluten/execution/DeltaProjectExecTransformer.scala
(100%)
rename gluten-delta/{src-delta-32 =>
src-delta-33}/main/scala/org/apache/spark/sql/perf/DeltaOptimizedWriterTransformer.scala
(100%)
create mode 100644
gluten-flink/runtime/src/main/java/org/apache/gluten/vectorized/ArrowVectorAccessor.java
delete mode 100644
gluten-substrait/src/main/scala/org/apache/spark/sql/execution/datasources/FakeRowEnhancement.scala
rename
{gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/batchcarrier
=>
shims/common/src/main/scala/org/apache/gluten/execution}/BatchCarrierRow.scala
(79%)
delete mode 100644
shims/common/src/main/scala/org/apache/spark/sql/execution/datasources/FakeRow.scala
delete mode 100644
shims/spark32/src/main/scala/org/apache/spark/sql/execution/datasources/BasicWriteStatsTracker.scala
delete mode 100644
shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/BasicWriteStatsTracker.scala
create mode 100644
shims/spark35/src/main/scala/org/apache/gluten/execution/PartitionedFileUtilShim.scala
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
