This is an automated email from the ASF dual-hosted git repository.
hongze pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
from 33d2f2d31 [VL] Following #6526, minor fixes and improvements (#6554)
add f75d7c154 [VL] Row based sort shuffle implementation (#6475)
No new revisions were added by this update.
Summary of changes:
.../gluten/backendsapi/velox/VeloxMetricsApi.scala | 24 +-
.../backendsapi/velox/VeloxSparkPlanExecApi.scala | 39 +--
cpp/core/CMakeLists.txt | 1 +
cpp/core/jni/JniWrapper.cc | 19 +-
.../operators/serializer/ColumnarBatchSerializer.h | 2 +-
cpp/core/shuffle/FallbackRangePartitioner.cc | 5 +-
cpp/core/shuffle/FallbackRangePartitioner.h | 6 +-
cpp/core/shuffle/HashPartitioner.cc | 14 +-
cpp/core/shuffle/HashPartitioner.h | 6 +-
cpp/core/shuffle/LocalPartitionWriter.cc | 201 ++++++++-----
cpp/core/shuffle/LocalPartitionWriter.h | 10 +-
cpp/core/shuffle/Options.h | 2 +-
cpp/core/shuffle/PartitionWriter.h | 7 +-
cpp/core/shuffle/Partitioner.cc | 5 +-
cpp/core/shuffle/Partitioner.h | 9 +-
cpp/core/shuffle/Payload.cc | 127 +++++----
cpp/core/shuffle/Payload.h | 18 +-
cpp/core/shuffle/RandomPartitioner.cc | 14 +-
cpp/core/shuffle/RandomPartitioner.h | 6 +-
cpp/core/shuffle/RoundRobinPartitioner.cc | 10 +-
cpp/core/shuffle/RoundRobinPartitioner.h | 6 +-
cpp/core/shuffle/ShuffleWriter.cc | 92 ++++++
cpp/core/shuffle/ShuffleWriter.h | 78 ++---
cpp/core/shuffle/SinglePartitioner.cc | 10 +-
cpp/core/shuffle/SinglePartitioner.h | 6 +-
cpp/core/shuffle/Spill.cc | 11 +-
cpp/core/shuffle/Spill.h | 6 +-
cpp/core/shuffle/rss/RssPartitionWriter.cc | 14 +-
cpp/core/shuffle/rss/RssPartitionWriter.h | 5 +-
cpp/core/tests/RoundRobinPartitionerTest.cc | 57 ++--
cpp/velox/CMakeLists.txt | 5 +-
cpp/velox/benchmarks/ColumnarToRowBenchmark.cc | 9 +-
cpp/velox/benchmarks/GenericBenchmark.cc | 7 +-
cpp/velox/compute/VeloxRuntime.cc | 4 +-
.../serializer/VeloxColumnarBatchSerializer.cc | 2 +-
...dShuffleWriter.cc => VeloxHashShuffleWriter.cc} | 140 ++++-----
...sedShuffleWriter.h => VeloxHashShuffleWriter.h} | 9 +-
...uffleWriter.cc => VeloxRssSortShuffleWriter.cc} | 43 ++-
...ShuffleWriter.h => VeloxRssSortShuffleWriter.h} | 5 +-
cpp/velox/shuffle/VeloxShuffleReader.cc | 232 +++++++++++----
cpp/velox/shuffle/VeloxShuffleReader.h | 52 +++-
cpp/velox/shuffle/VeloxShuffleWriter.cc | 16 +-
cpp/velox/shuffle/VeloxShuffleWriter.h | 31 +-
cpp/velox/shuffle/VeloxSortShuffleWriter.cc | 314 +++++++++++++++++++++
cpp/velox/shuffle/VeloxSortShuffleWriter.h | 119 ++++++++
cpp/velox/tests/VeloxShuffleWriterTest.cc | 43 ++-
cpp/velox/utils/tests/VeloxShuffleWriterTestBase.h | 23 +-
.../shuffle/CelebornColumnarShuffleWriter.scala | 4 +-
.../VeloxCelebornColumnarBatchSerializer.scala | 4 +-
.../execution/ColumnarShuffleExchangeExec.scala | 16 +-
.../gluten/vectorized/GlutenSplitResult.java | 24 +-
.../vectorized/ColumnarBatchSerializer.scala | 12 +-
.../spark/shuffle/ColumnarShuffleWriter.scala | 31 +-
.../writer/VeloxUniffleColumnarShuffleWriter.java | 4 +-
.../scala/org/apache/gluten/GlutenConfig.scala | 32 ++-
55 files changed, 1343 insertions(+), 648 deletions(-)
create mode 100644 cpp/core/shuffle/ShuffleWriter.cc
rename cpp/velox/shuffle/{VeloxHashBasedShuffleWriter.cc =>
VeloxHashShuffleWriter.cc} (91%)
rename cpp/velox/shuffle/{VeloxHashBasedShuffleWriter.h =>
VeloxHashShuffleWriter.h} (98%)
rename cpp/velox/shuffle/{VeloxSortBasedShuffleWriter.cc =>
VeloxRssSortShuffleWriter.cc} (85%)
rename cpp/velox/shuffle/{VeloxSortBasedShuffleWriter.h =>
VeloxRssSortShuffleWriter.h} (96%)
create mode 100644 cpp/velox/shuffle/VeloxSortShuffleWriter.cc
create mode 100644 cpp/velox/shuffle/VeloxSortShuffleWriter.h
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
