This is an automated email from the ASF dual-hosted git repository.
zhli pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new c0c8aa70c6 [VL] Refactor: move arenas_ from VeloxShuffleWriter to
VeloxHashShuffleWriter (#11111)
c0c8aa70c6 is described below
commit c0c8aa70c6eea19b55d24d4d35383bbbc67c7a08
Author: Zhen Li <[email protected]>
AuthorDate: Wed Nov 19 18:47:25 2025 +0800
[VL] Refactor: move arenas_ from VeloxShuffleWriter to
VeloxHashShuffleWriter (#11111)
Refactor: move arenas_ from VeloxShuffleWriter to VeloxHashShuffleWriter.
---
cpp/velox/shuffle/VeloxHashShuffleWriter.h | 6 +++++-
cpp/velox/shuffle/VeloxShuffleWriter.h | 3 ---
2 files changed, 5 insertions(+), 4 deletions(-)
diff --git a/cpp/velox/shuffle/VeloxHashShuffleWriter.h
b/cpp/velox/shuffle/VeloxHashShuffleWriter.h
index 06e13db773..fa2f3ea040 100644
--- a/cpp/velox/shuffle/VeloxHashShuffleWriter.h
+++ b/cpp/velox/shuffle/VeloxHashShuffleWriter.h
@@ -196,7 +196,9 @@ class VeloxHashShuffleWriter : public VeloxShuffleWriter {
MemoryManager* memoryManager)
: VeloxShuffleWriter(numPartitions, partitionWriter, options,
memoryManager),
splitBufferSize_(options->splitBufferSize),
- splitBufferReallocThreshold_(options->splitBufferReallocThreshold) {}
+ splitBufferReallocThreshold_(options->splitBufferReallocThreshold) {
+ arenas_.resize(numPartitions);
+ }
arrow::Status init();
@@ -412,6 +414,8 @@ class VeloxHashShuffleWriter : public VeloxShuffleWriter {
SplitState splitState_{kInit};
std::optional<uint32_t> partitionBufferInUse_{std::nullopt};
+
+ std::vector<std::unique_ptr<facebook::velox::StreamArena>> arenas_;
}; // class VeloxHashBasedShuffleWriter
} // namespace gluten
diff --git a/cpp/velox/shuffle/VeloxShuffleWriter.h
b/cpp/velox/shuffle/VeloxShuffleWriter.h
index 58131442ad..45276e3cd4 100644
--- a/cpp/velox/shuffle/VeloxShuffleWriter.h
+++ b/cpp/velox/shuffle/VeloxShuffleWriter.h
@@ -124,7 +124,6 @@ class VeloxShuffleWriter : public ShuffleWriter {
veloxPool_(dynamic_cast<VeloxMemoryManager*>(memoryManager)->getLeafMemoryPool()),
partitionWriter_(partitionWriter) {
partitioner_ = Partitioner::make(options->partitioning, numPartitions_,
options->startPartitionId);
- arenas_.resize(numPartitions);
serdeOptions_.useLosslessTimestamp = true;
}
@@ -142,8 +141,6 @@ class VeloxShuffleWriter : public ShuffleWriter {
std::shared_ptr<Partitioner> partitioner_;
- std::vector<std::unique_ptr<facebook::velox::StreamArena>> arenas_;
-
facebook::velox::serializer::presto::PrestoVectorSerde::PrestoOptions
serdeOptions_;
int32_t maxBatchSize_{0};
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
