This is an automated email from the ASF dual-hosted git repository.
marin-ma pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 4df6e9cd10 [VL][FOLLOWUP] Reduce Velox hash shuffle partition buffer
memory by evicting large partitions after split (#12156)
4df6e9cd10 is described below
commit 4df6e9cd10a2a5b232eebd1390f9baa0bc3ec6e5
Author: wankun <[email protected]>
AuthorDate: Thu May 28 17:19:01 2026 +0800
[VL][FOLLOWUP] Reduce Velox hash shuffle partition buffer memory by
evicting large partitions after split (#12156)
---
cpp/velox/shuffle/VeloxHashShuffleWriter.cc | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/cpp/velox/shuffle/VeloxHashShuffleWriter.cc
b/cpp/velox/shuffle/VeloxHashShuffleWriter.cc
index 89f1dbcda6..dfb799806a 100644
--- a/cpp/velox/shuffle/VeloxHashShuffleWriter.cc
+++ b/cpp/velox/shuffle/VeloxHashShuffleWriter.cc
@@ -440,16 +440,17 @@ arrow::Status VeloxHashShuffleWriter::doSplit(const
facebook::velox::RowVector&
printPartitionBuffer();
- setSplitState(SplitState::kInit);
if (partitionBufferEvictThreshold_ > 0) {
// After split, evict large partition buffers to free up memory for the
next input RowVector.
const auto partitionBytes = estimatePartitionBufferBytes();
for (uint32_t pid = 0; pid < partitionBytes.size(); ++pid) {
if (partitionBufferBase_[pid] > 0 && partitionBytes[pid] >=
partitionBufferEvictThreshold_) {
+ PartitionScopeGuard guard(partitionBufferInUse_, pid);
RETURN_NOT_OK(evictPartitionBuffers(pid, false));
}
}
}
+ setSplitState(SplitState::kInit);
return arrow::Status::OK();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
