This is an automated email from the ASF dual-hosted git repository.
wangzhen pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 7b7ef95c65 [GLUTEN-10920][VL] Allow disabling hash/sort shuffle reader
buffer (#10922)
7b7ef95c65 is described below
commit 7b7ef95c65cf911f03dc9bc7ed55c07fc408a03b
Author: Zhen Wang <[email protected]>
AuthorDate: Thu Oct 23 10:29:58 2025 +0800
[GLUTEN-10920][VL] Allow disabling hash/sort shuffle reader buffer (#10922)
---
cpp/velox/shuffle/VeloxShuffleReader.cc | 24 ++++++++++++++++--------
1 file changed, 16 insertions(+), 8 deletions(-)
diff --git a/cpp/velox/shuffle/VeloxShuffleReader.cc
b/cpp/velox/shuffle/VeloxShuffleReader.cc
index 00e2225744..5e7b78f00d 100644
--- a/cpp/velox/shuffle/VeloxShuffleReader.cc
+++ b/cpp/velox/shuffle/VeloxShuffleReader.cc
@@ -501,10 +501,14 @@ void VeloxHashShuffleReaderDeserializer::loadNextStream()
{
return;
}
- GLUTEN_ASSIGN_OR_THROW(
- in_,
- arrow::io::BufferedInputStream::Create(
- readerBufferSize_, memoryManager_->defaultArrowMemoryPool(),
std::move(in)));
+ if (readerBufferSize_ > 0) {
+ GLUTEN_ASSIGN_OR_THROW(
+ in_,
+ arrow::io::BufferedInputStream::Create(
+ readerBufferSize_, memoryManager_->defaultArrowMemoryPool(),
std::move(in)));
+ } else {
+ in_ = std::move(in);
+ }
}
std::shared_ptr<ColumnarBatch> VeloxHashShuffleReaderDeserializer::next() {
@@ -656,10 +660,14 @@ void VeloxSortShuffleReaderDeserializer::loadNextStream()
{
GLUTEN_ASSIGN_OR_THROW(
in_, CompressedInputStream::Make(codec_.get(), std::move(in),
memoryManager_->defaultArrowMemoryPool()));
} else {
- GLUTEN_ASSIGN_OR_THROW(
- in_,
- arrow::io::BufferedInputStream::Create(
- readerBufferSize_, memoryManager_->defaultArrowMemoryPool(),
std::move(in)));
+ if (readerBufferSize_ > 0) {
+ GLUTEN_ASSIGN_OR_THROW(
+ in_,
+ arrow::io::BufferedInputStream::Create(
+ readerBufferSize_, memoryManager_->defaultArrowMemoryPool(),
std::move(in)));
+ } else {
+ in_ = std::move(in);
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
