This is an automated email from the ASF dual-hosted git repository.
liuneng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new d0dcc1e40 [CH] Disable automatic switching of sort shuffle (#6015)
d0dcc1e40 is described below
commit d0dcc1e401ee1600f1e9716f5dffc1befb059634
Author: LiuNeng <[email protected]>
AuthorDate: Fri Jun 7 18:02:01 2024 +0800
[CH] Disable automatic switching of sort shuffle (#6015)
What changes were proposed in this pull request?
Disable automatic switching of sort shuffle
How was this patch tested?
unit tests
(If this patch involves UI changes, please attach a screenshot; otherwise,
remove this)
---
cpp-ch/local-engine/Shuffle/CachedShuffleWriter.cpp | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/cpp-ch/local-engine/Shuffle/CachedShuffleWriter.cpp
b/cpp-ch/local-engine/Shuffle/CachedShuffleWriter.cpp
index 559d90318..fd6f6fd81 100644
--- a/cpp-ch/local-engine/Shuffle/CachedShuffleWriter.cpp
+++ b/cpp-ch/local-engine/Shuffle/CachedShuffleWriter.cpp
@@ -133,11 +133,11 @@ void CachedShuffleWriter::lazyInitPartitionWriter(Block &
input_sample)
if (partition_writer)
return;
- auto avg_row_size = input_sample.allocatedBytes() / input_sample.rows();
- auto overhead_memory = std::max(avg_row_size, input_sample.columns() * 16)
* options.split_size * options.partition_num;
- auto use_sort_shuffle = overhead_memory > options.spill_threshold * 0.5 ||
options.partition_num >= 300;
+// auto avg_row_size = input_sample.allocatedBytes() / input_sample.rows();
+// auto overhead_memory = std::max(avg_row_size, input_sample.columns() *
16) * options.split_size * options.partition_num;
+// auto use_sort_shuffle = overhead_memory > options.spill_threshold * 0.5
|| options.partition_num >= 300;
auto use_external_sort_shuffle = options.force_external_sort;
- auto use_memory_sort_shuffle = options.force_mermory_sort ||
use_sort_shuffle;
+ auto use_memory_sort_shuffle = options.force_mermory_sort;
sort_shuffle = use_memory_sort_shuffle || use_external_sort_shuffle;
if (celeborn_client)
{
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
