This is an automated email from the ASF dual-hosted git repository.
zuston pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-uniffle.git
The following commit(s) were added to refs/heads/master by this push:
new 73c1ba2a [#889] improvement: Modify default value of single buffer
flush. (#1039)
73c1ba2a is described below
commit 73c1ba2acd5ae60ee0dd3c26004c0fc748ddd426
Author: Xianming Lei <[email protected]>
AuthorDate: Wed Jul 26 15:47:54 2023 +0800
[#889] improvement: Modify default value of single buffer flush. (#1039)
### What changes were proposed in this pull request?
Modify default value of single buffer flush.
### Why are the changes needed?
For release 0.8.0.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
exiting UTs.
---------
Co-authored-by: leixianming <[email protected]>
---
conf/server.conf | 2 ++
docs/server_guide.md | 4 ++--
.../src/main/java/org/apache/uniffle/server/ShuffleServerConf.java | 4 ++--
.../java/org/apache/uniffle/server/buffer/ShuffleBufferManager.java | 6 ++++--
.../org/apache/uniffle/server/buffer/ShuffleBufferManagerTest.java | 1 +
5 files changed, 11 insertions(+), 6 deletions(-)
diff --git a/conf/server.conf b/conf/server.conf
index 44f2ddef..c30cacc3 100644
--- a/conf/server.conf
+++ b/conf/server.conf
@@ -26,3 +26,5 @@ rss.server.flush.thread.alive 5
rss.server.flush.localfile.threadPool.size 10
rss.server.flush.hadoop.threadPool.size 10
rss.server.disk.capacity 1t
+rss.server.single.buffer.flush.enabled true
+rss.server.single.buffer.flush.threshold 128m
diff --git a/docs/server_guide.md b/docs/server_guide.md
index 5ab42b61..e796b3b2 100644
--- a/docs/server_guide.md
+++ b/docs/server_guide.md
@@ -83,8 +83,8 @@ This document will introduce how to deploy Uniffle shuffle
servers.
| rss.storage.type | - | Supports
MEMORY_LOCALFILE, MEMORY_HDFS, MEMORY_LOCALFILE_HDFS
|
| rss.server.flush.cold.storage.threshold.size | 64M | The
threshold of data size for LOACALFILE and HADOOP if MEMORY_LOCALFILE_HDFS is
used
|
| rss.server.tags | - | The
comma-separated list of tags to indicate the shuffle server's attributes. It
will be used as the assignment basis for the coordinator
|
-| rss.server.single.buffer.flush.enabled | false | Whether
single buffer flush when size exceeded rss.server.single.buffer.flush.threshold
|
-| rss.server.single.buffer.flush.threshold | 64M | The
threshold of single shuffle buffer flush
|
+| rss.server.single.buffer.flush.enabled | true | Whether
single buffer flush when size exceeded rss.server.single.buffer.flush.threshold
|
+| rss.server.single.buffer.flush.threshold | 128M | The
threshold of single shuffle buffer flush
|
| rss.server.disk.capacity | -1 | Disk
capacity that shuffle server can use. If negative, it will use disk whole space
* ratio
|
| rss.server.disk.capacity.ratio | 0.9 | When
`rss.server.disk.capacity` is negative, disk whole space * ratio is used
|
| rss.server.multistorage.fallback.strategy.class | - | The
fallback strategy for `MEMORY_LOCALFILE_HDFS`. Support
`org.apache.uniffle.server.storage.RotateStorageManagerFallbackStrategy`,`org.apache.uniffle.server.storage.LocalStorageManagerFallbackStrategy`
and `org.apache.uniffle.server.storage.HadoopStorageManagerFallbackStrategy`.
If not set,
`org.apache.uniffle.server.storage.HadoopStorageManagerFallbackStrategy` will
be used. |
diff --git
a/server/src/main/java/org/apache/uniffle/server/ShuffleServerConf.java
b/server/src/main/java/org/apache/uniffle/server/ShuffleServerConf.java
index 1db4b3c0..4e816183 100644
--- a/server/src/main/java/org/apache/uniffle/server/ShuffleServerConf.java
+++ b/server/src/main/java/org/apache/uniffle/server/ShuffleServerConf.java
@@ -339,14 +339,14 @@ public class ShuffleServerConf extends RssBaseConf {
public static final ConfigOption<Boolean> SINGLE_BUFFER_FLUSH_ENABLED =
ConfigOptions.key("rss.server.single.buffer.flush.enabled")
.booleanType()
- .defaultValue(false)
+ .defaultValue(true)
.withDescription(
"Whether single buffer flush when size exceeded
rss.server.single.buffer.flush.threshold");
public static final ConfigOption<Long> SINGLE_BUFFER_FLUSH_THRESHOLD =
ConfigOptions.key("rss.server.single.buffer.flush.threshold")
.longType()
- .defaultValue(64 * 1024 * 1024L)
+ .defaultValue(128 * 1024 * 1024L)
.withDescription("The threshold of single shuffle buffer flush");
public static final ConfigOption<Long> STORAGEMANAGER_CACHE_TIMEOUT =
diff --git
a/server/src/main/java/org/apache/uniffle/server/buffer/ShuffleBufferManager.java
b/server/src/main/java/org/apache/uniffle/server/buffer/ShuffleBufferManager.java
index db2ee531..2076f670 100644
---
a/server/src/main/java/org/apache/uniffle/server/buffer/ShuffleBufferManager.java
+++
b/server/src/main/java/org/apache/uniffle/server/buffer/ShuffleBufferManager.java
@@ -109,8 +109,10 @@ public class ShuffleBufferManager {
/ 100.0
*
conf.get(ShuffleServerConf.SERVER_MEMORY_SHUFFLE_LOWWATERMARK_PERCENTAGE));
this.bufferFlushEnabled =
conf.getBoolean(ShuffleServerConf.SINGLE_BUFFER_FLUSH_ENABLED);
- this.bufferFlushThreshold =
conf.getLong(ShuffleServerConf.SINGLE_BUFFER_FLUSH_THRESHOLD);
- this.shuffleFlushThreshold =
conf.getLong(ShuffleServerConf.SERVER_SHUFFLE_FLUSH_THRESHOLD);
+ this.bufferFlushThreshold =
+ conf.getSizeAsBytes(ShuffleServerConf.SINGLE_BUFFER_FLUSH_THRESHOLD);
+ this.shuffleFlushThreshold =
+ conf.getSizeAsBytes(ShuffleServerConf.SERVER_SHUFFLE_FLUSH_THRESHOLD);
this.hugePartitionSizeThreshold =
conf.getSizeAsBytes(ShuffleServerConf.HUGE_PARTITION_SIZE_THRESHOLD);
this.hugePartitionMemoryLimitSize =
diff --git
a/server/src/test/java/org/apache/uniffle/server/buffer/ShuffleBufferManagerTest.java
b/server/src/test/java/org/apache/uniffle/server/buffer/ShuffleBufferManagerTest.java
index ffdfb07c..6a466764 100644
---
a/server/src/test/java/org/apache/uniffle/server/buffer/ShuffleBufferManagerTest.java
+++
b/server/src/test/java/org/apache/uniffle/server/buffer/ShuffleBufferManagerTest.java
@@ -446,6 +446,7 @@ public class ShuffleBufferManagerTest extends
BufferTestBase {
shuffleConf.set(ShuffleServerConf.SERVER_BUFFER_CAPACITY, 200L);
shuffleConf.set(ShuffleServerConf.HUGE_PARTITION_MEMORY_USAGE_LIMITATION_RATIO,
0.1);
shuffleConf.set(ShuffleServerConf.HUGE_PARTITION_SIZE_THRESHOLD, 100L);
+ shuffleConf.set(ShuffleServerConf.SINGLE_BUFFER_FLUSH_ENABLED, false);
shuffleConf.setSizeAsBytes(ShuffleServerConf.SINGLE_BUFFER_FLUSH_THRESHOLD,
64L);
ShuffleServer mockShuffleServer = mock(ShuffleServer.class);
