[incubator-uniffle] branch master updated: [ISSUE-173][FOLLOWUP] The size of single buffer flush should reach rss.server.flush.cold.storage.threshold.size (#180)

[roryqi]

This is an automated email from the ASF dual-hosted git repository.

roryqi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-uniffle.git


The following commit(s) were added to refs/heads/master by this push:
     new e968c008 [ISSUE-173][FOLLOWUP] The size of single buffer flush should 
reach rss.server.flush.cold.storage.threshold.size (#180)
e968c008 is described below

commit e968c00800df0406e28b6e0ebc1d410ba3020f33
Author: Xianming Lei <31424839+le...@users.noreply.github.com>
AuthorDate: Tue Aug 23 11:10:01 2022 +0800

    [ISSUE-173][FOLLOWUP] The size of single buffer flush should reach 
rss.server.flush.cold.storage.threshold.size (#180)
    
    ### What changes were proposed in this pull request?
    follow https://github.com/apache/incubator-uniffle/issues/173, when single 
buffer flush is triggered, the buffer size should reach 
rss.server.flush.cold.storage.threshold.size, we keep the same logic as 
MultiStorageManager#selectStorageManager
    
    ### Why are the changes needed?
    Make sure cold storage is valid
    
    ### Does this PR introduce any user-facing change?
    No
    
    ### How was this patch tested?
    Already added
    
    Co-authored-by: leixianming <leixianm...@didiglobal.com>
---
 .../java/org/apache/uniffle/server/buffer/ShuffleBufferManager.java   | 4 +++-
 .../org/apache/uniffle/server/buffer/ShuffleBufferManagerTest.java    | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git 
a/server/src/main/java/org/apache/uniffle/server/buffer/ShuffleBufferManager.java
 
b/server/src/main/java/org/apache/uniffle/server/buffer/ShuffleBufferManager.java
index f934f0f6..660cd5c5 100644
--- 
a/server/src/main/java/org/apache/uniffle/server/buffer/ShuffleBufferManager.java
+++ 
b/server/src/main/java/org/apache/uniffle/server/buffer/ShuffleBufferManager.java
@@ -165,7 +165,9 @@ public class ShuffleBufferManager {
 
   void flushSingleBufferIfNecessary(ShuffleBuffer buffer, String appId,
       int shuffleId, int startPartition, int endPartition) {
-    if (this.bufferFlushEnabled && buffer.getSize() >= 
this.bufferFlushThreshold) {
+    // When we use multistorage and trigger single buffer flush, the buffer 
size should be bigger
+    // than rss.server.flush.cold.storage.threshold.size, otherwise cold 
storage will be useless.
+    if (this.bufferFlushEnabled && buffer.getSize() > 
this.bufferFlushThreshold) {
       flushBuffer(buffer, appId, shuffleId, startPartition, endPartition);
     }
   }
diff --git 
a/server/src/test/java/org/apache/uniffle/server/buffer/ShuffleBufferManagerTest.java
 
b/server/src/test/java/org/apache/uniffle/server/buffer/ShuffleBufferManagerTest.java
index 07be94f4..19a4ef3e 100644
--- 
a/server/src/test/java/org/apache/uniffle/server/buffer/ShuffleBufferManagerTest.java
+++ 
b/server/src/test/java/org/apache/uniffle/server/buffer/ShuffleBufferManagerTest.java
@@ -432,7 +432,7 @@ public class ShuffleBufferManagerTest extends 
BufferTestBase {
 
     shuffleBufferManager.cacheShuffleData(appId, shuffleId, false, 
createData(0, 32));
     assertEquals(64, shuffleBufferManager.getUsedMemory());
-    shuffleBufferManager.cacheShuffleData(appId, shuffleId, false, 
createData(1, 32));
+    shuffleBufferManager.cacheShuffleData(appId, shuffleId, false, 
createData(1, 48));
     waitForFlush(shuffleFlushManager, appId, shuffleId, 4);
     assertEquals(0, shuffleBufferManager.getUsedMemory());
     assertEquals(0, shuffleBufferManager.getInFlushSize());