[incubator-celeborn] branch celeborn-755 updated: [CELEBORN-755] Support to decide whether to compress shuffle data through configuration

[kerwinzhang]

This is an automated email from the ASF dual-hosted git repository.

kerwinzhang pushed a commit to branch celeborn-755
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git


The following commit(s) were added to refs/heads/celeborn-755 by this push:
     new 32ad992e3 [CELEBORN-755] Support to decide whether to compress shuffle 
data through configuration
32ad992e3 is described below

commit 32ad992e39f098690aba9ad897e248cf5bf39639
Author: xiyu.zk <xiyu...@alibaba-inc.com>
AuthorDate: Fri Jun 30 19:34:40 2023 +0800

    [CELEBORN-755] Support to decide whether to compress shuffle data through 
configuration
---
 .../org/apache/celeborn/client/ShuffleClientSuiteJ.java   | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git 
a/client/src/test/java/org/apache/celeborn/client/ShuffleClientSuiteJ.java 
b/client/src/test/java/org/apache/celeborn/client/ShuffleClientSuiteJ.java
index 9c5c070b2..311cde95e 100644
--- a/client/src/test/java/org/apache/celeborn/client/ShuffleClientSuiteJ.java
+++ b/client/src/test/java/org/apache/celeborn/client/ShuffleClientSuiteJ.java
@@ -133,9 +133,12 @@ public class ShuffleClientSuiteJ {
               1,
               1);
 
-      Compressor compressor = Compressor.getCompressor(conf);
-      compressor.compress(TEST_BUF1, 0, TEST_BUF1.length);
-      final int compressedTotalSize = compressor.getCompressedTotalSize();
+      int totalSize = TEST_BUF1.length;
+      if (!codec.equals(CompressionCodec.NONE)) {
+        Compressor compressor = Compressor.getCompressor(conf);
+        compressor.compress(TEST_BUF1, 0, TEST_BUF1.length);
+        totalSize = compressor.getCompressedTotalSize();
+      }
 
       shuffleClient.mergeData(
           TEST_SHUFFLE_ID,
@@ -148,7 +151,7 @@ public class ShuffleClientSuiteJ {
           1,
           1);
 
-      assert (mergeSize == compressedTotalSize + BATCH_HEADER_SIZE);
+      assert (mergeSize == totalSize + BATCH_HEADER_SIZE);
 
       byte[] buf1k = 
RandomStringUtils.random(4000).getBytes(StandardCharsets.UTF_8);
       int largeMergeSize =
@@ -163,9 +166,9 @@ public class ShuffleClientSuiteJ {
               1,
               1);
 
-      int totalSize = buf1k.length;
+      totalSize = buf1k.length;
       if (!codec.equals(CompressionCodec.NONE)) {
-        compressor = Compressor.getCompressor(conf);
+        Compressor compressor = Compressor.getCompressor(conf);
         compressor.compress(buf1k, 0, buf1k.length);
         totalSize = compressor.getCompressedTotalSize();
       }