(incubator-celeborn) branch main updated: [CELEBORN-1209][FOLLOW] Warning Shuffle Tracking

[chengpan]

This is an automated email from the ASF dual-hosted git repository.

chengpan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git


The following commit(s) were added to refs/heads/main by this push:
     new df9b6acb0 [CELEBORN-1209][FOLLOW] Warning Shuffle Tracking
df9b6acb0 is described below

commit df9b6acb069b38ae2eda21cef052a4f2246c41c7
Author: zwangsheng <binjiey...@apache.org>
AuthorDate: Tue Jan 9 19:51:39 2024 +0800

    [CELEBORN-1209][FOLLOW] Warning Shuffle Tracking
    
    ### What changes were proposed in this pull request?
    
    Follow up for https://github.com/apache/incubator-celeborn/pull/2204
    
    ### Why are the changes needed?
    
    [SPARK-39846](https://issues.apache.org/jira/browse/SPARK-39846) set 
shuffle tracking enabled default value to true, fix warning.
    
    ### Does this PR introduce _any_ user-facing change?
    Only log
    
    ### How was this patch tested?
    
    Pass CI and local build
    
    Closes #2216 from zwangsheng/CELEBORN-1209-2.
    
    Authored-by: zwangsheng <binjiey...@apache.org>
    Signed-off-by: Cheng Pan <cheng...@apache.org>
---
 .../apache/spark/shuffle/celeborn/SparkShuffleManager.java   | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git 
a/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/SparkShuffleManager.java
 
b/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/SparkShuffleManager.java
index 264bf7766..8064a8bab 100644
--- 
a/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/SparkShuffleManager.java
+++ 
b/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/SparkShuffleManager.java
@@ -21,6 +21,7 @@ import java.io.IOException;
 import java.util.concurrent.ConcurrentHashMap;
 
 import org.apache.spark.*;
+import org.apache.spark.internal.config.package$;
 import org.apache.spark.launcher.SparkLauncher;
 import org.apache.spark.rdd.DeterministicLevel;
 import org.apache.spark.shuffle.*;
@@ -94,11 +95,16 @@ public class SparkShuffleManager implements ShuffleManager {
               + "use Celeborn as Remote Shuffle Service to avoid performance 
degradation.",
           SQLConf.LOCAL_SHUFFLE_READER_ENABLED().key());
     }
-    if (conf.getBoolean("spark.dynamicAllocation.shuffleTracking.enabled", 
false)) {
+    if ((Boolean) 
conf.get(package$.MODULE$.DYN_ALLOCATION_SHUFFLE_TRACKING_ENABLED())) {
+      String key = 
package$.MODULE$.DYN_ALLOCATION_SHUFFLE_TRACKING_ENABLED().key();
+      Boolean defaultValue =
+          (Boolean) 
package$.MODULE$.DYN_ALLOCATION_SHUFFLE_TRACKING_ENABLED().defaultValue().get();
       logger.warn(
-          "Detected spark.dynamicAllocation.shuffleTracking.enabled (default 
is false) is enabled, "
+          "Detected {} (default is {}) is enabled, "
               + "it's highly recommended to disable it when use Celeborn as 
Remote Shuffle Service "
-              + "to avoid performance degradation.");
+              + "to avoid performance degradation.",
+          key,
+          defaultValue);
     }
     this.conf = conf;
     this.isDriver = isDriver;