(celeborn) branch main updated: [CELEBORN-2023] Spark4 Client incompatible with isLocalMaster method

[nicholasjiang]

This is an automated email from the ASF dual-hosted git repository.

nicholasjiang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/celeborn.git


The following commit(s) were added to refs/heads/main by this push:
     new 7eee202b5 [CELEBORN-2023] Spark4 Client incompatible with 
isLocalMaster method
7eee202b5 is described below

commit 7eee202b5572ff0d86a9fa0488bfaf41ceb9d34a
Author: sychen <syc...@ctrip.com>
AuthorDate: Tue Jun 3 20:30:50 2025 +0800

    [CELEBORN-2023] Spark4 Client incompatible with isLocalMaster method
    
    ### What changes were proposed in this pull request?
    
    ### Why are the changes needed?
    
    ```java
    Exception in thread "main" java.lang.NoSuchMethodError: 'boolean 
org.apache.spark.util.Utils.isLocalMaster(org.apache.spark.SparkConf)'
            at 
org.apache.spark.shuffle.celeborn.SparkShuffleManager.executorCores(SparkShuffleManager.java:464)
            at 
org.apache.spark.shuffle.celeborn.SparkShuffleManager.<init>(SparkShuffleManager.java:117)
            at 
java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native
 Method)
            at 
java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:75)
            at 
java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:53)
            at 
java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:502)
            at 
java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:486)
            at 
org.apache.spark.util.Utils$.instantiateSerializerOrShuffleManager(Utils.scala:2584)
            at 
org.apache.spark.shuffle.ShuffleManager$.create(ShuffleManager.scala:108)
            at 
org.apache.spark.SparkEnv.initializeShuffleManager(SparkEnv.scala:226)
            at org.apache.spark.SparkContext.<init>(SparkContext.scala:589)
            at 
org.apache.spark.SparkContext$.getOrCreate(SparkContext.scala:3055
    ```
    
    ### Does this PR introduce _any_ user-facing change?
    No
    
    ### How was this patch tested?
    local test
    
    Closes #3305 from cxzl25/CELEBORN-2023.
    
    Lead-authored-by: sychen <syc...@ctrip.com>
    Co-authored-by: cxzl25 <3898450+cxz...@users.noreply.github.com>
    Signed-off-by: 子懿 <ziyi....@antgroup.com>
---
 .../org/apache/spark/shuffle/celeborn/SparkShuffleManager.java     | 3 +--
 .../main/java/org/apache/spark/shuffle/celeborn/SparkUtils.java    | 7 +++++++
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git 
a/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/SparkShuffleManager.java
 
b/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/SparkShuffleManager.java
index 80ea5c256..151ce6e41 100644
--- 
a/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/SparkShuffleManager.java
+++ 
b/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/SparkShuffleManager.java
@@ -28,7 +28,6 @@ import org.apache.spark.rdd.DeterministicLevel;
 import org.apache.spark.shuffle.*;
 import org.apache.spark.shuffle.sort.SortShuffleManager;
 import org.apache.spark.sql.internal.SQLConf;
-import org.apache.spark.util.Utils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -461,7 +460,7 @@ public class SparkShuffleManager implements ShuffleManager {
   }
 
   private int executorCores(SparkConf conf) {
-    if (Utils.isLocalMaster(conf)) {
+    if (SparkUtils.isLocalMaster(conf)) {
       // SparkContext.numDriverCores is package private.
       return DynMethods.builder("numDriverCores")
           .impl("org.apache.spark.SparkContext$", String.class)
diff --git 
a/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/SparkUtils.java
 
b/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/SparkUtils.java
index fc5d605d8..d6f4ebb73 100644
--- 
a/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/SparkUtils.java
+++ 
b/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/SparkUtils.java
@@ -633,4 +633,11 @@ public class SparkUtils {
       SparkShuffleManager sparkShuffleManager, int celebornShuffleId) {
     
sparkShuffleManager.getFailedShuffleCleaner().removeCleanedShuffleId(celebornShuffleId);
   }
+
+  // Replica of Spark's `org.apache.spark.util.Utils.isLocalMaster`,
+  // due to SPARK-50515(4.0.0) changes the method signature.
+  public static boolean isLocalMaster(SparkConf conf) {
+    String master = conf.get("spark.master", "");
+    return master.equals("local") || master.startsWith("local[");
+  }
 }