This is an automated email from the ASF dual-hosted git repository.

zhouky pushed a commit to branch testmain
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git

commit 94e3337f5d34041604c236924f2f029187e26d44
Author: zky.zhoukeyong <[email protected]>
AuthorDate: Fri Jun 16 18:40:39 2023 +0800

    batch revive
---
 .../flink/readclient/FlinkShuffleClientImpl.java   |   8 +-
 .../org/apache/celeborn/client/ReviveManager.java  | 125 ++++++++
 .../apache/celeborn/client/ShuffleClientImpl.java  | 352 ++++++++++++++-------
 .../celeborn/client/ChangePartitionManager.scala   |  13 +-
 .../apache/celeborn/client/LifecycleManager.scala  | 108 ++++---
 .../client/RequestLocationCallContext.scala        |  44 ++-
 .../celeborn/client/ShuffleClientHelper.scala      |   7 +-
 .../celeborn/common/protocol/ReviveRequest.java    |  46 +++
 .../common/protocol/message/StatusCode.java        |   3 +-
 common/src/main/proto/TransportMessages.proto      |  25 +-
 .../common/protocol/message/ControlMessages.scala  |  56 +++-
 .../celeborn/tests/spark/RetryReviveTest.scala     |   3 +-
 12 files changed, 592 insertions(+), 198 deletions(-)

diff --git 
a/client-flink/common/src/main/java/org/apache/celeborn/plugin/flink/readclient/FlinkShuffleClientImpl.java
 
b/client-flink/common/src/main/java/org/apache/celeborn/plugin/flink/readclient/FlinkShuffleClientImpl.java
index 5d2d1a081..7d883b40f 100644
--- 
a/client-flink/common/src/main/java/org/apache/celeborn/plugin/flink/readclient/FlinkShuffleClientImpl.java
+++ 
b/client-flink/common/src/main/java/org/apache/celeborn/plugin/flink/readclient/FlinkShuffleClientImpl.java
@@ -49,6 +49,7 @@ import 
org.apache.celeborn.common.network.protocol.RegionFinish;
 import org.apache.celeborn.common.network.protocol.RegionStart;
 import org.apache.celeborn.common.network.util.TransportConf;
 import org.apache.celeborn.common.protocol.PartitionLocation;
+import org.apache.celeborn.common.protocol.PbChangeLocationPartitionInfo;
 import org.apache.celeborn.common.protocol.PbChangeLocationResponse;
 import org.apache.celeborn.common.protocol.TransportModuleConstants;
 import org.apache.celeborn.common.protocol.message.ControlMessages;
@@ -388,7 +389,6 @@ public class FlinkShuffleClientImpl extends 
ShuffleClientImpl {
                         applicationId,
                         shuffleId,
                         mapId,
-                        attemptId,
                         location.getId(),
                         location.getEpoch(),
                         location,
@@ -396,9 +396,11 @@ public class FlinkShuffleClientImpl extends 
ShuffleClientImpl {
                     conf.clientRpcRequestPartitionLocationRpcAskTimeout(),
                     ClassTag$.MODULE$.apply(PbChangeLocationResponse.class));
             // per partitionKey only serve single PartitionLocation in Client 
Cache.
-            StatusCode respStatus = Utils.toStatusCode(response.getStatus());
+            PbChangeLocationPartitionInfo partitionInfo = 
response.getPartitionInfo(0);
+            StatusCode respStatus = 
Utils.toStatusCode(partitionInfo.getStatus());
             if (StatusCode.SUCCESS.equals(respStatus)) {
-              return 
Optional.of(PbSerDeUtils.fromPbPartitionLocation(response.getLocation()));
+              return Optional.of(
+                  
PbSerDeUtils.fromPbPartitionLocation(partitionInfo.getPartition()));
             } else {
               // throw exception
               logger.error(
diff --git a/client/src/main/java/org/apache/celeborn/client/ReviveManager.java 
b/client/src/main/java/org/apache/celeborn/client/ReviveManager.java
new file mode 100644
index 000000000..8e889639d
--- /dev/null
+++ b/client/src/main/java/org/apache/celeborn/client/ReviveManager.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.celeborn.client;
+
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.celeborn.common.protocol.PartitionLocation;
+import org.apache.celeborn.common.protocol.ReviveRequest;
+import org.apache.celeborn.common.protocol.message.StatusCode;
+import org.apache.celeborn.common.util.ThreadUtils;
+
+class ReviveManager {
+  // appid -> shuffleId -> requests
+  HashMap<String, HashMap<Integer, Set<ReviveRequest>>> pendingRequests = new 
HashMap<>();
+  HashMap<String, HashMap<Integer, Set<ReviveRequest>>> inProcessRequests = 
null;
+  ShuffleClientImpl shuffleClient;
+  private ScheduledExecutorService batchReviveRequestScheduler =
+      
ThreadUtils.newDaemonSingleThreadScheduledExecutor("worker-forward-message-scheduler");
+
+  public ReviveManager(ShuffleClientImpl shuffleClient) {
+    this.shuffleClient = shuffleClient;
+
+    batchReviveRequestScheduler.scheduleAtFixedRate(
+        new Runnable() {
+          @Override
+          public void run() {
+            // We must ensure the swap is atomic
+            synchronized (this) {
+              inProcessRequests = pendingRequests;
+              pendingRequests = new HashMap<>();
+            }
+            if (!inProcessRequests.isEmpty()) {
+              for (Map.Entry<String, HashMap<Integer, Set<ReviveRequest>>> 
entry :
+                  inProcessRequests.entrySet()) {
+                String appId = entry.getKey();
+                HashMap<Integer, Set<ReviveRequest>> shuffleMap = 
entry.getValue();
+                for (Map.Entry<Integer, Set<ReviveRequest>> shuffleEntry : 
shuffleMap.entrySet()) {
+                  // Call reviveBatch for requests in the same (appId, 
shuffleId)
+                  int shuffleId = shuffleEntry.getKey();
+                  Set<ReviveRequest> requests = shuffleEntry.getValue();
+                  Set<Integer> mapIds = new HashSet<>();
+                  ArrayList<ReviveRequest> filteredRequests = new 
ArrayList<>();
+                  Map<Integer, ReviveRequest> requestsToSend = new HashMap<>();
+
+                  // Insert request that is not MapperEnded and with the max 
epoch
+                  // into requestsToSend
+                  Iterator<ReviveRequest> iter = requests.iterator();
+                  while (iter.hasNext()) {
+                    ReviveRequest req = iter.next();
+                    if (shuffleClient.checkRevivedLocation(
+                        shuffleId, req.mapId, req.partitionId, 
req.loc.getEpoch(), false)) {
+                      req.reviveStatus = StatusCode.SUCCESS;
+                    } else {
+                      filteredRequests.add(req);
+                      mapIds.add(req.mapId);
+                      PartitionLocation loc = req.loc;
+                      if (!requestsToSend.containsKey(loc.getId())
+                          || requestsToSend.get(loc.getId()).epoch < 
loc.getEpoch()) {
+                        requestsToSend.put(loc.getId(), req);
+                      }
+                    }
+                  }
+
+                  if (!requestsToSend.isEmpty()) {
+                    // Call reviveBatch. Return null means Exception caught or
+                    // SHUFFLE_NOT_REGISTERED
+                    Map<Integer, StatusCode> results =
+                        shuffleClient.reviveBatch(
+                            appId, shuffleId, mapIds, requestsToSend.values());
+                    if (results == null) {
+                      for (ReviveRequest req : filteredRequests) {
+                        req.reviveStatus = StatusCode.REVIVE_FAILED;
+                      }
+                    } else {
+                      for (ReviveRequest req : filteredRequests) {
+                        if (shuffleClient.mapperEnded(shuffleId, req.mapId)) {
+                          req.reviveStatus = StatusCode.MAP_ENDED;
+                        } else {
+                          req.reviveStatus = results.get(req.partitionId);
+                        }
+                      }
+                    }
+                  }
+                } // End foreach shuffleMap.entrySet
+              } // End foreach inProcessRequests.entrySet
+            }
+
+            inProcessRequests.clear();
+          }
+        },
+        100,
+        100,
+        TimeUnit.MILLISECONDS);
+  }
+
+  public void addRequest(String appId, int shuffleId, ReviveRequest request) {
+    shuffleClient.excludeWorkerByCause(request.cause, request.loc);
+    // This sync is necessary to ensure the add action is atomic
+    synchronized (this) {
+      HashMap<Integer, Set<ReviveRequest>> shuffleMap =
+          pendingRequests.computeIfAbsent(appId, (id) -> new HashMap());
+      Set<ReviveRequest> requests =
+          shuffleMap.computeIfAbsent(shuffleId, (id) -> 
ConcurrentHashMap.newKeySet());
+      requests.add(request);
+    }
+  }
+}
diff --git 
a/client/src/main/java/org/apache/celeborn/client/ShuffleClientImpl.java 
b/client/src/main/java/org/apache/celeborn/client/ShuffleClientImpl.java
index 799ba6a75..160fd807d 100644
--- a/client/src/main/java/org/apache/celeborn/client/ShuffleClientImpl.java
+++ b/client/src/main/java/org/apache/celeborn/client/ShuffleClientImpl.java
@@ -20,10 +20,7 @@ package org.apache.celeborn.client;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.*;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.*;
 
 import scala.reflect.ClassTag$;
 
@@ -109,7 +106,7 @@ public class ShuffleClientImpl extends ShuffleClient {
   private final ExecutorService partitionSplitPool;
   private final Map<Integer, Set<Integer>> splitting = 
JavaUtils.newConcurrentHashMap();
 
-  ThreadLocal<Compressor> compressorThreadLocal =
+  private ThreadLocal<Compressor> compressorThreadLocal =
       new ThreadLocal<Compressor>() {
         @Override
         protected Compressor initialValue() {
@@ -117,6 +114,8 @@ public class ShuffleClientImpl extends ShuffleClient {
         }
       };
 
+  private ReviveManager reviveManager = new ReviveManager(this);
+
   protected static class ReduceFileGroups {
     public Map<Integer, Set<PartitionLocation>> partitionGroups;
     public int[] mapAttempts;
@@ -202,21 +201,34 @@ public class ShuffleClientImpl extends ShuffleClient {
   private void submitRetryPushData(
       String applicationId,
       int shuffleId,
-      int mapId,
-      int attemptId,
       byte[] body,
       int batchId,
-      PartitionLocation loc,
       RpcResponseCallback wrappedCallback,
       PushState pushState,
-      StatusCode cause,
+      ReviveRequest request,
       int remainReviveTimes) {
+    int mapId = request.mapId;
+    int attemptId = request.attemptId;
+    PartitionLocation loc = request.loc;
+    StatusCode cause = request.cause;
     int partitionId = loc.getId();
-    if (!revive(
-        applicationId, shuffleId, mapId, attemptId, partitionId, 
loc.getEpoch(), loc, cause)) {
-      wrappedCallback.onFailure(
-          new CelebornIOException(cause + " then revive but " + 
StatusCode.REVIVE_FAILED));
-    } else if (mapperEnded(shuffleId, mapId)) {
+    long reviveWaitTime =
+        
conf.clientRpcRequestPartitionLocationRpcAskTimeout().duration().toMillis() + 
1000;
+    final long delta = 50;
+    long accumulatedTime = 0;
+    while (request.reviveStatus == StatusCode.REVIVE_INITIALIZED
+        && accumulatedTime <= reviveWaitTime) {
+      try {
+        Thread.sleep(50);
+        accumulatedTime += delta;
+      } catch (InterruptedException e) {
+        logger.error("Interrupted while waiting for Revive result!");
+        wrappedCallback.onFailure(
+            new CelebornIOException(cause + " then revive but " + 
StatusCode.REVIVE_FAILED));
+        return;
+      }
+    }
+    if (mapperEnded(shuffleId, mapId)) {
       logger.debug(
           "Revive for push data success, but the mapper already ended for 
shuffle {} map {} attempt {} partition {} batch {} location {}.",
           shuffleId,
@@ -226,6 +238,14 @@ public class ShuffleClientImpl extends ShuffleClient {
           batchId,
           loc);
       pushState.removeBatch(batchId, loc.hostAndPushPort());
+    } else if (request.reviveStatus != StatusCode.SUCCESS) {
+      wrappedCallback.onFailure(
+          new CelebornIOException(
+              cause
+                  + " then revive but "
+                  + StatusCode.REVIVE_FAILED
+                  + ", revive status "
+                  + request.reviveStatus));
     } else {
       PartitionLocation newLoc = 
reducePartitionMap.get(shuffleId).get(partitionId);
       logger.info(
@@ -270,6 +290,25 @@ public class ShuffleClientImpl extends ShuffleClient {
     }
   }
 
+  public ReviveRequest[] addAndGetReviveRequests(
+      String appId,
+      int shuffleId,
+      int mapId,
+      int attemptId,
+      ArrayList<DataBatches.DataBatch> batches,
+      StatusCode cause) {
+    ReviveRequest[] reviveRequests = new ReviveRequest[batches.size()];
+    for (int i = 0; i < batches.size(); i++) {
+      DataBatches.DataBatch batch = batches.get(i);
+      PartitionLocation loc = batch.loc;
+      ReviveRequest reviveRequest =
+          new ReviveRequest(mapId, attemptId, loc.getId(), loc.getEpoch(), 
loc, cause);
+      reviveManager.addRequest(appId, shuffleId, reviveRequest);
+      reviveRequests[i] = reviveRequest;
+    }
+    return reviveRequests;
+  }
+
   private void submitRetryPushMergedData(
       PushState pushState,
       String applicationId,
@@ -279,56 +318,77 @@ public class ShuffleClientImpl extends ShuffleClient {
       ArrayList<DataBatches.DataBatch> batches,
       StatusCode cause,
       Integer oldGroupedBatchId,
+      ReviveRequest[] reviveRequests,
       int remainReviveTimes) {
     HashMap<String, DataBatches> newDataBatchesMap = new HashMap<>();
     ArrayList<DataBatches.DataBatch> reviveFailedBatchesMap = new 
ArrayList<>();
-    for (DataBatches.DataBatch batch : batches) {
-      int partitionId = batch.loc.getId();
-      if (!revive(
-          applicationId,
-          shuffleId,
-          mapId,
-          attemptId,
-          partitionId,
-          batch.loc.getEpoch(),
-          batch.loc,
-          cause)) {
 
-        if (remainReviveTimes > 0) {
-          reviveFailedBatchesMap.add(batch);
+    long reviveWaitTime =
+        
conf.clientRpcRequestPartitionLocationRpcAskTimeout().duration().toMillis() + 
1000;
+    final long delta = 50;
+    long accumulatedTime = 0;
+    int index = 0;
+    while (index < reviveRequests.length && accumulatedTime <= reviveWaitTime) 
{
+      ReviveRequest request = reviveRequests[index];
+      DataBatches.DataBatch batch = batches.get(index);
+      if (request.reviveStatus != StatusCode.REVIVE_INITIALIZED) {
+        if (request.reviveStatus == StatusCode.SUCCESS) {
+          PartitionLocation newLoc = 
reducePartitionMap.get(shuffleId).get(request.partitionId);
+          DataBatches newDataBatches =
+              newDataBatchesMap.computeIfAbsent(genAddressPair(newLoc), (s) -> 
new DataBatches());
+          newDataBatches.addDataBatch(newLoc, batch.batchId, batch.body);
+        } else if (request.reviveStatus == StatusCode.MAP_ENDED) {
+          logger.debug(
+              "Revive for push merged data success, but the mapper already 
ended for shuffle {} map {} attempt {} partition {} batch {}.",
+              shuffleId,
+              mapId,
+              attemptId,
+              request.partitionId,
+              oldGroupedBatchId);
         } else {
-          String errorMsg =
-              String.format(
-                  "Revive failed while pushing merged for shuffle %d map %d 
attempt %d partition %d batch %d location %s.",
-                  shuffleId, mapId, attemptId, partitionId, oldGroupedBatchId, 
batch.loc);
-          pushState.exception.compareAndSet(
-              null,
-              new CelebornIOException(
-                  errorMsg,
-                  new CelebornIOException(cause + " then revive but " + 
StatusCode.REVIVE_FAILED)));
-          return;
+          if (remainReviveTimes > 0) {
+            reviveFailedBatchesMap.add(batch);
+          } else {
+            String errorMsg =
+                String.format(
+                    "Revive failed while pushing merged for shuffle %d map %d 
attempt %d partition %d batch %d location %s.",
+                    shuffleId, mapId, attemptId, request.partitionId, 
oldGroupedBatchId, batch.loc);
+            pushState.exception.compareAndSet(
+                null,
+                new CelebornIOException(
+                    errorMsg,
+                    new CelebornIOException(
+                        cause + " then revive but " + 
StatusCode.REVIVE_FAILED)));
+            return;
+          }
         }
-      } else if (mapperEnded(shuffleId, mapId)) {
-        logger.debug(
-            "Revive for push merged data success, but the mapper already ended 
for shuffle {} map {} attempt {} partition {} batch {}.",
-            shuffleId,
-            mapId,
-            attemptId,
-            partitionId,
-            oldGroupedBatchId);
+        index++;
       } else {
-        PartitionLocation newLoc = 
reducePartitionMap.get(shuffleId).get(partitionId);
-        logger.info(
-            "Revive for push merged data success, new location for shuffle {} 
map {} attempt {} partition {} batch {} is location {}.",
-            shuffleId,
-            mapId,
-            attemptId,
-            partitionId,
-            oldGroupedBatchId,
-            newLoc);
-        DataBatches newDataBatches =
-            newDataBatchesMap.computeIfAbsent(genAddressPair(newLoc), (s) -> 
new DataBatches());
-        newDataBatches.addDataBatch(newLoc, batch.batchId, batch.body);
+        try {
+          Thread.sleep(delta);
+        } catch (InterruptedException e) {
+          logger.warn("Interrupted while waiting for Revive result!");
+        }
+        accumulatedTime += delta;
+      }
+    }
+
+    for (int i = index; i < reviveRequests.length; i++) {
+      ReviveRequest request = reviveRequests[index];
+      DataBatches.DataBatch batch = batches.get(i);
+      if (remainReviveTimes > 0) {
+        reviveFailedBatchesMap.add(batch);
+      } else {
+        String errorMsg =
+            String.format(
+                "Revive failed while pushing merged for shuffle %d map %d 
attempt %d partition %d batch %d location %s.",
+                shuffleId, mapId, attemptId, request.partitionId, 
oldGroupedBatchId, batch.loc);
+        pushState.exception.compareAndSet(
+            null,
+            new CelebornIOException(
+                errorMsg,
+                new CelebornIOException(cause + " then revive but " + 
StatusCode.REVIVE_FAILED)));
+        return;
       }
     }
 
@@ -348,6 +408,9 @@ public class ShuffleClientImpl extends ShuffleClient {
     if (reviveFailedBatchesMap.isEmpty()) {
       pushState.removeBatch(oldGroupedBatchId, 
batches.get(0).loc.hostAndPushPort());
     } else {
+      ReviveRequest[] requests =
+          addAndGetReviveRequests(
+              applicationId, shuffleId, mapId, attemptId, 
reviveFailedBatchesMap, cause);
       pushDataRetryPool.submit(
           () ->
               submitRetryPushMergedData(
@@ -359,6 +422,7 @@ public class ShuffleClientImpl extends ShuffleClient {
                   reviveFailedBatchesMap,
                   cause,
                   oldGroupedBatchId,
+                  requests,
                   remainReviveTimes - 1));
     }
   }
@@ -503,36 +567,31 @@ public class ShuffleClientImpl extends ShuffleClient {
     }
   }
 
-  private boolean waitRevivedLocation(
-      ConcurrentHashMap<Integer, PartitionLocation> map, int partitionId, int 
epoch) {
+  boolean checkRevivedLocation(int shuffleId, int mapId, int partitionId, int 
epoch, boolean wait) {
+    ConcurrentHashMap<Integer, PartitionLocation> map = 
reducePartitionMap.get(shuffleId);
     PartitionLocation currentLocation = map.get(partitionId);
-    if (currentLocation != null && currentLocation.getEpoch() > epoch) {
+    if (mapperEnded(shuffleId, mapId)
+        || (currentLocation != null && currentLocation.getEpoch() > epoch)) {
       return true;
-    }
-
-    long sleepTimeMs = RND.nextInt(50);
-    if (sleepTimeMs > 30) {
-      try {
-        TimeUnit.MILLISECONDS.sleep(sleepTimeMs);
-      } catch (InterruptedException e) {
-        logger.error("Waiting revived location was interrupted.", e);
-        Thread.currentThread().interrupt();
+    } else if (wait) {
+      long sleepTimeMs = RND.nextInt(50);
+      if (sleepTimeMs > 30) {
+        try {
+          TimeUnit.MILLISECONDS.sleep(sleepTimeMs);
+        } catch (InterruptedException e) {
+          logger.error("Waiting revived location was interrupted.", e);
+          Thread.currentThread().interrupt();
+        }
       }
-    }
 
-    currentLocation = map.get(partitionId);
-    return currentLocation != null && currentLocation.getEpoch() > epoch;
+      currentLocation = map.get(partitionId);
+      return currentLocation != null && currentLocation.getEpoch() > epoch;
+    } else {
+      return false;
+    }
   }
 
-  private boolean revive(
-      String applicationId,
-      int shuffleId,
-      int mapId,
-      int attemptId,
-      int partitionId,
-      int epoch,
-      PartitionLocation oldLocation,
-      StatusCode cause) {
+  void excludeWorkerByCause(StatusCode cause, PartitionLocation oldLocation) {
     // Add ShuffleClient side blacklist
     if (shuffleClientPushBlacklistEnabled && oldLocation != null) {
       if (cause == StatusCode.PUSH_DATA_CREATE_CONNECTION_FAIL_MASTER) {
@@ -549,9 +608,22 @@ public class ShuffleClientImpl extends ShuffleClient {
         blacklist.add(oldLocation.getPeer().hostAndPushPort());
       }
     }
+  }
+
+  private boolean revive(
+      String applicationId,
+      int shuffleId,
+      int mapId,
+      int attemptId,
+      int partitionId,
+      int epoch,
+      PartitionLocation oldLocation,
+      StatusCode cause) {
+    excludeWorkerByCause(cause, oldLocation);
 
     ConcurrentHashMap<Integer, PartitionLocation> map = 
reducePartitionMap.get(shuffleId);
-    if (waitRevivedLocation(map, partitionId, epoch)) {
+
+    if (checkRevivedLocation(shuffleId, mapId, partitionId, epoch, true)) {
       logger.debug(
           "Revive already success for shuffle {} map {} attempt {} partition 
{} epoch {}, just return true(Assume revive successfully).",
           shuffleId,
@@ -561,34 +633,19 @@ public class ShuffleClientImpl extends ShuffleClient {
           epoch);
       return true;
     }
-    if (mapperEnded(shuffleId, mapId)) {
-      logger.debug(
-          "Revive success, but the mapper ended for shuffle {} map {} attempt 
{} partition {}, just return true(Assume revive successfully).",
-          shuffleId,
-          mapId,
-          attemptId,
-          partitionId);
-      return true;
-    }
 
     try {
       PbChangeLocationResponse response =
           driverRssMetaService.askSync(
               Revive$.MODULE$.apply(
-                  applicationId,
-                  shuffleId,
-                  mapId,
-                  attemptId,
-                  partitionId,
-                  epoch,
-                  oldLocation,
-                  cause),
+                  applicationId, shuffleId, mapId, partitionId, epoch, 
oldLocation, cause),
               conf.clientRpcRequestPartitionLocationRpcAskTimeout(),
               ClassTag$.MODULE$.apply(PbChangeLocationResponse.class));
       // per partitionKey only serve single PartitionLocation in Client Cache.
-      StatusCode respStatus = Utils.toStatusCode(response.getStatus());
+      PbChangeLocationPartitionInfo partitionInfo = 
response.getPartitionInfo(0);
+      StatusCode respStatus = Utils.toStatusCode(partitionInfo.getStatus());
       if (StatusCode.SUCCESS.equals(respStatus)) {
-        map.put(partitionId, 
PbSerDeUtils.fromPbPartitionLocation(response.getLocation()));
+        map.put(partitionId, 
PbSerDeUtils.fromPbPartitionLocation(partitionInfo.getPartition()));
         return true;
       } else if (StatusCode.MAP_ENDED.equals(respStatus)) {
         logger.debug(
@@ -615,6 +672,65 @@ public class ShuffleClientImpl extends ShuffleClient {
     }
   }
 
+  /** @return partitionId -> StatusCode */
+  Map<Integer, StatusCode> reviveBatch(
+      String applicationId,
+      int shuffleId,
+      Set<Integer> mapIds,
+      Collection<ReviveRequest> requests) {
+    Map<Integer, StatusCode> results = new HashMap<>();
+
+    ConcurrentHashMap<Integer, PartitionLocation> map = 
reducePartitionMap.get(shuffleId);
+
+    try {
+      PbChangeLocationResponse response =
+          driverRssMetaService.askSync(
+              Revive$.MODULE$.apply(applicationId, shuffleId, mapIds, 
requests),
+              conf.clientRpcRequestPartitionLocationRpcAskTimeout(),
+              ClassTag$.MODULE$.apply(PbChangeLocationResponse.class));
+
+      for (int i = 0; i < response.getEndedMapIdCount(); i++) {
+        int mapId = response.getEndedMapId(i);
+        mapperEndMap.computeIfAbsent(shuffleId, (id) -> 
ConcurrentHashMap.newKeySet()).add(mapId);
+      }
+
+      for (int i = 0; i < response.getPartitionInfoCount(); i++) {
+        PbChangeLocationPartitionInfo partitionInfo = 
response.getPartitionInfo(i);
+        int partitionId = partitionInfo.getPartitionId();
+        StatusCode statusCode = Utils.toStatusCode(partitionInfo.getStatus());
+
+        if (StatusCode.SUCCESS.equals(statusCode)) {
+          PartitionLocation loc =
+              
PbSerDeUtils.fromPbPartitionLocation(partitionInfo.getPartition());
+          map.put(partitionId, loc);
+        } else if (StatusCode.STAGE_ENDED.equals(statusCode)) {
+          stageEnded(shuffleId);
+          return results;
+        } else if (StatusCode.SHUFFLE_NOT_REGISTERED.equals(statusCode)) {
+          return null;
+        }
+        results.put(partitionId, statusCode);
+      }
+
+      return results;
+    } catch (Exception e) {
+      StringBuilder partitionIds = new StringBuilder();
+      StringBuilder epochs = new StringBuilder();
+      requests.forEach(
+          (req) -> {
+            partitionIds.append(req.partitionId).append(",");
+            epochs.append(req.epoch).append(",");
+          });
+      logger.error(
+          "Exception raised while reviving for shuffle {} partitionIds {} 
epochs {}.",
+          shuffleId,
+          partitionIds,
+          epochs,
+          e);
+      return null;
+    }
+  }
+
   public int pushOrMergeData(
       String applicationId,
       int shuffleId,
@@ -782,19 +898,25 @@ public class ShuffleClientImpl extends ShuffleClient {
                       attemptId,
                       partitionId,
                       nextBatchId);
+                  ReviveRequest reviveRequest =
+                      new ReviveRequest(
+                          mapId,
+                          attemptId,
+                          partitionId,
+                          loc.getEpoch(),
+                          loc,
+                          StatusCode.HARD_SPLIT);
+                  reviveManager.addRequest(applicationId, shuffleId, 
reviveRequest);
                   pushDataRetryPool.submit(
                       () ->
                           submitRetryPushData(
                               applicationId,
                               shuffleId,
-                              mapId,
-                              attemptId,
                               body,
                               nextBatchId,
-                              loc,
                               this,
                               pushState,
-                              StatusCode.HARD_SPLIT,
+                              reviveRequest,
                               remainReviveTimes));
                 } else if (reason == 
StatusCode.PUSH_DATA_SUCCESS_MASTER_CONGESTED.getValue()) {
                   logger.debug(
@@ -863,19 +985,19 @@ public class ShuffleClientImpl extends ShuffleClient {
                 if (!pushStatusIsBlacklisted(cause)) {
                   remainReviveTimes = remainReviveTimes - 1;
                 }
+                ReviveRequest reviveRequest =
+                    new ReviveRequest(mapId, attemptId, partitionId, 
loc.getEpoch(), loc, cause);
+                reviveManager.addRequest(applicationId, shuffleId, 
reviveRequest);
                 pushDataRetryPool.submit(
                     () ->
                         submitRetryPushData(
                             applicationId,
                             shuffleId,
-                            mapId,
-                            attemptId,
                             body,
                             nextBatchId,
-                            loc,
                             this,
                             pushState,
-                            cause,
+                            reviveRequest,
                             remainReviveTimes));
               } else {
                 pushState.removeBatch(nextBatchId, loc.hostAndPushPort());
@@ -1167,6 +1289,10 @@ public class ShuffleClientImpl extends ShuffleClient {
                     Arrays.toString(partitionIds),
                     groupedBatchId,
                     Arrays.toString(batchIds));
+
+                ReviveRequest[] requests =
+                    addAndGetReviveRequests(
+                        applicationId, shuffleId, mapId, attemptId, batches, 
StatusCode.HARD_SPLIT);
                 pushDataRetryPool.submit(
                     () ->
                         submitRetryPushMergedData(
@@ -1178,6 +1304,7 @@ public class ShuffleClientImpl extends ShuffleClient {
                             batches,
                             StatusCode.HARD_SPLIT,
                             groupedBatchId,
+                            requests,
                             remainReviveTimes));
               } else if (reason == 
StatusCode.PUSH_DATA_SUCCESS_MASTER_CONGESTED.getValue()) {
                 logger.debug(
@@ -1248,6 +1375,10 @@ public class ShuffleClientImpl extends ShuffleClient {
                 tmpRemainReviveTimes = tmpRemainReviveTimes - 1;
               }
               int finalRemainReviveTimes = tmpRemainReviveTimes;
+
+              ReviveRequest[] requests =
+                  addAndGetReviveRequests(
+                      applicationId, shuffleId, mapId, attemptId, batches, 
cause);
               pushDataRetryPool.submit(
                   () ->
                       submitRetryPushMergedData(
@@ -1259,6 +1390,7 @@ public class ShuffleClientImpl extends ShuffleClient {
                           batches,
                           cause,
                           groupedBatchId,
+                          requests,
                           finalRemainReviveTimes));
             } else {
               pushState.removeBatch(groupedBatchId, hostPort);
@@ -1525,7 +1657,7 @@ public class ShuffleClientImpl extends ShuffleClient {
     driverRssMetaService = endpointRef;
   }
 
-  protected boolean mapperEnded(int shuffleId, int mapId) {
+  boolean mapperEnded(int shuffleId, int mapId) {
     return (mapperEndMap.containsKey(shuffleId) && 
mapperEndMap.get(shuffleId).contains(mapId))
         || stageEnded(shuffleId);
   }
diff --git 
a/client/src/main/scala/org/apache/celeborn/client/ChangePartitionManager.scala 
b/client/src/main/scala/org/apache/celeborn/client/ChangePartitionManager.scala
index fd42f51d6..0a1838711 100644
--- 
a/client/src/main/scala/org/apache/celeborn/client/ChangePartitionManager.scala
+++ 
b/client/src/main/scala/org/apache/celeborn/client/ChangePartitionManager.scala
@@ -164,7 +164,7 @@ class ChangePartitionManager(
         // If new slot for the partition has been allocated, reply and return.
         // Else register and allocate for it.
         getLatestPartition(shuffleId, partitionId, oldEpoch).foreach { 
latestLoc =>
-          context.reply(StatusCode.SUCCESS, Some(latestLoc))
+          context.reply(partitionId, StatusCode.SUCCESS, Some(latestLoc))
           logDebug(s"New partition found, old partition $partitionId-$oldEpoch 
return it." +
             s" shuffleId: $shuffleId $latestLoc")
           return
@@ -226,9 +226,11 @@ class ChangePartitionManager(
           location -> Option(requestsMap.remove(location.getId))
         }
       }.foreach { case (newLocation, requests) =>
-        requests.map(_.asScala.toList.foreach(_.context.reply(
-          StatusCode.SUCCESS,
-          Option(newLocation))))
+        requests.map(_.asScala.toList.foreach(req =>
+          req.context.reply(
+            newLocation.getId,
+            StatusCode.SUCCESS,
+            Option(newLocation))))
       }
     }
 
@@ -242,7 +244,8 @@ class ChangePartitionManager(
           Option(requestsMap.remove(changePartition.partitionId))
         }
       }.foreach { requests =>
-        requests.map(_.asScala.toList.foreach(_.context.reply(status, None)))
+        requests.map(_.asScala.toList.foreach(req =>
+          req.context.reply(req.partitionId, status, None)))
       }
     }
 
diff --git 
a/client/src/main/scala/org/apache/celeborn/client/LifecycleManager.scala 
b/client/src/main/scala/org/apache/celeborn/client/LifecycleManager.scala
index 77dcfc559..9cf4119bf 100644
--- a/client/src/main/scala/org/apache/celeborn/client/LifecycleManager.scala
+++ b/client/src/main/scala/org/apache/celeborn/client/LifecycleManager.scala
@@ -240,30 +240,31 @@ class LifecycleManager(appId: String, val conf: 
CelebornConf) extends RpcEndpoin
     case pb: PbRevive =>
       val applicationId = pb.getApplicationId
       val shuffleId = pb.getShuffleId
-      val mapId = pb.getMapId
-      val attemptId = pb.getAttemptId
-      val partitionId = pb.getPartitionId
-      val epoch = pb.getEpoch
-      val oldPartition =
-        if (pb.hasOldPartition) {
-          PbSerDeUtils.fromPbPartitionLocation(pb.getOldPartition)
-        } else {
-          null
-        }
-      val cause = Utils.toStatusCode(pb.getStatus)
-      logTrace(s"Received Revive request, " +
-        s"$applicationId, $shuffleId, $mapId, $attemptId ,$partitionId," +
-        s" $epoch, $oldPartition, $cause.")
+      val mapIds = pb.getMapIdList
+      val partitionInfos = pb.getPartitionInfoList
+
+      val partitionIds = new util.ArrayList[Integer](partitionInfos.size)
+      val epochs = new util.ArrayList[Integer](partitionInfos.size())
+      val oldPartitions = new 
util.ArrayList[PartitionLocation](partitionInfos.size());
+      val causes = new util.ArrayList[StatusCode](partitionIds.size());
+      (0 until partitionInfos.size()).foreach { idx =>
+        val info = partitionInfos.get(idx)
+        partitionIds.add(info.getPartitionId)
+        epochs.add(info.getEpoch)
+        
oldPartitions.add(PbSerDeUtils.fromPbPartitionLocation(info.getPartition))
+        causes.add(Utils.toStatusCode(info.getStatus))
+      }
+      logInfo(
+        s"Received Revive request, shuffleId $shuffleId, partitions 
${partitionIds.size()}, partitionIds ${partitionIds.asScala.mkString(",")}")
       handleRevive(
         context,
         applicationId,
         shuffleId,
-        mapId,
-        attemptId,
-        partitionId,
-        epoch,
-        oldPartition,
-        cause)
+        mapIds,
+        partitionIds,
+        epochs,
+        oldPartitions,
+        causes)
 
     case pb: PbPartitionSplit =>
       val applicationId = pb.getApplicationId
@@ -274,7 +275,7 @@ class LifecycleManager(appId: String, val conf: 
CelebornConf) extends RpcEndpoin
       logTrace(s"Received split request, " +
         s"$applicationId, $shuffleId, $partitionId, $epoch, $oldPartition")
       changePartitionManager.handleRequestPartitionLocation(
-        ChangeLocationCallContext(context),
+        ChangeLocationsCallContext(context, 1),
         applicationId,
         shuffleId,
         partitionId,
@@ -497,43 +498,50 @@ class LifecycleManager(appId: String, val conf: 
CelebornConf) extends RpcEndpoin
       context: RpcCallContext,
       applicationId: String,
       shuffleId: Int,
-      mapId: Int,
-      attemptId: Int,
-      partitionId: Int,
-      oldEpoch: Int,
-      oldPartition: PartitionLocation,
-      cause: StatusCode): Unit = {
+      mapIds: util.List[Integer],
+      partitionIds: util.List[Integer],
+      oldEpochs: util.List[Integer],
+      oldPartitions: util.List[PartitionLocation],
+      causes: util.List[StatusCode]): Unit = {
+    val contextWrapper =
+      ChangeLocationsCallContext(context, partitionIds.size())
     // If shuffle not registered, reply ShuffleNotRegistered and return
     if (!registeredShuffle.contains(shuffleId)) {
       logError(s"[handleRevive] shuffle $shuffleId not registered!")
-      context.reply(ChangeLocationResponse(StatusCode.SHUFFLE_NOT_REGISTERED, 
None))
-      return
-    }
-
-    if (getPartitionType(shuffleId) == PartitionType.MAP) {
-      logError(s"[handleRevive] shuffle $shuffleId revived filed, because map 
partition don't support revive!")
-      context.reply(ChangeLocationResponse(StatusCode.REVIVE_FAILED, None))
+      contextWrapper.reply(
+        -1,
+        StatusCode.SHUFFLE_NOT_REGISTERED,
+        None)
       return
     }
 
-    if (commitManager.isMapperEnded(shuffleId, mapId)) {
-      logWarning(s"[handleRevive] Mapper ended, mapId $mapId, current 
attemptId $attemptId, " +
-        s"ended attemptId 
${commitManager.getMapperAttempts(shuffleId)(mapId)}, shuffleId $shuffleId.")
-      context.reply(ChangeLocationResponse(StatusCode.MAP_ENDED, None))
+    if (commitManager.isStageEnd(shuffleId)) {
+      logError(s"[handleRevive] shuffle $shuffleId stage ended!")
+      contextWrapper.reply(
+        -1,
+        StatusCode.STAGE_ENDED,
+        None)
       return
     }
 
-    logWarning(s"Do Revive for shuffle ${Utils.makeShuffleKey(applicationId, 
shuffleId)}, " +
-      s"oldPartition: $oldPartition, cause: $cause")
+    mapIds.asScala foreach (mapId => {
+      if (commitManager.isMapperEnded(shuffleId, mapId)) {
+        logWarning(s"[handleRevive] Mapper ended, mapId $mapId, ended 
attemptId ${commitManager.getMapperAttempts(
+          shuffleId)(mapId)}, shuffleId $shuffleId")
+        contextWrapper.markMapperEnd(mapId)
+      }
+    })
 
-    changePartitionManager.handleRequestPartitionLocation(
-      ChangeLocationCallContext(context),
-      applicationId,
-      shuffleId,
-      partitionId,
-      oldEpoch,
-      oldPartition,
-      Some(cause))
+    0 until partitionIds.size() foreach (idx => {
+      changePartitionManager.handleRequestPartitionLocation(
+        contextWrapper,
+        applicationId,
+        shuffleId,
+        partitionIds.get(idx),
+        oldEpochs.get(idx),
+        oldPartitions.get(idx),
+        Some(causes.get(idx)))
+    })
   }
 
   private def handleMapperEnd(
@@ -764,7 +772,7 @@ class LifecycleManager(appId: String, val conf: 
CelebornConf) extends RpcEndpoin
       }
     if (!destroyResource.isEmpty) {
       destroySlotsWithRetry(applicationId, shuffleId, destroyResource)
-      logInfo(s"Destroyed peer partitions for reserve buffer failed workers " +
+      logDebug(s"Destroyed peer partitions for reserve buffer failed workers " 
+
         s"${Utils.makeShuffleKey(applicationId, shuffleId)}, $destroyResource")
 
       val workerIds = new util.ArrayList[String]()
@@ -776,7 +784,7 @@ class LifecycleManager(appId: String, val conf: 
CelebornConf) extends RpcEndpoin
       }
       val msg = ReleaseSlots(applicationId, shuffleId, workerIds, 
workerSlotsPerDisk)
       requestMasterReleaseSlots(msg)
-      logInfo(s"Released slots for reserve buffer failed workers " +
+      logDebug(s"Released slots for reserve buffer failed workers " +
         s"${workerIds.asScala.mkString(",")}" + 
s"${slots.asScala.mkString(",")}" +
         s"${Utils.makeShuffleKey(applicationId, shuffleId)}, ")
     }
diff --git 
a/client/src/main/scala/org/apache/celeborn/client/RequestLocationCallContext.scala
 
b/client/src/main/scala/org/apache/celeborn/client/RequestLocationCallContext.scala
index 91cbf6df6..ebbee6cfb 100644
--- 
a/client/src/main/scala/org/apache/celeborn/client/RequestLocationCallContext.scala
+++ 
b/client/src/main/scala/org/apache/celeborn/client/RequestLocationCallContext.scala
@@ -17,23 +17,57 @@
 
 package org.apache.celeborn.client
 
+import java.util
+import java.util.concurrent.ConcurrentHashMap
+
+import org.apache.celeborn.common.internal.Logging
 import org.apache.celeborn.common.protocol.PartitionLocation
 import 
org.apache.celeborn.common.protocol.message.ControlMessages.{ChangeLocationResponse,
 RegisterShuffleResponse}
 import org.apache.celeborn.common.protocol.message.StatusCode
 import org.apache.celeborn.common.rpc.RpcCallContext
 
 trait RequestLocationCallContext {
-  def reply(status: StatusCode, partitionLocationOpt: 
Option[PartitionLocation]): Unit
+  def reply(
+      partitionId: Int,
+      status: StatusCode,
+      partitionLocationOpt: Option[PartitionLocation]): Unit
 }
 
-case class ChangeLocationCallContext(context: RpcCallContext) extends 
RequestLocationCallContext {
-  override def reply(status: StatusCode, partitionLocationOpt: 
Option[PartitionLocation]): Unit = {
-    context.reply(ChangeLocationResponse(status, partitionLocationOpt))
+case class ChangeLocationsCallContext(
+    context: RpcCallContext,
+    partitionCount: Int)
+  extends RequestLocationCallContext with Logging {
+  val endedMapIds = new util.HashSet[Integer]()
+  val newLocs = new ConcurrentHashMap[Integer, (StatusCode, 
PartitionLocation)](partitionCount)
+
+  def markMapperEnd(mapId: Int): Unit = this.synchronized {
+    endedMapIds.add(mapId)
+  }
+
+  override def reply(
+      partitionId: Int,
+      status: StatusCode,
+      partitionLocationOpt: Option[PartitionLocation]): Unit = {
+    newLocs.putIfAbsent(
+      partitionId,
+      (status, partitionLocationOpt.getOrElse(null)))
+
+    if (newLocs.size() == partitionCount || StatusCode.SHUFFLE_NOT_REGISTERED 
== status
+      || StatusCode.STAGE_ENDED == status) {
+      doReply()
+    }
+  }
+
+  private def doReply(): Unit = this.synchronized {
+    context.reply(ChangeLocationResponse(endedMapIds, newLocs))
   }
 }
 
 case class ApplyNewLocationCallContext(context: RpcCallContext) extends 
RequestLocationCallContext {
-  override def reply(status: StatusCode, partitionLocationOpt: 
Option[PartitionLocation]): Unit = {
+  override def reply(
+      partitionId: Int,
+      status: StatusCode,
+      partitionLocationOpt: Option[PartitionLocation]): Unit = {
     partitionLocationOpt match {
       case Some(partitionLocation) =>
         context.reply(RegisterShuffleResponse(status, 
Array(partitionLocation)))
diff --git 
a/client/src/main/scala/org/apache/celeborn/client/ShuffleClientHelper.scala 
b/client/src/main/scala/org/apache/celeborn/client/ShuffleClientHelper.scala
index d6b9be376..16c5c5198 100644
--- a/client/src/main/scala/org/apache/celeborn/client/ShuffleClientHelper.scala
+++ b/client/src/main/scala/org/apache/celeborn/client/ShuffleClientHelper.scala
@@ -42,9 +42,12 @@ object ShuffleClientHelper extends Logging {
       req,
       conf.clientRpcRequestPartitionLocationRpcAskTimeout).onComplete {
       case Success(resp) =>
-        val respStatus = Utils.toStatusCode(resp.getStatus)
+        val partitionInfo = resp.getPartitionInfo(0)
+        val respStatus = Utils.toStatusCode(partitionInfo.getStatus)
         if (respStatus == StatusCode.SUCCESS) {
-          shuffleLocs.put(partitionId, 
PbSerDeUtils.fromPbPartitionLocation(resp.getLocation))
+          shuffleLocs.put(
+            partitionId,
+            PbSerDeUtils.fromPbPartitionLocation(partitionInfo.getPartition))
         } else {
           logInfo(s"split failed for $respStatus, " +
             s"shuffle file can be larger than expected, try split again");
diff --git 
a/common/src/main/java/org/apache/celeborn/common/protocol/ReviveRequest.java 
b/common/src/main/java/org/apache/celeborn/common/protocol/ReviveRequest.java
new file mode 100644
index 000000000..77eb9a520
--- /dev/null
+++ 
b/common/src/main/java/org/apache/celeborn/common/protocol/ReviveRequest.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.celeborn.common.protocol;
+
+import org.apache.celeborn.common.protocol.message.StatusCode;
+
+public class ReviveRequest {
+  public int mapId;
+  public int attemptId;
+  public int partitionId;
+  public int epoch;
+  public PartitionLocation loc;
+  public StatusCode cause;
+  public volatile StatusCode reviveStatus;
+
+  public ReviveRequest(
+      int mapId,
+      int attemptId,
+      int partitionId,
+      int epoch,
+      PartitionLocation loc,
+      StatusCode cause) {
+    this.mapId = mapId;
+    this.attemptId = attemptId;
+    this.partitionId = partitionId;
+    this.epoch = epoch;
+    this.loc = loc;
+    this.cause = cause;
+    reviveStatus = StatusCode.REVIVE_INITIALIZED;
+  }
+}
diff --git 
a/common/src/main/java/org/apache/celeborn/common/protocol/message/StatusCode.java
 
b/common/src/main/java/org/apache/celeborn/common/protocol/message/StatusCode.java
index 688f0def8..5ad47bed8 100644
--- 
a/common/src/main/java/org/apache/celeborn/common/protocol/message/StatusCode.java
+++ 
b/common/src/main/java/org/apache/celeborn/common/protocol/message/StatusCode.java
@@ -77,7 +77,8 @@ public enum StatusCode {
   PUSH_DATA_MASTER_BLACKLISTED(44),
   PUSH_DATA_SLAVE_BLACKLISTED(45),
 
-  FETCH_DATA_TIMEOUT(46);
+  FETCH_DATA_TIMEOUT(46),
+  REVIVE_INITIALIZED(47);
 
   private final byte value;
 
diff --git a/common/src/main/proto/TransportMessages.proto 
b/common/src/main/proto/TransportMessages.proto
index d3f26e4e9..c326d156b 100644
--- a/common/src/main/proto/TransportMessages.proto
+++ b/common/src/main/proto/TransportMessages.proto
@@ -206,20 +206,29 @@ message PbRequestSlotsResponse {
   map<string, PbWorkerResource> workerResource = 2;
 }
 
+message PbRevivePartitionInfo {
+  int32 partitionId = 1;
+  int32 epoch = 2;
+  PbPartitionLocation partition = 3;
+  int32 status = 4;
+}
+
 message PbRevive {
   string applicationId = 1;
   int32 shuffleId = 2;
-  int32 mapId = 3;
-  int32 attemptId = 4;
-  int32 partitionId = 5;
-  int32 epoch = 6;
-  PbPartitionLocation oldPartition = 7;
-  int32 status = 8;
+  repeated int32 mapId = 3;
+  repeated PbRevivePartitionInfo partitionInfo = 4;
+}
+
+message PbChangeLocationPartitionInfo {
+  int32 partitionId = 1;
+  int32 status = 2;
+  PbPartitionLocation partition = 3;
 }
 
 message PbChangeLocationResponse {
-  int32 status = 1;
-  PbPartitionLocation location = 2;
+  repeated int32 endedMapId = 1;
+  repeated PbChangeLocationPartitionInfo partitionInfo = 2;
 }
 
 message PbPartitionSplit {
diff --git 
a/common/src/main/scala/org/apache/celeborn/common/protocol/message/ControlMessages.scala
 
b/common/src/main/scala/org/apache/celeborn/common/protocol/message/ControlMessages.scala
index 438369a42..61c4de4a7 100644
--- 
a/common/src/main/scala/org/apache/celeborn/common/protocol/message/ControlMessages.scala
+++ 
b/common/src/main/scala/org/apache/celeborn/common/protocol/message/ControlMessages.scala
@@ -188,27 +188,51 @@ object ControlMessages extends Logging {
     extends MasterMessage
 
   object Revive {
+    def apply(
+        appId: String,
+        shuffleId: Int,
+        mapIds: util.Set[Integer],
+        reviveRequests: util.Collection[ReviveRequest]): PbRevive = {
+      val builder = PbRevive.newBuilder()
+        .setApplicationId(appId)
+        .setShuffleId(shuffleId)
+        .addAllMapId(mapIds)
+
+      reviveRequests.asScala.foreach(req => {
+        val partitionInfoBuilder = PbRevivePartitionInfo.newBuilder()
+          .setPartitionId(req.partitionId)
+          .setEpoch(req.epoch)
+          .setStatus(req.cause.getValue)
+        if (req.loc != null) {
+          
partitionInfoBuilder.setPartition(PbSerDeUtils.toPbPartitionLocation(req.loc))
+        }
+        builder.addPartitionInfo(partitionInfoBuilder.build())
+      })
+
+      builder.build()
+    }
+
     def apply(
         appId: String,
         shuffleId: Int,
         mapId: Int,
-        attemptId: Int,
         partitionId: Int,
         epoch: Int,
         oldPartition: PartitionLocation,
         cause: StatusCode): PbRevive = {
-      val builder = PbRevive.newBuilder()
-      builder.setApplicationId(appId)
-        .setShuffleId(shuffleId)
-        .setMapId(mapId)
-        .setAttemptId(attemptId)
+      val partitionInfoBuilder = PbRevivePartitionInfo.newBuilder()
         .setPartitionId(partitionId)
         .setEpoch(epoch)
         .setStatus(cause.getValue)
       if (oldPartition != null) {
-        
builder.setOldPartition(PbSerDeUtils.toPbPartitionLocation(oldPartition))
+        
partitionInfoBuilder.setPartition(PbSerDeUtils.toPbPartitionLocation(oldPartition))
       }
-      builder.build()
+      val builder = PbRevive.newBuilder()
+      builder.setApplicationId(appId)
+        .setShuffleId(shuffleId)
+        .addMapId(mapId)
+        .addPartitionInfo(partitionInfoBuilder.build())
+        .build()
     }
   }
 
@@ -230,12 +254,18 @@ object ControlMessages extends Logging {
 
   object ChangeLocationResponse {
     def apply(
-        status: StatusCode,
-        partitionLocationOpt: Option[PartitionLocation]): 
PbChangeLocationResponse = {
+        mapIds: util.Set[Integer],
+        newLocs: util.Map[Integer, (StatusCode, PartitionLocation)]): 
PbChangeLocationResponse = {
       val builder = PbChangeLocationResponse.newBuilder()
-      builder.setStatus(status.getValue)
-      partitionLocationOpt.foreach { partitionLocation =>
-        
builder.setLocation(PbSerDeUtils.toPbPartitionLocation(partitionLocation))
+      builder.addAllEndedMapId(mapIds)
+      newLocs.asScala.foreach { case (partitionId, (status, loc)) =>
+        val pbChangeLocationPartitionInfoBuilder = 
PbChangeLocationPartitionInfo.newBuilder()
+          .setPartitionId(partitionId)
+          .setStatus(status.getValue)
+        if (loc != null) {
+          
pbChangeLocationPartitionInfoBuilder.setPartition(PbSerDeUtils.toPbPartitionLocation(loc))
+        }
+        builder.addPartitionInfo(pbChangeLocationPartitionInfoBuilder.build())
       }
       builder.build()
     }
diff --git 
a/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/RetryReviveTest.scala
 
b/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/RetryReviveTest.scala
index 791ac05df..b5c2c3599 100644
--- 
a/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/RetryReviveTest.scala
+++ 
b/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/RetryReviveTest.scala
@@ -50,8 +50,9 @@ class RetryReviveTest extends AnyFunSuite
     val ss = SparkSession.builder()
       .config(updateSparkConf(sparkConf, ShuffleMode.HASH))
       .getOrCreate()
-    ss.sparkContext.parallelize(1 to 1000, 2)
+    val result = ss.sparkContext.parallelize(1 to 1000, 2)
       .map { i => (i, Range(1, 1000).mkString(",")) }.groupByKey(16).collect()
+    assert(result.size == 1000)
     ss.stop()
   }
 }

Reply via email to