zhouyejoe commented on a change in pull request #33078:
URL: https://github.com/apache/spark/pull/33078#discussion_r671563694
##########
File path:
common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/RemoteBlockPushResolver.java
##########
@@ -112,34 +118,59 @@ public ShuffleIndexInformation load(File file) throws
IOException {
this.errorHandler = new ErrorHandler.BlockPushErrorHandler();
}
+ @VisibleForTesting
+ protected AppShuffleInfo validateAndGetAppShuffleInfo(String appId) {
+ // TODO: [SPARK-33236] Change the message when this service is able to
handle NM restart
+ AppShuffleInfo appShuffleInfo =
+ Preconditions.checkNotNull(appsShuffleInfo.get(appId),
Review comment:
Updated.
##########
File path:
common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/RemoteBlockPushResolver.java
##########
@@ -73,14 +72,22 @@
public class RemoteBlockPushResolver implements MergedShuffleFileManager {
private static final Logger logger =
LoggerFactory.getLogger(RemoteBlockPushResolver.class);
- @VisibleForTesting
- static final String MERGE_MANAGER_DIR = "merge_manager";
+
public static final String MERGED_SHUFFLE_FILE_NAME_PREFIX = "shuffleMerged";
+ public static final String SHUFFLE_META_DELIMITER = ":";
+ public static final String MERGE_DIR_KEY = "mergeDir";
+ public static final String ATTEMPT_ID_KEY = "attemptId";
+ private static final int UNDEFINED_ATTEMPT_ID = -1;
- private final ConcurrentMap<String, AppPathsInfo> appsPathInfo;
- private final ConcurrentMap<AppShuffleId, Map<Integer,
AppShufflePartitionInfo>> partitions;
+ /**
+ * A concurrent hashmap where the key is the applicationId, and the value
includes
+ * all the merged shuffle information for this application. AppShuffleInfo
stores
+ * the application attemptId, merged shuffle local directories and the
metadata
+ * for actively being merged shuffle partitions.
+ */
+ private final ConcurrentMap<String, AppShuffleInfo> appsShuffleInfo;
- private final Executor directoryCleaner;
+ private final Executor mergedShuffleCleanerExecutor;
Review comment:
Updated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
