zhouyejoe commented on a change in pull request #33078:
URL: https://github.com/apache/spark/pull/33078#discussion_r663427925
##########
File path:
common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/RemoteBlockPushResolver.java
##########
@@ -112,34 +116,48 @@ public ShuffleIndexInformation load(File file) throws
IOException {
this.errorHandler = new ErrorHandler.BlockPushErrorHandler();
}
+ private AppShuffleInfo validateAndGetAppShuffleInfo(String appId) {
+ // TODO: [SPARK-33236] Change the message when this service is able to
handle NM restart
+ AppShuffleInfo appShuffleInfo =
+ Preconditions.checkNotNull(appsShuffleInfo.get(appId),
+ "application " + appId + " is not registered or NM was restarted.");
+ return appShuffleInfo;
+ }
+
/**
* Given the appShuffleId and reduceId that uniquely identifies a given
shuffle partition of an
* application, retrieves the associated metadata. If not present and the
corresponding merged
* shuffle does not exist, initializes the metadata.
*/
private AppShufflePartitionInfo getOrCreateAppShufflePartitionInfo(
- AppShuffleId appShuffleId,
+ AppShuffleInfo appShuffleInfo,
+ int shuffleId,
int reduceId) {
- File dataFile = getMergedShuffleDataFile(appShuffleId, reduceId);
- if (!partitions.containsKey(appShuffleId) && dataFile.exists()) {
+ File dataFile = appShuffleInfo.getMergedShuffleDataFile(shuffleId,
reduceId);
+ ConcurrentMap<Integer, Map<Integer, AppShufflePartitionInfo>> partitions =
+ appShuffleInfo.partitions;
+ if (!partitions.containsKey(shuffleId) && dataFile.exists()) {
// If this partition is already finalized then the partitions map will
not contain
// the appShuffleId but the data file would exist. In that case the
block is considered late.
return null;
}
Review comment:
Updated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
