otterc commented on a change in pull request #32140:
URL: https://github.com/apache/spark/pull/32140#discussion_r646965372
##########
File path:
common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalBlockHandler.java
##########
@@ -333,14 +382,18 @@ public ShuffleMetrics() {
final int[] mapIdAndReduceIds = new int[2 * blockIds.length];
for (int i = 0; i < blockIds.length; i++) {
String[] blockIdParts = blockIds[i].split("_");
- if (blockIdParts.length != 4 || !blockIdParts[0].equals("shuffle")) {
+ if (blockIdParts.length != 4
+ || (!requestForMergedBlockChunks &&
!blockIdParts[0].equals(SHUFFLE_BLOCK_PREFIX))
+ || (requestForMergedBlockChunks &&
!blockIdParts[0].equals(SHUFFLE_CHUNK_PREFIX))) {
throw new IllegalArgumentException("Unexpected shuffle block id
format: " + blockIds[i]);
}
if (Integer.parseInt(blockIdParts[1]) != shuffleId) {
throw new IllegalArgumentException("Expected shuffleId=" + shuffleId
+
", got:" + blockIds[i]);
}
+ // For regular blocks this is mapId. For chunks this is reduceId.
mapIdAndReduceIds[2 * i] = Integer.parseInt(blockIdParts[2]);
+ // For regular blocks this is reduceId. For chunks this is chunkId.
mapIdAndReduceIds[2 * i + 1] = Integer.parseInt(blockIdParts[3]);
Review comment:
I have made this change in this PR:
https://github.com/apache/spark/pull/32811
So resolving the conversation here
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
