cloud-fan commented on a change in pull request #19788: [SPARK-9853][Core]
Optimize shuffle fetch of contiguous partition IDs
URL: https://github.com/apache/spark/pull/19788#discussion_r249478868
##########
File path:
common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/protocol/OpenBlocks.java
##########
@@ -71,20 +89,31 @@ public boolean equals(Object other) {
public int encodedLength() {
return Encoders.Strings.encodedLength(appId)
+ Encoders.Strings.encodedLength(execId)
- + Encoders.StringArrays.encodedLength(blockIds);
+ + Encoders.StringArrays.encodedLength(blockIds)
+ + (fetchContinuousShuffleBlocksInBatch ? 1 : 0);
}
@Override
public void encode(ByteBuf buf) {
Encoders.Strings.encode(buf, appId);
Encoders.Strings.encode(buf, execId);
Encoders.StringArrays.encode(buf, blockIds);
+ if (fetchContinuousShuffleBlocksInBatch) {
+ buf.writeBoolean(fetchContinuousShuffleBlocksInBatch);
+ }
}
public static OpenBlocks decode(ByteBuf buf) {
String appId = Encoders.Strings.decode(buf);
String execId = Encoders.Strings.decode(buf);
String[] blockIds = Encoders.StringArrays.decode(buf);
- return new OpenBlocks(appId, execId, blockIds);
+ boolean fetchContinuousShuffleBlocksInBatch = false;
+ if (buf.readableBytes() >= 1) {
+ // A sanity check. In `encode` we write true, so here we should read
true.
Review comment:
we should do the check `assert(buf.getBoolean)`
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
