squito commented on a change in pull request #23453: [SPARK-26089][CORE] Handle
corruption in large shuffle blocks
URL: https://github.com/apache/spark/pull/23453#discussion_r254874973
##########
File path:
core/src/test/scala/org/apache/spark/storage/ShuffleBlockFetcherIteratorSuite.scala
##########
@@ -449,54 +452,75 @@ class ShuffleBlockFetcherIteratorSuite extends
SparkFunSuite with PrivateMethodT
}
test("big blocks are also checked for corruption") {
- val corruptBuffer1 = mockCorruptBuffer(10000L, true)
-
+ val streamLength = 10000L
val blockManager = mock(classOf[BlockManager])
+
+ // This stream will throw IOException when the first byte is read
+ val localBuffer = mockCorruptBuffer(streamLength, 0)
val localBmId = BlockManagerId("test-client", "test-client", 1)
doReturn(localBmId).when(blockManager).blockManagerId
- doReturn(corruptBuffer1).when(blockManager).getBlockData(ShuffleBlockId(0,
0, 0))
+ doReturn(localBuffer).when(blockManager).getBlockData(ShuffleBlockId(0, 0,
0))
+ val localShuffleBlockId = ShuffleBlockId(0, 0, 0)
val localBlockLengths = Seq[Tuple2[BlockId, Long]](
- ShuffleBlockId(0, 0, 0) -> corruptBuffer1.size()
+ localShuffleBlockId -> localBuffer.size()
)
- val corruptBuffer2 = mockCorruptBuffer(10000L, false)
+ val streamNotCorruptTill = 8 * 1024
+ // This stream will throw exception after streamNotCorruptTill bytes are
read
+ val remoteBuffer = mockCorruptBuffer(streamLength, streamNotCorruptTill)
val remoteBmId = BlockManagerId("test-client-1", "test-client-1", 2)
Review comment:
I meant the blockManagerId should be "remote-client" or something. These
show up in the logs, when I was trying to step through I got very confused with
the names so similar.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
