wangyum commented on code in PR #40921:
URL: https://github.com/apache/spark/pull/40921#discussion_r1732131794
##########
core/src/test/scala/org/apache/spark/storage/ShuffleBlockFetcherIteratorSuite.scala:
##########
@@ -262,6 +262,33 @@ class ShuffleBlockFetcherIteratorSuite extends
SparkFunSuite with PrivateMethodT
}
}
+ test("SPARK-43242: Fix throw 'Unexpected type of BlockId' in shuffle
corruption diagnose") {
+ val remoteBmId = BlockManagerId("test-client-1", "test-client-1", 2)
+ val blocks = Map[BlockId, ManagedBuffer](
+ ShuffleBlockBatchId(0, 0, 0, 3) -> createMockManagedBuffer())
+ answerFetchBlocks { invocation =>
+ val listener = invocation.getArgument[BlockFetchingListener](4)
+ listener.onBlockFetchSuccess(ShuffleBlockBatchId(0, 0, 0, 3).toString,
mockCorruptBuffer())
+ }
+
+ val logAppender = new LogAppender("diagnose corruption")
+ withLogAppender(logAppender) {
+ val iterator = createShuffleBlockIteratorWithDefaults(
+ Map(remoteBmId -> toBlockList(blocks.keys, 1L, 0)),
+ streamWrapperLimitSize = Some(100)
+ )
+ intercept[FetchFailedException](iterator.next())
+ verify(transfer, times(2))
+ .fetchBlocks(any(), any(), any(), any(), any(), any())
+ assert(logAppender.loggingEvents.count(
+ _.getMessage.getFormattedMessage.contains("Start corruption
diagnosis")) === 1)
+ assert(logAppender.loggingEvents.exists(
+ _.getMessage.getFormattedMessage.contains("shuffle_0_0_0_3 is
corrupted " +
+ "but corruption diagnosis is skipped due to lack of " +
+ "shuffle checksum support for ShuffleBlockBatchId")))
+ }
+ }
Review Comment:
Could we move the test to the end of this file? We usually append the test.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
