dongjoon-hyun commented on code in PR #45228:
URL: https://github.com/apache/spark/pull/45228#discussion_r1533274587
##########
core/src/test/scala/org/apache/spark/storage/ShuffleBlockFetcherIteratorSuite.scala:
##########
@@ -447,7 +462,30 @@ class ShuffleBlockFetcherIteratorSuite extends
SparkFunSuite with PrivateMethodT
assert(!iterator.hasNext)
}
- test("fetch continuous blocks in batch successful 3 local + 4 host local + 2
remote reads") {
+ def createShuffleFile(shuffleId: Int, mapId: Int, reducerId: Int, conf:
SparkConf): Unit = {
+ var name = ShuffleIndexBlockId(shuffleId, mapId, reducerId).name
+ var hash = JavaUtils.nonNegativeHash(name)
+ val hadoopConf = SparkHadoopUtil.get.newConfiguration(conf)
+ val appId = conf.getAppId
+ val path: String = conf.get(STORAGE_DECOMMISSION_FALLBACK_STORAGE_PATH).get
+ val indexFile = new Path(path, s"$appId/$shuffleId/$hash/$name")
+ name = ShuffleDataBlockId(shuffleId, mapId, reducerId).name
+ hash = JavaUtils.nonNegativeHash(name)
+ val dataFile = new Path(path, s"$appId/$shuffleId/$hash/$name")
+ val fallbackFileSystem =
org.apache.hadoop.fs.FileSystem.get(indexFile.toUri, hadoopConf)
+ val indexOut = fallbackFileSystem.create(indexFile)
+ indexOut.writeLong(0L) // offset
+ indexOut.writeLong(10L) // next offset
+ indexOut.writeLong(10L) // next offset
+ indexOut.writeLong(10L) // next offset
+ val dataOut = fallbackFileSystem.create(dataFile)
+ dataOut.writeBytes("some data to write")
+ indexOut.close()
+ dataOut.close()
+ }
+
+ test("fetch continuous blocks in batch successful 3 local + 4 host local + 2
remote reads " +
+ "+ 2 read from external storage") {
Review Comment:
This PR seems to introduce a test failure (or flakiness) at this test case.
Could you take a look at the CI failure?
- https://github.com/maheshk114/spark/actions/runs/8369424178/job/22915159186
```
[info] ShuffleBlockFetcherIteratorSuite:
...
[info] - fetch continuous blocks in batch successful 3 local + 4 host local
+ 2 remote reads + 2 read from external storage *** FAILED *** (107
milliseconds)
[info] org.mockito.exceptions.verification.TooFewActualInvocations:
blockManager.getLocalBlockData(<any>);
[info] Wanted 2 times:
[info] -> at
org.apache.spark.storage.BlockManager.getLocalBlockData(BlockManager.scala:721)
[info] But was 1 time:
[info] -> at
org.apache.spark.storage.ShuffleBlockFetcherIterator.fetchLocalBlocks(ShuffleBlockFetcherIterator.scala:592)
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
