vanzin commented on a change in pull request #24499: [SPARK-27677][Core] Serve
local disk persisted blocks by the external service after releasing executor by
dynamic allocation
URL: https://github.com/apache/spark/pull/24499#discussion_r286249296
##########
File path: core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala
##########
@@ -1354,6 +1363,59 @@ class BlockManagerSuite extends SparkFunSuite with
Matchers with BeforeAndAfterE
assert(master.getLocations("item").isEmpty)
}
+ test("SPARK-25888: serving of removed file not detected by shuffle service")
{
+ // although the existence of the file is checked before serving it but a
delete can happen
+ // somewhere after that check
+ val store = makeBlockManager(8000, "executor1")
+ val emptyBlockFetcher = new MockBlockTransferService(0) {
+ override def fetchBlockSync(
+ host: String,
+ port: Int,
+ execId: String,
+ blockId: String,
+ tempFileManager: DownloadFileManager): ManagedBuffer = {
+ val transConf = SparkTransportConf.fromSparkConf(conf, "shuffle",
numUsableCores = 1)
+ // empty ManagedBuffer
+ new FileSegmentManagedBuffer(transConf, new File("missing.file"), 0, 0)
+ }
+ }
+ val store2 =
+ makeBlockManager(8000, "executor2", this.master, Some(emptyBlockFetcher))
Review comment:
fits in previous line
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
