attilapiros commented on a change in pull request #28848:
URL: https://github.com/apache/spark/pull/28848#discussion_r442079384
##########
File path:
core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala
##########
@@ -540,6 +540,43 @@ class DAGSchedulerSuite extends SparkFunSuite with
LocalSparkContext with TimeLi
assert(mapStatus2(2).location.host === "hostB")
}
+ test("[SPARK-32003] All shuffle files for executor should be cleaned up on
fetch failure") {
+ // reset the test context with the right shuffle service config
+ afterEach()
+ val conf = new SparkConf()
+ conf.set(config.SHUFFLE_SERVICE_ENABLED.key, "true")
+ init(conf)
+
+ val shuffleMapRdd = new MyRDD(sc, 3, Nil)
+ val shuffleDep = new ShuffleDependency(shuffleMapRdd, new
HashPartitioner(3))
+ val shuffleId = shuffleDep.shuffleId
+ val reduceRdd = new MyRDD(sc, 3, List(shuffleDep), tracker =
mapOutputTracker)
+
+ submit(reduceRdd, Array(0, 1, 2))
+ // Map stage completes successfully,
+ // two tasks are run on an executor on hostA and one on an executor on
hostB
+ completeShuffleMapStageSuccessfully(0, 0, 3, Seq("hostA", "hostA",
"hostB"))
+ // Now the executor on hostA is lost
+ runEvent(ExecutorLost("hostA-exec", ExecutorExited(-100, false, "Container
marked as failed")))
+
+ // The MapOutputTracker has all the shuffle files
+ val initialMapStatuses =
mapOutputTracker.shuffleStatuses(shuffleId).mapStatuses
+ assert(initialMapStatuses.count(_ != null) === 3)
+ assert(initialMapStatuses.count(s => s != null && s.location.executorId ==
"hostA-exec") === 2)
+ assert(initialMapStatuses.count(s => s != null && s.location.executorId ==
"hostB-exec") === 1)
+
+ // Now a fetch failure from the lost executor occurs
+ complete(taskSets(1), Seq(
+ (FetchFailed(makeBlockManagerId("hostA"), shuffleId, 0L, 0, 0,
"ignored"), null)
+ ))
+
+ // Shuffle files for hostA-exec should be lost
+ val mapStatuses = mapOutputTracker.shuffleStatuses(shuffleId).mapStatuses
+ assert(mapStatuses.count(_ != null) === 1)
+ assert(initialMapStatuses.count(s => s != null && s.location.executorId ==
"hostA-exec") === 0)
+ assert(initialMapStatuses.count(s => s != null && s.location.executorId ==
"hostB-exec") === 1)
Review comment:
As `mapStatuses` is reflecting the underlying change it is not needed to
get it in the second time:
```suggestion
val mapStatuses = mapOutputTracker.shuffleStatuses(shuffleId).mapStatuses
assert(mapStatuses.count(_ != null) === 3)
assert(mapStatuses.count(s => s != null && s.location.executorId ==
"hostA-exec") === 2)
assert(mapStatuses.count(s => s != null && s.location.executorId ==
"hostB-exec") === 1)
// Now a fetch failure from the lost executor occurs
complete(taskSets(1), Seq(
(FetchFailed(makeBlockManagerId("hostA"), shuffleId, 0L, 0, 0,
"ignored"), null)
))
// Shuffle files for hostA-exec should be lost
assert(mapStatuses.count(_ != null) === 1)
assert(mapStatuses.count(s => s != null && s.location.executorId ==
"hostA-exec") === 0)
assert(mapStatuses.count(s => s != null && s.location.executorId ==
"hostB-exec") === 1)
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
