Github user kayousterhout commented on a diff in the pull request:
https://github.com/apache/spark/pull/15249#discussion_r81423460
--- Diff:
core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala ---
@@ -809,32 +816,65 @@ class TaskSetManagerSuite extends SparkFunSuite with
LocalSparkContext with Logg
test("Kill other task attempts when one attempt belonging to the same
task succeeds") {
sc = new SparkContext("local", "test")
sched = new FakeTaskScheduler(sc, ("exec1", "host1"), ("exec2",
"host2"))
- val taskSet = FakeTask.createTaskSet(4)
+ val taskSet = FakeTask.createTaskSet(5)
// Set the speculation multiplier to be 0 so speculative tasks are
launched immediately
sc.conf.set("spark.speculation.multiplier", "0.0")
+ sc.conf.set("spark.speculation.quantile", "0.6")
val clock = new ManualClock()
val manager = new TaskSetManager(sched, taskSet, MAX_TASK_FAILURES,
clock)
val accumUpdatesByTask: Array[Seq[AccumulatorV2[_, _]]] =
taskSet.tasks.map { task =>
task.metrics.internalAccums
}
// Offer resources for 4 tasks to start
+ val tasks = new ArrayBuffer[TaskDescription]()
for ((k, v) <- List(
"exec1" -> "host1",
"exec1" -> "host1",
+ "exec1" -> "host1",
"exec2" -> "host2",
"exec2" -> "host2")) {
val taskOption = manager.resourceOffer(k, v, NO_PREF)
assert(taskOption.isDefined)
val task = taskOption.get
assert(task.executorId === k)
+ tasks += task
}
- assert(sched.startedTasks.toSet === Set(0, 1, 2, 3))
- // Complete the 3 tasks and leave 1 task in running
+ assert(sched.startedTasks.toSet === (0 until 5).toSet)
+ // Complete 3 tasks and leave 2 task in running
for (id <- Set(0, 1, 2)) {
manager.handleSuccessfulTask(id, createTaskResult(id,
accumUpdatesByTask(id)))
assert(sched.endedTasks(id) === Success)
}
+ def runningTaskForIndex(index: Int): TaskDescription = {
+ val t = tasks.find { task => task.index == index &&
!sched.endedTasks.contains(task.taskId) }
+ t match {
+ case Some(x) => x
+ case None =>
+ throw new RuntimeException(s"couldn't find index $index in " +
+ s"tasks: ${tasks.map{t => t.index -> t.taskId}} with
endedTasks:" +
+ s" ${sched.endedTasks.keys}")
+ }
+ }
+
+ // have each of the running tasks fail 3 times (not enough to abort
the stage)
+ (3 until 6).foreach { attempt =>
+ Seq(3, 4).foreach { index =>
+ val task = runningTaskForIndex(index)
+ logInfo(s"failing task $task")
+ val endReason = ExceptionFailure("a", "b", Array(), "c", None)
+ manager.handleFailedTask(task.taskId, TaskState.FAILED, endReason)
+ sched.endedTasks(task.taskId) = endReason
+ assert(!manager.isZombie)
+ val nextTask = manager.resourceOffer(s"exec2", s"host2", NO_PREF)
+ assert(nextTask.isDefined, s"no offer for attempt $attempt of
$index")
+ tasks += nextTask.get
+ }
+ }
+
+ // we can't be sure which one of our running tasks will get anothe
speculative copy
--- End diff --
another
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
