Github user kayousterhout commented on a diff in the pull request:
https://github.com/apache/spark/pull/13826#discussion_r68632131
--- Diff:
core/src/test/scala/org/apache/spark/scheduler/TaskSchedulerImplSuite.scala ---
@@ -272,6 +278,45 @@ class TaskSchedulerImplSuite extends SparkFunSuite
with LocalSparkContext with L
val taskDescriptions3 = taskScheduler.resourceOffers(e1Offers).flatten
assert(1 === taskDescriptions3.length)
assert("executor1" === taskDescriptions3(0).executorId)
+ assert(!failedTaskSet)
}
+ test("SPARK-16106 locality levels updated if executor added to existing
host") {
+ // val taskScheduler = setupScheduler("spark.locality.wait" -> "0s")
+ val taskScheduler = setupScheduler()
+
+ taskScheduler.submitTasks(FakeTask.createTaskSet(2, 0,
+ (0 until 2).map { _ => Seq(TaskLocation("host0", "executor2"))}: _*
+ ))
+
+ val taskDescs = taskScheduler.resourceOffers(Seq(
+ new WorkerOffer("executor0", "host0", 1),
+ new WorkerOffer("executor1", "host1", 1)
+ )).flatten
+ // only schedule one task because of locality
+ assert(taskDescs.size === 1)
+
+ val mgr =
taskScheduler.taskIdToTaskSetManager.get(taskDescs(0).taskId).get
+ assert(mgr.myLocalityLevels.toSet === Set(TaskLocality.NODE_LOCAL,
TaskLocality.ANY))
+ // we should know about both executors, even though we only scheduled
tasks on one of them
+ assert(taskScheduler.getExecutorsAliveOnHost("host0") ===
Some(Set("executor0")))
+ assert(taskScheduler.getExecutorsAliveOnHost("host1") ===
Some(Set("executor1")))
+
+ // suppose that now executor2 is added, we should realize that we can
run process-local tasks.
--- End diff --
"suppose that now" --> "when"? (a little hard to parse as-is)
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
