git commit: [SPARK-2350] Don't NPE while launching drivers
Repository: spark
Updated Branches:
refs/heads/branch-1.0 d2f253467 - 27a2afed6
[SPARK-2350] Don't NPE while launching drivers
Prior to this change, we could throw a NPE if we launch a driver while another
one is waiting, because removing from an iterator while iterating over it is
not safe.
Author: Aaron Davidson aa...@databricks.com
Closes #1289 from aarondav/master-fail and squashes the following commits:
1cf1cf4 [Aaron Davidson] SPARK-2350: Don't NPE while launching drivers
(cherry picked from commit 586feb5c9528042420f678f78bacb6c254a5eaf8)
Signed-off-by: Patrick Wendell pwend...@gmail.com
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/27a2afed
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/27a2afed
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/27a2afed
Branch: refs/heads/branch-1.0
Commit: 27a2afed60035474a100436798c8048506addb54
Parents: d2f2534
Author: Aaron Davidson aa...@databricks.com
Authored: Thu Jul 3 22:31:41 2014 -0700
Committer: Patrick Wendell pwend...@gmail.com
Committed: Thu Jul 3 22:32:05 2014 -0700
--
core/src/main/scala/org/apache/spark/deploy/master/Master.scala | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/27a2afed/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
--
diff --git a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
index 33ffcbd..4ed4827 100644
--- a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
@@ -481,7 +481,7 @@ private[spark] class Master(
// First schedule drivers, they take strict precedence over applications
val shuffledWorkers = Random.shuffle(workers) // Randomization helps
balance drivers
for (worker - shuffledWorkers if worker.state == WorkerState.ALIVE) {
- for (driver - waitingDrivers) {
+ for (driver - List(waitingDrivers: _*)) { // iterate over a copy of
waitingDrivers
if (worker.memoryFree = driver.desc.mem worker.coresFree =
driver.desc.cores) {
launchDriver(worker, driver)
waitingDrivers -= driver
git commit: [SPARK-2350] Don't NPE while launching drivers
Repository: spark
Updated Branches:
refs/heads/master 5fa0a0576 - 586feb5c9
[SPARK-2350] Don't NPE while launching drivers
Prior to this change, we could throw a NPE if we launch a driver while another
one is waiting, because removing from an iterator while iterating over it is
not safe.
Author: Aaron Davidson aa...@databricks.com
Closes #1289 from aarondav/master-fail and squashes the following commits:
1cf1cf4 [Aaron Davidson] SPARK-2350: Don't NPE while launching drivers
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/586feb5c
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/586feb5c
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/586feb5c
Branch: refs/heads/master
Commit: 586feb5c9528042420f678f78bacb6c254a5eaf8
Parents: 5fa0a05
Author: Aaron Davidson aa...@databricks.com
Authored: Thu Jul 3 22:31:41 2014 -0700
Committer: Patrick Wendell pwend...@gmail.com
Committed: Thu Jul 3 22:31:41 2014 -0700
--
core/src/main/scala/org/apache/spark/deploy/master/Master.scala | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/586feb5c/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
--
diff --git a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
index 11545b8..a304102 100644
--- a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
@@ -481,7 +481,7 @@ private[spark] class Master(
// First schedule drivers, they take strict precedence over applications
val shuffledWorkers = Random.shuffle(workers) // Randomization helps
balance drivers
for (worker - shuffledWorkers if worker.state == WorkerState.ALIVE) {
- for (driver - waitingDrivers) {
+ for (driver - List(waitingDrivers: _*)) { // iterate over a copy of
waitingDrivers
if (worker.memoryFree = driver.desc.mem worker.coresFree =
driver.desc.cores) {
launchDriver(worker, driver)
waitingDrivers -= driver
