Repository: spark
Updated Branches:
refs/heads/branch-2.3 32bec6ca3 -> 7fb11176f
[SPARK-24021][CORE] fix bug in BlacklistTracker's updateBlacklistForFetchFailure
## What changes were proposed in this pull request?
Thereâs a miswrite in BlacklistTracker's updateBlacklistForFetchFailure:
```
val blacklistedExecsOnNode =
nodeToBlacklistedExecs.getOrElseUpdate(exec, HashSet[String]())
blacklistedExecsOnNode += exec
```
where first **exec** should be **host**.
## How was this patch tested?
adjust existed test.
Author: wuyi <[email protected]>
Closes #21104 from Ngone51/SPARK-24021.
(cherry picked from commit 0deaa5251326a32a3d2d2b8851193ca926303972)
Signed-off-by: Imran Rashid <[email protected]>
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/7fb11176
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/7fb11176
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/7fb11176
Branch: refs/heads/branch-2.3
Commit: 7fb11176f285b4de47e61511c09acbbb79e5c44c
Parents: 32bec6c
Author: wuyi <[email protected]>
Authored: Thu Apr 19 09:00:33 2018 -0500
Committer: Imran Rashid <[email protected]>
Committed: Thu Apr 19 09:00:46 2018 -0500
----------------------------------------------------------------------
.../scala/org/apache/spark/scheduler/BlacklistTracker.scala | 2 +-
.../org/apache/spark/scheduler/BlacklistTrackerSuite.scala | 5 +++++
2 files changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/7fb11176/core/src/main/scala/org/apache/spark/scheduler/BlacklistTracker.scala
----------------------------------------------------------------------
diff --git
a/core/src/main/scala/org/apache/spark/scheduler/BlacklistTracker.scala
b/core/src/main/scala/org/apache/spark/scheduler/BlacklistTracker.scala
index 952598f..30cf75d 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/BlacklistTracker.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/BlacklistTracker.scala
@@ -210,7 +210,7 @@ private[scheduler] class BlacklistTracker (
updateNextExpiryTime()
killBlacklistedExecutor(exec)
- val blacklistedExecsOnNode =
nodeToBlacklistedExecs.getOrElseUpdate(exec, HashSet[String]())
+ val blacklistedExecsOnNode =
nodeToBlacklistedExecs.getOrElseUpdate(host, HashSet[String]())
blacklistedExecsOnNode += exec
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/7fb11176/core/src/test/scala/org/apache/spark/scheduler/BlacklistTrackerSuite.scala
----------------------------------------------------------------------
diff --git
a/core/src/test/scala/org/apache/spark/scheduler/BlacklistTrackerSuite.scala
b/core/src/test/scala/org/apache/spark/scheduler/BlacklistTrackerSuite.scala
index 88a57c5..00867ef 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/BlacklistTrackerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/BlacklistTrackerSuite.scala
@@ -574,6 +574,9 @@ class BlacklistTrackerSuite extends SparkFunSuite with
BeforeAndAfterEach with M
verify(allocationClientMock, never).killExecutors(any(), any(), any(),
any())
verify(allocationClientMock, never).killExecutorsOnHost(any())
+ assert(blacklist.nodeToBlacklistedExecs.contains("hostA"))
+ assert(blacklist.nodeToBlacklistedExecs("hostA").contains("1"))
+
// Enable auto-kill. Blacklist an executor and make sure killExecutors is
called.
conf.set(config.BLACKLIST_KILL_ENABLED, true)
blacklist = new BlacklistTracker(listenerBusMock, conf,
Some(allocationClientMock), clock)
@@ -589,6 +592,8 @@ class BlacklistTrackerSuite extends SparkFunSuite with
BeforeAndAfterEach with M
1000 + blacklist.BLACKLIST_TIMEOUT_MILLIS)
assert(blacklist.nextExpiryTime === 1000 +
blacklist.BLACKLIST_TIMEOUT_MILLIS)
assert(blacklist.nodeIdToBlacklistExpiryTime.isEmpty)
+ assert(blacklist.nodeToBlacklistedExecs.contains("hostA"))
+ assert(blacklist.nodeToBlacklistedExecs("hostA").contains("1"))
// Enable external shuffle service to see if all the executors on this
node will be killed.
conf.set(config.SHUFFLE_SERVICE_ENABLED, true)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
