This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new d945696 [SPARK-33029][CORE][WEBUI] Fix the UI executor page
incorrectly marking the driver as excluded
d945696 is described below
commit d94569658735dceb2351a869dcc1909b0ea7e453
Author: Baohe Zhang <[email protected]>
AuthorDate: Tue Jan 5 19:16:40 2021 -0800
[SPARK-33029][CORE][WEBUI] Fix the UI executor page incorrectly marking the
driver as excluded
### What changes were proposed in this pull request?
Filter out the driver entity when updating the exclusion status of live
executors(including the driver), so the UI won't be marked as excluded in the
UI even if the node that hosts the driver has been marked as excluded.
### Why are the changes needed?
Before this change, if we run spark with the standalone mode and with
spark.blacklist.enabled=true. The driver will be marked as excluded when the
host that hosts that driver has been marked as excluded. While it's incorrect
because the exclude list feature will exclude executors only and the driver is
still active.
After the fix, the driver won't be marked as excluded.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Manual test. Reopen the UI and see the driver is no longer marked as
excluded.
Closes #30954 from baohe-zhang/SPARK-33029.
Authored-by: Baohe Zhang <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
(cherry picked from commit 29510821a0e3b1e09a7710ed02a0fa1caab506af)
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../main/scala/org/apache/spark/status/AppStatusListener.scala | 8 +++++---
.../executor_memory_usage_expectation.json | 4 ++--
.../executor_node_excludeOnFailure_expectation.json | 4 ++--
3 files changed, 9 insertions(+), 7 deletions(-)
diff --git
a/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
b/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
index bf19897..6cb013b 100644
--- a/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
+++ b/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
@@ -366,10 +366,12 @@ private[spark] class AppStatusListener(
// Implicitly exclude every available executor for the stage associated
with this node
Option(liveStages.get((stageId, stageAttemptId))).foreach { stage =>
- val executorIds = liveExecutors.values.filter(_.host ==
hostId).map(_.executorId).toSeq
+ val executorIds = liveExecutors.values.filter(exec => exec.host == hostId
+ && exec.executorId !=
SparkContext.DRIVER_IDENTIFIER).map(_.executorId).toSeq
setStageExcludedStatus(stage, now, executorIds: _*)
}
- liveExecutors.values.filter(_.hostname == hostId).foreach { exec =>
+ liveExecutors.values.filter(exec => exec.hostname == hostId
+ && exec.executorId != SparkContext.DRIVER_IDENTIFIER).foreach { exec =>
addExcludedStageTo(exec, stageId, now)
}
}
@@ -416,7 +418,7 @@ private[spark] class AppStatusListener(
// Implicitly (un)exclude every executor associated with the node.
liveExecutors.values.foreach { exec =>
- if (exec.hostname == host) {
+ if (exec.hostname == host && exec.executorId !=
SparkContext.DRIVER_IDENTIFIER) {
updateExecExclusionStatus(exec, excluded, now)
}
}
diff --git
a/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json
b/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json
index 9adda27..5144934 100644
---
a/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json
@@ -16,7 +16,7 @@
"totalInputBytes" : 0,
"totalShuffleRead" : 0,
"totalShuffleWrite" : 0,
- "isBlacklisted" : true,
+ "isBlacklisted" : false,
"maxMemory" : 908381388,
"addTime" : "2016-11-16T22:33:31.477GMT",
"executorLogs" : { },
@@ -30,7 +30,7 @@
"attributes" : { },
"resources" : { },
"resourceProfileId" : 0,
- "isExcluded" : true,
+ "isExcluded" : false,
"excludedInStages" : [ ]
}, {
"id" : "3",
diff --git
a/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_expectation.json
b/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_expectation.json
index 65bd309..47a01b2 100644
---
a/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_expectation.json
@@ -16,7 +16,7 @@
"totalInputBytes" : 0,
"totalShuffleRead" : 0,
"totalShuffleWrite" : 0,
- "isBlacklisted" : true,
+ "isBlacklisted" : false,
"maxMemory" : 908381388,
"addTime" : "2016-11-16T22:33:31.477GMT",
"executorLogs" : { },
@@ -30,7 +30,7 @@
"attributes" : { },
"resources" : { },
"resourceProfileId" : 0,
- "isExcluded" : true,
+ "isExcluded" : false,
"excludedInStages" : [ ]
}, {
"id" : "3",
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
