This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 2d56903bea9 [SPARK-43397][CORE] Log executor decommission duration in
`executorLost` method
2d56903bea9 is described below
commit 2d56903bea9ebd0d02bab976eff8cc58265cade3
Author: Warren Zhu <[email protected]>
AuthorDate: Mon May 8 18:01:13 2023 -0700
[SPARK-43397][CORE] Log executor decommission duration in `executorLost`
method
### What changes were proposed in this pull request?
Log executor decommission duration
### Why are the changes needed?
Executor decommission duration is important info to understand resource
efficiency and troubleshoot decommission issue.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Manually tested.
Closes #41077 from warrenzhu25/decom-time.
Authored-by: Warren Zhu <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
diff --git
a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
index 0e916bf375e..0cb970fd278 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
@@ -1086,11 +1086,19 @@ private[spark] class TaskSchedulerImpl(
case ExecutorKilled =>
logInfo(s"Executor $executorId on $hostPort killed by driver.")
case _: ExecutorDecommission =>
- logInfo(s"Executor $executorId on $hostPort is decommissioned.")
+ logInfo(s"Executor $executorId on $hostPort is decommissioned" +
+ s"${getDecommissionDuration(executorId)}.")
case _ =>
logError(s"Lost executor $executorId on $hostPort: $reason")
}
+ // return decommission duration in string or "" if decommission startTime
not exists
+ private def getDecommissionDuration(executorId: String): String = {
+ executorsPendingDecommission.get(executorId)
+ .map(s => s" after ${Utils.msDurationToString(clock.getTimeMillis() -
s.startTime)}")
+ .getOrElse("")
+ }
+
/**
* Cleans up the TaskScheduler's state for tracking the given task.
*/
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
