Github user vanzin commented on a diff in the pull request:
https://github.com/apache/spark/pull/22209#discussion_r212705093
--- Diff:
core/src/main/scala/org/apache/spark/status/AppStatusListener.scala ---
@@ -608,14 +627,20 @@ private[spark] class AppStatusListener(
}
stage.executorSummaries.values.foreach(update(_, now))
- update(stage, now, last = true)
val executorIdsForStage = stage.blackListedExecutors
executorIdsForStage.foreach { executorId =>
liveExecutors.get(executorId).foreach { exec =>
removeBlackListedStageFrom(exec, event.stageInfo.stageId, now)
}
}
+
+ // Remove stage only if there are no active tasks remaining
+ val removeStage = stage.activeTasks == 0
+ update(stage, now, removeStage)
--- End diff --
`last = removeStage`.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
