Github user vanzin commented on a diff in the pull request:
https://github.com/apache/spark/pull/23038#discussion_r236426530
--- Diff:
core/src/main/scala/org/apache/spark/status/AppStatusListener.scala ---
@@ -565,7 +571,16 @@ private[spark] class AppStatusListener(
if (metricsDelta != null) {
esummary.metrics = LiveEntityHelpers.addMetrics(esummary.metrics,
metricsDelta)
}
- conditionalLiveUpdate(esummary, now, removeStage)
+
+ val isLastTask =
stage.activeTasksPerExecutor(event.taskInfo.executorId) == 0
+
+ // If the last task of the executor finished, then update the
esummary
+ // for both live and history events.
+ if (isLastTask) {
+ update(esummary, now)
--- End diff --
indentation is off
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
