Github user mccheah commented on a diff in the pull request:
https://github.com/apache/spark/pull/21221#discussion_r206333224
--- Diff:
core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala ---
@@ -155,7 +160,14 @@ private[spark] class EventLoggingListener(
}
// Events that do not trigger a flush
- override def onStageSubmitted(event: SparkListenerStageSubmitted): Unit
= logEvent(event)
+ override def onStageSubmitted(event: SparkListenerStageSubmitted): Unit
= {
+ logEvent(event)
+ if (shouldLogStageExecutorMetrics) {
+ // record the peak metrics for the new stage
+ liveStageExecutorMetrics.put((event.stageInfo.stageId,
event.stageInfo.attemptNumber()),
+ new HashMap[String, ExecutorMetrics]())
--- End diff --
`mutable.Map.empty[String, ExecutorMetrics]` if the above signature is
changed.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
