Github user vanzin commented on a diff in the pull request:
https://github.com/apache/spark/pull/20493#discussion_r166064864
--- Diff: core/src/main/scala/org/apache/spark/status/AppStatusUtils.scala
---
@@ -17,16 +17,23 @@
package org.apache.spark.status
-import org.apache.spark.status.api.v1.{TaskData, TaskMetrics}
+import org.apache.spark.status.api.v1.TaskData
private[spark] object AppStatusUtils {
+ private val TASK_FINISHED_STATES = Set("FAILED", "KILLED", "SUCCESS")
+
+ private def isTaskFinished(task: TaskData): Boolean = {
+ TASK_FINISHED_STATES.contains(task.status)
+ }
+
def schedulerDelay(task: TaskData): Long = {
- if (task.taskMetrics.isDefined && task.duration.isDefined) {
+ if (isTaskFinished(task) && task.taskMetrics.isDefined &&
task.duration.isDefined) {
--- End diff --
`task.duration.isDefined` should be redundant now, right?
(I remember the duration didn't use to be set for running tasks, so this
code worked, but apparently it changed while I worked on these changes...)
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
