Github user umehrot2 commented on a diff in the pull request:
https://github.com/apache/spark/pull/19426#discussion_r147537143
--- Diff: core/src/main/scala/org/apache/spark/executor/Executor.scala ---
@@ -406,6 +406,41 @@ private[spark] class Executor(
task.metrics.setJvmGCTime(computeTotalGcTime() - startGCTime)
task.metrics.setResultSerializationTime(afterSerialization -
beforeSerialization)
+ // Expose task metrics using the dropwizard metrics system.
+ // Update task metrics counters
+ executorSource.METRIC_CPU_TIME.inc(task.metrics.executorCpuTime)
+ executorSource.METRIC_RUN_TIME.inc(task.metrics.executorRunTime)
+ executorSource.METRIC_JVM_GC_TIME.inc(task.metrics.jvmGCTime)
+
executorSource.METRIC_DESERIALIZE_TIME.inc(task.metrics.executorDeserializeTime)
+
executorSource.METRIC_DESERIALIZE_CPU_TIME.inc(task.metrics.executorDeserializeCpuTime)
+
executorSource.METRIC_RESULT_SERIALIZE_TIME.inc(task.metrics.resultSerializationTime)
+ executorSource.METRIC_SHUFFLE_FETCH_WAIT_TIME
+ .inc(task.metrics.shuffleReadMetrics.fetchWaitTime)
+
executorSource.METRIC_SHUFFLE_WRITE_TIME.inc(task.metrics.shuffleWriteMetrics.writeTime)
+ executorSource.METRIC_SHUFFLE_TOTAL_BYTES_READ
+ .inc(task.metrics.shuffleReadMetrics.totalBytesRead)
+ executorSource.METRIC_SHUFFLE_REMOTE_BYTES_READ
+ .inc(task.metrics.shuffleReadMetrics.remoteBytesRead)
+ executorSource.METRIC_SHUFFLE_LOCAL_BYTES_READ
+ .inc(task.metrics.shuffleReadMetrics.localBytesRead)
+ executorSource.METRIC_SHUFFLE_RECORDS_READ
+ .inc(task.metrics.shuffleReadMetrics.recordsRead)
+ executorSource.METRIC_SHUFFLE_BYTES_WRITTEN
+ .inc(task.metrics.shuffleWriteMetrics.bytesWritten)
+ executorSource.METRIC_SHUFFLE_RECORDS_WRITTEN
+ .inc(task.metrics.shuffleWriteMetrics.recordsWritten)
+ executorSource.METRIC_INPUT_BYTES_READ
+ .inc(task.metrics.inputMetrics.bytesRead)
+ executorSource.METRIC_INPUT_RECORDS_READ
+ .inc(task.metrics.inputMetrics.recordsRead)
+ executorSource.METRIC_OUTPUT_BYTES_WRITTEN
+ .inc(task.metrics.outputMetrics.bytesWritten)
+ executorSource.METRIC_INPUT_RECORDS_READ
--- End diff --
This has been defined twice in this file. I think what you meant instead is
METRIC_OUTPUT_RECORDS_WRITTEN which is missing here.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
