Github user rxin commented on a diff in the pull request:
https://github.com/apache/spark/pull/12417#discussion_r59923303
--- Diff: core/src/main/scala/org/apache/spark/util/JsonProtocol.scala ---
@@ -326,33 +326,35 @@ private[spark] object JsonProtocol {
}
def taskMetricsToJson(taskMetrics: TaskMetrics): JValue = {
- val shuffleReadMetrics: JValue =
- taskMetrics.shuffleReadMetrics.map { rm =>
- ("Remote Blocks Fetched" -> rm.remoteBlocksFetched) ~
- ("Local Blocks Fetched" -> rm.localBlocksFetched) ~
- ("Fetch Wait Time" -> rm.fetchWaitTime) ~
- ("Remote Bytes Read" -> rm.remoteBytesRead) ~
- ("Local Bytes Read" -> rm.localBytesRead) ~
- ("Total Records Read" -> rm.recordsRead)
- }.getOrElse(JNothing)
- val shuffleWriteMetrics: JValue =
- taskMetrics.shuffleWriteMetrics.map { wm =>
- ("Shuffle Bytes Written" -> wm.bytesWritten) ~
- ("Shuffle Write Time" -> wm.writeTime) ~
- ("Shuffle Records Written" -> wm.recordsWritten)
- }.getOrElse(JNothing)
- val inputMetrics: JValue =
- taskMetrics.inputMetrics.map { im =>
- ("Data Read Method" -> im.readMethod.toString) ~
- ("Bytes Read" -> im.bytesRead) ~
- ("Records Read" -> im.recordsRead)
- }.getOrElse(JNothing)
- val outputMetrics: JValue =
- taskMetrics.outputMetrics.map { om =>
- ("Data Write Method" -> om.writeMethod.toString) ~
- ("Bytes Written" -> om.bytesWritten) ~
- ("Records Written" -> om.recordsWritten)
- }.getOrElse(JNothing)
+ val shuffleReadMetrics: JValue = if
(taskMetrics.shuffleReadMetrics.isUpdated) {
+ ("Remote Blocks Fetched" ->
taskMetrics.shuffleReadMetrics.remoteBlocksFetched) ~
+ ("Local Blocks Fetched" ->
taskMetrics.shuffleReadMetrics.localBlocksFetched) ~
+ ("Fetch Wait Time" ->
taskMetrics.shuffleReadMetrics.fetchWaitTime) ~
+ ("Remote Bytes Read" ->
taskMetrics.shuffleReadMetrics.remoteBytesRead) ~
+ ("Local Bytes Read" ->
taskMetrics.shuffleReadMetrics.localBytesRead) ~
+ ("Total Records Read" ->
taskMetrics.shuffleReadMetrics.recordsRead)
+ } else {
+ JNothing
+ }
+ val shuffleWriteMetrics: JValue = if
(taskMetrics.shuffleWriteMetrics.isUpdated) {
--- End diff --
can't we always output the metrics, and just fix the json protocol test?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
