Merge pull request #160 from xiajunluan/JIRA-923 Fix bug JIRA-923
Fix column sort issue in UI for JIRA-923. https://spark-project.atlassian.net/browse/SPARK-923 Conflicts: core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/b8bf04a0 Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/b8bf04a0 Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/b8bf04a0 Branch: refs/heads/master Commit: b8bf04a085162478a64ca5d7be15d1af2f6a930e Parents: dfd1ebc e13da05 Author: Matei Zaharia <[email protected]> Authored: Tue Nov 12 16:19:50 2013 -0800 Committer: Matei Zaharia <[email protected]> Committed: Tue Nov 12 16:19:50 2013 -0800 ---------------------------------------------------------------------- .../org/apache/spark/ui/jobs/StagePage.scala | 31 ++++++++++++++++---- .../org/apache/spark/ui/jobs/StageTable.scala | 11 ++++--- 2 files changed, 32 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/b8bf04a0/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala ---------------------------------------------------------------------- diff --cc core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala index 35b5d5f,6c67151..c1c7aa7 --- a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala @@@ -152,8 -152,23 +152,24 @@@ private[spark] class StagePage(parent: else metrics.map(m => parent.formatDuration(m.executorRunTime)).getOrElse("") val gcTime = metrics.map(m => m.jvmGCTime).getOrElse(0L) + var shuffleReadSortable: String = "" + var shuffleReadReadable: String = "" + if (shuffleRead) { + shuffleReadSortable = metrics.flatMap{m => m.shuffleReadMetrics}.map{s => s.remoteBytesRead}.toString() + shuffleReadReadable = metrics.flatMap{m => m.shuffleReadMetrics}.map{s => + Utils.bytesToString(s.remoteBytesRead)}.getOrElse("") + } + + var shuffleWriteSortable: String = "" + var shuffleWriteReadable: String = "" + if (shuffleWrite) { + shuffleWriteSortable = metrics.flatMap{m => m.shuffleWriteMetrics}.map{s => s.shuffleBytesWritten}.toString() + shuffleWriteReadable = metrics.flatMap{m => m.shuffleWriteMetrics}.map{s => + Utils.bytesToString(s.shuffleBytesWritten)}.getOrElse("") + } + <tr> + <td>{info.index}</td> <td>{info.taskId}</td> <td>{info.status}</td> <td>{info.taskLocality}</td> @@@ -166,14 -181,14 +182,17 @@@ {if (gcTime > 0) parent.formatDuration(gcTime) else ""} </td> {if (shuffleRead) { - <td>{metrics.flatMap{m => m.shuffleReadMetrics}.map{s => - Utils.bytesToString(s.remoteBytesRead)}.getOrElse("")}</td> + <td sorttable_customkey={shuffleReadSortable}> + {shuffleReadReadable} + </td> }} {if (shuffleWrite) { - <td>{metrics.flatMap{m => m.shuffleWriteMetrics}.map{s => - parent.formatDuration(s.shuffleWriteTime / (1000 * 1000))}.getOrElse("")}</td> - <td>{metrics.flatMap{m => m.shuffleWriteMetrics}.map{s => - Utils.bytesToString(s.shuffleBytesWritten)}.getOrElse("")}</td> ++ <td>{metrics.flatMap{m => m.shuffleWriteMetrics}.map{s => ++ parent.formatDuration(s.shuffleWriteTime / (1000 * 1000))}.getOrElse("")} ++ </td> + <td sorttable_customkey={shuffleWriteSortable}> + {shuffleWriteReadable} + </td> }} <td>{exception.map(e => <span> http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/b8bf04a0/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala ---------------------------------------------------------------------- diff --cc core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala index d7d0441,c9ee66f..9ad6de3 --- a/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala @@@ -79,11 -79,14 +79,14 @@@ private[spark] class StageTable(val sta case None => "Unknown" } - val shuffleRead = listener.stageIdToShuffleRead.getOrElse(s.stageId, 0L) match { - val shuffleReadSortable = listener.stageToShuffleRead.getOrElse(s.id, 0L) ++ val shuffleReadSortable = listener.stageIdToShuffleRead.getOrElse(s.stageId, 0L) + val shuffleRead = shuffleReadSortable match { case 0 => "" case b => Utils.bytesToString(b) } - val shuffleWrite = listener.stageIdToShuffleWrite.getOrElse(s.stageId, 0L) match { + - val shuffleWriteSortable = listener.stageToShuffleWrite.getOrElse(s.id, 0L) ++ val shuffleWriteSortable = listener.stageIdToShuffleWrite.getOrElse(s.stageId, 0L) + val shuffleWrite = shuffleWriteSortable match { case 0 => "" case b => Utils.bytesToString(b) }
