Github user srowen commented on a diff in the pull request:
https://github.com/apache/spark/pull/9487#discussion_r44016799
--- Diff: core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala ---
@@ -70,6 +71,33 @@ private[ui] class StagePage(parent: StagesTab) extends
WebUIPage("stage") {
private val displayPeakExecutionMemory =
parent.conf.getBoolean("spark.sql.unsafe.enabled", true)
+ private def getLocalitySummaryString(stageData: StageUIData): String = {
+ val localityCounts = new HashMap[String, Long]
+ localityCounts.put("Process local", 0L);
+ localityCounts.put("Node local", 0L);
+ localityCounts.put("Rack local", 0L);
+ localityCounts.put("Any", 0L);
+ stageData.taskData.values.foreach( taskUIData => {
+ taskUIData.taskInfo.taskLocality match {
+ case TaskLocality.PROCESS_LOCAL => {
+ localityCounts.put("Process local",
localityCounts.getOrElse("Process local", 0L) + 1)
+ }
+ case TaskLocality.NODE_LOCAL => {
+ localityCounts.put("Node local", localityCounts.getOrElse("Node
local", 0L) + 1)
+ }
+ case TaskLocality.RACK_LOCAL => {
+ localityCounts.put("Rack local", localityCounts.getOrElse("Rack
local", 0L) + 1)
+ }
+ case TaskLocality.ANY => {
+ localityCounts.put("Any", localityCounts.getOrElse("Any", 0L) +
1)
+ }
+ }
+ })
+ return localityCounts.map { _ match {
+ case (localityLevel, count) => s"$localityLevel: $count task(s)"
+ }}.mkString("; ")
+ }
+
--- End diff --
This might result in inconsistent order of output and always shows 0
counts, which may or may not be desirable. How about this which avoids some of
the repetition:
```
val localities = stageData.taskData.values.map(_.taskInfo.taskLocality)
val localityCounts = localities.groupBy(identity).mapValues(_.size)
val localityNamesAndCounts = localityCounts.toSeq.map { case (locality,
count) =>
val localityName = locality match {
case TaskLocality.PROCESS_LOCAL => "Process local"
case TaskLocality.NODE_LOCAL => "Node local"
case TaskLocality.RACK_LOCAL => "Rack local"
case TaskLocality.ANY => "Any"
}
(localityName, count)
}
localityNamesAndCounts.sorted.mkString("; ")
```
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
