This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 86b9dd9 [SPARK-32823][WEB UI] Fix the master ui resources reporting
86b9dd9 is described below
commit 86b9dd9b414cab9489f00ef7acec5c3264fa312d
Author: Thomas Graves <[email protected]>
AuthorDate: Wed Sep 9 10:33:21 2020 +0900
[SPARK-32823][WEB UI] Fix the master ui resources reporting
### What changes were proposed in this pull request?
Fixes the master UI for properly summing the resources total across
multiple workers.
field:
Resources in use: 0 / 8 gpu
The bug here is that it was creating MutableResourceInfo and then reducing
using the + operator. the + operator in MutableResourceInfo simple adds the
address from one to the addresses of the other. But its using a HashSet so if
the addresses are the same then you lose the correct amount. ie worker1 has
gpu addresses 0,1,2,3 and worker2 has addresses 0,1,2,3 then you only see 4
total GPUs when there are 8.
In this case we don't really need to create the MutableResourceInfo at all
because we just want the sums for used and total so just remove the use of it.
The other uses of it are per Worker so those should be ok.
### Why are the changes needed?
fix UI
### Does this PR introduce _any_ user-facing change?
UI
### How was this patch tested?
tested manually on standalone cluster with multiple workers and multiple
GPUs and multiple fpgas
Closes #29683 from tgravescs/SPARK-32823.
Lead-authored-by: Thomas Graves <[email protected]>
Co-authored-by: Thomas Graves <[email protected]>
Signed-off-by: HyukjinKwon <[email protected]>
(cherry picked from commit 514bf563a7fa1f470b2ab088c0838317500a9aab)
Signed-off-by: HyukjinKwon <[email protected]>
---
.../org/apache/spark/deploy/StandaloneResourceUtils.scala | 10 +++++-----
.../scala/org/apache/spark/deploy/master/ui/MasterPage.scala | 10 ++++------
2 files changed, 9 insertions(+), 11 deletions(-)
diff --git
a/core/src/main/scala/org/apache/spark/deploy/StandaloneResourceUtils.scala
b/core/src/main/scala/org/apache/spark/deploy/StandaloneResourceUtils.scala
index e08709e..c7c31a8 100644
--- a/core/src/main/scala/org/apache/spark/deploy/StandaloneResourceUtils.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/StandaloneResourceUtils.scala
@@ -149,11 +149,11 @@ private[spark] object StandaloneResourceUtils extends
Logging {
// used for UI
def formatResourcesUsed(
- resourcesTotal: Map[String, ResourceInformation],
- resourcesUsed: Map[String, ResourceInformation]): String = {
- resourcesTotal.map { case (rName, rInfo) =>
- val used = resourcesUsed(rName).addresses.length
- val total = rInfo.addresses.length
+ resourcesTotal: Map[String, Int],
+ resourcesUsed: Map[String, Int]): String = {
+ resourcesTotal.map { case (rName, totalSize) =>
+ val used = resourcesUsed(rName)
+ val total = totalSize
s"$used / $total $rName"
}.mkString(", ")
}
diff --git
a/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala
b/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala
index f64b449..fcbeba9 100644
--- a/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala
@@ -76,19 +76,17 @@ private[ui] class MasterPage(parent: MasterWebUI) extends
WebUIPage("") {
private def formatMasterResourcesInUse(aliveWorkers: Array[WorkerInfo]):
String = {
val totalInfo = aliveWorkers.map(_.resourcesInfo)
- .map(resources => toMutable(resources))
.flatMap(_.toIterator)
.groupBy(_._1) // group by resource name
.map { case (rName, rInfoArr) =>
- rName -> rInfoArr.map(_._2).reduce(_ + _)
- }.map { case (k, v) => (k, v.toResourceInformation) }
+ rName -> rInfoArr.map(_._2.addresses.size).sum
+ }
val usedInfo = aliveWorkers.map(_.resourcesInfoUsed)
- .map (resources => toMutable(resources))
.flatMap(_.toIterator)
.groupBy(_._1) // group by resource name
.map { case (rName, rInfoArr) =>
- rName -> rInfoArr.map(_._2).reduce(_ + _)
- }.map { case (k, v) => (k, v.toResourceInformation) }
+ rName -> rInfoArr.map(_._2.addresses.size).sum
+ }
formatResourcesUsed(totalInfo, usedInfo)
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
