Repository: hadoop
Updated Branches:
refs/heads/branch-2 7d66e2e66 -> baa2998e9
YARN-4285. Display resource usage as percentage of queue and cluster in the RM
UI (Varun Vasudev via wangda)
(cherry picked from commit 3cc73773eb26f7469c99b25a76814d6fad0be28e)
Conflicts:
hadoop-yarn-project/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/baa2998e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/baa2998e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/baa2998e
Branch: refs/heads/branch-2
Commit: baa2998e93b3c687310a530c578d78d441704031
Parents: 7d66e2e
Author: Wangda Tan <[email protected]>
Authored: Mon Oct 26 13:06:08 2015 -0700
Committer: Wangda Tan <[email protected]>
Committed: Mon Oct 26 13:12:56 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../records/ApplicationResourceUsageReport.java | 38 +++++++++++-
.../src/main/proto/yarn_protos.proto | 2 +
.../hadoop/yarn/client/cli/TestYarnCLI.java | 2 +-
.../ApplicationResourceUsageReportPBImpl.java | 24 ++++++++
...pplicationHistoryManagerOnTimelineStore.java | 5 +-
.../hadoop/yarn/server/webapp/WebPageUtils.java | 2 +-
.../scheduler/SchedulerApplicationAttempt.java | 10 ++-
.../scheduler/capacity/AbstractCSQueue.java | 4 +-
.../scheduler/capacity/LeafQueue.java | 2 +-
.../resourcemanager/webapp/RMAppsBlock.java | 40 ++++++++----
.../resourcemanager/webapp/dao/AppInfo.java | 6 +-
.../applicationsmanager/MockAsm.java | 2 +-
.../TestSchedulerApplicationAttempt.java | 64 +++++++++++++++++---
.../webapp/TestRMWebServicesApps.java | 9 ++-
15 files changed, 182 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/baa2998e/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 932da69..0d95dbe 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -491,6 +491,9 @@ Release 2.8.0 - UNRELEASED
YARN-3738. Add support for recovery of reserved apps running under dynamic
queues (subru via asuresh)
+ YARN-4285. Display resource usage as percentage of queue and cluster in the
+ RM UI (Varun Vasudev via wangda)
+
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not
http://git-wip-us.apache.org/repos/asf/hadoop/blob/baa2998e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
index b20d832..34efee8 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
@@ -36,7 +36,7 @@ public abstract class ApplicationResourceUsageReport {
public static ApplicationResourceUsageReport newInstance(
int numUsedContainers, int numReservedContainers, Resource usedResources,
Resource reservedResources, Resource neededResources, long memorySeconds,
- long vcoreSeconds) {
+ long vcoreSeconds, float queueUsagePerc, float clusterUsagePerc) {
ApplicationResourceUsageReport report =
Records.newRecord(ApplicationResourceUsageReport.class);
report.setNumUsedContainers(numUsedContainers);
@@ -46,6 +46,8 @@ public abstract class ApplicationResourceUsageReport {
report.setNeededResources(neededResources);
report.setMemorySeconds(memorySeconds);
report.setVcoreSeconds(vcoreSeconds);
+ report.setQueueUsagePercentage(queueUsagePerc);
+ report.setClusterUsagePercentage(clusterUsagePerc);
return report;
}
@@ -152,4 +154,38 @@ public abstract class ApplicationResourceUsageReport {
@Public
@Unstable
public abstract long getVcoreSeconds();
+
+ /**
+ * Get the percentage of resources of the queue that the app is using.
+ * @return the percentage of resources of the queue that the app is using.
+ */
+ @Public
+ @Stable
+ public abstract float getQueueUsagePercentage();
+
+ /**
+ * Set the percentage of resources of the queue that the app is using.
+ * @param queueUsagePerc the percentage of resources of the queue that
+ * the app is using.
+ */
+ @Private
+ @Unstable
+ public abstract void setQueueUsagePercentage(float queueUsagePerc);
+
+ /**
+ * Get the percentage of resources of the cluster that the app is using.
+ * @return the percentage of resources of the cluster that the app is using.
+ */
+ @Public
+ @Stable
+ public abstract float getClusterUsagePercentage();
+
+ /**
+ * Set the percentage of resources of the cluster that the app is using.
+ * @param clusterUsagePerc the percentage of resources of the cluster that
+ * the app is using.
+ */
+ @Private
+ @Unstable
+ public abstract void setClusterUsagePercentage(float clusterUsagePerc);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/baa2998e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index 687ee89..09d2bd5 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -171,6 +171,8 @@ message ApplicationResourceUsageReportProto {
optional ResourceProto needed_resources = 5;
optional int64 memory_seconds = 6;
optional int64 vcore_seconds = 7;
+ optional float queue_usage_percentage = 8;
+ optional float cluster_usage_percentage = 9;
}
message ApplicationReportProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/baa2998e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
index 3dab504..9ee7001 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
@@ -105,7 +105,7 @@ public class TestYarnCLI {
ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
ApplicationResourceUsageReport usageReport = i == 0 ? null :
ApplicationResourceUsageReport.newInstance(
- 2, 0, null, null, null, 123456, 4567);
+ 2, 0, null, null, null, 123456, 4567, 0, 0);
ApplicationReport newApplicationReport = ApplicationReport.newInstance(
applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
"user", "queue", "appname", "host", 124, null,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/baa2998e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java
index 1cabadd..5b1b85e 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java
@@ -231,4 +231,28 @@ extends ApplicationResourceUsageReport {
private ResourceProto convertToProtoFormat(Resource t) {
return ((ResourcePBImpl)t).getProto();
}
+
+ @Override
+ public synchronized float getQueueUsagePercentage() {
+ ApplicationResourceUsageReportProtoOrBuilder p = viaProto ? proto :
builder;
+ return (p.getQueueUsagePercentage());
+ }
+
+ @Override
+ public synchronized void setQueueUsagePercentage(float queueUsagePerc) {
+ maybeInitBuilder();
+ builder.setQueueUsagePercentage((queueUsagePerc));
+ }
+
+ @Override
+ public synchronized float getClusterUsagePercentage() {
+ ApplicationResourceUsageReportProtoOrBuilder p = viaProto ? proto :
builder;
+ return (p.getClusterUsagePercentage());
+ }
+
+ @Override
+ public synchronized void setClusterUsagePercentage(float clusterUsagePerc) {
+ maybeInitBuilder();
+ builder.setClusterUsagePercentage((clusterUsagePerc));
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/baa2998e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
index 96ad5ed..9b4e78e 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
@@ -334,8 +334,9 @@ public class ApplicationHistoryManagerOnTimelineStore
extends AbstractService
ApplicationMetricsConstants.APP_CPU_METRICS).toString());
long memorySeconds=Long.parseLong(entityInfo.get(
ApplicationMetricsConstants.APP_MEM_METRICS).toString());
- appResources=ApplicationResourceUsageReport
- .newInstance(0, 0, null, null, null, memorySeconds, vcoreSeconds);
+ appResources = ApplicationResourceUsageReport
+ .newInstance(0, 0, null, null, null, memorySeconds, vcoreSeconds,
0,
+ 0);
}
if (entityInfo.containsKey(ApplicationMetricsConstants.APP_TAGS_INFO)) {
appTags = new HashSet<String>();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/baa2998e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java
index f32dd0c..d51c632 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java
@@ -57,7 +57,7 @@ public class WebPageUtils {
if (isFairSchedulerPage) {
sb.append("[13]");
} else if (isResourceManager) {
- sb.append("[13]");
+ sb.append("[15]");
} else {
sb.append("[9]");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/baa2998e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
index 005fa71..23f00e0 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
@@ -676,11 +676,19 @@ public class SchedulerApplicationAttempt implements
SchedulableEntity {
Resources.clone(attemptResourceUsage.getAllUsed());
Resource reservedResourceClone =
Resources.clone(attemptResourceUsage.getReserved());
+ Resource cluster = rmContext.getScheduler().getClusterResource();
+ ResourceCalculator calc = rmContext.getScheduler().getResourceCalculator();
+ float queueUsagePerc = calc.divide(cluster, usedResourceClone, Resources
+ .multiply(cluster, queue.getQueueInfo(false, false).getCapacity()))
+ * 100;
+ float clusterUsagePerc =
+ calc.divide(cluster, usedResourceClone, cluster) * 100;
return ApplicationResourceUsageReport.newInstance(liveContainers.size(),
reservedContainers.size(), usedResourceClone, reservedResourceClone,
Resources.add(usedResourceClone, reservedResourceClone),
runningResourceUsage.getMemorySeconds(),
- runningResourceUsage.getVcoreSeconds());
+ runningResourceUsage.getVcoreSeconds(),
+ queueUsagePerc, clusterUsagePerc);
}
public synchronized Map<ContainerId, RMContainer> getLiveContainersMap() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/baa2998e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index 9f61b11..b40ac27 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -145,7 +145,7 @@ public abstract class AbstractCSQueue implements CSQueue {
}
@Override
- public synchronized float getUsedCapacity() {
+ public float getUsedCapacity() {
return queueCapacities.getUsedCapacity();
}
@@ -198,7 +198,7 @@ public abstract class AbstractCSQueue implements CSQueue {
}
@Override
- public synchronized void setUsedCapacity(float usedCapacity) {
+ public void setUsedCapacity(float usedCapacity) {
queueCapacities.setUsedCapacity(usedCapacity);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/baa2998e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index 1c133d3..a993ece 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -365,7 +365,7 @@ public class LeafQueue extends AbstractCSQueue {
}
@Override
- public synchronized QueueInfo getQueueInfo(
+ public QueueInfo getQueueInfo(
boolean includeChildQueues, boolean recursive) {
QueueInfo queueInfo = getQueueInfo();
return queueInfo;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/baa2998e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
index a1d8633..9f2f308 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
@@ -27,6 +27,7 @@ import java.util.Set;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ApplicationBaseProtocol;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
@@ -63,8 +64,11 @@ public class RMAppsBlock extends AppsBlock {
.th(".runningcontainer", "Running Containers")
.th(".allocatedCpu", "Allocated CPU VCores")
.th(".allocatedMemory", "Allocated Memory MB")
+ .th(".queuePercentage", "% of Queue")
+ .th(".clusterPercentage", "% of Cluster")
.th(".progress", "Progress")
- .th(".ui", "Tracking UI").th(".blacklisted", "Blacklisted Nodes")._()
+ .th(".ui", "Tracking UI")
+ .th(".blacklisted", "Blacklisted Nodes")._()
._().tbody();
StringBuilder appsTableData = new StringBuilder("[\n");
@@ -78,11 +82,21 @@ public class RMAppsBlock extends AppsBlock {
}
AppInfo app = new AppInfo(appReport);
+ ApplicationAttemptId appAttemptId =
+ ConverterUtils.toApplicationAttemptId(app.getCurrentAppAttemptId());
+ String queuePercent = "N/A";
+ String clusterPercent = "N/A";
+ if(appReport.getApplicationResourceUsageReport() != null) {
+ queuePercent = String.format("%.1f",
+ appReport.getApplicationResourceUsageReport()
+ .getQueueUsagePercentage());
+ clusterPercent = String.format("%.1f",
+
appReport.getApplicationResourceUsageReport().getClusterUsagePercentage());
+ }
+
String blacklistedNodesCount = "N/A";
Set<String> nodes =
- RMAppAttemptBlock
- .getBlacklistedNodes(rm, ConverterUtils.toApplicationAttemptId(app
- .getCurrentAppAttemptId()));
+ RMAppAttemptBlock.getBlacklistedNodes(rm, appAttemptId);
if (nodes != null) {
blacklistedNodesCount = String.valueOf(nodes.size());
}
@@ -94,12 +108,12 @@ public class RMAppsBlock extends AppsBlock {
.append(app.getAppId())
.append("</a>\",\"")
.append(
- StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml(app
- .getUser())))
+ StringEscapeUtils.escapeJavaScript(
+ StringEscapeUtils.escapeHtml(app.getUser())))
.append("\",\"")
.append(
- StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml(app
- .getName())))
+ StringEscapeUtils.escapeJavaScript(
+ StringEscapeUtils.escapeHtml(app.getName())))
.append("\",\"")
.append(
StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml(app
@@ -122,11 +136,15 @@ public class RMAppsBlock extends AppsBlock {
.append(app.getAllocatedCpuVcores() == -1 ? "N/A" : String
.valueOf(app.getAllocatedCpuVcores()))
.append("\",\"")
- .append(app.getAllocatedMemoryMB() == -1 ? "N/A" : String
- .valueOf(app.getAllocatedMemoryMB()))
+ .append(app.getAllocatedMemoryMB() == -1 ? "N/A" :
+ String.valueOf(app.getAllocatedMemoryMB()))
+ .append("\",\"")
+ .append(queuePercent)
+ .append("\",\"")
+ .append(clusterPercent)
.append("\",\"")
// Progress bar
- .append("<br title='").append(percent).append("'> <div class='")
+ .append("<br title='").append(percent).append("'> <div class='")
.append(C_PROGRESSBAR).append("' title='").append(join(percent, '%'))
.append("'> ").append("<div class='").append(C_PROGRESSBAR_VALUE)
.append("' style='").append(join("width:", percent, '%'))
http://git-wip-us.apache.org/repos/asf/hadoop/blob/baa2998e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
index a035aeb..f51197b 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
@@ -87,7 +87,9 @@ public class AppInfo {
protected int runningContainers;
protected long memorySeconds;
protected long vcoreSeconds;
-
+ protected float queueUsagePercentage;
+ protected float clusterUsagePercentage;
+
// preemption info fields
protected int preemptedResourceMB;
protected int preemptedResourceVCores;
@@ -175,6 +177,8 @@ public class AppInfo {
allocatedMB = usedResources.getMemory();
allocatedVCores = usedResources.getVirtualCores();
runningContainers = resourceReport.getNumUsedContainers();
+ queueUsagePercentage = resourceReport.getQueueUsagePercentage();
+ clusterUsagePercentage =
resourceReport.getClusterUsagePercentage();
}
resourceRequests = rm.getRMContext().getScheduler()
.getPendingResourceRequestsForAttempt(attempt.getAppAttemptId());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/baa2998e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
index 87f96eb..af46dcd 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
@@ -305,7 +305,7 @@ public abstract class MockAsm extends MockApps {
String clientUserName, boolean allowAccess) {
ApplicationResourceUsageReport usageReport =
ApplicationResourceUsageReport.newInstance(0, 0, null, null, null,
- 0, 0);
+ 0, 0, 0, 0);
ApplicationReport report = ApplicationReport.newInstance(
getApplicationId(), appAttemptId, getUser(), getQueue(),
getName(), null, 0, null, null, getDiagnostics().toString(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/baa2998e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java
index c648b83..95d7129 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+import org.apache.hadoop.yarn.api.records.*;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.*;
@@ -27,15 +28,9 @@ import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
+import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.junit.After;
import org.junit.Test;
@@ -149,11 +144,17 @@ public class TestSchedulerApplicationAttempt {
}
private Queue createQueue(String name, Queue parent) {
+ return createQueue(name, parent, 1.0f);
+ }
+
+ private Queue createQueue(String name, Queue parent, float capacity) {
QueueMetrics metrics = QueueMetrics.forQueue(name, parent, false, conf);
+ QueueInfo queueInfo = QueueInfo.newInstance(name, capacity, 1.0f, 0, null,
null, QueueState.RUNNING, null, "", null);
ActiveUsersManager activeUsersManager = new ActiveUsersManager(metrics);
Queue queue = mock(Queue.class);
when(queue.getMetrics()).thenReturn(metrics);
when(queue.getActiveUsersManager()).thenReturn(activeUsersManager);
+ when(queue.getQueueInfo(false, false)).thenReturn(queueInfo);
return queue;
}
@@ -163,4 +164,51 @@ public class TestSchedulerApplicationAttempt {
ApplicationAttemptId.newInstance(appIdImpl, attemptId);
return attId;
}
+
+ @Test
+ public void testAppPercentages() throws Exception {
+ FifoScheduler scheduler = mock(FifoScheduler.class);
+ when(scheduler.getClusterResource())
+ .thenReturn(Resource.newInstance(10 * 1024, 10));
+ when(scheduler.getResourceCalculator())
+ .thenReturn(new DefaultResourceCalculator());
+
+ ApplicationAttemptId appAttId = createAppAttemptId(0, 0);
+ RMContext rmContext = mock(RMContext.class);
+ when(rmContext.getEpoch()).thenReturn(3L);
+ when(rmContext.getScheduler()).thenReturn(scheduler);
+
+ final String user = "user1";
+ Queue queue = createQueue("test", null);
+ SchedulerApplicationAttempt app =
+ new SchedulerApplicationAttempt(appAttId, user, queue,
+ queue.getActiveUsersManager(), rmContext);
+
+ // Resource request
+ Resource requestedResource = Resource.newInstance(1536, 2);
+ app.attemptResourceUsage.incUsed(requestedResource);
+
+ assertEquals(15.0f, app.getResourceUsageReport().getQueueUsagePercentage(),
+ 0.01f);
+ assertEquals(15.0f,
+ app.getResourceUsageReport().getClusterUsagePercentage(), 0.01f);
+
+ queue = createQueue("test2", null, 0.5f);
+ app = new SchedulerApplicationAttempt(appAttId, user, queue,
+ queue.getActiveUsersManager(), rmContext);
+ app.attemptResourceUsage.incUsed(requestedResource);
+ assertEquals(30.0f, app.getResourceUsageReport().getQueueUsagePercentage(),
+ 0.01f);
+ assertEquals(15.0f,
+ app.getResourceUsageReport().getClusterUsagePercentage(), 0.01f);
+
+ app.attemptResourceUsage.incUsed(requestedResource);
+ app.attemptResourceUsage.incUsed(requestedResource);
+ app.attemptResourceUsage.incUsed(requestedResource);
+
+ assertEquals(120.0f,
app.getResourceUsageReport().getQueueUsagePercentage(),
+ 0.01f);
+ assertEquals(60.0f,
+ app.getResourceUsageReport().getClusterUsagePercentage(), 0.01f);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/baa2998e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
index 47b44d2..d7e80e8 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
@@ -1305,6 +1305,8 @@ public class TestRMWebServicesApps extends JerseyTestBase
{
WebServicesTestUtils.getXmlInt(element, "allocatedMB"),
WebServicesTestUtils.getXmlInt(element, "allocatedVCores"),
WebServicesTestUtils.getXmlInt(element, "runningContainers"),
+ WebServicesTestUtils.getXmlFloat(element, "queueUsagePercentage"),
+ WebServicesTestUtils.getXmlFloat(element, "clusterUsagePercentage"),
WebServicesTestUtils.getXmlInt(element, "preemptedResourceMB"),
WebServicesTestUtils.getXmlInt(element, "preemptedResourceVCores"),
WebServicesTestUtils.getXmlInt(element,
"numNonAMContainerPreempted"),
@@ -1319,7 +1321,7 @@ public class TestRMWebServicesApps extends JerseyTestBase
{
public void verifyAppInfo(JSONObject info, RMApp app) throws JSONException,
Exception {
- int expectedNumberOfElements = 30;
+ int expectedNumberOfElements = 32;
String appNodeLabelExpression = null;
String amNodeLabelExpression = null;
if (app.getApplicationSubmissionContext()
@@ -1344,6 +1346,8 @@ public class TestRMWebServicesApps extends JerseyTestBase
{
info.getLong("elapsedTime"), info.getString("amHostHttpAddress"),
info.getString("amContainerLogs"), info.getInt("allocatedMB"),
info.getInt("allocatedVCores"), info.getInt("runningContainers"),
+ (float) info.getDouble("queueUsagePercentage"),
+ (float) info.getDouble("clusterUsagePercentage"),
info.getInt("preemptedResourceMB"),
info.getInt("preemptedResourceVCores"),
info.getInt("numNonAMContainerPreempted"),
@@ -1360,6 +1364,7 @@ public class TestRMWebServicesApps extends JerseyTestBase
{
String diagnostics, long clusterId, long startedTime, long finishedTime,
long elapsedTime, String amHostHttpAddress, String amContainerLogs,
int allocatedMB, int allocatedVCores, int numContainers,
+ float queueUsagePerc, float clusterUsagePerc,
int preemptedResourceMB, int preemptedResourceVCores,
int numNonAMContainerPreempted, int numAMContainerPreempted,
String logAggregationStatus, boolean unmanagedApplication,
@@ -1399,6 +1404,8 @@ public class TestRMWebServicesApps extends JerseyTestBase
{
amContainerLogs.endsWith("/" + app.getUser()));
assertEquals("allocatedMB doesn't match", 1024, allocatedMB);
assertEquals("allocatedVCores doesn't match", 1, allocatedVCores);
+ assertEquals("queueUsagePerc doesn't match", 50.0f, queueUsagePerc, 0.01f);
+ assertEquals("clusterUsagePerc doesn't match", 50.0f, clusterUsagePerc,
0.01f);
assertEquals("numContainers doesn't match", 1, numContainers);
assertEquals("preemptedResourceMB doesn't match", app
.getRMAppMetrics().getResourcePreempted().getMemory(),
