Repository: hadoop
Updated Branches:
refs/heads/branch-2 c6cf74898 -> d070597a0
YARN-2360. Fair Scheduler: Display dynamic fair share for queues on the
scheduler page. (Ashwin Shankar and Wei Yan via kasha)
(cherry picked from commit 270a271f53f52a1f33e63a0520ae6032536dd4c6)
Conflicts:
hadoop-yarn-project/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d070597a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d070597a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d070597a
Branch: refs/heads/branch-2
Commit: d070597a0ee9d9566855e5aa2d07aedcc0bcee89
Parents: c6cf748
Author: Karthik Kambatla <[email protected]>
Authored: Fri Aug 29 17:15:38 2014 -0700
Committer: Karthik Kambatla <[email protected]>
Committed: Tue Jan 6 03:28:48 2015 +0530
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 5 +++
.../webapp/FairSchedulerPage.java | 31 ++++++++++++------
.../webapp/dao/FairSchedulerQueueInfo.java | 34 ++++++++++++++++----
.../src/site/apt/FairScheduler.apt.vm | 20 ++++++++----
4 files changed, 67 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d070597a/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 83acc08..c737a9a 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -129,6 +129,9 @@ Release 2.7.0 - UNRELEASED
YARN-2998. Abstract out scheduler independent PlanFollower components.
(Anubhav Dhoot via kasha)
+ YARN-2360. Fair Scheduler: Display dynamic fair share for queues on the
+ scheduler page. (Ashwin Shankar and Wei Yan via kasha)
+
OPTIMIZATIONS
@@ -522,6 +525,7 @@ Release 2.6.0 - 2014-11-18
YARN-2406. Move RM recovery related proto to
yarn_server_resourcemanager_recovery.proto. (Tsuyoshi Ozawa via jianhe)
+<<<<<<< HEAD
YARN-1506. Changed RMNode/SchedulerNode to update resource with event
notification. (Junping Du via jianhe)
@@ -657,6 +661,7 @@ Release 2.6.0 - 2014-11-18
YARN-2635. TestRM, TestRMRestart, TestClientToAMTokens should run
with both CS and FS. (Wei Yan and kasha via kasha)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d070597a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java
index 0d3c297..8c54f4e 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java
@@ -44,10 +44,12 @@ public class FairSchedulerPage extends RmView {
static final float Q_MAX_WIDTH = 0.8f;
static final float Q_STATS_POS = Q_MAX_WIDTH + 0.05f;
static final String Q_END = "left:101%";
- static final String Q_GIVEN = "left:0%;background:none;border:1px dashed
rgba(0,0,0,0.25)";
+ static final String Q_GIVEN = "left:0%;background:none;border:1px solid
rgba(0,0,0,1)";
+ static final String Q_INSTANTANEOUS_FS = "left:0%;background:none;border:1px
dashed rgba(0,0,0,1)";
static final String Q_OVER = "background:rgba(255, 140, 0, 0.8)";
static final String Q_UNDER = "background:rgba(50, 205, 50, 0.8)";
-
+ static final String STEADY_FAIR_SHARE = "Steady Fair Share";
+ static final String INSTANTANEOUS_FAIR_SHARE = "Instantaneous Fair Share";
@RequestScoped
static class FSQInfo {
FairSchedulerQueueInfo qinfo;
@@ -73,8 +75,8 @@ public class FairSchedulerPage extends RmView {
if (maxApps < Integer.MAX_VALUE) {
ri._("Max Running Applications:", qinfo.getMaxApplications());
}
- ri._("Fair Share:", qinfo.getFairShare().toString());
-
+ ri._(STEADY_FAIR_SHARE + ":", qinfo.getSteadyFairShare().toString());
+ ri._(INSTANTANEOUS_FAIR_SHARE + ":", qinfo.getFairShare().toString());
html._(InfoBlock.class);
// clear the info contents so this queue's info doesn't accumulate into
another queue's info
@@ -95,16 +97,21 @@ public class FairSchedulerPage extends RmView {
UL<Hamlet> ul = html.ul("#pq");
for (FairSchedulerQueueInfo info : subQueues) {
float capacity = info.getMaxResourcesFraction();
- float fairShare = info.getFairShareMemoryFraction();
+ float steadyFairShare = info.getSteadyFairShareMemoryFraction();
+ float instantaneousFairShare = info.getFairShareMemoryFraction();
float used = info.getUsedMemoryFraction();
LI<UL<Hamlet>> li = ul.
li().
a(_Q).$style(width(capacity * Q_MAX_WIDTH)).
- $title(join("Fair Share:", percent(fairShare))).
- span().$style(join(Q_GIVEN, ";font-size:1px;",
width(fairShare/capacity))).
+ $title(join(join(STEADY_FAIR_SHARE + ":",
percent(steadyFairShare)),
+ join(" " + INSTANTANEOUS_FAIR_SHARE + ":",
percent(instantaneousFairShare)))).
+ span().$style(join(Q_GIVEN, ";font-size:1px;",
width(steadyFairShare / capacity))).
+ _('.')._().
+ span().$style(join(Q_INSTANTANEOUS_FS, ";font-size:1px;",
+ width(instantaneousFairShare/capacity))).
_('.')._().
span().$style(join(width(used/capacity),
- ";font-size:1px;left:0%;", used > fairShare ? Q_OVER :
Q_UNDER)).
+ ";font-size:1px;left:0%;", used > instantaneousFairShare ?
Q_OVER : Q_UNDER)).
_('.')._().
span(".q", info.getQueueName())._().
span().$class("qstats").$style(left(Q_STATS_POS)).
@@ -156,7 +163,13 @@ public class FairSchedulerPage extends RmView {
li().$style("margin-bottom: 1em").
span().$style("font-weight: bold")._("Legend:")._().
span().$class("qlegend ui-corner-all").$style(Q_GIVEN).
- _("Fair Share")._().
+ $title("The steady fair shares consider all queues, " +
+ "both active (with running applications) and inactive.").
+ _(STEADY_FAIR_SHARE)._().
+ span().$class("qlegend ui-corner-all").$style(Q_INSTANTANEOUS_FS).
+ $title("The instantaneous fair shares consider only active " +
+ "queues (with running applications).").
+ _(INSTANTANEOUS_FAIR_SHARE)._().
span().$class("qlegend ui-corner-all").$style(Q_UNDER).
_("Used")._().
span().$class("qlegend ui-corner-all").$style(Q_OVER).
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d070597a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
index d76fc53..5fbfe51 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
@@ -28,7 +28,6 @@ import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.XmlSeeAlso;
import javax.xml.bind.annotation.XmlTransient;
-import org.apache.hadoop.yarn.api.records.Resource;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationConfiguration;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue;
@@ -44,6 +43,8 @@ public class FairSchedulerQueueInfo {
@XmlTransient
private float fractionMemUsed;
@XmlTransient
+ private float fractionMemSteadyFairShare;
+ @XmlTransient
private float fractionMemFairShare;
@XmlTransient
private float fractionMemMinShare;
@@ -53,6 +54,7 @@ public class FairSchedulerQueueInfo {
private ResourceInfo minResources;
private ResourceInfo maxResources;
private ResourceInfo usedResources;
+ private ResourceInfo steadyFairResources;
private ResourceInfo fairResources;
private ResourceInfo clusterResources;
@@ -75,15 +77,19 @@ public class FairSchedulerQueueInfo {
usedResources = new ResourceInfo(queue.getResourceUsage());
fractionMemUsed = (float)usedResources.getMemory() /
clusterResources.getMemory();
-
+
+ steadyFairResources = new ResourceInfo(queue.getSteadyFairShare());
fairResources = new ResourceInfo(queue.getFairShare());
minResources = new ResourceInfo(queue.getMinShare());
maxResources = new ResourceInfo(queue.getMaxShare());
maxResources = new ResourceInfo(
Resources.componentwiseMin(queue.getMaxShare(),
scheduler.getClusterResource()));
-
- fractionMemFairShare = (float)fairResources.getMemory() /
clusterResources.getMemory();
+
+ fractionMemSteadyFairShare =
+ (float)steadyFairResources.getMemory() / clusterResources.getMemory();
+ fractionMemFairShare = (float) fairResources.getMemory()
+ / clusterResources.getMemory();
fractionMemMinShare = (float)minResources.getMemory() /
clusterResources.getMemory();
fractionMemMaxShare = (float)maxResources.getMemory() /
clusterResources.getMemory();
@@ -106,19 +112,33 @@ public class FairSchedulerQueueInfo {
}
/**
+ * Returns the steady fair share as a fraction of the entire cluster
capacity.
+ */
+ public float getSteadyFairShareMemoryFraction() {
+ return fractionMemSteadyFairShare;
+ }
+
+ /**
* Returns the fair share as a fraction of the entire cluster capacity.
*/
public float getFairShareMemoryFraction() {
return fractionMemFairShare;
}
-
+
+ /**
+ * Returns the steady fair share of this queue in megabytes.
+ */
+ public ResourceInfo getSteadyFairShare() {
+ return steadyFairResources;
+ }
+
/**
- * Returns the fair share of this queue in megabytes.
+ * Returns the fair share of this queue in megabytes
*/
public ResourceInfo getFairShare() {
return fairResources;
}
-
+
public ResourceInfo getMinResources() {
return minResources;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d070597a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
index 2f488b2..13bf7f9 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
@@ -453,13 +453,19 @@ Monitoring through web UI
* Max Resources - The configured maximum resources that are allowed to the
queue.
- * Fair Share - The queue's fair share of resources. Queues may be allocated
- resources beyond their fair share when other queues aren't using them. A
- queue whose resource consumption lies at or below its fair share will never
- have its containers preempted.
-
- In addition to the information that the ResourceManager normally displays
- about each application, the web interface includes the application's fair
share.
+ * Instantaneous Fair Share - The queue's instantaneous fair share of
resources.
+ These shares consider only actives queues (those with running applications),
+ and are used for scheduling decisions. Queues may be allocated resources
+ beyond their shares when other queues aren't using them. A queue whose
+ resource consumption lies at or below its instantaneous fair share will
never
+ have its containers preempted.
+
+ * Steady Fair Share - The queue's steady fair share of resources. These shares
+ consider all the queues irrespective of whether they are active (have
+ running applications) or not. These are computed less frequently and
+ change only when the configuration or capacity changes.They are meant to
+ provide visibility into resources the user can expect, and hence displayed
+ in the Web UI.
Moving applications between queues
