hadoop git commit: YARN-3432. Cluster metrics have wrong Total Memory when there is reserved memory on CS. (Brahma Reddy Battula via curino)

2016-10-30 Thread brahma 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.7 5aa26aa66 -> 2de984ac3


YARN-3432.  Cluster metrics have wrong Total Memory when there is reserved 
memory on CS. (Brahma Reddy Battula via curino)

(cherry picked from commit 892a8348fceb42069ea9877251c413fe33415e16)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2de984ac
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2de984ac
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2de984ac

Branch: refs/heads/branch-2.7
Commit: 2de984ac3dd6dd3473addab7e9388b3d49d60a08
Parents: 5aa26aa
Author: Carlo Curino 
Authored: Thu Oct 27 15:12:10 2016 -0700
Committer: Brahma Reddy Battula 
Committed: Sun Oct 30 23:28:13 2016 +0530

--
 hadoop-yarn-project/CHANGES.txt  |  3 +++
 .../resourcemanager/webapp/dao/ClusterMetricsInfo.java   | 11 +--
 2 files changed, 12 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/2de984ac/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index a2d1bce..4bc8ad0 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -53,6 +53,9 @@ Release 2.7.4 - UNRELEASED
 YARN-5382. RM does not audit log kill request for active applications
 (Vrushali C via jlowe)
 
+YARN-3432.  Cluster metrics have wrong Total Memory when there is reserved
+memory on CS. (Brahma Reddy Battula via curino)
+
 Release 2.7.3 - 2016-08-25
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2de984ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
index 16a5c01..e815fac 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
@@ -25,6 +25,7 @@ import 
org.apache.hadoop.yarn.server.resourcemanager.ClusterMetrics;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
 
 @XmlRootElement(name = "clusterMetrics")
 @XmlAccessorType(XmlAccessType.FIELD)
@@ -85,8 +86,14 @@ public class ClusterMetricsInfo {
 this.containersPending = metrics.getPendingContainers();
 this.containersReserved = metrics.getReservedContainers();
 
-this.totalMB = availableMB + allocatedMB;
-this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores;
+if (rs instanceof CapacityScheduler) {
+  this.totalMB = availableMB + allocatedMB + reservedMB;
+  this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores
+  + containersReserved;
+} else {
+  this.totalMB = availableMB + allocatedMB;
+  this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores;
+}
 this.activeNodes = clusterMetrics.getNumActiveNMs();
 this.lostNodes = clusterMetrics.getNumLostNMs();
 this.unhealthyNodes = clusterMetrics.getUnhealthyNMs();


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[18/50] [abbrv] hadoop git commit: YARN-3432. Cluster metrics have wrong Total Memory when there is reserved memory on CS. (Brahma Reddy Battula via curino)

2016-10-28 Thread virajith 
YARN-3432.  Cluster metrics have wrong Total Memory when there is reserved 
memory on CS. (Brahma Reddy Battula via curino)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/892a8348
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/892a8348
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/892a8348

Branch: refs/heads/HDFS-9806
Commit: 892a8348fceb42069ea9877251c413fe33415e16
Parents: 79ae78d
Author: Carlo Curino 
Authored: Thu Oct 27 15:12:10 2016 -0700
Committer: Carlo Curino 
Committed: Thu Oct 27 15:15:49 2016 -0700

--
 .../resourcemanager/webapp/dao/ClusterMetricsInfo.java   | 11 +--
 1 file changed, 9 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/892a8348/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
index 1789e09..f083b05 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
@@ -25,6 +25,7 @@ import 
org.apache.hadoop.yarn.server.resourcemanager.ClusterMetrics;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
 
 @XmlRootElement(name = "clusterMetrics")
 @XmlAccessorType(XmlAccessType.FIELD)
@@ -87,8 +88,14 @@ public class ClusterMetricsInfo {
 this.containersPending = metrics.getPendingContainers();
 this.containersReserved = metrics.getReservedContainers();
 
-this.totalMB = availableMB + allocatedMB;
-this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores;
+if (rs instanceof CapacityScheduler) {
+  this.totalMB = availableMB + allocatedMB + reservedMB;
+  this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores
+  + containersReserved;
+} else {
+  this.totalMB = availableMB + allocatedMB;
+  this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores;
+}
 this.activeNodes = clusterMetrics.getNumActiveNMs();
 this.lostNodes = clusterMetrics.getNumLostNMs();
 this.unhealthyNodes = clusterMetrics.getUnhealthyNMs();


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-3432. Cluster metrics have wrong Total Memory when there is reserved memory on CS. (Brahma Reddy Battula via curino)

2016-10-27 Thread curino 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 55ba22072 -> b32f14d80


YARN-3432.  Cluster metrics have wrong Total Memory when there is reserved 
memory on CS. (Brahma Reddy Battula via curino)

(cherry picked from commit 892a8348fceb42069ea9877251c413fe33415e16)
(cherry picked from commit 58ac40b55296834a8e3f3375caddc03bee901e9a)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b32f14d8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b32f14d8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b32f14d8

Branch: refs/heads/branch-2.8
Commit: b32f14d8061cf1c201a253b32e689016df709954
Parents: 55ba220
Author: Carlo Curino 
Authored: Thu Oct 27 15:12:10 2016 -0700
Committer: Carlo Curino 
Committed: Thu Oct 27 15:23:04 2016 -0700

--
 .../resourcemanager/webapp/dao/ClusterMetricsInfo.java   | 11 +--
 1 file changed, 9 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b32f14d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
index 3012d0d..d441658 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
@@ -25,6 +25,7 @@ import 
org.apache.hadoop.yarn.server.resourcemanager.ClusterMetrics;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
 
 @XmlRootElement(name = "clusterMetrics")
 @XmlAccessorType(XmlAccessType.FIELD)
@@ -87,8 +88,14 @@ public class ClusterMetricsInfo {
 this.containersPending = metrics.getPendingContainers();
 this.containersReserved = metrics.getReservedContainers();
 
-this.totalMB = availableMB + allocatedMB;
-this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores;
+if (rs instanceof CapacityScheduler) {
+  this.totalMB = availableMB + allocatedMB + reservedMB;
+  this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores
+  + containersReserved;
+} else {
+  this.totalMB = availableMB + allocatedMB;
+  this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores;
+}
 this.activeNodes = clusterMetrics.getNumActiveNMs();
 this.lostNodes = clusterMetrics.getNumLostNMs();
 this.unhealthyNodes = clusterMetrics.getUnhealthyNMs();


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-3432. Cluster metrics have wrong Total Memory when there is reserved memory on CS. (Brahma Reddy Battula via curino)

2016-10-27 Thread curino 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 4274600b9 -> 58ac40b55


YARN-3432.  Cluster metrics have wrong Total Memory when there is reserved 
memory on CS. (Brahma Reddy Battula via curino)

(cherry picked from commit 892a8348fceb42069ea9877251c413fe33415e16)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/58ac40b5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/58ac40b5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/58ac40b5

Branch: refs/heads/branch-2
Commit: 58ac40b55296834a8e3f3375caddc03bee901e9a
Parents: 4274600
Author: Carlo Curino 
Authored: Thu Oct 27 15:12:10 2016 -0700
Committer: Carlo Curino 
Committed: Thu Oct 27 15:22:04 2016 -0700

--
 .../resourcemanager/webapp/dao/ClusterMetricsInfo.java   | 11 +--
 1 file changed, 9 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/58ac40b5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
index 1789e09..f083b05 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
@@ -25,6 +25,7 @@ import 
org.apache.hadoop.yarn.server.resourcemanager.ClusterMetrics;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
 
 @XmlRootElement(name = "clusterMetrics")
 @XmlAccessorType(XmlAccessType.FIELD)
@@ -87,8 +88,14 @@ public class ClusterMetricsInfo {
 this.containersPending = metrics.getPendingContainers();
 this.containersReserved = metrics.getReservedContainers();
 
-this.totalMB = availableMB + allocatedMB;
-this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores;
+if (rs instanceof CapacityScheduler) {
+  this.totalMB = availableMB + allocatedMB + reservedMB;
+  this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores
+  + containersReserved;
+} else {
+  this.totalMB = availableMB + allocatedMB;
+  this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores;
+}
 this.activeNodes = clusterMetrics.getNumActiveNMs();
 this.lostNodes = clusterMetrics.getNumLostNMs();
 this.unhealthyNodes = clusterMetrics.getUnhealthyNMs();


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-3432. Cluster metrics have wrong Total Memory when there is reserved memory on CS. (Brahma Reddy Battula via curino)

2016-10-27 Thread curino 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 79ae78dcb -> 892a8348f


YARN-3432.  Cluster metrics have wrong Total Memory when there is reserved 
memory on CS. (Brahma Reddy Battula via curino)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/892a8348
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/892a8348
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/892a8348

Branch: refs/heads/trunk
Commit: 892a8348fceb42069ea9877251c413fe33415e16
Parents: 79ae78d
Author: Carlo Curino 
Authored: Thu Oct 27 15:12:10 2016 -0700
Committer: Carlo Curino 
Committed: Thu Oct 27 15:15:49 2016 -0700

--
 .../resourcemanager/webapp/dao/ClusterMetricsInfo.java   | 11 +--
 1 file changed, 9 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/892a8348/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
index 1789e09..f083b05 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
@@ -25,6 +25,7 @@ import 
org.apache.hadoop.yarn.server.resourcemanager.ClusterMetrics;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
 
 @XmlRootElement(name = "clusterMetrics")
 @XmlAccessorType(XmlAccessType.FIELD)
@@ -87,8 +88,14 @@ public class ClusterMetricsInfo {
 this.containersPending = metrics.getPendingContainers();
 this.containersReserved = metrics.getReservedContainers();
 
-this.totalMB = availableMB + allocatedMB;
-this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores;
+if (rs instanceof CapacityScheduler) {
+  this.totalMB = availableMB + allocatedMB + reservedMB;
+  this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores
+  + containersReserved;
+} else {
+  this.totalMB = availableMB + allocatedMB;
+  this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores;
+}
 this.activeNodes = clusterMetrics.getNumActiveNMs();
 this.lostNodes = clusterMetrics.getNumLostNMs();
 this.unhealthyNodes = clusterMetrics.getUnhealthyNMs();


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org