Author: atm
Date: Wed Jan 30 18:32:00 2013
New Revision: 1440579
URL: http://svn.apache.org/viewvc?rev=1440579&view=rev
Log:
Merge trunk into HDFS-347 branch.
Modified:
hadoop/common/branches/HDFS-347/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java
hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServer.java
hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/CHANGES.txt?rev=1440579&r1=1440578&r2=1440579&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/CHANGES.txt Wed Jan 30
18:32:00 2013
@@ -274,10 +274,16 @@ Release 0.23.7 - UNRELEASED
IMPROVEMENTS
+ YARN-133 Update web services docs for RM clusterMetrics (Ravi Prakash via
+ kihwal)
+
OPTIMIZATIONS
BUG FIXES
+ YARN-343. Capacity Scheduler maximum-capacity value -1 is invalid (Xuan
+ Gong via tgraves)
+
Release 0.23.6 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -335,6 +341,9 @@ Release 0.23.6 - UNRELEASED
YARN-334. Maven RAT plugin is not checking all source files (tgraves)
+ YARN-354. WebAppProxyServer exits immediately after startup (Liang Xie via
+ jlowe)
+
Release 0.23.5 - 2012-11-28
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml?rev=1440579&r1=1440578&r2=1440579&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
(original)
+++
hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
Wed Jan 30 18:32:00 2013
@@ -93,6 +93,7 @@
<executions>
<execution>
<id>version-info</id>
+ <phase>generate-resources</phase>
<goals>
<goal>version-info</goal>
</goals>
Modified:
hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java?rev=1440579&r1=1440578&r2=1440579&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
(original)
+++
hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
Wed Jan 30 18:32:00 2013
@@ -100,6 +100,9 @@ public class CapacitySchedulerConfigurat
public static final float MAXIMUM_CAPACITY_VALUE = 100;
@Private
+ public static final float DEFAULT_MAXIMUM_CAPACITY_VALUE = -1.0f;
+
+ @Private
public static final int DEFAULT_USER_LIMIT = 100;
@Private
@@ -206,6 +209,8 @@ public class CapacitySchedulerConfigurat
public float getMaximumCapacity(String queue) {
float maxCapacity = getFloat(getQueuePrefix(queue) + MAXIMUM_CAPACITY,
MAXIMUM_CAPACITY_VALUE);
+ maxCapacity = (maxCapacity == DEFAULT_MAXIMUM_CAPACITY_VALUE) ?
+ MAXIMUM_CAPACITY_VALUE : maxCapacity;
return maxCapacity;
}
Modified:
hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java?rev=1440579&r1=1440578&r2=1440579&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
(original)
+++
hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
Wed Jan 30 18:32:00 2013
@@ -244,6 +244,18 @@ public class TestCapacityScheduler {
}
@Test
+ public void testMaximumCapacitySetup() {
+ float delta = 0.0000001f;
+ CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+
assertEquals(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE,conf.getMaximumCapacity(A),delta);
+ conf.setMaximumCapacity(A, 50.0f);
+ assertEquals(50.0f, conf.getMaximumCapacity(A),delta);
+ conf.setMaximumCapacity(A, -1);
+
assertEquals(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE,conf.getMaximumCapacity(A),delta);
+ }
+
+
+ @Test
public void testRefreshQueues() throws Exception {
CapacityScheduler cs = new CapacityScheduler();
CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
Modified:
hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java?rev=1440579&r1=1440578&r2=1440579&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java
(original)
+++
hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java
Wed Jan 30 18:32:00 2013
@@ -114,4 +114,13 @@ public class WebAppProxy extends Abstrac
}
super.stop();
}
+
+ public void join() {
+ if(proxyServer != null) {
+ try {
+ proxyServer.join();
+ } catch (InterruptedException e) {
+ }
+ }
+ }
}
Modified:
hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServer.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServer.java?rev=1440579&r1=1440578&r2=1440579&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServer.java
(original)
+++
hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServer.java
Wed Jan 30 18:32:00 2013
@@ -73,6 +73,14 @@ public class WebAppProxyServer extends C
YarnConfiguration.PROXY_PRINCIPAL);
}
+ /**
+ * Wait for service to finish.
+ * (Normally, it runs forever.)
+ */
+ private void join() {
+ proxy.join();
+ }
+
public static void main(String[] args) {
Thread.setDefaultUncaughtExceptionHandler(new
YarnUncaughtExceptionHandler());
StringUtils.startupShutdownMessage(WebAppProxyServer.class, args, LOG);
@@ -84,6 +92,7 @@ public class WebAppProxyServer extends C
YarnConfiguration conf = new YarnConfiguration();
proxy.init(conf);
proxy.start();
+ proxy.join();
} catch (Throwable t) {
LOG.fatal("Error starting Proxy server", t);
System.exit(-1);
Modified:
hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm?rev=1440579&r1=1440578&r2=1440579&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
(original)
+++
hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
Wed Jan 30 18:32:00 2013
@@ -177,6 +177,16 @@ ResourceManager REST API's.
*---------------+--------------+-------------------------------+
| appsSubmitted | int | The number of applications submitted |
*---------------+--------------+-------------------------------+
+| appsCompleted | int | The number of applications completed |
+*---------------+--------------+-------------------------------+
+| appsPending | int | The number of applications pending |
+*---------------+--------------+-------------------------------+
+| appsRunning | int | The number of applications running |
+*---------------+--------------+-------------------------------+
+| appsFailed | int | The number of applications failed |
+*---------------+--------------+-------------------------------+
+| appsKilled | int | The number of applications killed |
+*---------------+--------------+-------------------------------+
| reservedMB | long | The amount of memory reserved in MB |
*---------------+--------------+-------------------------------+
| availableMB | long | The amount of memory available in MB |
@@ -187,6 +197,10 @@ ResourceManager REST API's.
*---------------+--------------+-------------------------------+
| containersAllocated | int | The number of containers allocated |
*---------------+--------------+-------------------------------+
+| containersReserved | int | The number of containers reserved |
+*---------------+--------------+-------------------------------+
+| containersPending | int | The number of containers pending |
+*---------------+--------------+-------------------------------+
| totalNodes | int | The total number of nodes |
*---------------+--------------+-------------------------------+
| activeNodes | int | The number of active nodes |
@@ -223,20 +237,26 @@ ResourceManager REST API's.
+---+
{
- "clusterMetrics":
- {
- "appsSubmitted":4,
+ "clusterMetrics":{
+ "appsSubmitted":0,
+ "appsCompleted":0,
+ "appsPending":0,
+ "appsRunning":0,
+ "appsFailed":0,
+ "appsKilled":0,
"reservedMB":0,
- "availableMB":8192,
+ "availableMB":17408,
"allocatedMB":0,
- "totalMB":8192,
"containersAllocated":0,
+ "containersReserved":0,
+ "containersPending":0,
+ "totalMB":17408,
"totalNodes":1,
- "activeNodes":1,
"lostNodes":0,
"unhealthyNodes":0,
"decommissionedNodes":0,
- "rebootedNodes":0
+ "rebootedNodes":0,
+ "activeNodes":1
}
}
+---+
@@ -264,18 +284,25 @@ ResourceManager REST API's.
+---+
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<clusterMetrics>
- <appsSubmitted>4</appsSubmitted>
+ <appsSubmitted>0</appsSubmitted>
+ <appsCompleted>0</appsCompleted>
+ <appsPending>0</appsPending>
+ <appsRunning>0</appsRunning>
+ <appsFailed>0</appsFailed>
+ <appsKilled>0</appsKilled>
<reservedMB>0</reservedMB>
- <availableMB>8192</availableMB>
+ <availableMB>17408</availableMB>
<allocatedMB>0</allocatedMB>
<containersAllocated>0</containersAllocated>
- <totalMB>8192</totalMB>
+ <containersReserved>0</containersReserved>
+ <containersPending>0</containersPending>
+ <totalMB>17408</totalMB>
<totalNodes>1</totalNodes>
- <activeNodes>1</activeNodes>
<lostNodes>0</lostNodes>
<unhealthyNodes>0</unhealthyNodes>
<decommissionedNodes>0</decommissionedNodes>
<rebootedNodes>0</rebootedNodes>
+ <activeNodes>1</activeNodes>
</clusterMetrics>
+---+
