Author: szetszwo
Date: Mon Mar 18 11:45:07 2013
New Revision: 1457716
URL: http://svn.apache.org/r1457716
Log:
Merge r1455389 through r1457712 from trunk.
Modified:
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/JQueryUI.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NavBlock.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt?rev=1457716&r1=1457715&r2=1457716&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt Mon Mar 18
11:45:07 2013
@@ -66,6 +66,12 @@ Release 2.0.5-beta - UNRELEASED
the per-application page are translated to html line breaks. (Omkar Vinit
Joshi via vinodkv)
+ YARN-198. Added a link to RM pages from the NodeManager web app. (Jian He
+ via vinodkv)
+
+ YARN-237. Refreshing the RM page forgets how many rows I had in my
+ Datatables (jian he via bobby)
+
OPTIMIZATIONS
BUG FIXES
@@ -91,6 +97,9 @@ Release 2.0.5-beta - UNRELEASED
YARN-376. Fixes a bug which would prevent the NM knowing about completed
containers and applications. (Jason Lowe via sseth)
+ YARN-196. Nodemanager should be more robust in handling connection failure
+ to ResourceManager when a cluster is started (Xuan Gong via hitesh)
+
Release 2.0.4-alpha - UNRELEASED
INCOMPATIBLE CHANGES
@@ -396,6 +405,8 @@ Release 0.23.7 - UNRELEASED
YARN-443. allow OS scheduling priority of NM to be different than the
containers it launches (tgraves)
+ YARN-468. coverage fix for org.apache.hadoop.yarn.server.webproxy.amfilter
+ (Aleksey Gorshkov via bobby)
OPTIMIZATIONS
Modified:
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1457716&r1=1457715&r2=1457716&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
Mon Mar 18 11:45:07 2013
@@ -621,6 +621,20 @@ public class YarnConfiguration extends C
public static final long DEFAULT_NM_PROCESS_KILL_WAIT_MS =
2000;
+ /** Max time to wait to establish a connection to RM when NM starts
+ */
+ public static final String RESOURCEMANAGER_CONNECT_WAIT_SECS =
+ NM_PREFIX + "resourcemanager.connect.wait.secs";
+ public static final int DEFAULT_RESOURCEMANAGER_CONNECT_WAIT_SECS =
+ 15*60;
+
+ /** Time interval between each NM attempt to connect to RM
+ */
+ public static final String RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_SECS =
+ NM_PREFIX + "resourcemanager.connect.retry_interval.secs";
+ public static final long DEFAULT_RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_SECS
+ = 30;
+
/**
* CLASSPATH for YARN applications. A comma-separated list of CLASSPATH
* entries
Modified:
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/JQueryUI.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/JQueryUI.java?rev=1457716&r1=1457715&r2=1457716&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/JQueryUI.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/JQueryUI.java
Mon Mar 18 11:45:07 2013
@@ -107,12 +107,21 @@ public class JQueryUI extends HtmlBlock
protected void initDataTables(List<String> list) {
String defaultInit = "{bJQueryUI: true, sPaginationType: 'full_numbers'}";
+ String stateSaveInit = "bStateSave : true, " +
+ "\"fnStateSave\": function (oSettings, oData) { " +
+ "sessionStorage.setItem( oSettings.sTableId,
JSON.stringify(oData) ); }, " +
+ "\"fnStateLoad\": function (oSettings) { " +
+ "return JSON.parse( sessionStorage.getItem(oSettings.sTableId)
);}, ";
+
for (String id : split($(DATATABLES_ID))) {
if (Html.isValidId(id)) {
String init = $(initID(DATATABLES, id));
if (init.isEmpty()) {
init = defaultInit;
}
+ // for inserting stateSaveInit
+ int pos = init.indexOf('{') + 1;
+ init = new StringBuffer(init).insert(pos, stateSaveInit).toString();
list.add(join(id,"DataTable = $('#", id, "').dataTable(", init,
").fnSetFilteringDelay(188);"));
String postInit = $(postInitID(DATATABLES, id));
@@ -126,9 +135,12 @@ public class JQueryUI extends HtmlBlock
String init = $(initSelector(DATATABLES));
if (init.isEmpty()) {
init = defaultInit;
- }
+ }
+ int pos = init.indexOf('{') + 1;
+ init = new StringBuffer(init).insert(pos, stateSaveInit).toString();
list.add(join(" $('", escapeJavaScript(selector), "').dataTable(", init,
- ").fnSetFilteringDelay(288);"));
+ ").fnSetFilteringDelay(288);"));
+
}
}
Modified:
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml?rev=1457716&r1=1457715&r2=1457716&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
Mon Mar 18 11:45:07 2013
@@ -597,6 +597,20 @@
<value>2000</value>
</property>
+ <property>
+ <description>Max time, in seconds, to wait to establish a connection to RM
when NM starts.
+ The NM will shutdown if it cannot connect to RM within the specified max
time period.
+ If the value is set as -1, then NM will retry forever.</description>
+ <name>yarn.nodemanager.resourcemanager.connect.wait.secs</name>
+ <value>900</value>
+ </property>
+
+ <property>
+ <description>Time interval, in seconds, between each NM attempt to connect
to RM.</description>
+ <name>yarn.nodemanager.resourcemanager.connect.retry_interval.secs</name>
+ <value>30</value>
+ </property>
+
<!--Map Reduce configuration-->
<property>
<name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
Modified:
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java?rev=1457716&r1=1457715&r2=1457716&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
Mon Mar 18 11:45:07 2013
@@ -350,7 +350,7 @@ public class NodeManager extends Composi
ContainerManagerImpl getContainerManager() {
return containerManager;
}
-
+
public static void main(String[] args) {
Thread.setDefaultUncaughtExceptionHandler(new
YarnUncaughtExceptionHandler());
StringUtils.startupShutdownMessage(NodeManager.class, args, LOG);
Modified:
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java?rev=1457716&r1=1457715&r2=1457716&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
Mon Mar 18 11:45:07 2013
@@ -151,7 +151,6 @@ public class NodeStatusUpdaterImpl exten
YarnConfiguration.DEFAULT_NM_WEBAPP_ADDRESS,
YarnConfiguration.DEFAULT_NM_WEBAPP_PORT);
try {
- // this.hostName =
InetAddress.getLocalHost().getCanonicalHostName();
this.httpPort = httpBindAddress.getPort();
// Registration has to be in start so that ContainerManager can get the
// perNM tokens needed to authenticate ContainerTokens.
@@ -189,15 +188,84 @@ public class NodeStatusUpdaterImpl exten
}
private void registerWithRM() throws YarnRemoteException {
- this.resourceTracker = getRMClient();
- LOG.info("Connecting to ResourceManager at " + this.rmAddress);
-
- RegisterNodeManagerRequest request =
recordFactory.newRecordInstance(RegisterNodeManagerRequest.class);
+ Configuration conf = getConfig();
+ long rmConnectWaitMS =
+ conf.getInt(
+ YarnConfiguration.RESOURCEMANAGER_CONNECT_WAIT_SECS,
+ YarnConfiguration.DEFAULT_RESOURCEMANAGER_CONNECT_WAIT_SECS)
+ * 1000;
+ long rmConnectionRetryIntervalMS =
+ conf.getLong(
+ YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_SECS,
+ YarnConfiguration
+ .DEFAULT_RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_SECS)
+ * 1000;
+
+ if(rmConnectionRetryIntervalMS < 0) {
+ throw new YarnException("Invalid Configuration. " +
+ YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_SECS +
+ " should not be negative.");
+ }
+
+ boolean waitForEver = (rmConnectWaitMS == -1000);
+
+ if(! waitForEver) {
+ if(rmConnectWaitMS < 0) {
+ throw new YarnException("Invalid Configuration. " +
+ YarnConfiguration.RESOURCEMANAGER_CONNECT_WAIT_SECS +
+ " can be -1, but can not be other negative numbers");
+ }
+
+ //try connect once
+ if(rmConnectWaitMS < rmConnectionRetryIntervalMS) {
+ LOG.warn(YarnConfiguration.RESOURCEMANAGER_CONNECT_WAIT_SECS
+ + " is smaller than "
+ + YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_SECS
+ + ". Only try connect once.");
+ rmConnectWaitMS = 0;
+ }
+ }
+
+ int rmRetryCount = 0;
+ long waitStartTime = System.currentTimeMillis();
+
+ RegisterNodeManagerRequest request =
+ recordFactory.newRecordInstance(RegisterNodeManagerRequest.class);
request.setHttpPort(this.httpPort);
request.setResource(this.totalResource);
request.setNodeId(this.nodeId);
- RegistrationResponse regResponse =
-
this.resourceTracker.registerNodeManager(request).getRegistrationResponse();
+ RegistrationResponse regResponse;
+
+ while(true) {
+ try {
+ rmRetryCount++;
+ LOG.info("Connecting to ResourceManager at " + this.rmAddress
+ + ". current no. of attempts is " + rmRetryCount);
+ this.resourceTracker = getRMClient();
+ regResponse =
+ this.resourceTracker.registerNodeManager(request)
+ .getRegistrationResponse();
+ break;
+ } catch(Throwable e) {
+ LOG.warn("Trying to connect to ResourceManager, " +
+ "current no. of failed attempts is "+rmRetryCount);
+ if(System.currentTimeMillis() - waitStartTime < rmConnectWaitMS
+ || waitForEver) {
+ try {
+ LOG.info("Sleeping for " + rmConnectionRetryIntervalMS/1000
+ + " seconds before next connection retry to RM");
+ Thread.sleep(rmConnectionRetryIntervalMS);
+ } catch(InterruptedException ex) {
+ //done nothing
+ }
+ } else {
+ String errorMessage = "Failed to Connect to RM, " +
+ "no. of failed attempts is "+rmRetryCount;
+ LOG.error(errorMessage,e);
+ throw new YarnException(errorMessage,e);
+ }
+ }
+ }
// if the Resourcemanager instructs NM to shutdown.
if (NodeAction.SHUTDOWN.equals(regResponse.getNodeAction())) {
throw new YarnException(
Modified:
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NavBlock.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NavBlock.java?rev=1457716&r1=1457715&r2=1457716&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NavBlock.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NavBlock.java
Mon Mar 18 11:45:07 2013
@@ -18,16 +18,32 @@
package org.apache.hadoop.yarn.server.nodemanager.webapp;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.webapp.YarnWebParams;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
+import com.google.inject.Inject;
+
public class NavBlock extends HtmlBlock implements YarnWebParams {
+ private Configuration conf;
+
+ @Inject
+ public NavBlock(Configuration conf) {
+ this.conf = conf;
+ }
+
@Override
protected void render(Block html) {
- html
+
+ String RMWebAppURL = YarnConfiguration.getRMWebAppURL(this.conf);
+ html
.div("#nav")
- .h3()._("NodeManager")._() // TODO: Problem if no header like this
+ .h3()._("ResourceManager")._()
+ .ul()
+ .li().a(RMWebAppURL, "RM Home")._()._()
+ .h3()._("NodeManager")._() // TODO: Problem if no header like this
.ul()
.li()
.a(url("node"), "Node Information")._()
@@ -37,7 +53,7 @@ public class NavBlock extends HtmlBlock
.li()
.a(url("allContainers"), "List of Containers")._()
._()
- .h3("Tools")
+ .h3("Tools")
.ul()
.li().a("/conf", "Configuration")._()
.li().a("/logs", "Local logs")._()
Modified:
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java?rev=1457716&r1=1457715&r2=1457716&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java
Mon Mar 18 11:45:07 2013
@@ -42,7 +42,7 @@ public class NodePage extends NMView {
protected void commonPreHead(HTML<_> html) {
super.commonPreHead(html);
- set(initID(ACCORDION, "nav"), "{autoHeight:false, active:0}");
+ set(initID(ACCORDION, "nav"), "{autoHeight:false, active:1}");
}
@Override
Modified:
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java?rev=1457716&r1=1457715&r2=1457716&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
Mon Mar 18 11:45:07 2013
@@ -267,6 +267,36 @@ public class TestNodeStatusUpdater {
}
}
+ private class MyNodeStatusUpdater4 extends NodeStatusUpdaterImpl {
+ public ResourceTracker resourceTracker =
+ new MyResourceTracker(this.context);
+ private Context context;
+ private final long waitStartTime;
+ private final long rmStartIntervalMS;
+ private final boolean rmNeverStart;
+
+ public MyNodeStatusUpdater4(Context context, Dispatcher dispatcher,
+ NodeHealthCheckerService healthChecker, NodeManagerMetrics metrics,
+ long rmStartIntervalMS, boolean rmNeverStart) {
+ super(context, dispatcher, healthChecker, metrics);
+ this.context = context;
+ this.waitStartTime = System.currentTimeMillis();
+ this.rmStartIntervalMS = rmStartIntervalMS;
+ this.rmNeverStart = rmNeverStart;
+ }
+
+ @Override
+ protected ResourceTracker getRMClient() {
+ if(System.currentTimeMillis() - waitStartTime <= rmStartIntervalMS
+ || rmNeverStart) {
+ throw new YarnException("Faking RM start failure as start " +
+ "delay timer has not expired.");
+ } else {
+ return resourceTracker;
+ }
+ }
+ }
+
private class MyNodeManager extends NodeManager {
private MyNodeStatusUpdater3 nodeStatusUpdater;
@@ -580,6 +610,73 @@ public class TestNodeStatusUpdater {
+ "Recieved SHUTDOWN signal from Resourcemanager ,Registration of
NodeManager failed");
}
+ @Test (timeout = 15000)
+ public void testNMConnectionToRM() {
+ final long delta = 1500;
+ final long connectionWaitSecs = 5;
+ final long connectionRetryIntervalSecs = 1;
+ //Waiting for rmStartIntervalMS, RM will be started
+ final long rmStartIntervalMS = 2*1000;
+ YarnConfiguration conf = createNMConfig();
+ conf.setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_WAIT_SECS,
+ connectionWaitSecs);
+ conf.setLong(YarnConfiguration
+ .RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_SECS,
+ connectionRetryIntervalSecs);
+
+ //Test NM try to connect to RM Several times, but finally fail
+ nm = new NodeManager() {
+ @Override
+ protected NodeStatusUpdater createNodeStatusUpdater(Context context,
+ Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
+ NodeStatusUpdater nodeStatusUpdater = new MyNodeStatusUpdater4(
+ context, dispatcher, healthChecker, metrics,
+ rmStartIntervalMS, true);
+ return nodeStatusUpdater;
+ }
+ };
+ nm.init(conf);
+ long waitStartTime = System.currentTimeMillis();
+ try {
+ nm.start();
+ Assert.fail("NM should have failed to start due to RM connect failure");
+ } catch(Exception e) {
+ Assert.assertTrue("NM should have tried re-connecting to RM during " +
+ "period of at least " + connectionWaitSecs + " seconds, but " +
+ "stopped retrying within " + (connectionWaitSecs + delta/1000) +
+ " seconds", (System.currentTimeMillis() - waitStartTime
+ >= connectionWaitSecs*1000) && (System.currentTimeMillis()
+ - waitStartTime < (connectionWaitSecs*1000+delta)));
+ }
+
+ //Test NM connect to RM, fail at first several attempts,
+ //but finally success.
+ nm = new NodeManager() {
+ @Override
+ protected NodeStatusUpdater createNodeStatusUpdater(Context context,
+ Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
+ NodeStatusUpdater nodeStatusUpdater = new MyNodeStatusUpdater4(
+ context, dispatcher, healthChecker, metrics, rmStartIntervalMS,
+ false);
+ return nodeStatusUpdater;
+ }
+ };
+
+ nm.init(conf);
+ waitStartTime = System.currentTimeMillis();
+ try {
+ nm.start();
+ } catch (Exception ex){
+ Assert.fail("NM should have started successfully " +
+ "after connecting to RM.");
+ }
+ Assert.assertTrue("NM should have connected to RM within " + delta/1000
+ +" seconds of RM starting up.",
+ (System.currentTimeMillis() - waitStartTime >= rmStartIntervalMS)
+ && (System.currentTimeMillis() - waitStartTime
+ < (rmStartIntervalMS+delta)));
+ }
+
/**
* Verifies that if for some reason NM fails to start ContainerManager RPC
* server, RM is oblivious to NM's presence. The behaviour is like this
Modified:
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java?rev=1457716&r1=1457715&r2=1457716&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java
Mon Mar 18 11:45:07 2013
@@ -19,41 +19,39 @@
package org.apache.hadoop.yarn.server.webproxy.amfilter;
import java.io.IOException;
-import java.util.Collections;
-import java.util.Enumeration;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
import java.util.concurrent.atomic.AtomicBoolean;
-import javax.servlet.Filter;
-import javax.servlet.FilterChain;
-import javax.servlet.FilterConfig;
-import javax.servlet.ServletContext;
-import javax.servlet.ServletException;
-import javax.servlet.ServletRequest;
-import javax.servlet.ServletResponse;
+import javax.servlet.*;
+import javax.servlet.http.Cookie;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
-import junit.framework.Assert;
+import static junit.framework.Assert.*;
+import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServlet;
+import org.glassfish.grizzly.servlet.HttpServletResponseImpl;
import org.junit.Test;
import org.mockito.Mockito;
+/**
+ * Test AmIpFilter. Requests to a no declared hosts should has way through
+ * proxy. Another requests can be filtered with (without) user name.
+ *
+ */
+public class TestAmFilter {
-public class TestAmFilter {
-
- private String proxyHost = "bogushost.com";
+ private String proxyHost = "localhost";
private String proxyUri = "http://bogus";;
+ private String doFilterRequest;
+ private AmIpServletRequestWrapper servletWrapper;
private class TestAmIpFilter extends AmIpFilter {
private Set<String> proxyAddresses = null;
protected Set<String> getProxyAddresses() {
- if(proxyAddresses == null) {
+ if (proxyAddresses == null) {
proxyAddresses = new HashSet<String>();
}
proxyAddresses.add(proxyHost);
@@ -61,12 +59,10 @@ public class TestAmFilter {
}
}
-
private static class DummyFilterConfig implements FilterConfig {
final Map<String, String> map;
-
- DummyFilterConfig(Map<String,String> map) {
+ DummyFilterConfig(Map<String, String> map) {
this.map = map;
}
@@ -74,22 +70,24 @@ public class TestAmFilter {
public String getFilterName() {
return "dummy";
}
+
@Override
public String getInitParameter(String arg0) {
return map.get(arg0);
}
+
@Override
public Enumeration<String> getInitParameterNames() {
return Collections.enumeration(map.keySet());
}
+
@Override
public ServletContext getServletContext() {
return null;
}
}
-
- @Test
+ @Test(timeout = 5000)
public void filterNullCookies() throws Exception {
HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
@@ -97,13 +95,12 @@ public class TestAmFilter {
Mockito.when(request.getRemoteAddr()).thenReturn(proxyHost);
HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
-
final AtomicBoolean invoked = new AtomicBoolean();
FilterChain chain = new FilterChain() {
@Override
- public void doFilter(ServletRequest servletRequest, ServletResponse
servletResponse)
- throws IOException, ServletException {
+ public void doFilter(ServletRequest servletRequest,
+ ServletResponse servletResponse) throws IOException,
ServletException {
invoked.set(true);
}
};
@@ -115,7 +112,93 @@ public class TestAmFilter {
Filter filter = new TestAmIpFilter();
filter.init(conf);
filter.doFilter(request, response, chain);
- Assert.assertTrue(invoked.get());
+ assertTrue(invoked.get());
filter.destroy();
}
+
+ /**
+ * Test AmIpFilter
+ */
+ @Test(timeout = 1000)
+ public void testFilter() throws Exception {
+ Map<String, String> params = new HashMap<String, String>();
+ params.put(AmIpFilter.PROXY_HOST, proxyHost);
+ params.put(AmIpFilter.PROXY_URI_BASE, proxyUri);
+ FilterConfig config = new DummyFilterConfig(params);
+
+ // dummy filter
+ FilterChain chain = new FilterChain() {
+ @Override
+ public void doFilter(ServletRequest servletRequest,
+ ServletResponse servletResponse) throws IOException,
ServletException {
+ doFilterRequest = servletRequest.getClass().getName();
+ if (servletRequest instanceof AmIpServletRequestWrapper) {
+ servletWrapper = (AmIpServletRequestWrapper) servletRequest;
+
+ }
+ }
+ };
+ AmIpFilter testFilter = new AmIpFilter();
+ testFilter.init(config);
+
+ HttpServletResponseForTest response = new HttpServletResponseForTest();
+ // Test request should implements HttpServletRequest
+
+ ServletRequest failRequest = Mockito.mock(ServletRequest.class);
+ try {
+ testFilter.doFilter(failRequest, response, chain);
+ fail();
+ } catch (ServletException e) {
+ assertEquals("This filter only works for HTTP/HTTPS", e.getMessage());
+ }
+
+ // request with HttpServletRequest
+ HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
+ Mockito.when(request.getRemoteAddr()).thenReturn("redirect");
+ Mockito.when(request.getRequestURI()).thenReturn("/redirect");
+ testFilter.doFilter(request, response, chain);
+ // address "redirect" is not in host list
+ assertEquals("http://bogus/redirect";, response.getRedirect());
+ // "127.0.0.1" contains in host list. Without cookie
+ Mockito.when(request.getRemoteAddr()).thenReturn("127.0.0.1");
+ testFilter.doFilter(request, response, chain);
+
+ assertTrue(doFilterRequest
+ .contains("javax.servlet.http.HttpServletRequest"));
+ // cookie added
+ Cookie[] cookies = new Cookie[1];
+ cookies[0] = new Cookie(WebAppProxyServlet.PROXY_USER_COOKIE_NAME, "user");
+
+ Mockito.when(request.getCookies()).thenReturn(cookies);
+ testFilter.doFilter(request, response, chain);
+
+ assertEquals(
+
"org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpServletRequestWrapper",
+ doFilterRequest);
+ // request contains principal from cookie
+ assertEquals("user", servletWrapper.getUserPrincipal().getName());
+ assertEquals("user", servletWrapper.getRemoteUser());
+ assertFalse(servletWrapper.isUserInRole(""));
+
+ }
+
+ private class HttpServletResponseForTest extends HttpServletResponseImpl {
+ String redirectLocation = "";
+
+ public String getRedirect() {
+ return redirectLocation;
+ }
+
+ @Override
+ public void sendRedirect(String location) throws IOException {
+ redirectLocation = location;
+ }
+
+ @Override
+ public String encodeRedirectURL(String url) {
+ return url;
+ }
+
+ }
+
}
