Author: szetszwo
Date: Tue Sep 3 19:01:19 2013
New Revision: 1519796
URL: http://svn.apache.org/r1519796
Log:
Merge r1517887 through r1518850 from trunk.
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogDumper.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestLogDumper.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt?rev=1519796&r1=1519795&r2=1519796&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt Tue Sep 3
19:01:19 2013
@@ -54,6 +54,18 @@ Release 2.1.1-beta - UNRELEASED
YARN-942. In Fair Scheduler documentation, inconsistency on which
properties have prefix (Akira Ajisaka via Sandy Ryza)
+ YARN-1083. Changed ResourceManager to fail when the expiry interval is less
+ than the configured node-heartbeat interval. (Zhijie Shen via vinodkv)
+
+ YARN-1081. Made a trivial change to YARN node CLI header to avoid potential
+ confusion. (Akira AJISAKA via vinodkv)
+
+ YARN-1034. Remove "experimental" in the Fair Scheduler documentation.
+ (Karthik Kambatla via Sandy Ryza)
+
+ YARN-1080. Improved help message for "yarn logs" command. (Xuan Gong via
+ vinodkv)
+
OPTIMIZATIONS
BUG FIXES
@@ -105,6 +117,15 @@ Release 2.1.1-beta - UNRELEASED
YARN-1008. MiniYARNCluster with multiple nodemanagers, all nodes have same
key for allocations. (tucu)
+ YARN-981. Fixed YARN webapp so that /logs servlet works like before. (Jian
He
+ via vinodkv)
+
+ YARN-602. Fixed NodeManager to not let users override some mandatory
+ environmental variables. (Kenji Kikushima via vinodkv)
+
+ YARN-1101. Active nodes can be decremented below 0 (Robert Parker
+ via tgraves)
+
Release 2.1.0-beta - 2013-08-22
INCOMPATIBLE CHANGES
@@ -1223,6 +1244,9 @@ Release 0.23.10 - UNRELEASED
YARN-337. RM handles killed application tracking URL poorly (jlowe)
+ YARN-1101. Active nodes can be decremented below 0 (Robert Parker
+ via tgraves)
+
Release 0.23.9 - 2013-07-08
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java?rev=1519796&r1=1519795&r2=1519796&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java
Tue Sep 3 19:01:19 2013
@@ -43,7 +43,7 @@ import org.apache.hadoop.yarn.util.Conve
@Private
@Unstable
public class NodeCLI extends YarnCLI {
- private static final String NODES_PATTERN = "%16s\t%15s\t%17s\t%18s" +
+ private static final String NODES_PATTERN = "%16s\t%15s\t%17s\t%28s" +
System.getProperty("line.separator");
private static final String NODE_STATE_CMD = "states";
@@ -133,7 +133,7 @@ public class NodeCLI extends YarnCLI {
nodeStates.toArray(new NodeState[0]));
writer.println("Total Nodes:" + nodesReport.size());
writer.printf(NODES_PATTERN, "Node-Id", "Node-State", "Node-Http-Address",
- "Running-Containers");
+ "Number-of-Running-Containers");
for (NodeReport nodeReport : nodesReport) {
writer.printf(NODES_PATTERN, nodeReport.getNodeId(), nodeReport
.getNodeState(), nodeReport.getHttpAddress(), nodeReport
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java?rev=1519796&r1=1519795&r2=1519796&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
Tue Sep 3 19:01:19 2013
@@ -543,9 +543,9 @@ public class TestYarnCLI {
PrintWriter pw = new PrintWriter(baos);
pw.println("Total Nodes:1");
pw.print(" Node-Id\t Node-State\tNode-Http-Address\t");
- pw.println("Running-Containers");
- pw.print(" host0:0\t NEW\t host1:8888");
- pw.println("\t 0");
+ pw.println("Number-of-Running-Containers");
+ pw.print(" host0:0\t NEW\t host1:8888\t");
+ pw.println(" 0");
pw.close();
String nodesReportStr = baos.toString("UTF-8");
Assert.assertEquals(nodesReportStr, sysOutStream.toString());
@@ -564,11 +564,11 @@ public class TestYarnCLI {
pw = new PrintWriter(baos);
pw.println("Total Nodes:2");
pw.print(" Node-Id\t Node-State\tNode-Http-Address\t");
- pw.println("Running-Containers");
- pw.print(" host0:0\t RUNNING\t host1:8888");
- pw.println("\t 0");
- pw.print(" host1:0\t RUNNING\t host1:8888");
- pw.println("\t 0");
+ pw.println("Number-of-Running-Containers");
+ pw.print(" host0:0\t RUNNING\t host1:8888\t");
+ pw.println(" 0");
+ pw.print(" host1:0\t RUNNING\t host1:8888\t");
+ pw.println(" 0");
pw.close();
nodesReportStr = baos.toString("UTF-8");
Assert.assertEquals(nodesReportStr, sysOutStream.toString());
@@ -593,9 +593,9 @@ public class TestYarnCLI {
pw = new PrintWriter(baos);
pw.println("Total Nodes:1");
pw.print(" Node-Id\t Node-State\tNode-Http-Address\t");
- pw.println("Running-Containers");
- pw.print(" host0:0\t UNHEALTHY\t host1:8888");
- pw.println("\t 0");
+ pw.println("Number-of-Running-Containers");
+ pw.print(" host0:0\t UNHEALTHY\t host1:8888\t");
+ pw.println(" 0");
pw.close();
nodesReportStr = baos.toString("UTF-8");
Assert.assertEquals(nodesReportStr, sysOutStream.toString());
@@ -614,9 +614,9 @@ public class TestYarnCLI {
pw = new PrintWriter(baos);
pw.println("Total Nodes:1");
pw.print(" Node-Id\t Node-State\tNode-Http-Address\t");
- pw.println("Running-Containers");
- pw.print(" host0:0\t DECOMMISSIONED\t host1:8888");
- pw.println("\t 0");
+ pw.println("Number-of-Running-Containers");
+ pw.print(" host0:0\t DECOMMISSIONED\t host1:8888\t");
+ pw.println(" 0");
pw.close();
nodesReportStr = baos.toString("UTF-8");
Assert.assertEquals(nodesReportStr, sysOutStream.toString());
@@ -635,9 +635,9 @@ public class TestYarnCLI {
pw = new PrintWriter(baos);
pw.println("Total Nodes:1");
pw.print(" Node-Id\t Node-State\tNode-Http-Address\t");
- pw.println("Running-Containers");
- pw.print(" host0:0\t REBOOTED\t host1:8888");
- pw.println("\t 0");
+ pw.println("Number-of-Running-Containers");
+ pw.print(" host0:0\t REBOOTED\t host1:8888\t");
+ pw.println(" 0");
pw.close();
nodesReportStr = baos.toString("UTF-8");
Assert.assertEquals(nodesReportStr, sysOutStream.toString());
@@ -656,9 +656,9 @@ public class TestYarnCLI {
pw = new PrintWriter(baos);
pw.println("Total Nodes:1");
pw.print(" Node-Id\t Node-State\tNode-Http-Address\t");
- pw.println("Running-Containers");
- pw.print(" host0:0\t LOST\t host1:8888");
- pw.println("\t 0");
+ pw.println("Number-of-Running-Containers");
+ pw.print(" host0:0\t LOST\t host1:8888\t");
+ pw.println(" 0");
pw.close();
nodesReportStr = baos.toString("UTF-8");
Assert.assertEquals(nodesReportStr, sysOutStream.toString());
@@ -681,17 +681,17 @@ public class TestYarnCLI {
pw = new PrintWriter(baos);
pw.println("Total Nodes:5");
pw.print(" Node-Id\t Node-State\tNode-Http-Address\t");
- pw.println("Running-Containers");
- pw.print(" host0:0\t NEW\t host1:8888");
- pw.println("\t 0");
- pw.print(" host0:0\t RUNNING\t host1:8888");
- pw.println("\t 0");
- pw.print(" host1:0\t RUNNING\t host1:8888");
- pw.println("\t 0");
- pw.print(" host0:0\t REBOOTED\t host1:8888");
- pw.println("\t 0");
- pw.print(" host0:0\t LOST\t host1:8888");
- pw.println("\t 0");
+ pw.println("Number-of-Running-Containers");
+ pw.print(" host0:0\t NEW\t host1:8888\t");
+ pw.println(" 0");
+ pw.print(" host0:0\t RUNNING\t host1:8888\t");
+ pw.println(" 0");
+ pw.print(" host1:0\t RUNNING\t host1:8888\t");
+ pw.println(" 0");
+ pw.print(" host0:0\t REBOOTED\t host1:8888\t");
+ pw.println(" 0");
+ pw.print(" host0:0\t LOST\t host1:8888\t");
+ pw.println(" 0");
pw.close();
nodesReportStr = baos.toString("UTF-8");
Assert.assertEquals(nodesReportStr, sysOutStream.toString());
@@ -712,21 +712,21 @@ public class TestYarnCLI {
pw = new PrintWriter(baos);
pw.println("Total Nodes:7");
pw.print(" Node-Id\t Node-State\tNode-Http-Address\t");
- pw.println("Running-Containers");
- pw.print(" host0:0\t NEW\t host1:8888");
- pw.println("\t 0");
- pw.print(" host0:0\t RUNNING\t host1:8888");
- pw.println("\t 0");
- pw.print(" host1:0\t RUNNING\t host1:8888");
- pw.println("\t 0");
- pw.print(" host0:0\t UNHEALTHY\t host1:8888");
- pw.println("\t 0");
- pw.print(" host0:0\t DECOMMISSIONED\t host1:8888");
- pw.println("\t 0");
- pw.print(" host0:0\t REBOOTED\t host1:8888");
- pw.println("\t 0");
- pw.print(" host0:0\t LOST\t host1:8888");
- pw.println("\t 0");
+ pw.println("Number-of-Running-Containers");
+ pw.print(" host0:0\t NEW\t host1:8888\t");
+ pw.println(" 0");
+ pw.print(" host0:0\t RUNNING\t host1:8888\t");
+ pw.println(" 0");
+ pw.print(" host1:0\t RUNNING\t host1:8888\t");
+ pw.println(" 0");
+ pw.print(" host0:0\t UNHEALTHY\t host1:8888\t");
+ pw.println(" 0");
+ pw.print(" host0:0\t DECOMMISSIONED\t host1:8888\t");
+ pw.println(" 0");
+ pw.print(" host0:0\t REBOOTED\t host1:8888\t");
+ pw.println(" 0");
+ pw.print(" host0:0\t LOST\t host1:8888\t");
+ pw.println(" 0");
pw.close();
nodesReportStr = baos.toString("UTF-8");
Assert.assertEquals(nodesReportStr, sysOutStream.toString());
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogDumper.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogDumper.java?rev=1519796&r1=1519795&r2=1519796&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogDumper.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogDumper.java
Tue Sep 3 19:01:19 2013
@@ -72,10 +72,18 @@ public class LogDumper extends Configure
+ "nodename:port (must be specified if container id is specified)");
opts.addOption(APP_OWNER_OPTION, true,
"AppOwner (assumed to be current user if not specified)");
+ opts.getOption(APPLICATION_ID_OPTION).setArgName("Application ID");
+ opts.getOption(CONTAINER_ID_OPTION).setArgName("Container ID");
+ opts.getOption(NODE_ADDRESS_OPTION).setArgName("Node Address");
+ opts.getOption(APP_OWNER_OPTION).setArgName("Application Owner");
+
+ Options printOpts = new Options();
+ printOpts.addOption(opts.getOption(CONTAINER_ID_OPTION));
+ printOpts.addOption(opts.getOption(NODE_ADDRESS_OPTION));
+ printOpts.addOption(opts.getOption(APP_OWNER_OPTION));
if (args.length < 1) {
- HelpFormatter formatter = new HelpFormatter();
- formatter.printHelp("general options are: ", opts);
+ printHelpMessage(printOpts);
return -1;
}
@@ -92,16 +100,13 @@ public class LogDumper extends Configure
appOwner = commandLine.getOptionValue(APP_OWNER_OPTION);
} catch (ParseException e) {
System.out.println("options parsing failed: " + e.getMessage());
-
- HelpFormatter formatter = new HelpFormatter();
- formatter.printHelp("general options are: ", opts);
+ printHelpMessage(printOpts);
return -1;
}
if (appIdStr == null) {
System.out.println("ApplicationId cannot be null!");
- HelpFormatter formatter = new HelpFormatter();
- formatter.printHelp("general options are: ", opts);
+ printHelpMessage(printOpts);
return -1;
}
@@ -119,8 +124,7 @@ public class LogDumper extends Configure
} else if ((containerIdStr == null && nodeAddress != null)
|| (containerIdStr != null && nodeAddress == null)) {
System.out.println("ContainerId or NodeAddress cannot be null!");
- HelpFormatter formatter = new HelpFormatter();
- formatter.printHelp("general options are: ", opts);
+ printHelpMessage(printOpts);
resultCode = -1;
} else {
Path remoteRootLogDir =
@@ -255,4 +259,12 @@ public class LogDumper extends Configure
int exitCode = logDumper.run(args);
System.exit(exitCode);
}
+
+ private void printHelpMessage(Options options) {
+ System.out.println("Retrieve logs for completed YARN applications.");
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp("yarn logs -applicationId <application ID> [OPTIONS]",
new Options());
+ formatter.setSyntaxPrefix("");
+ formatter.printHelp("general options are:", options);
+ }
}
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java?rev=1519796&r1=1519795&r2=1519796&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
Tue Sep 3 19:01:19 2013
@@ -242,7 +242,10 @@ public class WebApps {
for(Map.Entry<String, Object> entry : attributes.entrySet()) {
server.setAttribute(entry.getKey(), entry.getValue());
}
- server.addGlobalFilter("guice", GuiceFilter.class.getName(), null);
+ String webAppPath = "/" + name + "/*";
+ server.defineFilter(server.getWebAppContext(), "guice",
+ GuiceFilter.class.getName(), null, new String[] { webAppPath, "/" });
+
webapp.setConf(conf);
webapp.setHttpServer(server);
server.start();
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestLogDumper.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestLogDumper.java?rev=1519796&r1=1519795&r2=1519796&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestLogDumper.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestLogDumper.java
Tue Sep 3 19:01:19 2013
@@ -19,14 +19,30 @@
package org.apache.hadoop.yarn.logaggregation;
import static org.junit.Assert.assertTrue;
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+import java.io.PrintWriter;
+
+import junit.framework.Assert;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.junit.Before;
import org.junit.Test;
public class TestLogDumper {
+ ByteArrayOutputStream sysOutStream;
+ private PrintStream sysOut;
+
+ @Before
+ public void setUp() {
+ sysOutStream = new ByteArrayOutputStream();
+ sysOut = new PrintStream(sysOutStream);
+ System.setOut(sysOut);
+ }
+
@Test
public void testFailResultCodes() throws Exception {
Configuration conf = new YarnConfiguration();
@@ -44,4 +60,30 @@ public class TestLogDumper {
"nonexistentnode:1234", "nobody");
assertTrue("Should return an error code", exitCode != 0);
}
+
+ @Test
+ public void testHelpMessage() throws Exception {
+ Configuration conf = new YarnConfiguration();
+ LogDumper dumper = new LogDumper();
+ dumper.setConf(conf);
+
+ int exitCode = dumper.run(new String[]{});
+ assertTrue(exitCode == -1);
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintWriter pw = new PrintWriter(baos);
+ pw.println("Retrieve logs for completed YARN applications.");
+ pw.println("usage: yarn logs -applicationId <application ID> [OPTIONS]");
+ pw.println();
+ pw.println("general options are:");
+ pw.println(" -appOwner <Application Owner> AppOwner (assumed to be
current user if");
+ pw.println(" not specified)");
+ pw.println(" -containerId <Container ID> ContainerId (must be
specified if node");
+ pw.println(" address is specified)");
+ pw.println(" -nodeAddress <Node Address> NodeAddress in the format
nodename:port");
+ pw.println(" (must be specified if
container id is");
+ pw.println(" specified)");
+ pw.close();
+ String appReportStr = baos.toString("UTF-8");
+ Assert.assertEquals(appReportStr, sysOutStream.toString());
+ }
}
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java?rev=1519796&r1=1519795&r2=1519796&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
Tue Sep 3 19:01:19 2013
@@ -588,20 +588,18 @@ public class ContainerLaunch implements
environment.put(Environment.LOG_DIRS.name(),
StringUtils.join(",", containerLogDirs));
- putEnvIfNotNull(environment, Environment.USER.name(), container.getUser());
+ environment.put(Environment.USER.name(), container.getUser());
- putEnvIfNotNull(environment,
- Environment.LOGNAME.name(),container.getUser());
-
- putEnvIfNotNull(environment,
- Environment.HOME.name(),
+ environment.put(Environment.LOGNAME.name(), container.getUser());
+
+ environment.put(Environment.HOME.name(),
conf.get(
YarnConfiguration.NM_USER_HOME_DIR,
YarnConfiguration.DEFAULT_NM_USER_HOME_DIR
)
);
- putEnvIfNotNull(environment, Environment.PWD.name(), pwd.toString());
+ environment.put(Environment.PWD.name(), pwd.toString());
putEnvIfNotNull(environment,
Environment.HADOOP_CONF_DIR.name(),
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java?rev=1519796&r1=1519795&r2=1519796&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
Tue Sep 3 19:01:19 2013
@@ -346,7 +346,6 @@ public class TestContainerLaunch extends
ApplicationAttemptId appAttemptId =
ApplicationAttemptId.newInstance(appId, 1);
- int port = 12345;
ContainerId cId = ContainerId.newInstance(appAttemptId, 0);
Map<String, String> userSetEnv = new HashMap<String, String>();
userSetEnv.put(Environment.CONTAINER_ID.name(), "user_set_container_id");
@@ -354,6 +353,11 @@ public class TestContainerLaunch extends
userSetEnv.put(Environment.NM_PORT.name(), "user_set_NM_PORT");
userSetEnv.put(Environment.NM_HTTP_PORT.name(), "user_set_NM_HTTP_PORT");
userSetEnv.put(Environment.LOCAL_DIRS.name(), "user_set_LOCAL_DIR");
+ userSetEnv.put(Environment.USER.key(), "user_set_" +
+ Environment.USER.key());
+ userSetEnv.put(Environment.LOGNAME.name(), "user_set_LOGNAME");
+ userSetEnv.put(Environment.PWD.name(), "user_set_PWD");
+ userSetEnv.put(Environment.HOME.name(), "user_set_HOME");
containerLaunchContext.setEnvironment(userSetEnv);
File scriptFile = Shell.appendScriptExtension(tmpDir, "scriptFile");
@@ -371,6 +375,14 @@ public class TestContainerLaunch extends
+ processStartFile);
fileWriter.println("@echo " + Environment.LOCAL_DIRS.$() + ">> "
+ processStartFile);
+ fileWriter.println("@echo " + Environment.USER.$() + ">> "
+ + processStartFile);
+ fileWriter.println("@echo " + Environment.LOGNAME.$() + ">> "
+ + processStartFile);
+ fileWriter.println("@echo " + Environment.PWD.$() + ">> "
+ + processStartFile);
+ fileWriter.println("@echo " + Environment.HOME.$() + ">> "
+ + processStartFile);
fileWriter.println("@echo " + cId + ">> " + processStartFile);
fileWriter.println("@ping -n 100 127.0.0.1 >nul");
} else {
@@ -385,6 +397,14 @@ public class TestContainerLaunch extends
+ processStartFile);
fileWriter.write("\necho $" + Environment.LOCAL_DIRS.name() + " >> "
+ processStartFile);
+ fileWriter.write("\necho $" + Environment.USER.name() + " >> "
+ + processStartFile);
+ fileWriter.write("\necho $" + Environment.LOGNAME.name() + " >> "
+ + processStartFile);
+ fileWriter.write("\necho $" + Environment.PWD.name() + " >> "
+ + processStartFile);
+ fileWriter.write("\necho $" + Environment.HOME.name() + " >> "
+ + processStartFile);
fileWriter.write("\necho $$ >> " + processStartFile);
fileWriter.write("\nexec sleep 100");
}
@@ -452,6 +472,22 @@ public class TestContainerLaunch extends
reader.readLine());
Assert.assertEquals(String.valueOf(HTTP_PORT), reader.readLine());
Assert.assertEquals(StringUtils.join(",", appDirs), reader.readLine());
+ Assert.assertEquals(user, reader.readLine());
+ Assert.assertEquals(user, reader.readLine());
+ String obtainedPWD = reader.readLine();
+ boolean found = false;
+ for (Path localDir : appDirs) {
+ if (new Path(localDir, cId.toString()).toString().equals(obtainedPWD)) {
+ found = true;
+ break;
+ }
+ }
+ Assert.assertTrue("Wrong local-dir found : " + obtainedPWD, found);
+ Assert.assertEquals(
+ conf.get(
+ YarnConfiguration.NM_USER_HOME_DIR,
+ YarnConfiguration.DEFAULT_NM_USER_HOME_DIR),
+ reader.readLine());
Assert.assertEquals(cId.toString(), containerLaunchContext
.getEnvironment().get(Environment.CONTAINER_ID.name()));
@@ -465,6 +501,26 @@ public class TestContainerLaunch extends
.getEnvironment().get(Environment.LOCAL_DIRS.name()));
Assert.assertEquals(StringUtils.join(",", containerLogDirs),
containerLaunchContext.getEnvironment().get(Environment.LOG_DIRS.name()));
+ Assert.assertEquals(user, containerLaunchContext.getEnvironment()
+ .get(Environment.USER.name()));
+ Assert.assertEquals(user, containerLaunchContext.getEnvironment()
+ .get(Environment.LOGNAME.name()));
+ found = false;
+ obtainedPWD =
+ containerLaunchContext.getEnvironment().get(Environment.PWD.name());
+ for (Path localDir : appDirs) {
+ if (new Path(localDir, cId.toString()).toString().equals(obtainedPWD)) {
+ found = true;
+ break;
+ }
+ }
+ Assert.assertTrue("Wrong local-dir found : " + obtainedPWD, found);
+ Assert.assertEquals(
+ conf.get(
+ YarnConfiguration.NM_USER_HOME_DIR,
+ YarnConfiguration.DEFAULT_NM_USER_HOME_DIR),
+ containerLaunchContext.getEnvironment()
+ .get(Environment.HOME.name()));
// Get the pid of the process
String pid = reader.readLine().trim();
@@ -538,7 +594,6 @@ public class TestContainerLaunch extends
ContainerLaunchContext containerLaunchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
- int port = 12345;
// upload the script file so that the container can run it
URL resource_alpha =
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java?rev=1519796&r1=1519795&r2=1519796&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
Tue Sep 3 19:01:19 2013
@@ -365,6 +365,20 @@ public class ResourceManager extends Com
+ ", " + YarnConfiguration.RM_AM_MAX_ATTEMPTS
+ "=" + globalMaxAppAttempts + ", it should be a positive integer.");
}
+
+ // validate expireIntvl >= heartbeatIntvl
+ long expireIntvl = conf.getLong(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS,
+ YarnConfiguration.DEFAULT_RM_NM_EXPIRY_INTERVAL_MS);
+ long heartbeatIntvl =
+ conf.getLong(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS,
+ YarnConfiguration.DEFAULT_RM_NM_HEARTBEAT_INTERVAL_MS);
+ if (expireIntvl < heartbeatIntvl) {
+ throw new YarnRuntimeException("Nodemanager expiry interval should be no"
+ + " less than heartbeat interval, "
+ + YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS + "=" + expireIntvl
+ + ", " + YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS + "="
+ + heartbeatIntvl);
+ }
}
@Private
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java?rev=1519796&r1=1519795&r2=1519796&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
Tue Sep 3 19:01:19 2013
@@ -393,9 +393,18 @@ public class RMNodeImpl implements RMNod
}
}
- private void updateMetricsForDeactivatedNode(NodeState finalState) {
+ private void updateMetricsForDeactivatedNode(NodeState initialState,
+ NodeState finalState) {
ClusterMetrics metrics = ClusterMetrics.getMetrics();
- metrics.decrNumActiveNodes();
+
+ switch (initialState) {
+ case RUNNING:
+ metrics.decrNumActiveNodes();
+ break;
+ case UNHEALTHY:
+ metrics.decrNumUnhealthyNMs();
+ break;
+ }
switch (finalState) {
case DECOMMISSIONED:
@@ -505,7 +514,8 @@ public class RMNodeImpl implements RMNod
// If the current state is NodeState.UNHEALTHY
// Then node is already been removed from the
// Scheduler
- if (!rmNode.getState().equals(NodeState.UNHEALTHY)) {
+ NodeState initialState = rmNode.getState();
+ if (!initialState.equals(NodeState.UNHEALTHY)) {
rmNode.context.getDispatcher().getEventHandler()
.handle(new NodeRemovedSchedulerEvent(rmNode));
}
@@ -520,7 +530,7 @@ public class RMNodeImpl implements RMNod
rmNode.context.getInactiveRMNodes().put(rmNode.nodeId.getHost(), rmNode);
//Update the metrics
- rmNode.updateMetricsForDeactivatedNode(finalState);
+ rmNode.updateMetricsForDeactivatedNode(initialState, finalState);
}
}
@@ -550,7 +560,8 @@ public class RMNodeImpl implements RMNod
new NodesListManagerEvent(
NodesListManagerEventType.NODE_UNUSABLE, rmNode));
// Update metrics
- rmNode.updateMetricsForDeactivatedNode(NodeState.UNHEALTHY);
+ rmNode.updateMetricsForDeactivatedNode(rmNode.getState(),
+ NodeState.UNHEALTHY);
return NodeState.UNHEALTHY;
}
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java?rev=1519796&r1=1519795&r2=1519796&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
Tue Sep 3 19:01:19 2013
@@ -267,7 +267,21 @@ public class TestRMNodeTransitions {
@Test
public void testUnhealthyExpire() {
RMNodeImpl node = getUnhealthyNode();
+ ClusterMetrics cm = ClusterMetrics.getMetrics();
+ int initialActive = cm.getNumActiveNMs();
+ int initialLost = cm.getNumLostNMs();
+ int initialUnhealthy = cm.getUnhealthyNMs();
+ int initialDecommissioned = cm.getNumDecommisionedNMs();
+ int initialRebooted = cm.getNumRebootedNMs();
node.handle(new RMNodeEvent(node.getNodeID(), RMNodeEventType.EXPIRE));
+ Assert.assertEquals("Active Nodes", initialActive, cm.getNumActiveNMs());
+ Assert.assertEquals("Lost Nodes", initialLost + 1, cm.getNumLostNMs());
+ Assert.assertEquals("Unhealthy Nodes",
+ initialUnhealthy - 1, cm.getUnhealthyNMs());
+ Assert.assertEquals("Decommissioned Nodes",
+ initialDecommissioned, cm.getNumDecommisionedNMs());
+ Assert.assertEquals("Rebooted Nodes",
+ initialRebooted, cm.getNumRebootedNMs());
Assert.assertEquals(NodeState.LOST, node.getState());
}
@@ -291,8 +305,22 @@ public class TestRMNodeTransitions {
@Test
public void testUnhealthyDecommission() {
RMNodeImpl node = getUnhealthyNode();
+ ClusterMetrics cm = ClusterMetrics.getMetrics();
+ int initialActive = cm.getNumActiveNMs();
+ int initialLost = cm.getNumLostNMs();
+ int initialUnhealthy = cm.getUnhealthyNMs();
+ int initialDecommissioned = cm.getNumDecommisionedNMs();
+ int initialRebooted = cm.getNumRebootedNMs();
node.handle(new RMNodeEvent(node.getNodeID(),
RMNodeEventType.DECOMMISSION));
+ Assert.assertEquals("Active Nodes", initialActive, cm.getNumActiveNMs());
+ Assert.assertEquals("Lost Nodes", initialLost, cm.getNumLostNMs());
+ Assert.assertEquals("Unhealthy Nodes",
+ initialUnhealthy - 1, cm.getUnhealthyNMs());
+ Assert.assertEquals("Decommissioned Nodes",
+ initialDecommissioned + 1, cm.getNumDecommisionedNMs());
+ Assert.assertEquals("Rebooted Nodes",
+ initialRebooted, cm.getNumRebootedNMs());
Assert.assertEquals(NodeState.DECOMMISSIONED, node.getState());
}
@@ -307,8 +335,22 @@ public class TestRMNodeTransitions {
@Test
public void testUnhealthyRebooting() {
RMNodeImpl node = getUnhealthyNode();
+ ClusterMetrics cm = ClusterMetrics.getMetrics();
+ int initialActive = cm.getNumActiveNMs();
+ int initialLost = cm.getNumLostNMs();
+ int initialUnhealthy = cm.getUnhealthyNMs();
+ int initialDecommissioned = cm.getNumDecommisionedNMs();
+ int initialRebooted = cm.getNumRebootedNMs();
node.handle(new RMNodeEvent(node.getNodeID(),
RMNodeEventType.REBOOTING));
+ Assert.assertEquals("Active Nodes", initialActive, cm.getNumActiveNMs());
+ Assert.assertEquals("Lost Nodes", initialLost, cm.getNumLostNMs());
+ Assert.assertEquals("Unhealthy Nodes",
+ initialUnhealthy - 1, cm.getUnhealthyNMs());
+ Assert.assertEquals("Decommissioned Nodes",
+ initialDecommissioned, cm.getNumDecommisionedNMs());
+ Assert.assertEquals("Rebooted Nodes",
+ initialRebooted + 1, cm.getNumRebootedNMs());
Assert.assertEquals(NodeState.REBOOTED, node.getState());
}
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java?rev=1519796&r1=1519795&r2=1519796&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java
Tue Sep 3 19:01:19 2013
@@ -203,4 +203,21 @@ public class TestResourceManager {
}
}
+ @Test
+ public void testNMExpiryAndHeartbeatIntervalsValidation() throws Exception {
+ Configuration conf = new YarnConfiguration();
+ conf.setLong(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, 1000);
+ conf.setLong(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, 1001);
+ resourceManager = new ResourceManager();;
+ try {
+ resourceManager.init(conf);
+ } catch (YarnRuntimeException e) {
+ // Exception is expected.
+ if (!e.getMessage().startsWith("Nodemanager expiry interval should be no"
+ + " less than heartbeat interval")) {
+ throw e;
+ }
+ }
+ }
+
}
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm?rev=1519796&r1=1519795&r2=1519796&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
Tue Sep 3 19:01:19 2013
@@ -25,8 +25,7 @@ Hadoop MapReduce Next Generation - Fair
* {Purpose}
This document describes the <<<FairScheduler>>>, a pluggable scheduler for
Hadoop
- which provides a way to share large clusters. <<NOTE:>> The Fair Scheduler
- implementation is currently under development and should be considered
experimental.
+ that allows YARN applications to share resources in large clusters fairly.
* {Introduction}
