Author: szetszwo
Date: Wed Nov 21 21:08:45 2012
New Revision: 1412297
URL: http://svn.apache.org/viewvc?rev=1412297&view=rev
Log:
Merge r1410998 through r1412282 from trunk.
Modified:
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt?rev=1412297&r1=1412296&r2=1412297&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt Wed Nov 21
21:08:45 2012
@@ -71,6 +71,8 @@ Release 2.0.3-alpha - Unreleased
YARN-183. Clean up fair scheduler code. (Sandy Ryza via tomwhite)
+ YARN-129. Simplify classpath construction for mini YARN tests. (tomwhite)
+
OPTIMIZATIONS
BUG FIXES
@@ -245,6 +247,8 @@ Release 0.23.5 - UNRELEASED
YARN-212. NM state machine ignores an APPLICATION_CONTAINER_FINISHED event
when it shouldn't (Nathan Roberts via jlowe)
+ YARN-219. NM should aggregate logs when application finishes. (bobby)
+
Release 0.23.4
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml?rev=1412297&r1=1412296&r2=1412297&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml
Wed Nov 21 21:08:45 2012
@@ -91,23 +91,6 @@
</configuration>
</plugin>
<plugin>
- <artifactId>maven-dependency-plugin</artifactId>
- <executions>
- <execution>
- <id>build-classpath</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>build-classpath</goal>
- </goals>
- <configuration>
- <!-- needed to run the unit test for DS to generate the required
classpath
- that is required in the env of the launch container in the
mini yarn cluster -->
-
<outputFile>target/classes/yarn-apps-ds-generated-classpath</outputFile>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
Modified:
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java?rev=1412297&r1=1412296&r2=1412297&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
Wed Nov 21 21:08:45 2012
@@ -494,9 +494,10 @@ public class Client extends YarnClientIm
classPathEnv.append(":./log4j.properties");
// add the runtime classpath needed for tests to work
- String testRuntimeClassPath = Client.getTestRuntimeClasspath();
- classPathEnv.append(':');
- classPathEnv.append(testRuntimeClassPath);
+ if (conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) {
+ classPathEnv.append(':');
+ classPathEnv.append(System.getProperty("java.class.path"));
+ }
env.put("CLASSPATH", classPathEnv.toString());
@@ -663,50 +664,4 @@ public class Client extends YarnClientIm
super.killApplication(appId);
}
- private static String getTestRuntimeClasspath() {
-
- InputStream classpathFileStream = null;
- BufferedReader reader = null;
- String envClassPath = "";
-
- LOG.info("Trying to generate classpath for app master from current
thread's classpath");
- try {
-
- // Create classpath from generated classpath
- // Check maven ppom.xml for generated classpath info
- // Works if compile time env is same as runtime. Mainly tests.
- ClassLoader thisClassLoader =
- Thread.currentThread().getContextClassLoader();
- String generatedClasspathFile = "yarn-apps-ds-generated-classpath";
- classpathFileStream =
- thisClassLoader.getResourceAsStream(generatedClasspathFile);
- if (classpathFileStream == null) {
- LOG.info("Could not classpath resource from class loader");
- return envClassPath;
- }
- LOG.info("Readable bytes from stream=" +
classpathFileStream.available());
- reader = new BufferedReader(new InputStreamReader(classpathFileStream));
- String cp = reader.readLine();
- if (cp != null) {
- envClassPath += cp.trim() + ":";
- }
- // Put the file itself on classpath for tasks.
- envClassPath +=
thisClassLoader.getResource(generatedClasspathFile).getFile();
- } catch (IOException e) {
- LOG.info("Could not find the necessary resource to generate class path
for tests. Error=" + e.getMessage());
- }
-
- try {
- if (classpathFileStream != null) {
- classpathFileStream.close();
- }
- if (reader != null) {
- reader.close();
- }
- } catch (IOException e) {
- LOG.info("Failed to close class path file stream or reader. Error=" +
e.getMessage());
- }
- return envClassPath;
- }
-
}
Modified:
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml?rev=1412297&r1=1412296&r2=1412297&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml
Wed Nov 21 21:08:45 2012
@@ -87,23 +87,6 @@
<build>
<plugins>
<plugin>
- <artifactId>maven-dependency-plugin</artifactId>
- <executions>
- <execution>
- <id>build-classpath</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>build-classpath</goal>
- </goals>
- <configuration>
- <!-- needed to run the unit test for DS to generate the required
classpath
- that is required in the env of the launch container in the
mini yarn cluster -->
-
<outputFile>target/classes/yarn-apps-am-generated-classpath</outputFile>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
Modified:
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java?rev=1412297&r1=1412296&r2=1412297&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java
Wed Nov 21 21:08:45 2012
@@ -18,12 +18,9 @@
package org.apache.hadoop.yarn.applications.unmanagedamlauncher;
-import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
import java.io.OutputStream;
import java.net.URL;
@@ -80,51 +77,17 @@ public class TestUnmanagedAMLauncher {
}
private static String getTestRuntimeClasspath() {
-
- InputStream classpathFileStream = null;
- BufferedReader reader = null;
- String envClassPath = "";
-
LOG.info("Trying to generate classpath for app master from current
thread's classpath");
- try {
-
- // Create classpath from generated classpath
- // Check maven pom.xml for generated classpath info
- // Works if compile time env is same as runtime. Mainly tests.
- ClassLoader thisClassLoader = Thread.currentThread()
- .getContextClassLoader();
- String generatedClasspathFile = "yarn-apps-am-generated-classpath";
- classpathFileStream = thisClassLoader
- .getResourceAsStream(generatedClasspathFile);
- if (classpathFileStream == null) {
- LOG.info("Could not classpath resource from class loader");
- return envClassPath;
- }
- LOG.info("Readable bytes from stream=" +
classpathFileStream.available());
- reader = new BufferedReader(new InputStreamReader(classpathFileStream));
- String cp = reader.readLine();
- if (cp != null) {
- envClassPath += cp.trim() + File.pathSeparator;
- }
- // yarn-site.xml at this location contains proper config for mini cluster
- URL url = thisClassLoader.getResource("yarn-site.xml");
- envClassPath += new File(url.getFile()).getParent();
- } catch (IOException e) {
- LOG.info("Could not find the necessary resource to generate class path
for tests. Error="
- + e.getMessage());
- }
-
- try {
- if (classpathFileStream != null) {
- classpathFileStream.close();
- }
- if (reader != null) {
- reader.close();
- }
- } catch (IOException e) {
- LOG.info("Failed to close class path file stream or reader. Error="
- + e.getMessage());
- }
+ String envClassPath = "";
+ String cp = System.getProperty("java.class.path");
+ if (cp != null) {
+ envClassPath += cp.trim() + File.pathSeparator;
+ }
+ // yarn-site.xml at this location contains proper config for mini cluster
+ ClassLoader thisClassLoader = Thread.currentThread()
+ .getContextClassLoader();
+ URL url = thisClassLoader.getResource("yarn-site.xml");
+ envClassPath += new File(url.getFile()).getParent();
return envClassPath;
}
Modified:
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java?rev=1412297&r1=1412296&r2=1412297&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
Wed Nov 21 21:08:45 2012
@@ -149,16 +149,13 @@ public class AppLogAggregatorImpl implem
ContainerId containerId;
while (!this.appFinishing.get()) {
- try {
- containerId = this.pendingContainers.poll();
- if (containerId == null) {
- Thread.sleep(THREAD_SLEEP_TIME);
- } else {
- uploadLogsForContainer(containerId);
+ synchronized(this) {
+ try {
+ wait(THREAD_SLEEP_TIME);
+ } catch (InterruptedException e) {
+ LOG.warn("PendingContainers queue is interrupted");
+ this.appFinishing.set(true);
}
- } catch (InterruptedException e) {
- LOG.warn("PendingContainers queue is interrupted");
- this.appFinishing.set(true);
}
}
@@ -251,8 +248,9 @@ public class AppLogAggregatorImpl implem
}
@Override
- public void finishLogAggregation() {
+ public synchronized void finishLogAggregation() {
LOG.info("Application just finished : " + this.applicationId);
this.appFinishing.set(true);
+ this.notifyAll();
}
}
