Author: cnauroth
Date: Fri Oct 4 05:51:51 2013
New Revision: 1529081
URL: http://svn.apache.org/r1529081
Log:
MAPREDUCE-5442. Merging change r1529079 from branch-2 to branch-2.1-beta
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/CHANGES.txt?rev=1529081&r1=1529080&r2=1529081&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/CHANGES.txt
Fri Oct 4 05:51:51 2013
@@ -87,6 +87,9 @@ Release 2.1.2 - UNRELEASED
MAPREDUCE-5489. MR jobs hangs as it does not use the node-blacklisting
feature in RM requests (Zhijie Shen via bikas)
+ MAPREDUCE-5442. $HADOOP_MAPRED_HOME/$HADOOP_CONF_DIR setting not working on
+ Windows. (Yingda Chen via cnauroth)
+
Release 2.1.1-beta - 2013-09-23
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java?rev=1529081&r1=1529080&r2=1529081&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
(original)
+++
hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
Fri Oct 4 05:51:51 2013
@@ -48,6 +48,7 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState;
import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
+import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.ContainerLogAppender;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
@@ -150,7 +151,8 @@ public class MRApps extends Apps {
}
for (String c : conf.getStrings(
MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH,
- MRJobConfig.DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH)) {
+ StringUtils.getStrings(
+ MRJobConfig.DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH))){
Apps.addToEnvironment(environment, Environment.CLASSPATH.name(), c
.trim());
}
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java?rev=1529081&r1=1529080&r2=1529081&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
(original)
+++
hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
Fri Oct 4 05:51:51 2013
@@ -192,7 +192,9 @@ public class TestMRApps {
}
assertTrue(environment.get("CLASSPATH").contains(yarnAppClasspath));
String mrAppClasspath =
-
job.getConfiguration().get(MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH);
+ job.getConfiguration().get(
+ MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH,
+ MRJobConfig.DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH);
if (mrAppClasspath != null) {
mrAppClasspath = mrAppClasspath.replaceAll(",\\s*", File.pathSeparator)
.trim();
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java?rev=1529081&r1=1529080&r2=1529081&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
(original)
+++
hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
Fri Oct 4 05:51:51 2013
@@ -19,6 +19,7 @@ package org.apache.hadoop.mapreduce;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.util.Shell;
@InterfaceAudience.Private
@InterfaceStability.Evolving
@@ -647,10 +648,12 @@ public interface MRJobConfig {
/**
* Default CLASSPATH for all YARN MapReduce applications.
*/
- public static final String[] DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH = {
- "$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*",
- "$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*",
- };
+ public final String
+ DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH = Shell.WINDOWS ?
+ "%HADOOP_MAPRED_HOME%\\share\\hadoop\\mapreduce\\*,"
+ + "%HADOOP_MAPRED_HOME%\\share\\hadoop\\mapreduce\\lib\\*" :
+ "$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*,"
+ + "$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*";
public static final String WORKFLOW_ID = "mapreduce.workflow.id";
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml?rev=1529081&r1=1529080&r2=1529081&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
(original)
+++
hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
Fri Oct 4 05:51:51 2013
@@ -1458,9 +1458,20 @@
<property>
<description>CLASSPATH for MR applications. A comma-separated list
- of CLASSPATH entries</description>
+ of CLASSPATH entries. If mapreduce.application.framework is set then this
+ must specify the appropriate classpath for that archive, and the name of
+ the archive must be present in the classpath.
+ When this value is empty, the following default CLASSPATH for MR
+ applications would be used.
+ For Linux:
+ $HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*,
+ $HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*.
+ For Windows:
+ %HADOOP_MAPRED_HOME%/share/hadoop/mapreduce/*,
+ %HADOOP_MAPRED_HOME%/share/hadoop/mapreduce/lib/*.
+ </description>
<name>mapreduce.application.classpath</name>
-
<value>$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*,$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*</value>
+ <value></value>
</property>
<property>
