Author: cnauroth
Date: Wed Jul 10 23:14:09 2013
New Revision: 1502054
URL: http://svn.apache.org/r1502054
Log:
MAPREDUCE-4374. Merging change r1502048 from branch-2 to branch-2.1-beta.
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/CHANGES.txt?rev=1502054&r1=1502053&r2=1502054&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/CHANGES.txt
Wed Jul 10 23:14:09 2013
@@ -457,6 +457,9 @@ Release 2.1.0-beta - 2013-07-02
MAPREDUCE-5187. Create mapreduce command scripts on Windows. (Chuan Liu via
cnauroth)
+ MAPREDUCE-4374. Fix child task environment variable config and add support
+ for Windows. (Chuan Liu via cnauroth)
+
MAPREDUCE-5291. Change MR App to use updated property names in
container-log4j.properties. (Zhijie Shen via sseth)
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java?rev=1502054&r1=1502053&r2=1502054&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
(original)
+++
hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
Wed Jul 10 23:14:09 2013
@@ -280,12 +280,14 @@ public class JobConf extends Configurati
* Configuration key to set the environment of the child map/reduce tasks.
*
* The format of the value is <code>k1=v1,k2=v2</code>. Further it can
- * reference existing environment variables via <code>$key</code>.
+ * reference existing environment variables via <code>$key</code> on
+ * Linux or <code>%key%</code> on Windows.
*
* Example:
* <ul>
* <li> A=foo - This will set the env variable A to foo. </li>
- * <li> B=$X:c This is inherit tasktracker's X env variable. </li>
+ * <li> B=$X:c This is inherit tasktracker's X env variable on Linux. </li>
+ * <li> B=%X%;c This is inherit tasktracker's X env variable on Windows.
</li>
* </ul>
*
* @deprecated Use {@link #MAPRED_MAP_TASK_ENV} or
@@ -295,31 +297,33 @@ public class JobConf extends Configurati
public static final String MAPRED_TASK_ENV = "mapred.child.env";
/**
- * Configuration key to set the maximum virutal memory available to the
- * map tasks.
+ * Configuration key to set the environment of the child map tasks.
*
- * The format of the value is <code>k1=v1,k2=v2</code>. Further it can
- * reference existing environment variables via <code>$key</code>.
+ * The format of the value is <code>k1=v1,k2=v2</code>. Further it can
+ * reference existing environment variables via <code>$key</code> on
+ * Linux or <code>%key%</code> on Windows.
*
* Example:
* <ul>
* <li> A=foo - This will set the env variable A to foo. </li>
- * <li> B=$X:c This is inherit tasktracker's X env variable. </li>
+ * <li> B=$X:c This is inherit tasktracker's X env variable on Linux. </li>
+ * <li> B=%X%;c This is inherit tasktracker's X env variable on Windows.
</li>
* </ul>
*/
public static final String MAPRED_MAP_TASK_ENV = JobContext.MAP_ENV;
/**
- * Configuration key to set the maximum virutal memory available to the
- * reduce tasks.
+ * Configuration key to set the environment of the child reduce tasks.
*
* The format of the value is <code>k1=v1,k2=v2</code>. Further it can
- * reference existing environment variables via <code>$key</code>.
+ * reference existing environment variables via <code>$key</code> on
+ * Linux or <code>%key%</code> on Windows.
*
* Example:
* <ul>
* <li> A=foo - This will set the env variable A to foo. </li>
- * <li> B=$X:c This is inherit tasktracker's X env variable. </li>
+ * <li> B=$X:c This is inherit tasktracker's X env variable on Linux. </li>
+ * <li> B=%X%;c This is inherit tasktracker's X env variable on Windows.
</li>
* </ul>
*/
public static final String MAPRED_REDUCE_TASK_ENV = JobContext.REDUCE_ENV;
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml?rev=1502054&r1=1502053&r2=1502054&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
(original)
+++
hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
Wed Jul 10 23:14:09 2013
@@ -432,7 +432,8 @@
<description>User added environment variables for the task tracker child
processes. Example :
1) A=foo This will set the env variable A to foo
- 2) B=$B:c This is inherit tasktracker's B env variable.
+ 2) B=$B:c This is inherit nodemanager's B env variable on Unix.
+ 3) B=%B%;c This is inherit nodemanager's B env variable on Windows.
</description>
</property>
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java?rev=1502054&r1=1502053&r2=1502054&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java
(original)
+++
hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java
Wed Jul 10 23:14:09 2013
@@ -45,6 +45,7 @@ import org.apache.hadoop.mapreduce.MRJob
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.mapreduce.v2.MiniMRYarnCluster;
+import org.apache.hadoop.util.Shell;
/**
* Class to test mapred task's
@@ -172,10 +173,10 @@ public class TestMiniMRChildTask {
private static void checkEnv(String envName, String expValue, String mode) {
String envValue = System.getenv(envName).trim();
if ("append".equals(mode)) {
- if (envValue == null || !envValue.contains(":")) {
+ if (envValue == null || !envValue.contains(File.pathSeparator)) {
throw new RuntimeException("Missing env variable");
} else {
- String parts[] = envValue.split(":");
+ String parts[] = envValue.split(File.pathSeparator);
// check if the value is appended
if (!parts[parts.length - 1].equals(expValue)) {
throw new RuntimeException("Wrong env variable in append mode");
@@ -225,10 +226,10 @@ public class TestMiniMRChildTask {
// check if X=/tmp for a new env variable
checkEnv("MY_PATH", "/tmp", "noappend");
// check if X=$X:/tmp works for a new env var and results into :/tmp
- checkEnv("NEW_PATH", ":/tmp", "noappend");
+ checkEnv("NEW_PATH", File.pathSeparator + "/tmp", "noappend");
// check if X=$(tt's X var):/tmp for an old env variable inherited from
// the tt
- checkEnv("PATH", path + ":/tmp", "noappend");
+ checkEnv("PATH", path + File.pathSeparator + "/tmp", "noappend");
String jobLocalDir = job.get(MRJobConfig.JOB_LOCAL_DIR);
assertNotNull(MRJobConfig.JOB_LOCAL_DIR + " is null",
@@ -279,10 +280,10 @@ public class TestMiniMRChildTask {
// check if X=/tmp for a new env variable
checkEnv("MY_PATH", "/tmp", "noappend");
// check if X=$X:/tmp works for a new env var and results into :/tmp
- checkEnv("NEW_PATH", ":/tmp", "noappend");
+ checkEnv("NEW_PATH", File.pathSeparator + "/tmp", "noappend");
// check if X=$(tt's X var):/tmp for an old env variable inherited from
// the tt
- checkEnv("PATH", path + ":/tmp", "noappend");
+ checkEnv("PATH", path + File.pathSeparator + "/tmp", "noappend");
}
@@ -437,12 +438,18 @@ public class TestMiniMRChildTask {
mapTaskJavaOptsKey = reduceTaskJavaOptsKey =
JobConf.MAPRED_TASK_JAVA_OPTS;
mapTaskJavaOpts = reduceTaskJavaOpts = TASK_OPTS_VAL;
}
- conf.set(mapTaskEnvKey,
-
"MY_PATH=/tmp,LANG=en_us_8859_1,LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/tmp," +
- "PATH=$PATH:/tmp,NEW_PATH=$NEW_PATH:/tmp");
- conf.set(reduceTaskEnvKey,
-
"MY_PATH=/tmp,LANG=en_us_8859_1,LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/tmp," +
- "PATH=$PATH:/tmp,NEW_PATH=$NEW_PATH:/tmp");
+ conf.set(
+ mapTaskEnvKey,
+ Shell.WINDOWS ?
"MY_PATH=/tmp,LANG=en_us_8859_1,LD_LIBRARY_PATH=%LD_LIBRARY_PATH%;/tmp,"
+ + "PATH=%PATH%;/tmp,NEW_PATH=%NEW_PATH%;/tmp"
+ :
"MY_PATH=/tmp,LANG=en_us_8859_1,LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/tmp,"
+ + "PATH=$PATH:/tmp,NEW_PATH=$NEW_PATH:/tmp");
+ conf.set(
+ reduceTaskEnvKey,
+ Shell.WINDOWS ?
"MY_PATH=/tmp,LANG=en_us_8859_1,LD_LIBRARY_PATH=%LD_LIBRARY_PATH%;/tmp,"
+ + "PATH=%PATH%;/tmp,NEW_PATH=%NEW_PATH%;/tmp"
+ :
"MY_PATH=/tmp,LANG=en_us_8859_1,LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/tmp,"
+ + "PATH=$PATH:/tmp,NEW_PATH=$NEW_PATH:/tmp");
conf.set("path", System.getenv("PATH"));
conf.set(mapTaskJavaOptsKey, mapTaskJavaOpts);
conf.set(reduceTaskJavaOptsKey, reduceTaskJavaOpts);
