Author: ddas
Date: Sat Feb 16 04:37:28 2008
New Revision: 628285
URL: http://svn.apache.org/viewvc?rev=628285&view=rev
Log:
Merge -r 628283:628284 from trunk to 0.16 branch. Fixes HADOOP-2735.
Added:
hadoop/core/branches/branch-0.16/src/test/org/apache/hadoop/mapred/TestMiniMRTaskTempDir.java
- copied unchanged from r628284,
hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestMiniMRTaskTempDir.java
Modified:
hadoop/core/branches/branch-0.16/CHANGES.txt
hadoop/core/branches/branch-0.16/conf/hadoop-default.xml
hadoop/core/branches/branch-0.16/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java
hadoop/core/branches/branch-0.16/src/java/org/apache/hadoop/mapred/TaskRunner.java
hadoop/core/branches/branch-0.16/src/java/org/apache/hadoop/mapred/pipes/Application.java
Modified: hadoop/core/branches/branch-0.16/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.16/CHANGES.txt?rev=628285&r1=628284&r2=628285&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.16/CHANGES.txt (original)
+++ hadoop/core/branches/branch-0.16/CHANGES.txt Sat Feb 16 04:37:28 2008
@@ -50,6 +50,9 @@
HADOOP-2811. Dump of counters in job history does not add comma between
groups. (runping via omalley)
+ HADOOP-2735. Enables setting TMPDIR for tasks.
+ (Amareshwari Sri Ramadasu via ddas)
+
Release 0.16.0 - 2008-02-07
INCOMPATIBLE CHANGES
Modified: hadoop/core/branches/branch-0.16/conf/hadoop-default.xml
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.16/conf/hadoop-default.xml?rev=628285&r1=628284&r2=628285&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.16/conf/hadoop-default.xml (original)
+++ hadoop/core/branches/branch-0.16/conf/hadoop-default.xml Sat Feb 16
04:37:28 2008
@@ -682,6 +682,18 @@
</property>
<property>
+ <name>mapred.child.tmp</name>
+ <value>./tmp</value>
+ <description> To set the value of tmp directory for map and reduce tasks.
+ If the value is an absolute path, it is directly assigned. Otherwise, it is
+ prepended with task's working directory. The java tasks are executed with
+ option -Djava.io.tmpdir='the absolute path of the tmp dir'. Pipes and
+ streaming are set with environment variable,
+ TMPDIR='the absolute path of the tmp dir'
+ </description>
+</property>
+
+<property>
<name>mapred.inmem.merge.threshold</name>
<value>1000</value>
<description>The threshold, in terms of the number of files
Modified:
hadoop/core/branches/branch-0.16/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.16/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java?rev=628285&r1=628284&r2=628285&view=diff
==============================================================================
---
hadoop/core/branches/branch-0.16/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java
(original)
+++
hadoop/core/branches/branch-0.16/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java
Sat Feb 16 04:37:28 2008
@@ -161,6 +161,8 @@
Environment childEnv = (Environment) StreamUtil.env().clone();
addJobConfToEnvironment(job_, childEnv);
addEnvironment(childEnv, job_.get("stream.addenvironment"));
+ // add TMPDIR environment variable with the value of java.io.tmpdir
+ envPut(childEnv, "TMPDIR", System.getProperty("java.io.tmpdir"));
sim = Runtime.getRuntime().exec(argvSplit, childEnv.toArray());
/* // This way required jdk1.5
Modified:
hadoop/core/branches/branch-0.16/src/java/org/apache/hadoop/mapred/TaskRunner.java
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.16/src/java/org/apache/hadoop/mapred/TaskRunner.java?rev=628285&r1=628284&r2=628285&view=diff
==============================================================================
---
hadoop/core/branches/branch-0.16/src/java/org/apache/hadoop/mapred/TaskRunner.java
(original)
+++
hadoop/core/branches/branch-0.16/src/java/org/apache/hadoop/mapred/TaskRunner.java
Sat Feb 16 04:37:28 2008
@@ -323,6 +323,21 @@
vargs.add(javaOptsSplit[i]);
}
+ // add java.io.tmpdir given by mapred.child.tmp
+ String tmp = conf.get("mapred.child.tmp", "./tmp");
+ Path tmpDir = new Path(tmp);
+
+ // if temp directory path is not absolute
+ // prepend it with workDir.
+ if (!tmpDir.isAbsolute()) {
+ tmpDir = new Path(workDir.toString(), tmp);
+ }
+ FileSystem localFs = FileSystem.getLocal(conf);
+ if (!localFs.mkdirs(tmpDir) && !localFs.getFileStatus(tmpDir).isDir()) {
+ throw new IOException("Mkdirs failed to create " + tmpDir.toString());
+ }
+ vargs.add("-Djava.io.tmpdir=" + tmpDir.toString());
+
// Add classpath.
vargs.add("-classpath");
vargs.add(classPath.toString());
Modified:
hadoop/core/branches/branch-0.16/src/java/org/apache/hadoop/mapred/pipes/Application.java
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.16/src/java/org/apache/hadoop/mapred/pipes/Application.java?rev=628285&r1=628284&r2=628285&view=diff
==============================================================================
---
hadoop/core/branches/branch-0.16/src/java/org/apache/hadoop/mapred/pipes/Application.java
(original)
+++
hadoop/core/branches/branch-0.16/src/java/org/apache/hadoop/mapred/pipes/Application.java
Sat Feb 16 04:37:28 2008
@@ -69,6 +69,8 @@
) throws IOException, InterruptedException {
serverSocket = new ServerSocket(0);
Map<String, String> env = new HashMap<String,String>();
+ // add TMPDIR environment variable with the value of java.io.tmpdir
+ env.put("TMPDIR", System.getProperty("java.io.tmpdir"));
env.put("hadoop.pipes.command.port",
Integer.toString(serverSocket.getLocalPort()));
List<String> cmd = new ArrayList<String>();
