Author: ddas
Date: Thu Mar 26 10:24:22 2009
New Revision: 758584
URL: http://svn.apache.org/viewvc?rev=758584&view=rev
Log:
HADOOP-5194. Disables setsid for tasks run on cygwin. Contributed by Ravi
Gummadi.
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/mapred/mapred-default.xml
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Child.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JvmManager.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Task.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskLog.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskRunner.java
hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestKillSubProcesses.java
Modified: hadoop/core/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=758584&r1=758583&r2=758584&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Thu Mar 26 10:24:22 2009
@@ -333,6 +333,9 @@
HADOOP-5477. Fix rare failure in TestCLI for hosts returning variations of
'localhost'. (Jacob Homan via cdouglas)
+ HADOOP-5194. Disables setsid for tasks run on cygwin.
+ (Ravi Gummadi via ddas)
+
Release 0.20.0 - Unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/core/trunk/src/mapred/mapred-default.xml
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/mapred-default.xml?rev=758584&r1=758583&r2=758584&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/mapred-default.xml (original)
+++ hadoop/core/trunk/src/mapred/mapred-default.xml Thu Mar 26 10:24:22 2009
@@ -315,7 +315,8 @@
<name>mapred.tasktracker.tasks.sleeptime-before-sigkill</name>
<value>5000</value>
<description>The time, in milliseconds, the tasktracker waits for sending a
- SIGKILL to a process, after it has been sent a SIGTERM.
+ SIGKILL to a task, after it has been sent a SIGTERM. This is currently
+ not used on WINDOWS where tasks are just sent a SIGTERM.
</description>
</property>
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Child.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Child.java?rev=758584&r1=758583&r2=758584&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Child.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Child.java Thu Mar 26
10:24:22 2009
@@ -35,6 +35,7 @@
import org.apache.hadoop.metrics.MetricsContext;
import org.apache.hadoop.metrics.MetricsUtil;
import org.apache.hadoop.metrics.jvm.JvmMetrics;
+import org.apache.hadoop.util.Shell;
import org.apache.log4j.LogManager;
/**
@@ -123,13 +124,13 @@
//are viewable immediately
TaskLog.syncLogs(firstTaskid, taskid, isCleanup);
JobConf job = new JobConf(task.getJobFile());
- if (srcPidPath == null) {
+ if (srcPidPath == null && !Shell.WINDOWS) {
// get the first task's path for the first time
srcPidPath = new Path(task.getPidFile());
}
//since the JVM is running multiple tasks potentially, we need
//to do symlink stuff only for the subsequent tasks
- if (!taskid.equals(firstTaskid)) {
+ if (!taskid.equals(firstTaskid) && !Shell.WINDOWS) {
dstPidPath = new Path(task.getPidFile());
FileUtil.symLink(srcPidPath.toUri().getPath(),
dstPidPath.toUri().getPath());
@@ -155,7 +156,7 @@
task.run(job, umbilical); // run the task
} finally {
TaskLog.syncLogs(firstTaskid, taskid, isCleanup);
- if (!taskid.equals(firstTaskid)) {
+ if (!taskid.equals(firstTaskid) && !Shell.WINDOWS) {
// delete the pid-file's symlink
new File(dstPidPath.toUri().getPath()).delete();
}
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JvmManager.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JvmManager.java?rev=758584&r1=758583&r2=758584&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JvmManager.java
(original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JvmManager.java Thu
Mar 26 10:24:22 2009
@@ -33,6 +33,7 @@
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.TaskTracker.TaskInProgress;
+import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.Shell.ShellCommandExecutor;
import org.apache.hadoop.util.ProcessTree;
@@ -366,19 +367,24 @@
if (shexec != null) {
Process process = shexec.getProcess();
if (process != null) {
- Path pidFilePath = new Path(env.pidFile);
- String pid = ProcessTree.getPidFromPidFile(
+ if (Shell.WINDOWS) {
+ process.destroy();
+ }
+ else {
+ Path pidFilePath = new Path(env.pidFile);
+ String pid = ProcessTree.getPidFromPidFile(
pidFilePath.toString());
- long sleeptimeBeforeSigkill = env.conf.getLong(
- "mapred.tasktracker.tasks.sleeptime-before-sigkill",
- ProcessTree.DEFAULT_SLEEPTIME_BEFORE_SIGKILL);
-
- ProcessTree.destroy(pid, sleeptimeBeforeSigkill,
- ProcessTree.isSetsidAvailable, false);
- try {
- LOG.info("Process exited with exit code:" + process.waitFor());
- } catch (InterruptedException ie) {}
+ long sleeptimeBeforeSigkill = env.conf.getLong(
+ "mapred.tasktracker.tasks.sleeptime-before-sigkill",
+ ProcessTree.DEFAULT_SLEEPTIME_BEFORE_SIGKILL);
+
+ ProcessTree.destroy(pid, sleeptimeBeforeSigkill,
+ ProcessTree.isSetsidAvailable, false);
+ try {
+ LOG.info("Process exited with exit code:" + process.waitFor());
+ } catch (InterruptedException ie) {}
+ }
}
}
removeJvm(jvmId);
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Task.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Task.java?rev=758584&r1=758583&r2=758584&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Task.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Task.java Thu Mar 26
10:24:22 2009
@@ -50,6 +50,7 @@
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.Shell;
/** Base class for tasks. */
abstract class Task implements Writable, Configurable {
@@ -313,7 +314,9 @@
out.writeBoolean(jobSetup);
out.writeBoolean(writeSkipRecs);
out.writeBoolean(taskCleanup);
- Text.writeString(out, pidFile);
+ if(!Shell.WINDOWS) {
+ Text.writeString(out, pidFile);
+ }
}
public void readFields(DataInput in) throws IOException {
@@ -333,7 +336,9 @@
if (taskCleanup) {
setPhase(TaskStatus.Phase.CLEANUP);
}
- pidFile = Text.readString(in);
+ if(!Shell.WINDOWS) {
+ pidFile = Text.readString(in);
+ }
}
@Override
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskLog.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskLog.java?rev=758584&r1=758583&r2=758584&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskLog.java
(original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskLog.java Thu Mar
26 10:24:22 2009
@@ -35,6 +35,7 @@
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.util.ProcessTree;
+import org.apache.hadoop.util.Shell;
import org.apache.log4j.Appender;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
@@ -461,7 +462,7 @@
StringBuffer mergedCmd = new StringBuffer();
// Spit out the pid to pidFileName
- if (pidFileName != null) {
+ if (pidFileName != null && !Shell.WINDOWS) {
mergedCmd.append("echo $$ > ");
mergedCmd.append(pidFileName);
mergedCmd.append(" ;");
@@ -473,7 +474,8 @@
}
if (tailLength > 0) {
mergedCmd.append("(");
- } else if(ProcessTree.isSetsidAvailable && useSetsid) {
+ } else if(ProcessTree.isSetsidAvailable && useSetsid &&
+ !Shell.WINDOWS) {
mergedCmd.append("exec setsid ");
} else {
mergedCmd.append("exec ");
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskRunner.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskRunner.java?rev=758584&r1=758583&r2=758584&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskRunner.java
(original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskRunner.java Thu
Mar 26 10:24:22 2009
@@ -368,11 +368,15 @@
vargs.add(Integer.toString(address.getPort()));
vargs.add(taskid.toString()); // pass task
identifier
- String pidFile = lDirAlloc.getLocalPathForWrite(
+ String pidFile = "";
+ if (!Shell.WINDOWS) {
+ pidFile = lDirAlloc.getLocalPathForWrite(
(TaskTracker.getPidFile(t.getJobID().toString(),
t.getTaskID().toString(), t.isTaskCleanupTask())),
this.conf).toString();
- t.setPidFile(pidFile);
+ t.setPidFile(pidFile);
+ }
+
tracker.addToMemoryManager(t.getTaskID(), conf, pidFile);
// set memory limit using ulimit if feasible and necessary ...
Modified:
hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestKillSubProcesses.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestKillSubProcesses.java?rev=758584&r1=758583&r2=758584&view=diff
==============================================================================
---
hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestKillSubProcesses.java
(original)
+++
hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestKillSubProcesses.java
Thu Mar 26 10:24:22 2009
@@ -35,6 +35,7 @@
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.ProcessTree;
+import org.apache.hadoop.util.Shell;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -87,7 +88,6 @@
JobInProgress jip = jt.getJob(job.getID());
for(TaskReport tr : mapReports) {
TaskInProgress tip = jip.getTaskInProgress(tr.getTaskID());
- assertTrue(tip.isRunning());
// for this tip, get active tasks of all attempts
while(tip.getActiveTasks().size() == 0) {
@@ -240,6 +240,12 @@
}
public void testJobKill() throws IOException {
+ if (Shell.WINDOWS) {
+ System.out.println(
+ "setsid doesn't work on WINDOWS as expected. Not testing");
+ return;
+ }
+
JobConf conf=null;
try {
mr = new MiniMRCluster(1, "file:///", 1);
