Author: suresh
Date: Fri May 24 21:58:04 2013
New Revision: 1486231
URL: http://svn.apache.org/r1486231
Log:
HADOOP-8562. Merge r1469667 for YARN-493 and HADOOP-9486 and r1469996 for
HADOOP-9488.
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ShellCommandFencer.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/winutils/task.c
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestNodeFencer.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1486231&r1=1486230&r2=1486231&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Fri May 24 21:58:04 2013
@@ -341,7 +341,11 @@ Release 2.0.5-beta - UNRELEASED
HADOOP-9490. LocalFileSystem#reportChecksumFailure not closing the
checksum file handle before rename. (Ivan Mitic via suresh)
-
+
+ HADOOP-9488. FileUtil#createJarWithClassPath only substitutes environment
+ variables from current process environment/does not support overriding
+ when launching new process (Chris Nauroth via bikas)
+
Release 2.0.4-beta - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ShellCommandFencer.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ShellCommandFencer.java?rev=1486231&r1=1486230&r2=1486231&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ShellCommandFencer.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ShellCommandFencer.java
Fri May 24 21:58:04 2013
@@ -26,6 +26,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.conf.Configured;
import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.util.Shell;
/**
* Fencing method that runs a shell command. It should be specified
@@ -33,8 +34,8 @@ import com.google.common.annotations.Vis
* <code>
* shell(/path/to/my/script.sh arg1 arg2 ...)
* </code><br>
- * The string between '(' and ')' is passed directly to a bash shell and
- * may not include any closing parentheses.<p>
+ * The string between '(' and ')' is passed directly to a bash shell
+ * (cmd.exe on Windows) and may not include any closing parentheses.<p>
*
* The shell command will be run with an environment set up to contain
* all of the current Hadoop configuration variables, with the '_' character
@@ -58,11 +59,11 @@ public class ShellCommandFencer
/** Prefix for target parameters added to the environment */
private static final String TARGET_PREFIX = "target_";
-
+
@VisibleForTesting
static Log LOG = LogFactory.getLog(
ShellCommandFencer.class);
-
+
@Override
public void checkArgs(String args) throws BadFencingConfigurationException {
if (args == null || args.isEmpty()) {
@@ -74,8 +75,14 @@ public class ShellCommandFencer
@Override
public boolean tryFence(HAServiceTarget target, String cmd) {
- ProcessBuilder builder = new ProcessBuilder(
- "bash", "-e", "-c", cmd);
+ ProcessBuilder builder;
+
+ if (!Shell.WINDOWS) {
+ builder = new ProcessBuilder("bash", "-e", "-c", cmd);
+ } else {
+ builder = new ProcessBuilder("cmd.exe", "/c", cmd);
+ }
+
setConfAsEnvVars(builder.environment());
addTargetInfoAsEnvVars(target, builder.environment());
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java?rev=1486231&r1=1486230&r2=1486231&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
Fri May 24 21:58:04 2013
@@ -123,6 +123,56 @@ abstract public class Shell {
: new String[] { "ln", "-s", target, link };
}
+ /** Return a command for determining if process with specified pid is alive.
*/
+ public static String[] getCheckProcessIsAliveCommand(String pid) {
+ return Shell.WINDOWS ?
+ new String[] { Shell.WINUTILS, "task", "isAlive", pid } :
+ new String[] { "kill", "-0", isSetsidAvailable ? "-" + pid : pid };
+ }
+
+ /** Return a command to send a signal to a given pid */
+ public static String[] getSignalKillCommand(int code, String pid) {
+ return Shell.WINDOWS ? new String[] { Shell.WINUTILS, "task", "kill", pid
} :
+ new String[] { "kill", "-" + code, isSetsidAvailable ? "-" + pid : pid };
+ }
+
+ /**
+ * Returns a File referencing a script with the given basename, inside the
+ * given parent directory. The file extension is inferred by platform:
".cmd"
+ * on Windows, or ".sh" otherwise.
+ *
+ * @param parent File parent directory
+ * @param basename String script file basename
+ * @return File referencing the script in the directory
+ */
+ public static File appendScriptExtension(File parent, String basename) {
+ return new File(parent, appendScriptExtension(basename));
+ }
+
+ /**
+ * Returns a script file name with the given basename. The file extension is
+ * inferred by platform: ".cmd" on Windows, or ".sh" otherwise.
+ *
+ * @param basename String script file basename
+ * @return String script file name
+ */
+ public static String appendScriptExtension(String basename) {
+ return basename + (WINDOWS ? ".cmd" : ".sh");
+ }
+
+ /**
+ * Returns a command to run the given script. The script interpreter is
+ * inferred by platform: cmd on Windows or bash otherwise.
+ *
+ * @param script File script to run
+ * @return String[] command to run the script
+ */
+ public static String[] getRunScriptCommand(File script) {
+ String absolutePath = script.getAbsolutePath();
+ return WINDOWS ? new String[] { "cmd", "/c", absolutePath } :
+ new String[] { "/bin/bash", absolutePath };
+ }
+
/** a Unix command to set permission */
public static final String SET_PERMISSION_COMMAND = "chmod";
/** a Unix command to set owner */
@@ -243,6 +293,26 @@ abstract public class Shell {
return winUtilsPath;
}
+ public static final boolean isSetsidAvailable = isSetsidSupported();
+ private static boolean isSetsidSupported() {
+ if (Shell.WINDOWS) {
+ return false;
+ }
+ ShellCommandExecutor shexec = null;
+ boolean setsidSupported = true;
+ try {
+ String[] args = {"setsid", "bash", "-c", "echo $$"};
+ shexec = new ShellCommandExecutor(args);
+ shexec.execute();
+ } catch (IOException ioe) {
+ LOG.warn("setsid is not available on this machine. So not using it.");
+ setsidSupported = false;
+ } finally { // handle the exit code
+ LOG.info("setsid exited with exit code " + shexec.getExitCode());
+ }
+ return setsidSupported;
+ }
+
/** Token separator regex used to parse Shell tool outputs */
public static final String TOKEN_SEPARATOR_REGEX
= WINDOWS ? "[|\n\r]" : "[ \t\n\r\f]";
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/winutils/task.c
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/winutils/task.c?rev=1486231&r1=1486230&r2=1486231&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/winutils/task.c
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/winutils/task.c
Fri May 24 21:58:04 2013
@@ -24,6 +24,10 @@
#define ERROR_TASK_NOT_ALIVE 1
+// This exit code for killed processes is compatible with Unix, where a killed
+// process exits with 128 + signal. For SIGKILL, this would be 128 + 9 = 137.
+#define KILLED_PROCESS_EXIT_CODE 137
+
// List of different task related command line options supported by
// winutils.
typedef enum TaskCommandOptionType
@@ -266,7 +270,7 @@ DWORD killTask(PCWSTR jobObjName)
return err;
}
- if(TerminateJobObject(jobObject, 1) == 0)
+ if(TerminateJobObject(jobObject, KILLED_PROCESS_EXIT_CODE) == 0)
{
return GetLastError();
}
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestNodeFencer.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestNodeFencer.java?rev=1486231&r1=1486230&r2=1486231&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestNodeFencer.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestNodeFencer.java
Fri May 24 21:58:04 2013
@@ -24,6 +24,7 @@ import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.util.Shell;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
@@ -33,7 +34,12 @@ import com.google.common.collect.Lists;
public class TestNodeFencer {
private HAServiceTarget MOCK_TARGET;
-
+
+ // Fencer shell commands that always return true on Unix and Windows
+ // respectively. Lacking the POSIX 'true' command on Windows, we use
+ // the batch command 'rem'.
+ private static String FENCER_TRUE_COMMAND_UNIX = "shell(true)";
+ private static String FENCER_TRUE_COMMAND_WINDOWS = "shell(rem)";
@Before
public void clearMockState() {
@@ -48,6 +54,11 @@ public class TestNodeFencer {
.when(MOCK_TARGET).getAddress();
}
+ private static String getFencerTrueCommand() {
+ return Shell.WINDOWS ?
+ FENCER_TRUE_COMMAND_WINDOWS : FENCER_TRUE_COMMAND_UNIX;
+ }
+
@Test
public void testSingleFencer() throws BadFencingConfigurationException {
NodeFencer fencer = setupFencer(
@@ -100,7 +111,7 @@ public class TestNodeFencer {
@Test
public void testShortNameShell() throws BadFencingConfigurationException {
- NodeFencer fencer = setupFencer("shell(true)");
+ NodeFencer fencer = setupFencer(getFencerTrueCommand());
assertTrue(fencer.fence(MOCK_TARGET));
}
