Author: suresh
Date: Sat Apr 20 20:22:21 2013
New Revision: 1470225
URL: http://svn.apache.org/r1470225
Log:
Merge trunk to HDFS-2802 branch. This involves fixing many conflict with
HDFS-4434.
Modified:
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/CHANGES.txt
(contents, props changed)
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/docs/
(props changed)
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/
(props changed)
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/winutils/task.c
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/core/
(props changed)
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
Modified:
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1470225&r1=1470224&r2=1470225&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/CHANGES.txt
Sat Apr 20 20:22:21 2013
@@ -161,7 +161,10 @@ Trunk (Unreleased)
HADOOP-9218 Document the Rpc-wrappers used internally (sanjay Radia)
- HADOOP-9258 Add stricter tests to FileSystemContractTestBase (stevel)
+ HADOOP-9258 Add stricter tests to FileSystemContractTestBase (stevel)
+
+ HADOOP-9486. Promoted Windows and Shell related utils from YARN to Hadoop
+ Common. (Chris Nauroth via vinodkv)
BUG FIXES
@@ -367,6 +370,10 @@ Trunk (Unreleased)
HADOOP-9433 TestLocalFileSystem#testHasFileDescriptor leaks file handle
(Chris Nauroth via sanjay)
+ HADOOP-9488. FileUtil#createJarWithClassPath only substitutes environment
+ variables from current process environment/does not support overriding
+ when launching new process (Chris Nauroth via bikas)
+
OPTIMIZATIONS
HADOOP-7761. Improve the performance of raw comparisons. (todd)
@@ -1641,6 +1648,9 @@ Release 0.23.8 - UNRELEASED
HADOOP-9233. Cover package org.apache.hadoop.io.compress.zlib with unit
tests (Vadim Bondarev via jlowe)
+ HADOOP-9469. mapreduce/yarn source jars not included in dist tarball
+ (Robert Parker via tgraves)
+
Release 0.23.7 - UNRELEASED
INCOMPATIBLE CHANGES
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/CHANGES.txt
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:r1469644-1470194
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/docs/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs:r1469644-1470194
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1469644-1470194
Modified:
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java?rev=1470225&r1=1470224&r2=1470225&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
Sat Apr 20 20:22:21 2013
@@ -1039,15 +1039,17 @@ public class FileUtil {
*
* @param inputClassPath String input classpath to bundle into the jar
manifest
* @param pwd Path to working directory to save jar
+ * @param callerEnv Map<String, String> caller's environment variables to use
+ * for expansion
* @return String absolute path to new jar
* @throws IOException if there is an I/O error while writing the jar file
*/
- public static String createJarWithClassPath(String inputClassPath, Path pwd)
- throws IOException {
+ public static String createJarWithClassPath(String inputClassPath, Path pwd,
+ Map<String, String> callerEnv) throws IOException {
// Replace environment variables, case-insensitive on Windows
@SuppressWarnings("unchecked")
- Map<String, String> env = Shell.WINDOWS ?
- new CaseInsensitiveMap(System.getenv()) : System.getenv();
+ Map<String, String> env = Shell.WINDOWS ? new
CaseInsensitiveMap(callerEnv) :
+ callerEnv;
String[] classPathEntries = inputClassPath.split(File.pathSeparator);
for (int i = 0; i < classPathEntries.length; ++i) {
classPathEntries[i] = StringUtils.replaceTokens(classPathEntries[i],
@@ -1078,9 +1080,22 @@ public class FileUtil {
}
}
} else {
- // Append just this jar
- classPathEntryList.add(new File(classPathEntry).toURI().toURL()
- .toExternalForm());
+ // Append just this entry
+ String classPathEntryUrl = new File(classPathEntry).toURI().toURL()
+ .toExternalForm();
+
+ // File.toURI only appends trailing '/' if it can determine that it is
a
+ // directory that already exists. (See JavaDocs.) If this entry had a
+ // trailing '/' specified by the caller, then guarantee that the
+ // classpath entry in the manifest has a trailing '/', and thus refers
to
+ // a directory instead of a file. This can happen if the caller is
+ // creating a classpath jar referencing a directory that hasn't been
+ // created yet, but will definitely be created before running.
+ if (classPathEntry.endsWith(Path.SEPARATOR) &&
+ !classPathEntryUrl.endsWith(Path.SEPARATOR)) {
+ classPathEntryUrl = classPathEntryUrl + Path.SEPARATOR;
+ }
+ classPathEntryList.add(classPathEntryUrl);
}
}
String jarClassPath = StringUtils.join(" ", classPathEntryList);
Modified:
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java?rev=1470225&r1=1470224&r2=1470225&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
Sat Apr 20 20:22:21 2013
@@ -123,6 +123,56 @@ abstract public class Shell {
: new String[] { "ln", "-s", target, link };
}
+ /** Return a command for determining if process with specified pid is alive.
*/
+ public static String[] getCheckProcessIsAliveCommand(String pid) {
+ return Shell.WINDOWS ?
+ new String[] { Shell.WINUTILS, "task", "isAlive", pid } :
+ new String[] { "kill", "-0", isSetsidAvailable ? "-" + pid : pid };
+ }
+
+ /** Return a command to send a signal to a given pid */
+ public static String[] getSignalKillCommand(int code, String pid) {
+ return Shell.WINDOWS ? new String[] { Shell.WINUTILS, "task", "kill", pid
} :
+ new String[] { "kill", "-" + code, isSetsidAvailable ? "-" + pid : pid };
+ }
+
+ /**
+ * Returns a File referencing a script with the given basename, inside the
+ * given parent directory. The file extension is inferred by platform:
".cmd"
+ * on Windows, or ".sh" otherwise.
+ *
+ * @param parent File parent directory
+ * @param basename String script file basename
+ * @return File referencing the script in the directory
+ */
+ public static File appendScriptExtension(File parent, String basename) {
+ return new File(parent, appendScriptExtension(basename));
+ }
+
+ /**
+ * Returns a script file name with the given basename. The file extension is
+ * inferred by platform: ".cmd" on Windows, or ".sh" otherwise.
+ *
+ * @param basename String script file basename
+ * @return String script file name
+ */
+ public static String appendScriptExtension(String basename) {
+ return basename + (WINDOWS ? ".cmd" : ".sh");
+ }
+
+ /**
+ * Returns a command to run the given script. The script interpreter is
+ * inferred by platform: cmd on Windows or bash otherwise.
+ *
+ * @param script File script to run
+ * @return String[] command to run the script
+ */
+ public static String[] getRunScriptCommand(File script) {
+ String absolutePath = script.getAbsolutePath();
+ return WINDOWS ? new String[] { "cmd", "/c", absolutePath } :
+ new String[] { "/bin/bash", absolutePath };
+ }
+
/** a Unix command to set permission */
public static final String SET_PERMISSION_COMMAND = "chmod";
/** a Unix command to set owner */
@@ -243,6 +293,26 @@ abstract public class Shell {
return winUtilsPath;
}
+ public static final boolean isSetsidAvailable = isSetsidSupported();
+ private static boolean isSetsidSupported() {
+ if (Shell.WINDOWS) {
+ return false;
+ }
+ ShellCommandExecutor shexec = null;
+ boolean setsidSupported = true;
+ try {
+ String[] args = {"setsid", "bash", "-c", "echo $$"};
+ shexec = new ShellCommandExecutor(args);
+ shexec.execute();
+ } catch (IOException ioe) {
+ LOG.warn("setsid is not available on this machine. So not using it.");
+ setsidSupported = false;
+ } finally { // handle the exit code
+ LOG.info("setsid exited with exit code " + shexec.getExitCode());
+ }
+ return setsidSupported;
+ }
+
/** Token separator regex used to parse Shell tool outputs */
public static final String TOKEN_SEPARATOR_REGEX
= WINDOWS ? "[|\n\r]" : "[ \t\n\r\f]";
Modified:
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/winutils/task.c
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/winutils/task.c?rev=1470225&r1=1470224&r2=1470225&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/winutils/task.c
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/winutils/task.c
Sat Apr 20 20:22:21 2013
@@ -24,6 +24,10 @@
#define ERROR_TASK_NOT_ALIVE 1
+// This exit code for killed processes is compatible with Unix, where a killed
+// process exits with 128 + signal. For SIGKILL, this would be 128 + 9 = 137.
+#define KILLED_PROCESS_EXIT_CODE 137
+
// List of different task related command line options supported by
// winutils.
typedef enum TaskCommandOptionType
@@ -264,7 +268,7 @@ DWORD killTask(_TCHAR* jobObjName)
return err;
}
- if(TerminateJobObject(jobObject, 1) == 0)
+ if(TerminateJobObject(jobObject, KILLED_PROCESS_EXIT_CODE) == 0)
{
return GetLastError();
}
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/core/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/core:r1469644-1470194
Modified:
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java?rev=1470225&r1=1470224&r2=1470225&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
Sat Apr 20 20:22:21 2013
@@ -755,11 +755,13 @@ public class TestFileUtil {
// create classpath jar
String wildcardPath = tmp.getCanonicalPath() + File.separator + "*";
+ String nonExistentSubdir = tmp.getCanonicalPath() + Path.SEPARATOR +
"subdir"
+ + Path.SEPARATOR;
List<String> classPaths = Arrays.asList("cp1.jar", "cp2.jar", wildcardPath,
- "cp3.jar");
+ "cp3.jar", nonExistentSubdir);
String inputClassPath = StringUtils.join(File.pathSeparator, classPaths);
String classPathJar = FileUtil.createJarWithClassPath(inputClassPath,
- new Path(tmp.getCanonicalPath()));
+ new Path(tmp.getCanonicalPath()), System.getenv());
// verify classpath by reading manifest from jar file
JarFile jarFile = null;
@@ -774,15 +776,20 @@ public class TestFileUtil {
Assert.assertNotNull(classPathAttr);
List<String> expectedClassPaths = new ArrayList<String>();
for (String classPath: classPaths) {
- if (!wildcardPath.equals(classPath)) {
- expectedClassPaths.add(new File(classPath).toURI().toURL()
- .toExternalForm());
- } else {
+ if (wildcardPath.equals(classPath)) {
// add wildcard matches
for (File wildcardMatch: wildcardMatches) {
expectedClassPaths.add(wildcardMatch.toURI().toURL()
.toExternalForm());
}
+ } else if (nonExistentSubdir.equals(classPath)) {
+ // expect to maintain trailing path separator if present in input,
even
+ // if directory doesn't exist yet
+ expectedClassPaths.add(new File(classPath).toURI().toURL()
+ .toExternalForm() + Path.SEPARATOR);
+ } else {
+ expectedClassPaths.add(new File(classPath).toURI().toURL()
+ .toExternalForm());
}
}
List<String> actualClassPaths = Arrays.asList(classPathAttr.split(" "));
