Author: suresh
Date: Fri May 24 16:03:32 2013
New Revision: 1486101
URL: http://svn.apache.org/r1486101
Log:
HADOOP-8562. Merge r1469996 for HADOOP-9488, r1469998 for YARN-593
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1486101&r1=1486100&r2=1486101&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Fri May 24 16:03:32 2013
@@ -329,6 +329,10 @@ Release 2.0.5-beta - UNRELEASED
HADOOP-9437. TestNativeIO#testRenameTo fails on Windows due to assumption
that POSIX errno is embedded in NativeIOException. (Chris Nauroth via
suresh)
+
+ HADOOP-9488. FileUtil#createJarWithClassPath only substitutes environment
+ variables from current process environment/does not support overriding
+ when launching new process (Chris Nauroth via bikas)
Release 2.0.4-beta - UNRELEASED
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java?rev=1486101&r1=1486100&r2=1486101&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
Fri May 24 16:03:32 2013
@@ -967,15 +967,17 @@ public class FileUtil {
*
* @param inputClassPath String input classpath to bundle into the jar
manifest
* @param pwd Path to working directory to save jar
+ * @param callerEnv Map<String, String> caller's environment variables to use
+ * for expansion
* @return String absolute path to new jar
* @throws IOException if there is an I/O error while writing the jar file
*/
- public static String createJarWithClassPath(String inputClassPath, Path pwd)
- throws IOException {
+ public static String createJarWithClassPath(String inputClassPath, Path pwd,
+ Map<String, String> callerEnv) throws IOException {
// Replace environment variables, case-insensitive on Windows
@SuppressWarnings("unchecked")
- Map<String, String> env = Shell.WINDOWS ?
- new CaseInsensitiveMap(System.getenv()) : System.getenv();
+ Map<String, String> env = Shell.WINDOWS ? new
CaseInsensitiveMap(callerEnv) :
+ callerEnv;
String[] classPathEntries = inputClassPath.split(File.pathSeparator);
for (int i = 0; i < classPathEntries.length; ++i) {
classPathEntries[i] = StringUtils.replaceTokens(classPathEntries[i],
@@ -1006,9 +1008,22 @@ public class FileUtil {
}
}
} else {
- // Append just this jar
- classPathEntryList.add(new File(classPathEntry).toURI().toURL()
- .toExternalForm());
+ // Append just this entry
+ String classPathEntryUrl = new File(classPathEntry).toURI().toURL()
+ .toExternalForm();
+
+ // File.toURI only appends trailing '/' if it can determine that it is
a
+ // directory that already exists. (See JavaDocs.) If this entry had a
+ // trailing '/' specified by the caller, then guarantee that the
+ // classpath entry in the manifest has a trailing '/', and thus refers
to
+ // a directory instead of a file. This can happen if the caller is
+ // creating a classpath jar referencing a directory that hasn't been
+ // created yet, but will definitely be created before running.
+ if (classPathEntry.endsWith(Path.SEPARATOR) &&
+ !classPathEntryUrl.endsWith(Path.SEPARATOR)) {
+ classPathEntryUrl = classPathEntryUrl + Path.SEPARATOR;
+ }
+ classPathEntryList.add(classPathEntryUrl);
}
}
String jarClassPath = StringUtils.join(" ", classPathEntryList);
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java?rev=1486101&r1=1486100&r2=1486101&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
Fri May 24 16:03:32 2013
@@ -585,11 +585,13 @@ public class TestFileUtil {
// create classpath jar
String wildcardPath = tmp.getCanonicalPath() + File.separator + "*";
+ String nonExistentSubdir = tmp.getCanonicalPath() + Path.SEPARATOR +
"subdir"
+ + Path.SEPARATOR;
List<String> classPaths = Arrays.asList("cp1.jar", "cp2.jar", wildcardPath,
- "cp3.jar");
+ "cp3.jar", nonExistentSubdir);
String inputClassPath = StringUtils.join(File.pathSeparator, classPaths);
String classPathJar = FileUtil.createJarWithClassPath(inputClassPath,
- new Path(tmp.getCanonicalPath()));
+ new Path(tmp.getCanonicalPath()), System.getenv());
// verify classpath by reading manifest from jar file
JarFile jarFile = null;
@@ -604,15 +606,20 @@ public class TestFileUtil {
Assert.assertNotNull(classPathAttr);
List<String> expectedClassPaths = new ArrayList<String>();
for (String classPath: classPaths) {
- if (!wildcardPath.equals(classPath)) {
- expectedClassPaths.add(new File(classPath).toURI().toURL()
- .toExternalForm());
- } else {
+ if (wildcardPath.equals(classPath)) {
// add wildcard matches
for (File wildcardMatch: wildcardMatches) {
expectedClassPaths.add(wildcardMatch.toURI().toURL()
.toExternalForm());
}
+ } else if (nonExistentSubdir.equals(classPath)) {
+ // expect to maintain trailing path separator if present in input,
even
+ // if directory doesn't exist yet
+ expectedClassPaths.add(new File(classPath).toURI().toURL()
+ .toExternalForm() + Path.SEPARATOR);
+ } else {
+ expectedClassPaths.add(new File(classPath).toURI().toURL()
+ .toExternalForm());
}
}
List<String> actualClassPaths = Arrays.asList(classPathAttr.split(" "));
