Author: arp Date: Fri Aug 23 20:57:00 2013 New Revision: 1517030 URL: http://svn.apache.org/r1517030 Log: Merge all changes from trunk to branch HDFS-2832
Modified: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt (contents, props changed) hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/ (props changed) hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CommandExecutor.java hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-nfs/pom.xml Modified: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1517030&r1=1517029&r2=1517030&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt Fri Aug 23 20:57:00 2013 @@ -420,6 +420,9 @@ Release 2.1.1-beta - UNRELEASED HADOOP-9880. SASL changes from HADOOP-9421 breaks Secure HA NN. (daryn via jing9) + HADOOP-9887. globStatus does not correctly handle paths starting with a drive + spec on Windows. (Chuan Liu via cnauroth) + Release 2.1.0-beta - 2013-08-22 INCOMPATIBLE CHANGES Propchange: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:r1516229-1517028 Propchange: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1516229-1517028 Modified: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java?rev=1517030&r1=1517029&r2=1517030&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java Fri Aug 23 20:57:00 2013 @@ -97,7 +97,7 @@ class Globber { /** * Translate an absolute path into a list of path components. * We merge double slashes into a single slash here. - * The first path component (i.e. root) does not get an entry in the list. + * POSIX root path, i.e. '/', does not get an entry in the list. */ private static List<String> getPathComponents(String path) throws IOException { @@ -167,8 +167,8 @@ class Globber { // Get the absolute path for this flattened pattern. We couldn't do // this prior to flattening because of patterns like {/,a}, where which // path you go down influences how the path must be made absolute. - Path absPattern = - fixRelativePart(new Path(flatPattern .isEmpty() ? "." : flatPattern )); + Path absPattern = fixRelativePart(new Path( + flatPattern.isEmpty() ? Path.CUR_DIR : flatPattern)); // Now we break the flattened, absolute pattern into path components. // For example, /a/*/c would be broken into the list [a, *, c] List<String> components = @@ -176,9 +176,19 @@ class Globber { // Starting out at the root of the filesystem, we try to match // filesystem entries against pattern components. ArrayList<FileStatus> candidates = new ArrayList<FileStatus>(1); - candidates.add(new FileStatus(0, true, 0, 0, 0, - new Path(scheme, authority, "/"))); - + if (Path.WINDOWS && !components.isEmpty() + && Path.isWindowsAbsolutePath(absPattern.toUri().getPath(), true)) { + // On Windows the path could begin with a drive letter, e.g. /E:/foo. + // We will skip matching the drive letter and start from listing the + // root of the filesystem on that drive. + String driveLetter = components.remove(0); + candidates.add(new FileStatus(0, true, 0, 0, 0, new Path(scheme, + authority, Path.SEPARATOR + driveLetter + Path.SEPARATOR))); + } else { + candidates.add(new FileStatus(0, true, 0, 0, 0, + new Path(scheme, authority, Path.SEPARATOR))); + } + for (String component : components) { ArrayList<FileStatus> newCandidates = new ArrayList<FileStatus>(candidates.size()); Modified: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java?rev=1517030&r1=1517029&r2=1517030&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java Fri Aug 23 20:57:00 2013 @@ -106,10 +106,12 @@ public class PathData implements Compara /** * Validates the given Windows path. - * Throws IOException on failure. * @param pathString a String of the path suppliued by the user. + * @return true if the URI scheme was not present in the pathString but + * inferred; false, otherwise. + * @throws IOException if anything goes wrong */ - private void ValidateWindowsPath(String pathString) + private static boolean checkIfSchemeInferredFromPath(String pathString) throws IOException { if (windowsNonUriAbsolutePath1.matcher(pathString).find()) { @@ -118,23 +120,21 @@ public class PathData implements Compara throw new IOException("Invalid path string " + pathString); } - inferredSchemeFromPath = true; - return; + return true; } // Is it a forward slash-separated absolute path? if (windowsNonUriAbsolutePath2.matcher(pathString).find()) { - inferredSchemeFromPath = true; - return; + return true; } // Does it look like a URI? If so then just leave it alone. if (potentialUri.matcher(pathString).find()) { - return; + return false; } // Looks like a relative path on Windows. - return; + return false; } /** @@ -153,7 +153,7 @@ public class PathData implements Compara setStat(stat); if (Path.WINDOWS) { - ValidateWindowsPath(pathString); + inferredSchemeFromPath = checkIfSchemeInferredFromPath(pathString); } } @@ -302,7 +302,7 @@ public class PathData implements Compara // check getPath() so scheme slashes aren't considered part of the path String separator = uri.getPath().endsWith(Path.SEPARATOR) ? "" : Path.SEPARATOR; - return uri + separator + basename; + return uriToString(uri, inferredSchemeFromPath) + separator + basename; } protected enum PathType { HAS_SCHEME, SCHEMELESS_ABSOLUTE, RELATIVE }; @@ -356,7 +356,7 @@ public class PathData implements Compara if (globUri.getAuthority() == null) { matchUri = removeAuthority(matchUri); } - globMatch = matchUri.toString(); + globMatch = uriToString(matchUri, false); break; case SCHEMELESS_ABSOLUTE: // take just the uri's path globMatch = matchUri.getPath(); @@ -438,6 +438,10 @@ public class PathData implements Compara */ @Override public String toString() { + return uriToString(uri, inferredSchemeFromPath); + } + + private static String uriToString(URI uri, boolean inferredSchemeFromPath) { String scheme = uri.getScheme(); // No interpretation of symbols. Just decode % escaped chars. String decodedRemainder = uri.getSchemeSpecificPart(); Modified: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CommandExecutor.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CommandExecutor.java?rev=1517030&r1=1517029&r2=1517030&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CommandExecutor.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CommandExecutor.java Fri Aug 23 20:57:00 2013 @@ -24,6 +24,9 @@ import java.io.ByteArrayOutputStream; import java.io.File; import java.io.PrintStream; import java.util.StringTokenizer; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.ArrayList; /** * @@ -32,23 +35,31 @@ import java.util.StringTokenizer; public abstract class CommandExecutor { protected String[] getCommandAsArgs(final String cmd, final String masterKey, final String master) { - StringTokenizer tokenizer = new StringTokenizer(cmd, " "); - String[] args = new String[tokenizer.countTokens()]; - - int i = 0; - while (tokenizer.hasMoreTokens()) { - args[i] = tokenizer.nextToken(); - - args[i] = args[i].replaceAll(masterKey, master); - args[i] = args[i].replaceAll("CLITEST_DATA", - new File(CLITestHelper.TEST_CACHE_DATA_DIR). - toURI().toString().replace(' ', '+')); - args[i] = args[i].replaceAll("USERNAME", System.getProperty("user.name")); + String regex = "\'([^\']*)\'|\"([^\"]*)\"|(\\S+)"; + Matcher matcher = Pattern.compile(regex).matcher(cmd); - i++; - } - - return args; + ArrayList<String> args = new ArrayList<String>(); + String arg = null; + + while (matcher.find()) { + if (matcher.group(1) != null) { + arg = matcher.group(1); + } else if (matcher.group(2) != null) { + arg = matcher.group(2); + } else { + arg = matcher.group(3); + } + + arg = arg.replaceAll(masterKey, master); + arg = arg.replaceAll("CLITEST_DATA", + new File(CLITestHelper.TEST_CACHE_DATA_DIR). + toURI().toString().replace(' ', '+')); + arg = arg.replaceAll("USERNAME", System.getProperty("user.name")); + + args.add(arg); + } + + return args.toArray(new String[0]); } public Result executeCommand(final String cmd) throws Exception { Modified: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-nfs/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-nfs/pom.xml?rev=1517030&r1=1517029&r2=1517030&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-nfs/pom.xml (original) +++ hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-nfs/pom.xml Fri Aug 23 20:57:00 2013 @@ -95,4 +95,50 @@ <version>11.0.2</version> </dependency> </dependencies> + + + <profiles> + <profile> + <id>dist</id> + <activation> + <activeByDefault>false</activeByDefault> + </activation> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-assembly-plugin</artifactId> + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-assemblies</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> + <executions> + <execution> + <id>dist</id> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + <configuration> + <finalName>${project.artifactId}-${project.version}</finalName> + <appendAssemblyId>false</appendAssemblyId> + <attach>false</attach> + <!--<descriptorRefs> + <descriptorRef>hadoop-nfs-dist</descriptorRef> + </descriptorRefs>--> + <descriptors> + <descriptor>../../hadoop-assemblies/src/main/resources/assemblies/hadoop-nfs-dist.xml</descriptor> + </descriptors> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + </profiles> + </project>