Author: jlowe Date: Tue Jul 30 14:42:59 2013 New Revision: 1508471 URL: http://svn.apache.org/r1508471 Log: Revert HADOOP-9652 from branch-2
Removed: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Stat.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestStat.java Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestSymlinkLocalFS.java Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1508471&r1=1508470&r2=1508471&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Tue Jul 30 14:42:59 2013 @@ -33,9 +33,6 @@ Release 2.3.0 - UNRELEASED HADOOP-9582. Non-existent file to "hadoop fs -conf" doesn't throw error (Ashwin Shankar via jlowe) - HADOOP-9652. RawLocalFs#getFileLinkStatus does not fill in the link owner - and mode. (Andrew Wang via Colin Patrick McCabe) - Release 2.1.1-beta - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java?rev=1508471&r1=1508470&r2=1508471&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java Tue Jul 30 14:42:59 2013 @@ -113,14 +113,7 @@ public abstract class DelegateToFileSyst @Override public FileStatus getFileLinkStatus(final Path f) throws IOException { - FileStatus status = fsImpl.getFileLinkStatus(f); - // FileSystem#getFileLinkStatus qualifies the link target - // AbstractFileSystem needs to return it plain since it's qualified - // in FileContext, so re-get and set the plain target - if (status.isSymlink()) { - status.setSymlink(fsImpl.getLinkTarget(f)); - } - return status; + return getFileStatus(f); } @Override @@ -206,18 +199,22 @@ public abstract class DelegateToFileSyst @Override public boolean supportsSymlinks() { - return fsImpl.supportsSymlinks(); + return false; } @Override public void createSymlink(Path target, Path link, boolean createParent) throws IOException { - fsImpl.createSymlink(target, link, createParent); + throw new IOException("File system does not support symlinks"); } @Override public Path getLinkTarget(final Path f) throws IOException { - return fsImpl.getLinkTarget(f); + /* We should never get here. Any file system that threw an + * UnresolvedLinkException, causing this function to be called, + * should override getLinkTarget. + */ + throw new AssertionError(); } @Override //AbstractFileSystem Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java?rev=1508471&r1=1508470&r2=1508471&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java Tue Jul 30 14:42:59 2013 @@ -26,7 +26,6 @@ import java.io.InputStreamReader; import java.util.Arrays; import org.apache.hadoop.util.Shell; -import org.apache.hadoop.util.Shell.OSType; /** * Class for creating hardlinks. @@ -42,7 +41,15 @@ import org.apache.hadoop.util.Shell.OSTy */ public class HardLink { - public static OSType osType = Shell.osType; + public enum OSType { + OS_TYPE_UNIX, + OS_TYPE_WIN, + OS_TYPE_SOLARIS, + OS_TYPE_MAC, + OS_TYPE_FREEBSD + } + + public static OSType osType; private static HardLinkCommandGetter getHardLinkCommand; public final LinkStats linkStats; //not static @@ -50,11 +57,12 @@ public class HardLink { //initialize the command "getters" statically, so can use their //methods without instantiating the HardLink object static { + osType = getOSType(); if (osType == OSType.OS_TYPE_WIN) { // Windows getHardLinkCommand = new HardLinkCGWin(); } else { - // Unix or Linux + // Unix getHardLinkCommand = new HardLinkCGUnix(); //override getLinkCountCommand for the particular Unix variant //Linux is already set as the default - {"stat","-c%h", null} @@ -71,7 +79,27 @@ public class HardLink { public HardLink() { linkStats = new LinkStats(); } - + + static private OSType getOSType() { + String osName = System.getProperty("os.name"); + if (Shell.WINDOWS) { + return OSType.OS_TYPE_WIN; + } + else if (osName.contains("SunOS") + || osName.contains("Solaris")) { + return OSType.OS_TYPE_SOLARIS; + } + else if (osName.contains("Mac")) { + return OSType.OS_TYPE_MAC; + } + else if (osName.contains("FreeBSD")) { + return OSType.OS_TYPE_FREEBSD; + } + else { + return OSType.OS_TYPE_UNIX; + } + } + /** * This abstract class bridges the OS-dependent implementations of the * needed functionality for creating hardlinks and querying link counts. Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java?rev=1508471&r1=1508470&r2=1508471&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java Tue Jul 30 14:42:59 2013 @@ -379,9 +379,6 @@ public class RawLocalFileSystem extends throw new FileNotFoundException("File " + f + " does not exist"); } if (localf.isFile()) { - if (!Shell.WINDOWS) { - return new FileStatus[] { getFileStatus(f) }; - } return new FileStatus[] { new RawLocalFileStatus(localf, getDefaultBlockSize(f), this) }; } @@ -513,10 +510,6 @@ public class RawLocalFileSystem extends @Override public FileStatus getFileStatus(Path f) throws IOException { - if (!Shell.WINDOWS) { - return getFileLinkStatusInternal(f, true); - } - File path = pathToFile(f); if (path.exists()) { return new RawLocalFileStatus(pathToFile(f), getDefaultBlockSize(f), this); @@ -525,7 +518,6 @@ public class RawLocalFileSystem extends } } - @Deprecated static class RawLocalFileStatus extends FileStatus { /* We can add extra fields here. It breaks at least CopyFiles.FilePair(). * We recognize if the information is already loaded by check if @@ -707,7 +699,6 @@ public class RawLocalFileSystem extends * the given path does not refer to a symlink or there is an error * accessing the symlink. */ - @Deprecated private String readLink(Path p) { /* NB: Use readSymbolicLink in java.nio.file.Path once available. Could * use getCanonicalPath in File to get the target of the symlink but that @@ -728,12 +719,7 @@ public class RawLocalFileSystem extends */ @Override public FileStatus getFileLinkStatus(final Path f) throws IOException { - FileStatus fi; - if (!Shell.WINDOWS) { - fi = getFileLinkStatusInternal(f, false); - } else { - fi = getFileLinkStatusInternal(f); - } + FileStatus fi = getFileLinkStatusInternal(f); // getFileLinkStatus is supposed to return a symlink with a // qualified path if (fi.isSymlink()) { @@ -744,12 +730,6 @@ public class RawLocalFileSystem extends return fi; } - /** - * Deprecated. Remains for windows support. Should be removed in favor of - * {@link #getFileLinkStatusInternal(Path, boolean)} when {@link Stat} gains - * support for windows. - */ - @Deprecated private FileStatus getFileLinkStatusInternal(final Path f) throws IOException { String target = readLink(f); @@ -787,17 +767,9 @@ public class RawLocalFileSystem extends } } - private FileStatus getFileLinkStatusInternal(final Path f, - boolean dereference) throws IOException { - checkPath(f); - Stat stat = new Stat(f, getDefaultBlockSize(f), dereference, this); - FileStatus status = stat.getFileStatus(); - return status; - } - @Override public Path getLinkTarget(Path f) throws IOException { - FileStatus fi = getFileLinkStatusInternal(f, false); + FileStatus fi = getFileLinkStatusInternal(f); // return an unqualified symlink target return fi.getSymlink(); } Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java?rev=1508471&r1=1508470&r2=1508471&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java Tue Jul 30 14:42:59 2013 @@ -17,6 +17,7 @@ */ package org.apache.hadoop.fs.local; +import java.io.FileNotFoundException; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; @@ -26,9 +27,13 @@ import org.apache.hadoop.classification. import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.AbstractFileSystem; import org.apache.hadoop.fs.DelegateToFileSystem; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FsConstants; import org.apache.hadoop.fs.FsServerDefaults; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RawLocalFileSystem; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.util.Shell; /** * The RawLocalFs implementation of AbstractFileSystem. @@ -67,6 +72,92 @@ public class RawLocalFs extends Delegate return LocalConfigKeys.getServerDefaults(); } + @Override + public boolean supportsSymlinks() { + return true; + } + + @Override + public void createSymlink(Path target, Path link, boolean createParent) + throws IOException { + final String targetScheme = target.toUri().getScheme(); + if (targetScheme != null && !"file".equals(targetScheme)) { + throw new IOException("Unable to create symlink to non-local file "+ + "system: "+target.toString()); + } + if (createParent) { + mkdir(link.getParent(), FsPermission.getDirDefault(), true); + } + // NB: Use createSymbolicLink in java.nio.file.Path once available + try { + Shell.execCommand(Shell.getSymlinkCommand( + Path.getPathWithoutSchemeAndAuthority(target).toString(), + Path.getPathWithoutSchemeAndAuthority(link).toString())); + } catch (IOException x) { + throw new IOException("Unable to create symlink: "+x.getMessage()); + } + } + + /** + * Returns the target of the given symlink. Returns the empty string if + * the given path does not refer to a symlink or there is an error + * acessing the symlink. + */ + private String readLink(Path p) { + /* NB: Use readSymbolicLink in java.nio.file.Path once available. Could + * use getCanonicalPath in File to get the target of the symlink but that + * does not indicate if the given path refers to a symlink. + */ + try { + final String path = p.toUri().getPath(); + return Shell.execCommand(Shell.READ_LINK_COMMAND, path).trim(); + } catch (IOException x) { + return ""; + } + } + + /** + * Return a FileStatus representing the given path. If the path refers + * to a symlink return a FileStatus representing the link rather than + * the object the link refers to. + */ + @Override + public FileStatus getFileLinkStatus(final Path f) throws IOException { + String target = readLink(f); + try { + FileStatus fs = getFileStatus(f); + // If f refers to a regular file or directory + if ("".equals(target)) { + return fs; + } + // Otherwise f refers to a symlink + return new FileStatus(fs.getLen(), + false, + fs.getReplication(), + fs.getBlockSize(), + fs.getModificationTime(), + fs.getAccessTime(), + fs.getPermission(), + fs.getOwner(), + fs.getGroup(), + new Path(target), + f); + } catch (FileNotFoundException e) { + /* The exists method in the File class returns false for dangling + * links so we can get a FileNotFoundException for links that exist. + * It's also possible that we raced with a delete of the link. Use + * the readBasicFileAttributes method in java.nio.file.attributes + * when available. + */ + if (!"".equals(target)) { + return new FileStatus(0, false, 0, 0, 0, 0, FsPermission.getDefault(), + "", "", new Path(target), f); + } + // f refers to a file or directory that does not exist + throw e; + } + } + @Override public boolean isValidName(String src) { // Different local file systems have different validation rules. Skip @@ -74,4 +165,14 @@ public class RawLocalFs extends Delegate // RawLocalFileSystem. return true; } + + @Override + public Path getLinkTarget(Path f) throws IOException { + /* We should never get here. Valid local links are resolved transparently + * by the underlying local file system and accessing a dangling link will + * result in an IOException, not an UnresolvedLinkException, so FileContext + * should never call this function. + */ + throw new AssertionError(); + } } Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java?rev=1508471&r1=1508470&r2=1508471&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java Tue Jul 30 14:42:59 2013 @@ -58,33 +58,6 @@ abstract public class Shell { /** Windows CreateProcess synchronization object */ public static final Object WindowsProcessLaunchLock = new Object(); - // OSType detection - - public enum OSType { - OS_TYPE_LINUX, - OS_TYPE_WIN, - OS_TYPE_SOLARIS, - OS_TYPE_MAC, - OS_TYPE_FREEBSD - } - - public static final OSType osType = getOSType(); - - static private OSType getOSType() { - String osName = System.getProperty("os.name"); - if (Shell.WINDOWS) { - return OSType.OS_TYPE_WIN; - } else if (osName.contains("SunOS") || osName.contains("Solaris")) { - return OSType.OS_TYPE_SOLARIS; - } else if (osName.contains("Mac")) { - return OSType.OS_TYPE_MAC; - } else if (osName.contains("FreeBSD")) { - return OSType.OS_TYPE_FREEBSD; - } else { - return OSType.OS_TYPE_LINUX; - } - } - /** a Unix command to get the current user's groups list */ public static String[] getGroupsCommand() { return (WINDOWS)? new String[]{"cmd", "/c", "groups"} Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java?rev=1508471&r1=1508470&r2=1508471&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java Tue Jul 30 14:42:59 2013 @@ -358,12 +358,12 @@ public class TestLocalFileSystem { FileStatus status = fileSys.getFileStatus(path); assertTrue("check we're actually changing something", newModTime != status.getModificationTime()); - long accessTime = status.getAccessTime(); + assertEquals(0, status.getAccessTime()); fileSys.setTimes(path, newModTime, -1); status = fileSys.getFileStatus(path); assertEquals(newModTime, status.getModificationTime()); - assertEquals(accessTime, status.getAccessTime()); + assertEquals(0, status.getAccessTime()); } /** Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestSymlinkLocalFS.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestSymlinkLocalFS.java?rev=1508471&r1=1508470&r2=1508471&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestSymlinkLocalFS.java (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestSymlinkLocalFS.java Tue Jul 30 14:42:59 2013 @@ -30,7 +30,6 @@ import java.net.URI; import java.net.URISyntaxException; import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.security.UserGroupInformation; import org.junit.Test; /** @@ -124,7 +123,6 @@ abstract public class TestSymlinkLocalFS Path fileAbs = new Path(testBaseDir1()+"/file"); Path fileQual = new Path(testURI().toString(), fileAbs); Path link = new Path(testBaseDir1()+"/linkToFile"); - Path linkQual = new Path(testURI().toString(), link.toString()); wrapper.createSymlink(fileAbs, link, false); // Deleting the link using FileContext currently fails because // resolve looks up LocalFs rather than RawLocalFs for the path @@ -142,15 +140,18 @@ abstract public class TestSymlinkLocalFS // Expected. File's exists method returns false for dangling links } // We can stat a dangling link - UserGroupInformation user = UserGroupInformation.getCurrentUser(); FileStatus fsd = wrapper.getFileLinkStatus(link); assertEquals(fileQual, fsd.getSymlink()); assertTrue(fsd.isSymlink()); assertFalse(fsd.isDirectory()); - assertEquals(user.getUserName(), fsd.getOwner()); - // Compare against user's primary group - assertEquals(user.getGroupNames()[0], fsd.getGroup()); - assertEquals(linkQual, fsd.getPath()); + assertEquals("", fsd.getOwner()); + assertEquals("", fsd.getGroup()); + assertEquals(link, fsd.getPath()); + assertEquals(0, fsd.getLen()); + assertEquals(0, fsd.getBlockSize()); + assertEquals(0, fsd.getReplication()); + assertEquals(0, fsd.getAccessTime()); + assertEquals(FsPermission.getDefault(), fsd.getPermission()); // Accessing the link try { readFile(link);