HADOOP-9631. ViewFs should use underlying FileSystem's server side defaults. Contributed by Lohit Vijayarenu and Erik Krogen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/59d69257 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/59d69257 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/59d69257 Branch: refs/heads/HDFS-10467 Commit: 59d69257a888347f0fb9c51bb000afc986b64f98 Parents: ed2d03f Author: Zhe Zhang <z...@apache.org> Authored: Wed Mar 22 22:21:30 2017 -0700 Committer: Zhe Zhang <z...@apache.org> Committed: Wed Mar 22 22:22:22 2017 -0700 ---------------------------------------------------------------------- .../apache/hadoop/fs/AbstractFileSystem.java | 16 ++++++- .../java/org/apache/hadoop/fs/ChecksumFs.java | 13 +++--- .../apache/hadoop/fs/DelegateToFileSystem.java | 6 +++ .../java/org/apache/hadoop/fs/FilterFs.java | 5 ++ .../java/org/apache/hadoop/fs/ftp/FtpFs.java | 7 +++ .../org/apache/hadoop/fs/local/RawLocalFs.java | 8 ++++ .../org/apache/hadoop/fs/viewfs/ChRootedFs.java | 6 +++ .../org/apache/hadoop/fs/viewfs/ViewFs.java | 20 +++++++- .../org/apache/hadoop/fs/TestAfsCheckPath.java | 1 + .../apache/hadoop/fs/viewfs/ViewFsBaseTest.java | 49 ++++++++++++++++++++ .../main/java/org/apache/hadoop/fs/Hdfs.java | 6 +++ 11 files changed, 128 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/59d69257/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java index d1e7a1b..ef68437 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java @@ -450,10 +450,22 @@ public abstract class AbstractFileSystem { * @return server default configuration values * * @throws IOException an I/O error occurred + * @deprecated use {@link #getServerDefaults(Path)} instead */ + @Deprecated public abstract FsServerDefaults getServerDefaults() throws IOException; /** + * Return a set of server default configuration values based on path. + * @param f path to fetch server defaults + * @return server default configuration values for path + * @throws IOException an I/O error occurred + */ + public FsServerDefaults getServerDefaults(final Path f) throws IOException { + return getServerDefaults(); + } + + /** * Return the fully-qualified path of path f resolving the path * through any internal symlinks or mount point * @param p path to be resolved @@ -548,7 +560,7 @@ public abstract class AbstractFileSystem { } - FsServerDefaults ssDef = getServerDefaults(); + FsServerDefaults ssDef = getServerDefaults(f); if (ssDef.getBlockSize() % ssDef.getBytesPerChecksum() != 0) { throw new IOException("Internal error: default blockSize is" + " not a multiple of default bytesPerChecksum "); @@ -626,7 +638,7 @@ public abstract class AbstractFileSystem { */ public FSDataInputStream open(final Path f) throws AccessControlException, FileNotFoundException, UnresolvedLinkException, IOException { - return open(f, getServerDefaults().getFileBufferSize()); + return open(f, getServerDefaults(f).getFileBufferSize()); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/59d69257/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java index 3972033..0a8cc73 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java @@ -57,7 +57,7 @@ public abstract class ChecksumFs extends FilterFs { throws IOException, URISyntaxException { super(theFs); defaultBytesPerChecksum = - getMyFs().getServerDefaults().getBytesPerChecksum(); + getMyFs().getServerDefaults(new Path("/")).getBytesPerChecksum(); } /** @@ -96,9 +96,10 @@ public abstract class ChecksumFs extends FilterFs { return defaultBytesPerChecksum; } - private int getSumBufferSize(int bytesPerSum, int bufferSize) + private int getSumBufferSize(int bytesPerSum, int bufferSize, Path file) throws IOException { - int defaultBufferSize = getMyFs().getServerDefaults().getFileBufferSize(); + int defaultBufferSize = getMyFs().getServerDefaults(file) + .getFileBufferSize(); int proportionalBufferSize = bufferSize / bytesPerSum; return Math.max(bytesPerSum, Math.max(proportionalBufferSize, defaultBufferSize)); @@ -121,7 +122,7 @@ public abstract class ChecksumFs extends FilterFs { public ChecksumFSInputChecker(ChecksumFs fs, Path file) throws IOException, UnresolvedLinkException { - this(fs, file, fs.getServerDefaults().getFileBufferSize()); + this(fs, file, fs.getServerDefaults(file).getFileBufferSize()); } public ChecksumFSInputChecker(ChecksumFs fs, Path file, int bufferSize) @@ -132,7 +133,7 @@ public abstract class ChecksumFs extends FilterFs { Path sumFile = fs.getChecksumFile(file); try { int sumBufferSize = fs.getSumBufferSize(fs.getBytesPerSum(), - bufferSize); + bufferSize, file); sums = fs.getRawFs().open(sumFile, sumBufferSize); byte[] version = new byte[CHECKSUM_VERSION.length]; @@ -353,7 +354,7 @@ public abstract class ChecksumFs extends FilterFs { // Now create the chekcsumfile; adjust the buffsize int bytesPerSum = fs.getBytesPerSum(); - int sumBufferSize = fs.getSumBufferSize(bytesPerSum, bufferSize); + int sumBufferSize = fs.getSumBufferSize(bytesPerSum, bufferSize, file); this.sums = fs.getRawFs().createInternal(fs.getChecksumFile(file), EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE), absolutePermission, sumBufferSize, replication, blockSize, progress, http://git-wip-us.apache.org/repos/asf/hadoop/blob/59d69257/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java index d2550dc..a5ab75e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java @@ -149,11 +149,17 @@ public abstract class DelegateToFileSystem extends AbstractFileSystem { } @Override + @Deprecated public FsServerDefaults getServerDefaults() throws IOException { return fsImpl.getServerDefaults(); } @Override + public FsServerDefaults getServerDefaults(final Path f) throws IOException { + return fsImpl.getServerDefaults(f); + } + + @Override public Path getHomeDirectory() { return fsImpl.getHomeDirectory(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/59d69257/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java index d7e313f..6b1093e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java @@ -147,10 +147,15 @@ public abstract class FilterFs extends AbstractFileSystem { } @Override + @Deprecated public FsServerDefaults getServerDefaults() throws IOException { return myFs.getServerDefaults(); } + @Override + public FsServerDefaults getServerDefaults(final Path f) throws IOException { + return myFs.getServerDefaults(f); + } @Override public Path resolvePath(final Path p) throws FileNotFoundException, http://git-wip-us.apache.org/repos/asf/hadoop/blob/59d69257/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpFs.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpFs.java index 0a97375..6d54e36 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpFs.java @@ -29,6 +29,7 @@ import org.apache.hadoop.fs.AbstractFileSystem; import org.apache.hadoop.fs.DelegateToFileSystem; import org.apache.hadoop.fs.FsConstants; import org.apache.hadoop.fs.FsServerDefaults; +import org.apache.hadoop.fs.Path; /** * The FtpFs implementation of AbstractFileSystem. @@ -57,7 +58,13 @@ public class FtpFs extends DelegateToFileSystem { } @Override + @Deprecated public FsServerDefaults getServerDefaults() throws IOException { return FtpConfigKeys.getServerDefaults(); } + + @Override + public FsServerDefaults getServerDefaults(final Path f) throws IOException { + return FtpConfigKeys.getServerDefaults(); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/59d69257/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java index 6cb2792..8aebe02 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java @@ -28,6 +28,7 @@ import org.apache.hadoop.fs.AbstractFileSystem; import org.apache.hadoop.fs.DelegateToFileSystem; import org.apache.hadoop.fs.FsConstants; import org.apache.hadoop.fs.FsServerDefaults; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RawLocalFileSystem; /** @@ -63,6 +64,13 @@ public class RawLocalFs extends DelegateToFileSystem { } @Override + public FsServerDefaults getServerDefaults(final Path f) + throws IOException { + return LocalConfigKeys.getServerDefaults(); + } + + @Override + @Deprecated public FsServerDefaults getServerDefaults() throws IOException { return LocalConfigKeys.getServerDefaults(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/59d69257/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java index b7e47c2..d77ad8b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java @@ -223,11 +223,17 @@ class ChRootedFs extends AbstractFileSystem { } @Override + @Deprecated public FsServerDefaults getServerDefaults() throws IOException { return myFs.getServerDefaults(); } @Override + public FsServerDefaults getServerDefaults(final Path f) throws IOException { + return myFs.getServerDefaults(fullPath(f)); + } + + @Override public int getUriDefaultPort() { return myFs.getUriDefaultPort(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/59d69257/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java index 1a54a81..3a34a91 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java @@ -240,11 +240,23 @@ public class ViewFs extends AbstractFileSystem { } @Override + @Deprecated public FsServerDefaults getServerDefaults() throws IOException { return LocalConfigKeys.getServerDefaults(); } @Override + public FsServerDefaults getServerDefaults(final Path f) throws IOException { + InodeTree.ResolveResult<AbstractFileSystem> res; + try { + res = fsState.resolve(getUriPath(f), true); + } catch (FileNotFoundException fnfe) { + return LocalConfigKeys.getServerDefaults(); + } + return res.targetFileSystem.getServerDefaults(res.remainingPath); + } + + @Override public int getUriDefaultPort() { return -1; } @@ -927,8 +939,14 @@ public class ViewFs extends AbstractFileSystem { } @Override + @Deprecated public FsServerDefaults getServerDefaults() throws IOException { - throw new IOException("FsServerDefaults not implemented yet"); + return LocalConfigKeys.getServerDefaults(); + } + + @Override + public FsServerDefaults getServerDefaults(final Path f) throws IOException { + return LocalConfigKeys.getServerDefaults(); } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/59d69257/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAfsCheckPath.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAfsCheckPath.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAfsCheckPath.java index 3bd14f1..da429ff 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAfsCheckPath.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAfsCheckPath.java @@ -117,6 +117,7 @@ public class TestAfsCheckPath { } @Override + @Deprecated public FsServerDefaults getServerDefaults() throws IOException { // deliberately empty return null; http://git-wip-us.apache.org/repos/asf/hadoop/blob/59d69257/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java index 8f6df32..50237d1 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java @@ -39,6 +39,7 @@ import java.net.URI; import java.net.URISyntaxException; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; +import java.util.EnumSet; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -46,8 +47,10 @@ import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.AbstractFileSystem; import org.apache.hadoop.fs.BlockLocation; +import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileContextTestHelper; +import org.apache.hadoop.fs.FsServerDefaults; import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.FileContextTestHelper.fileType; @@ -55,6 +58,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FsConstants; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.UnresolvedLinkException; +import org.apache.hadoop.fs.local.LocalConfigKeys; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclStatus; import org.apache.hadoop.fs.permission.AclUtil; @@ -839,6 +843,51 @@ abstract public class ViewFsBaseTest { }); } + @Test + public void testRespectsServerDefaults() throws Exception { + FsServerDefaults targetDefs = + fcTarget.getDefaultFileSystem().getServerDefaults(new Path("/")); + FsServerDefaults viewDefs = + fcView.getDefaultFileSystem().getServerDefaults(new Path("/data")); + assertEquals(targetDefs.getReplication(), viewDefs.getReplication()); + assertEquals(targetDefs.getBlockSize(), viewDefs.getBlockSize()); + assertEquals(targetDefs.getBytesPerChecksum(), + viewDefs.getBytesPerChecksum()); + assertEquals(targetDefs.getFileBufferSize(), + viewDefs.getFileBufferSize()); + assertEquals(targetDefs.getWritePacketSize(), + viewDefs.getWritePacketSize()); + assertEquals(targetDefs.getEncryptDataTransfer(), + viewDefs.getEncryptDataTransfer()); + assertEquals(targetDefs.getTrashInterval(), viewDefs.getTrashInterval()); + assertEquals(targetDefs.getChecksumType(), viewDefs.getChecksumType()); + + fcView.create(new Path("/data/file"), EnumSet.of(CreateFlag.CREATE)) + .close(); + FileStatus stat = + fcTarget.getFileStatus(new Path(targetTestRoot, "data/file")); + assertEquals(targetDefs.getReplication(), stat.getReplication()); + } + + @Test + public void testServerDefaultsInternalDir() throws Exception { + FsServerDefaults localDefs = LocalConfigKeys.getServerDefaults(); + FsServerDefaults viewDefs = fcView + .getDefaultFileSystem().getServerDefaults(new Path("/internalDir")); + assertEquals(localDefs.getReplication(), viewDefs.getReplication()); + assertEquals(localDefs.getBlockSize(), viewDefs.getBlockSize()); + assertEquals(localDefs.getBytesPerChecksum(), + viewDefs.getBytesPerChecksum()); + assertEquals(localDefs.getFileBufferSize(), + viewDefs.getFileBufferSize()); + assertEquals(localDefs.getWritePacketSize(), + viewDefs.getWritePacketSize()); + assertEquals(localDefs.getEncryptDataTransfer(), + viewDefs.getEncryptDataTransfer()); + assertEquals(localDefs.getTrashInterval(), viewDefs.getTrashInterval()); + assertEquals(localDefs.getChecksumType(), viewDefs.getChecksumType()); + } + // Confirm that listLocatedStatus is delegated properly to the underlying // AbstractFileSystem to allow for optimizations @Test http://git-wip-us.apache.org/repos/asf/hadoop/blob/59d69257/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java index 82ee41a..645f1ad 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java @@ -155,11 +155,17 @@ public class Hdfs extends AbstractFileSystem { } @Override + @Deprecated public FsServerDefaults getServerDefaults() throws IOException { return dfs.getServerDefaults(); } @Override + public FsServerDefaults getServerDefaults(final Path f) throws IOException { + return dfs.getServerDefaults(); + } + + @Override public RemoteIterator<LocatedFileStatus> listLocatedStatus( final Path p) throws FileNotFoundException, IOException { --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
