Repository: hadoop Updated Branches: refs/heads/branch-2 4d4442cb3 -> ae316705b
HADOOP-11510. Expose truncate API via FileContext. (yliu) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ae316705 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ae316705 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ae316705 Branch: refs/heads/branch-2 Commit: ae316705bb79479038a13b80bab6febbe8f3c75f Parents: 4d4442c Author: yliu <y...@apache.org> Authored: Tue Feb 10 01:43:08 2015 +0800 Committer: yliu <y...@apache.org> Committed: Tue Feb 10 01:43:08 2015 +0800 ---------------------------------------------------------------------- hadoop-common-project/hadoop-common/CHANGES.txt | 2 + .../apache/hadoop/fs/AbstractFileSystem.java | 9 ++++ .../java/org/apache/hadoop/fs/ChecksumFs.java | 5 +++ .../apache/hadoop/fs/DelegateToFileSystem.java | 6 +++ .../java/org/apache/hadoop/fs/FileContext.java | 43 ++++++++++++++++++++ .../java/org/apache/hadoop/fs/FilterFs.java | 8 ++++ .../org/apache/hadoop/fs/viewfs/ChRootedFs.java | 6 +++ .../org/apache/hadoop/fs/viewfs/ViewFs.java | 17 +++++++- .../org/apache/hadoop/fs/TestAfsCheckPath.java | 6 +++ .../main/java/org/apache/hadoop/fs/Hdfs.java | 6 +++ .../fs/TestHDFSFileContextMainOperations.java | 32 ++++++++++++++- 11 files changed, 138 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae316705/hadoop-common-project/hadoop-common/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 8d1047f..a61c349 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -31,6 +31,8 @@ Release 2.7.0 - UNRELEASED HADOOP-11045. Introducing a tool to detect flaky tests of hadoop jenkins testing job. (Yongjun Zhang and Todd Lipcon via ozawa) + HADOOP-11510. Expose truncate API via FileContext. (yliu) + IMPROVEMENTS HADOOP-11483. HardLink.java should use the jdk7 createLink method (aajisaka) http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae316705/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java index 92d4eca..975cc3c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java @@ -639,6 +639,15 @@ public abstract class AbstractFileSystem { /** * The specification of this method matches that of + * {@link FileContext#truncate(Path, long)} except that Path f must be for + * this file system. + */ + public abstract boolean truncate(Path f, long newLength) + throws AccessControlException, FileNotFoundException, + UnresolvedLinkException, IOException; + + /** + * The specification of this method matches that of * {@link FileContext#setReplication(Path, short)} except that Path f must be * for this file system. */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae316705/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java index ab5cd13..7dc4a80 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java @@ -297,6 +297,11 @@ public abstract class ChecksumFs extends FilterFs { } + @Override + public boolean truncate(Path f, long newLength) throws IOException { + throw new IOException("Not supported"); + } + /** * Opens an FSDataInputStream at the indicated Path. * @param f the file name to open http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae316705/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java index 1cdcb27..09707c6 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java @@ -170,6 +170,12 @@ public abstract class DelegateToFileSystem extends AbstractFileSystem { } @Override + public boolean truncate(Path f, long newLength) throws IOException { + checkPath(f); + return fsImpl.truncate(f, newLength); + } + + @Override @SuppressWarnings("deprecation") // call to rename public void renameInternal(Path src, Path dst) throws IOException { checkPath(src); http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae316705/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java index 3c5e9ab..705a8d8 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java @@ -829,6 +829,49 @@ public class FileContext { } /** + * Truncate the file in the indicated path to the indicated size. + * <ul> + * <li>Fails if path is a directory. + * <li>Fails if path does not exist. + * <li>Fails if path is not closed. + * <li>Fails if new size is greater than current size. + * </ul> + * @param f The path to the file to be truncated + * @param newLength The size the file is to be truncated to + * + * @return <code>true</code> if the file has been truncated to the desired + * <code>newLength</code> and is immediately available to be reused for + * write operations such as <code>append</code>, or + * <code>false</code> if a background process of adjusting the length of + * the last block has been started, and clients should wait for it to + * complete before proceeding with further file updates. + * + * @throws AccessControlException If access is denied + * @throws FileNotFoundException If file <code>f</code> does not exist + * @throws UnsupportedFileSystemException If file system for <code>f</code> is + * not supported + * @throws IOException If an I/O error occurred + * + * Exceptions applicable to file systems accessed over RPC: + * @throws RpcClientException If an exception occurred in the RPC client + * @throws RpcServerException If an exception occurred in the RPC server + * @throws UnexpectedServerException If server implementation throws + * undeclared exception to RPC server + */ + public boolean truncate(final Path f, final long newLength) + throws AccessControlException, FileNotFoundException, + UnsupportedFileSystemException, IOException { + final Path absF = fixRelativePart(f); + return new FSLinkResolver<Boolean>() { + @Override + public Boolean next(final AbstractFileSystem fs, final Path p) + throws IOException, UnresolvedLinkException { + return fs.truncate(p, newLength); + } + }.resolve(this, absF); + } + + /** * Set replication for an existing file. * * @param f file name http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae316705/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java index b6e1d96..4f28a9a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java @@ -213,6 +213,14 @@ public abstract class FilterFs extends AbstractFileSystem { } @Override + public boolean truncate(Path f, long newLength) + throws AccessControlException, FileNotFoundException, + UnresolvedLinkException, IOException { + checkPath(f); + return myFs.truncate(f, newLength); + } + + @Override public void renameInternal(Path src, Path dst) throws IOException, UnresolvedLinkException { checkPath(src); http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae316705/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java index 9569e10..68e756a8 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java @@ -248,6 +248,12 @@ class ChRootedFs extends AbstractFileSystem { } @Override + public boolean truncate(final Path f, final long newLength) + throws IOException, UnresolvedLinkException { + return myFs.truncate(fullPath(f), newLength); + } + + @Override public void renameInternal(final Path src, final Path dst) throws IOException, UnresolvedLinkException { // note fullPath will check that paths are relative to this FileSystem. http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae316705/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java index 014f488..975496a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java @@ -452,7 +452,15 @@ public class ViewFs extends AbstractFileSystem { return res.targetFileSystem.open(res.remainingPath, bufferSize); } - + @Override + public boolean truncate(final Path f, final long newLength) + throws AccessControlException, FileNotFoundException, + UnresolvedLinkException, IOException { + InodeTree.ResolveResult<AbstractFileSystem> res = + fsState.resolve(getUriPath(f), true); + return res.targetFileSystem.truncate(res.remainingPath, newLength); + } + @Override public void renameInternal(final Path src, final Path dst, final boolean overwrite) throws IOException, UnresolvedLinkException { @@ -878,6 +886,13 @@ public class ViewFs extends AbstractFileSystem { } @Override + public boolean truncate(final Path f, final long newLength) + throws FileNotFoundException, IOException { + checkPathIsSlash(f); + throw readOnlyMountTable("truncate", f); + } + + @Override public void renameInternal(final Path src, final Path dst) throws AccessControlException, IOException { checkPathIsSlash(src); http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae316705/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAfsCheckPath.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAfsCheckPath.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAfsCheckPath.java index 3bd14f1..6b9378d 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAfsCheckPath.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAfsCheckPath.java @@ -141,6 +141,12 @@ public class TestAfsCheckPath { } @Override + public boolean truncate(Path f, long newLength) throws IOException { + // deliberately empty + return false; + } + + @Override public void renameInternal(Path src, Path dst) throws IOException { // deliberately empty } http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae316705/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java index 1b9f515..8c09193 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java @@ -318,6 +318,12 @@ public class Hdfs extends AbstractFileSystem { } @Override + public boolean truncate(Path f, long newLength) + throws IOException, UnresolvedLinkException { + return dfs.truncate(getUriPath(f), newLength); + } + + @Override public void renameInternal(Path src, Path dst) throws IOException, UnresolvedLinkException { dfs.rename(getUriPath(src), getUriPath(dst), Options.Rename.NONE); http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae316705/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestHDFSFileContextMainOperations.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestHDFSFileContextMainOperations.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestHDFSFileContextMainOperations.java index 8a215b0..94fb0fb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestHDFSFileContextMainOperations.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestHDFSFileContextMainOperations.java @@ -28,6 +28,7 @@ import java.net.URISyntaxException; import javax.security.auth.login.LoginException; import org.apache.hadoop.fs.Options.Rename; +import org.apache.hadoop.hdfs.AppendTestUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; @@ -114,7 +115,36 @@ public class TestHDFSFileContextMainOperations extends private Path getTestRootPath(FileContext fc, String path) { return fileContextTestHelper.getTestRootPath(fc, path); } - + + @Test + public void testTruncate() throws Exception { + final short repl = 3; + final int blockSize = 1024; + final int numOfBlocks = 2; + DistributedFileSystem fs = cluster.getFileSystem(); + Path dir = getTestRootPath(fc, "test/hadoop"); + Path file = getTestRootPath(fc, "test/hadoop/file"); + + final byte[] data = FileSystemTestHelper.getFileData( + numOfBlocks, blockSize); + FileSystemTestHelper.createFile(fs, file, data, blockSize, repl); + + final int newLength = blockSize; + + boolean isReady = fc.truncate(file, newLength); + + Assert.assertTrue("Recovery is not expected.", isReady); + + FileStatus fileStatus = fc.getFileStatus(file); + Assert.assertEquals(fileStatus.getLen(), newLength); + AppendTestUtil.checkFullFile(fs, file, newLength, data, file.toString()); + + ContentSummary cs = fs.getContentSummary(dir); + Assert.assertEquals("Bad disk space usage", cs.getSpaceConsumed(), + newLength * repl); + Assert.assertTrue(fs.delete(dir, true)); + } + @Test public void testOldRenameWithQuota() throws Exception { DistributedFileSystem fs = cluster.getFileSystem();
