HDFS-7656. Expose truncate API for HDFS httpfs. (yliu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2fd02afe Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2fd02afe Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2fd02afe Branch: refs/heads/HDFS-7285 Commit: 2fd02afeca3710f487b6a039a65c1a666322b229 Parents: 64a8375 Author: yliu <y...@apache.org> Authored: Thu Feb 19 08:36:31 2015 +0800 Committer: yliu <y...@apache.org> Committed: Thu Feb 19 08:36:31 2015 +0800 ---------------------------------------------------------------------- .../hadoop/fs/http/client/HttpFSFileSystem.java | 24 ++++++++++- .../hadoop/fs/http/server/FSOperations.java | 43 +++++++++++++++++++- .../http/server/HttpFSParametersProvider.java | 20 +++++++++ .../hadoop/fs/http/server/HttpFSServer.java | 10 +++++ .../fs/http/client/BaseTestHttpFSWith.java | 40 ++++++++++++++++-- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + 6 files changed, 133 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/2fd02afe/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java index 5b079e9..20b212e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java @@ -109,12 +109,15 @@ public class HttpFSFileSystem extends FileSystem public static final String XATTR_VALUE_PARAM = "xattr.value"; public static final String XATTR_SET_FLAG_PARAM = "flag"; public static final String XATTR_ENCODING_PARAM = "encoding"; + public static final String NEW_LENGTH_PARAM = "newlength"; public static final Short DEFAULT_PERMISSION = 0755; public static final String ACLSPEC_DEFAULT = ""; public static final String RENAME_JSON = "boolean"; + public static final String TRUNCATE_JSON = "boolean"; + public static final String DELETE_JSON = "boolean"; public static final String MKDIRS_JSON = "boolean"; @@ -191,7 +194,7 @@ public class HttpFSFileSystem extends FileSystem GETHOMEDIRECTORY(HTTP_GET), GETCONTENTSUMMARY(HTTP_GET), GETFILECHECKSUM(HTTP_GET), GETFILEBLOCKLOCATIONS(HTTP_GET), INSTRUMENTATION(HTTP_GET), GETACLSTATUS(HTTP_GET), - APPEND(HTTP_POST), CONCAT(HTTP_POST), + APPEND(HTTP_POST), CONCAT(HTTP_POST), TRUNCATE(HTTP_POST), CREATE(HTTP_PUT), MKDIRS(HTTP_PUT), RENAME(HTTP_PUT), SETOWNER(HTTP_PUT), SETPERMISSION(HTTP_PUT), SETREPLICATION(HTTP_PUT), SETTIMES(HTTP_PUT), MODIFYACLENTRIES(HTTP_PUT), REMOVEACLENTRIES(HTTP_PUT), @@ -568,6 +571,25 @@ public class HttpFSFileSystem extends FileSystem } /** + * Truncate a file. + * + * @param f the file to be truncated. + * @param newLength The size the file is to be truncated to. + * + * @throws IOException + */ + @Override + public boolean truncate(Path f, long newLength) throws IOException { + Map<String, String> params = new HashMap<String, String>(); + params.put(OP_PARAM, Operation.TRUNCATE.toString()); + params.put(NEW_LENGTH_PARAM, Long.toString(newLength)); + HttpURLConnection conn = getConnection(Operation.TRUNCATE.getMethod(), + params, f, true); + JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); + return (Boolean) json.get(TRUNCATE_JSON); + } + + /** * Concat existing files together. * @param f the path to the target destination. * @param psrcs the paths to the sources to use for the concatenation. http://git-wip-us.apache.org/repos/asf/hadoop/blob/2fd02afe/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java index 4b72a51..bc290a2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java @@ -364,7 +364,7 @@ public class FSOperations { } /** - * Executor that performs an append FileSystemAccess files system operation. + * Executor that performs a concat FileSystemAccess files system operation. */ @InterfaceAudience.Private public static class FSConcat implements FileSystemAccess.FileSystemExecutor<Void> { @@ -405,6 +405,47 @@ public class FSOperations { } /** + * Executor that performs a truncate FileSystemAccess files system operation. + */ + @InterfaceAudience.Private + public static class FSTruncate implements + FileSystemAccess.FileSystemExecutor<JSONObject> { + private Path path; + private long newLength; + + /** + * Creates a Truncate executor. + * + * @param path target path to truncate to. + * @param newLength The size the file is to be truncated to. + */ + public FSTruncate(String path, long newLength) { + this.path = new Path(path); + this.newLength = newLength; + } + + /** + * Executes the filesystem operation. + * + * @param fs filesystem instance to use. + * + * @return <code>true</code> if the file has been truncated to the desired, + * <code>false</code> if a background process of adjusting the + * length of the last block has been started, and clients should + * wait for it to complete before proceeding with further file + * updates. + * + * @throws IOException thrown if an IO error occured. + */ + @Override + public JSONObject execute(FileSystem fs) throws IOException { + boolean result = fs.truncate(path, newLength); + return toJSON(HttpFSFileSystem.TRUNCATE_JSON.toLowerCase(), result); + } + + } + + /** * Executor that performs a content-summary FileSystemAccess files system operation. */ @InterfaceAudience.Private http://git-wip-us.apache.org/repos/asf/hadoop/blob/2fd02afe/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java index fb06667..73853c4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java @@ -63,6 +63,7 @@ public class HttpFSParametersProvider extends ParametersProvider { PARAMS_DEF.put(Operation.INSTRUMENTATION, new Class[]{}); PARAMS_DEF.put(Operation.APPEND, new Class[]{DataParam.class}); PARAMS_DEF.put(Operation.CONCAT, new Class[]{SourcesParam.class}); + PARAMS_DEF.put(Operation.TRUNCATE, new Class[]{NewLengthParam.class}); PARAMS_DEF.put(Operation.CREATE, new Class[]{PermissionParam.class, OverwriteParam.class, ReplicationParam.class, BlockSizeParam.class, DataParam.class}); @@ -290,6 +291,25 @@ public class HttpFSParametersProvider extends ParametersProvider { } /** + * Class for newlength parameter. + */ + @InterfaceAudience.Private + public static class NewLengthParam extends LongParam { + + /** + * Parameter name. + */ + public static final String NAME = HttpFSFileSystem.NEW_LENGTH_PARAM; + + /** + * Constructor. + */ + public NewLengthParam() { + super(NAME, 0l); + } + } + + /** * Class for overwrite parameter. */ @InterfaceAudience.Private http://git-wip-us.apache.org/repos/asf/hadoop/blob/2fd02afe/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java index 9103718..1f903ba 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java @@ -33,6 +33,7 @@ import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.FilterParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.GroupParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.LenParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.ModifiedTimeParam; +import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.NewLengthParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.OffsetParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.OperationParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.OverwriteParam; @@ -427,6 +428,15 @@ public class HttpFSServer { response = Response.ok().build(); break; } + case TRUNCATE: { + Long newLength = params.get(NewLengthParam.NAME, NewLengthParam.class); + FSOperations.FSTruncate command = + new FSOperations.FSTruncate(path, newLength); + JSONObject json = fsExecute(user, command); + AUDIT_LOG.info("Truncate [{}] to length [{}]", path, newLength); + response = Response.ok(json).type(MediaType.APPLICATION_JSON).build(); + break; + } default: { throw new IOException( MessageFormat.format("Invalid HTTP POST operation [{0}]", http://git-wip-us.apache.org/repos/asf/hadoop/blob/2fd02afe/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java index f063e33..2cc67d4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java @@ -24,12 +24,14 @@ import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FileChecksum; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileSystemTestHelper; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.http.server.HttpFSServerWebApp; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclStatus; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.AppendTestUtil; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.security.UserGroupInformation; @@ -192,7 +194,7 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase { Assert.fail("the create should have failed because the file exists " + "and override is FALSE"); } catch (IOException ex) { -System.out.println("#"); + System.out.println("#"); } catch (Exception ex) { Assert.fail(ex.toString()); } @@ -222,6 +224,31 @@ System.out.println("#"); } } + private void testTruncate() throws Exception { + if (!isLocalFS()) { + final short repl = 3; + final int blockSize = 1024; + final int numOfBlocks = 2; + FileSystem fs = FileSystem.get(getProxiedFSConf()); + fs.mkdirs(getProxiedFSTestDir()); + Path file = new Path(getProxiedFSTestDir(), "foo.txt"); + final byte[] data = FileSystemTestHelper.getFileData( + numOfBlocks, blockSize); + FileSystemTestHelper.createFile(fs, file, data, blockSize, repl); + + final int newLength = blockSize; + + boolean isReady = fs.truncate(file, newLength); + Assert.assertTrue("Recovery is not expected.", isReady); + + FileStatus fileStatus = fs.getFileStatus(file); + Assert.assertEquals(fileStatus.getLen(), newLength); + AppendTestUtil.checkFullFile(fs, file, newLength, data, file.toString()); + + fs.close(); + } + } + private void testConcat() throws Exception { Configuration config = getProxiedFSConf(); config.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 1024); @@ -784,9 +811,10 @@ System.out.println("#"); } protected enum Operation { - GET, OPEN, CREATE, APPEND, CONCAT, RENAME, DELETE, LIST_STATUS, WORKING_DIRECTORY, MKDIRS, - SET_TIMES, SET_PERMISSION, SET_OWNER, SET_REPLICATION, CHECKSUM, CONTENT_SUMMARY, - FILEACLS, DIRACLS, SET_XATTR, GET_XATTRS, REMOVE_XATTR, LIST_XATTRS + GET, OPEN, CREATE, APPEND, TRUNCATE, CONCAT, RENAME, DELETE, LIST_STATUS, + WORKING_DIRECTORY, MKDIRS, SET_TIMES, SET_PERMISSION, SET_OWNER, + SET_REPLICATION, CHECKSUM, CONTENT_SUMMARY, FILEACLS, DIRACLS, SET_XATTR, + GET_XATTRS, REMOVE_XATTR, LIST_XATTRS } private void operation(Operation op) throws Exception { @@ -803,8 +831,12 @@ System.out.println("#"); case APPEND: testAppend(); break; + case TRUNCATE: + testTruncate(); + break; case CONCAT: testConcat(); + break; case RENAME: testRename(); break; http://git-wip-us.apache.org/repos/asf/hadoop/blob/2fd02afe/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 3735e90..80a086a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -337,6 +337,8 @@ Release 2.7.0 - UNRELEASED HDFS-7584. Enable Quota Support for Storage Types (See breakdown of tasks below) + HDFS-7656. Expose truncate API for HDFS httpfs. (yliu) + IMPROVEMENTS HDFS-7055. Add tracing to DFSInputStream (cmccabe)