HDFS-12117. HttpFS does not seem to support SNAPSHOT related methods for WebHDFS REST Interface. Contributed by Wellington Chevreuil.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8a4bff02 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8a4bff02 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8a4bff02 Branch: refs/heads/HDFS-10467 Commit: 8a4bff02c1534c6bf529726f2bbe414ac4c172e8 Parents: 9a3c237 Author: Wei-Chiu Chuang <weic...@apache.org> Authored: Tue Aug 8 23:58:53 2017 -0700 Committer: Wei-Chiu Chuang <weic...@apache.org> Committed: Tue Aug 8 23:58:53 2017 -0700 ---------------------------------------------------------------------- .../hadoop/fs/http/client/HttpFSFileSystem.java | 47 ++++++- .../hadoop/fs/http/server/FSOperations.java | 105 ++++++++++++++ .../http/server/HttpFSParametersProvider.java | 45 ++++++ .../hadoop/fs/http/server/HttpFSServer.java | 36 +++++ .../fs/http/client/BaseTestHttpFSWith.java | 110 ++++++++++++++- .../hadoop/fs/http/server/TestHttpFSServer.java | 140 ++++++++++++++++++- 6 files changed, 479 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a4bff02/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java index d139100..1059a02 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java @@ -124,6 +124,8 @@ public class HttpFSFileSystem extends FileSystem public static final String POLICY_NAME_PARAM = "storagepolicy"; public static final String OFFSET_PARAM = "offset"; public static final String LENGTH_PARAM = "length"; + public static final String SNAPSHOT_NAME_PARAM = "snapshotname"; + public static final String OLD_SNAPSHOT_NAME_PARAM = "oldsnapshotname"; public static final Short DEFAULT_PERMISSION = 0755; public static final String ACLSPEC_DEFAULT = ""; @@ -144,6 +146,8 @@ public class HttpFSFileSystem extends FileSystem public static final String UPLOAD_CONTENT_TYPE= "application/octet-stream"; + public static final String SNAPSHOT_JSON = "Path"; + public enum FILE_TYPE { FILE, DIRECTORY, SYMLINK; @@ -229,7 +233,9 @@ public class HttpFSFileSystem extends FileSystem DELETE(HTTP_DELETE), SETXATTR(HTTP_PUT), GETXATTRS(HTTP_GET), REMOVEXATTR(HTTP_PUT), LISTXATTRS(HTTP_GET), LISTSTATUS_BATCH(HTTP_GET), GETALLSTORAGEPOLICY(HTTP_GET), GETSTORAGEPOLICY(HTTP_GET), - SETSTORAGEPOLICY(HTTP_PUT), UNSETSTORAGEPOLICY(HTTP_POST); + SETSTORAGEPOLICY(HTTP_PUT), UNSETSTORAGEPOLICY(HTTP_POST), + CREATESNAPSHOT(HTTP_PUT), DELETESNAPSHOT(HTTP_DELETE), + RENAMESNAPSHOT(HTTP_PUT); private String httpMethod; @@ -1434,4 +1440,43 @@ public class HttpFSFileSystem extends FileSystem Operation.UNSETSTORAGEPOLICY.getMethod(), params, src, true); HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); } + + @Override + public final Path createSnapshot(Path path, String snapshotName) + throws IOException { + Map<String, String> params = new HashMap<String, String>(); + params.put(OP_PARAM, Operation.CREATESNAPSHOT.toString()); + if (snapshotName != null) { + params.put(SNAPSHOT_NAME_PARAM, snapshotName); + } + HttpURLConnection conn = getConnection(Operation.CREATESNAPSHOT.getMethod(), + params, path, true); + HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); + JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); + return new Path((String) json.get(SNAPSHOT_JSON)); + } + + @Override + public void renameSnapshot(Path path, String snapshotOldName, + String snapshotNewName) throws IOException { + Map<String, String> params = new HashMap<String, String>(); + params.put(OP_PARAM, Operation.RENAMESNAPSHOT.toString()); + params.put(SNAPSHOT_NAME_PARAM, snapshotNewName); + params.put(OLD_SNAPSHOT_NAME_PARAM, snapshotOldName); + HttpURLConnection conn = getConnection(Operation.RENAMESNAPSHOT.getMethod(), + params, path, true); + HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); + } + + @Override + public void deleteSnapshot(Path path, String snapshotName) + throws IOException { + Map<String, String> params = new HashMap<String, String>(); + params.put(OP_PARAM, Operation.DELETESNAPSHOT.toString()); + params.put(SNAPSHOT_NAME_PARAM, snapshotName); + HttpURLConnection conn = getConnection(Operation.DELETESNAPSHOT.getMethod(), + params, path, true); + HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); + } + } http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a4bff02/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java index f1615c3..c008802 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java @@ -1492,4 +1492,109 @@ public class FSOperations { return JsonUtil.toJsonMap(locations); } } + + /** + * Executor that performs a createSnapshot FileSystemAccess operation. + */ + @InterfaceAudience.Private + public static class FSCreateSnapshot implements + FileSystemAccess.FileSystemExecutor<String> { + + private Path path; + private String snapshotName; + + /** + * Creates a createSnapshot executor. + * @param path directory path to be snapshotted. + * @param snapshotName the snapshot name. + */ + public FSCreateSnapshot(String path, String snapshotName) { + this.path = new Path(path); + this.snapshotName = snapshotName; + } + + /** + * Executes the filesystem operation. + * @param fs filesystem instance to use. + * @return <code>Path</code> the complete path for newly created snapshot + * @throws IOException thrown if an IO error occurred. + */ + @Override + public String execute(FileSystem fs) throws IOException { + Path snapshotPath = fs.createSnapshot(path, snapshotName); + JSONObject json = toJSON(HttpFSFileSystem.HOME_DIR_JSON, + snapshotPath.toString()); + return json.toJSONString().replaceAll("\\\\", ""); + } + } + + /** + * Executor that performs a deleteSnapshot FileSystemAccess operation. + */ + @InterfaceAudience.Private + public static class FSDeleteSnapshot implements + FileSystemAccess.FileSystemExecutor<Void> { + + private Path path; + private String snapshotName; + + /** + * Creates a deleteSnapshot executor. + * @param path path for the snapshot to be deleted. + * @param snapshotName snapshot name. + */ + public FSDeleteSnapshot(String path, String snapshotName) { + this.path = new Path(path); + this.snapshotName = snapshotName; + } + + /** + * Executes the filesystem operation. + * @param fs filesystem instance to use. + * @return void + * @throws IOException thrown if an IO error occurred. + */ + @Override + public Void execute(FileSystem fs) throws IOException { + fs.deleteSnapshot(path, snapshotName); + return null; + } + } + + /** + * Executor that performs a renameSnapshot FileSystemAccess operation. + */ + @InterfaceAudience.Private + public static class FSRenameSnapshot implements + FileSystemAccess.FileSystemExecutor<Void> { + private Path path; + private String oldSnapshotName; + private String snapshotName; + + /** + * Creates a renameSnapshot executor. + * @param path directory path of the snapshot to be renamed. + * @param oldSnapshotName current snapshot name. + * @param snapshotName new snapshot name to be set. + */ + public FSRenameSnapshot(String path, String oldSnapshotName, + String snapshotName) { + this.path = new Path(path); + this.oldSnapshotName = oldSnapshotName; + this.snapshotName = snapshotName; + } + + /** + * Executes the filesystem operation. + * @param fs filesystem instance to use. + * @return void + * @throws IOException thrown if an IO error occurred. + */ + @Override + public Void execute(FileSystem fs) throws IOException { + fs.renameSnapshot(path, oldSnapshotName, snapshotName); + return null; + } + } + } http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a4bff02/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java index 347a747..5f265c0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java @@ -100,6 +100,13 @@ public class HttpFSParametersProvider extends ParametersProvider { PARAMS_DEF.put(Operation.SETSTORAGEPOLICY, new Class[] {PolicyNameParam.class}); PARAMS_DEF.put(Operation.UNSETSTORAGEPOLICY, new Class[] {}); + PARAMS_DEF.put(Operation.CREATESNAPSHOT, + new Class[] {SnapshotNameParam.class}); + PARAMS_DEF.put(Operation.DELETESNAPSHOT, + new Class[] {SnapshotNameParam.class}); + PARAMS_DEF.put(Operation.RENAMESNAPSHOT, + new Class[] {OldSnapshotNameParam.class, + SnapshotNameParam.class}); } public HttpFSParametersProvider() { @@ -565,4 +572,42 @@ public class HttpFSParametersProvider extends ParametersProvider { super(NAME, null); } } + + /** + * Class for SnapshotName parameter. + */ + public static class SnapshotNameParam extends StringParam { + + /** + * Parameter name. + */ + public static final String NAME = HttpFSFileSystem.SNAPSHOT_NAME_PARAM; + + /** + * Constructor. + */ + public SnapshotNameParam() { + super(NAME, null); + } + + } + + /** + * Class for OldSnapshotName parameter. + */ + public static class OldSnapshotNameParam extends StringParam { + + /** + * Parameter name. + */ + public static final String NAME = HttpFSFileSystem.OLD_SNAPSHOT_NAME_PARAM; + + /** + * Constructor. + */ + public OldSnapshotNameParam() { + super(NAME, null); + } + } + } http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a4bff02/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java index 5c0c9b5..03ccb4c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java @@ -37,6 +37,7 @@ import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.LenParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.ModifiedTimeParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.NewLengthParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.OffsetParam; +import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.OldSnapshotNameParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.OperationParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.OverwriteParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.OwnerParam; @@ -45,6 +46,7 @@ import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.PolicyNameParam import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.RecursiveParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.ReplicationParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.SourcesParam; +import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.SnapshotNameParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.XAttrEncodingParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.XAttrNameParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.XAttrSetFlagParam; @@ -430,6 +432,16 @@ public class HttpFSServer { response = Response.ok(json).type(MediaType.APPLICATION_JSON).build(); break; } + case DELETESNAPSHOT: { + String snapshotName = params.get(SnapshotNameParam.NAME, + SnapshotNameParam.class); + FSOperations.FSDeleteSnapshot command = + new FSOperations.FSDeleteSnapshot(path, snapshotName); + fsExecute(user, command); + AUDIT_LOG.info("[{}] deleted snapshot [{}]", path, snapshotName); + response = Response.ok().build(); + break; + } default: { throw new IOException( MessageFormat.format("Invalid HTTP DELETE operation [{0}]", @@ -602,6 +614,16 @@ public class HttpFSServer { } break; } + case CREATESNAPSHOT: { + String snapshotName = params.get(SnapshotNameParam.NAME, + SnapshotNameParam.class); + FSOperations.FSCreateSnapshot command = + new FSOperations.FSCreateSnapshot(path, snapshotName); + String json = fsExecute(user, command); + AUDIT_LOG.info("[{}] snapshot created as [{}]", path, snapshotName); + response = Response.ok(json).type(MediaType.APPLICATION_JSON).build(); + break; + } case SETXATTR: { String xattrName = params.get(XAttrNameParam.NAME, XAttrNameParam.class); @@ -617,6 +639,20 @@ public class HttpFSServer { response = Response.ok().build(); break; } + case RENAMESNAPSHOT: { + String oldSnapshotName = params.get(OldSnapshotNameParam.NAME, + OldSnapshotNameParam.class); + String snapshotName = params.get(SnapshotNameParam.NAME, + SnapshotNameParam.class); + FSOperations.FSRenameSnapshot command = + new FSOperations.FSRenameSnapshot(path, oldSnapshotName, + snapshotName); + fsExecute(user, command); + AUDIT_LOG.info("[{}] renamed snapshot [{}] to [{}]", path, + oldSnapshotName, snapshotName); + response = Response.ok().build(); + break; + } case REMOVEXATTR: { String xattrName = params.get(XAttrNameParam.NAME, XAttrNameParam.class); FSOperations.FSRemoveXAttr command = new FSOperations.FSRemoveXAttr( http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a4bff02/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java index ca11c66..553bbce 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java @@ -38,6 +38,7 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.AppendTestUtil; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; +import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.security.UserGroupInformation; @@ -74,6 +75,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.regex.Pattern; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; @@ -1034,11 +1036,12 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase { } protected enum Operation { - GET, OPEN, CREATE, APPEND, TRUNCATE, CONCAT, RENAME, DELETE, LIST_STATUS, + GET, OPEN, CREATE, APPEND, TRUNCATE, CONCAT, RENAME, DELETE, LIST_STATUS, WORKING_DIRECTORY, MKDIRS, SET_TIMES, SET_PERMISSION, SET_OWNER, SET_REPLICATION, CHECKSUM, CONTENT_SUMMARY, FILEACLS, DIRACLS, SET_XATTR, GET_XATTRS, REMOVE_XATTR, LIST_XATTRS, ENCRYPTION, LIST_STATUS_BATCH, - GETTRASHROOT, STORAGEPOLICY, ERASURE_CODING, GETFILEBLOCKLOCATIONS + GETTRASHROOT, STORAGEPOLICY, ERASURE_CODING, GETFILEBLOCKLOCATIONS, + CREATE_SNAPSHOT, RENAME_SNAPSHOT, DELETE_SNAPSHOT } private void operation(Operation op) throws Exception { @@ -1130,6 +1133,15 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase { case GETFILEBLOCKLOCATIONS: testGetFileBlockLocations(); break; + case CREATE_SNAPSHOT: + testCreateSnapshot(); + break; + case RENAME_SNAPSHOT: + testRenameSnapshot(); + break; + case DELETE_SNAPSHOT: + testDeleteSnapshot(); + break; } } @@ -1257,4 +1269,98 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase { location2.getTopologyPaths()); } } + + private void testCreateSnapshot(String snapshotName) throws Exception { + if (!this.isLocalFS()) { + Path snapshottablePath = new Path("/tmp/tmp-snap-test"); + createSnapshotTestsPreconditions(snapshottablePath); + //Now get the FileSystem instance that's being tested + FileSystem fs = this.getHttpFSFileSystem(); + if (snapshotName == null) { + fs.createSnapshot(snapshottablePath); + } else { + fs.createSnapshot(snapshottablePath, snapshotName); + } + Path snapshotsDir = new Path("/tmp/tmp-snap-test/.snapshot"); + FileStatus[] snapshotItems = fs.listStatus(snapshotsDir); + assertTrue("Should have exactly one snapshot.", + snapshotItems.length == 1); + String resultingSnapName = snapshotItems[0].getPath().getName(); + if (snapshotName == null) { + assertTrue("Snapshot auto generated name not matching pattern", + Pattern.matches("(s)(\\d{8})(-)(\\d{6})(\\.)(\\d{3})", + resultingSnapName)); + } else { + assertTrue("Snapshot name is not same as passed name.", + snapshotName.equals(resultingSnapName)); + } + cleanSnapshotTests(snapshottablePath, resultingSnapName); + } + } + + private void testCreateSnapshot() throws Exception { + testCreateSnapshot(null); + testCreateSnapshot("snap-with-name"); + } + + private void createSnapshotTestsPreconditions(Path snapshottablePath) + throws Exception { + //Needed to get a DistributedFileSystem instance, in order to + //call allowSnapshot on the newly created directory + DistributedFileSystem distributedFs = (DistributedFileSystem) + FileSystem.get(snapshottablePath.toUri(), this.getProxiedFSConf()); + distributedFs.mkdirs(snapshottablePath); + distributedFs.allowSnapshot(snapshottablePath); + Path subdirPath = new Path("/tmp/tmp-snap-test/subdir"); + distributedFs.mkdirs(subdirPath); + + } + + private void cleanSnapshotTests(Path snapshottablePath, + String resultingSnapName) throws Exception { + DistributedFileSystem distributedFs = (DistributedFileSystem) + FileSystem.get(snapshottablePath.toUri(), this.getProxiedFSConf()); + distributedFs.deleteSnapshot(snapshottablePath, resultingSnapName); + distributedFs.delete(snapshottablePath, true); + } + + private void testRenameSnapshot() throws Exception { + if (!this.isLocalFS()) { + Path snapshottablePath = new Path("/tmp/tmp-snap-test"); + createSnapshotTestsPreconditions(snapshottablePath); + //Now get the FileSystem instance that's being tested + FileSystem fs = this.getHttpFSFileSystem(); + fs.createSnapshot(snapshottablePath, "snap-to-rename"); + fs.renameSnapshot(snapshottablePath, "snap-to-rename", + "snap-new-name"); + Path snapshotsDir = new Path("/tmp/tmp-snap-test/.snapshot"); + FileStatus[] snapshotItems = fs.listStatus(snapshotsDir); + assertTrue("Should have exactly one snapshot.", + snapshotItems.length == 1); + String resultingSnapName = snapshotItems[0].getPath().getName(); + assertTrue("Snapshot name is not same as passed name.", + "snap-new-name".equals(resultingSnapName)); + cleanSnapshotTests(snapshottablePath, resultingSnapName); + } + } + + private void testDeleteSnapshot() throws Exception { + if (!this.isLocalFS()) { + Path snapshottablePath = new Path("/tmp/tmp-snap-test"); + createSnapshotTestsPreconditions(snapshottablePath); + //Now get the FileSystem instance that's being tested + FileSystem fs = this.getHttpFSFileSystem(); + fs.createSnapshot(snapshottablePath, "snap-to-delete"); + Path snapshotsDir = new Path("/tmp/tmp-snap-test/.snapshot"); + FileStatus[] snapshotItems = fs.listStatus(snapshotsDir); + assertTrue("Should have exactly one snapshot.", + snapshotItems.length == 1); + fs.deleteSnapshot(snapshottablePath, "snap-to-delete"); + snapshotItems = fs.listStatus(snapshotsDir); + assertTrue("There should be no snapshot anymore.", + snapshotItems.length == 0); + fs.delete(snapshottablePath, true); + } + } + } http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a4bff02/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java index 0e1cc20..60e70d2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java @@ -18,6 +18,7 @@ package org.apache.hadoop.fs.http.server; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.security.authentication.util.SignerSecretProvider; import org.apache.hadoop.security.authentication.util.StringSignerSecretProviderCreator; import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; @@ -71,6 +72,7 @@ import org.eclipse.jetty.webapp.WebAppContext; import com.google.common.collect.Maps; import java.util.Properties; +import java.util.regex.Pattern; import org.apache.hadoop.security.authentication.server.AuthenticationFilter; /** @@ -465,6 +467,20 @@ public class TestHttpFSServer extends HFSTestCase { */ private void putCmd(String filename, String command, String params) throws Exception { + Assert.assertEquals(HttpURLConnection.HTTP_OK, + putCmdWithReturn(filename, command, params).getResponseCode()); + } + + /** + * General-purpose http PUT command to the httpfs server, + * which returns relted HttpURLConnection instance. + * @param filename The file to operate upon + * @param command The command to perform (SETACL, etc) + * @param params Parameters, like "aclspec=..." + * @return HttpURLConnection the HttpURLConnection instance for the given PUT + */ + private HttpURLConnection putCmdWithReturn(String filename, String command, + String params) throws Exception { String user = HadoopUsersConfTestHelper.getHadoopUsers()[0]; // Remove leading / from filename if (filename.charAt(0) == '/') { @@ -478,7 +494,7 @@ public class TestHttpFSServer extends HFSTestCase { HttpURLConnection conn = (HttpURLConnection) url.openConnection(); conn.setRequestMethod("PUT"); conn.connect(); - Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); + return conn; } /** @@ -882,6 +898,108 @@ public class TestHttpFSServer extends HFSTestCase { delegationTokenCommonTests(false); } + private HttpURLConnection snapshotTestPreconditions(String httpMethod, + String snapOperation, + String additionalParams) + throws Exception { + String user = HadoopUsersConfTestHelper.getHadoopUsers()[0]; + URL url = new URL(TestJettyHelper.getJettyURL(), MessageFormat.format( + "/webhdfs/v1/tmp/tmp-snap-test/subdir?user.name={0}&op=MKDIRS", + user)); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("PUT"); + conn.connect(); + + Assert.assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_OK); + + //needed to make the given dir snapshottable + Path snapshottablePath = new Path("/tmp/tmp-snap-test"); + DistributedFileSystem dfs = + (DistributedFileSystem) FileSystem.get(snapshottablePath.toUri(), + TestHdfsHelper.getHdfsConf()); + dfs.allowSnapshot(snapshottablePath); + + //Try to create snapshot passing snapshot name + url = new URL(TestJettyHelper.getJettyURL(), MessageFormat.format( + "/webhdfs/v1/tmp/tmp-snap-test?user.name={0}&op={1}&{2}", user, + snapOperation, additionalParams)); + conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod(httpMethod); + conn.connect(); + return conn; + } + + @Test + @TestDir + @TestJetty + @TestHdfs + public void testCreateSnapshot() throws Exception { + createHttpFSServer(false, false); + final HttpURLConnection conn = snapshotTestPreconditions("PUT", + "CREATESNAPSHOT", + "snapshotname=snap-with-name"); + Assert.assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_OK); + final BufferedReader reader = + new BufferedReader(new InputStreamReader(conn.getInputStream())); + String result = reader.readLine(); + //Validates if the content format is correct + Assert.assertTrue(result. + equals("{\"Path\":\"/tmp/tmp-snap-test/.snapshot/snap-with-name\"}")); + //Validates if the snapshot is properly created under .snapshot folder + result = getStatus("/tmp/tmp-snap-test/.snapshot", + "LISTSTATUS"); + Assert.assertTrue(result.contains("snap-with-name")); + } + + @Test + @TestDir + @TestJetty + @TestHdfs + public void testCreateSnapshotNoSnapshotName() throws Exception { + createHttpFSServer(false, false); + final HttpURLConnection conn = snapshotTestPreconditions("PUT", + "CREATESNAPSHOT", + ""); + Assert.assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_OK); + final BufferedReader reader = new BufferedReader( + new InputStreamReader(conn.getInputStream())); + String result = reader.readLine(); + //Validates if the content format is correct + Assert.assertTrue(Pattern.matches( + "(\\{\\\"Path\\\"\\:\\\"/tmp/tmp-snap-test/.snapshot/s)" + + "(\\d{8})(-)(\\d{6})(\\.)(\\d{3})(\\\"\\})", result)); + //Validates if the snapshot is properly created under .snapshot folder + result = getStatus("/tmp/tmp-snap-test/.snapshot", + "LISTSTATUS"); + + Assert.assertTrue(Pattern.matches("(.+)(\\\"pathSuffix\\\":\\\"s)" + + "(\\d{8})(-)(\\d{6})(\\.)(\\d{3})(\\\")(.+)", + result)); + } + + @Test + @TestDir + @TestJetty + @TestHdfs + public void testRenameSnapshot() throws Exception { + createHttpFSServer(false, false); + HttpURLConnection conn = snapshotTestPreconditions("PUT", + "CREATESNAPSHOT", + "snapshotname=snap-to-rename"); + Assert.assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_OK); + conn = snapshotTestPreconditions("PUT", + "RENAMESNAPSHOT", + "oldsnapshotname=snap-to-rename" + + "&snapshotname=snap-renamed"); + Assert.assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_OK); + //Validates the snapshot is properly renamed under .snapshot folder + String result = getStatus("/tmp/tmp-snap-test/.snapshot", + "LISTSTATUS"); + Assert.assertTrue(result.contains("snap-renamed")); + //There should be no snapshot named snap-to-rename now + Assert.assertFalse(result.contains("snap-to-rename")); + } + @Test @TestDir @TestJetty @@ -890,4 +1008,24 @@ public class TestHttpFSServer extends HFSTestCase { createHttpFSServer(true, true); delegationTokenCommonTests(true); } + + @Test + @TestDir + @TestJetty + @TestHdfs + public void testDeleteSnapshot() throws Exception { + createHttpFSServer(false, false); + HttpURLConnection conn = snapshotTestPreconditions("PUT", + "CREATESNAPSHOT", + "snapshotname=snap-to-delete"); + Assert.assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_OK); + conn = snapshotTestPreconditions("DELETE", + "DELETESNAPSHOT", + "snapshotname=snap-to-delete"); + Assert.assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_OK); + //Validates the snapshot is not under .snapshot folder anymore + String result = getStatus("/tmp/tmp-snap-test/.snapshot", + "LISTSTATUS"); + Assert.assertFalse(result.contains("snap-to-delete")); + } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
