Repository: hadoop Updated Branches: refs/heads/branch-2.8 749e106c1 -> 4b0b466f8
HDFS-13109. Support fully qualified hdfs path in EZ commands. Contributed by Hanisha Koneru. (cherry picked from commit edf9445708ffb7a9e59cb933e049b540f99add1e) (cherry picked from commit 321766bcc8f8c7ac3d0091a065ccabf8a16c13c5) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4b0b466f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4b0b466f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4b0b466f Branch: refs/heads/branch-2.8 Commit: 4b0b466f83584aa7f87da61029b918ea530fc8fd Parents: 749e106 Author: Xiaoyu Yao <[email protected]> Authored: Tue Mar 6 16:44:20 2018 -0800 Committer: Xiaoyu Yao <[email protected]> Committed: Tue Mar 6 19:35:17 2018 -0800 ---------------------------------------------------------------------- .../hadoop/hdfs/DistributedFileSystem.java | 65 ++++++++++++++++++++ .../apache/hadoop/hdfs/client/HdfsAdmin.java | 45 +------------- .../apache/hadoop/hdfs/TestEncryptionZones.java | 44 +++++++++++++ 3 files changed, 112 insertions(+), 42 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b0b466f/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index 50a2576..d8a7419 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -41,6 +41,7 @@ import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FSLinkResolver; +import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.fs.FileChecksum; import org.apache.hadoop.fs.FileEncryptionInfo; import org.apache.hadoop.fs.FileStatus; @@ -2306,6 +2307,70 @@ public class DistributedFileSystem extends FileSystem }.resolve(this, absF); } + /* HDFS only */ + public void provisionEZTrash(final Path path, + final FsPermission trashPermission) throws IOException { + Path absF = fixRelativePart(path); + new FileSystemLinkResolver<Void>() { + @Override + public Void doCall(Path p) throws IOException { + provisionEZTrash(getPathName(p), trashPermission); + return null; + } + + @Override + public Void next(FileSystem fs, Path p) throws IOException { + if (fs instanceof DistributedFileSystem) { + DistributedFileSystem myDfs = (DistributedFileSystem)fs; + myDfs.provisionEZTrash(p, trashPermission); + return null; + } + throw new UnsupportedOperationException("Cannot provisionEZTrash " + + "through a symlink to a non-DistributedFileSystem: " + fs + " -> " + + p); + } + }.resolve(this, absF); + } + + private void provisionEZTrash(String path, FsPermission trashPermission) + throws IOException { + // make sure the path is an EZ + EncryptionZone ez = dfs.getEZForPath(path); + if (ez == null) { + throw new IllegalArgumentException(path + " is not an encryption zone."); + } + + String ezPath = ez.getPath(); + if (!path.toString().equals(ezPath)) { + throw new IllegalArgumentException(path + " is not the root of an " + + "encryption zone. Do you mean " + ez.getPath() + "?"); + } + + // check if the trash directory exists + Path trashPath = new Path(ez.getPath(), FileSystem.TRASH_PREFIX); + try { + FileStatus trashFileStatus = getFileStatus(trashPath); + String errMessage = "Will not provision new trash directory for " + + "encryption zone " + ez.getPath() + ". Path already exists."; + if (!trashFileStatus.isDirectory()) { + errMessage += "\r\n" + + "Warning: " + trashPath.toString() + " is not a directory"; + } + if (!trashFileStatus.getPermission().equals(trashPermission)) { + errMessage += "\r\n" + + "Warning: the permission of " + + trashPath.toString() + " is not " + trashPermission; + } + throw new FileAlreadyExistsException(errMessage); + } catch (FileNotFoundException ignored) { + // no trash path + } + + // Update the permission bits + mkdir(trashPath, trashPermission); + setPermission(trashPath, trashPermission); + } + @Override public void setXAttr(Path path, final String name, final byte[] value, final EnumSet<XAttrSetFlag> flag) throws IOException { http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b0b466f/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java index 189c7c6..a880897 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java @@ -31,7 +31,6 @@ import org.apache.hadoop.crypto.key.KeyProvider; import org.apache.hadoop.fs.BlockStoragePolicySpi; import org.apache.hadoop.fs.CacheFlag; import org.apache.hadoop.fs.FileEncryptionInfo; -import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; @@ -323,7 +322,7 @@ public class HdfsAdmin { throw new HadoopIllegalArgumentException( "can not have both PROVISION_TRASH and NO_TRASH flags"); } - this.provisionEZTrash(path); + dfs.provisionEZTrash(path, TRASH_PERMISSION); } } @@ -334,7 +333,7 @@ public class HdfsAdmin { * @throws IOException if the trash directory can not be created. */ public void provisionEncryptionZoneTrash(Path path) throws IOException { - this.provisionEZTrash(path); + dfs.provisionEZTrash(path, TRASH_PERMISSION); } /** @@ -472,45 +471,7 @@ public class HdfsAdmin { return dfs.getAllStoragePolicies(); } - private void provisionEZTrash(Path path) throws IOException { - // make sure the path is an EZ - EncryptionZone ez = dfs.getEZForPath(path); - if (ez == null) { - throw new IllegalArgumentException(path + " is not an encryption zone."); - } - - String ezPath = ez.getPath(); - if (!path.toString().equals(ezPath)) { - throw new IllegalArgumentException(path + " is not the root of an " + - "encryption zone. Do you mean " + ez.getPath() + "?"); - } - - // check if the trash directory exists - - Path trashPath = new Path(ez.getPath(), FileSystem.TRASH_PREFIX); - - if (dfs.exists(trashPath)) { - String errMessage = "Will not provision new trash directory for " + - "encryption zone " + ez.getPath() + ". Path already exists."; - FileStatus trashFileStatus = dfs.getFileStatus(trashPath); - if (!trashFileStatus.isDirectory()) { - errMessage += "\r\n" + - "Warning: " + trashPath.toString() + " is not a directory"; - } - if (!trashFileStatus.getPermission().equals(TRASH_PERMISSION)) { - errMessage += "\r\n" + - "Warning: the permission of " + - trashPath.toString() + " is not " + TRASH_PERMISSION; - } - throw new IOException(errMessage); - } - - // Update the permission bits - dfs.mkdir(trashPath, TRASH_PERMISSION); - dfs.setPermission(trashPath, TRASH_PERMISSION); - } - - /** + /** * Returns a RemoteIterator which can be used to list all open files * currently managed by the NameNode. For large numbers of open files, * iterator will fetch the list in batches of configured size. http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b0b466f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java index d9b7b6b..276cc60 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java @@ -538,6 +538,50 @@ public class TestEncryptionZones { assertZonePresent(null, rootDir.toString()); } + @Test + public void testEZwithFullyQualifiedPath() throws Exception { + /* Test failure of create EZ on a directory that doesn't exist. */ + final Path zoneParent = new Path("/zones"); + final Path zone1 = new Path(zoneParent, "zone1"); + final Path zone1FQP = new Path(cluster.getURI().toString(), zone1); + final Path zone2 = new Path(zoneParent, "zone2"); + final Path zone2FQP = new Path(cluster.getURI().toString(), zone2); + + int numZones = 0; + EnumSet<CreateEncryptionZoneFlag> withTrash = EnumSet + .of(CreateEncryptionZoneFlag.PROVISION_TRASH); + + // Create EZ with Trash using FQP + fsWrapper.mkdir(zone1FQP, FsPermission.getDirDefault(), true); + dfsAdmin.createEncryptionZone(zone1FQP, TEST_KEY, withTrash); + assertNumZones(++numZones); + assertZonePresent(TEST_KEY, zone1.toString()); + // Check that zone1 contains a .Trash directory + final Path zone1Trash = new Path(zone1, fs.TRASH_PREFIX); + assertTrue("CreateEncryptionZone with trash enabled should create a " + + ".Trash directory in the EZ", fs.exists(zone1Trash)); + + // getEncryptionZoneForPath for FQP should return the path component + EncryptionZone ezForZone1 = dfsAdmin.getEncryptionZoneForPath(zone1FQP); + assertTrue("getEncryptionZoneForPath for fully qualified path should " + + "return the path component", + ezForZone1.getPath().equals(zone1.toString())); + + // Create EZ without Trash + fsWrapper.mkdir(zone2FQP, FsPermission.getDirDefault(), true); + dfsAdmin.createEncryptionZone(zone2FQP, TEST_KEY, NO_TRASH); + assertNumZones(++numZones); + assertZonePresent(TEST_KEY, zone2.toString()); + + // Provision Trash on zone2 using FQP + dfsAdmin.provisionEncryptionZoneTrash(zone2FQP); + EncryptionZone ezForZone2 = dfsAdmin.getEncryptionZoneForPath(zone2FQP); + Path ezTrashForZone2 = new Path(ezForZone2.getPath(), + FileSystem.TRASH_PREFIX); + assertTrue("provisionEZTrash with fully qualified path should create " + + "trash directory ", fsWrapper.exists(ezTrashForZone2)); + } + /** * Test listing encryption zones as a non super user. */ --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
