hadoop git commit: HDFS-12336. Listing encryption zones still fails when deleted EZ is not a direct child of snapshottable directory. Contributed by Wellington Chevreuil.

2017-08-30 Thread xiao 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 8724e3656 -> 98436a2eb


HDFS-12336. Listing encryption zones still fails when deleted EZ is not a 
direct child of snapshottable directory. Contributed by Wellington Chevreuil.

(cherry picked from commit 441c472552eadf66c748ab9493fd02d2c914994a)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/98436a2e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/98436a2e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/98436a2e

Branch: refs/heads/branch-2.8
Commit: 98436a2eb77e2ddee3776262e55b01c90397f26a
Parents: 8724e36
Author: Xiao Chen 
Authored: Wed Aug 30 09:42:04 2017 -0700
Committer: Xiao Chen 
Committed: Wed Aug 30 09:43:27 2017 -0700

--
 .../server/namenode/EncryptionZoneManager.java  |  2 +-
 .../hadoop/hdfs/server/namenode/INode.java  | 11 -
 .../apache/hadoop/hdfs/TestEncryptionZones.java | 47 
 .../namenode/TestEncryptionZoneManager.java | 26 +++
 .../src/test/resources/testCryptoConf.xml   | 33 ++
 5 files changed, 117 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/98436a2e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
index 3e670ce..40bc36a 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
@@ -373,7 +373,7 @@ public class EncryptionZoneManager {
   final String pathName = getFullPathName(ezi);
   INode inode = dir.getInode(ezi.getINodeId());
   INode lastINode = null;
-  if (inode.getParent() != null || inode.isRoot()) {
+  if (INode.isValidAbsolutePath(pathName)) {
 INodesInPath iip = dir.getINodesInPath(pathName, DirOp.READ_LINK);
 lastINode = iip.getLastINode();
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/98436a2e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
index c6258a1..779ae13 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
@@ -776,8 +776,17 @@ public abstract class INode implements INodeAttributes, 
Diff.Element {
 return StringUtils.split(path, Path.SEPARATOR_CHAR);
   }
 
+  /**
+   * Verifies if the path informed is a valid absolute path.
+   * @param path the absolute path to validate.
+   * @return true if the path is valid.
+   */
+  static boolean isValidAbsolutePath(final String path){
+return path != null && path.startsWith(Path.SEPARATOR);
+  }
+
   private static void checkAbsolutePath(final String path) {
-if (path == null || !path.startsWith(Path.SEPARATOR)) {
+if (!isValidAbsolutePath(path)) {
   throw new AssertionError("Absolute path required, but got '"
   + path + "'");
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/98436a2e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
index 458ee28..e0c9c17 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
@@ -1850,4 +1850,51 @@ public class TestEncryptionZones {
 // Read them back in and compare byte-by-byte
 verifyFilesEqual(fs, baseFile, encFile1, len);
   }
+
+  /**
+   * Test listing encryption zones after zones had been deleted,
+   * but still exist under snapshots. This test first moves EZs
+   * to trash folder, so that an inodereference is created for the EZ,
+   * then it removes the EZ

hadoop git commit: HDFS-12336. Listing encryption zones still fails when deleted EZ is not a direct child of snapshottable directory. Contributed by Wellington Chevreuil.

2017-08-30 Thread xiao 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 877d96b83 -> 858715d05


HDFS-12336. Listing encryption zones still fails when deleted EZ is not a 
direct child of snapshottable directory. Contributed by Wellington Chevreuil.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/858715d0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/858715d0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/858715d0

Branch: refs/heads/branch-2
Commit: 858715d054e38fa98add5df8fc089737f31a0218
Parents: 877d96b
Author: Xiao Chen 
Authored: Wed Aug 30 09:42:04 2017 -0700
Committer: Xiao Chen 
Committed: Wed Aug 30 09:43:12 2017 -0700

--
 .../server/namenode/EncryptionZoneManager.java  |  2 +-
 .../hadoop/hdfs/server/namenode/INode.java  | 11 -
 .../apache/hadoop/hdfs/TestEncryptionZones.java | 47 
 .../namenode/TestEncryptionZoneManager.java | 26 +++
 .../src/test/resources/testCryptoConf.xml   | 33 ++
 5 files changed, 117 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/858715d0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
index 3e670ce..40bc36a 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
@@ -373,7 +373,7 @@ public class EncryptionZoneManager {
   final String pathName = getFullPathName(ezi);
   INode inode = dir.getInode(ezi.getINodeId());
   INode lastINode = null;
-  if (inode.getParent() != null || inode.isRoot()) {
+  if (INode.isValidAbsolutePath(pathName)) {
 INodesInPath iip = dir.getINodesInPath(pathName, DirOp.READ_LINK);
 lastINode = iip.getLastINode();
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/858715d0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
index fcae42e..91b1154 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
@@ -782,8 +782,17 @@ public abstract class INode implements INodeAttributes, 
Diff.Element {
 return StringUtils.split(path, Path.SEPARATOR_CHAR);
   }
 
+  /**
+   * Verifies if the path informed is a valid absolute path.
+   * @param path the absolute path to validate.
+   * @return true if the path is valid.
+   */
+  static boolean isValidAbsolutePath(final String path){
+return path != null && path.startsWith(Path.SEPARATOR);
+  }
+
   private static void checkAbsolutePath(final String path) {
-if (path == null || !path.startsWith(Path.SEPARATOR)) {
+if (!isValidAbsolutePath(path)) {
   throw new AssertionError("Absolute path required, but got '"
   + path + "'");
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/858715d0/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
index 195f3bd..1dda8b3 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
@@ -1885,4 +1885,51 @@ public class TestEncryptionZones {
 // Read them back in and compare byte-by-byte
 verifyFilesEqual(fs, baseFile, encFile1, len);
   }
+
+  /**
+   * Test listing encryption zones after zones had been deleted,
+   * but still exist under snapshots. This test first moves EZs
+   * to trash folder, so that an inodereference is created for the EZ,
+   * then it removes the EZ from trash folder to emulate condition where
+   * the EZ inode will not

hadoop git commit: HDFS-12336. Listing encryption zones still fails when deleted EZ is not a direct child of snapshottable directory. Contributed by Wellington Chevreuil.

2017-08-29 Thread xiao 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 8201ed800 -> f3661fd08


HDFS-12336. Listing encryption zones still fails when deleted EZ is not a 
direct child of snapshottable directory. Contributed by Wellington Chevreuil.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f3661fd0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f3661fd0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f3661fd0

Branch: refs/heads/trunk
Commit: f3661fd08e03440d02452b46ba3ae3cff2b75ba7
Parents: 8201ed8
Author: Xiao Chen 
Authored: Tue Aug 29 21:28:02 2017 -0700
Committer: Xiao Chen 
Committed: Tue Aug 29 21:30:02 2017 -0700

--
 .../server/namenode/EncryptionZoneManager.java  |  2 +-
 .../hadoop/hdfs/server/namenode/INode.java  | 11 -
 .../apache/hadoop/hdfs/TestEncryptionZones.java | 47 
 .../namenode/TestEncryptionZoneManager.java | 26 +++
 .../src/test/resources/testCryptoConf.xml   | 32 +
 5 files changed, 116 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3661fd0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
index d6302ba..f4cf8f2 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
@@ -573,7 +573,7 @@ public class EncryptionZoneManager {
   return false;
 }
 INode lastINode = null;
-if (inode.getParent() != null || inode.isRoot()) {
+if (INode.isValidAbsolutePath(zonePath)) {
   INodesInPath iip = dir.getINodesInPath(zonePath, DirOp.READ_LINK);
   lastINode = iip.getLastINode();
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3661fd0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
index 874563d..34bfe10 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
@@ -783,8 +783,17 @@ public abstract class INode implements INodeAttributes, 
Diff.Element {
 return StringUtils.split(path, Path.SEPARATOR_CHAR);
   }
 
+  /**
+   * Verifies if the path informed is a valid absolute path.
+   * @param path the absolute path to validate.
+   * @return true if the path is valid.
+   */
+  static boolean isValidAbsolutePath(final String path){
+return path != null && path.startsWith(Path.SEPARATOR);
+  }
+
   private static void checkAbsolutePath(final String path) {
-if (path == null || !path.startsWith(Path.SEPARATOR)) {
+if (!isValidAbsolutePath(path)) {
   throw new AssertionError("Absolute path required, but got '"
   + path + "'");
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3661fd0/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
index bf02db3..870023b 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
@@ -1867,4 +1867,51 @@ public class TestEncryptionZones {
 // Read them back in and compare byte-by-byte
 verifyFilesEqual(fs, baseFile, encFile1, len);
   }
+
+  /**
+   * Test listing encryption zones after zones had been deleted,
+   * but still exist under snapshots. This test first moves EZs
+   * to trash folder, so that an inodereference is created for the EZ,
+   * then it removes the EZ from trash folder to emulate condition where
+   * the EZ inode will not be complete.
+   */
+  @Test
+  public void testListEncryptionZonesWithSnapshots() throws