[08/52] [abbrv] hadoop git commit: HDFS-10939. Reduce performance penalty of encryption zones. Contributed by Daryn sharp.

2016-10-12 Thread cnauroth 
HDFS-10939. Reduce performance penalty of encryption zones. Contributed by 
Daryn sharp.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f32e9fc8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f32e9fc8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f32e9fc8

Branch: refs/heads/HADOOP-13037
Commit: f32e9fc8f7150f0e889c0774b3ad712af26fbd65
Parents: 72a2ae6
Author: Kihwal Lee 
Authored: Thu Oct 6 15:11:14 2016 -0500
Committer: Kihwal Lee 
Committed: Thu Oct 6 15:11:14 2016 -0500

--
 .../namenode/EncryptionFaultInjector.java   |   6 +
 .../server/namenode/EncryptionZoneManager.java  |  25 +--
 .../server/namenode/FSDirEncryptionZoneOp.java  | 144 +---
 .../server/namenode/FSDirErasureCodingOp.java   |   2 +-
 .../hdfs/server/namenode/FSDirRenameOp.java |   4 +-
 .../server/namenode/FSDirStatAndListingOp.java  |  20 +--
 .../hdfs/server/namenode/FSDirWriteFileOp.java  | 163 +--
 .../hdfs/server/namenode/FSDirXAttrOp.java  |  21 +--
 .../hdfs/server/namenode/FSDirectory.java   |   5 +-
 .../hdfs/server/namenode/FSEditLogLoader.java   |   3 +-
 .../hdfs/server/namenode/FSNamesystem.java  | 115 ++---
 .../hdfs/server/namenode/XAttrStorage.java  |   7 +-
 .../apache/hadoop/hdfs/TestEncryptionZones.java |  50 --
 13 files changed, 295 insertions(+), 270 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f32e9fc8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java
index 27d8f50..104d8c3 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java
@@ -35,5 +35,11 @@ public class EncryptionFaultInjector {
   }
 
   @VisibleForTesting
+  public void startFileNoKey() throws IOException {}
+
+  @VisibleForTesting
+  public void startFileBeforeGenerateKey() throws IOException {}
+
+  @VisibleForTesting
   public void startFileAfterGenerateKey() throws IOException {}
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f32e9fc8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
index 511c616..ceeccf6 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
@@ -260,12 +260,14 @@ public class EncryptionZoneManager {
*
* @param srcIIP source IIP
* @param dstIIP destination IIP
-   * @param srcsource path, used for debugging
* @throws IOException if the src cannot be renamed to the dst
*/
-  void checkMoveValidity(INodesInPath srcIIP, INodesInPath dstIIP, String src)
+  void checkMoveValidity(INodesInPath srcIIP, INodesInPath dstIIP)
   throws IOException {
 assert dir.hasReadLock();
+if (!hasCreatedEncryptionZone()) {
+  return;
+}
 final EncryptionZoneInt srcParentEZI =
 getParentEncryptionZoneForPath(srcIIP);
 final EncryptionZoneInt dstParentEZI =
@@ -274,17 +276,17 @@ public class EncryptionZoneManager {
 final boolean dstInEZ = (dstParentEZI != null);
 if (srcInEZ && !dstInEZ) {
   throw new IOException(
-  src + " can't be moved from an encryption zone.");
+  srcIIP.getPath() + " can't be moved from an encryption zone.");
 } else if (dstInEZ && !srcInEZ) {
   throw new IOException(
-  src + " can't be moved into an encryption zone.");
+  srcIIP.getPath() + " can't be moved into an encryption zone.");
 }
 
 if (srcInEZ) {
   if (srcParentEZI != dstParentEZI) {
 final String srcEZPath = getFullPathName(srcParentEZI);
 final String dstEZPath = getFullPathName(dstParentEZI);
-final StringBuilder sb = new StringBuilder(src);
+final StringBuilder sb = new StringBuilder(srcIIP.getPath());
 sb.append(" can't be

[34/50] [abbrv] hadoop git commit: HDFS-10939. Reduce performance penalty of encryption zones. Contributed by Daryn sharp.

2016-10-08 Thread drankye 
HDFS-10939. Reduce performance penalty of encryption zones. Contributed by 
Daryn sharp.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f32e9fc8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f32e9fc8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f32e9fc8

Branch: refs/heads/HADOOP-12756
Commit: f32e9fc8f7150f0e889c0774b3ad712af26fbd65
Parents: 72a2ae6
Author: Kihwal Lee 
Authored: Thu Oct 6 15:11:14 2016 -0500
Committer: Kihwal Lee 
Committed: Thu Oct 6 15:11:14 2016 -0500

--
 .../namenode/EncryptionFaultInjector.java   |   6 +
 .../server/namenode/EncryptionZoneManager.java  |  25 +--
 .../server/namenode/FSDirEncryptionZoneOp.java  | 144 +---
 .../server/namenode/FSDirErasureCodingOp.java   |   2 +-
 .../hdfs/server/namenode/FSDirRenameOp.java |   4 +-
 .../server/namenode/FSDirStatAndListingOp.java  |  20 +--
 .../hdfs/server/namenode/FSDirWriteFileOp.java  | 163 +--
 .../hdfs/server/namenode/FSDirXAttrOp.java  |  21 +--
 .../hdfs/server/namenode/FSDirectory.java   |   5 +-
 .../hdfs/server/namenode/FSEditLogLoader.java   |   3 +-
 .../hdfs/server/namenode/FSNamesystem.java  | 115 ++---
 .../hdfs/server/namenode/XAttrStorage.java  |   7 +-
 .../apache/hadoop/hdfs/TestEncryptionZones.java |  50 --
 13 files changed, 295 insertions(+), 270 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f32e9fc8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java
index 27d8f50..104d8c3 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java
@@ -35,5 +35,11 @@ public class EncryptionFaultInjector {
   }
 
   @VisibleForTesting
+  public void startFileNoKey() throws IOException {}
+
+  @VisibleForTesting
+  public void startFileBeforeGenerateKey() throws IOException {}
+
+  @VisibleForTesting
   public void startFileAfterGenerateKey() throws IOException {}
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f32e9fc8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
index 511c616..ceeccf6 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
@@ -260,12 +260,14 @@ public class EncryptionZoneManager {
*
* @param srcIIP source IIP
* @param dstIIP destination IIP
-   * @param srcsource path, used for debugging
* @throws IOException if the src cannot be renamed to the dst
*/
-  void checkMoveValidity(INodesInPath srcIIP, INodesInPath dstIIP, String src)
+  void checkMoveValidity(INodesInPath srcIIP, INodesInPath dstIIP)
   throws IOException {
 assert dir.hasReadLock();
+if (!hasCreatedEncryptionZone()) {
+  return;
+}
 final EncryptionZoneInt srcParentEZI =
 getParentEncryptionZoneForPath(srcIIP);
 final EncryptionZoneInt dstParentEZI =
@@ -274,17 +276,17 @@ public class EncryptionZoneManager {
 final boolean dstInEZ = (dstParentEZI != null);
 if (srcInEZ && !dstInEZ) {
   throw new IOException(
-  src + " can't be moved from an encryption zone.");
+  srcIIP.getPath() + " can't be moved from an encryption zone.");
 } else if (dstInEZ && !srcInEZ) {
   throw new IOException(
-  src + " can't be moved into an encryption zone.");
+  srcIIP.getPath() + " can't be moved into an encryption zone.");
 }
 
 if (srcInEZ) {
   if (srcParentEZI != dstParentEZI) {
 final String srcEZPath = getFullPathName(srcParentEZI);
 final String dstEZPath = getFullPathName(dstParentEZI);
-final StringBuilder sb = new StringBuilder(src);
+final StringBuilder sb = new StringBuilder(srcIIP.getPath());
 sb.append(" can't be

[15/50] [abbrv] hadoop git commit: HDFS-10939. Reduce performance penalty of encryption zones. Contributed by Daryn sharp.

2016-10-07 Thread subru 
HDFS-10939. Reduce performance penalty of encryption zones. Contributed by 
Daryn sharp.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f32e9fc8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f32e9fc8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f32e9fc8

Branch: refs/heads/YARN-2915
Commit: f32e9fc8f7150f0e889c0774b3ad712af26fbd65
Parents: 72a2ae6
Author: Kihwal Lee 
Authored: Thu Oct 6 15:11:14 2016 -0500
Committer: Kihwal Lee 
Committed: Thu Oct 6 15:11:14 2016 -0500

--
 .../namenode/EncryptionFaultInjector.java   |   6 +
 .../server/namenode/EncryptionZoneManager.java  |  25 +--
 .../server/namenode/FSDirEncryptionZoneOp.java  | 144 +---
 .../server/namenode/FSDirErasureCodingOp.java   |   2 +-
 .../hdfs/server/namenode/FSDirRenameOp.java |   4 +-
 .../server/namenode/FSDirStatAndListingOp.java  |  20 +--
 .../hdfs/server/namenode/FSDirWriteFileOp.java  | 163 +--
 .../hdfs/server/namenode/FSDirXAttrOp.java  |  21 +--
 .../hdfs/server/namenode/FSDirectory.java   |   5 +-
 .../hdfs/server/namenode/FSEditLogLoader.java   |   3 +-
 .../hdfs/server/namenode/FSNamesystem.java  | 115 ++---
 .../hdfs/server/namenode/XAttrStorage.java  |   7 +-
 .../apache/hadoop/hdfs/TestEncryptionZones.java |  50 --
 13 files changed, 295 insertions(+), 270 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f32e9fc8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java
index 27d8f50..104d8c3 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java
@@ -35,5 +35,11 @@ public class EncryptionFaultInjector {
   }
 
   @VisibleForTesting
+  public void startFileNoKey() throws IOException {}
+
+  @VisibleForTesting
+  public void startFileBeforeGenerateKey() throws IOException {}
+
+  @VisibleForTesting
   public void startFileAfterGenerateKey() throws IOException {}
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f32e9fc8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
index 511c616..ceeccf6 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
@@ -260,12 +260,14 @@ public class EncryptionZoneManager {
*
* @param srcIIP source IIP
* @param dstIIP destination IIP
-   * @param srcsource path, used for debugging
* @throws IOException if the src cannot be renamed to the dst
*/
-  void checkMoveValidity(INodesInPath srcIIP, INodesInPath dstIIP, String src)
+  void checkMoveValidity(INodesInPath srcIIP, INodesInPath dstIIP)
   throws IOException {
 assert dir.hasReadLock();
+if (!hasCreatedEncryptionZone()) {
+  return;
+}
 final EncryptionZoneInt srcParentEZI =
 getParentEncryptionZoneForPath(srcIIP);
 final EncryptionZoneInt dstParentEZI =
@@ -274,17 +276,17 @@ public class EncryptionZoneManager {
 final boolean dstInEZ = (dstParentEZI != null);
 if (srcInEZ && !dstInEZ) {
   throw new IOException(
-  src + " can't be moved from an encryption zone.");
+  srcIIP.getPath() + " can't be moved from an encryption zone.");
 } else if (dstInEZ && !srcInEZ) {
   throw new IOException(
-  src + " can't be moved into an encryption zone.");
+  srcIIP.getPath() + " can't be moved into an encryption zone.");
 }
 
 if (srcInEZ) {
   if (srcParentEZI != dstParentEZI) {
 final String srcEZPath = getFullPathName(srcParentEZI);
 final String dstEZPath = getFullPathName(dstParentEZI);
-final StringBuilder sb = new StringBuilder(src);
+final StringBuilder sb = new StringBuilder(srcIIP.getPath());
 sb.append(" can't be

hadoop git commit: HDFS-10939. Reduce performance penalty of encryption zones. Contributed by Daryn Sharp.

2016-10-07 Thread kihwal 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 dfc0e3f54 -> d55b7bf1c


HDFS-10939. Reduce performance penalty of encryption zones. Contributed by 
Daryn Sharp.

(cherry picked from commit 76c03acc6df8f379e9e1b09b82d77b9db655)

Conflicts:

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d55b7bf1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d55b7bf1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d55b7bf1

Branch: refs/heads/branch-2.8
Commit: d55b7bf1c7fd2376fe7d4db3e75a1f941dfbf724
Parents: dfc0e3f
Author: Kihwal Lee 
Authored: Fri Oct 7 13:25:56 2016 -0500
Committer: Kihwal Lee 
Committed: Fri Oct 7 13:25:56 2016 -0500

--
 .../namenode/EncryptionFaultInjector.java   |   6 +
 .../server/namenode/EncryptionZoneManager.java  |  25 +--
 .../server/namenode/FSDirEncryptionZoneOp.java  | 144 ++---
 .../hdfs/server/namenode/FSDirRenameOp.java |   4 +-
 .../server/namenode/FSDirStatAndListingOp.java  |  20 +--
 .../hdfs/server/namenode/FSDirWriteFileOp.java  | 162 +--
 .../hdfs/server/namenode/FSDirXAttrOp.java  |  21 +--
 .../hdfs/server/namenode/FSDirectory.java   |   5 +-
 .../hdfs/server/namenode/FSEditLogLoader.java   |   3 +-
 .../hdfs/server/namenode/FSNamesystem.java  |  93 ++-
 .../hdfs/server/namenode/XAttrStorage.java  |   7 +-
 .../apache/hadoop/hdfs/TestEncryptionZones.java |  50 --
 12 files changed, 287 insertions(+), 253 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d55b7bf1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java
index 27d8f50..104d8c3 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java
@@ -35,5 +35,11 @@ public class EncryptionFaultInjector {
   }
 
   @VisibleForTesting
+  public void startFileNoKey() throws IOException {}
+
+  @VisibleForTesting
+  public void startFileBeforeGenerateKey() throws IOException {}
+
+  @VisibleForTesting
   public void startFileAfterGenerateKey() throws IOException {}
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d55b7bf1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
index ad5869b..0f6d4a6 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
@@ -260,12 +260,14 @@ public class EncryptionZoneManager {
*
* @param srcIIP source IIP
* @param dstIIP destination IIP
-   * @param srcsource path, used for debugging
* @throws IOException if the src cannot be renamed to the dst
*/
-  void checkMoveValidity(INodesInPath srcIIP, INodesInPath dstIIP, String src)
+  void checkMoveValidity(INodesInPath srcIIP, INodesInPath dstIIP)
   throws IOException {
 assert dir.hasReadLock();
+if (!hasCreatedEncryptionZone()) {
+  return;
+}
 final EncryptionZoneInt srcParentEZI =
 getParentEncryptionZoneForPath(srcIIP);
 final EncryptionZoneInt dstParentEZI =
@@ -274,17 +276,17 @@ public class EncryptionZoneManager {
 final boolean dstInEZ = (dstParentEZI != null);
 if (srcInEZ && !dstInEZ) {
   throw new IOException(
-  src + " can't be moved from an encryption zone.");
+  srcIIP.getPath() + " can't be moved from an encryption zone.");
 } else if (dstInEZ && !srcInEZ) {
   throw new IOException(
-  src + " can't be moved into an encryption zone.");
+  srcIIP.getPath() + " can't be moved into an encryption zone.");
 }
 
 if (srcInEZ) {
   if (srcParentEZI != dstParentEZI) {
 final String srcEZPath = getFullPathName(srcParentEZI);

hadoop git commit: HDFS-10939. Reduce performance penalty of encryption zones. Contributed by Daryn Sharp.

2016-10-07 Thread kihwal 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 cddd35e1f -> 76c03acc6


HDFS-10939. Reduce performance penalty of encryption zones. Contributed by 
Daryn Sharp.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/76c03acc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/76c03acc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/76c03acc

Branch: refs/heads/branch-2
Commit: 76c03acc6df8f379e9e1b09b82d77b9db655
Parents: cddd35e
Author: Kihwal Lee 
Authored: Fri Oct 7 13:20:05 2016 -0500
Committer: Kihwal Lee 
Committed: Fri Oct 7 13:20:05 2016 -0500

--
 .../namenode/EncryptionFaultInjector.java   |   6 +
 .../server/namenode/EncryptionZoneManager.java  |  25 +--
 .../server/namenode/FSDirEncryptionZoneOp.java  | 144 ++---
 .../hdfs/server/namenode/FSDirRenameOp.java |   4 +-
 .../server/namenode/FSDirStatAndListingOp.java  |  20 +--
 .../hdfs/server/namenode/FSDirWriteFileOp.java  | 162 +--
 .../hdfs/server/namenode/FSDirXAttrOp.java  |  21 +--
 .../hdfs/server/namenode/FSDirectory.java   |   5 +-
 .../hdfs/server/namenode/FSEditLogLoader.java   |   3 +-
 .../hdfs/server/namenode/FSNamesystem.java  |  93 ++-
 .../hdfs/server/namenode/XAttrStorage.java  |   7 +-
 .../apache/hadoop/hdfs/TestEncryptionZones.java |  50 --
 12 files changed, 287 insertions(+), 253 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/76c03acc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java
index 27d8f50..104d8c3 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java
@@ -35,5 +35,11 @@ public class EncryptionFaultInjector {
   }
 
   @VisibleForTesting
+  public void startFileNoKey() throws IOException {}
+
+  @VisibleForTesting
+  public void startFileBeforeGenerateKey() throws IOException {}
+
+  @VisibleForTesting
   public void startFileAfterGenerateKey() throws IOException {}
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/76c03acc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
index ad5869b..0f6d4a6 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
@@ -260,12 +260,14 @@ public class EncryptionZoneManager {
*
* @param srcIIP source IIP
* @param dstIIP destination IIP
-   * @param srcsource path, used for debugging
* @throws IOException if the src cannot be renamed to the dst
*/
-  void checkMoveValidity(INodesInPath srcIIP, INodesInPath dstIIP, String src)
+  void checkMoveValidity(INodesInPath srcIIP, INodesInPath dstIIP)
   throws IOException {
 assert dir.hasReadLock();
+if (!hasCreatedEncryptionZone()) {
+  return;
+}
 final EncryptionZoneInt srcParentEZI =
 getParentEncryptionZoneForPath(srcIIP);
 final EncryptionZoneInt dstParentEZI =
@@ -274,17 +276,17 @@ public class EncryptionZoneManager {
 final boolean dstInEZ = (dstParentEZI != null);
 if (srcInEZ && !dstInEZ) {
   throw new IOException(
-  src + " can't be moved from an encryption zone.");
+  srcIIP.getPath() + " can't be moved from an encryption zone.");
 } else if (dstInEZ && !srcInEZ) {
   throw new IOException(
-  src + " can't be moved into an encryption zone.");
+  srcIIP.getPath() + " can't be moved into an encryption zone.");
 }
 
 if (srcInEZ) {
   if (srcParentEZI != dstParentEZI) {
 final String srcEZPath = getFullPathName(srcParentEZI);
 final String dstEZPath = getFullPathName(dstParentEZI);
-final StringBuilder sb = new StringBuilder(src);
+final StringBuilder sb = new StringBuilder(srcIIP.getPath());

hadoop git commit: HDFS-10939. Reduce performance penalty of encryption zones. Contributed by Daryn sharp.

2016-10-06 Thread kihwal 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 72a2ae645 -> f32e9fc8f


HDFS-10939. Reduce performance penalty of encryption zones. Contributed by 
Daryn sharp.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f32e9fc8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f32e9fc8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f32e9fc8

Branch: refs/heads/trunk
Commit: f32e9fc8f7150f0e889c0774b3ad712af26fbd65
Parents: 72a2ae6
Author: Kihwal Lee 
Authored: Thu Oct 6 15:11:14 2016 -0500
Committer: Kihwal Lee 
Committed: Thu Oct 6 15:11:14 2016 -0500

--
 .../namenode/EncryptionFaultInjector.java   |   6 +
 .../server/namenode/EncryptionZoneManager.java  |  25 +--
 .../server/namenode/FSDirEncryptionZoneOp.java  | 144 +---
 .../server/namenode/FSDirErasureCodingOp.java   |   2 +-
 .../hdfs/server/namenode/FSDirRenameOp.java |   4 +-
 .../server/namenode/FSDirStatAndListingOp.java  |  20 +--
 .../hdfs/server/namenode/FSDirWriteFileOp.java  | 163 +--
 .../hdfs/server/namenode/FSDirXAttrOp.java  |  21 +--
 .../hdfs/server/namenode/FSDirectory.java   |   5 +-
 .../hdfs/server/namenode/FSEditLogLoader.java   |   3 +-
 .../hdfs/server/namenode/FSNamesystem.java  | 115 ++---
 .../hdfs/server/namenode/XAttrStorage.java  |   7 +-
 .../apache/hadoop/hdfs/TestEncryptionZones.java |  50 --
 13 files changed, 295 insertions(+), 270 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f32e9fc8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java
index 27d8f50..104d8c3 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java
@@ -35,5 +35,11 @@ public class EncryptionFaultInjector {
   }
 
   @VisibleForTesting
+  public void startFileNoKey() throws IOException {}
+
+  @VisibleForTesting
+  public void startFileBeforeGenerateKey() throws IOException {}
+
+  @VisibleForTesting
   public void startFileAfterGenerateKey() throws IOException {}
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f32e9fc8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
index 511c616..ceeccf6 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
@@ -260,12 +260,14 @@ public class EncryptionZoneManager {
*
* @param srcIIP source IIP
* @param dstIIP destination IIP
-   * @param srcsource path, used for debugging
* @throws IOException if the src cannot be renamed to the dst
*/
-  void checkMoveValidity(INodesInPath srcIIP, INodesInPath dstIIP, String src)
+  void checkMoveValidity(INodesInPath srcIIP, INodesInPath dstIIP)
   throws IOException {
 assert dir.hasReadLock();
+if (!hasCreatedEncryptionZone()) {
+  return;
+}
 final EncryptionZoneInt srcParentEZI =
 getParentEncryptionZoneForPath(srcIIP);
 final EncryptionZoneInt dstParentEZI =
@@ -274,17 +276,17 @@ public class EncryptionZoneManager {
 final boolean dstInEZ = (dstParentEZI != null);
 if (srcInEZ && !dstInEZ) {
   throw new IOException(
-  src + " can't be moved from an encryption zone.");
+  srcIIP.getPath() + " can't be moved from an encryption zone.");
 } else if (dstInEZ && !srcInEZ) {
   throw new IOException(
-  src + " can't be moved into an encryption zone.");
+  srcIIP.getPath() + " can't be moved into an encryption zone.");
 }
 
 if (srcInEZ) {
   if (srcParentEZI != dstParentEZI) {
 final String srcEZPath = getFullPathName(srcParentEZI);
 final String dstEZPath = getFullPathName(dstParentEZI);
-final StringBuilder sb = new StringBuilder(src);
+final StringBuilder