hadoop git commit: HDFS-8146. Protobuf changes for BlockECRecoveryCommand and its fields for making it ready for transfer to DN (Contributed by Uma Maheswara Rao G)

2015-04-18 Thread vinayakumarb 
Repository: hadoop
Updated Branches:
  refs/heads/HDFS-7285 e01dab0a9 - c6114e64a


HDFS-8146. Protobuf changes for BlockECRecoveryCommand and its fields for 
making it ready for transfer to DN (Contributed by Uma Maheswara Rao G)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c6114e64
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c6114e64
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c6114e64

Branch: refs/heads/HDFS-7285
Commit: c6114e64a4ba38f489d45c7c34234c17d30a35f1
Parents: e01dab0
Author: Vinayakumar B vinayakum...@apache.org
Authored: Sat Apr 18 23:20:45 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Sat Apr 18 23:20:45 2015 +0530

--
 .../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt|   3 +
 .../apache/hadoop/hdfs/protocolPB/PBHelper.java | 137 ++-
 .../blockmanagement/DatanodeDescriptor.java |  31 +
 .../server/blockmanagement/DatanodeManager.java |   4 +-
 .../server/protocol/BlockECRecoveryCommand.java |  80 ++-
 .../hdfs/server/protocol/DatanodeProtocol.java  |   2 +-
 .../src/main/proto/DatanodeProtocol.proto   |   8 ++
 .../src/main/proto/erasurecoding.proto  |  13 ++
 .../hadoop/hdfs/protocolPB/TestPBHelper.java|  88 
 .../namenode/TestRecoverStripedBlocks.java  |  10 +-
 10 files changed, 335 insertions(+), 41 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c6114e64/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
index 0ed61cd..40517e7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
@@ -87,3 +87,6 @@
 startup. (Hui Zheng via szetszwo)
 
 HDFS-8167. BlockManager.addBlockCollectionWithCheck should check if the 
block is a striped block. (Hui Zheng via zhz).
+
+HDFS-8146. Protobuf changes for BlockECRecoveryCommand and its fields for
+making it ready for transfer to DN (Uma Maheswara Rao G via vinayakumarb)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c6114e64/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
index 33b6a2a..5cd256f 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
@@ -28,6 +28,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.List;
@@ -99,7 +100,7 @@ import 
org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclEntryProto.AclEntryTyp
 import 
org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclEntryProto.FsActionProto;
 import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclStatusProto;
 import 
org.apache.hadoop.hdfs.protocol.proto.AclProtos.GetAclStatusResponseProto;
-import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos;
+import org.apache.hadoop.hdfs.protocol.proto.*;
 import 
org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveEntryProto;
 import 
org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveInfoExpirationProto;
 import 
org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveInfoProto;
@@ -120,6 +121,7 @@ import 
org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.ShortCircuitShmI
 import 
org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.ShortCircuitShmSlotProto;
 import 
org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.BalancerBandwidthCommandProto;
 import 
org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.BlockCommandProto;
+import 
org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.BlockECRecoveryCommandProto;
 import 
org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.BlockIdCommandProto;
 import 
org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.BlockRecoveryCommandProto;
 import 
org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.DatanodeCommandProto;
@@ -131,11 +133,11 @@ import 
org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.ReceivedDele
 import

hadoop git commit: YARN-3136. Fixed a synchronization problem of AbstractYarnScheduler#getTransferredContainers. Contributed by Sunil G

2015-04-18 Thread jianhe 
Repository: hadoop
Updated Branches:
  refs/heads/trunk f47a5763a - 497c86b48


YARN-3136. Fixed a synchronization problem of 
AbstractYarnScheduler#getTransferredContainers. Contributed by Sunil G


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/497c86b4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/497c86b4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/497c86b4

Branch: refs/heads/trunk
Commit: 497c86b485b1bb8a2eba52308646d8e1ee76bce3
Parents: f47a576
Author: Jian He jia...@apache.org
Authored: Sat Apr 18 12:45:38 2015 -0700
Committer: Jian He jia...@apache.org
Committed: Sat Apr 18 12:46:37 2015 -0700

--
 hadoop-yarn-project/CHANGES.txt |  3 ++
 .../dev-support/findbugs-exclude.xml|  8 
 .../ApplicationMasterService.java   | 47 +++-
 .../scheduler/AbstractYarnScheduler.java| 17 ++-
 4 files changed, 51 insertions(+), 24 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/497c86b4/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index dfedc8a..71fde68 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -227,6 +227,9 @@ Release 2.8.0 - UNRELEASED
 YARN-3493. RM fails to come up with error Failed to load/recover state 
 when mem settings are changed. (Jian He via wangda)
 
+YARN-3136. Fixed a synchronization problem of
+AbstractYarnScheduler#getTransferredContainers. (Sunil G via jianhe)
+
 Release 2.7.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/497c86b4/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
--
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml 
b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index 8aae152..ece8548 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -469,6 +469,14 @@
 Method name=recoverContainersOnNode /
 Bug pattern=RCN_REDUNDANT_NULLCHECK_OF_NONNULL_VALUE /
   /Match
+  Match
+Class 
name=org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler
 /
+Or
+  Field name=rmContext /
+  Field name=applications /
+/Or
+Bug pattern=IS2_INCONSISTENT_SYNC /
+  /Match
   
   !-- Following fields are used in ErrorsAndWarningsBlock, which is not a 
part of analysis of findbugs --
   Match

http://git-wip-us.apache.org/repos/asf/hadoop/blob/497c86b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
index 4ad8e87..7244b17 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
@@ -298,32 +298,35 @@ public class ApplicationMasterService extends 
AbstractService implements
 
   // For work-preserving AM restart, retrieve previous attempts' containers
   // and corresponding NM tokens.
-  ListContainer transferredContainers =
-  ((AbstractYarnScheduler) rScheduler)
+  if (app.getApplicationSubmissionContext()
+  .getKeepContainersAcrossApplicationAttempts()) {
+ListContainer transferredContainers = ((AbstractYarnScheduler) 
rScheduler)
 .getTransferredContainers(applicationAttemptId);
-  if (!transferredContainers.isEmpty()) {
-response.setContainersFromPreviousAttempts(transferredContainers);
-ListNMToken nmTokens = new ArrayListNMToken();
-for (Container container : transferredContainers) {
-  try {
-NMToken token = rmContext.getNMTokenSecretManager()
-.createAndGetNMToken(app.getUser(), applicationAttemptId,
-container);
-if (null != token) {
-

hadoop git commit: YARN-3136. Fixed a synchronization problem of AbstractYarnScheduler#getTransferredContainers. Contributed by Sunil G (cherry picked from commit 497c86b485b1bb8a2eba52308646d8e1ee76b

2015-04-18 Thread jianhe 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 3316cd435 - 6f70f1b1a


YARN-3136. Fixed a synchronization problem of 
AbstractYarnScheduler#getTransferredContainers. Contributed by Sunil G
(cherry picked from commit 497c86b485b1bb8a2eba52308646d8e1ee76bce3)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6f70f1b1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6f70f1b1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6f70f1b1

Branch: refs/heads/branch-2
Commit: 6f70f1b1afb349eaaf71e29dc4fcdeb910354e4d
Parents: 3316cd4
Author: Jian He jia...@apache.org
Authored: Sat Apr 18 12:45:38 2015 -0700
Committer: Jian He jia...@apache.org
Committed: Sat Apr 18 12:47:30 2015 -0700

--
 hadoop-yarn-project/CHANGES.txt |  3 ++
 .../dev-support/findbugs-exclude.xml|  8 
 .../ApplicationMasterService.java   | 47 +++-
 .../scheduler/AbstractYarnScheduler.java| 17 ++-
 4 files changed, 51 insertions(+), 24 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f70f1b1/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index bd5439b..a30d517 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -179,6 +179,9 @@ Release 2.8.0 - UNRELEASED
 YARN-3493. RM fails to come up with error Failed to load/recover state 
 when mem settings are changed. (Jian He via wangda)
 
+YARN-3136. Fixed a synchronization problem of
+AbstractYarnScheduler#getTransferredContainers. (Sunil G via jianhe)
+
 Release 2.7.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f70f1b1/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
--
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml 
b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index 8aae152..ece8548 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -469,6 +469,14 @@
 Method name=recoverContainersOnNode /
 Bug pattern=RCN_REDUNDANT_NULLCHECK_OF_NONNULL_VALUE /
   /Match
+  Match
+Class 
name=org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler
 /
+Or
+  Field name=rmContext /
+  Field name=applications /
+/Or
+Bug pattern=IS2_INCONSISTENT_SYNC /
+  /Match
   
   !-- Following fields are used in ErrorsAndWarningsBlock, which is not a 
part of analysis of findbugs --
   Match

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f70f1b1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
index 4ad8e87..7244b17 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
@@ -298,32 +298,35 @@ public class ApplicationMasterService extends 
AbstractService implements
 
   // For work-preserving AM restart, retrieve previous attempts' containers
   // and corresponding NM tokens.
-  ListContainer transferredContainers =
-  ((AbstractYarnScheduler) rScheduler)
+  if (app.getApplicationSubmissionContext()
+  .getKeepContainersAcrossApplicationAttempts()) {
+ListContainer transferredContainers = ((AbstractYarnScheduler) 
rScheduler)
 .getTransferredContainers(applicationAttemptId);
-  if (!transferredContainers.isEmpty()) {
-response.setContainersFromPreviousAttempts(transferredContainers);
-ListNMToken nmTokens = new ArrayListNMToken();
-for (Container container : transferredContainers) {
-  try {
-NMToken token = rmContext.getNMTokenSecretManager()
-.createAndGetNMToken(app.getUser(), applicationAttemptId,
-