hadoop git commit: YARN-4438. Implement RM leader election with curator. Contributed by Jian He

2016-12-13 Thread junping_du 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 6a18ae849 -> 120f3a0ff


YARN-4438. Implement RM leader election with curator. Contributed by Jian He

(cherry picked from commit 89022f8d4bac0e9d0b848fd91e9c4d700fe1cdbe)
(cherry picked from commit 2cbbf76c3dd9570e08914077de8e610d93ee0c9e)

Conflicts:

hadoop-yarn-project/CHANGES.txt


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/120f3a0f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/120f3a0f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/120f3a0f

Branch: refs/heads/branch-2.8
Commit: 120f3a0ff9da6252aa8004ee43242d75d36805f0
Parents: 6a18ae8
Author: Xuan 
Authored: Thu Jan 7 14:33:06 2016 -0800
Committer: Junping Du 
Committed: Tue Dec 13 13:30:15 2016 -0800

--
 .../hadoop/yarn/conf/YarnConfiguration.java |   5 +
 .../yarn/conf/TestYarnConfigurationFields.java  |   1 +
 .../server/resourcemanager/AdminService.java|  49 +++-
 .../resourcemanager/LeaderElectorService.java   | 144 ++
 .../yarn/server/resourcemanager/RMContext.java  |   4 +
 .../server/resourcemanager/RMContextImpl.java   |  11 +
 .../server/resourcemanager/ResourceManager.java |  24 +-
 .../TestLeaderElectorService.java   | 269 +++
 .../yarn/server/resourcemanager/TestRMHA.java   |   8 +-
 9 files changed, 494 insertions(+), 21 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/120f3a0f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 5ce171c..8f2d02c 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -550,6 +550,11 @@ public class YarnConfiguration extends Configuration {
   public static final String RM_HA_FC_ELECTOR_ZK_RETRIES_KEY = RM_HA_PREFIX
   + "failover-controller.active-standby-elector.zk.retries";
 
+  @Private
+  public static final String CURATOR_LEADER_ELECTOR =
+  RM_HA_PREFIX + "curator-leader-elector.enabled";
+  public static final boolean DEFAULT_CURATOR_LEADER_ELECTOR_ENABLED = false;
+
   
   // RM state store configs
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/120f3a0f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
index dcfbd5d..65e1c9c 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
@@ -90,6 +90,7 @@ public class TestYarnConfigurationFields extends 
TestConfigurationFieldsBase {
 .add(YarnConfiguration.DEFAULT_SHARED_CACHE_CHECKSUM_ALGO_IMPL);
 configurationPropsToSkipCompare
 .add(YarnConfiguration.DEFAULT_AMRM_PROXY_INTERCEPTOR_CLASS_PIPELINE);
+
configurationPropsToSkipCompare.add(YarnConfiguration.CURATOR_LEADER_ELECTOR);
 
 // Ignore blacklisting nodes for AM failures feature since it is still a
 // "work in progress"

http://git-wip-us.apache.org/repos/asf/hadoop/blob/120f3a0f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
index 20ce76b..82e8578 100644
---

hadoop git commit: YARN-4438. Implement RM leader election with curator. Contributed by Jian He

2016-01-07 Thread xgong 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 ee9611719 -> 2cbbf76c3


YARN-4438. Implement RM leader election with curator. Contributed by Jian He

(cherry picked from commit 89022f8d4bac0e9d0b848fd91e9c4d700fe1cdbe)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2cbbf76c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2cbbf76c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2cbbf76c

Branch: refs/heads/branch-2
Commit: 2cbbf76c3dd9570e08914077de8e610d93ee0c9e
Parents: ee96117
Author: Xuan 
Authored: Thu Jan 7 14:33:06 2016 -0800
Committer: Xuan 
Committed: Thu Jan 7 14:34:14 2016 -0800

--
 hadoop-yarn-project/CHANGES.txt |   2 +
 .../hadoop/yarn/conf/YarnConfiguration.java |   5 +
 .../yarn/conf/TestYarnConfigurationFields.java  |   1 +
 .../server/resourcemanager/AdminService.java|  49 +++-
 .../resourcemanager/LeaderElectorService.java   | 144 ++
 .../yarn/server/resourcemanager/RMContext.java  |   4 +
 .../server/resourcemanager/RMContextImpl.java   |  11 +
 .../server/resourcemanager/ResourceManager.java |  24 +-
 .../TestLeaderElectorService.java   | 269 +++
 .../yarn/server/resourcemanager/TestRMHA.java   |   8 +-
 10 files changed, 496 insertions(+), 21 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/2cbbf76c/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index fc1f221..17280e3 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -28,6 +28,8 @@ Release 2.9.0 - UNRELEASED
 
 YARN-4524. Cleanup AppSchedulingInfo. (Karthik Kambatla via wangda)
 
+YARN-4438. Implement RM leader election with curator. (Jian He via xgong)
+
   OPTIMIZATIONS
 
   BUG FIXES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2cbbf76c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 6a7857a..4316a37 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -545,6 +545,11 @@ public class YarnConfiguration extends Configuration {
   public static final String RM_HA_FC_ELECTOR_ZK_RETRIES_KEY = RM_HA_PREFIX
   + "failover-controller.active-standby-elector.zk.retries";
 
+  @Private
+  public static final String CURATOR_LEADER_ELECTOR =
+  RM_HA_PREFIX + "curator-leader-elector.enabled";
+  public static final boolean DEFAULT_CURATOR_LEADER_ELECTOR_ENABLED = false;
+
   
   // RM state store configs
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2cbbf76c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
index 41c3d87..0e508ed 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
@@ -90,6 +90,7 @@ public class TestYarnConfigurationFields extends 
TestConfigurationFieldsBase {
 .add(YarnConfiguration.DEFAULT_SHARED_CACHE_CHECKSUM_ALGO_IMPL);
 configurationPropsToSkipCompare
 .add(YarnConfiguration.DEFAULT_AMRM_PROXY_INTERCEPTOR_CLASS_PIPELINE);
+
configurationPropsToSkipCompare.add(YarnConfiguration.CURATOR_LEADER_ELECTOR);
 
 // Ignore all YARN Application Timeline Service (version 1) properties
 configurationPrefixToSkipCompare.add("yarn.timeline-service.");

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2cbbf76c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java

hadoop git commit: YARN-4438. Implement RM leader election with curator. Contributed by Jian He

2016-01-07 Thread xgong 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 52b77577c -> 89022f8d4


YARN-4438. Implement RM leader election with curator. Contributed by Jian He


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/89022f8d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/89022f8d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/89022f8d

Branch: refs/heads/trunk
Commit: 89022f8d4bac0e9d0b848fd91e9c4d700fe1cdbe
Parents: 52b7757
Author: Xuan 
Authored: Thu Jan 7 14:33:06 2016 -0800
Committer: Xuan 
Committed: Thu Jan 7 14:33:06 2016 -0800

--
 hadoop-yarn-project/CHANGES.txt |   2 +
 .../hadoop/yarn/conf/YarnConfiguration.java |   5 +
 .../yarn/conf/TestYarnConfigurationFields.java  |   1 +
 .../server/resourcemanager/AdminService.java|  49 +++-
 .../resourcemanager/LeaderElectorService.java   | 144 ++
 .../yarn/server/resourcemanager/RMContext.java  |   4 +
 .../server/resourcemanager/RMContextImpl.java   |  11 +
 .../server/resourcemanager/ResourceManager.java |  24 +-
 .../TestLeaderElectorService.java   | 269 +++
 .../yarn/server/resourcemanager/TestRMHA.java   |   8 +-
 10 files changed, 496 insertions(+), 21 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/89022f8d/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 5273614..00d31d8 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -86,6 +86,8 @@ Release 2.9.0 - UNRELEASED
 
 YARN-4524. Cleanup AppSchedulingInfo. (Karthik Kambatla via wangda)
 
+YARN-4438. Implement RM leader election with curator. (Jian He via xgong)
+
   OPTIMIZATIONS
 
   BUG FIXES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/89022f8d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 9a1eb54..37c81ec 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -545,6 +545,11 @@ public class YarnConfiguration extends Configuration {
   public static final String RM_HA_FC_ELECTOR_ZK_RETRIES_KEY = RM_HA_PREFIX
   + "failover-controller.active-standby-elector.zk.retries";
 
+  @Private
+  public static final String CURATOR_LEADER_ELECTOR =
+  RM_HA_PREFIX + "curator-leader-elector.enabled";
+  public static final boolean DEFAULT_CURATOR_LEADER_ELECTOR_ENABLED = false;
+
   
   // RM state store configs
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/89022f8d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
index 41c3d87..0e508ed 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
@@ -90,6 +90,7 @@ public class TestYarnConfigurationFields extends 
TestConfigurationFieldsBase {
 .add(YarnConfiguration.DEFAULT_SHARED_CACHE_CHECKSUM_ALGO_IMPL);
 configurationPropsToSkipCompare
 .add(YarnConfiguration.DEFAULT_AMRM_PROXY_INTERCEPTOR_CLASS_PIPELINE);
+
configurationPropsToSkipCompare.add(YarnConfiguration.CURATOR_LEADER_ELECTOR);
 
 // Ignore all YARN Application Timeline Service (version 1) properties
 configurationPrefixToSkipCompare.add("yarn.timeline-service.");

http://git-wip-us.apache.org/repos/asf/hadoop/blob/89022f8d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
--
diff --git

[49/50] [abbrv] hadoop git commit: YARN-4438. Implement RM leader election with curator. Contributed by Jian He

2016-01-07 Thread arp 
YARN-4438. Implement RM leader election with curator. Contributed by Jian He


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/89022f8d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/89022f8d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/89022f8d

Branch: refs/heads/HDFS-1312
Commit: 89022f8d4bac0e9d0b848fd91e9c4d700fe1cdbe
Parents: 52b7757
Author: Xuan 
Authored: Thu Jan 7 14:33:06 2016 -0800
Committer: Xuan 
Committed: Thu Jan 7 14:33:06 2016 -0800

--
 hadoop-yarn-project/CHANGES.txt |   2 +
 .../hadoop/yarn/conf/YarnConfiguration.java |   5 +
 .../yarn/conf/TestYarnConfigurationFields.java  |   1 +
 .../server/resourcemanager/AdminService.java|  49 +++-
 .../resourcemanager/LeaderElectorService.java   | 144 ++
 .../yarn/server/resourcemanager/RMContext.java  |   4 +
 .../server/resourcemanager/RMContextImpl.java   |  11 +
 .../server/resourcemanager/ResourceManager.java |  24 +-
 .../TestLeaderElectorService.java   | 269 +++
 .../yarn/server/resourcemanager/TestRMHA.java   |   8 +-
 10 files changed, 496 insertions(+), 21 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/89022f8d/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 5273614..00d31d8 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -86,6 +86,8 @@ Release 2.9.0 - UNRELEASED
 
 YARN-4524. Cleanup AppSchedulingInfo. (Karthik Kambatla via wangda)
 
+YARN-4438. Implement RM leader election with curator. (Jian He via xgong)
+
   OPTIMIZATIONS
 
   BUG FIXES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/89022f8d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 9a1eb54..37c81ec 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -545,6 +545,11 @@ public class YarnConfiguration extends Configuration {
   public static final String RM_HA_FC_ELECTOR_ZK_RETRIES_KEY = RM_HA_PREFIX
   + "failover-controller.active-standby-elector.zk.retries";
 
+  @Private
+  public static final String CURATOR_LEADER_ELECTOR =
+  RM_HA_PREFIX + "curator-leader-elector.enabled";
+  public static final boolean DEFAULT_CURATOR_LEADER_ELECTOR_ENABLED = false;
+
   
   // RM state store configs
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/89022f8d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
index 41c3d87..0e508ed 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
@@ -90,6 +90,7 @@ public class TestYarnConfigurationFields extends 
TestConfigurationFieldsBase {
 .add(YarnConfiguration.DEFAULT_SHARED_CACHE_CHECKSUM_ALGO_IMPL);
 configurationPropsToSkipCompare
 .add(YarnConfiguration.DEFAULT_AMRM_PROXY_INTERCEPTOR_CLASS_PIPELINE);
+
configurationPropsToSkipCompare.add(YarnConfiguration.CURATOR_LEADER_ELECTOR);
 
 // Ignore all YARN Application Timeline Service (version 1) properties
 configurationPrefixToSkipCompare.add("yarn.timeline-service.");

http://git-wip-us.apache.org/repos/asf/hadoop/blob/89022f8d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
--
diff --git