Repository: hadoop Updated Branches: refs/heads/branch-2 85b5481d8 -> 9988b57e7
YARN-4201. AMBlacklist does not work for minicluster. Contributed by Jun Gong. (cherry picked from commit 049c6e8dc0f952b5ff7d394aa564ab7db13e4f26) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9988b57e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9988b57e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9988b57e Branch: refs/heads/branch-2 Commit: 9988b57e715e32c082046a17ea11305964f420de Parents: 85b5481 Author: Zhihai Xu <[email protected]> Authored: Mon Oct 12 00:13:30 2015 -0700 Committer: Zhihai Xu <[email protected]> Committed: Mon Oct 12 00:16:40 2015 -0700 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 2 ++ .../rmapp/attempt/RMAppAttemptImpl.java | 9 ++++++++- .../scheduler/AbstractYarnScheduler.java | 1 + .../resourcemanager/scheduler/YarnScheduler.java | 9 +++++++++ .../applicationsmanager/TestAMRestart.java | 15 +++++++++++++++ 5 files changed, 35 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/9988b57e/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 8ef524f..83bd63c 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -872,6 +872,8 @@ Release 2.8.0 - UNRELEASED YARN-4140. RM container allocation delayed incase of app submitted to Nodelabel partition. (Bibin A Chundatt via wangda) + YARN-4201. AMBlacklist does not work for minicluster. (Jun Gong via zxu) + Release 2.7.2 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/9988b57e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 962a810..73d2da0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -94,6 +94,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAt import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeFinishedContainersPulledByAMEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent; @@ -1811,7 +1812,13 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { } private void addAMNodeToBlackList(NodeId nodeId) { - blacklistedNodesForAM.addNode(nodeId.getHost().toString()); + SchedulerNode schedulerNode = scheduler.getSchedulerNode(nodeId); + if (schedulerNode != null) { + blacklistedNodesForAM.addNode(schedulerNode.getNodeName()); + } else { + LOG.info(nodeId + " is not added to AM blacklist for " + + applicationAttemptId + ", because it has been removed"); + } } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/9988b57e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java index 6a4efa1..949ec6b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java @@ -604,6 +604,7 @@ public abstract class AbstractYarnScheduler SchedContainerChangeRequest decreaseRequest, SchedulerApplicationAttempt attempt); + @Override public SchedulerNode getSchedulerNode(NodeId nodeId) { return nodes.get(nodeId); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/9988b57e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java index 0c2ae36..0aff669 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java @@ -352,4 +352,13 @@ public interface YarnScheduler extends EventHandler<SchedulerEvent> { * @return maximum priority of cluster */ Priority getMaxClusterLevelAppPriority(); + + /** + * Get SchedulerNode corresponds to nodeId. + * + * @param nodeId the node id of RMNode + * + * @return SchedulerNode corresponds to nodeId + */ + SchedulerNode getSchedulerNode(NodeId nodeId); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/9988b57e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java index 168280a..688ca9a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java @@ -383,6 +383,21 @@ public class TestAMRestart { public void testAMBlacklistPreventsRestartOnSameNode() throws Exception { YarnConfiguration conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.AM_BLACKLISTING_ENABLED, true); + testAMBlacklistPreventRestartOnSameNode(conf); + } + + @Test(timeout = 100000) + public void testAMBlacklistPreventsRestartOnSameNodeForMinicluster() + throws Exception { + YarnConfiguration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.AM_BLACKLISTING_ENABLED, true); + conf.setBoolean(YarnConfiguration.RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME, + true); + testAMBlacklistPreventRestartOnSameNode(conf); + } + + private void testAMBlacklistPreventRestartOnSameNode(YarnConfiguration conf) + throws Exception{ MemoryRMStateStore memStore = new MemoryRMStateStore(); memStore.init(conf); final DrainDispatcher dispatcher = new DrainDispatcher();
