Repository: hadoop Updated Branches: refs/heads/branch-2 28805cb8d -> 64a7617eb
YARN-4066. Large number of queues choke fair scheduler. (Johan Gustavsson via kasha) (cherry picked from commit a0b5a0a419dfc07b7ac45c06b11b4c8dc7e79958) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/64a7617e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/64a7617e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/64a7617e Branch: refs/heads/branch-2 Commit: 64a7617eb9c4b3ade49e0e90c8d113500bb33114 Parents: 28805cb Author: Karthik Kambatla <[email protected]> Authored: Tue Sep 29 07:55:34 2015 -0700 Committer: Karthik Kambatla <[email protected]> Committed: Tue Sep 29 08:13:25 2015 -0700 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 + .../scheduler/fair/QueueManager.java | 80 ++++++++++++++------ 2 files changed, 60 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/64a7617e/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index cd7c8ca..e8e57a1 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -457,6 +457,9 @@ Release 2.8.0 - UNRELEASED YARN-3635. Refactored current queue mapping implementation in CapacityScheduler to use a generic PlacementManager framework. (Wangda Tan via jianhe) + YARN-4066. Large number of queues choke fair scheduler. + (Johan Gustavsson via kasha) + BUG FIXES YARN-3197. Confusing log generated by CapacityScheduler. (Varun Saxena http://git-wip-us.apache.org/repos/asf/hadoop/blob/64a7617e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java index 0092845..51a298b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java @@ -87,7 +87,19 @@ public class QueueManager { * could be referred to as just "parent1.queue2". */ public FSLeafQueue getLeafQueue(String name, boolean create) { - FSQueue queue = getQueue(name, create, FSQueueType.LEAF); + return getLeafQueue(name, create, true); + } + + public FSLeafQueue getLeafQueue( + String name, + boolean create, + boolean recomputeSteadyShares) { + FSQueue queue = getQueue( + name, + create, + FSQueueType.LEAF, + recomputeSteadyShares + ); if (queue instanceof FSParentQueue) { return null; } @@ -117,28 +129,46 @@ public class QueueManager { * could be referred to as just "parent1.queue2". */ public FSParentQueue getParentQueue(String name, boolean create) { - FSQueue queue = getQueue(name, create, FSQueueType.PARENT); + return getParentQueue(name, create, true); + } + + public FSParentQueue getParentQueue( + String name, + boolean create, + boolean recomputeSteadyShares) { + FSQueue queue = getQueue( + name, + create, + FSQueueType.PARENT, + recomputeSteadyShares + ); if (queue instanceof FSLeafQueue) { return null; } return (FSParentQueue) queue; } - - private FSQueue getQueue(String name, boolean create, FSQueueType queueType) { + + private FSQueue getQueue( + String name, + boolean create, + FSQueueType queueType, + boolean recomputeSteadyShares) { + boolean recompute = recomputeSteadyShares; name = ensureRootPrefix(name); + FSQueue queue; synchronized (queues) { - FSQueue queue = queues.get(name); + queue = queues.get(name); if (queue == null && create) { // if the queue doesn't exist,create it and return queue = createQueue(name, queueType); - - // Update steady fair share for all queues - if (queue != null) { - rootQueue.recomputeSteadyShares(); - } + } else { + recompute = false; } - return queue; } + if (recompute) { + rootQueue.recomputeSteadyShares(); + } + return queue; } /** @@ -376,21 +406,25 @@ public class QueueManager { public void updateAllocationConfiguration(AllocationConfiguration queueConf) { // Create leaf queues and the parent queues in a leaf's ancestry if they do not exist - for (String name : queueConf.getConfiguredQueues().get(FSQueueType.LEAF)) { - if (removeEmptyIncompatibleQueues(name, FSQueueType.LEAF)) { - getLeafQueue(name, true); + synchronized (queues) { + for (String name : queueConf.getConfiguredQueues().get( + FSQueueType.LEAF)) { + if (removeEmptyIncompatibleQueues(name, FSQueueType.LEAF)) { + getLeafQueue(name, true, false); + } } - } - - // At this point all leaves and 'parents with at least one child' would have been created. - // Now create parents with no configured leaf. - for (String name : queueConf.getConfiguredQueues().get( - FSQueueType.PARENT)) { - if (removeEmptyIncompatibleQueues(name, FSQueueType.PARENT)) { - getParentQueue(name, true); + // At this point all leaves and 'parents with + // at least one child' would have been created. + // Now create parents with no configured leaf. + for (String name : queueConf.getConfiguredQueues().get( + FSQueueType.PARENT)) { + if (removeEmptyIncompatibleQueues(name, FSQueueType.PARENT)) { + getParentQueue(name, true, false); + } } } - + rootQueue.recomputeSteadyShares(); + for (FSQueue queue : queues.values()) { // Update queue metrics FSQueueMetrics queueMetrics = queue.getMetrics();
