Repository: hadoop Updated Branches: refs/heads/branch-2 53e9973af -> d06d2a96f
YARN-3489. RMServerUtils.validateResourceRequests should only obtain queue info once. (Varun Saxena via wangda) (cherry picked from commit d6f6741296639a73f5306e3ebefec84a40ca03e5) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d06d2a96 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d06d2a96 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d06d2a96 Branch: refs/heads/branch-2 Commit: d06d2a96f56569c92c7508736c1e5d7c3e00c7f4 Parents: 53e9973 Author: Wangda Tan <wan...@apache.org> Authored: Mon May 11 17:31:15 2015 -0700 Committer: Wangda Tan <wan...@apache.org> Committed: Mon May 11 17:35:01 2015 -0700 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 ++ .../server/resourcemanager/RMServerUtils.java | 10 ++++++- .../scheduler/SchedulerUtils.java | 30 +++++++++++++++----- 3 files changed, 35 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d06d2a96/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 7048e06..13214ca 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -360,6 +360,9 @@ Release 2.7.1 - UNRELEASED YARN-3243. CapacityScheduler should pass headroom from parent to children to make sure ParentQueue obey its capacity limits. (Wangda Tan via jianhe) + YARN-3489. RMServerUtils.validateResourceRequests should only obtain queue + info once. (Varun Saxena via wangda) + OPTIMIZATIONS BUG FIXES http://git-wip-us.apache.org/repos/asf/hadoop/blob/d06d2a96/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java index 4669a28..4d2e41c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java @@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; import org.apache.hadoop.yarn.api.records.ResourceRequest; @@ -94,9 +95,16 @@ public class RMServerUtils { Resource maximumResource, String queueName, YarnScheduler scheduler, RMContext rmContext) throws InvalidResourceRequestException { + // Get queue from scheduler + QueueInfo queueInfo = null; + try { + queueInfo = scheduler.getQueueInfo(queueName, false, false); + } catch (IOException e) { + } + for (ResourceRequest resReq : ask) { SchedulerUtils.normalizeAndvalidateRequest(resReq, maximumResource, - queueName, scheduler, rmContext); + queueName, scheduler, rmContext, queueInfo); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/d06d2a96/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java index 0ef5c1e..8047d0b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java @@ -213,13 +213,21 @@ public class SchedulerUtils { Resource maximumResource, String queueName, YarnScheduler scheduler, boolean isRecovery, RMContext rmContext) throws InvalidResourceRequestException { + normalizeAndValidateRequest(resReq, maximumResource, queueName, scheduler, + isRecovery, rmContext, null); + } - QueueInfo queueInfo = null; - try { - queueInfo = scheduler.getQueueInfo(queueName, false, false); - } catch (IOException e) { - // it is possible queue cannot get when queue mapping is set, just ignore - // the queueInfo here, and move forward + public static void normalizeAndValidateRequest(ResourceRequest resReq, + Resource maximumResource, String queueName, YarnScheduler scheduler, + boolean isRecovery, RMContext rmContext, QueueInfo queueInfo) + throws InvalidResourceRequestException { + if (null == queueInfo) { + try { + queueInfo = scheduler.getQueueInfo(queueName, false, false); + } catch (IOException e) { + // it is possible queue cannot get when queue mapping is set, just ignore + // the queueInfo here, and move forward + } } SchedulerUtils.normalizeNodeLabelExpressionInRequest(resReq, queueInfo); if (!isRecovery) { @@ -231,8 +239,16 @@ public class SchedulerUtils { Resource maximumResource, String queueName, YarnScheduler scheduler, RMContext rmContext) throws InvalidResourceRequestException { + normalizeAndvalidateRequest(resReq, maximumResource, queueName, scheduler, + rmContext, null); + } + + public static void normalizeAndvalidateRequest(ResourceRequest resReq, + Resource maximumResource, String queueName, YarnScheduler scheduler, + RMContext rmContext, QueueInfo queueInfo) + throws InvalidResourceRequestException { normalizeAndValidateRequest(resReq, maximumResource, queueName, scheduler, - false, rmContext); + false, rmContext, queueInfo); } /**
