Repository: hadoop Updated Branches: refs/heads/branch-3.1 c7973292b -> 00a469138
YARN-8720. CapacityScheduler does not enforce max resource allocation check at queue level. Contributed by Tarun Parimi. (cherry picked from commit f1a893fdbc2dbe949cae786f08bdb2651b88d673) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/00a46913 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/00a46913 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/00a46913 Branch: refs/heads/branch-3.1 Commit: 00a469138dfd2c09bfd02f99cbd5eb3f43feee1b Parents: c797329 Author: Weiwei Yang <[email protected]> Authored: Fri Sep 14 16:33:51 2018 +0800 Committer: Weiwei Yang <[email protected]> Committed: Fri Sep 14 16:40:35 2018 +0800 ---------------------------------------------------------------------- .../hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java | 3 ++- .../apache/hadoop/yarn/server/resourcemanager/RMAppManager.java | 5 +++-- .../hadoop/yarn/server/resourcemanager/TestAppManager.java | 3 +++ .../hadoop/yarn/server/resourcemanager/TestClientRMService.java | 3 +++ 4 files changed, 11 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/00a46913/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java index 9c6166c..5168bd3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java @@ -223,7 +223,8 @@ final class DefaultAMSProcessor implements ApplicationMasterServiceProcessor { } } - Resource maximumCapacity = getScheduler().getMaximumResourceCapability(); + Resource maximumCapacity = + getScheduler().getMaximumResourceCapability(app.getQueue()); // sanity check try { http://git-wip-us.apache.org/repos/asf/hadoop/blob/00a46913/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index 7011aaa..c54051e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -525,10 +525,11 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>, } // Normalize all requests + String queue = submissionContext.getQueue(); for (ResourceRequest amReq : amReqs) { SchedulerUtils.normalizeAndValidateRequest(amReq, - scheduler.getMaximumResourceCapability(), - submissionContext.getQueue(), scheduler, isRecovery, rmContext); + scheduler.getMaximumResourceCapability(queue), + queue, scheduler, isRecovery, rmContext); amReq.setCapability( scheduler.getNormalizedResource(amReq.getCapability())); http://git-wip-us.apache.org/repos/asf/hadoop/blob/00a46913/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java index e79ba08..0fb5cab 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java @@ -1070,6 +1070,9 @@ public class TestAppManager{ when(scheduler.getMaximumResourceCapability()).thenReturn( Resources.createResource( YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB)); + when(scheduler.getMaximumResourceCapability(any(String.class))).thenReturn( + Resources.createResource( + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB)); ResourceCalculator rs = mock(ResourceCalculator.class); when(scheduler.getResourceCalculator()).thenReturn(rs); http://git-wip-us.apache.org/repos/asf/hadoop/blob/00a46913/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index d66a866..82e105b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -1473,6 +1473,9 @@ public class TestClientRMService { when(yarnScheduler.getMaximumResourceCapability()).thenReturn( Resources.createResource( YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB)); + when(yarnScheduler.getMaximumResourceCapability(any(String.class))) + .thenReturn(Resources.createResource( + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB)); when(yarnScheduler.getAppsInQueue(QUEUE_1)).thenReturn( Arrays.asList(getApplicationAttemptId(101), getApplicationAttemptId(102))); when(yarnScheduler.getAppsInQueue(QUEUE_2)).thenReturn( --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
