Repository: hadoop Updated Branches: refs/heads/branch-3.0 f918e4d89 -> ef17728c5
YARN-8720. CapacityScheduler does not enforce max resource allocation check at queue level. Contributed by Tarun Parimi. (cherry picked from commit f1a893fdbc2dbe949cae786f08bdb2651b88d673) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ef17728c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ef17728c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ef17728c Branch: refs/heads/branch-3.0 Commit: ef17728c5854c6d63858cc73b0d3b7cbf3d768cd Parents: f918e4d Author: Weiwei Yang <[email protected]> Authored: Fri Sep 14 16:33:51 2018 +0800 Committer: Weiwei Yang <[email protected]> Committed: Fri Sep 14 16:43:28 2018 +0800 ---------------------------------------------------------------------- .../hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java | 3 ++- .../apache/hadoop/yarn/server/resourcemanager/RMAppManager.java | 5 +++-- .../hadoop/yarn/server/resourcemanager/TestAppManager.java | 3 +++ .../hadoop/yarn/server/resourcemanager/TestClientRMService.java | 3 +++ 4 files changed, 11 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef17728c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java index 99d3871..3ee858e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java @@ -213,7 +213,8 @@ final class DefaultAMSProcessor implements ApplicationMasterServiceProcessor { } } - Resource maximumCapacity = getScheduler().getMaximumResourceCapability(); + Resource maximumCapacity = + getScheduler().getMaximumResourceCapability(app.getQueue()); // sanity check try { http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef17728c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index c5a9da4..7d1d8f9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -515,10 +515,11 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>, } // Normalize all requests + String queue = submissionContext.getQueue(); for (ResourceRequest amReq : amReqs) { SchedulerUtils.normalizeAndValidateRequest(amReq, - scheduler.getMaximumResourceCapability(), - submissionContext.getQueue(), scheduler, isRecovery, rmContext); + scheduler.getMaximumResourceCapability(queue), + queue, scheduler, isRecovery, rmContext); amReq.setCapability( scheduler.getNormalizedResource(amReq.getCapability())); http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef17728c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java index 8179321..aab0ee1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java @@ -898,6 +898,9 @@ public class TestAppManager{ when(scheduler.getMaximumResourceCapability()).thenReturn( Resources.createResource( YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB)); + when(scheduler.getMaximumResourceCapability(any(String.class))).thenReturn( + Resources.createResource( + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB)); ResourceCalculator rs = mock(ResourceCalculator.class); when(scheduler.getResourceCalculator()).thenReturn(rs); http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef17728c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index 68aec27..e204406 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -1473,6 +1473,9 @@ public class TestClientRMService { when(yarnScheduler.getMaximumResourceCapability()).thenReturn( Resources.createResource( YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB)); + when(yarnScheduler.getMaximumResourceCapability(any(String.class))) + .thenReturn(Resources.createResource( + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB)); when(yarnScheduler.getAppsInQueue(QUEUE_1)).thenReturn( Arrays.asList(getApplicationAttemptId(101), getApplicationAttemptId(102))); when(yarnScheduler.getAppsInQueue(QUEUE_2)).thenReturn( --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
