Repository: hadoop Updated Branches: refs/heads/YARN-1011 8d217ee3c -> efd852449
YARN-8808. Use aggregate container utilization instead of node utilization to determine resources available for oversubscription. (Haibo Chen via asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/efd85244 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/efd85244 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/efd85244 Branch: refs/heads/YARN-1011 Commit: efd852449f8db899553fa351ece06cbb53823fc0 Parents: 8d217ee Author: Arun Suresh <asur...@apache.org> Authored: Wed Oct 3 11:22:16 2018 -0700 Committer: Arun Suresh <asur...@apache.org> Committed: Wed Oct 3 11:22:16 2018 -0700 ---------------------------------------------------------------------- .../scheduler/SchedulerNode.java | 10 ++++- .../yarn/server/resourcemanager/MockNodes.java | 8 +--- .../scheduler/fair/TestFairScheduler.java | 42 ++++++++++---------- 3 files changed, 32 insertions(+), 28 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/efd85244/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java index 9e38d52..8d47f34 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java @@ -731,8 +731,16 @@ public abstract class SchedulerNode { return Resources.none(); } + ResourceUtilization aggregateContainersUtilization = + getAggregatedContainersUtilization(); + if (aggregateContainersUtilization == null) { + // be conservative if the aggregate container utilization is unknown + return Resources.none(); + + } + ResourceUtilization projectedNodeUtilization = ResourceUtilization. - newInstance(getNodeUtilization()); + newInstance(aggregateContainersUtilization); // account for resources allocated in this heartbeat projectedNodeUtilization.addTo( (int) (resourceAllocatedPendingLaunch.getMemorySize()), 0, http://git-wip-us.apache.org/repos/asf/hadoop/blob/efd85244/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java index bff6125..d841ff0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java @@ -330,17 +330,13 @@ public class MockNodes { return this.physicalResource; } - public void updateResourceUtilization(ResourceUtilization utilization) { - this.nodeUtilization = utilization; - } - - public void updateContainersAndNodeUtilization( + public void updateContainersInfoAndUtilization( UpdatedContainerInfo updatedContainerInfo, ResourceUtilization resourceUtilization) { if (updatedContainerInfo != null) { containerUpdates = Collections.singletonList(updatedContainerInfo); } - this.nodeUtilization = resourceUtilization; + this.containersUtilization = resourceUtilization; } }; http://git-wip-us.apache.org/repos/asf/hadoop/blob/efd85244/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 1ac20f7..5847ca9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -2758,7 +2758,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { ContainerStatus containerStatus = ContainerStatus.newInstance( allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "", ContainerExitStatus.SUCCESS); - node.updateContainersAndNodeUtilization( + node.updateContainersInfoAndUtilization( new UpdatedContainerInfo(Collections.singletonList(containerStatus), Collections.emptyList()), ResourceUtilization.newInstance(1024, 0, 0.1f)); @@ -2850,7 +2850,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { ContainerStatus containerStatus = ContainerStatus.newInstance( allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "", ContainerExitStatus.SUCCESS); - node.updateContainersAndNodeUtilization( + node.updateContainersInfoAndUtilization( new UpdatedContainerInfo(Collections.singletonList(containerStatus), Collections.emptyList()), ResourceUtilization.newInstance(1024, 0, 0.1f)); @@ -2942,7 +2942,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { ContainerStatus containerStatus = ContainerStatus.newInstance( allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "", ContainerExitStatus.SUCCESS); - node.updateContainersAndNodeUtilization( + node.updateContainersInfoAndUtilization( new UpdatedContainerInfo(Collections.singletonList(containerStatus), Collections.emptyList()), ResourceUtilization.newInstance(2000, 0, 0.8f)); @@ -3026,7 +3026,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { ContainerStatus containerStatus = ContainerStatus.newInstance( allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "", ContainerExitStatus.SUCCESS); - node.updateContainersAndNodeUtilization( + node.updateContainersInfoAndUtilization( new UpdatedContainerInfo(Collections.singletonList(containerStatus), Collections.emptyList()), ResourceUtilization.newInstance(1800, 0, 0.5f)); @@ -3112,7 +3112,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { ContainerStatus containerStatus = ContainerStatus.newInstance( allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "", ContainerExitStatus.SUCCESS); - node.updateContainersAndNodeUtilization( + node.updateContainersInfoAndUtilization( new UpdatedContainerInfo(Collections.singletonList(containerStatus), Collections.emptyList()), ResourceUtilization.newInstance(1800, 0, 0.5f)); @@ -3201,7 +3201,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { ContainerStatus containerStatus = ContainerStatus.newInstance( allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "", ContainerExitStatus.SUCCESS); - node.updateContainersAndNodeUtilization( + node.updateContainersInfoAndUtilization( new UpdatedContainerInfo(Collections.singletonList(containerStatus), Collections.emptyList()), ResourceUtilization.newInstance(512, 0, 0.1f)); @@ -3249,7 +3249,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { List<ContainerStatus> finishedContainers = Collections.singletonList( ContainerStatus.newInstance(allocatedContainers3.get(0).getId(), ContainerState.COMPLETE, "", ContainerExitStatus.SUCCESS)); - node.updateContainersAndNodeUtilization( + node.updateContainersInfoAndUtilization( new UpdatedContainerInfo(Collections.emptyList(), finishedContainers), ResourceUtilization.newInstance(1024, 0, 0.1f)); scheduler.handle(new NodeUpdateSchedulerEvent(node)); @@ -3325,7 +3325,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { ContainerStatus containerStatus1 = ContainerStatus.newInstance( allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "", ContainerExitStatus.SUCCESS); - node.updateContainersAndNodeUtilization( + node.updateContainersInfoAndUtilization( new UpdatedContainerInfo(Collections.singletonList(containerStatus1), Collections.emptyList()), ResourceUtilization.newInstance(0, 0, 0.0f)); @@ -3348,7 +3348,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { ContainerStatus containerStatus2 = ContainerStatus.newInstance( allocatedContainers2.get(0).getId(), ContainerState.RUNNING, "", ContainerExitStatus.SUCCESS); - node.updateContainersAndNodeUtilization( + node.updateContainersInfoAndUtilization( new UpdatedContainerInfo(Collections.singletonList(containerStatus2), Collections.emptyList()), ResourceUtilization.newInstance(0, 0, 0.0f)); @@ -3443,7 +3443,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { List<ContainerStatus> containerStatuses = new ArrayList<>(2); containerStatuses.add(container1Status); containerStatuses.add(container2Status); - node.updateContainersAndNodeUtilization( + node.updateContainersInfoAndUtilization( new UpdatedContainerInfo(containerStatuses, Collections.emptyList()), ResourceUtilization.newInstance(1024, 0, 0.1f)); @@ -3467,7 +3467,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { List<ContainerStatus> finishedContainers = Collections.singletonList( ContainerStatus.newInstance(allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "", ContainerExitStatus.SUCCESS)); - node.updateContainersAndNodeUtilization( + node.updateContainersInfoAndUtilization( new UpdatedContainerInfo(Collections.emptyList(), finishedContainers), ResourceUtilization.newInstance(1024, 0, 0.1f)); scheduler.handle(new NodeUpdateSchedulerEvent(node)); @@ -3552,7 +3552,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { List<ContainerStatus> containerStatuses = new ArrayList<>(2); containerStatuses.add(container1Status); containerStatuses.add(container2Status); - node.updateContainersAndNodeUtilization( + node.updateContainersInfoAndUtilization( new UpdatedContainerInfo(containerStatuses, Collections.emptyList()), ResourceUtilization.newInstance(1024, 0, 0.1f)); @@ -3578,7 +3578,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { ContainerStatus container3Status = ContainerStatus.newInstance( allocatedContainers3.get(0).getId(), ContainerState.RUNNING, "", ContainerExitStatus.SUCCESS); - node.updateContainersAndNodeUtilization( + node.updateContainersInfoAndUtilization( new UpdatedContainerInfo(Collections.singletonList(container3Status), Collections.emptyList()), ResourceUtilization.newInstance(2000, 0, 0.2f)); @@ -3603,7 +3603,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { List<ContainerStatus> finishedContainers = Collections.singletonList( ContainerStatus.newInstance(allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "", ContainerExitStatus.SUCCESS)); - node.updateContainersAndNodeUtilization( + node.updateContainersInfoAndUtilization( new UpdatedContainerInfo(Collections.emptyList(), finishedContainers), ResourceUtilization.newInstance(1024, 0, 0.1f)); scheduler.handle(new NodeUpdateSchedulerEvent(node)); @@ -3622,7 +3622,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { finishedContainers = Collections.singletonList( ContainerStatus.newInstance(allocatedContainers2.get(0).getId(), ContainerState.RUNNING, "", ContainerExitStatus.SUCCESS)); - node.updateContainersAndNodeUtilization( + node.updateContainersInfoAndUtilization( new UpdatedContainerInfo(Collections.emptyList(), finishedContainers), ResourceUtilization.newInstance(3000, 0, 0.1f)); scheduler.handle(new NodeUpdateSchedulerEvent(node)); @@ -3709,7 +3709,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { List<ContainerStatus> containerStatuses = new ArrayList<>(2); containerStatuses.add(container1Status); containerStatuses.add(container2Status); - node.updateContainersAndNodeUtilization( + node.updateContainersInfoAndUtilization( new UpdatedContainerInfo(containerStatuses, Collections.emptyList()), ResourceUtilization.newInstance(1024, 0, 0.1f)); @@ -3748,7 +3748,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { List<ContainerStatus> finishedContainers = Collections.singletonList( ContainerStatus.newInstance(allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "", ContainerExitStatus.SUCCESS)); - node.updateContainersAndNodeUtilization( + node.updateContainersInfoAndUtilization( new UpdatedContainerInfo(Collections.emptyList(), finishedContainers), ResourceUtilization.newInstance(1024, 0, 0.1f)); scheduler.handle(new NodeUpdateSchedulerEvent(node)); @@ -3774,7 +3774,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { finishedContainers = Collections.singletonList( ContainerStatus.newInstance(allocatedContainers2.get(0).getId(), ContainerState.RUNNING, "", ContainerExitStatus.SUCCESS)); - node.updateContainersAndNodeUtilization( + node.updateContainersInfoAndUtilization( new UpdatedContainerInfo(Collections.emptyList(), finishedContainers), ResourceUtilization.newInstance(3000, 0, 0.1f)); scheduler.handle(new NodeUpdateSchedulerEvent(node)); @@ -3863,7 +3863,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { List<ContainerStatus> containerStatuses = new ArrayList<>(2); containerStatuses.add(container1Status); containerStatuses.add(container2Status); - node.updateContainersAndNodeUtilization( + node.updateContainersInfoAndUtilization( new UpdatedContainerInfo(containerStatuses, Collections.emptyList()), ResourceUtilization.newInstance(1024, 0, 0.1f)); @@ -3901,7 +3901,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { List<ContainerStatus> finishedContainers = Collections.singletonList( ContainerStatus.newInstance(allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "", ContainerExitStatus.SUCCESS)); - node.updateContainersAndNodeUtilization( + node.updateContainersInfoAndUtilization( new UpdatedContainerInfo(Collections.emptyList(), finishedContainers), ResourceUtilization.newInstance(1024, 0, 0.1f)); scheduler.handle(new NodeUpdateSchedulerEvent(node)); @@ -3919,7 +3919,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { finishedContainers = Collections.singletonList( ContainerStatus.newInstance(allocatedContainers2.get(0).getId(), ContainerState.RUNNING, "", ContainerExitStatus.SUCCESS)); - node.updateContainersAndNodeUtilization( + node.updateContainersInfoAndUtilization( new UpdatedContainerInfo(Collections.emptyList(), finishedContainers), ResourceUtilization.newInstance(3000, 0, 0.1f)); scheduler.handle(new NodeUpdateSchedulerEvent(node)); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
