Repository: hadoop Updated Branches: refs/heads/branch-2 5de6f1b79 -> e259a818a
YARN-6144. FairScheduler: preempted resources can become negative. (Miklos Szegedi via kasha) (cherry picked from commit 08f93978f3ec724b24a93d7ef538f158da75802f) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e259a818 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e259a818 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e259a818 Branch: refs/heads/branch-2 Commit: e259a818adb9c95bf2966f0898defc4a4d41bb01 Parents: 5de6f1b Author: Karthik Kambatla <ka...@cloudera.com> Authored: Thu Feb 9 14:45:04 2017 -0800 Committer: Karthik Kambatla <ka...@cloudera.com> Committed: Thu Feb 9 14:45:21 2017 -0800 ---------------------------------------------------------------------- .../scheduler/fair/FSAppAttempt.java | 16 +++++++++------- .../scheduler/fair/TestFairScheduler.java | 16 ++++++++++++++++ 2 files changed, 25 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e259a818/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java index 5f5d14c..9333caa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java @@ -542,18 +542,20 @@ public class FSAppAttempt extends SchedulerApplicationAttempt } void trackContainerForPreemption(RMContainer container) { - containersToPreempt.add(container); - synchronized (preemptedResources) { - Resources.addTo(preemptedResources, container.getAllocatedResource()); + if (containersToPreempt.add(container)) { + synchronized (preemptedResources) { + Resources.addTo(preemptedResources, container.getAllocatedResource()); + } } } private void untrackContainerForPreemption(RMContainer container) { - synchronized (preemptedResources) { - Resources.subtractFrom(preemptedResources, - container.getAllocatedResource()); + if (containersToPreempt.remove(container)) { + synchronized (preemptedResources) { + Resources.subtractFrom(preemptedResources, + container.getAllocatedResource()); + } } - containersToPreempt.remove(container); } Set<RMContainer> getPreemptionContainers() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/e259a818/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 5c86f98..cc68b36 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -3640,6 +3640,8 @@ public class TestFairScheduler extends FairSchedulerTestBase { scheduler.handle(updateEvent); assertEquals("Application1's AM should be finished", 0, app1.getLiveContainers().size()); + assertEquals("Finished application usage should be none", + Resources.none(), app1.getResourceUsage()); assertEquals("Application3's AM should be running", 1, app3.getLiveContainers().size()); assertEquals("Application3's AM requests 1024 MB memory", @@ -3658,6 +3660,8 @@ public class TestFairScheduler extends FairSchedulerTestBase { 0, app4.getAMResource().getMemorySize()); assertEquals("Application4's AM should not be running", 0, app4.getLiveContainers().size()); + assertEquals("Finished application usage should be none", + Resources.none(), app4.getResourceUsage()); assertEquals("Queue1's AM resource usage should be 2048 MB memory", 2048, queue1.getAmResourceUsage().getMemorySize()); @@ -3672,6 +3676,8 @@ public class TestFairScheduler extends FairSchedulerTestBase { 0, app5.getAMResource().getMemorySize()); assertEquals("Application5's AM should not be running", 0, app5.getLiveContainers().size()); + assertEquals("Finished application usage should be none", + Resources.none(), app5.getResourceUsage()); assertEquals("Queue1's AM resource usage should be 2048 MB memory", 2048, queue1.getAmResourceUsage().getMemorySize()); @@ -3683,6 +3689,8 @@ public class TestFairScheduler extends FairSchedulerTestBase { scheduler.handle(updateEvent); assertEquals("Application5's AM should not be running", 0, app5.getLiveContainers().size()); + assertEquals("Finished application usage should be none", + Resources.none(), app5.getResourceUsage()); assertEquals("Queue1's AM resource usage should be 2048 MB memory", 2048, queue1.getAmResourceUsage().getMemorySize()); @@ -3697,8 +3705,12 @@ public class TestFairScheduler extends FairSchedulerTestBase { scheduler.handle(updateEvent); assertEquals("Application2's AM should be finished", 0, app2.getLiveContainers().size()); + assertEquals("Finished application usage should be none", + Resources.none(), app2.getResourceUsage()); assertEquals("Application3's AM should be finished", 0, app3.getLiveContainers().size()); + assertEquals("Finished application usage should be none", + Resources.none(), app3.getResourceUsage()); assertEquals("Application5's AM should be running", 1, app5.getLiveContainers().size()); assertEquals("Application5's AM requests 2048 MB memory", @@ -3718,6 +3730,8 @@ public class TestFairScheduler extends FairSchedulerTestBase { scheduler.handle(containerExpired); assertEquals("Application5's AM should have 0 container", 0, app5.getLiveContainers().size()); + assertEquals("Finished application usage should be none", + Resources.none(), app5.getResourceUsage()); assertEquals("Queue1's AM resource usage should be 2048 MB memory", 2048, queue1.getAmResourceUsage().getMemorySize()); scheduler.update(); @@ -3740,6 +3754,8 @@ public class TestFairScheduler extends FairSchedulerTestBase { scheduler.handle(updateEvent); assertEquals("Application6's AM should not be running", 0, app6.getLiveContainers().size()); + assertEquals("Finished application usage should be none", + Resources.none(), app6.getResourceUsage()); assertEquals("Application6's AM resource shouldn't be updated", 0, app6.getAMResource().getMemorySize()); assertEquals("Queue1's AM resource usage should be 2048 MB memory", --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
