YARN-4565. Fix a bug that leads to AM resource limit not hornored when sizeBasedWeight enabled for FairOrderingPolicy. Contributed by Wangda Tan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/edc43a90 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/edc43a90 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/edc43a90 Branch: refs/heads/HDFS-1312 Commit: edc43a9097530fd469dee47d4fefd091818331e5 Parents: 92c5f56 Author: Jian He <[email protected]> Authored: Mon Jan 18 18:28:05 2016 -0800 Committer: Jian He <[email protected]> Committed: Mon Jan 18 21:04:36 2016 -0800 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 ++ .../scheduler/SchedulerApplicationAttempt.java | 2 +- .../scheduler/capacity/CapacityScheduler.java | 3 +- .../CapacitySchedulerConfiguration.java | 13 +++++ .../policy/TestFairOrderingPolicy.java | 54 ++++++++++++++++++++ 5 files changed, 73 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/edc43a90/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index c1430d7..29ee57b 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -1280,6 +1280,9 @@ Release 2.8.0 - UNRELEASED YARN-4502. Fix two AM containers get allocated when AM restart. (Vinod Kumar Vavilapalli via wangda) + YARN-4565. Fix a bug that leads to AM resource limit not hornored when + sizeBasedWeight enabled for FairOrderingPolicy. (wtan via jianhe) + Release 2.7.3 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/edc43a90/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java index d91c79e..ca05fe9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java @@ -863,7 +863,7 @@ public class SchedulerApplicationAttempt implements SchedulableEntity { } @Override - public synchronized ResourceUsage getSchedulingResourceUsage() { + public ResourceUsage getSchedulingResourceUsage() { return attemptResourceUsage; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/edc43a90/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index f4129ab..e773384 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -978,7 +978,8 @@ public class CapacityScheduler extends clusterResource, getMinimumResourceCapability()); } - if (updateDemandForQueue != null) { + if (updateDemandForQueue != null && !application + .isWaitingForAMContainer()) { updateDemandForQueue.getOrderingPolicy().demandUpdated(application); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/edc43a90/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 1e62b44..2e8fd24 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -30,6 +30,7 @@ import java.util.Map.Entry; import java.util.Set; import java.util.StringTokenizer; +import com.google.common.annotations.VisibleForTesting; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -959,4 +960,16 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur DEFAULT_CONFIGURATION_APPLICATION_PRIORITY); return defaultPriority; } + + @VisibleForTesting + public void setOrderingPolicy(String queue, String policy) { + set(getQueuePrefix(queue) + ORDERING_POLICY, policy); + } + + @VisibleForTesting + public void setOrderingPolicyParameter(String queue, + String parameterKey, String parameterValue) { + set(getQueuePrefix(queue) + ORDERING_POLICY + "." + + parameterKey, parameterValue); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/edc43a90/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFairOrderingPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFairOrderingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFairOrderingPolicy.java index ffb9d93..98cfdab 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFairOrderingPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFairOrderingPolicy.java @@ -20,6 +20,18 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy; import java.util.*; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.NodeLabel; +import org.apache.hadoop.yarn.server.resourcemanager.MockAM; +import org.apache.hadoop.yarn.server.resourcemanager.MockNM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; import org.junit.Assert; import org.junit.Test; @@ -138,6 +150,48 @@ public class TestFairOrderingPolicy { checkIds(schedOrder.getPreemptionIterator(), new String[]{"2", "1", "3"}); } + @Test + public void testSizeBasedWeightNotAffectAppActivation() throws Exception { + CapacitySchedulerConfiguration csConf = + new CapacitySchedulerConfiguration(); + + // Define top-level queues + String queuePath = CapacitySchedulerConfiguration.ROOT + ".default"; + csConf.setOrderingPolicy(queuePath, CapacitySchedulerConfiguration.FAIR_ORDERING_POLICY); + csConf.setOrderingPolicyParameter(queuePath, + FairOrderingPolicy.ENABLE_SIZE_BASED_WEIGHT, "true"); + csConf.setMaximumApplicationMasterResourcePerQueuePercent(queuePath, 0.1f); + + // inject node label manager + MockRM rm = new MockRM(csConf); + rm.start(); + + CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); + + // Get LeafQueue + LeafQueue lq = (LeafQueue) cs.getQueue("default"); + OrderingPolicy<FiCaSchedulerApp> policy = lq.getOrderingPolicy(); + Assert.assertTrue(policy instanceof FairOrderingPolicy); + Assert.assertTrue(((FairOrderingPolicy<FiCaSchedulerApp>)policy).getSizeBasedWeight()); + + rm.registerNode("h1:1234", 10 * GB); + + // Submit 4 apps + rm.submitApp(1 * GB, "app", "user", null, "default"); + rm.submitApp(1 * GB, "app", "user", null, "default"); + rm.submitApp(1 * GB, "app", "user", null, "default"); + rm.submitApp(1 * GB, "app", "user", null, "default"); + + Assert.assertEquals(1, lq.getNumActiveApplications()); + Assert.assertEquals(3, lq.getNumPendingApplications()); + + // Try allocate once, #active-apps and #pending-apps should be still correct + cs.handle(new NodeUpdateSchedulerEvent( + rm.getRMContext().getRMNodes().get(NodeId.newInstance("h1", 1234)))); + Assert.assertEquals(1, lq.getNumActiveApplications()); + Assert.assertEquals(3, lq.getNumPendingApplications()); + } + public void checkIds(Iterator<MockSchedulableEntity> si, String[] ids) { for (int i = 0;i < ids.length;i++) {
