Repository: hadoop Updated Branches: refs/heads/trunk ccaf03666 -> 0b7afc060
Addendum patch for YARN-5731 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0b7afc06 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0b7afc06 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0b7afc06 Branch: refs/heads/trunk Commit: 0b7afc060c2024a882bd1934d0f722bfca731742 Parents: ccaf036 Author: Sunil G <sun...@apache.org> Authored: Tue Jul 18 11:49:09 2017 +0530 Committer: Sunil G <sun...@apache.org> Committed: Tue Jul 18 11:49:09 2017 +0530 ---------------------------------------------------------------------- .../ProportionalCapacityPreemptionPolicy.java | 25 ++++-------------- .../CapacitySchedulerConfiguration.java | 27 ++++++++++++++++++++ ...TestCapacitySchedulerSurgicalPreemption.java | 6 +++-- 3 files changed, 36 insertions(+), 22 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b7afc06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java index 719d2eb..fc8ad2b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java @@ -231,28 +231,13 @@ public class ProportionalCapacityPreemptionPolicy .add(new ReservedContainerCandidatesSelector(this)); } + boolean additionalPreemptionBasedOnReservedResource = csConfig.getBoolean( + CapacitySchedulerConfiguration.ADDITIONAL_RESOURCE_BALANCE_BASED_ON_RESERVED_CONTAINERS, + CapacitySchedulerConfiguration.DEFAULT_ADDITIONAL_RESOURCE_BALANCE_BASED_ON_RESERVED_CONTAINERS); + // initialize candidates preemption selection policies - // When select candidates for reserved containers is enabled, exclude reserved - // resource in fifo policy (less aggressive). Otherwise include reserved - // resource. - // - // Why doing this? In YARN-4390, we added preemption-based-on-reserved-container - // Support. To reduce unnecessary preemption for large containers. We will - // not include reserved resources while calculating ideal-allocation in - // FifoCandidatesSelector. - // - // Changes in YARN-4390 will significantly reduce number of containers preempted - // When cluster has heterogeneous container requests. (Please check test - // report: https://issues.apache.org/jira/secure/attachment/12796197/YARN-4390-test-results.pdf - // - // However, on the other hand, in some corner cases, especially for - // fragmented cluster. It could lead to preemption cannot kick in in some - // cases. Please see YARN-5731. - // - // So to solve the problem, we will include reserved when surgical preemption - // for reserved container, which reverts behavior when YARN-4390 is disabled. candidatesSelectionPolicies.add(new FifoCandidatesSelector(this, - !selectCandidatesForResevedContainers)); + additionalPreemptionBasedOnReservedResource)); // Do we need to specially consider intra queue boolean isIntraQueuePreemptionEnabled = csConfig.getBoolean( http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b7afc06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 90a7e65..1e29d50 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -1206,6 +1206,33 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur 0.2f; /** + * By default, reserved resource will be excluded while balancing capacities + * of queues. + * + * Why doing this? In YARN-4390, we added preemption-based-on-reserved-container + * Support. To reduce unnecessary preemption for large containers. We will + * not include reserved resources while calculating ideal-allocation in + * FifoCandidatesSelector. + * + * Changes in YARN-4390 will significantly reduce number of containers preempted + * When cluster has heterogeneous container requests. (Please check test + * report: https://issues.apache.org/jira/secure/attachment/12796197/YARN-4390-test-results.pdf + * + * However, on the other hand, in some corner cases, especially for + * fragmented cluster. It could lead to preemption cannot kick in in some + * cases. Please see YARN-5731. + * + * So to solve the problem, make this change to be configurable, and please + * note that it is an experimental option. + */ + public static final String + ADDITIONAL_RESOURCE_BALANCE_BASED_ON_RESERVED_CONTAINERS = + PREEMPTION_CONFIG_PREFIX + + "additional_res_balance_based_on_reserved_containers"; + public static final boolean + DEFAULT_ADDITIONAL_RESOURCE_BALANCE_BASED_ON_RESERVED_CONTAINERS = false; + + /** * When calculating which containers to be preempted, we will try to preempt * containers for reserved containers first. By default is false. */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b7afc06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java index afd2f82..9146373 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java @@ -813,9 +813,11 @@ public class TestCapacitySchedulerSurgicalPreemption @Test(timeout = 60000) public void testPreemptionForFragmentatedCluster() throws Exception { + // Set additional_balance_queue_based_on_reserved_res to true to get + // additional preemptions. conf.setBoolean( - CapacitySchedulerConfiguration.PREEMPTION_SELECT_CANDIDATES_FOR_RESERVED_CONTAINERS, - false); + CapacitySchedulerConfiguration.ADDITIONAL_RESOURCE_BALANCE_BASED_ON_RESERVED_CONTAINERS, + true); /** * Two queues, a/b, each of them are 50/50 --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org