AMBARI-20821. Update default values for YARN preemption in Ambari.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/99f16607 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/99f16607 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/99f16607 Branch: refs/heads/branch-feature-AMBARI-12556 Commit: 99f16607f804992b4c06ea3f0e9295399604e7f6 Parents: 84a8b76 Author: Vitaly Brodetskyi <[email protected]> Authored: Fri Apr 21 18:53:06 2017 +0300 Committer: Vitaly Brodetskyi <[email protected]> Committed: Fri Apr 21 18:53:58 2017 +0300 ---------------------------------------------------------------------- .../YARN/3.0.0.3.0/configuration/yarn-site.xml | 12 ++++++++++++ .../common-services/YARN/3.0.0.3.0/service_advisor.py | 4 ++++ .../HDP/2.6/services/YARN/configuration/yarn-site.xml | 12 ++++++++++++ .../resources/stacks/HDP/2.6/services/stack_advisor.py | 5 +++++ .../test/python/stacks/2.6/common/test_stack_advisor.py | 3 +++ 5 files changed, 36 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/99f16607/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/configuration/yarn-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/configuration/yarn-site.xml b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/configuration/yarn-site.xml index 394fae3..cc96cd7 100644 --- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/configuration/yarn-site.xml +++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/configuration/yarn-site.xml @@ -1177,4 +1177,16 @@ yarn.node-labels.manager-class </value-attributes> <on-ambari-upgrade add="false"/> </property> + <property> + <name>yarn.resourcemanager.monitor.capacity.preemption.total_preemption_per_round</name> + <value>0.1</value> + <description>This option controls the pace at which containers-marked-for-preemption are actually preempted in each period.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>yarn.resourcemanager.monitor.capacity.preemption.natural_termination_factor</name> + <value>1</value> + <description>Similar to total_preemption_per_round, we can apply this factor to slowdown resource preemption after preemption-target is computed for each queue.</description> + <on-ambari-upgrade add="false"/> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/99f16607/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py index 1ac7849..34fe117 100644 --- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py +++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py @@ -430,6 +430,10 @@ class YARNRecommender(service_advisor.ServiceAdvisor): else: putYarnSiteProperty('yarn.scheduler.capacity.ordering-policy.priority-utilization.underutilized-preemption.enabled', "false") + # calculate total_preemption_per_round + total_preemption_per_round = str(round(max(float(1)/len(hosts['items']), 0.1),2)) + putYarnSiteProperty('yarn.resourcemanager.monitor.capacity.preemption.total_preemption_per_round', total_preemption_per_round) + if 'yarn-env' in services['configurations'] and 'yarn_user' in services['configurations']['yarn-env']['properties']: yarn_user = services['configurations']['yarn-env']['properties']['yarn_user'] else: http://git-wip-us.apache.org/repos/asf/ambari/blob/99f16607/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml index acdae65..c434c12 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml @@ -42,6 +42,18 @@ <on-ambari-upgrade add="false"/> </property> <property> + <name>yarn.resourcemanager.monitor.capacity.preemption.total_preemption_per_round</name> + <value>0.1</value> + <description>This option controls the pace at which containers-marked-for-preemption are actually preempted in each period.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>yarn.resourcemanager.monitor.capacity.preemption.natural_termination_factor</name> + <value>1</value> + <description>Similar to total_preemption_per_round, we can apply this factor to slowdown resource preemption after preemption-target is computed for each queue.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> <name>yarn.timeline-service.entity-group-fs-store.app-cache-size</name> <value>10</value> <description> http://git-wip-us.apache.org/repos/asf/ambari/blob/99f16607/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py index 4e1b4b6..da3c98f 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py @@ -189,6 +189,11 @@ class HDP26StackAdvisor(HDP25StackAdvisor): else: putYarnSiteProperty('yarn.scheduler.capacity.ordering-policy.priority-utilization.underutilized-preemption.enabled', "false") + # calculate total_preemption_per_round + total_preemption_per_round = str(round(max(float(1)/len(hosts['items']), 0.1),2)) + putYarnSiteProperty('yarn.resourcemanager.monitor.capacity.preemption.total_preemption_per_round', total_preemption_per_round) + + if 'yarn-env' in services['configurations'] and 'yarn_user' in services['configurations']['yarn-env']['properties']: yarn_user = services['configurations']['yarn-env']['properties']['yarn_user'] else: http://git-wip-us.apache.org/repos/asf/ambari/blob/99f16607/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py index 114dd3a..d4d28c9 100644 --- a/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py @@ -1162,6 +1162,7 @@ class TestHDP26StackAdvisor(TestCase): 'yarn.timeline-service.entity-group-fs-store.app-cache-size': '10', 'yarn.scheduler.minimum-allocation-mb': '256', 'yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath': '', + 'yarn.resourcemanager.monitor.capacity.preemption.total_preemption_per_round': '1.0', 'yarn.nodemanager.resource.cpu-vcores': '4', 'yarn.scheduler.maximum-allocation-mb': '768', 'yarn.nodemanager.linux-container-executor.group': 'hadoop', @@ -1336,6 +1337,7 @@ class TestHDP26StackAdvisor(TestCase): 'yarn.nodemanager.resource.memory-mb': '1280', 'yarn.scheduler.minimum-allocation-mb': '256', 'yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath': '', + 'yarn.resourcemanager.monitor.capacity.preemption.total_preemption_per_round': '1.0', 'yarn.nodemanager.resource.cpu-vcores': '4', 'yarn.scheduler.maximum-allocation-mb': '1280', 'yarn.nodemanager.linux-container-executor.group': 'hadoop', @@ -1442,6 +1444,7 @@ class TestHDP26StackAdvisor(TestCase): 'yarn.nodemanager.resource.memory-mb': '1280', 'yarn.scheduler.minimum-allocation-mb': '256', 'yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath': '', + 'yarn.resourcemanager.monitor.capacity.preemption.total_preemption_per_round': '1.0', 'yarn.nodemanager.resource.cpu-vcores': '4', 'yarn.scheduler.maximum-allocation-mb': '1280', 'yarn.nodemanager.linux-container-executor.group': 'hadoop',
