AMBARI-19947. To make sure LLAP preemption happens correctly, should enable priority-utilization.underutilized-preemption.enabled=true when YARN preemption is enabled.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/59545f7f Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/59545f7f Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/59545f7f Branch: refs/heads/branch-feature-AMBARI-12556 Commit: 59545f7fde971c352c7f7f44a19f436887fcdfe7 Parents: b15b606 Author: Vitaly Brodetskyi <[email protected]> Authored: Fri Feb 10 14:41:12 2017 +0200 Committer: Vitaly Brodetskyi <[email protected]> Committed: Fri Feb 10 14:41:12 2017 +0200 ---------------------------------------------------------------------- .../YARN/3.0.0.3.0/service_advisor.py | 15 ++++++++++++++- .../2.6/services/YARN/configuration/yarn-site.xml | 12 ++++++++++++ .../stacks/HDP/2.6/services/stack_advisor.py | 12 ++++++++++++ .../3.0/services/YARN/configuration/yarn-site.xml | 13 +++++++++++++ 4 files changed, 51 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/59545f7f/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py index 30dd474..aecf1e3 100644 --- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py +++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py @@ -130,6 +130,7 @@ class YARNServiceAdvisor(service_advisor.ServiceAdvisor): recommender.recommendYARNConfigurationsFromHDP22(configurations, clusterData, services, hosts) recommender.recommendYARNConfigurationsFromHDP23(configurations, clusterData, services, hosts) recommender.recommendYARNConfigurationsFromHDP25(configurations, clusterData, services, hosts) + recommender.recommendYARNConfigurationsFromHDP26(configurations, clusterData, services, hosts) def getServiceConfigurationsValidationItems(self, configurations, recommendedDefaults, services, hosts): """ @@ -435,6 +436,18 @@ class YARNRecommender(service_advisor.ServiceAdvisor): putYarnSiteProperty('yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes', ",".join(timeline_plugin_classes_values)) putYarnSiteProperty('yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath', ":".join(timeline_plugin_classpath_values)) + + def recommendYARNConfigurationsFromHDP26(self, configurations, clusterData, services, hosts): + putYarnSiteProperty = self.putProperty(configurations, "yarn-site", services) + + if "yarn-site" in services["configurations"] and \ + "yarn.resourcemanager.scheduler.monitor.enable" in services["configurations"]["yarn-site"]["properties"]: + scheduler_monitor_enabled = services["configurations"]["yarn-site"]["properties"]["yarn.resourcemanager.scheduler.monitor.enable"] + if scheduler_monitor_enabled.lower() == 'true': + putYarnSiteProperty('yarn.scheduler.capacity.ordering-policy.priority-utilization.underutilized-preemption.enabled', "true") + else: + putYarnSiteProperty('yarn.scheduler.capacity.ordering-policy.priority-utilization.underutilized-preemption.enabled', "false") + #region LLAP def updateLlapConfigs(self, configurations, services, hosts, llap_queue_name): """ @@ -1787,4 +1800,4 @@ class MAPREDUCE2Validator(service_advisor.ServiceAdvisor): if yarnAppMapreduceAmCommandOpts > yarnAppMapreduceAmResourceMb: validationItems.append({"config-name": 'yarn.app.mapreduce.am.command-opts', "item": self.getWarnItem("yarn.app.mapreduce.am.command-opts Xmx should be less than yarn.app.mapreduce.am.resource.mb ({0})".format(yarnAppMapreduceAmResourceMb))}) - return self.toConfigurationValidationProblems(validationItems, "mapred-site") \ No newline at end of file + return self.toConfigurationValidationProblems(validationItems, "mapred-site") http://git-wip-us.apache.org/repos/asf/ambari/blob/59545f7f/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml index 58b528e..70a2cbe 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml @@ -23,4 +23,16 @@ <description>When HA is enabled, the class to be used by Clients, AMs and NMs to failover to the Active RM. It should extend org.apache.hadoop.yarn.client.RMFailoverProxyProvider</description> <on-ambari-upgrade add="false"/> </property> + <property> + <name>yarn.scheduler.capacity.ordering-policy.priority-utilization.underutilized-preemption.enabled</name> + <value>false</value> + <description>This property need to be set to make preemption can happen under a fragmented cluster.</description> + <depends-on> + <property> + <type>yarn-site</type> + <name>yarn.resourcemanager.scheduler.monitor.enable</name> + </property> + </depends-on> + <on-ambari-upgrade add="false"/> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/59545f7f/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py index e6ebd6f..969c3dd 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py @@ -138,6 +138,18 @@ class HDP26StackAdvisor(HDP25StackAdvisor): elif superset_database_type == "postgresql": putSupersetProperty("SUPERSET_DATABASE_PORT", "5432") + def recommendYARNConfigurations(self, configurations, clusterData, services, hosts): + super(HDP26StackAdvisor, self).recommendYARNConfigurations(configurations, clusterData, services, hosts) + putYarnSiteProperty = self.putProperty(configurations, "yarn-site", services) + + if "yarn-site" in services["configurations"] and \ + "yarn.resourcemanager.scheduler.monitor.enable" in services["configurations"]["yarn-site"]["properties"]: + scheduler_monitor_enabled = services["configurations"]["yarn-site"]["properties"]["yarn.resourcemanager.scheduler.monitor.enable"] + if scheduler_monitor_enabled.lower() == 'true': + putYarnSiteProperty('yarn.scheduler.capacity.ordering-policy.priority-utilization.underutilized-preemption.enabled', "true") + else: + putYarnSiteProperty('yarn.scheduler.capacity.ordering-policy.priority-utilization.underutilized-preemption.enabled', "false") + def getMetadataConnectionString(self, database_type): driverDict = { 'mysql': 'jdbc:mysql://{0}:{2}/{1}?createDatabaseIfNotExist=true', http://git-wip-us.apache.org/repos/asf/ambari/blob/59545f7f/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/yarn-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/yarn-site.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/yarn-site.xml index 0f46d75..01551c4 100644 --- a/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/yarn-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/yarn-site.xml @@ -25,6 +25,19 @@ <on-ambari-upgrade add="false"/> </property> + <property> + <name>yarn.scheduler.capacity.ordering-policy.priority-utilization.underutilized-preemption.enabled</name> + <value>false</value> + <description>This property need to be set to make preemption can happen under a fragmented cluster.</description> + <depends-on> + <property> + <type>yarn-site</type> + <name>yarn.resourcemanager.scheduler.monitor.enable</name> + </property> + </depends-on> + <on-ambari-upgrade add="false"/> + </property> + <!-- These configs were inherited from HDP 2.5 --> <property> <name>yarn.nodemanager.aux-services.spark2_shuffle.classpath</name>
