AMBARI-17200. Tez, Pig, Hive, MapReduce service checks will fail on non standard queue names (dgrinenko via dlysnichenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/16175189 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/16175189 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/16175189 Branch: refs/heads/branch-2.4 Commit: 16175189b0818734ef29861e8253a249e155c084 Parents: 92c8a26 Author: Lisnichenko Dmitro <[email protected]> Authored: Tue Jun 21 17:59:44 2016 +0300 Committer: Lisnichenko Dmitro <[email protected]> Committed: Tue Jun 21 18:00:40 2016 +0300 ---------------------------------------------------------------------- .../server/upgrade/AbstractUpgradeCatalog.java | 84 +++++++- .../server/upgrade/UpgradeCatalog240.java | 52 +++++ .../0.12.0.2.0/configuration/webhcat-site.xml | 14 ++ .../TEZ/0.4.0.2.1/configuration/tez-site.xml | 14 ++ .../configuration-mapred/mapred-site.xml | 14 ++ .../YARN/2.1.0.2.0/configuration/yarn-env.xml | 6 + .../stacks/HDP/2.0.6/services/stack_advisor.py | 35 +++- .../stacks/HDP/2.1/services/stack_advisor.py | 4 +- .../stacks/HDP/2.2/services/stack_advisor.py | 18 +- .../server/upgrade/UpgradeCatalog240Test.java | 200 +++++++++++++++++-- .../stacks/2.0.6/common/test_stack_advisor.py | 53 +---- .../stacks/2.2/common/test_stack_advisor.py | 55 +++-- .../stacks/2.3/common/test_stack_advisor.py | 13 +- .../stacks/2.5/common/test_stack_advisor.py | 22 +- 14 files changed, 471 insertions(+), 113 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/16175189/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java index 7856969..bff2305 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java @@ -17,10 +17,6 @@ */ package org.apache.ambari.server.upgrade; -import javax.persistence.EntityManager; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; - import java.io.StringReader; import java.sql.ResultSet; import java.sql.SQLException; @@ -34,8 +30,14 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; +import java.util.Stack; +import java.util.StringTokenizer; import java.util.TreeMap; +import javax.persistence.EntityManager; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; + import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.configuration.Configuration; @@ -115,6 +117,10 @@ public abstract class AbstractUpgradeCatalog implements UpgradeCatalog { public static final String PROPERTY_RANGER_HBASE_PLUGIN_ENABLED = "ranger-hbase-plugin-enabled"; public static final String PROPERTY_RANGER_KNOX_PLUGIN_ENABLED = "ranger-knox-plugin-enabled"; + public static final String YARN_SCHEDULER_CAPACITY_ROOT_QUEUE = "yarn.scheduler.capacity.root"; + public static final String YARN_SCHEDULER_CAPACITY_ROOT_QUEUES = "yarn.scheduler.capacity.root.queues"; + public static final String QUEUES = "queues"; + public static final String ALERT_URL_PROPERTY_CONNECTION_TIMEOUT = "connection_timeout"; private static final Logger LOG = LoggerFactory.getLogger @@ -913,4 +919,74 @@ public abstract class AbstractUpgradeCatalog implements UpgradeCatalog { public void onPostUpgrade() throws AmbariException, SQLException { // NOOP } + + /** + * Validate queueNameProperty exists for configType in cluster and corresponds to one of validLeafQueues + * @param cluster cluster to operate with + * @param validLeafQueues Set of YARN capacity-scheduler leaf queues + * @param queueNameProperty queue name property to check and update + * @param configType config type name + * @return + */ + protected boolean isQueueNameValid(Cluster cluster, Set<String> validLeafQueues, String queueNameProperty, String configType) { + Config site = cluster.getDesiredConfigByType(configType); + Map<String, String> properties = site.getProperties(); + boolean result = properties.containsKey(queueNameProperty) && validLeafQueues.contains(properties.get(queueNameProperty)); + if (!result){ + LOG.info("Queue name " + queueNameProperty + " in " + configType + " not defined or not corresponds to valid capacity-scheduler queue"); + } + return result; + } + + + /** + * Update property queueNameProperty from configType of cluster to first of validLeafQueues + * @param cluster cluster to operate with + * @param validLeafQueues Set of YARN capacity-scheduler leaf queues + * @param queueNameProperty queue name property to check and update + * @param configType config type name + * @throws AmbariException if an error occurs while updating the configurations + */ + protected void updateQueueName(Cluster cluster, Set<String> validLeafQueues, String queueNameProperty, String configType) throws AmbariException { + String recommendQueue = validLeafQueues.iterator().next(); + LOG.info("Update " + queueNameProperty + " in " + configType + " set to " + recommendQueue); + Map<String, String> updates = Collections.singletonMap(queueNameProperty, recommendQueue); + updateConfigurationPropertiesForCluster(cluster, configType, updates, true, true); + } + + /** + * Pars Capacity Scheduler Properties and get all YARN Capacity Scheduler leaf queue names + * @param capacitySchedulerMap capacity-scheduler properties map + * @return all YARN Capacity Scheduler leaf queue names + */ + protected Set<String> getCapacitySchedulerLeafQueues(Map<String, String> capacitySchedulerMap) { + Set<String> leafQueues= new HashSet<>(); + Stack<String> toProcessQueues = new Stack<>(); + if (capacitySchedulerMap.containsKey(YARN_SCHEDULER_CAPACITY_ROOT_QUEUES)){ + StringTokenizer queueTokenizer = new StringTokenizer(capacitySchedulerMap.get( + YARN_SCHEDULER_CAPACITY_ROOT_QUEUES), ","); + while (queueTokenizer.hasMoreTokens()){ + toProcessQueues.push(queueTokenizer.nextToken()); + } + } + while (!toProcessQueues.empty()){ + String queue = toProcessQueues.pop(); + String queueKey = YARN_SCHEDULER_CAPACITY_ROOT_QUEUE + "." + queue + "." + QUEUES; + if (capacitySchedulerMap.containsKey(queueKey)){ + StringTokenizer queueTokenizer = new StringTokenizer(capacitySchedulerMap.get(queueKey), ","); + while (queueTokenizer.hasMoreTokens()){ + toProcessQueues.push(queue + "." + queueTokenizer.nextToken()); + } + } else { + if (!queue.endsWith(".")){ + String queueName = queue.substring(queue.lastIndexOf('.')+1); + leafQueues.add(queueName); + } else { + LOG.warn("Queue " + queue + " is not valid"); + } + } + } + return leafQueues; + } + } http://git-wip-us.apache.org/repos/asf/ambari/blob/16175189/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java index 357b776..72f5052 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java @@ -144,6 +144,10 @@ public class UpgradeCatalog240 extends AbstractUpgradeCatalog { protected static final String CLUSTER_TABLE = "clusters"; protected static final String CLUSTER_UPGRADE_ID_COLUMN = "upgrade_id"; protected static final String YARN_ENV_CONFIG = "yarn-env"; + protected static final String CAPACITY_SCHEDULER_CONFIG = "capacity-scheduler"; + protected static final String WEBHCAT_SITE_CONFIG = "webhcat-site"; + protected static final String TEZ_SITE_CONFIG = "tez-site"; + protected static final String MAPRED_SITE_CONFIG = "mapred-site"; public static final String DESIRED_VERSION_COLUMN_NAME = "desired_version"; public static final String BLUEPRINT_SETTING_TABLE = "blueprint_setting"; public static final String BLUEPRINT_NAME_COL = "blueprint_name"; @@ -182,6 +186,10 @@ public class UpgradeCatalog240 extends AbstractUpgradeCatalog { protected static final String EXTENSION_LINK_ID_COLUMN = "link_id"; private static final Map<String, Integer> ROLE_ORDER; + public static final String WEBHCAT_SITE_QUEUE_NAME = "templeton.hadoop.queue.name"; + public static final String TEZ_SITE_QUEUE_NAME = "tez.queue.name"; + public static final String YARN_ENV_QUEUE_NAME = "service_check.queue.name"; + public static final String MAPRED_SITE_QUEUE_NAME = "mapreduce.job.queuename"; static { // Manually create role order since there really isn't any mechanism for this @@ -375,6 +383,7 @@ public class UpgradeCatalog240 extends AbstractUpgradeCatalog { updateSparkConfigs(); updateHBaseConfigs(); updateFalconConfigs(); + updateQueueNameConfigs(); updateKerberosDescriptorArtifacts(); removeHiveOozieDBConnectionConfigs(); updateClustersAndHostsVersionStateTableDML(); @@ -1964,6 +1973,49 @@ public class UpgradeCatalog240 extends AbstractUpgradeCatalog { } /** + * Updates the YARN Capacity Scheduler related configurations for the clusters managed by this Ambari + * Update services configuration to set proper leaf queue name for services + * update mapred-site, webhcat-site, tez-site, yarn-env + * + * @throws AmbariException if an error occurs while updating the configurations + */ + protected void updateQueueNameConfigs() throws AmbariException { + AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class); + Clusters clusters = ambariManagementController.getClusters(); + Map<String, Cluster> clusterMap = getCheckedClusterMap(clusters); + + for (final Cluster cluster : clusterMap.values()) { + Config capacityScheduler = cluster.getDesiredConfigByType(CAPACITY_SCHEDULER_CONFIG); + if (capacityScheduler != null) { + Map<String, String> capacitySchedulerProperties = capacityScheduler.getProperties(); + Set<String> leafQueues; + leafQueues = getCapacitySchedulerLeafQueues(capacitySchedulerProperties); + Set<String> installedServices = cluster.getServices().keySet(); + if (leafQueues ==null || leafQueues.isEmpty()) { + LOG.warn("There is no leafQueues in capacity-scheduler"); + return; + } + if (installedServices.contains(cluster.getServiceByConfigType(WEBHCAT_SITE_CONFIG)) && + !isQueueNameValid(cluster, leafQueues, WEBHCAT_SITE_QUEUE_NAME, WEBHCAT_SITE_CONFIG)){ + updateQueueName(cluster, leafQueues, WEBHCAT_SITE_QUEUE_NAME, WEBHCAT_SITE_CONFIG); + } + if (installedServices.contains(cluster.getServiceByConfigType(TEZ_SITE_CONFIG)) && + !isQueueNameValid(cluster, leafQueues, TEZ_SITE_QUEUE_NAME, TEZ_SITE_CONFIG)){ + updateQueueName(cluster, leafQueues, TEZ_SITE_QUEUE_NAME, TEZ_SITE_CONFIG); + } + if (installedServices.contains(cluster.getServiceByConfigType(YARN_ENV_CONFIG)) && + !isQueueNameValid(cluster, leafQueues, YARN_ENV_QUEUE_NAME, YARN_ENV_CONFIG)){ + updateQueueName(cluster, leafQueues, YARN_ENV_QUEUE_NAME, YARN_ENV_CONFIG); + } + if (installedServices.contains(cluster.getServiceByConfigType(MAPRED_SITE_CONFIG)) && + !isQueueNameValid(cluster, leafQueues, MAPRED_SITE_QUEUE_NAME, MAPRED_SITE_CONFIG)){ + updateQueueName(cluster, leafQueues, MAPRED_SITE_QUEUE_NAME, MAPRED_SITE_CONFIG); + } + } + } + } + + /** * Updates the Spark-related configurations for the clusters managed by this Ambari * Removes falcon_store_uri from falcon-env. * Updates {{hdp_full_version}} to {{full_stack_version}} http://git-wip-us.apache.org/repos/asf/ambari/blob/16175189/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/webhcat-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/webhcat-site.xml b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/webhcat-site.xml index eaa5334..d8012dd 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/webhcat-site.xml +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/webhcat-site.xml @@ -143,4 +143,18 @@ limitations under the License. <description>Time out for templeton api</description> <on-ambari-upgrade add="true"/> </property> + <property> + <name>templeton.hadoop.queue.name</name> + <value>default</value> + <description> + MapReduce queue name where WebHCat map-only jobs will be submitted to. Can be used to avoid a deadlock where all map slots in the cluster are taken over by Templeton launcher tasks. + </description> + <depends-on> + <property> + <type>capacity-scheduler</type> + <name>yarn.scheduler.capacity.root.queues</name> + </property> + </depends-on> + <on-ambari-upgrade add="true"/> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/16175189/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/configuration/tez-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/configuration/tez-site.xml b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/configuration/tez-site.xml index 9d54dd6..e7a851c 100644 --- a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/configuration/tez-site.xml +++ b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/configuration/tez-site.xml @@ -231,4 +231,18 @@ <description>Whether to send history events to YARN Application Timeline Server</description> <on-ambari-upgrade add="true"/> </property> + <property> + <name>tez.queue.name</name> + <value>default</value> + <description> + Specify which queue will be used for Tez jobs. + </description> + <depends-on> + <property> + <type>capacity-scheduler</type> + <name>yarn.scheduler.capacity.root.queues</name> + </property> + </depends-on> + <on-ambari-upgrade add="true"/> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/16175189/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml index 0d6c617..6951db0 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml @@ -464,4 +464,18 @@ </description> <on-ambari-upgrade add="true"/> </property> + <property> + <name>mapreduce.job.queuename</name> + <value>default</value> + <description> + Queue to which a job is submitted. + </description> + <depends-on> + <property> + <type>capacity-scheduler</type> + <name>yarn.scheduler.capacity.root.queues</name> + </property> + </depends-on> + <on-ambari-upgrade add="true"/> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/16175189/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml index b035ea3..152c463 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml @@ -249,6 +249,12 @@ YARN_OPTS="$YARN_OPTS -Djava.io.tmpdir={{hadoop_java_io_tmpdir}}" <description> The queue that used by service check. </description> + <depends-on> + <property> + <type>capacity-scheduler</type> + <name>yarn.scheduler.capacity.root.queues</name> + </property> + </depends-on> <on-ambari-upgrade add="true"/> </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/16175189/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py index dd03a9f..373403c 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py @@ -112,6 +112,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor): putYarnProperty('yarn.scheduler.minimum-allocation-mb', int(clusterData['ramPerContainer'])) putYarnProperty('yarn.scheduler.maximum-allocation-mb', int(configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"])) putYarnEnvProperty('min_user_id', self.get_system_min_uid()) + putYarnEnvProperty("service_check.queue.name", self.recommendYarnQueue(services)) containerExecutorGroup = 'hadoop' if 'cluster-env' in services['configurations'] and 'user_group' in services['configurations']['cluster-env']['properties']: containerExecutorGroup = services['configurations']['cluster-env']['properties']['user_group'] @@ -138,6 +139,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor): putMapredProperty('mapreduce.map.java.opts', "-Xmx" + str(int(round(0.8 * clusterData['mapMemory']))) + "m") putMapredProperty('mapreduce.reduce.java.opts', "-Xmx" + str(int(round(0.8 * clusterData['reduceMemory']))) + "m") putMapredProperty('mapreduce.task.io.sort.mb', min(int(round(0.4 * clusterData['mapMemory'])), 1024)) + putMapredProperty("mapreduce.job.queuename", self.recommendYarnQueue(services)) def getAmbariUser(self, services): ambari_user = services['ambari-server-properties']['ambari-server.user'] @@ -1354,6 +1356,26 @@ class HDP206StackAdvisor(DefaultStackAdvisor): return None + def validatorYarnQueue(self, properties, recommendedDefaults, propertyName, services): + if not propertyName in properties: + return self.getErrorItem("Value should be set") + capacity_scheduler_properties, received_as_key_value_pair = self.getCapacitySchedulerProperties(services) + leafQueueNames = self.getAllYarnLeafQueues(capacity_scheduler_properties) + value = properties[propertyName] + if len(leafQueueNames) == 0: + return None + if value not in leafQueueNames: + return self.getErrorItem("Queue is not exist, or not corresponds to existing YARN leaf queue") + return None + + def recommendYarnQueue(self, services): + if services: + if 'configurations' in services: + capacity_scheduler_properties, received_as_key_value_pair = self.getCapacitySchedulerProperties(services) + leafQueueNames = self.getAllYarnLeafQueues(capacity_scheduler_properties) + if leafQueueNames: + return leafQueueNames.pop() + return "default" def validateXmxValue(self, properties, recommendedDefaults, propertyName): if not propertyName in properties: @@ -1382,7 +1404,8 @@ class HDP206StackAdvisor(DefaultStackAdvisor): {"config-name": 'mapreduce.map.memory.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'mapreduce.map.memory.mb')}, {"config-name": 'mapreduce.reduce.memory.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'mapreduce.reduce.memory.mb')}, {"config-name": 'yarn.app.mapreduce.am.resource.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.app.mapreduce.am.resource.mb')}, - {"config-name": 'yarn.app.mapreduce.am.command-opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'yarn.app.mapreduce.am.command-opts')} ] + {"config-name": 'yarn.app.mapreduce.am.command-opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'yarn.app.mapreduce.am.command-opts')}, + {"config-name": 'mapreduce.job.queuename', "item": self.validatorYarnQueue(properties, recommendedDefaults, 'mapreduce.job.queuename', services)} ] return self.toConfigurationValidationProblems(validationItems, "mapred-site") def validateYARNConfigurations(self, properties, recommendedDefaults, configurations, services, hosts): @@ -1394,15 +1417,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor): return self.toConfigurationValidationProblems(validationItems, "yarn-site") def validateYARNEnvConfigurations(self, properties, recommendedDefaults, configurations, services, hosts): - validationItems = [ ] - if not services: - return self.toConfigurationValidationProblems(validationItems, "yarn-env") - yarnEnvProperties = getSiteProperties(configurations, "yarn-env") - capacity_scheduler_properties, received_as_key_value_pair = self.getCapacitySchedulerProperties(services) - leafQueueNames = self.getAllYarnLeafQueues(capacity_scheduler_properties) - service_checkQueueName=yarnEnvProperties.get("service_check.queue.name") - if service_checkQueueName not in leafQueueNames: - validationItems.append({"config-name": 'service_check.queue.name', "item": self.getErrorItem("service_check.queue.name is not exist, or not corresponds to existing leaf queue")}) + validationItems = [{"config-name": 'service_check.queue.name', "item": self.validatorYarnQueue(properties, recommendedDefaults, 'service_check.queue.name', services)} ] return self.toConfigurationValidationProblems(validationItems, "yarn-env") def validateHbaseEnvConfigurations(self, properties, recommendedDefaults, configurations, services, hosts): http://git-wip-us.apache.org/repos/asf/ambari/blob/16175189/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py index f63ca86..3d6ca4c 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py @@ -113,6 +113,7 @@ class HDP21StackAdvisor(HDP206StackAdvisor): putTezProperty("tez.am.java.opts", "-server -Xmx" + str(int(0.8 * clusterData["amMemory"])) + "m -Djava.net.preferIPv4Stack=true -XX:+UseNUMA -XX:+UseParallelGC") + putTezProperty("tez.queue.name", self.recommendYarnQueue(services)) def getNotPreferableOnServerComponents(self): @@ -153,7 +154,8 @@ class HDP21StackAdvisor(HDP206StackAdvisor): def validateTezConfigurations(self, properties, recommendedDefaults, configurations, services, hosts): validationItems = [ {"config-name": 'tez.am.resource.memory.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.am.resource.memory.mb')}, - {"config-name": 'tez.am.java.opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'tez.am.java.opts')} ] + {"config-name": 'tez.am.java.opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'tez.am.java.opts')}, + {"config-name": 'tez.queue.name', "item": self.validatorYarnQueue(properties, recommendedDefaults, 'tez.queue.name', services)} ] return self.toConfigurationValidationProblems(validationItems, "tez-site") def getDBDriver(self, databaseType): http://git-wip-us.apache.org/repos/asf/ambari/blob/16175189/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py index 85f679b..38586e4 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py @@ -267,6 +267,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor): putHiveServerProperty = self.putProperty(configurations, "hiveserver2-site", services) putHiveEnvProperty = self.putProperty(configurations, "hive-env", services) putHiveSiteProperty = self.putProperty(configurations, "hive-site", services) + putWebhcatSiteProperty = self.putProperty(configurations, "webhcat-site", services) putHiveSitePropertyAttribute = self.putPropertyAttribute(configurations, "hive-site") putHiveEnvPropertyAttributes = self.putPropertyAttribute(configurations, "hive-env") servicesList = [service["StackServices"]["service_name"] for service in services["services"]] @@ -405,6 +406,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor): leafQueues = sorted(leafQueues, key=lambda q:q['value']) putHiveSitePropertyAttribute("hive.server2.tez.default.queues", "entries", leafQueues) putHiveSiteProperty("hive.server2.tez.default.queues", ",".join([leafQueue['value'] for leafQueue in leafQueues])) + putWebhcatSiteProperty("templeton.hadoop.queue.name", self.recommendYarnQueue(services)) # Recommend Ranger Hive authorization as per Ranger Hive plugin property @@ -769,6 +771,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor): putTezProperty("tez.runtime.io.sort.mb", min(int(taskResourceMemory * 0.4), 2047)) putTezProperty("tez.runtime.unordered.output.buffer.size-mb", int(taskResourceMemory * 0.075)) putTezProperty("tez.session.am.dag.submit.timeout.secs", "600") + putTezProperty("tez.queue.name", self.recommendYarnQueue(services)) serverProperties = services["ambari-server-properties"] latest_tez_jar_version = None @@ -934,7 +937,8 @@ class HDP22StackAdvisor(HDP21StackAdvisor): "ranger-yarn-plugin-properties": self.validateYARNRangerPluginConfigurations}, "HIVE": {"hiveserver2-site": self.validateHiveServer2Configurations, "hive-site": self.validateHiveConfigurations, - "hive-env": self.validateHiveConfigurationsEnv}, + "hive-env": self.validateHiveConfigurationsEnv, + "webhcat-site": self.validateWebhcatConfigurations}, "HBASE": {"hbase-site": self.validateHBASEConfigurations, "hbase-env": self.validateHBASEEnvConfigurations, "ranger-hbase-plugin-properties": self.validateHBASERangerPluginConfigurations}, @@ -976,7 +980,8 @@ class HDP22StackAdvisor(HDP21StackAdvisor): validationItems = [ {"config-name": 'tez.am.resource.memory.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.am.resource.memory.mb')}, {"config-name": 'tez.task.resource.memory.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.task.resource.memory.mb')}, {"config-name": 'tez.runtime.io.sort.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.runtime.io.sort.mb')}, - {"config-name": 'tez.runtime.unordered.output.buffer.size-mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.runtime.unordered.output.buffer.size-mb')},] + {"config-name": 'tez.runtime.unordered.output.buffer.size-mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.runtime.unordered.output.buffer.size-mb')}, + {"config-name": 'tez.queue.name', "item": self.validatorYarnQueue(properties, recommendedDefaults, 'tez.queue.name', services)} ] if "tez.tez-ui.history-url.base" in recommendedDefaults: validationItems.append({"config-name": 'tez.tez-ui.history-url.base', "item": self.validatorEqualsToRecommendedItem(properties, recommendedDefaults, 'tez.tez-ui.history-url.base')}) @@ -1031,6 +1036,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor): putMapredPropertyAttribute("yarn.app.mapreduce.am.resource.mb", "minimum", yarnMinAllocationSize) # Hadoop MR limitation putMapredPropertyAttribute("mapreduce.task.io.sort.mb", "maximum", "2047") + putMapredProperty("mapreduce.job.queuename", self.recommendYarnQueue(services)) def validateMapReduce2Configurations(self, properties, recommendedDefaults, configurations, services, hosts): validationItems = [ {"config-name": 'mapreduce.map.java.opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'mapreduce.map.java.opts')}, @@ -1039,7 +1045,8 @@ class HDP22StackAdvisor(HDP21StackAdvisor): {"config-name": 'mapreduce.map.memory.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'mapreduce.map.memory.mb')}, {"config-name": 'mapreduce.reduce.memory.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'mapreduce.reduce.memory.mb')}, {"config-name": 'yarn.app.mapreduce.am.resource.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.app.mapreduce.am.resource.mb')}, - {"config-name": 'yarn.app.mapreduce.am.command-opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'yarn.app.mapreduce.am.command-opts')}] + {"config-name": 'yarn.app.mapreduce.am.command-opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'yarn.app.mapreduce.am.command-opts')}, + {"config-name": 'mapreduce.job.queuename', "item": self.validatorYarnQueue(properties, recommendedDefaults, 'mapreduce.job.queuename', services)} ] if 'mapreduce.map.java.opts' in properties and \ checkXmxValueFormat(properties['mapreduce.map.java.opts']): @@ -1295,6 +1302,11 @@ class HDP22StackAdvisor(HDP21StackAdvisor): " {0} needs to be set to {1}".format(prop_name,prop_val))}) return self.toConfigurationValidationProblems(validationItems, "hiveserver2-site") + def validateWebhcatConfigurations(self, properties, recommendedDefaults, configurations, services, hosts): + validationItems = [{"config-name": 'templeton.hadoop.queue.name', "item": self.validatorYarnQueue(properties, recommendedDefaults, 'templeton.hadoop.queue.name', services)}] + return self.toConfigurationValidationProblems(validationItems, "webhcat-site") + + def validateHiveConfigurationsEnv(self, properties, recommendedDefaults, configurations, services, hosts): validationItems = [] hive_env = properties http://git-wip-us.apache.org/repos/asf/ambari/blob/16175189/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java index 14b626b..3eea4d3 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java @@ -19,26 +19,6 @@ package org.apache.ambari.server.upgrade; -import javax.persistence.EntityManager; -import junit.framework.Assert; -import static org.easymock.EasyMock.anyObject; -import static org.easymock.EasyMock.anyString; -import static org.easymock.EasyMock.capture; -import static org.easymock.EasyMock.createMockBuilder; -import static org.easymock.EasyMock.createNiceMock; -import static org.easymock.EasyMock.createStrictMock; -import static org.easymock.EasyMock.eq; -import static org.easymock.EasyMock.expect; -import static org.easymock.EasyMock.expectLastCall; -import static org.easymock.EasyMock.newCapture; -import static org.easymock.EasyMock.replay; -import static org.easymock.EasyMock.reset; -import static org.easymock.EasyMock.verify; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - import java.io.File; import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; @@ -55,6 +35,8 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import javax.persistence.EntityManager; + import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.actionmanager.ActionManager; import org.apache.ambari.server.api.services.AmbariMetaInfo; @@ -126,7 +108,32 @@ import com.google.inject.Injector; import com.google.inject.Module; import com.google.inject.Provider; +import junit.framework.Assert; + +import static org.easymock.EasyMock.anyObject; +import static org.easymock.EasyMock.anyString; +import static org.easymock.EasyMock.capture; +import static org.easymock.EasyMock.createMockBuilder; +import static org.easymock.EasyMock.createNiceMock; +import static org.easymock.EasyMock.createStrictMock; +import static org.easymock.EasyMock.eq; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.expectLastCall; +import static org.easymock.EasyMock.newCapture; +import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.reset; +import static org.easymock.EasyMock.verify; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + public class UpgradeCatalog240Test { + private static final String CAPACITY_SCHEDULER_CONFIG_TYPE = "capacity-scheduler"; + private static final String WEBHCAT_SITE_CONFIG_TYPE = "webhcat-site"; + private static final String TEZ_SITE_CONFIG_TYPE = "tez-site"; + private static final String MAPRED_SITE_CONFIG_TYPE = "mapred-site"; + private static final String YARN_ENV_CONFIG_TYPE = "yarn-env"; private static Injector injector; private Provider<EntityManager> entityManagerProvider = createStrictMock(Provider.class); private EntityManager entityManager = createNiceMock(EntityManager.class); @@ -541,6 +548,7 @@ public class UpgradeCatalog240Test { Method updateHostRoleCommandTableDML = UpgradeCatalog240.class.getDeclaredMethod("updateHostRoleCommandTableDML"); Method updateKerberosEnv = UpgradeCatalog240.class.getDeclaredMethod("updateKerberosConfigs"); Method updateYarnEnv = UpgradeCatalog240.class.getDeclaredMethod("updateYarnEnv"); + Method updateQueueNameConfigs = UpgradeCatalog240.class.getDeclaredMethod("updateQueueNameConfigs"); Method removeHiveOozieDBConnectionConfigs = UpgradeCatalog240.class.getDeclaredMethod("removeHiveOozieDBConnectionConfigs"); Method updateClustersAndHostsVersionStateTableDML = UpgradeCatalog240.class.getDeclaredMethod("updateClustersAndHostsVersionStateTableDML"); Method removeStandardDeviationAlerts = UpgradeCatalog240.class.getDeclaredMethod("removeStandardDeviationAlerts"); @@ -582,6 +590,7 @@ public class UpgradeCatalog240Test { .addMockedMethod(updateHostRoleCommandTableDML) .addMockedMethod(updateKerberosEnv) .addMockedMethod(updateYarnEnv) + .addMockedMethod(updateQueueNameConfigs) .addMockedMethod(removeHiveOozieDBConnectionConfigs) .addMockedMethod(updateClustersAndHostsVersionStateTableDML) .addMockedMethod(removeStandardDeviationAlerts) @@ -618,6 +627,7 @@ public class UpgradeCatalog240Test { upgradeCatalog240.updateHostRoleCommandTableDML(); upgradeCatalog240.updateKerberosConfigs(); upgradeCatalog240.updateYarnEnv(); + upgradeCatalog240.updateQueueNameConfigs(); upgradeCatalog240.removeHiveOozieDBConnectionConfigs(); upgradeCatalog240.updateClustersAndHostsVersionStateTableDML(); upgradeCatalog240.removeStandardDeviationAlerts(); @@ -996,6 +1006,156 @@ public class UpgradeCatalog240Test { assertTrue(Maps.difference(newPropertiesYarnEnv, updatedProperties).areEqual()); } + /** + * Test that queue names updated in mapred-site, webhcat-site, tez-site, yarn-env + * @throws Exception + */ + @SuppressWarnings("unchecked") + @Test + public void testQueueNameUpdateConfigs() throws Exception{ + Map<String, String> oldPropertiesCapacityScheduler = new HashMap<String, String>() { + { + put("yarn.scheduler.capacity.root.queues", "default3,d3"); + put("yarn.scheduler.capacity.root.d3.queues", "default4"); + put("yarn.scheduler.capacity.root.default3.queues", "default5"); + } + }; + Map<String, String> oldPropertiesTezSite = new HashMap<String, String>() { + { + put("tez.queue.name", "default3"); + } + }; + Map<String, String> oldPropertiesWebhcatSite = new HashMap<String, String>() { + { + put("templeton.hadoop.queue.name", "default3"); + } + }; + Map<String, String> oldPropertiesMapredSite = new HashMap<String, String>() { + { + put("mapreduce.job.queuename", "default3"); + } + }; + Map<String, String> oldPropertiesYarnEnv = new HashMap<String, String>() { + { + put("service_check.queue.name", "default3"); + } + }; + + Map<String, String> newPropertiesTezSite = new HashMap<String, String>() { + { + put("tez.queue.name", "default5"); + } + }; + Map<String, String> newPropertiesWebhcatSite = new HashMap<String, String>() { + { + put("templeton.hadoop.queue.name", "default5"); + } + }; + Map<String, String> newPropertiesMapredSite = new HashMap<String, String>() { + { + put("mapreduce.job.queuename", "default5"); + } + }; + Map<String, String> newPropertiesYarnEnv = new HashMap<String, String>() { + { + put("service_check.queue.name", "default5"); + } + }; + + EasyMockSupport easyMockSupport = new EasyMockSupport(); + + Clusters clusters = easyMockSupport.createNiceMock(Clusters.class); + final Cluster cluster = easyMockSupport.createNiceMock(Cluster.class); + Config mockCapacityScheduler = easyMockSupport.createNiceMock(Config.class); + Config mockWebhcatSite = easyMockSupport.createNiceMock(Config.class); + Config mockTezSite = easyMockSupport.createNiceMock(Config.class); + Config mockMapredSite = easyMockSupport.createNiceMock(Config.class); + Config mockYarnEnv = easyMockSupport.createNiceMock(Config.class); + + expect(clusters.getClusters()).andReturn(new HashMap<String, Cluster>() {{ + put("normal", cluster); + }}).anyTimes(); + + expect(cluster.getServices()).andReturn(new HashMap<String, Service>() { + { + put("YARN", null); + put("HIVE", null); + put("MAPREDUCE", null); + put("TEZ", null); + } + }).atLeastOnce(); + + expect(cluster.getServiceByConfigType(WEBHCAT_SITE_CONFIG_TYPE)).andReturn("HIVE").atLeastOnce(); + expect(cluster.getServiceByConfigType(TEZ_SITE_CONFIG_TYPE)).andReturn("TEZ").atLeastOnce(); + expect(cluster.getServiceByConfigType(MAPRED_SITE_CONFIG_TYPE)).andReturn("MAPREDUCE").atLeastOnce(); + expect(cluster.getServiceByConfigType(YARN_ENV_CONFIG_TYPE)).andReturn("YARN").atLeastOnce(); + + expect(cluster.getDesiredConfigByType(CAPACITY_SCHEDULER_CONFIG_TYPE)).andReturn(mockCapacityScheduler).atLeastOnce(); + expect(cluster.getDesiredConfigByType(WEBHCAT_SITE_CONFIG_TYPE)).andReturn(mockWebhcatSite).atLeastOnce(); + expect(cluster.getDesiredConfigByType(TEZ_SITE_CONFIG_TYPE)).andReturn(mockTezSite).atLeastOnce(); + expect(cluster.getDesiredConfigByType(MAPRED_SITE_CONFIG_TYPE)).andReturn(mockMapredSite).atLeastOnce(); + expect(cluster.getDesiredConfigByType(YARN_ENV_CONFIG_TYPE)).andReturn(mockYarnEnv).atLeastOnce(); + + + + expect(mockCapacityScheduler.getProperties()).andReturn(oldPropertiesCapacityScheduler).anyTimes(); + expect(mockWebhcatSite.getProperties()).andReturn(oldPropertiesWebhcatSite).anyTimes(); + expect(mockTezSite.getProperties()).andReturn(oldPropertiesTezSite).anyTimes(); + expect(mockMapredSite.getProperties()).andReturn(oldPropertiesMapredSite).anyTimes(); + expect(mockYarnEnv.getProperties()).andReturn(oldPropertiesYarnEnv).anyTimes(); + + Injector injector = easyMockSupport.createNiceMock(Injector.class); + expect(injector.getInstance(Gson.class)).andReturn(null).anyTimes(); + expect(injector.getInstance(MaintenanceStateHelper.class)).andReturn(null).anyTimes(); + expect(injector.getInstance(KerberosHelper.class)).andReturn(createNiceMock(KerberosHelper.class)).anyTimes(); + + replay(injector, clusters, mockCapacityScheduler, mockWebhcatSite, mockTezSite, mockMapredSite, + mockYarnEnv, cluster); + + AmbariManagementControllerImpl controller = createMockBuilder(AmbariManagementControllerImpl.class) + .addMockedMethod("createConfiguration") + .addMockedMethod("getClusters", new Class[] { }) + .addMockedMethod("createConfig") + .withConstructor(createNiceMock(ActionManager.class), clusters, injector) + .createNiceMock(); + + Injector injector2 = easyMockSupport.createNiceMock(Injector.class); + Capture<Map> propertiesCapacityScheduler = EasyMock.newCapture(); + Capture<Map> propertiesCaptureYarnEnv = EasyMock.newCapture(); + Capture<Map> propertiesCaptureWebhcatSite = EasyMock.newCapture(); + Capture<Map> propertiesCaptureTezSite= EasyMock.newCapture(); + Capture<Map> propertiesCaptureMapredSite = EasyMock.newCapture(); + + expect(injector2.getInstance(AmbariManagementController.class)).andReturn(controller).anyTimes(); + expect(controller.getClusters()).andReturn(clusters).anyTimes(); + expect(controller.createConfig(anyObject(Cluster.class), eq(CAPACITY_SCHEDULER_CONFIG_TYPE), capture(propertiesCapacityScheduler), anyString(), + anyObject(Map.class))).andReturn(createNiceMock(Config.class)).once(); + expect(controller.createConfig(anyObject(Cluster.class), eq(YARN_ENV_CONFIG_TYPE), capture(propertiesCaptureYarnEnv), anyString(), + anyObject(Map.class))).andReturn(createNiceMock(Config.class)).once(); + expect(controller.createConfig(anyObject(Cluster.class), eq(WEBHCAT_SITE_CONFIG_TYPE), capture(propertiesCaptureWebhcatSite), anyString(), + anyObject(Map.class))).andReturn(createNiceMock(Config.class)).once(); + expect(controller.createConfig(anyObject(Cluster.class), eq(TEZ_SITE_CONFIG_TYPE), capture(propertiesCaptureTezSite), anyString(), + anyObject(Map.class))).andReturn(createNiceMock(Config.class)).once(); + expect(controller.createConfig(anyObject(Cluster.class), eq(MAPRED_SITE_CONFIG_TYPE), capture(propertiesCaptureMapredSite), anyString(), + anyObject(Map.class))).andReturn(createNiceMock(Config.class)).once(); + + replay(controller, injector2); + new UpgradeCatalog240(injector2).updateQueueNameConfigs(); + easyMockSupport.verifyAll(); + + Map<String, String> updatedPropertiesYarnEnv = propertiesCaptureYarnEnv.getValue(); + assertTrue(Maps.difference(newPropertiesYarnEnv, updatedPropertiesYarnEnv).areEqual()); + + Map<String, String> updatedPropertiesWebhcatSite = propertiesCaptureWebhcatSite.getValue(); + assertTrue(Maps.difference(newPropertiesWebhcatSite, updatedPropertiesWebhcatSite).areEqual()); + + Map<String, String> updatedPropertiesTezSite = propertiesCaptureTezSite.getValue(); + assertTrue(Maps.difference(newPropertiesTezSite, updatedPropertiesTezSite).areEqual()); + + Map<String, String> updatedPropertiesMapredSite = propertiesCaptureMapredSite.getValue(); + assertTrue(Maps.difference(newPropertiesMapredSite, updatedPropertiesMapredSite).areEqual()); + } + @Test public void testSparkConfigUpdate() throws Exception{ http://git-wip-us.apache.org/repos/asf/ambari/blob/16175189/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py index eee00de..6a8f3e2 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py @@ -218,57 +218,16 @@ class TestHDP206StackAdvisor(TestCase): ] services = self.prepareServices(servicesInfo) services["configurations"] = {"yarn-env":{"properties":{"service_check.queue.name": "default"}}, - "capacity-scheduler":{"properties":{"capacity-scheduler": - "yarn.scheduler.capacity.ndfqueue.minimum-user-limit-percent=100\n" + - "yarn.scheduler.capacity.maximum-am-resource-percent=0.2\n" + - "yarn.scheduler.capacity.maximum-applications=10000\n" + - "yarn.scheduler.capacity.node-locality-delay=40\n" + - "yarn.scheduler.capacity.root.accessible-node-labels=*\n" + - "yarn.scheduler.capacity.root.acl_administer_queue=*\n" + - "yarn.scheduler.capacity.root.capacity=100\n" + - "yarn.scheduler.capacity.root.ndfqueue.acl_administer_jobs=*\n" + - "yarn.scheduler.capacity.root.ndfqueue.acl_submit_applications=*\n" + - "yarn.scheduler.capacity.root.ndfqueue.capacity=100\n" + - "yarn.scheduler.capacity.root.ndfqueue.maximum-capacity=100\n" + - "yarn.scheduler.capacity.root.ndfqueue.state=RUNNING\n" + - "yarn.scheduler.capacity.root.ndfqueue.user-limit-factor=1\n" + - "yarn.scheduler.capacity.root.queues=ndfqueue\n"}}} + "capacity-scheduler":{"properties":{"capacity-scheduler": "yarn.scheduler.capacity.root.queues=ndfqueue\n"}}} hosts = self.prepareHosts([]) result = self.stackAdvisor.validateConfigurations(services, hosts) - expectedItems = [ - {'message': 'service_check.queue.name is not exist, or not corresponds to existing leaf queue', 'level': 'ERROR'} + {'message': 'Queue is not exist, or not corresponds to existing YARN leaf queue', 'level': 'ERROR'} ] self.assertValidationResult(expectedItems, result) - - def test_validationYARNServicecheckQueueNameDefault(self): - servicesInfo = [ - { - "name": "YARN", - "components": [] - } - ] - services = self.prepareServices(servicesInfo) - services["configurations"] = {"yarn-env":{"properties":{"service_check.queue.name": "default"}}, - "capacity-scheduler":{"properties":{"capacity-scheduler": - "yarn.scheduler.capacity.default.minimum-user-limit-percent=100\n" + - "yarn.scheduler.capacity.maximum-am-resource-percent=0.2\n" + - "yarn.scheduler.capacity.maximum-applications=10000\n" + - "yarn.scheduler.capacity.node-locality-delay=40\n" + - "yarn.scheduler.capacity.root.accessible-node-labels=*\n" + - "yarn.scheduler.capacity.root.acl_administer_queue=*\n" + - "yarn.scheduler.capacity.root.capacity=100\n" + - "yarn.scheduler.capacity.root.default.acl_administer_jobs=*\n" + - "yarn.scheduler.capacity.root.default.acl_submit_applications=*\n" + - "yarn.scheduler.capacity.root.default.capacity=100\n" + - "yarn.scheduler.capacity.root.default.maximum-capacity=100\n" + - "yarn.scheduler.capacity.root.default.state=RUNNING\n" + - "yarn.scheduler.capacity.root.default.user-limit-factor=1\n" + - "yarn.scheduler.capacity.root.queues=default\n"}}} - hosts = self.prepareHosts([]) + services["configurations"]["yarn-env"]["properties"]["service_check.queue.name"] = "ndfqueue" + expectedItems = [] result = self.stackAdvisor.validateConfigurations(services, hosts) - - expectedItems = [ ] self.assertValidationResult(expectedItems, result) def test_validationMinMax(self): @@ -581,7 +540,8 @@ class TestHDP206StackAdvisor(TestCase): expected = { "yarn-env": { "properties": { - "min_user_id": "500" + "min_user_id": "500", + 'service_check.queue.name': 'default' } }, "yarn-site": { @@ -607,6 +567,7 @@ class TestHDP206StackAdvisor(TestCase): expected = { "mapred-site": { "properties": { + 'mapreduce.job.queuename': 'default', "yarn.app.mapreduce.am.resource.mb": "123", "yarn.app.mapreduce.am.command-opts": "-Xmx99m", "mapreduce.map.memory.mb": "567", http://git-wip-us.apache.org/repos/asf/ambari/blob/16175189/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py index a7e4072..08b9554 100644 --- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py @@ -95,6 +95,7 @@ class TestHDP22StackAdvisor(TestCase): "tez-site": { "properties": { # tez.am.resource.memory.mb must be <= yarn.scheduler.maximum-allocation-mb + 'tez.queue.name': 'default', "tez.am.resource.memory.mb": "2048", "tez.task.resource.memory.mb": "768", "tez.runtime.io.sort.mb": "307", @@ -188,6 +189,7 @@ class TestHDP22StackAdvisor(TestCase): "tez-site": { "properties": { # tez.am.resource.memory.mb must be <= yarn.scheduler.maximum-allocation-mb + 'tez.queue.name': 'default', "tez.am.resource.memory.mb": "2048", "tez.task.resource.memory.mb": "768", "tez.runtime.io.sort.mb": "307", @@ -272,6 +274,7 @@ class TestHDP22StackAdvisor(TestCase): "tez-site": { "properties": { # tez.am.resource.memory.mb must be <= yarn.scheduler.maximum-allocation-mb + 'tez.queue.name': 'default', "tez.am.resource.memory.mb": "2048", "tez.task.resource.memory.mb": "760", "tez.runtime.io.sort.mb": "304", @@ -865,7 +868,8 @@ class TestHDP22StackAdvisor(TestCase): expected = { "yarn-env": { "properties": { - "min_user_id": "500" + "min_user_id": "500", + 'service_check.queue.name': 'default' } }, "yarn-site": { @@ -909,7 +913,8 @@ class TestHDP22StackAdvisor(TestCase): expected = { "yarn-env": { "properties": { - "min_user_id": "500" + "min_user_id": "500", + 'service_check.queue.name': 'default' } }, "yarn-site": { @@ -1226,6 +1231,11 @@ class TestHDP22StackAdvisor(TestCase): 'hive.security.authenticator.manager': {'delete': 'true'}, 'hive.conf.restricted.list': {'delete': 'true'} } + }, + 'webhcat-site': { + 'properties': { + 'templeton.hadoop.queue.name': 'queue1' + } } } @@ -1606,11 +1616,13 @@ class TestHDP22StackAdvisor(TestCase): }, "yarn-env": { "properties": { - "min_user_id": "500" + "min_user_id": "500", + 'service_check.queue.name': 'default' } }, "mapred-site": { "properties": { + 'mapreduce.job.queuename': 'default', "mapreduce.map.memory.mb": "1536", "mapreduce.reduce.memory.mb": "1536", "yarn.app.mapreduce.am.command-opts": "-Xmx80m -Dhdp.version=${hdp.version}", @@ -1867,11 +1879,13 @@ class TestHDP22StackAdvisor(TestCase): expected = { "yarn-env": { "properties": { - "min_user_id": "500" + "min_user_id": "500", + 'service_check.queue.name': 'default' } }, "mapred-site": { "properties": { + 'mapreduce.job.queuename': 'default', "mapreduce.map.memory.mb": "100", "mapreduce.reduce.memory.mb": "200", "yarn.app.mapreduce.am.command-opts": "-Xmx80m -Dhdp.version=${hdp.version}", @@ -2080,11 +2094,13 @@ class TestHDP22StackAdvisor(TestCase): expected = { "yarn-env": { "properties": { - "min_user_id": "500" + "min_user_id": "500", + 'service_check.queue.name': 'default' } }, "mapred-site": { "properties": { + 'mapreduce.job.queuename': 'default', "mapreduce.map.memory.mb": "700", "mapreduce.reduce.memory.mb": "1280", "yarn.app.mapreduce.am.command-opts": "-Xmx560m -Dhdp.version=${hdp.version}", @@ -3264,7 +3280,12 @@ class TestHDP22StackAdvisor(TestCase): 'tez.tez-ui.history-url.base' : 'http://host:8080/#/main/views/TEZ/0.7.0.2.3.0.0-2155/TEZ_CLUSTER_INSTANCE'} - res_expected = [{'config-name': 'tez.tez-ui.history-url.base', + res_expected = [{'config-name': 'tez.queue.name', + 'config-type': 'tez-site', + 'level': 'ERROR', + 'message': 'Value should be set', + 'type': 'configuration'}, + {'config-name': 'tez.tez-ui.history-url.base', 'config-type': 'tez-site', 'level': 'WARN', 'message': "It is recommended to set value https://host:8443/#/main/views/TEZ/0.7.0.2.3.0.0-2155/TEZ_CLUSTER_INSTANCE for property tez.tez-ui.history-url.base", @@ -3318,15 +3339,18 @@ class TestHDP22StackAdvisor(TestCase): def test_validateYARNConfigurationsEnv(self): configurations = {} + services = {} + services['configurations'] = configurations # 1) ok: No yarn_cgroups_enabled recommendedDefaults = {'namenode_heapsize': '1024', 'namenode_opt_newsize' : '256', 'namenode_opt_maxnewsize' : '256'} properties = {} + properties['service_check.queue.name'] = 'default' res_expected = [] - res = self.stackAdvisor.validateYARNEnvConfigurations(properties, recommendedDefaults, configurations, '', '') + res = self.stackAdvisor.validateYARNEnvConfigurations(properties, recommendedDefaults, configurations, services, '') self.assertEquals(res, res_expected) # 2) ok: yarn_cgroups_enabled=false, but security enabled @@ -3340,13 +3364,13 @@ class TestHDP22StackAdvisor(TestCase): } } res_expected = [] - res = self.stackAdvisor.validateYARNEnvConfigurations(properties, recommendedDefaults, configurations, '', '') + res = self.stackAdvisor.validateYARNEnvConfigurations(properties, recommendedDefaults, configurations, services, '') self.assertEquals(res, res_expected) # 3) ok: yarn_cgroups_enabled=true, but security enabled properties['yarn_cgroups_enabled'] = 'true' res_expected = [] - res = self.stackAdvisor.validateYARNEnvConfigurations(properties, recommendedDefaults, configurations, '', '') + res = self.stackAdvisor.validateYARNEnvConfigurations(properties, recommendedDefaults, configurations, services, '') self.assertEquals(res, res_expected) # 4) fail: yarn_cgroups_enabled=true, but security disabled @@ -3356,7 +3380,7 @@ class TestHDP22StackAdvisor(TestCase): 'type': 'configuration', 'config-name': 'yarn_cgroups_enabled', 'level': 'WARN'}] - res = self.stackAdvisor.validateYARNEnvConfigurations(properties, recommendedDefaults, configurations, '', '') + res = self.stackAdvisor.validateYARNEnvConfigurations(properties, recommendedDefaults, configurations, services, '') self.assertEquals(res, res_expected) def test_validateMR2XmxOptsEnv(self): @@ -3390,6 +3414,11 @@ class TestHDP22StackAdvisor(TestCase): 'type': 'configuration', 'config-name': 'yarn.app.mapreduce.am.command-opts', 'level': 'WARN'}, + {'config-name': 'mapreduce.job.queuename', + 'config-type': 'mapred-site', + 'level': 'ERROR', + 'message': 'Value should be set', + 'type': 'configuration'}, {'config-type': 'mapred-site', 'message': 'yarn.app.mapreduce.am.command-opts Xmx should be less than yarn.app.mapreduce.am.resource.mb (410)', 'type': 'configuration', @@ -3571,7 +3600,8 @@ class TestHDP22StackAdvisor(TestCase): expected = { "yarn-env": { "properties": { - "min_user_id": "500" + "min_user_id": "500", + "service_check.queue.name": "default" } }, "yarn-site": { @@ -3626,7 +3656,8 @@ class TestHDP22StackAdvisor(TestCase): expected = { "yarn-env": { "properties": { - "min_user_id": "500" + "min_user_id": "500", + 'service_check.queue.name': 'default' } }, "yarn-site": { http://git-wip-us.apache.org/repos/asf/ambari/blob/16175189/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py index 9d830da..62d926e 100644 --- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py @@ -1342,6 +1342,11 @@ class TestHDP23StackAdvisor(TestCase): 'hive.security.authorization.manager': {'delete': 'true'}, 'hive.security.authenticator.manager': {'delete': 'true'} } + }, + 'webhcat-site': { + 'properties': { + 'templeton.hadoop.queue.name': 'queue1' + } } } services = { @@ -1603,6 +1608,11 @@ class TestHDP23StackAdvisor(TestCase): 'hive.security.authorization.manager': {'delete': 'true'}, 'hive.security.authenticator.manager': {'delete': 'true'} } + }, + 'webhcat-site': { + 'properties': { + 'templeton.hadoop.queue.name': 'queue1' + } } } services = { @@ -1821,7 +1831,8 @@ class TestHDP23StackAdvisor(TestCase): "tez.runtime.io.sort.mb": "307", "tez.session.am.dag.submit.timeout.secs": "600", "tez.runtime.unordered.output.buffer.size-mb": "57", - "tez.am.resource.memory.mb": "4000" + "tez.am.resource.memory.mb": "4000", + "tez.queue.name": "queue1", } }, "yarn-site": { http://git-wip-us.apache.org/repos/asf/ambari/blob/16175189/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py index 1bc53ea..c9c21ae 100644 --- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py @@ -7130,27 +7130,17 @@ class TestHDP25StackAdvisor(TestCase): ] services = self.prepareServices(servicesInfo) services["configurations"] = {"yarn-env":{"properties":{"service_check.queue.name": "default"}}, - "capacity-scheduler":{"properties":{"capacity-scheduler": - "yarn.scheduler.capacity.ndfqueue.minimum-user-limit-percent=100\n" + - "yarn.scheduler.capacity.maximum-am-resource-percent=0.2\n" + - "yarn.scheduler.capacity.maximum-applications=10000\n" + - "yarn.scheduler.capacity.node-locality-delay=40\n" + - "yarn.scheduler.capacity.root.accessible-node-labels=*\n" + - "yarn.scheduler.capacity.root.acl_administer_queue=*\n" + - "yarn.scheduler.capacity.root.capacity=100\n" + - "yarn.scheduler.capacity.root.ndfqueue.acl_administer_jobs=*\n" + - "yarn.scheduler.capacity.root.ndfqueue.acl_submit_applications=*\n" + - "yarn.scheduler.capacity.root.ndfqueue.capacity=100\n" + - "yarn.scheduler.capacity.root.ndfqueue.maximum-capacity=100\n" + - "yarn.scheduler.capacity.root.ndfqueue.state=RUNNING\n" + - "yarn.scheduler.capacity.root.ndfqueue.user-limit-factor=1\n" + - "yarn.scheduler.capacity.root.queues=ndfqueue\n"}}} + "capacity-scheduler":{"properties":{"capacity-scheduler": "yarn.scheduler.capacity.root.queues=ndfqueue\n"}}} hosts = self.prepareHosts([]) result = self.stackAdvisor.validateConfigurations(services, hosts) expectedItems = [ - {'message': 'service_check.queue.name is not exist, or not corresponds to existing leaf queue', 'level': 'ERROR'} + {'message': 'Queue is not exist, or not corresponds to existing YARN leaf queue', 'level': 'ERROR'} ] self.assertValidationResult(expectedItems, result) + services["configurations"]["yarn-env"]["properties"]["service_check.queue.name"] = "ndfqueue" + expectedItems = [] + result = self.stackAdvisor.validateConfigurations(services, hosts) + self.assertValidationResult(expectedItems, result) def assertValidationResult(self, expectedItems, result): actualItems = []
