Repository: ambari Updated Branches: refs/heads/branch-2.4 d08da2d25 -> 7d2df9fff refs/heads/trunk dc0722265 -> 2b7c97fc1
AMBARI-18034. Issues with custom default queue support (dgrinenko via dlysnichenko) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7d2df9ff Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7d2df9ff Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7d2df9ff Branch: refs/heads/branch-2.4 Commit: 7d2df9fff48a90b675672fbd1de817431f67e56a Parents: d08da2d Author: Lisnichenko Dmitro <[email protected]> Authored: Fri Aug 5 19:27:45 2016 +0300 Committer: Lisnichenko Dmitro <[email protected]> Committed: Fri Aug 5 19:27:45 2016 +0300 ---------------------------------------------------------------------- .../server/upgrade/UpgradeCatalog240.java | 48 ------ .../0.12.0.2.0/configuration/webhcat-site.xml | 2 +- .../4.0.0.2.0/package/scripts/params_linux.py | 2 +- .../1.2.1/configuration/spark-defaults.xml | 2 +- .../configuration/spark-thrift-sparkconf.xml | 2 +- .../2.0.0/configuration/spark2-defaults.xml | 2 +- .../configuration/spark2-thrift-sparkconf.xml | 2 +- .../TEZ/0.4.0.2.1/configuration/tez-site.xml | 2 +- .../configuration-mapred/mapred-site.xml | 2 +- .../YARN/2.1.0.2.0/configuration/yarn-env.xml | 2 +- .../2.1.0.2.0/package/scripts/params_linux.py | 2 +- .../stacks/HDP/2.0.6/services/stack_advisor.py | 62 ++++---- .../stacks/HDP/2.1/services/stack_advisor.py | 4 +- .../stacks/HDP/2.2/services/stack_advisor.py | 24 ++- .../stacks/HDP/2.5/services/stack_advisor.py | 9 +- .../server/upgrade/UpgradeCatalog240Test.java | 153 ------------------- .../stacks/2.0.6/common/test_stack_advisor.py | 2 +- .../stacks/2.2/common/test_stack_advisor.py | 2 +- .../stacks/2.3/common/test_stack_advisor.py | 6 +- .../stacks/2.5/common/test_stack_advisor.py | 2 +- 20 files changed, 80 insertions(+), 252 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/7d2df9ff/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java index 5c96e5a..8c22eeb 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java @@ -192,10 +192,6 @@ public class UpgradeCatalog240 extends AbstractUpgradeCatalog { protected static final String EXTENSION_LINK_ID_COLUMN = "link_id"; private static final Map<String, Integer> ROLE_ORDER; - public static final String WEBHCAT_SITE_QUEUE_NAME = "templeton.hadoop.queue.name"; - public static final String TEZ_SITE_QUEUE_NAME = "tez.queue.name"; - public static final String YARN_ENV_QUEUE_NAME = "service_check.queue.name"; - public static final String MAPRED_SITE_QUEUE_NAME = "mapreduce.job.queuename"; private static final String AMS_HBASE_SITE = "ams-hbase-site"; private static final String HBASE_RPC_TIMEOUT_PROPERTY = "hbase.rpc.timeout"; private static final String AMS_HBASE_SITE_NORMALIZER_ENABLED_PROPERTY = "hbase.normalizer.enabled"; @@ -402,7 +398,6 @@ public class UpgradeCatalog240 extends AbstractUpgradeCatalog { updateSparkConfigs(); updateHBaseConfigs(); updateFalconConfigs(); - updateQueueNameConfigs(); updateKerberosDescriptorArtifacts(); removeHiveOozieDBConnectionConfigs(); updateClustersAndHostsVersionStateTableDML(); @@ -2245,49 +2240,6 @@ public class UpgradeCatalog240 extends AbstractUpgradeCatalog { } /** - * Updates the YARN Capacity Scheduler related configurations for the clusters managed by this Ambari - * Update services configuration to set proper leaf queue name for services - * update mapred-site, webhcat-site, tez-site, yarn-env - * - * @throws AmbariException if an error occurs while updating the configurations - */ - protected void updateQueueNameConfigs() throws AmbariException { - AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class); - Clusters clusters = ambariManagementController.getClusters(); - Map<String, Cluster> clusterMap = getCheckedClusterMap(clusters); - - for (final Cluster cluster : clusterMap.values()) { - Config capacityScheduler = cluster.getDesiredConfigByType(CAPACITY_SCHEDULER_CONFIG); - if (capacityScheduler != null) { - Map<String, String> capacitySchedulerProperties = capacityScheduler.getProperties(); - Set<String> leafQueues; - leafQueues = getCapacitySchedulerLeafQueues(capacitySchedulerProperties); - Set<String> installedServices = cluster.getServices().keySet(); - if (leafQueues ==null || leafQueues.isEmpty()) { - LOG.warn("There is no leafQueues in capacity-scheduler"); - return; - } - if (installedServices.contains(cluster.getServiceByConfigType(WEBHCAT_SITE_CONFIG)) && - !isQueueNameValid(cluster, leafQueues, WEBHCAT_SITE_QUEUE_NAME, WEBHCAT_SITE_CONFIG)){ - updateQueueName(cluster, leafQueues, WEBHCAT_SITE_QUEUE_NAME, WEBHCAT_SITE_CONFIG); - } - if (installedServices.contains(cluster.getServiceByConfigType(TEZ_SITE_CONFIG)) && - !isQueueNameValid(cluster, leafQueues, TEZ_SITE_QUEUE_NAME, TEZ_SITE_CONFIG)){ - updateQueueName(cluster, leafQueues, TEZ_SITE_QUEUE_NAME, TEZ_SITE_CONFIG); - } - if (installedServices.contains(cluster.getServiceByConfigType(YARN_ENV_CONFIG)) && - !isQueueNameValid(cluster, leafQueues, YARN_ENV_QUEUE_NAME, YARN_ENV_CONFIG)){ - updateQueueName(cluster, leafQueues, YARN_ENV_QUEUE_NAME, YARN_ENV_CONFIG); - } - if (installedServices.contains(cluster.getServiceByConfigType(MAPRED_SITE_CONFIG)) && - !isQueueNameValid(cluster, leafQueues, MAPRED_SITE_QUEUE_NAME, MAPRED_SITE_CONFIG)){ - updateQueueName(cluster, leafQueues, MAPRED_SITE_QUEUE_NAME, MAPRED_SITE_CONFIG); - } - } - } - } - - /** * Updates the Spark-related configurations for the clusters managed by this Ambari * Removes falcon_store_uri from falcon-env. * Updates {{hdp_full_version}} to {{full_stack_version}} http://git-wip-us.apache.org/repos/asf/ambari/blob/7d2df9ff/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/webhcat-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/webhcat-site.xml b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/webhcat-site.xml index d8012dd..cc5de73 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/webhcat-site.xml +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/webhcat-site.xml @@ -155,6 +155,6 @@ limitations under the License. <name>yarn.scheduler.capacity.root.queues</name> </property> </depends-on> - <on-ambari-upgrade add="true"/> + <on-ambari-upgrade add="false"/> </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/7d2df9ff/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py index ca57c81..db048bb 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py @@ -113,7 +113,7 @@ oozie_user = config['configurations']['oozie-env']['oozie_user'] smokeuser = config['configurations']['cluster-env']['smokeuser'] smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name'] smoke_hdfs_user_mode = 0770 -service_check_queue_name = default('/configurations/yarn-env/service_check.queue.name','default') +service_check_queue_name = default('/configurations/yarn-env/service_check.queue.name', 'default') # This config actually contains {oozie_user} oozie_admin_users = format(config['configurations']['oozie-env']['oozie_admin_users']) http://git-wip-us.apache.org/repos/asf/ambari/blob/7d2df9ff/ambari-server/src/main/resources/common-services/SPARK/1.2.1/configuration/spark-defaults.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/configuration/spark-defaults.xml b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/configuration/spark-defaults.xml index ecb103c..748c270 100644 --- a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/configuration/spark-defaults.xml +++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/configuration/spark-defaults.xml @@ -75,7 +75,7 @@ <name>yarn.scheduler.capacity.root.queues</name> </property> </depends-on> - <on-ambari-upgrade add="true"/> + <on-ambari-upgrade add="false"/> </property> <property> <name>spark.yarn.containerLauncherMaxThreads</name> http://git-wip-us.apache.org/repos/asf/ambari/blob/7d2df9ff/ambari-server/src/main/resources/common-services/SPARK/1.5.2/configuration/spark-thrift-sparkconf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.5.2/configuration/spark-thrift-sparkconf.xml b/ambari-server/src/main/resources/common-services/SPARK/1.5.2/configuration/spark-thrift-sparkconf.xml index 865aa73..42ed316 100644 --- a/ambari-server/src/main/resources/common-services/SPARK/1.5.2/configuration/spark-thrift-sparkconf.xml +++ b/ambari-server/src/main/resources/common-services/SPARK/1.5.2/configuration/spark-thrift-sparkconf.xml @@ -91,7 +91,7 @@ <name>yarn.scheduler.capacity.root.queues</name> </property> </depends-on> - <on-ambari-upgrade add="true"/> + <on-ambari-upgrade add="false"/> </property> <property> <name>spark.yarn.containerLauncherMaxThreads</name> http://git-wip-us.apache.org/repos/asf/ambari/blob/7d2df9ff/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/configuration/spark2-defaults.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/configuration/spark2-defaults.xml b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/configuration/spark2-defaults.xml index 2f9334e..4c28d46 100755 --- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/configuration/spark2-defaults.xml +++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/configuration/spark2-defaults.xml @@ -31,7 +31,7 @@ <name>yarn.scheduler.capacity.root.queues</name> </property> </depends-on> - <on-ambari-upgrade add="true"/> + <on-ambari-upgrade add="false"/> </property> <property> <name>spark.driver.extraLibraryPath</name> http://git-wip-us.apache.org/repos/asf/ambari/blob/7d2df9ff/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/configuration/spark2-thrift-sparkconf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/configuration/spark2-thrift-sparkconf.xml b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/configuration/spark2-thrift-sparkconf.xml index dfe87cb..6d73946 100755 --- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/configuration/spark2-thrift-sparkconf.xml +++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/configuration/spark2-thrift-sparkconf.xml @@ -31,7 +31,7 @@ <name>yarn.scheduler.capacity.root.queues</name> </property> </depends-on> - <on-ambari-upgrade add="true"/> + <on-ambari-upgrade add="false"/> </property> <property> <name>spark.driver.extraLibraryPath</name> http://git-wip-us.apache.org/repos/asf/ambari/blob/7d2df9ff/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/configuration/tez-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/configuration/tez-site.xml b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/configuration/tez-site.xml index e7a851c..a39583c 100644 --- a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/configuration/tez-site.xml +++ b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/configuration/tez-site.xml @@ -243,6 +243,6 @@ <name>yarn.scheduler.capacity.root.queues</name> </property> </depends-on> - <on-ambari-upgrade add="true"/> + <on-ambari-upgrade add="false"/> </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/7d2df9ff/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml index 6951db0..434eea0 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml @@ -476,6 +476,6 @@ <name>yarn.scheduler.capacity.root.queues</name> </property> </depends-on> - <on-ambari-upgrade add="true"/> + <on-ambari-upgrade add="false"/> </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/7d2df9ff/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml index 152c463..c3bbcb6 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml @@ -255,6 +255,6 @@ YARN_OPTS="$YARN_OPTS -Djava.io.tmpdir={{hadoop_java_io_tmpdir}}" <name>yarn.scheduler.capacity.root.queues</name> </property> </depends-on> - <on-ambari-upgrade add="true"/> + <on-ambari-upgrade add="false"/> </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/7d2df9ff/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py index b2c635d..4d42861 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py @@ -183,7 +183,7 @@ mapred_log_dir_prefix = config['configurations']['mapred-env']['mapred_log_dir_p mapred_env_sh_template = config['configurations']['mapred-env']['content'] yarn_env_sh_template = config['configurations']['yarn-env']['content'] yarn_nodemanager_recovery_dir = default('/configurations/yarn-site/yarn.nodemanager.recovery.dir', None) -service_check_queue_name = default('/configurations/yarn-env/service_check.queue.name','default') +service_check_queue_name = default('/configurations/yarn-env/service_check.queue.name', 'default') if len(rm_hosts) > 1: additional_rm_host = rm_hosts[1] http://git-wip-us.apache.org/repos/asf/ambari/blob/7d2df9ff/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py index 2efb60d..87618a8 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py @@ -113,7 +113,11 @@ class HDP206StackAdvisor(DefaultStackAdvisor): putYarnProperty('yarn.scheduler.minimum-allocation-mb', int(clusterData['ramPerContainer'])) putYarnProperty('yarn.scheduler.maximum-allocation-mb', int(configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"])) putYarnEnvProperty('min_user_id', self.get_system_min_uid()) - putYarnEnvProperty("service_check.queue.name", self.recommendYarnQueue(services)) + + sc_queue_name = self.recommendYarnQueue(services, "yarn-env", "service_check.queue.name") + if sc_queue_name is not None: + putYarnEnvProperty("service_check.queue.name", sc_queue_name) + containerExecutorGroup = 'hadoop' if 'cluster-env' in services['configurations'] and 'user_group' in services['configurations']['cluster-env']['properties']: containerExecutorGroup = services['configurations']['cluster-env']['properties']['user_group'] @@ -140,7 +144,9 @@ class HDP206StackAdvisor(DefaultStackAdvisor): putMapredProperty('mapreduce.map.java.opts', "-Xmx" + str(int(round(0.8 * clusterData['mapMemory']))) + "m") putMapredProperty('mapreduce.reduce.java.opts', "-Xmx" + str(int(round(0.8 * clusterData['reduceMemory']))) + "m") putMapredProperty('mapreduce.task.io.sort.mb', min(int(round(0.4 * clusterData['mapMemory'])), 1024)) - putMapredProperty("mapreduce.job.queuename", self.recommendYarnQueue(services)) + mr_queue = self.recommendYarnQueue(services, "mapred-site", "mapreduce.job.queuename") + if mr_queue is not None: + putMapredProperty("mapreduce.job.queuename", mr_queue) def getAmbariUser(self, services): ambari_user = services['ambari-server-properties']['ambari-server.user'] @@ -1382,34 +1388,36 @@ class HDP206StackAdvisor(DefaultStackAdvisor): return None def validatorYarnQueue(self, properties, recommendedDefaults, propertyName, services): - if not propertyName in properties: + if propertyName not in properties: return self.getErrorItem("Value should be set") - capacity_scheduler_properties, received_as_key_value_pair = self.getCapacitySchedulerProperties(services) - leafQueueNames = set() - leafQueues = self.getAllYarnLeafQueues(capacity_scheduler_properties) - for queue in leafQueues: - queue = queue.rstrip('.') - queueName = queue.split('.')[-1] - leafQueueNames.add(queueName) - value = properties[propertyName] - if len(leafQueueNames) == 0: + + capacity_scheduler_properties, _ = self.getCapacitySchedulerProperties(services) + leaf_queue_names = self.getAllYarnLeafQueues(capacity_scheduler_properties) + queue_name = properties[propertyName] + + if len(leaf_queue_names) == 0: return None - if value not in leafQueueNames: - return self.getErrorItem("Queue is not exist, or not corresponds to existing YARN leaf queue") + elif queue_name not in leaf_queue_names: + return self.getErrorItem("Queue is not exist or not corresponds to existing YARN leaf queue") + return None - def recommendYarnQueue(self, services): - if services: - if 'configurations' in services: - capacity_scheduler_properties, received_as_key_value_pair = self.getCapacitySchedulerProperties(services) - leafQueueNames = set() - leafQueues = self.getAllYarnLeafQueues(capacity_scheduler_properties) - for queue in leafQueues: - queue = queue.rstrip('.') - queueName = queue.split('.')[-1] - leafQueueNames.add(queueName) - if leafQueueNames: - return leafQueueNames.pop() + def recommendYarnQueue(self, services, catalog_name=None, queue_property=None): + old_queue_name = None + + if services and 'configurations' in services: + configurations = services["configurations"] + if catalog_name in configurations and queue_property in configurations[catalog_name]["properties"]: + old_queue_name = configurations[catalog_name]["properties"][queue_property] + + capacity_scheduler_properties, _ = self.getCapacitySchedulerProperties(services) + leaf_queues = sorted(self.getAllYarnLeafQueues(capacity_scheduler_properties)) + + if leaf_queues and (old_queue_name is None or old_queue_name not in leaf_queues): + return leaf_queues.pop() + elif old_queue_name and old_queue_name in leaf_queues: + return None + return "default" def validateXmxValue(self, properties, recommendedDefaults, propertyName): @@ -1641,7 +1649,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor): """ def getAllYarnLeafQueues(self, capacitySchedulerProperties): config_list = capacitySchedulerProperties.keys() - yarn_queues = [] + yarn_queues = None leafQueueNames = set() if 'yarn.scheduler.capacity.root.queues' in config_list: yarn_queues = capacitySchedulerProperties.get('yarn.scheduler.capacity.root.queues') http://git-wip-us.apache.org/repos/asf/ambari/blob/7d2df9ff/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py index 6ec4b8d..34b4614 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py @@ -127,7 +127,9 @@ class HDP21StackAdvisor(HDP206StackAdvisor): putTezProperty("tez.am.java.opts", "-server -Xmx" + str(int(0.8 * clusterData["amMemory"])) + "m -Djava.net.preferIPv4Stack=true -XX:+UseNUMA -XX:+UseParallelGC") - putTezProperty("tez.queue.name", self.recommendYarnQueue(services)) + recommended_tez_queue = self.recommendYarnQueue(services, "tez-site", "tez.queue.name") + if recommended_tez_queue is not None: + putTezProperty("tez.queue.name", recommended_tez_queue) def getNotPreferableOnServerComponents(self): http://git-wip-us.apache.org/repos/asf/ambari/blob/7d2df9ff/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py index 9ec1f05..4854514 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py @@ -57,12 +57,17 @@ class HDP22StackAdvisor(HDP21StackAdvisor): :type hosts dict """ putSparkProperty = self.putProperty(configurations, "spark-defaults", services) - putSparkProperty("spark.yarn.queue", self.recommendYarnQueue(services)) + + spark_queue = self.recommendYarnQueue(services, "spark-defaults", "spark.yarn.queue") + if spark_queue is not None: + putSparkProperty("spark.yarn.queue", spark_queue) # add only if spark supports this config if "configurations" in services and "spark-thrift-sparkconf" in services["configurations"]: putSparkThriftSparkConf = self.putProperty(configurations, "spark-thrift-sparkconf", services) - putSparkThriftSparkConf("spark.yarn.queue", self.recommendYarnQueue(services)) + recommended_spark_queue = self.recommendYarnQueue(services, "spark-thrift-sparkconf", "spark.yarn.queue") + if recommended_spark_queue is not None: + putSparkThriftSparkConf("spark.yarn.queue", recommended_spark_queue) def recommendYARNConfigurations(self, configurations, clusterData, services, hosts): @@ -445,7 +450,10 @@ class HDP22StackAdvisor(HDP21StackAdvisor): leafQueues = sorted(leafQueues, key=lambda q:q['value']) putHiveSitePropertyAttribute("hive.server2.tez.default.queues", "entries", leafQueues) putHiveSiteProperty("hive.server2.tez.default.queues", ",".join([leafQueue['value'] for leafQueue in leafQueues])) - putWebhcatSiteProperty("templeton.hadoop.queue.name", self.recommendYarnQueue(services)) + + webhcat_queue = self.recommendYarnQueue(services, "webhcat-site", "templeton.hadoop.queue.name") + if webhcat_queue is not None: + putWebhcatSiteProperty("templeton.hadoop.queue.name", webhcat_queue) # Recommend Ranger Hive authorization as per Ranger Hive plugin property @@ -812,7 +820,10 @@ class HDP22StackAdvisor(HDP21StackAdvisor): putTezProperty("tez.runtime.io.sort.mb", min(int(taskResourceMemory * 0.4), 2047)) putTezProperty("tez.runtime.unordered.output.buffer.size-mb", int(taskResourceMemory * 0.075)) putTezProperty("tez.session.am.dag.submit.timeout.secs", "600") - putTezProperty("tez.queue.name", self.recommendYarnQueue(services)) + + tez_queue = self.recommendYarnQueue(services, "tez-site", "tez.queue.name") + if tez_queue is not None: + putTezProperty("tez.queue.name", tez_queue) serverProperties = services["ambari-server-properties"] latest_tez_jar_version = None @@ -1079,7 +1090,10 @@ class HDP22StackAdvisor(HDP21StackAdvisor): putMapredPropertyAttribute("yarn.app.mapreduce.am.resource.mb", "minimum", yarnMinAllocationSize) # Hadoop MR limitation putMapredPropertyAttribute("mapreduce.task.io.sort.mb", "maximum", "2047") - putMapredProperty("mapreduce.job.queuename", self.recommendYarnQueue(services)) + + mr_queue = self.recommendYarnQueue(services, "mapred-site", "mapreduce.job.queuename") + if mr_queue is not None: + putMapredProperty("mapreduce.job.queuename", mr_queue) def validateMapReduce2Configurations(self, properties, recommendedDefaults, configurations, services, hosts): validationItems = [ {"config-name": 'mapreduce.map.java.opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'mapreduce.map.java.opts')}, http://git-wip-us.apache.org/repos/asf/ambari/blob/7d2df9ff/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py index 68b03ad..e982285 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py @@ -405,8 +405,13 @@ class HDP25StackAdvisor(HDP24StackAdvisor): putSparkProperty = self.putProperty(configurations, "spark2-defaults", services) putSparkThriftSparkConf = self.putProperty(configurations, "spark2-thrift-sparkconf", services) - putSparkProperty("spark.yarn.queue", self.recommendYarnQueue(services)) - putSparkThriftSparkConf("spark.yarn.queue", self.recommendYarnQueue(services)) + spark_queue = self.recommendYarnQueue(services, "spark2-defaults", "spark.yarn.queue") + if spark_queue is not None: + putSparkProperty("spark.yarn.queue", spark_queue) + + spart_thrift_queue = self.recommendYarnQueue(services, "spark2-thrift-sparkconf", "spark.yarn.queue") + if spart_thrift_queue is not None: + putSparkThriftSparkConf("spark.yarn.queue", spart_thrift_queue) def recommendStormConfigurations(self, configurations, clusterData, services, hosts): super(HDP25StackAdvisor, self).recommendStormConfigurations(configurations, clusterData, services, hosts) http://git-wip-us.apache.org/repos/asf/ambari/blob/7d2df9ff/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java index f7c33ad..3b42984 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java @@ -564,7 +564,6 @@ public class UpgradeCatalog240Test { Method updateHostRoleCommandTableDML = UpgradeCatalog240.class.getDeclaredMethod("updateHostRoleCommandTableDML"); Method updateKerberosEnv = UpgradeCatalog240.class.getDeclaredMethod("updateKerberosConfigs"); Method updateYarnEnv = UpgradeCatalog240.class.getDeclaredMethod("updateYarnEnv"); - Method updateQueueNameConfigs = UpgradeCatalog240.class.getDeclaredMethod("updateQueueNameConfigs"); Method removeHiveOozieDBConnectionConfigs = UpgradeCatalog240.class.getDeclaredMethod("removeHiveOozieDBConnectionConfigs"); Method updateClustersAndHostsVersionStateTableDML = UpgradeCatalog240.class.getDeclaredMethod("updateClustersAndHostsVersionStateTableDML"); Method removeStandardDeviationAlerts = UpgradeCatalog240.class.getDeclaredMethod("removeStandardDeviationAlerts"); @@ -614,7 +613,6 @@ public class UpgradeCatalog240Test { .addMockedMethod(updateHostRoleCommandTableDML) .addMockedMethod(updateKerberosEnv) .addMockedMethod(updateYarnEnv) - .addMockedMethod(updateQueueNameConfigs) .addMockedMethod(removeHiveOozieDBConnectionConfigs) .addMockedMethod(updateClustersAndHostsVersionStateTableDML) .addMockedMethod(removeStandardDeviationAlerts) @@ -655,7 +653,6 @@ public class UpgradeCatalog240Test { upgradeCatalog240.updateHostRoleCommandTableDML(); upgradeCatalog240.updateKerberosConfigs(); upgradeCatalog240.updateYarnEnv(); - upgradeCatalog240.updateQueueNameConfigs(); upgradeCatalog240.removeHiveOozieDBConnectionConfigs(); upgradeCatalog240.updateClustersAndHostsVersionStateTableDML(); upgradeCatalog240.removeStandardDeviationAlerts(); @@ -1147,156 +1144,6 @@ public class UpgradeCatalog240Test { assertTrue(Maps.difference(newPropertiesYarnEnv, updatedProperties).areEqual()); } - /** - * Test that queue names updated in mapred-site, webhcat-site, tez-site, yarn-env - * @throws Exception - */ - @SuppressWarnings("unchecked") - @Test - public void testQueueNameUpdateConfigs() throws Exception{ - Map<String, String> oldPropertiesCapacityScheduler = new HashMap<String, String>() { - { - put("yarn.scheduler.capacity.root.queues", "default3,d3"); - put("yarn.scheduler.capacity.root.d3.queues", "default4"); - put("yarn.scheduler.capacity.root.default3.queues", "default5"); - } - }; - Map<String, String> oldPropertiesTezSite = new HashMap<String, String>() { - { - put("tez.queue.name", "default3"); - } - }; - Map<String, String> oldPropertiesWebhcatSite = new HashMap<String, String>() { - { - put("templeton.hadoop.queue.name", "default3"); - } - }; - Map<String, String> oldPropertiesMapredSite = new HashMap<String, String>() { - { - put("mapreduce.job.queuename", "default3"); - } - }; - Map<String, String> oldPropertiesYarnEnv = new HashMap<String, String>() { - { - put("service_check.queue.name", "default3"); - } - }; - - Map<String, String> newPropertiesTezSite = new HashMap<String, String>() { - { - put("tez.queue.name", "default5"); - } - }; - Map<String, String> newPropertiesWebhcatSite = new HashMap<String, String>() { - { - put("templeton.hadoop.queue.name", "default5"); - } - }; - Map<String, String> newPropertiesMapredSite = new HashMap<String, String>() { - { - put("mapreduce.job.queuename", "default5"); - } - }; - Map<String, String> newPropertiesYarnEnv = new HashMap<String, String>() { - { - put("service_check.queue.name", "default5"); - } - }; - - EasyMockSupport easyMockSupport = new EasyMockSupport(); - - Clusters clusters = easyMockSupport.createNiceMock(Clusters.class); - final Cluster cluster = easyMockSupport.createNiceMock(Cluster.class); - Config mockCapacityScheduler = easyMockSupport.createNiceMock(Config.class); - Config mockWebhcatSite = easyMockSupport.createNiceMock(Config.class); - Config mockTezSite = easyMockSupport.createNiceMock(Config.class); - Config mockMapredSite = easyMockSupport.createNiceMock(Config.class); - Config mockYarnEnv = easyMockSupport.createNiceMock(Config.class); - - expect(clusters.getClusters()).andReturn(new HashMap<String, Cluster>() {{ - put("normal", cluster); - }}).anyTimes(); - - expect(cluster.getServices()).andReturn(new HashMap<String, Service>() { - { - put("YARN", null); - put("HIVE", null); - put("MAPREDUCE", null); - put("TEZ", null); - } - }).atLeastOnce(); - - expect(cluster.getServiceByConfigType(WEBHCAT_SITE_CONFIG_TYPE)).andReturn("HIVE").atLeastOnce(); - expect(cluster.getServiceByConfigType(TEZ_SITE_CONFIG_TYPE)).andReturn("TEZ").atLeastOnce(); - expect(cluster.getServiceByConfigType(MAPRED_SITE_CONFIG_TYPE)).andReturn("MAPREDUCE").atLeastOnce(); - expect(cluster.getServiceByConfigType(YARN_ENV_CONFIG_TYPE)).andReturn("YARN").atLeastOnce(); - - expect(cluster.getDesiredConfigByType(CAPACITY_SCHEDULER_CONFIG_TYPE)).andReturn(mockCapacityScheduler).atLeastOnce(); - expect(cluster.getDesiredConfigByType(WEBHCAT_SITE_CONFIG_TYPE)).andReturn(mockWebhcatSite).atLeastOnce(); - expect(cluster.getDesiredConfigByType(TEZ_SITE_CONFIG_TYPE)).andReturn(mockTezSite).atLeastOnce(); - expect(cluster.getDesiredConfigByType(MAPRED_SITE_CONFIG_TYPE)).andReturn(mockMapredSite).atLeastOnce(); - expect(cluster.getDesiredConfigByType(YARN_ENV_CONFIG_TYPE)).andReturn(mockYarnEnv).atLeastOnce(); - - - - expect(mockCapacityScheduler.getProperties()).andReturn(oldPropertiesCapacityScheduler).anyTimes(); - expect(mockWebhcatSite.getProperties()).andReturn(oldPropertiesWebhcatSite).anyTimes(); - expect(mockTezSite.getProperties()).andReturn(oldPropertiesTezSite).anyTimes(); - expect(mockMapredSite.getProperties()).andReturn(oldPropertiesMapredSite).anyTimes(); - expect(mockYarnEnv.getProperties()).andReturn(oldPropertiesYarnEnv).anyTimes(); - - Injector injector = easyMockSupport.createNiceMock(Injector.class); - expect(injector.getInstance(Gson.class)).andReturn(null).anyTimes(); - expect(injector.getInstance(MaintenanceStateHelper.class)).andReturn(null).anyTimes(); - expect(injector.getInstance(KerberosHelper.class)).andReturn(createNiceMock(KerberosHelper.class)).anyTimes(); - - replay(injector, clusters, mockCapacityScheduler, mockWebhcatSite, mockTezSite, mockMapredSite, - mockYarnEnv, cluster); - - AmbariManagementControllerImpl controller = createMockBuilder(AmbariManagementControllerImpl.class) - .addMockedMethod("createConfiguration") - .addMockedMethod("getClusters", new Class[] { }) - .addMockedMethod("createConfig") - .withConstructor(createNiceMock(ActionManager.class), clusters, injector) - .createNiceMock(); - - Injector injector2 = easyMockSupport.createNiceMock(Injector.class); - Capture<Map> propertiesCapacityScheduler = EasyMock.newCapture(); - Capture<Map> propertiesCaptureYarnEnv = EasyMock.newCapture(); - Capture<Map> propertiesCaptureWebhcatSite = EasyMock.newCapture(); - Capture<Map> propertiesCaptureTezSite= EasyMock.newCapture(); - Capture<Map> propertiesCaptureMapredSite = EasyMock.newCapture(); - - expect(injector2.getInstance(AmbariManagementController.class)).andReturn(controller).anyTimes(); - expect(controller.getClusters()).andReturn(clusters).anyTimes(); - expect(controller.createConfig(anyObject(Cluster.class), eq(CAPACITY_SCHEDULER_CONFIG_TYPE), capture(propertiesCapacityScheduler), anyString(), - anyObject(Map.class))).andReturn(createNiceMock(Config.class)).once(); - expect(controller.createConfig(anyObject(Cluster.class), eq(YARN_ENV_CONFIG_TYPE), capture(propertiesCaptureYarnEnv), anyString(), - anyObject(Map.class))).andReturn(createNiceMock(Config.class)).once(); - expect(controller.createConfig(anyObject(Cluster.class), eq(WEBHCAT_SITE_CONFIG_TYPE), capture(propertiesCaptureWebhcatSite), anyString(), - anyObject(Map.class))).andReturn(createNiceMock(Config.class)).once(); - expect(controller.createConfig(anyObject(Cluster.class), eq(TEZ_SITE_CONFIG_TYPE), capture(propertiesCaptureTezSite), anyString(), - anyObject(Map.class))).andReturn(createNiceMock(Config.class)).once(); - expect(controller.createConfig(anyObject(Cluster.class), eq(MAPRED_SITE_CONFIG_TYPE), capture(propertiesCaptureMapredSite), anyString(), - anyObject(Map.class))).andReturn(createNiceMock(Config.class)).once(); - - replay(controller, injector2); - new UpgradeCatalog240(injector2).updateQueueNameConfigs(); - easyMockSupport.verifyAll(); - - Map<String, String> updatedPropertiesYarnEnv = propertiesCaptureYarnEnv.getValue(); - assertTrue(Maps.difference(newPropertiesYarnEnv, updatedPropertiesYarnEnv).areEqual()); - - Map<String, String> updatedPropertiesWebhcatSite = propertiesCaptureWebhcatSite.getValue(); - assertTrue(Maps.difference(newPropertiesWebhcatSite, updatedPropertiesWebhcatSite).areEqual()); - - Map<String, String> updatedPropertiesTezSite = propertiesCaptureTezSite.getValue(); - assertTrue(Maps.difference(newPropertiesTezSite, updatedPropertiesTezSite).areEqual()); - - Map<String, String> updatedPropertiesMapredSite = propertiesCaptureMapredSite.getValue(); - assertTrue(Maps.difference(newPropertiesMapredSite, updatedPropertiesMapredSite).areEqual()); - } - @Test public void testSparkConfigUpdate() throws Exception{ http://git-wip-us.apache.org/repos/asf/ambari/blob/7d2df9ff/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py index 4a12920..e28389e 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py @@ -222,7 +222,7 @@ class TestHDP206StackAdvisor(TestCase): hosts = self.prepareHosts([]) result = self.stackAdvisor.validateConfigurations(services, hosts) expectedItems = [ - {'message': 'Queue is not exist, or not corresponds to existing YARN leaf queue', 'level': 'ERROR'} + {'message': 'Queue is not exist or not corresponds to existing YARN leaf queue', 'level': 'ERROR'} ] self.assertValidationResult(expectedItems, result) services["configurations"]["yarn-env"]["properties"]["service_check.queue.name"] = "ndfqueue" http://git-wip-us.apache.org/repos/asf/ambari/blob/7d2df9ff/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py index 75a26d2..c3c039b 100644 --- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py @@ -1283,7 +1283,7 @@ class TestHDP22StackAdvisor(TestCase): }, 'webhcat-site': { 'properties': { - 'templeton.hadoop.queue.name': 'queue1' + 'templeton.hadoop.queue.name': 'queue2' } } } http://git-wip-us.apache.org/repos/asf/ambari/blob/7d2df9ff/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py index 272269d..137fd47 100644 --- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py @@ -657,7 +657,7 @@ class TestHDP23StackAdvisor(TestCase): }, 'webhcat-site': { 'properties': { - 'templeton.hadoop.queue.name': 'queue1' + 'templeton.hadoop.queue.name': 'queue2' } } } @@ -933,7 +933,7 @@ class TestHDP23StackAdvisor(TestCase): }, 'webhcat-site': { 'properties': { - 'templeton.hadoop.queue.name': 'queue1' + 'templeton.hadoop.queue.name': 'queue2' } } } @@ -1163,7 +1163,7 @@ class TestHDP23StackAdvisor(TestCase): "tez.session.am.dag.submit.timeout.secs": "600", "tez.runtime.unordered.output.buffer.size-mb": "57", "tez.am.resource.memory.mb": "4000", - "tez.queue.name": "queue1", + "tez.queue.name": "queue2", } }, "yarn-site": { http://git-wip-us.apache.org/repos/asf/ambari/blob/7d2df9ff/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py index d1f6d88..0ed1761 100644 --- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py @@ -7252,7 +7252,7 @@ class TestHDP25StackAdvisor(TestCase): hosts = self.prepareHosts([]) result = self.stackAdvisor.validateConfigurations(services, hosts) expectedItems = [ - {'message': 'Queue is not exist, or not corresponds to existing YARN leaf queue', 'level': 'ERROR'} + {'message': 'Queue is not exist or not corresponds to existing YARN leaf queue', 'level': 'ERROR'} ] self.assertValidationResult(expectedItems, result) services["configurations"]["yarn-env"]["properties"]["service_check.queue.name"] = "ndfqueue2"
