Repository: ambari Updated Branches: refs/heads/branch-2.4 42a892ac3 -> 2fa199878
AMBARI-17503. Service check fails on non default queue due to change in a way of leaf queue retrivieng (dgrinenko via dlysnichenko) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2fa19987 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2fa19987 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2fa19987 Branch: refs/heads/branch-2.4 Commit: 2fa199878156b2da9834c62f8b4402a5debf9930 Parents: 42a892a Author: Lisnichenko Dmitro <[email protected]> Authored: Thu Jun 30 19:57:31 2016 +0300 Committer: Lisnichenko Dmitro <[email protected]> Committed: Thu Jun 30 19:58:04 2016 +0300 ---------------------------------------------------------------------- .../stacks/HDP/2.0.6/services/stack_advisor.py | 14 ++++- .../stacks/2.5/common/test_stack_advisor.py | 54 +++++++++++++++++++- 2 files changed, 64 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/2fa19987/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py index 9a310b5..06f7cfe 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py @@ -1360,7 +1360,12 @@ class HDP206StackAdvisor(DefaultStackAdvisor): if not propertyName in properties: return self.getErrorItem("Value should be set") capacity_scheduler_properties, received_as_key_value_pair = self.getCapacitySchedulerProperties(services) - leafQueueNames = self.getAllYarnLeafQueues(capacity_scheduler_properties) + leafQueueNames = set() + leafQueues = self.getAllYarnLeafQueues(capacity_scheduler_properties) + for queue in leafQueues: + queue = queue.rstrip('.') + queueName = queue.split('.')[-1] + leafQueueNames.add(queueName) value = properties[propertyName] if len(leafQueueNames) == 0: return None @@ -1372,7 +1377,12 @@ class HDP206StackAdvisor(DefaultStackAdvisor): if services: if 'configurations' in services: capacity_scheduler_properties, received_as_key_value_pair = self.getCapacitySchedulerProperties(services) - leafQueueNames = self.getAllYarnLeafQueues(capacity_scheduler_properties) + leafQueueNames = set() + leafQueues = self.getAllYarnLeafQueues(capacity_scheduler_properties) + for queue in leafQueues: + queue = queue.rstrip('.') + queueName = queue.split('.')[-1] + leafQueueNames.add(queueName) if leafQueueNames: return leafQueueNames.pop() return "default" http://git-wip-us.apache.org/repos/asf/ambari/blob/2fa19987/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py index 558d32d..a5bfefc 100644 --- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py @@ -7060,17 +7060,67 @@ class TestHDP25StackAdvisor(TestCase): ] services = self.prepareServices(servicesInfo) services["configurations"] = {"yarn-env":{"properties":{"service_check.queue.name": "default"}}, - "capacity-scheduler":{"properties":{"capacity-scheduler": "yarn.scheduler.capacity.root.queues=ndfqueue\n"}}} + "capacity-scheduler":{"properties":{ + "capacity-scheduler": "yarn.scheduler.capacity.root.queues=ndfqueue,leaf\n" + + "yarn.scheduler.capacity.root.ndfqueue.queues=ndfqueue1,ndfqueue2\n"}}} hosts = self.prepareHosts([]) result = self.stackAdvisor.validateConfigurations(services, hosts) expectedItems = [ {'message': 'Queue is not exist, or not corresponds to existing YARN leaf queue', 'level': 'ERROR'} ] self.assertValidationResult(expectedItems, result) - services["configurations"]["yarn-env"]["properties"]["service_check.queue.name"] = "ndfqueue" + services["configurations"]["yarn-env"]["properties"]["service_check.queue.name"] = "ndfqueue2" expectedItems = [] result = self.stackAdvisor.validateConfigurations(services, hosts) self.assertValidationResult(expectedItems, result) + services["configurations"]["yarn-env"]["properties"]["service_check.queue.name"] = "leaf" + expectedItems = [] + result = self.stackAdvisor.validateConfigurations(services, hosts) + self.assertValidationResult(expectedItems, result) + + def test_recommendYARNQueueConfigurations(self): + configurations = {"yarn-env":{"properties":{"service_check.queue.name": "default"}}, + "capacity-scheduler":{"properties":{ + "capacity-scheduler": "yarn.scheduler.capacity.root.queues=ndfqueue\n" + + "yarn.scheduler.capacity.root.ndfqueue.queues=ndfqueue1,ndfqueue2\n"}}} + services = {"configurations": configurations, "services": [], "ambari-server-properties": {}} + clusterData = { + "containers" : 5, + "ramPerContainer": 256, + "mapMemory": 567, + "reduceMemory": 345.6666666666666, + "amMemory": 123.54, + "cpu": 4, + "referenceNodeManagerHost" : { + "total_mem" : 328960 * 1024 + } + } + hosts = { + "items" : [ + { + "href" : "/api/v1/hosts/c6401.ambari.apache.org", + "Hosts" : { + "cpu_count" : 1, + "host_name" : "c6401.ambari.apache.org", + "os_arch" : "x86_64", + "os_type" : "centos6", + "ph_cpu_count" : 1, + "public_host_name" : "c6401.ambari.apache.org", + "rack_info" : "/default-rack", + "total_mem" : 1922680 + } + } + ] + } + + self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, hosts) + self.stackAdvisor.recommendMapReduce2Configurations(configurations, clusterData, services, hosts) + self.stackAdvisor.recommendHIVEConfigurations(configurations, clusterData, services, hosts) + self.stackAdvisor.recommendTezConfigurations(configurations, clusterData, services, hosts) + self.assertEquals(configurations["yarn-env"]["properties"]["service_check.queue.name"], "ndfqueue2") + self.assertEquals(configurations["mapred-site"]["properties"]["mapreduce.job.queuename"], "ndfqueue2") + self.assertEquals(configurations["webhcat-site"]["properties"]["templeton.hadoop.queue.name"], "ndfqueue2") + self.assertEquals(configurations["tez-site"]["properties"]["tez.queue.name"], "ndfqueue2") def assertValidationResult(self, expectedItems, result): actualItems = []
