http://git-wip-us.apache.org/repos/asf/ambari/blob/ab9acef4/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py index f4538d7..54fe42a 100644 --- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py @@ -548,6 +548,7 @@ class TestHDP25StackAdvisor(TestCase): pass + ''' TODO: Commenting, Need to fix validations in 2.5/stack_advisor and then fix the test code """ Tests validations for Hive Server Interactive site. """ @@ -667,2441 +668,14 @@ class TestHDP25StackAdvisor(TestCase): self.assertEquals(res4, res_expected4) pass + ''' - - - # Tests related to 'recommendYARNConfigurations()' - - - # Test 1 : (1). Only default queue exists in capacity-scheduler and 'capacity-scheduler' configs are passed-in as - # single "/n" separated string (2). enable_hive_interactive' is 'On' and 'llap_queue_capacity is 0. - def test_recommendYARNConfigurations_create_llap_queue_1(self): - - services = { - "Versions": { - "parent_stack_version": "2.4", - "stack_name": "HDP", - "stack_version": "2.5", - "stack_hierarchy": { - "stack_name": "HDP", - "stack_versions": ["2.4", "2.3", "2.2", "2.1", "2.0.6"] - } - }, - "services": [{ - "StackServices": { - "service_name": "YARN", - }, - "components": [ - { - "StackServiceComponents": { - "component_name": "NODEMANAGER", - "hostnames": ["c6401.ambari.apache.org"] - }, - } - ] - }, { - "href": "/api/v1/stacks/HDP/versions/2.5/services/HIVE", - "StackServices": { - "service_name": "HIVE", - "service_version": "1.2.1.2.5", - "stack_name": "HDP", - "stack_version": "2.5" - }, - "components": [ - { - "href": "/api/v1/stacks/HDP/versions/2.5/services/HIVE/components/HIVE_SERVER_INTERACTIVE", - "StackServiceComponents": { - "advertise_version": "true", - "bulk_commands_display_name": "", - "bulk_commands_master_component_name": "", - "cardinality": "0-1", - "component_category": "MASTER", - "component_name": "HIVE_SERVER_INTERACTIVE", - "custom_commands": ["RESTART_LLAP"], - "decommission_allowed": "false", - "display_name": "HiveServer2 Interactive", - "has_bulk_commands_definition": "false", - "is_client": "false", - "is_master": "true", - "reassign_allowed": "false", - "recovery_enabled": "false", - "service_name": "HIVE", - "stack_name": "HDP", - "stack_version": "2.5", - "hostnames": ["c6401.ambari.apache.org"] - }, - "dependencies": [] - } - ] - } - ], - "changed-configurations": [ - { - u'old_value': u'', - u'type': u'', - u'name': u'' - } - ], - "configurations": { - "capacity-scheduler": { - "properties": { - "capacity-scheduler": "yarn.scheduler.capacity.root.queues=default\n" - "yarn.scheduler.capacity.root.default.user-limit-factor=1\n" - "yarn.scheduler.capacity.root.default.state=RUNNING\n" - "yarn.scheduler.capacity.root.default.maximum-capacity=100\n" - "yarn.scheduler.capacity.root.default.capacity=100\n" - "yarn.scheduler.capacity.root.default.acl_submit_applications=*\n" - "yarn.scheduler.capacity.root.capacity=100\n" - "yarn.scheduler.capacity.root.acl_administer_queue=*\n" - "yarn.scheduler.capacity.root.accessible-node-labels=*\n" - "yarn.scheduler.capacity.node-locality-delay=40\n" - "yarn.scheduler.capacity.maximum-applications=10000\n" - "yarn.scheduler.capacity.maximum-am-resource-percent=1\n" - "yarn.scheduler.capacity.queue-mappings-override.enable=false\n" - } - }, - "hive-interactive-env": - { - 'properties': { - 'enable_hive_interactive': 'true', - 'llap_queue_capacity':'0' - } - }, - "yarn-site": { - "properties": { - "yarn.scheduler.minimum-allocation-mb": "682", - "yarn.nodemanager.resource.memory-mb": "10240", - "yarn.nodemanager.resource.cpu-vcores": "1" - } - }, - "hive-interactive-site": - { - 'properties': { - 'hive.llap.daemon.queue.name':'default' - } - }, - "tez-site": { - "properties": { - "tez.am.resource.memory.mb": "341" - } - }, - "hive-env": - { - 'properties': { - 'hive_user': 'hive' - } - }, - "hive-site": - { - 'properties': { - 'hive.tez.container.size': '341' - } - }, - } - } - - clusterData = { - "cpu": 4, - "mapMemory": 30000, - "amMemory": 20000, - "reduceMemory": 20560, - "containers": 30, - "ramPerContainer": 512, - "referenceNodeManagerHost" : { - "total_mem" : 10240 * 1024 - } - } - - configurations = { - } - self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, self.hosts) - # Check output - self.assertEquals(configurations['hive-interactive-site']['properties']['hive.llap.daemon.queue.name'], - self.expected_hive_interactive_site_llap['hive-interactive-site']['properties']['hive.llap.daemon.queue.name']) - self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name'], - self.expected_hive_interactive_site_llap['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name']) - self.assertEquals(configurations['hive-interactive-env']['properties']['llap_queue_capacity'], - self.expected_llap_queue_capacity_20) - - cap_sched_output_dict = convertToDict(configurations['capacity-scheduler']['properties']['capacity-scheduler']) - cap_sched_expected_dict = convertToDict(self.expected_capacity_scheduler_llap_queue_size_20['properties']['capacity-scheduler']) - self.assertEqual(cap_sched_output_dict, cap_sched_expected_dict) - self.assertEquals(configurations['hive-interactive-env']['property_attributes']['llap_queue_capacity'], - {'maximum': '100', 'minimum': '20', 'visible': 'true'}) - - - - - - # Test 2 : (1). Only default queue exists in capacity-scheduler and capacity-scheduler is passed-in as a dictionary, - # and services['configurations']["capacity-scheduler"]["properties"]["capacity-scheduler"] is set to value "null" - # (2). enable_hive_interactive' is 'On' and 'llap_queue_capacity is set a -ve value (-10). - def test_recommendYARNConfigurations_create_llap_queue_2(self): - - services = { - "services": [{ - "StackServices": { - "service_name": "YARN", - }, - "Versions": { - "stack_version": "2.5" - }, - "components": [ - { - "StackServiceComponents": { - "component_name": "NODEMANAGER", - "hostnames": ["c6401.ambari.apache.org"] - }, - } - ] - }, { - "href": "/api/v1/stacks/HDP/versions/2.5/services/HIVE", - "StackServices": { - "service_name": "HIVE", - "service_version": "1.2.1.2.5", - "stack_name": "HDP", - "stack_version": "2.5" - }, - "components": [ - { - "href": "/api/v1/stacks/HDP/versions/2.5/services/HIVE/components/HIVE_SERVER_INTERACTIVE", - "StackServiceComponents": { - "advertise_version": "true", - "bulk_commands_display_name": "", - "bulk_commands_master_component_name": "", - "cardinality": "0-1", - "component_category": "MASTER", - "component_name": "HIVE_SERVER_INTERACTIVE", - "custom_commands": ["RESTART_LLAP"], - "decommission_allowed": "false", - "display_name": "HiveServer2 Interactive", - "has_bulk_commands_definition": "false", - "is_client": "false", - "is_master": "true", - "reassign_allowed": "false", - "recovery_enabled": "false", - "service_name": "HIVE", - "stack_name": "HDP", - "stack_version": "2.5", - "hostnames": ["c6401.ambari.apache.org"] - }, - "dependencies": [] - } - ] - } - ], - "changed-configurations": [ - { - u'old_value': u'', - u'type': u'', - u'name': u'' - } - ], - "configurations": { - "capacity-scheduler" : { - "properties" : { - "capacity-scheduler" : "null", - "yarn.scheduler.capacity.root.accessible-node-labels" : "*", - "yarn.scheduler.capacity.maximum-am-resource-percent" : "1", - "yarn.scheduler.capacity.root.acl_administer_queue" : "*", - 'yarn.scheduler.capacity.queue-mappings-override.enable' : 'false', - "yarn.scheduler.capacity.root.default.capacity" : "100", - "yarn.scheduler.capacity.root.default.user-limit-factor" : "1", - "yarn.scheduler.capacity.root.queues" : "default", - "yarn.scheduler.capacity.root.capacity" : "100", - "yarn.scheduler.capacity.root.default.acl_submit_applications" : "*", - "yarn.scheduler.capacity.root.default.maximum-capacity" : "100", - "yarn.scheduler.capacity.node-locality-delay" : "40", - "yarn.scheduler.capacity.maximum-applications" : "10000", - "yarn.scheduler.capacity.root.default.state" : "RUNNING" - } - }, - "hive-interactive-env": - { - 'properties': { - 'enable_hive_interactive': 'true', - 'llap_queue_capacity':'-10' - } - }, - "yarn-site": { - "properties": { - "yarn.scheduler.minimum-allocation-mb": "682", - "yarn.nodemanager.resource.memory-mb": "10240", - "yarn.nodemanager.resource.cpu-vcores": "1" - } - }, - "hive-interactive-site": - { - 'properties': { - 'hive.llap.daemon.queue.name':'default' - } - }, - "tez-site": { - "properties": { - "tez.am.resource.memory.mb": "341" - } - }, - "hive-env": - { - 'properties': { - 'hive_user': 'hive' - } - }, - "hive-site": - { - 'properties': { - 'hive.tez.container.size': '341' - } - }, - } - } - - clusterData = { - "cpu": 4, - "mapMemory": 30000, - "amMemory": 20000, - "reduceMemory": 20560, - "containers": 30, - "ramPerContainer": 512, - "referenceNodeManagerHost" : { - "total_mem" : 10240 * 1024 - } - } - - - configurations = { - } - - self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, self.hosts) - # Check output - - self.assertEquals(configurations['hive-interactive-site']['properties']['hive.llap.daemon.queue.name'], - self.expected_hive_interactive_site_llap['hive-interactive-site']['properties']['hive.llap.daemon.queue.name']) - self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name'], - self.expected_hive_interactive_site_llap['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name']) - self.assertEquals(configurations['hive-interactive-env']['properties']['llap_queue_capacity'], - self.expected_llap_queue_capacity_20) - - cap_sched_output_dict = configurations['capacity-scheduler']['properties'] - self.assertTrue(isinstance(cap_sched_output_dict, dict)) - cap_sched_expected_dict = convertToDict(self.expected_capacity_scheduler_llap_queue_size_20['properties']['capacity-scheduler']) - self.assertEqual(cap_sched_output_dict, cap_sched_expected_dict) - self.assertEquals(configurations['hive-interactive-env']['property_attributes']['llap_queue_capacity'], - {'maximum': '100', 'minimum': '20', 'visible': 'true'}) - - - # Test 3 : (1). Only default queue exists in capacity-scheduler and capacity-scheduler is passed-in as a dictionary, - # and services['configurations']["capacity-scheduler"]["properties"]["capacity-scheduler"] is set to value "null" - # (2). enable_hive_interactive' is 'On' and 'llap_queue_capacity is set a value grater than upper bound 100 (=101). - def test_recommendYARNConfigurations_create_llap_queue_3(self): - - services = { - "services": [{ - "StackServices": { - "service_name": "YARN", - }, - "Versions": { - "stack_version": "2.5" - }, - "components": [ - { - "StackServiceComponents": { - "component_name": "NODEMANAGER", - "hostnames": ["c6401.ambari.apache.org"] - }, - } - ] - }, { - "href": "/api/v1/stacks/HDP/versions/2.5/services/HIVE", - "StackServices": { - "service_name": "HIVE", - "service_version": "1.2.1.2.5", - "stack_name": "HDP", - "stack_version": "2.5" - }, - "components": [ - { - "href": "/api/v1/stacks/HDP/versions/2.5/services/HIVE/components/HIVE_SERVER_INTERACTIVE", - "StackServiceComponents": { - "advertise_version": "true", - "bulk_commands_display_name": "", - "bulk_commands_master_component_name": "", - "cardinality": "0-1", - "component_category": "MASTER", - "component_name": "HIVE_SERVER_INTERACTIVE", - "custom_commands": ["RESTART_LLAP"], - "decommission_allowed": "false", - "display_name": "HiveServer2 Interactive", - "has_bulk_commands_definition": "false", - "is_client": "false", - "is_master": "true", - "reassign_allowed": "false", - "recovery_enabled": "false", - "service_name": "HIVE", - "stack_name": "HDP", - "stack_version": "2.5", - "hostnames": ["c6401.ambari.apache.org"] - }, - "dependencies": [] - } - ] - } - ], - "changed-configurations": [ - { - u'old_value': u'', - u'type': u'', - u'name': u'' - } - ], - "configurations": { - "capacity-scheduler" : { - "properties" : { - "capacity-scheduler" : "null", - "yarn.scheduler.capacity.root.accessible-node-labels" : "*", - "yarn.scheduler.capacity.maximum-am-resource-percent" : "1", - "yarn.scheduler.capacity.root.acl_administer_queue" : "*", - 'yarn.scheduler.capacity.queue-mappings-override.enable' : 'false', - "yarn.scheduler.capacity.root.default.capacity" : "100", - "yarn.scheduler.capacity.root.default.user-limit-factor" : "1", - "yarn.scheduler.capacity.root.queues" : "default", - "yarn.scheduler.capacity.root.capacity" : "100", - "yarn.scheduler.capacity.root.default.acl_submit_applications" : "*", - "yarn.scheduler.capacity.root.default.maximum-capacity" : "100", - "yarn.scheduler.capacity.node-locality-delay" : "40", - "yarn.scheduler.capacity.maximum-applications" : "10000", - "yarn.scheduler.capacity.root.default.state" : "RUNNING" - } - }, - "hive-interactive-env": - { - 'properties': { - 'enable_hive_interactive': 'true', - 'llap_queue_capacity':'-101' - } - }, - "yarn-site": { - "properties": { - "yarn.scheduler.minimum-allocation-mb": "682", - "yarn.nodemanager.resource.memory-mb": "10240", - "yarn.nodemanager.resource.cpu-vcores": "1" - } - }, - "hive-interactive-site": - { - 'properties': { - 'hive.llap.daemon.queue.name':'default' - } - }, - "tez-site": { - "properties": { - "tez.am.resource.memory.mb": "341" - } - }, - "hive-env": - { - 'properties': { - 'hive_user': 'hive' - } - }, - "hive-site": - { - 'properties': { - 'hive.tez.container.size': '341' - } - }, - } - } - - clusterData = { - "cpu": 4, - "mapMemory": 30000, - "amMemory": 20000, - "reduceMemory": 20560, - "containers": 30, - "ramPerContainer": 512, - "referenceNodeManagerHost" : { - "total_mem" : 10240 * 1024 - } - } - - - configurations = { - } - self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, self.hosts) - # Check output - - self.assertEquals(configurations['hive-interactive-site']['properties']['hive.llap.daemon.queue.name'], - self.expected_hive_interactive_site_llap['hive-interactive-site']['properties']['hive.llap.daemon.queue.name']) - self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name'], - self.expected_hive_interactive_site_llap['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name']) - self.assertEquals(configurations['hive-interactive-env']['properties']['llap_queue_capacity'], - self.expected_llap_queue_capacity_20) - - cap_sched_output_dict = configurations['capacity-scheduler']['properties'] - self.assertTrue(isinstance(cap_sched_output_dict, dict)) - cap_sched_expected_dict = convertToDict(self.expected_capacity_scheduler_llap_queue_size_20['properties']['capacity-scheduler']) - self.assertEqual(cap_sched_output_dict, cap_sched_expected_dict) - - - - # Test 4: (1). Only default queue exists in capacity-scheduler and 'capacity-scheduler' configs are passed-in as - # single "/n" separated string (2). enable_hive_interactive' is 'On' and 'llap_queue_capacity is 40. - def test_recommendYARNConfigurations_create_llap_queue_4(self): - services = { - "services": [{ - "StackServices": { - "service_name": "YARN", - }, - "Versions": { - "stack_version": "2.5" - }, - "components": [ - { - "StackServiceComponents": { - "component_name": "NODEMANAGER", - "hostnames": ["c6401.ambari.apache.org"] - } - } - ] - }, { - "href": "/api/v1/stacks/HDP/versions/2.5/services/HIVE", - "StackServices": { - "service_name": "HIVE", - "service_version": "1.2.1.2.5", - "stack_name": "HDP", - "stack_version": "2.5" - }, - "components": [ - { - "href": "/api/v1/stacks/HDP/versions/2.5/services/HIVE/components/HIVE_SERVER_INTERACTIVE", - "StackServiceComponents": { - "advertise_version": "true", - "bulk_commands_display_name": "", - "bulk_commands_master_component_name": "", - "cardinality": "0-1", - "component_category": "MASTER", - "component_name": "HIVE_SERVER_INTERACTIVE", - "custom_commands": ["RESTART_LLAP"], - "decommission_allowed": "false", - "display_name": "HiveServer2 Interactive", - "has_bulk_commands_definition": "false", - "is_client": "false", - "is_master": "true", - "reassign_allowed": "false", - "recovery_enabled": "false", - "service_name": "HIVE", - "stack_name": "HDP", - "stack_version": "2.5", - "hostnames": ["c6401.ambari.apache.org"] - }, - "dependencies": [] - } - ] - } - ], - "changed-configurations": [ - { - u'old_value': u'', - u'type': u'', - u'name': u'' - } - ], - "configurations": { - "capacity-scheduler": { - "properties": { - "capacity-scheduler": "yarn.scheduler.capacity.root.queues=default\n" - "yarn.scheduler.capacity.root.default.user-limit-factor=1\n" - "yarn.scheduler.capacity.root.default.state=RUNNING\n" - "yarn.scheduler.capacity.root.default.maximum-capacity=100\n" - "yarn.scheduler.capacity.root.default.capacity=100\n" - "yarn.scheduler.capacity.root.default.acl_submit_applications=*\n" - "yarn.scheduler.capacity.root.capacity=100\n" - "yarn.scheduler.capacity.root.acl_administer_queue=*\n" - "yarn.scheduler.capacity.root.accessible-node-labels=*\n" - "yarn.scheduler.capacity.node-locality-delay=40\n" - "yarn.scheduler.capacity.maximum-applications=10000\n" - "yarn.scheduler.capacity.maximum-am-resource-percent=1\n" - "yarn.scheduler.capacity.queue-mappings-override.enable=false\n" - } - }, - "hive-interactive-env": - { - 'properties': { - 'enable_hive_interactive': 'true', - 'llap_queue_capacity':'40' - } - }, - "yarn-site": { - "properties": { - "yarn.scheduler.minimum-allocation-mb": "682", - "yarn.nodemanager.resource.memory-mb": "2048", - "yarn.nodemanager.resource.cpu-vcores": "1" - } - }, - "hive-interactive-site": - { - 'properties': { - 'hive.llap.daemon.queue.name':'default', - } - }, - "tez-site": { - "properties": { - "tez.am.resource.memory.mb": "341" - } - }, - "hive-env": - { - 'properties': { - 'hive_user': 'hive' - } - }, - "hive-site": - { - 'properties': { - 'hive.tez.container.size': '341' - } - }, - } - } - - clusterData = { - "cpu": 4, - "mapMemory": 30000, - "amMemory": 20000, - "reduceMemory": 20560, - "containers": 30, - "ramPerContainer": 512, - "referenceNodeManagerHost" : { - "total_mem" : 10240 * 1024 - } - } - - configurations = { - } - - self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, self.hosts) - - # Check output - self.assertEquals(configurations['hive-interactive-site']['properties']['hive.llap.daemon.queue.name'], - self.expected_hive_interactive_site_llap['hive-interactive-site']['properties']['hive.llap.daemon.queue.name']) - self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name'], - self.expected_hive_interactive_site_llap['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name']) - self.assertTrue('llap_queue_capacity' not in configurations['hive-interactive-env']['properties']) - - cap_sched_output_dict = convertToDict(configurations['capacity-scheduler']['properties']['capacity-scheduler']) - cap_sched_expected_dict = convertToDict(self.expected_capacity_scheduler_llap_queue_size_40['properties']['capacity-scheduler']) - self.assertEqual(cap_sched_output_dict, cap_sched_expected_dict) - self.assertEquals(configurations['hive-interactive-env']['property_attributes']['llap_queue_capacity'], - {'maximum': '100', 'minimum': '100', 'visible': 'true'}) - - - - # Test 5: (1). Only default queue exists in capacity-scheduler and capacity-scheduler is passed-in as a dictionary - # and services['configurations']["capacity-scheduler"]["properties"]["capacity-scheduler"] is null - # (2). enable_hive_interactive' is 'On' and 'llap_queue_capacity is 40. - def test_recommendYARNConfigurations_create_llap_queue_5(self): - services = { - "services": [{ - "StackServices": { - "service_name": "YARN", - }, - "Versions": { - "stack_version": "2.5" - }, - "components": [ - { - "StackServiceComponents": { - "component_name": "NODEMANAGER", - "hostnames": ["c6401.ambari.apache.org"] - } - } - ] - }, { - "href": "/api/v1/stacks/HDP/versions/2.5/services/HIVE", - "StackServices": { - "service_name": "HIVE", - "service_version": "1.2.1.2.5", - "stack_name": "HDP", - "stack_version": "2.5" - }, - "components": [ - { - "href": "/api/v1/stacks/HDP/versions/2.5/services/HIVE/components/HIVE_SERVER_INTERACTIVE", - "StackServiceComponents": { - "advertise_version": "true", - "bulk_commands_display_name": "", - "bulk_commands_master_component_name": "", - "cardinality": "0-1", - "component_category": "MASTER", - "component_name": "HIVE_SERVER_INTERACTIVE", - "custom_commands": ["RESTART_LLAP"], - "decommission_allowed": "false", - "display_name": "HiveServer2 Interactive", - "has_bulk_commands_definition": "false", - "is_client": "false", - "is_master": "true", - "reassign_allowed": "false", - "recovery_enabled": "false", - "service_name": "HIVE", - "stack_name": "HDP", - "stack_version": "2.5", - "hostnames": ["c6401.ambari.apache.org"] - }, - "dependencies": [] - } - ] - } - ], - "changed-configurations": [ - { - u'old_value': u'', - u'type': u'', - u'name': u'' - } - ], - "configurations": { - "capacity-scheduler" : { - "properties" : { - "capacity-scheduler" : None, - "yarn.scheduler.capacity.root.accessible-node-labels" : "*", - "yarn.scheduler.capacity.maximum-am-resource-percent" : "1", - "yarn.scheduler.capacity.root.acl_administer_queue" : "*", - 'yarn.scheduler.capacity.queue-mappings-override.enable' : 'false', - "yarn.scheduler.capacity.root.default.capacity" : "100", - "yarn.scheduler.capacity.root.default.user-limit-factor" : "1", - "yarn.scheduler.capacity.root.queues" : "default", - "yarn.scheduler.capacity.root.capacity" : "100", - "yarn.scheduler.capacity.root.default.acl_submit_applications" : "*", - "yarn.scheduler.capacity.root.default.maximum-capacity" : "100", - "yarn.scheduler.capacity.node-locality-delay" : "40", - "yarn.scheduler.capacity.maximum-applications" : "10000", - "yarn.scheduler.capacity.root.default.state" : "RUNNING" - } - }, - "hive-interactive-env": - { - 'properties': { - 'enable_hive_interactive': 'true', - 'llap_queue_capacity':'40' - } - }, - "yarn-site": { - "properties": { - "yarn.scheduler.minimum-allocation-mb": "682", - "yarn.nodemanager.resource.memory-mb": "8192", - "yarn.nodemanager.resource.cpu-vcores": "1" - } - }, - "hive-interactive-site": - { - 'properties': { - 'hive.llap.daemon.queue.name':'default', - } - }, - "tez-site": { - "properties": { - "tez.am.resource.memory.mb": "341" - } - }, - "hive-env": - { - 'properties': { - 'hive_user': 'hive' - } - }, - "hive-site": - { - 'properties': { - 'hive.tez.container.size': '341' - } - }, - } - } - - - clusterData = { - "cpu": 4, - "mapMemory": 30000, - "amMemory": 20000, - "reduceMemory": 20560, - "containers": 30, - "ramPerContainer": 512, - "referenceNodeManagerHost" : { - "total_mem" : 10240 * 1024 - } - } - - configurations = { - } - - self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, self.hosts) - # Check output - self.assertEquals(configurations['hive-interactive-site']['properties']['hive.llap.daemon.queue.name'], - self.expected_hive_interactive_site_llap['hive-interactive-site']['properties']['hive.llap.daemon.queue.name']) - self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name'], - self.expected_hive_interactive_site_llap['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name']) - self.assertTrue('llap_queue_capacity' not in configurations['hive-interactive-env']['properties']) - - cap_sched_output_dict = configurations['capacity-scheduler']['properties'] - self.assertTrue(isinstance(cap_sched_output_dict, dict)) - cap_sched_expected_dict = convertToDict(self.expected_capacity_scheduler_llap_queue_size_40['properties']['capacity-scheduler']) - self.assertEqual(cap_sched_output_dict, cap_sched_expected_dict) - self.assertEquals(configurations['hive-interactive-env']['property_attributes']['llap_queue_capacity'], - {'maximum': '100', 'minimum': '25', 'visible': 'true'}) - - - # Test 6: (1). 'llap' (0%) and 'default' (100%) queues exists at leaf level in capacity-scheduler and 'capacity-scheduler' - # configs are passed-in as single "/n" separated string - # (2). llap is state = STOPPED, (3). llap_queue_capacity = 0, and (4). enable_hive_interactive' is 'ON'. - # Expected : llap queue state = RUNNING, llap_queue_capacity = 20 - def test_recommendYARNConfigurations_update_llap_queue_1(self): - services = { - "services": [{ - "StackServices": { - "service_name": "YARN", - }, - "Versions": { - "stack_version": "2.5" - }, - "components": [ - { - "StackServiceComponents": { - "component_name": "NODEMANAGER", - "hostnames": ["c6401.ambari.apache.org"] - } - } - ] - }, { - "href": "/api/v1/stacks/HDP/versions/2.5/services/HIVE", - "StackServices": { - "service_name": "HIVE", - "service_version": "1.2.1.2.5", - "stack_name": "HDP", - "stack_version": "2.5" - }, - "components": [ - { - "href": "/api/v1/stacks/HDP/versions/2.5/services/HIVE/components/HIVE_SERVER_INTERACTIVE", - "StackServiceComponents": { - "advertise_version": "true", - "bulk_commands_display_name": "", - "bulk_commands_master_component_name": "", - "cardinality": "0-1", - "component_category": "MASTER", - "component_name": "HIVE_SERVER_INTERACTIVE", - "custom_commands": ["RESTART_LLAP"], - "decommission_allowed": "false", - "display_name": "HiveServer2 Interactive", - "has_bulk_commands_definition": "false", - "is_client": "false", - "is_master": "true", - "reassign_allowed": "false", - "recovery_enabled": "false", - "service_name": "HIVE", - "stack_name": "HDP", - "stack_version": "2.5", - "hostnames": ["c6401.ambari.apache.org"] - }, - "dependencies": [] - } - ] - } - ], - "changed-configurations": [ - { - u'old_value': u'off', - u'type': u'hive-interactive-env', - u'name': u'enable_hive_interactive' - } - ], - "configurations": { - "capacity-scheduler": { - "properties": { - "capacity-scheduler": "yarn.scheduler.capacity.root.accessible-node-labels=*\n" - "yarn.scheduler.capacity.root.capacity=100\n" - "yarn.scheduler.capacity.root.queues=default,llap\n" - "yarn.scheduler.capacity.maximum-applications=10000\n" - "yarn.scheduler.capacity.root.default.user-limit-factor=1\n" - "yarn.scheduler.capacity.root.default.state=RUNNING\n" - "yarn.scheduler.capacity.maximum-am-resource-percent=1\n" - "yarn.scheduler.capacity.root.default.acl_submit_applications=*\n" - "yarn.scheduler.capacity.root.default.capacity=100\n" - "yarn.scheduler.capacity.root.acl_administer_queue=*\n" - "yarn.scheduler.capacity.node-locality-delay=40\n" - "yarn.scheduler.capacity.queue-mappings-override.enable=false\n" - "yarn.scheduler.capacity.root.llap.user-limit-factor=1\n" - "yarn.scheduler.capacity.root.llap.state=STOPPED\n" - "yarn.scheduler.capacity.root.llap.ordering-policy=fifo\n" - "yarn.scheduler.capacity.root.llap.minimum-user-limit-percent=100\n" - "yarn.scheduler.capacity.root.llap.maximum-capacity=0\n" - "yarn.scheduler.capacity.root.default.maximum-capacity=100\n" - "yarn.scheduler.capacity.root.llap.capacity=0\n" - "yarn.scheduler.capacity.root.llap.acl_submit_applications=hive\n" - "yarn.scheduler.capacity.root.llap.acl_administer_queue=hive\n" - "yarn.scheduler.capacity.root.llap.maximum-am-resource-percent=1\n" - } - }, - "hive-interactive-env": - { - 'properties': { - 'enable_hive_interactive': 'true', - 'llap_queue_capacity':'0' - } - }, - "yarn-site": { - "properties": { - "yarn.scheduler.minimum-allocation-mb": "341", - "yarn.nodemanager.resource.memory-mb": "20000", - "yarn.nodemanager.resource.cpu-vcores": '1' - } - }, - "hive-interactive-site": - { - 'properties': { - 'hive.llap.daemon.queue.name':'llap', - 'hive.server2.tez.sessions.per.default.queue' : '1' - } - }, - "tez-site": { - "properties": { - "tez.am.resource.memory.mb": "341" - } - }, - "hive-env": - { - 'properties': { - 'hive_user': 'hive' - } - }, - "hive-site": - { - 'properties': { - 'hive.tez.container.size': '341' - } - }, - } - } - - - clusterData = { - "cpu": 4, - "mapMemory": 30000, - "amMemory": 20000, - "reduceMemory": 20560, - "containers": 30, - "ramPerContainer": 512, - "referenceNodeManagerHost" : { - "total_mem" : 10240 * 1024 - } - } - - - configurations = { - } - - self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, self.hosts) - - # Check output - self.assertEquals(configurations['hive-interactive-site']['properties']['hive.llap.daemon.queue.name'], - self.expected_hive_interactive_site_llap['hive-interactive-site']['properties']['hive.llap.daemon.queue.name']) - self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name'], - self.expected_hive_interactive_site_llap['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name']) - self.assertEquals(configurations['hive-interactive-env']['properties']['llap_queue_capacity'], - self.expected_llap_queue_capacity_20) - - cap_sched_output_dict = convertToDict(configurations['capacity-scheduler']['properties']['capacity-scheduler']) - cap_sched_expected_dict = convertToDict(self.expected_capacity_scheduler_llap_queue_size_20['properties']['capacity-scheduler']) - self.assertEqual(cap_sched_output_dict, cap_sched_expected_dict) - self.assertEquals(configurations['hive-interactive-env']['property_attributes']['llap_queue_capacity'], - {'maximum': '100', 'minimum': '20', 'visible': 'false'}) - - - - # Test 7: (1). 'llap' (20%) and 'default' (80%) queues exists at leaf level in capacity-scheduler and 'capacity-scheduler' - # configs are passed-in as single "/n" separated string - # (2). llap is state = STOPPED, (3). llap_queue_capacity = 40, and (4). enable_hive_interactive' is 'ON'. - # Expected : llap state goes RUNNING. - def test_recommendYARNConfigurations_update_llap_queue_2(self): - services = { - "services": [{ - "StackServices": { - "service_name": "YARN", - }, - "Versions": { - "stack_version": "2.5" - }, - "components": [ - { - "StackServiceComponents": { - "component_name": "NODEMANAGER", - "hostnames": ["c6401.ambari.apache.org"] - } - } - ] - }, { - "href": "/api/v1/stacks/HDP/versions/2.5/services/HIVE", - "StackServices": { - "service_name": "HIVE", - "service_version": "1.2.1.2.5", - "stack_name": "HDP", - "stack_version": "2.5" - }, - "components": [ - { - "href": "/api/v1/stacks/HDP/versions/2.5/services/HIVE/components/HIVE_SERVER_INTERACTIVE", - "StackServiceComponents": { - "advertise_version": "true", - "bulk_commands_display_name": "", - "bulk_commands_master_component_name": "", - "cardinality": "0-1", - "component_category": "MASTER", - "component_name": "HIVE_SERVER_INTERACTIVE", - "custom_commands": ["RESTART_LLAP"], - "decommission_allowed": "false", - "display_name": "HiveServer2 Interactive", - "has_bulk_commands_definition": "false", - "is_client": "false", - "is_master": "true", - "reassign_allowed": "false", - "recovery_enabled": "false", - "service_name": "HIVE", - "stack_name": "HDP", - "stack_version": "2.5", - "hostnames": ["c6401.ambari.apache.org"] - }, - "dependencies": [] - } - ] - } - ], - "changed-configurations": [ - { - u'old_value': u'', - u'type': u'', - u'name': u'' - } - ], - "configurations": { - "capacity-scheduler": { - "properties": { - "capacity-scheduler": "yarn.scheduler.capacity.root.accessible-node-labels=*\n" - "yarn.scheduler.capacity.root.capacity=100\n" - "yarn.scheduler.capacity.root.queues=default,llap\n" - "yarn.scheduler.capacity.maximum-applications=10000\n" - "yarn.scheduler.capacity.root.default.user-limit-factor=1\n" - "yarn.scheduler.capacity.root.default.state=RUNNING\n" - "yarn.scheduler.capacity.maximum-am-resource-percent=1\n" - "yarn.scheduler.capacity.root.default.acl_submit_applications=*\n" - "yarn.scheduler.capacity.root.default.capacity=80\n" - "yarn.scheduler.capacity.root.acl_administer_queue=*\n" - "yarn.scheduler.capacity.node-locality-delay=40\n" - "yarn.scheduler.capacity.queue-mappings-override.enable=false\n" - "yarn.scheduler.capacity.root.llap.user-limit-factor=1\n" - "yarn.scheduler.capacity.root.llap.state=RUNNING\n" - "yarn.scheduler.capacity.root.llap.ordering-policy=fifo\n" - "yarn.scheduler.capacity.root.llap.minimum-user-limit-percent=100\n" - "yarn.scheduler.capacity.root.llap.maximum-capacity=20\n" - "yarn.scheduler.capacity.root.default.maximum-capacity=80\n" - "yarn.scheduler.capacity.root.llap.capacity=20\n" - "yarn.scheduler.capacity.root.llap.acl_submit_applications=hive\n" - "yarn.scheduler.capacity.root.llap.acl_administer_queue=hive\n" - "yarn.scheduler.capacity.root.llap.maximum-am-resource-percent=1\n" - } - }, - "hive-interactive-env": - { - 'properties': { - 'enable_hive_interactive': 'true', - 'llap_queue_capacity':'40' - } - }, - "hive-interactive-site": - { - 'properties': { - 'hive.llap.daemon.queue.name':'llap' - } - }, - "tez-site": { - "properties": { - "tez.am.resource.memory.mb": "341" - } - }, - "yarn-site": { - "properties": { - "yarn.scheduler.minimum-allocation-mb": "341", - "yarn.nodemanager.resource.memory-mb": "20000", - "yarn.nodemanager.resource.cpu-vcores": '1' - } - }, - "hive-env": - { - 'properties': { - 'hive_user': 'hive' - } - }, - "hive-site": - { - 'properties': { - 'hive.tez.container.size': '341' - } - }, - } - } - - - clusterData = { - "cpu": 4, - "mapMemory": 30000, - "amMemory": 20000, - "reduceMemory": 20560, - "containers": 30, - "ramPerContainer": 512, - "referenceNodeManagerHost" : { - "total_mem" : 10240 * 1024 - } - } - - configurations = { - } - - - self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, self.hosts) - - # Check output - self.assertEquals(configurations['hive-interactive-site']['properties']['hive.llap.daemon.queue.name'], - self.expected_hive_interactive_site_llap['hive-interactive-site']['properties']['hive.llap.daemon.queue.name']) - self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name'], - self.expected_hive_interactive_site_llap['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name']) - self.assertTrue('llap_queue_capacity' not in configurations['hive-interactive-env']['properties']) - - cap_sched_output_dict = convertToDict(configurations['capacity-scheduler']['properties']['capacity-scheduler']) - cap_sched_expected_dict = convertToDict(self.expected_capacity_scheduler_llap_queue_size_40['properties']['capacity-scheduler']) - self.assertEqual(cap_sched_output_dict, cap_sched_expected_dict) - self.assertEquals(configurations['hive-interactive-env']['property_attributes']['llap_queue_capacity'], - {'maximum': '100', 'minimum': '20', 'visible': 'true'}) - - - - - # Test 8: (1). 'llap' (20%) and 'default' (60%) queues exists at leaf level in capacity-scheduler and 'capacity-scheduler' - # configs are passed-in as single "/n" separated string - # (2). llap is state = RUNNING, (3). llap_queue_capacity = 40, and (4). enable_hive_interactive' is 'ON'. - # Expected : Existing llap queue's capacity in capacity-scheduler set to 40. - def test_recommendYARNConfigurations_update_llap_queue_3(self): - services = { - "services": [{ - "StackServices": { - "service_name": "YARN", - }, - "Versions": { - "stack_version": "2.5" - }, - "components": [ - { - "StackServiceComponents": { - "component_name": "NODEMANAGER", - "hostnames": ["c6401.ambari.apache.org"] - } - } - ] - }, { - "href": "/api/v1/stacks/HDP/versions/2.5/services/HIVE", - "StackServices": { - "service_name": "HIVE", - "service_version": "1.2.1.2.5", - "stack_name": "HDP", - "stack_version": "2.5" - }, - "components": [ - { - "href": "/api/v1/stacks/HDP/versions/2.5/services/HIVE/components/HIVE_SERVER_INTERACTIVE", - "StackServiceComponents": { - "advertise_version": "true", - "bulk_commands_display_name": "", - "bulk_commands_master_component_name": "", - "cardinality": "0-1", - "component_category": "MASTER", - "component_name": "HIVE_SERVER_INTERACTIVE", - "custom_commands": ["RESTART_LLAP"], - "decommission_allowed": "false", - "display_name": "HiveServer2 Interactive", - "has_bulk_commands_definition": "false", - "is_client": "false", - "is_master": "true", - "reassign_allowed": "false", - "recovery_enabled": "false", - "service_name": "HIVE", - "stack_name": "HDP", - "stack_version": "2.5", - "hostnames": ["c6401.ambari.apache.org"] - }, - "dependencies": [] - } - ] - } - ], - "changed-configurations": [ - { - u'old_value': u'', - u'type': u'', - u'name': u'' - } - ], - "configurations": { - "capacity-scheduler": { - "properties": { - "capacity-scheduler": "yarn.scheduler.capacity.root.accessible-node-labels=*\n" - "yarn.scheduler.capacity.root.capacity=100\n" - "yarn.scheduler.capacity.root.queues=default,llap\n" - "yarn.scheduler.capacity.maximum-applications=10000\n" - "yarn.scheduler.capacity.root.default.user-limit-factor=1\n" - "yarn.scheduler.capacity.root.default.state=RUNNING\n" - "yarn.scheduler.capacity.maximum-am-resource-percent=1\n" - "yarn.scheduler.capacity.root.default.acl_submit_applications=*\n" - "yarn.scheduler.capacity.root.default.capacity=80\n" - "yarn.scheduler.capacity.root.acl_administer_queue=*\n" - "yarn.scheduler.capacity.node-locality-delay=40\n" - "yarn.scheduler.capacity.queue-mappings-override.enable=false\n" - "yarn.scheduler.capacity.root.llap.user-limit-factor=1\n" - "yarn.scheduler.capacity.root.llap.state=RUNNING\n" - "yarn.scheduler.capacity.root.llap.ordering-policy=fifo\n" - "yarn.scheduler.capacity.root.llap.minimum-user-limit-percent=100\n" - "yarn.scheduler.capacity.root.llap.maximum-capacity=20\n" - "yarn.scheduler.capacity.root.default.maximum-capacity=80\n" - "yarn.scheduler.capacity.root.llap.capacity=20\n" - "yarn.scheduler.capacity.root.llap.acl_submit_applications=hive\n" - "yarn.scheduler.capacity.root.llap.acl_administer_queue=hive\n" - "yarn.scheduler.capacity.root.llap.maximum-am-resource-percent=1\n" - } - }, - "hive-interactive-env": - { - 'properties': { - 'enable_hive_interactive': 'true', - 'llap_queue_capacity':'40' - } - }, - "hive-interactive-site": - { - 'properties': { - 'hive.llap.daemon.queue.name':'llap' - } - }, - "tez-site": { - "properties": { - "tez.am.resource.memory.mb": "341" - } - }, - "yarn-site": { - "properties": { - "yarn.scheduler.minimum-allocation-mb": "341", - "yarn.nodemanager.resource.memory-mb": "20000", - "yarn.nodemanager.resource.cpu-vcores": '1' - } - }, - "hive-env": - { - 'properties': { - 'hive_user': 'hive' - } - }, - "hive-site": - { - 'properties': { - 'hive.tez.container.size': '341' - } - }, - } - } - - - clusterData = { - "cpu": 4, - "mapMemory": 30000, - "amMemory": 20000, - "reduceMemory": 20560, - "containers": 30, - "ramPerContainer": 512, - "referenceNodeManagerHost" : { - "total_mem" : 10240 * 1024 - } - } - - - configurations = { - } - - self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, self.hosts) - - # Check output - self.assertEquals(configurations['hive-interactive-site']['properties']['hive.llap.daemon.queue.name'], - self.expected_hive_interactive_site_llap['hive-interactive-site']['properties']['hive.llap.daemon.queue.name']) - self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name'], - self.expected_hive_interactive_site_llap['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name']) - self.assertTrue('llap_queue_capacity' not in configurations['hive-interactive-env']['properties']) - - cap_sched_output_dict = convertToDict(configurations['capacity-scheduler']['properties']['capacity-scheduler']) - cap_sched_expected_dict = convertToDict(self.expected_capacity_scheduler_llap_queue_size_40['properties']['capacity-scheduler']) - self.assertEqual(cap_sched_output_dict, cap_sched_expected_dict) - self.assertEquals(configurations['hive-interactive-env']['property_attributes']['llap_queue_capacity'], - {'maximum': '100', 'minimum': '20', 'visible': 'true'}) - - - - - # Test 9: (1). Only default queue exists in capacity-scheduler and 'capacity-scheduler' configs are passed-in as - # single "/n" separated string (2). enable_hive_interactive' is 'Off' and - # 'llap_queue_capacity is 0. - # Expected : No changes - def test_recommendYARNConfigurations_no_update_to_llap_queue_1(self): - services = { - "services": [{ - "StackServices": { - "service_name": "YARN", - }, - "Versions": { - "stack_version": "2.5" - }, - "components": [ - { - "StackServiceComponents": { - "component_name": "NODEMANAGER", - "hostnames": ["c6401.ambari.apache.org"] - } - } - ] - }, { - "href": "/api/v1/stacks/HDP/versions/2.5/services/HIVE", - "StackServices": { - "service_name": "HIVE", - "service_version": "1.2.1.2.5", - "stack_name": "HDP", - "stack_version": "2.5" - }, - "components": [ - { - "href": "/api/v1/stacks/HDP/versions/2.5/services/HIVE/components/HIVE_SERVER_INTERACTIVE", - "StackServiceComponents": { - "advertise_version": "true", - "bulk_commands_display_name": "", - "bulk_commands_master_component_name": "", - "cardinality": "0-1", - "component_category": "MASTER", - "component_name": "HIVE_SERVER_INTERACTIVE", - "custom_commands": ["RESTART_LLAP"], - "decommission_allowed": "false", - "display_name": "HiveServer2 Interactive", - "has_bulk_commands_definition": "false", - "is_client": "false", - "is_master": "true", - "reassign_allowed": "false", - "recovery_enabled": "false", - "service_name": "HIVE", - "stack_name": "HDP", - "stack_version": "2.5", - "hostnames": ["c6401.ambari.apache.org"] - }, - "dependencies": [] - } - ] - } - ], - "changed-configurations": [ - { - u'old_value': u'', - u'type': u'', - u'name': u'' - } - ], - "configurations": { - "capacity-scheduler": { - "properties": { - "capacity-scheduler": "yarn.scheduler.capacity.root.queues=default\n" - "yarn.scheduler.capacity.root.default.user-limit-factor=1\n" - "yarn.scheduler.capacity.root.default.state=RUNNING\n" - "yarn.scheduler.capacity.root.default.maximum-capacity=100\n" - "yarn.scheduler.capacity.root.default.capacity=100\n" - "yarn.scheduler.capacity.root.default.acl_submit_applications=*\n" - "yarn.scheduler.capacity.root.capacity=100\n" - "yarn.scheduler.capacity.root.acl_administer_queue=*\n" - "yarn.scheduler.capacity.root.accessible-node-labels=*\n" - "yarn.scheduler.capacity.node-locality-delay=40\n" - "yarn.scheduler.capacity.maximum-applications=10000\n" - "yarn.scheduler.capacity.maximum-am-resource-percent=1\n" - "yarn.scheduler.capacity.queue-mappings-override.enable=false\n" - } - }, - "hive-interactive-env": - { - 'properties': { - 'enable_hive_interactive': 'false', - 'llap_queue_capacity':'0' - } - }, - "yarn-site": { - "properties": { - "yarn.scheduler.minimum-allocation-mb": "682", - "yarn.nodemanager.resource.memory-mb": "2048" - } - }, - "tez-interactive-site": { - "properties": { - "tez.am.resource.memory.mb": "341" - } - }, - "hive-env": - { - 'properties': { - 'hive_user': 'hive' - } - } - } - } - - clusterData = { - "cpu": 4, - "mapMemory": 30000, - "amMemory": 20000, - "reduceMemory": 20560, - "containers": 30, - "ramPerContainer": 512, - "referenceNodeManagerHost" : { - "total_mem" : 10240 * 1024 - } - } - - - configurations = { - } - - self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, self.hosts) - - # Check output - self.assertTrue('hive.llap.daemon.queue.name' not in configurations['hive-interactive-site']['properties']) - self.assertTrue('property_attributes' not in configurations['hive-interactive-site']) - self.assertTrue('hive-interactive-env' not in configurations) - self.assertEquals(configurations['capacity-scheduler']['properties'],self.expected_capacity_scheduler_empty['properties']) - - - # Test 10: (1). 'default' and 'llap' (State : RUNNING) queue exists at root level in capacity-scheduler and - # 'capacity-scheduler' configs are passed-in as single "/n" separated string , and - # (2). enable_hive_interactive' is 'off'. - # Expected : 'default' queue set to Size 100, 'llap' queue state set to STOPPED and sized to 0. - def test_recommendYARNConfigurations_llap_queue_set_to_stopped_1(self): - services = { - "services": [{ - "StackServices": { - "service_name": "YARN", - }, - "Versions": { - "stack_version": "2.5" - }, - "components": [ - { - "StackServiceComponents": { - "component_name": "NODEMANAGER", - "hostnames": ["c6401.ambari.apache.org"] - } - } - ] - }, { - "href": "/api/v1/stacks/HDP/versions/2.5/services/HIVE", - "StackServices": { - "service_name": "HIVE", - "service_version": "1.2.1.2.5", - "stack_name": "HDP", - "stack_version": "2.5" - }, - "components": [ - { - "href": "/api/v1/stacks/HDP/versions/2.5/services/HIVE/components/HIVE_SERVER_INTERACTIVE", - "StackServiceComponents": { - "advertise_version": "true", - "bulk_commands_display_name": "", - "bulk_commands_master_component_name": "", - "cardinality": "0-1", - "component_category": "MASTER", - "component_name": "HIVE_SERVER_INTERACTIVE", - "custom_commands": ["RESTART_LLAP"], - "decommission_allowed": "false", - "display_name": "HiveServer2 Interactive", - "has_bulk_commands_definition": "false", - "is_client": "false", - "is_master": "true", - "reassign_allowed": "false", - "recovery_enabled": "false", - "service_name": "HIVE", - "stack_name": "HDP", - "stack_version": "2.5", - "hostnames": ["c6401.ambari.apache.org"] - }, - "dependencies": [] - } - ] - } - ], - "changed-configurations": [ - { - u'old_value': u'', - u'type': u'', - u'name': u'' - } - ], - "configurations": { - "capacity-scheduler": { - "properties": { - "capacity-scheduler": "yarn.scheduler.capacity.root.accessible-node-labels=*\n" - "yarn.scheduler.capacity.root.capacity=100\n" - "yarn.scheduler.capacity.root.queues=default,llap\n" - "yarn.scheduler.capacity.maximum-applications=10000\n" - "yarn.scheduler.capacity.root.default.user-limit-factor=1\n" - "yarn.scheduler.capacity.root.default.state=RUNNING\n" - "yarn.scheduler.capacity.maximum-am-resource-percent=1\n" - "yarn.scheduler.capacity.root.default.acl_submit_applications=*\n" - "yarn.scheduler.capacity.root.default.capacity=80\n" - "yarn.scheduler.capacity.root.acl_administer_queue=*\n" - "yarn.scheduler.capacity.node-locality-delay=40\n" - "yarn.scheduler.capacity.queue-mappings-override.enable=false\n" - "yarn.scheduler.capacity.root.llap.user-limit-factor=1\n" - "yarn.scheduler.capacity.root.llap.state=RUNNING\n" - "yarn.scheduler.capacity.root.llap.ordering-policy=fifo\n" - "yarn.scheduler.capacity.root.llap.minimum-user-limit-percent=100\n" - "yarn.scheduler.capacity.root.llap.maximum-capacity=20\n" - "yarn.scheduler.capacity.root.default.maximum-capacity=100\n" - "yarn.scheduler.capacity.root.llap.capacity=20\n" - "yarn.scheduler.capacity.root.llap.acl_submit_applications=hive\n" - "yarn.scheduler.capacity.root.llap.acl_administer_queue=hive\n" - "yarn.scheduler.capacity.root.llap.maximum-am-resource-percent=1\n" - } - }, - "hive-interactive-env": - { - 'properties': { - 'enable_hive_interactive': 'false' - } - }, - "hive-interactive-site": - { - 'properties': { - 'hive.llap.daemon.queue.name':'default' - } - }, - "yarn-site": { - "properties": { - "yarn.scheduler.minimum-allocation-mb": "682", - "yarn.nodemanager.resource.memory-mb": "2048" - }, - "tez-interactive-site": { - "properties": { - "tez.am.resource.memory.mb": "341" - } - }, - }, - "hive-env": - { - 'properties': { - 'hive_user': 'hive' - } - } - } - } - - clusterData = { - "cpu": 4, - "mapMemory": 30000, - "amMemory": 20000, - "reduceMemory": 20560, - "containers": 30, - "ramPerContainer": 512, - "referenceNodeManagerHost" : { - "total_mem" : 10240 * 1024 - } - } - - configurations = { - } - - self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, self.hosts) - - # Check output - self.assertEquals(configurations['hive-interactive-site']['properties']['hive.llap.daemon.queue.name'], - self.expected_hive_interactive_site_default['hive-interactive-site']['properties']['hive.llap.daemon.queue.name']) - self.assertFalse('property_attributes' in configurations['hive-interactive-site']) - self.assertFalse('hive-interactive-env' in configurations) - - cap_sched_output_dict = convertToDict(configurations['capacity-scheduler']['properties']['capacity-scheduler']) - cap_sched_expected_dict = convertToDict(self.expected_capacity_scheduler_llap_Stopped_size_0['properties']['capacity-scheduler']) - self.assertEqual(cap_sched_output_dict, cap_sched_expected_dict) - - - - - # Test 11: (1). More than 2 queues at leaf level exists in capacity-scheduler (no queue is named 'llap') and - # 'capacity-scheduler' configs are passed-in as single "/n" separated string - # (2). enable_hive_interactive' is 'off'. - # Expected : No changes. - def test_recommendYARNConfigurations_no_update_to_llap_queue_2(self): - services= { - "services": [{ - "StackServices": { - "service_name": "YARN", - }, - "Versions": { - "stack_version": "2.5" - }, - "components": [ - { - "StackServiceComponents": { - "component_name": "NODEMANAGER", - "hostnames": ["c6401.ambari.apache.org"] - } - } - ] - }, { - "href": "/api/v1/stacks/HDP/versions/2.5/services/HIVE", - "StackServices": { - "service_name": "HIVE", - "service_version": "1.2.1.2.5", - "stack_name": "HDP", - "stack_version": "2.5" - }, - "components": [ - { - "href": "/api/v1/stacks/HDP/versions/2.5/services/HIVE/components/HIVE_SERVER_INTERACTIVE", - "StackServiceComponents": { - "advertise_version": "true", - "bulk_commands_display_name": "", - "bulk_commands_master_component_name": "", - "cardinality": "0-1", - "component_category": "MASTER", - "component_name": "HIVE_SERVER_INTERACTIVE", - "custom_commands": ["RESTART_LLAP"], - "decommission_allowed": "false", - "display_name": "HiveServer2 Interactive", - "has_bulk_commands_definition": "false", - "is_client": "false", - "is_master": "true", - "reassign_allowed": "false", - "recovery_enabled": "false", - "service_name": "HIVE", - "stack_name": "HDP", - "stack_version": "2.5", - "hostnames": ["c6401.ambari.apache.org"] - }, - "dependencies": [] - } - ] - } - ], - "changed-configurations": [ - { - u'old_value': u'', - u'type': u'', - u'name': u'' - } - ], - "configurations": { - "capacity-scheduler": { - "properties": { - "capacity-scheduler": "yarn.scheduler.capacity.maximum-am-resource-percent=0.2\n" - "yarn.scheduler.capacity.maximum-applications=10000\n" - "yarn.scheduler.capacity.node-locality-delay=40\n" - "yarn.scheduler.capacity.queue-mappings-override.enable=false\n" - "yarn.scheduler.capacity.resource-calculator=org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator\n" - "yarn.scheduler.capacity.root.accessible-node-labels=*\n" - "yarn.scheduler.capacity.root.acl_administer_queue=*\n" - "yarn.scheduler.capacity.root.capacity=100\n" - "yarn.scheduler.capacity.root.default.a.a1.acl_administer_queue=*\n" - "yarn.scheduler.capacity.root.default.a.a1.acl_submit_applications=*\n" - "yarn.scheduler.capacity.root.default.a.a1.capacity=75\n" - "yarn.scheduler.capacity.root.default.a.a1.maximum-capacity=100\n" - "yarn.scheduler.capacity.root.default.a.a1.minimum-user-limit-percent=100\n" - "yarn.scheduler.capacity.root.default.a.a1.ordering-policy=fifo\n" - "yarn.scheduler.capacity.root.default.a.a1.state=RUNNING\n" - "yarn.scheduler.capacity.root.default.a.a1.user-limit-factor=1\n" - "yarn.scheduler.capacity.root.default.a.a2.acl_administer_queue=*\n" - "yarn.scheduler.capacity.root.default.a.a2.acl_submit_applications=*\n" - "yarn.scheduler.capacity.root.default.a.a2.capacity=25\n" - "yarn.scheduler.capacity.root.default.a.a2.maximum-capacity=25\n" - "yarn.scheduler.capacity.root.default.a.a2.minimum-user-limit-percent=100\n" - "yarn.scheduler.capacity.root.default.a.a2.ordering-policy=fifo\n" - "yarn.scheduler.capacity.root.default.a.a2.state=RUNNING\n" - "yarn.scheduler.capacity.root.default.a.a2.user-limit-factor=1\n" - "yarn.scheduler.capacity.root.default.a.acl_administer_queue=*\n" - "yarn.scheduler.capacity.root.default.a.acl_submit_applications=*\n" - "yarn.scheduler.capacity.root.default.a.capacity=50\n" - "yarn.scheduler.capacity.root.default.a.maximum-capacity=100\n" - "yarn.scheduler.capacity.root.default.a.minimum-user-limit-percent=100\n" - "yarn.scheduler.capacity.root.default.a.ordering-policy=fifo\n" - "yarn.scheduler.capacity.root.default.a.queues=a1,a2\n" - "yarn.scheduler.capacity.root.default.a.state=RUNNING\n" - "yarn.scheduler.capacity.root.default.a.user-limit-factor=1\n" - "yarn.scheduler.capacity.root.default.acl_submit_applications=*\n" - "yarn.scheduler.capacity.root.default.b.acl_administer_queue=*\n" - "yarn.scheduler.capacity.root.default.b.acl_submit_applications=*\n" - "yarn.scheduler.capacity.root.default.b.capacity=50\n" - "yarn.scheduler.capacity.root.default.b.maximum-capacity=50\n" - "yarn.scheduler.capacity.root.default.b.minimum-user-limit-percent=100\n" - "yarn.scheduler.capacity.root.default.b.ordering-policy=fifo\n" - "yarn.scheduler.capacity.root.default.b.state=RUNNING\n" - "yarn.scheduler.capacity.root.default.b.user-limit-factor=1\n" - "yarn.scheduler.capacity.root.default.capacity=100\n" - "yarn.scheduler.capacity.root.default.maximum-capacity=100\n" - "yarn.scheduler.capacity.root.default.queues=a,b\n" - "yarn.scheduler.capacity.root.default.state=RUNNING\n" - "yarn.scheduler.capacity.root.default.user-limit-factor=1\n" - "yarn.scheduler.capacity.root.queues=default" - } - }, - "hive-interactive-env": - { - 'properties': { - 'enable_hive_interactive': 'false', - 'llap_queue_capacity':'0' - } - }, - "tez-interactive-site": { - "properties": { - "tez.am.resource.memory.mb": "341" - } - }, - "hive-env": - { - 'properties': { - 'hive_user': 'hive' - } - } - } - } - - - clusterData = { - "cpu": 4, - "mapMemory": 30000, - "amMemory": 20000, - "reduceMemory": 20560, - "containers": 30, - "ramPerContainer": 512, - "referenceNodeManagerHost" : { - "total_mem" : 10240 * 1024 - } - } - - - configurations = { - } - - self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, self.hosts) - - # Check output - self.assertEquals(configurations['hive-interactive-site']['properties'], - self.expected_hive_interactive_site_empty['hive-interactive-site']['properties']) - self.assertEquals(configurations['capacity-scheduler']['properties'], - self.expected_capacity_scheduler_empty['properties']) - self.assertFalse('hive-interactive-env' in configurations) - - - - - # Test 12: (1). More than 2 queues at leaf level exists in capacity-scheduler (one queue is named 'llap') and - # 'capacity-scheduler' configs are passed-in as single "/n" separated string - # (2). enable_hive_interactive' is 'off'. - # Expected : No changes. - def test_recommendYARNConfigurations_no_update_to_llap_queue_3(self): - services= { - "services": [{ - "StackServices": { - "service_name": "YARN", - }, - "Versions": { - "stack_version": "2.5" - }, - "components": [ - { - "StackServiceComponents": { - "component_name": "NODEMANAGER", - "hostnames": ["c6401.ambari.apache.org"] - } - } - ] - }, { - "href": "/api/v1/stacks/HDP/versions/2.5/services/HIVE", - "StackServices": { - "service_name": "HIVE", - "service_version": "1.2.1.2.5", - "stack_name": "HDP", - "stack_version": "2.5" - }, - "components": [ - { - "href": "/api/v1/stacks/HDP/versions/2.5/services/HIVE/components/HIVE_SERVER_INTERACTIVE", - "StackServiceComponents": { - "advertise_version": "true", - "bulk_commands_display_name": "", - "bulk_commands_master_component_name": "", - "cardinality": "0-1", - "component_category": "MASTER", - "component_name": "HIVE_SERVER_INTERACTIVE", - "custom_commands": ["RESTART_LLAP"], - "decommission_allowed": "false", - "display_name": "HiveServer2 Interactive", - "has_bulk_commands_definition": "false", - "is_client": "false", - "is_master": "true", - "reassign_allowed": "false", - "recovery_enabled": "false", - "service_name": "HIVE", - "stack_name": "HDP", - "stack_version": "2.5", - "hostnames": ["c6401.ambari.apache.org"] - }, - "dependencies": [] - } - ] - } - ], - "changed-configurations": [ - { - u'old_value': u'', - u'type': u'', - u'name': u'' - } - ], - "configurations": { - "capacity-scheduler": { - "properties": { - "capacity-scheduler": "yarn.scheduler.capacity.maximum-am-resource-percent=0.2\n" - "yarn.scheduler.capacity.maximum-applications=10000\n" - "yarn.scheduler.capacity.node-locality-delay=40\n" - "yarn.scheduler.capacity.queue-mappings-override.enable=false\n" - "yarn.scheduler.capacity.resource-calculator=org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator\n" - "yarn.scheduler.capacity.root.accessible-node-labels=*\n" - "yarn.scheduler.capacity.root.acl_administer_queue=*\n" - "yarn.scheduler.capacity.root.capacity=100\n" - "yarn.scheduler.capacity.root.default.a.a1.acl_administer_queue=*\n" - "yarn.scheduler.capacity.root.default.a.a1.acl_submit_applications=*\n" - "yarn.scheduler.capacity.root.default.a.a1.capacity=75\n" - "yarn.scheduler.capacity.root.default.a.a1.maximum-capacity=100\n" - "yarn.scheduler.capacity.root.default.a.a1.minimum-user-limit-percent=100\n" - "yarn.scheduler.capacity.root.default.a.a1.ordering-policy=fifo\n" - "yarn.scheduler.capacity.root.default.a.a1.state=RUNNING\n" - "yarn.scheduler.capacity.root.default.a.a1.user-limit-factor=1\n" - "yarn.scheduler.capacity.root.default.a.llap.acl_administer_queue=*\n" - "yarn.scheduler.capacity.root.default.a.llap.acl_submit_applications=*\n" - "yarn.scheduler.capacity.root.default.a.llap.capacity=25\n" - "yarn.scheduler.capacity.root.default.a.llap.maximum-capacity=25\n" - "yarn.scheduler.capacity.root.default.a.llap.minimum-user-limit-percent=100\n" - "yarn.scheduler.capacity.root.default.a.llap.ordering-policy=fifo\n" - "yarn.scheduler.capacity.root.default.a.llap.state=RUNNING\n" - "yarn.scheduler.capacity.root.default.a.llap.user-limit-factor=1\n" - "yarn.scheduler.capacity.root.default.a.acl_administer_queue=*\n" - "yarn.scheduler.capacity.root.default.a.acl_submit_applications=*\n" - "yarn.scheduler.capacity.root.default.a.capacity=50\n" - "yarn.scheduler.capacity.root.default.a.maximum-capacity=100\n" - "yarn.scheduler.capacity.root.default.a.minimum-user-limit-percent=100\n" - "yarn.scheduler.capacity.root.default.a.ordering-policy=fifo\n" - "yarn.scheduler.capacity.root.default.a.queues=a1,llap\n" - "yarn.scheduler.capacity.root.default.a.state=RUNNING\n" - "yarn.scheduler.capacity.root.default.a.user-limit-factor=1\n" - "yarn.scheduler.capacity.root.default.acl_submit_applications=*\n" - "yarn.scheduler.capacity.root.default.b.acl_administer_queue=*\n" - "yarn.scheduler.capacity.root.default.b.acl_submit_applications=*\n" - "yarn.scheduler.capacity.root.default.b.capacity=50\n" - "yarn.scheduler.capacity.root.default.b.maximum-capacity=50\n" - "yarn.scheduler.capacity.root.default.b.minimum-user-limit-percent=100\n" - "yarn.scheduler.capacity.root.default.b.ordering-policy=fifo\n" - "yarn.scheduler.capacity.root.default.b.state=RUNNING\n" - "yarn.scheduler.capacity.root.default.b.user-limit-factor=1\n" - "yarn.scheduler.capacity.root.default.capacity=100\n" - "yarn.scheduler.capacity.root.default.maximum-capacity=100\n" - "yarn.scheduler.capacity.root.default.queues=a,b\n" - "yarn.scheduler.capacity.root.default.state=RUNNING\n" - "yarn.scheduler.capacity.root.default.user-limit-factor=1\n" - "yarn.scheduler.capacity.root.queues=default" - } - }, - "hive-interactive-env": - { - 'properties': { - 'enable_hive_interactive': 'false', - 'llap_queue_capacity':'0' - } - }, - "hive-env": - { - 'properties': { - 'hive_user': 'hive' - } - } - } - } - - clusterData = { - "cpu": 4, - "mapMemory": 30000, - "amMemory": 20000, - "reduceMemory": 20560, - "containers": 30, - "ramPerContainer": 512, - "referenceNodeManagerHost" : { - "total_mem" : 10240 * 1024 - } - } - - configurations = { - } - - self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, self.hosts) - - # Check output - self.assertEquals(configurations['hive-interactive-site']['properties'], - self.expected_hive_interactive_site_empty['hive-interactive-site']['properties']) - self.assertEquals(configurations['capacity-scheduler']['properties'], - self.expected_capacity_scheduler_empty['properties']) - self.assertFalse('hive-interactive-env' in configurations) - - - - - # Test 13: (1). 'llap' (Cap: 0%, State: STOPPED) and 'default' (100%) queues exists at leaf level - # in capacity-scheduler and 'capacity-scheduler' configs are passed-in as single "/n" separated string - # (2). enable_hive_interactive' is 'off'. - # Expected : No changes. - def test_recommendYARNConfigurations_no_update_to_llap_queue_4(self): - services = { - "services": [{ - "StackServices": { - "service_name": "YARN", - }, - "Versions": { - "stack_version": "2.5" - }, - "components": [ - { - "StackServiceComponents": { - "component_name": "NODEMANAGER", - "hostnames": ["c6401.ambari.apache.org"] - } - } - ] - }, { - "href": "/api/v1/stacks/HDP/versions/2.5/services/HIVE", - "StackServices": { - "service_name": "HIVE", - "service_version": "1.2.1.2.5", - "stack_name": "HDP", - "stack_version": "2.5" - }, - "components": [ - { - "href": "/api/v1/stacks/HDP/versions/2.5/services/HIVE/components/HIVE_SERVER_INTERACTIVE", - "StackServiceComponents": { - "advertise_version": "true", - "bulk_commands_display_name": "", - "bulk_commands_master_component_name": "", - "cardinality": "0-1", - "component_category": "MASTER", - "component_name": "HIVE_SERVER_INTERACTIVE", - "custom_commands": ["RESTART_LLAP"], - "decommission_allowed": "false", - "display_name": "HiveServer2 Interactive", - "has_bulk_commands_definition": "false", - "is_client": "false", - "is_master": "true", - "reassign_allowed": "false", - "recovery_enabled": "false", - "service_name": "HIVE", - "stack_name": "HDP", - "stack_version": "2.5", - "hostnames": ["c6401.ambari.apache.org"] - }, - "dependencies": [] - } - ] - } - ], - "changed-configurations": [ - { - u'old_value': u'', - u'type': u'', - u'name': u'' - } - ], - "configurations": { - "capacity-scheduler": { - "properties": { - "capacity-scheduler": "yarn.scheduler.capacity.root.accessible-node-labels=*\n" - "yarn.scheduler.capacity.root.capacity=100\n" - "yarn.scheduler.capacity.root.queues=default,llap\n" - "yarn.scheduler.capacity.maximum-applications=10000\n" - "yarn.scheduler.capacity.root.default.user-limit-factor=1\n" - "yarn.scheduler.capacity.root.default.state=RUNNING\n" - "yarn.scheduler.capacity.maximum-am-resource-percent=1\n" - "yarn.scheduler.capacity.root.default.acl_submit_applications=*\n" - "yarn.scheduler.capacity.root.default.capacity=100\n" - "yarn.scheduler.capacity.root.acl_administer_queue=*\n" - "yarn.scheduler.capacity.node-locality-delay=40\n" - "yarn.scheduler.capacity.queue-mappings-override.enable=false\n" - "yarn.scheduler.capacity.root.llap.user-limit-factor=1\n" - "yarn.scheduler.capacity.root.llap.state=STOPPED\n" - "yarn.scheduler.capacity.root.llap.ordering-policy=fifo\n" - "yarn.scheduler.capacity.root.llap.minimum-user-limit-percent=100\n" - "yarn.scheduler.capacity.root.llap.maximum-capacity=0\n" - "yarn.scheduler.capacity.root.default.maximum-capacity=100\n" - "yarn.scheduler.capacity.root.llap.capacity=0\n" - "yarn.scheduler.capacity.root.llap.acl_submit_applications=hive\n" - "yarn.scheduler.capacity.root.llap.acl_administer_queue=hive\n" - "yarn.scheduler.capacity.root.llap.maximum-am-resource-percent=1\n" - } - }, - "hive-interactive-env": - { - 'properties': { - 'enable_hive_interactive': 'false' - } - }, - "hive-env": - { - 'properties': { - 'hive_user': 'hive' - } - } - } - } - - clusterData = { - "cpu": 4, - "mapMemory": 30000, - "amMemory": 20000, - "reduceMemory": 20560, - "containers": 30, - "ramPerContainer": 512, - "referenceNodeManagerHost" : { - "total_mem" : 10240 * 1024 - } - } - - - configurations = { - } - - self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, self.hosts) - - # Check output - self.assertEquals(configurations['hive-interactive-site']['properties'], - self.expected_hive_interactive_site_empty['hive-interactive-site']['properties']) - self.assertEquals(configurations['capacity-scheduler']['properties'], - self.expected_capacity_scheduler_empty['properties']) - self.assertFalse('hive-interactive-env' in configurations) - - - - # Test 14: YARN service with : (1). 'capacity scheduler' having 'llap' (state:stopped) and 'default' queue at - # root level and and 'capacity-scheduler' configs are passed-in as single "/n" separated string - # (2). 'enable_hive_interactive' is ON and (3). 'hive.llap.daemon.queue.name' == 'default' - def test_recommendYARNConfigurations_no_update_to_llap_queue_5(self): - services_15 = { - "services": [{ - "StackServices": { - "service_name": "YARN", - }, - "Versions": { - "stack_version": "2.5" - }, - "components": [ - { - "StackServiceComponents": { - "component_name": "NODEMANAGER", - "hostnames": ["c6401.ambari.apache.org"] - } - } - ] - }, { - "href": "/api/v1/stacks/HDP/versions/2.5/services/HIVE", - "StackServices": { - "service_name": "HIVE", - "service_version": "1.2.1.2.5", - "stack_name": "HDP", - "stack_version": "2.5" - }, - "components": [ - { - "href": "/api/v1/stacks/HDP/versions/2.5/services/HIVE/components/HIVE_SERVER_INTERACTIVE", - "StackServiceComponents": { - "advertise_version": "true", - "bulk_commands_display_name": "", - "bulk_commands_master_component_name": "", - "cardinality": "0-1", - "component_category": "MASTER", - "component_name": "HIVE_SERVER_INTERACTIVE", - "custom_commands": ["RESTART_LLAP"], - "decommission_allowed": "false", - "display_name": "HiveServer2 Interactive", - "has_bulk_commands_definition": "false", - "is_client": "false", - "is_master": "true", - "reassign_allowed": "false", - "recovery_enabled": "false", - "service_name": "HIVE", - "stack_name": "HDP", - "stack_version": "2.5", - "hostnames": ["c6401.ambari.apache.org"] - }, - "dependencies": [] - }, - { - "StackServiceComponents": { - "advertise_version": "true", - "cardinality": "1+", - "component_category": "SLAVE", - "component_name": "NODEMANAGER", - "display_name": "NodeManager", - "is_client": "false", - "is_master": "false", - "hostnames": [ - "c6403.ambari.apache.org" - ] -
<TRUNCATED>
