Repository: ambari Updated Branches: refs/heads/trunk 13248dc10 -> 86bf4dcaa
AMBARI-17075. Set 'hive.server2.tez.default.queues' config value to be the one used by config hive.llap.daemon.queue.name for 'llap' app. Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/86bf4dca Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/86bf4dca Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/86bf4dca Branch: refs/heads/trunk Commit: 86bf4dcaac4c46dafe25c4db21cbecfbb3aeadbb Parents: 13248dc Author: Swapan Shridhar <[email protected]> Authored: Mon Jun 6 16:54:21 2016 -0700 Committer: Swapan Shridhar <[email protected]> Committed: Tue Jun 7 00:08:19 2016 -0700 ---------------------------------------------------------------------- .../configuration/hive-interactive-site.xml | 4 + .../stacks/HDP/2.5/services/stack_advisor.py | 42 ++-- .../stacks/2.5/common/test_stack_advisor.py | 197 ++++++++++++++++++- 3 files changed, 227 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/86bf4dca/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml index 4133e5b..62823a2 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml @@ -195,6 +195,10 @@ limitations under the License. <type>capacity-scheduler</type> <name>yarn.scheduler.capacity.root.queues</name> </property> + <property> + <type>hive-interactive-site</type> + <name>hive.llap.daemon.queue.name</name> + </property> </depends-on> <on-ambari-upgrade add="false" change="true" delete="true"/> <on-stack-upgrade add="true" change="true" delete="false"/> http://git-wip-us.apache.org/repos/asf/ambari/blob/86bf4dca/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py index 379c268..6ecebc6 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py @@ -29,6 +29,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor): super(HDP25StackAdvisor, self).__init__() Logger.initialize_logger() self.HIVE_INTERACTIVE_SITE = 'hive-interactive-site' + self.YARN_ROOT_DEFAULT_QUEUE_NAME = 'default' def createComponentLayoutRecommendations(self, services, hosts): parentComponentLayoutRecommendations = super(HDP25StackAdvisor, self).createComponentLayoutRecommendations( @@ -254,10 +255,23 @@ class HDP25StackAdvisor(HDP24StackAdvisor): putHiveInteractiveEnvProperty('enable_hive_interactive', 'true') putHiveInteractiveEnvProperty('hive_server_interactive_host', hsi_host) - # Update 'hive.llap.daemon.queue.name' if capacity scheduler is changed. - if self.HIVE_INTERACTIVE_SITE in services['configurations'] and \ - 'hive.llap.daemon.queue.name' in services['configurations'][self.HIVE_INTERACTIVE_SITE]['properties']: - self.setLlapDaemonQueueNamePropAttributes(services, configurations) + # Update 'hive.llap.daemon.queue.name' property attributes if capacity scheduler is changed. + if self.HIVE_INTERACTIVE_SITE in services['configurations']: + if 'hive.llap.daemon.queue.name' in services['configurations'][self.HIVE_INTERACTIVE_SITE]['properties']: + self.setLlapDaemonQueueNamePropAttributes(services, configurations) + + # Update 'hive.server2.tez.default.queues' value + hive_tez_default_queue = None + if 'hive-interactive-site' in configurations and \ + 'hive.llap.daemon.queue.name' in configurations[self.HIVE_INTERACTIVE_SITE]['properties']: + hive_tez_default_queue = configurations[self.HIVE_INTERACTIVE_SITE]['properties']['hive.llap.daemon.queue.name'] + Logger.info("'hive.llap.daemon.queue.name' value from configurations : '{0}'".format(hive_tez_default_queue)) + if not hive_tez_default_queue: + hive_tez_default_queue = services['configurations'][self.HIVE_INTERACTIVE_SITE]['properties']['hive.llap.daemon.queue.name'] + Logger.info("'hive.llap.daemon.queue.name' value from services : '{0}'".format(hive_tez_default_queue)) + if hive_tez_default_queue: + putHiveInteractiveSiteProperty("hive.server2.tez.default.queues", hive_tez_default_queue) + Logger.info("Updated 'hive.server2.tez.default.queues' config : '{0}'".format(hive_tez_default_queue)) else: putHiveInteractiveEnvProperty('enable_hive_interactive', 'false') @@ -643,13 +657,13 @@ class HDP25StackAdvisor(HDP24StackAdvisor): # Check if 'hive.tez.container.size' is modified in current ST invocation. if 'hive-site' in configurations and 'hive.tez.container.size' in configurations['hive-site']['properties']: hive_container_size = float(configurations['hive-site']['properties']['hive.tez.container.size']) - Logger.info("''hive.tez.container.size'' read from configurations as : {0}".format(hive_container_size)) + Logger.info("'hive.tez.container.size' read from configurations as : {0}".format(hive_container_size)) if not hive_container_size: # Check if 'hive.tez.container.size' is input in services array. if 'hive.tez.container.size' in services['configurations']['hive-site']['properties']: hive_container_size = float(services['configurations']['hive-site']['properties']['hive.tez.container.size']) - Logger.info("''hive.tez.container.size'' read from services as : {0}".format(hive_container_size)) + Logger.info("'hive.tez.container.size' read from services as : {0}".format(hive_container_size)) if not hive_container_size: raise Fail("Couldn't retrieve Hive Server 'hive.tez.container.size' config.") @@ -668,14 +682,14 @@ class HDP25StackAdvisor(HDP24StackAdvisor): # Check if 'yarn.scheduler.minimum-allocation-mb' is modified in current ST invocation. if 'yarn-site' in configurations and 'yarn.scheduler.minimum-allocation-mb' in configurations['yarn-site']['properties']: yarn_min_container_size = float(configurations['yarn-site']['properties']['yarn.scheduler.minimum-allocation-mb']) - Logger.info("''yarn.scheduler.minimum-allocation-mb'' read from configurations as : {0}".format(yarn_min_container_size)) + Logger.info("'yarn.scheduler.minimum-allocation-mb' read from configurations as : {0}".format(yarn_min_container_size)) if not yarn_min_container_size: # Check if 'yarn.scheduler.minimum-allocation-mb' is input in services array. if 'yarn-site' in services['configurations'] and \ 'yarn.scheduler.minimum-allocation-mb' in services['configurations']['yarn-site']['properties']: yarn_min_container_size = float(services['configurations']['yarn-site']['properties']['yarn.scheduler.minimum-allocation-mb']) - Logger.info("''yarn.scheduler.minimum-allocation-mb'' read from services as : {0}".format(yarn_min_container_size)) + Logger.info("'yarn.scheduler.minimum-allocation-mb' read from services as : {0}".format(yarn_min_container_size)) if not yarn_min_container_size: raise Fail("Couldn't retrieve YARN's 'yarn.scheduler.minimum-allocation-mb' config.") @@ -705,7 +719,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor): # Check if 'yarn.nodemanager.resource.memory-mb' is modified in current ST invocation. if 'yarn-site' in configurations and 'yarn.nodemanager.resource.memory-mb' in configurations['yarn-site']['properties']: yarn_nm_mem_in_mb = float(configurations['yarn-site']['properties']['yarn.nodemanager.resource.memory-mb']) - Logger.info("''yarn.nodemanager.resource.memory-mb'' read from configurations as : {0}".format(yarn_nm_mem_in_mb)) + Logger.info("'yarn.nodemanager.resource.memory-mb' read from configurations as : {0}".format(yarn_nm_mem_in_mb)) print "from services : ",services['configurations']['yarn-site']['properties']['yarn.nodemanager.resource.memory-mb'] if not yarn_nm_mem_in_mb: @@ -732,14 +746,14 @@ class HDP25StackAdvisor(HDP24StackAdvisor): # Check if 'tez.am.resource.memory.mb' is modified in current ST invocation. if 'tez-interactive-site' in configurations and 'tez.am.resource.memory.mb' in configurations['tez-interactive-site']['properties']: llap_daemon_container_size = float(configurations['tez-interactive-site']['properties']['tez.am.resource.memory.mb']) - Logger.info("''tez.am.resource.memory.mb'' read from configurations as : {0}".format(llap_daemon_container_size)) + Logger.info("'tez.am.resource.memory.mb' read from configurations as : {0}".format(llap_daemon_container_size)) if not llap_daemon_container_size: # Check if 'tez.am.resource.memory.mb' is input in services array. if self.HIVE_INTERACTIVE_SITE in services['configurations'] and \ 'tez.am.resource.memory.mb' in services['configurations']['tez-interactive-site']['properties']: llap_daemon_container_size = float(services['configurations']['tez-interactive-site']['properties']['tez.am.resource.memory.mb']) - Logger.info("''tez.am.resource.memory.mb'' read from services as : {0}".format(llap_daemon_container_size)) + Logger.info("'tez.am.resource.memory.mb' read from services as : {0}".format(llap_daemon_container_size)) if not llap_daemon_container_size: @@ -940,7 +954,8 @@ class HDP25StackAdvisor(HDP24StackAdvisor): Logger.info("Adjusted 'default' queue capacity to : {0}%".format(adjusted_default_queue_cap)) # Update Hive 'hive.llap.daemon.queue.name' prop to use 'llap' queue. - putHiveInteractiveSiteProperty('hive.llap.daemon.queue.name', 'llap') + putHiveInteractiveSiteProperty('hive.llap.daemon.queue.name', llap_queue_name) + putHiveInteractiveSiteProperty('hive.server2.tez.default.queues', llap_queue_name) putHiveInteractiveEnvPropertyAttribute('llap_queue_capacity', "minimum", llap_min_reqd_cap_percentage) putHiveInteractiveEnvPropertyAttribute('llap_queue_capacity', "maximum", 100) @@ -1014,7 +1029,8 @@ class HDP25StackAdvisor(HDP24StackAdvisor): .format(llap_queue_name, DEFAULT_MAX_CAPACITY)) # Update Hive 'hive.llap.daemon.queue.name' prop to use 'default' queue. - putHiveInteractiveSiteProperty('hive.llap.daemon.queue.name', 'default') + putHiveInteractiveSiteProperty('hive.llap.daemon.queue.name', self.YARN_ROOT_DEFAULT_QUEUE_NAME) + putHiveInteractiveSiteProperty('hive.server2.tez.default.queues', self.YARN_ROOT_DEFAULT_QUEUE_NAME) else: Logger.debug("Not removing '{0}' queue as number of Queues not equal to 2. Current YARN queues : {1}".format(llap_queue_name, list(leafQueueNames))) else: http://git-wip-us.apache.org/repos/asf/ambari/blob/86bf4dca/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py index 46e9f47..1627458 100644 --- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py @@ -310,6 +310,7 @@ class TestHDP25StackAdvisor(TestCase): self.expected_hive_interactive_site_default = { "hive-interactive-site": { "properties": { + "hive.server2.tez.default.queues": "default", "hive.llap.daemon.queue.name": "default" }, "property_attributes": { @@ -2922,7 +2923,6 @@ class TestHDP25StackAdvisor(TestCase): } self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, self.hosts) - cap_sched_output_dict = convertToDict(configurations['capacity-scheduler']['properties']['capacity-scheduler']) cap_sched_expected_dict = convertToDict(self.expected_capacity_scheduler_llap_Stopped_size_0['properties']['capacity-scheduler']) self.assertEqual(cap_sched_output_dict, cap_sched_expected_dict) @@ -3281,6 +3281,7 @@ class TestHDP25StackAdvisor(TestCase): self.assertEqual(configurations['hive-interactive-env']['properties']['llap_heap_size'], '409') self.assertEqual(configurations['hive-interactive-env']['properties']['slider_am_container_size'], '512') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.default.queues'], 'llap') # Test 19: (1). 'default' and 'llap' (State : RUNNING) queue exists at root level in capacity-scheduler, and @@ -3470,6 +3471,8 @@ class TestHDP25StackAdvisor(TestCase): self.assertEqual(configurations['hive-interactive-env']['properties']['slider_am_container_size'], '512') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.default.queues'], 'llap') + @@ -3661,6 +3664,8 @@ class TestHDP25StackAdvisor(TestCase): self.assertEqual(configurations['hive-interactive-env']['properties']['slider_am_container_size'], '1024') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.default.queues'], 'llap') + ####################### 'Three Node Managers' cluster - tests for calculating llap configs ################ @@ -3853,6 +3858,8 @@ class TestHDP25StackAdvisor(TestCase): self.assertEqual(configurations['hive-interactive-env']['properties']['slider_am_container_size'], '1024') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.default.queues'], 'llap') + # Test 22: (1). 'default' and 'llap' (State : RUNNING) queue exists at root level in capacity-scheduler, and @@ -4040,6 +4047,8 @@ class TestHDP25StackAdvisor(TestCase): self.assertEqual(configurations['hive-interactive-env']['properties']['slider_am_container_size'], '341') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.default.queues'], 'llap') + # Test 23: (1). 'default' and 'llap' (State : RUNNING) queue exists at root level in capacity-scheduler, and @@ -4224,6 +4233,8 @@ class TestHDP25StackAdvisor(TestCase): self.assertEqual(configurations['hive-interactive-env']['properties']['slider_am_container_size'], '1024') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.default.queues'], 'llap') + @@ -4416,6 +4427,8 @@ class TestHDP25StackAdvisor(TestCase): self.assertEqual(configurations['hive-interactive-env']['properties']['slider_am_container_size'], '1024') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.default.queues'], 'llap') + @@ -4604,6 +4617,8 @@ class TestHDP25StackAdvisor(TestCase): self.assertEqual(configurations['hive-interactive-env']['properties']['slider_am_container_size'], '341') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.default.queues'], 'llap') + @@ -4771,7 +4786,6 @@ class TestHDP25StackAdvisor(TestCase): configurations = { } - print "\n\n\n\n\ yyyyyyy" self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, self.hosts) self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes'], '2') @@ -4787,6 +4801,8 @@ class TestHDP25StackAdvisor(TestCase): self.assertEqual(configurations['hive-interactive-env']['properties']['slider_am_container_size'], '1024') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.default.queues'], 'llap') + @@ -4962,6 +4978,8 @@ class TestHDP25StackAdvisor(TestCase): self.assertEqual(configurations['hive-interactive-env']['properties']['slider_am_container_size'], '1024') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.default.queues'], 'llap') + @@ -5139,6 +5157,8 @@ class TestHDP25StackAdvisor(TestCase): self.assertEqual(configurations['hive-interactive-env']['properties']['slider_am_container_size'], '1024') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.default.queues'], 'llap') + @@ -5147,6 +5167,7 @@ class TestHDP25StackAdvisor(TestCase): # Test 29: (1). only 'default' queue exists at root level in capacity-scheduler, and # 'capacity-scheduler' configs are passed-in as single "/n" separated string and # Expected : 'hive.llap.daemon.queue.name' property attributes getting set with current YARN leaf queues. + # 'hive.server2.tez.default.queues' value getting set to value of 'hive.llap.daemon.queue.name' (llap). def test_recommendHIVEConfigurations_for_llap_queue_prop_attributes_1(self): services = { "services": [{ @@ -5300,7 +5321,7 @@ class TestHDP25StackAdvisor(TestCase): self.stackAdvisor.recommendHIVEConfigurations(configurations, clusterData, services, self.hosts) self.assertEquals(configurations['hive-interactive-site']['properties']['hive.llap.daemon.queue.name'], self.expected_hive_interactive_site_llap['hive-interactive-site']['properties']['hive.llap.daemon.queue.name']) - + self.assertEquals(configurations['hive-interactive-site']['properties']['hive.server2.tez.default.queues'], 'llap') @@ -5310,6 +5331,7 @@ class TestHDP25StackAdvisor(TestCase): # Test 30: (1). More than 2 queues at leaf level exists in capacity-scheduler (one queue is named 'llap') and # 'capacity-scheduler' configs are passed-in as single "/n" separated string # Expected : 'hive.llap.daemon.queue.name' property attributes getting set with current YARN leaf queues. + # 'hive.server2.tez.default.queues' value getting set to value of 'hive.llap.daemon.queue.name' (llap). def test_recommendHIVEConfigurations_for_llap_queue_prop_attributes_2(self): services= { "services": [{ @@ -5481,6 +5503,175 @@ class TestHDP25StackAdvisor(TestCase): self.stackAdvisor.recommendHIVEConfigurations(configurations, clusterData, services, self.hosts) self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name'], self.expected_hive_interactive_site_prop_attr_as_a1_b_llap['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name']) + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.default.queues'], 'llap') + + + + + + + # Test 31: (1). only 'default' queue exists at root level in capacity-scheduler, and + # 'capacity-scheduler' configs are passed-in as single "/n" separated string and + # change in 'hive.llap.daemon.queue.name' value detected. + # Expected : 'hive.llap.daemon.queue.name' property attributes getting set with current YARN leaf queues. + # 'hive.server2.tez.default.queues' value getting set to value of 'hive.llap.daemon.queue.name' (default). + def test_recommendHIVEConfigurations_for_llap_queue_prop_attributes_1(self): + services = { + "services": [{ + "StackServices": { + "service_name": "YARN", + }, + "Versions": { + "stack_version": "2.5" + }, + "components": [ + { + "StackServiceComponents": { + "component_name": "NODEMANAGER", + "hostnames": ["c6401.ambari.apache.org"] + } + } + ] + }, { + "href": "/api/v1/stacks/HDP/versions/2.5/services/HIVE", + "StackServices": { + "service_name": "HIVE", + "service_version": "1.2.1.2.5", + "stack_name": "HDP", + "stack_version": "2.5" + }, + "components": [ + { + "href": "/api/v1/stacks/HDP/versions/2.5/services/HIVE/components/HIVE_SERVER_INTERACTIVE", + "StackServiceComponents": { + "advertise_version": "true", + "bulk_commands_display_name": "", + "bulk_commands_master_component_name": "", + "cardinality": "0-1", + "component_category": "MASTER", + "component_name": "HIVE_SERVER_INTERACTIVE", + "custom_commands": ["RESTART_LLAP"], + "decommission_allowed": "false", + "display_name": "HiveServer2 Interactive", + "has_bulk_commands_definition": "false", + "is_client": "false", + "is_master": "true", + "reassign_allowed": "false", + "recovery_enabled": "false", + "service_name": "HIVE", + "stack_name": "HDP", + "stack_version": "2.5", + "hostnames": ["c6401.ambari.apache.org"] + }, + "dependencies": [] + }, + { + "StackServiceComponents": { + "advertise_version": "true", + "cardinality": "1+", + "component_category": "SLAVE", + "component_name": "NODEMANAGER", + "display_name": "NodeManager", + "is_client": "false", + "is_master": "false", + "hostnames": [ + "c6401.ambari.apache.org" + ] + }, + "dependencies": [] + }, + ] + } + ], + "changed-configurations": [ + { + u'old_value': u'llap', + u'type': u'hive-interactive-site', + u'name': u'hive.llap.daemon.queue.name' + } + ], + "configurations": { + "capacity-scheduler": { + "properties": { + "capacity-scheduler": "yarn.scheduler.capacity.root.queues=default\n" + "yarn.scheduler.capacity.root.default.user-limit-factor=1\n" + "yarn.scheduler.capacity.root.default.state=RUNNING\n" + "yarn.scheduler.capacity.root.default.maximum-capacity=100\n" + "yarn.scheduler.capacity.root.default.capacity=100\n" + "yarn.scheduler.capacity.root.default.acl_submit_applications=*\n" + "yarn.scheduler.capacity.root.capacity=100\n" + "yarn.scheduler.capacity.root.acl_administer_queue=*\n" + "yarn.scheduler.capacity.root.accessible-node-labels=*\n" + "yarn.scheduler.capacity.node-locality-delay=40\n" + "yarn.scheduler.capacity.maximum-applications=10000\n" + "yarn.scheduler.capacity.maximum-am-resource-percent=1\n" + "yarn.scheduler.capacity.queue-mappings-override.enable=false\n" + } + }, + "hive-interactive-env": + { + 'properties': { + 'enable_hive_interactive': 'true', + 'llap_queue_capacity':'50' + } + }, + "hive-interactive-site": + { + 'properties': { + 'hive.llap.daemon.queue.name': 'default', + 'hive.server2.tez.sessions.per.default.queue': '1' + } + }, + "hive-env": + { + 'properties': { + 'hive_user': 'hive' + } + }, + "yarn-site": { + "properties": { + "yarn.scheduler.minimum-allocation-mb": "2048", + "yarn.nodemanager.resource.memory-mb": "204800", + "yarn.nodemanager.resource.cpu-vcores": '3' + } + }, + "tez-interactive-site": { + "properties": { + "tez.am.resource.memory.mb": "1024" + } + }, + "hive-site": + { + 'properties': { + 'hive.tez.container.size': '1024' + } + }, + } + } + + + clusterData = { + "cpu": 4, + "mapMemory": 30000, + "amMemory": 20000, + "reduceMemory": 20560, + "containers": 3, + "ramPerContainer": 82240, + "referenceNodeManagerHost" : { + "total_mem" : 328960 * 1024 + } + } + + configurations = { + } + + self.stackAdvisor.recommendHIVEConfigurations(configurations, clusterData, services, self.hosts) + self.assertEquals(configurations['hive-interactive-site']['properties']['hive.llap.daemon.queue.name'], + self.expected_hive_interactive_site_default['hive-interactive-site']['properties']['hive.llap.daemon.queue.name']) + self.assertEquals(configurations['hive-interactive-site']['properties']['hive.server2.tez.default.queues'], 'default') + + +
