Repository: ambari Updated Branches: refs/heads/trunk a3c4d46c7 -> 051f09a40
AMBARI-16178. llap queue minimum recommended size is not persistent across HSI on and off (Swapan Sridhar via smohanty) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/051f09a4 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/051f09a4 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/051f09a4 Branch: refs/heads/trunk Commit: 051f09a400509fe1e38b768553a6bcd755e4ed5e Parents: a3c4d46 Author: Sumit Mohanty <[email protected]> Authored: Fri Apr 29 05:14:06 2016 -0700 Committer: Sumit Mohanty <[email protected]> Committed: Fri Apr 29 05:14:06 2016 -0700 ---------------------------------------------------------------------- .../stacks/HDP/2.5/services/stack_advisor.py | 22 +++--- .../stacks/2.5/common/test_stack_advisor.py | 76 +++++++++++++------- 2 files changed, 62 insertions(+), 36 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/051f09a4/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py index 03af7c0..ca92157 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py @@ -506,7 +506,8 @@ class HDP25StackAdvisor(HDP24StackAdvisor): node_manager_hosts = self.get_node_manager_hosts(services, hosts) node_manager_host_cnt = len(node_manager_hosts) - num_llap_nodes_in_changed_configs = self.are_config_props_in_changed_configs(services, "hive-interactive-env", "num_llap_nodes", False) + num_llap_nodes_in_changed_configs = self.are_config_props_in_changed_configs(services, "hive-interactive-env", + set(['num_llap_nodes']), False) if not num_llap_nodes_in_changed_configs: num_llap_nodes, num_llap_nodes_max_limit = self.calculate_num_llap_nodes(services, hosts, configurations) else: @@ -736,12 +737,11 @@ class HDP25StackAdvisor(HDP24StackAdvisor): if 'llap_queue_capacity' in services['configurations']['hive-interactive-env']['properties']: llap_slider_cap_percentage = int( services['configurations']['hive-interactive-env']['properties']['llap_queue_capacity']) + llap_min_reqd_cap_percentage = self.min_llap_queue_perc_required_in_cluster(services, hosts) if llap_slider_cap_percentage not in range(1,101): - llap_slider_cap_percentage = self.min_llap_queue_perc_required_in_cluster(services, hosts) - Logger.info("Adjusting HIVE 'llap_queue_capacity' from {0}% to {1}%".format(llap_slider_cap_percentage, llap_slider_cap_percentage)) - - putHiveInteractiveEnvProperty('llap_queue_capacity', llap_slider_cap_percentage) - putHiveInteractiveEnvPropertyAttribute('llap_queue_capacity', "minimum", llap_slider_cap_percentage) + Logger.info("Adjusting HIVE 'llap_queue_capacity' from {0}% to {1}%".format(llap_slider_cap_percentage, llap_min_reqd_cap_percentage)) + putHiveInteractiveEnvProperty('llap_queue_capacity', llap_min_reqd_cap_percentage) + llap_slider_cap_percentage = llap_min_reqd_cap_percentage else: Logger.error("Problem retrieving LLAP Queue Capacity. Skipping creating {0} queue".format(llap_queue_name)) return @@ -749,8 +749,8 @@ class HDP25StackAdvisor(HDP24StackAdvisor): capSchedConfigKeys = capacitySchedulerProperties.keys() yarn_default_queue_capacity = -1 - if 'yarn.scheduler.capacity.root.capacity' in capSchedConfigKeys: - yarn_default_queue_capacity = capacitySchedulerProperties.get('yarn.scheduler.capacity.root.capacity') + if 'yarn.scheduler.capacity.root.default.capacity' in capSchedConfigKeys: + yarn_default_queue_capacity = capacitySchedulerProperties.get('yarn.scheduler.capacity.root.default.capacity') # Get 'llap' queue state currLlapQueueState = '' @@ -760,7 +760,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor): # Get 'llap' queue capacity currLlapQueueCap = -1 if 'yarn.scheduler.capacity.root.'+llap_queue_name+'.capacity' in capSchedConfigKeys: - currLlapQueueCap = capacitySchedulerProperties.get('yarn.scheduler.capacity.root.'+llap_queue_name+'.capacity') + currLlapQueueCap = int(capacitySchedulerProperties.get('yarn.scheduler.capacity.root.'+llap_queue_name+'.capacity')) if self.HIVE_INTERACTIVE_SITE in services['configurations'] and \ 'hive.llap.daemon.queue.name' in services['configurations'][self.HIVE_INTERACTIVE_SITE]['properties']: @@ -784,7 +784,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor): ((len(leafQueueNames) == 1 and int(yarn_default_queue_capacity) == 100) or \ ((len(leafQueueNames) == 2 and llap_queue_name in leafQueueNames) and \ ((currLlapQueueState == 'STOPPED' and enabled_hive_int_in_changed_configs) or (currLlapQueueState == 'RUNNING' and currLlapQueueCap != llap_slider_cap_percentage)))): - adjusted_default_queue_cap = str(int(yarn_default_queue_capacity) - llap_slider_cap_percentage) + adjusted_default_queue_cap = str(100 - llap_slider_cap_percentage) for prop, val in capacitySchedulerProperties.items(): if llap_queue_name not in prop: if prop == 'yarn.scheduler.capacity.root.queues': @@ -831,6 +831,8 @@ class HDP25StackAdvisor(HDP24StackAdvisor): # Update Hive 'hive.llap.daemon.queue.name' prop to use 'llap' queue. putHiveInteractiveSiteProperty('hive.llap.daemon.queue.name', 'llap') + putHiveInteractiveEnvPropertyAttribute('llap_queue_capacity', "minimum", llap_min_reqd_cap_percentage) + # Update 'hive.llap.daemon.queue.name' prop combo entries. self.setLlapDaemonQueueName(services, configurations) else: http://git-wip-us.apache.org/repos/asf/ambari/blob/051f09a4/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py index 8f4a5c2..574964f 100644 --- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py @@ -553,7 +553,8 @@ class TestHDP25StackAdvisor(TestCase): "hive-interactive-site": { 'properties': { - 'hive.llap.daemon.queue.name':'default' + 'hive.llap.daemon.queue.name':'default', + 'hive.tez.container.size': '341' } }, "tez-interactive-site": { @@ -1437,9 +1438,22 @@ class TestHDP25StackAdvisor(TestCase): "hive-interactive-site": { 'properties': { - 'hive.llap.daemon.queue.name':'llap' + 'hive.llap.daemon.queue.name':'llap', + 'hive.tez.container.size': '341' } }, + "tez-interactive-site": { + "properties": { + "tez.am.resource.memory.mb": "341" + } + }, + "yarn-site": { + "properties": { + "yarn.scheduler.minimum-allocation-mb": "341", + "yarn.nodemanager.resource.memory-mb": "20000", + "yarn.nodemanager.resource.cpu-vcores": '1' + } + }, "hive-env": { 'properties': { @@ -1690,7 +1704,7 @@ class TestHDP25StackAdvisor(TestCase): { 'properties': { 'enable_hive_interactive': 'true', - 'llap_queue_capacity':'40' + 'llap_queue_capacity':'41' } }, "hive-interactive-site": @@ -2144,7 +2158,7 @@ class TestHDP25StackAdvisor(TestCase): { 'properties': { 'enable_hive_interactive': 'true', - 'llap_queue_capacity':'40' + 'llap_queue_capacity':'41' } }, "hive-interactive-site": @@ -2564,7 +2578,7 @@ class TestHDP25StackAdvisor(TestCase): { 'properties': { 'enable_hive_interactive': 'true', - 'llap_queue_capacity':'40', + 'llap_queue_capacity':'41', 'num_llap_nodes': 1 } }, @@ -2930,6 +2944,8 @@ class TestHDP25StackAdvisor(TestCase): # expected vals. expected_visibility_false = {'visible': 'false'} expected_visibility_true = {'visible': 'true'} + expected_llap_queue_attributes_min_25_visible_true = {'minimum': '25', 'visible': 'true'} + expected_llap_queue_attributes_min_21_visible_true = {'minimum': '21', 'visible': 'true'} @@ -2943,6 +2959,7 @@ class TestHDP25StackAdvisor(TestCase): } self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services_1, hosts) # Check output + print "\n\n\n\n conf = ",configurations self.assertEquals(configurations['hive-interactive-site']['properties']['hive.llap.daemon.queue.name'], expected_hive_interactive_site_llap['hive-interactive-site']['properties']['hive.llap.daemon.queue.name']) self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name'], @@ -3167,29 +3184,32 @@ class TestHDP25StackAdvisor(TestCase): } # expected vals. - yarn_cont_mb_visibility = {'minimum': '682', 'maximum': '682'} - llap_io_mem_size_visibility = {'minimum': '0', 'maximum': '341'} - llap_num_executors_visibility = {'minimum': '1', 'maximum': '1'} - num_llap_nodes_visibility = {'minimum': '1', 'maximum': '1'} + print "\n\n\n\n\n\n\n\n\n\n\n" + yarn_cont_mb_attributes = {'minimum': '682', 'maximum': '682'} + llap_io_mem_size_attributes = {'minimum': '0', 'maximum': '341'} + llap_num_executors_attributes = {'minimum': '1', 'maximum': '1'} + num_llap_nodes_attributes = {'minimum': '1', 'maximum': '1'} + self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services_13, hosts) self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '682') - self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.yarn.container.mb'], yarn_cont_mb_visibility) + self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.yarn.container.mb'], yarn_cont_mb_attributes) self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors'], '1') - self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.num.executors'], llap_num_executors_visibility) + self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.num.executors'], llap_num_executors_attributes) self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size'], '341') - self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.io.memory.size'], llap_io_mem_size_visibility) + self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.io.memory.size'], llap_io_mem_size_attributes) self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes'], '1') - self.assertEquals(configurations['hive-interactive-env']['property_attributes']['num_llap_nodes'], num_llap_nodes_visibility) + self.assertEquals(configurations['hive-interactive-env']['property_attributes']['num_llap_nodes'], num_llap_nodes_attributes) self.assertEqual(configurations['hive-interactive-env']['properties']['llap_heap_size'], '272') self.assertEqual(configurations['hive-interactive-env']['properties']['slider_am_container_mb'], '341') - self.assertEquals(configurations['hive-interactive-env']['property_attributes']['llap_queue_capacity'], expected_visibility_true) + self.assertEquals(configurations['hive-interactive-env']['property_attributes']['llap_queue_capacity'], + expected_llap_queue_attributes_min_25_visible_true) @@ -3200,10 +3220,10 @@ class TestHDP25StackAdvisor(TestCase): } # expected vals. - yarn_cont_mb_visibility = {'minimum': '682', 'maximum': '682'} - llap_io_mem_size_visibility = {'minimum': '0', 'maximum': '341'} - llap_num_executors_visibility = {'minimum': '1', 'maximum': '1'} - num_llap_nodes_visibility = {'minimum': '1', 'maximum': '1'} + yarn_cont_mb_attributes = {'minimum': '682', 'maximum': '682'} + llap_io_mem_size_attributes = {'minimum': '0', 'maximum': '341'} + llap_num_executors_attributes = {'minimum': '1', 'maximum': '1'} + num_llap_nodes_attributes = {'minimum': '1', 'maximum': '1'} @@ -3211,21 +3231,22 @@ class TestHDP25StackAdvisor(TestCase): self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '682') - self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.yarn.container.mb'], yarn_cont_mb_visibility) + self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.yarn.container.mb'], yarn_cont_mb_attributes) self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors'], '1') - self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.num.executors'], llap_num_executors_visibility) + self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.num.executors'], llap_num_executors_attributes) self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size'], '341') - self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.io.memory.size'], llap_io_mem_size_visibility) + self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.io.memory.size'], llap_io_mem_size_attributes) self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes'], '1') - self.assertEquals(configurations['hive-interactive-env']['property_attributes']['num_llap_nodes'], num_llap_nodes_visibility) + self.assertEquals(configurations['hive-interactive-env']['property_attributes']['num_llap_nodes'], num_llap_nodes_attributes) self.assertEqual(configurations['hive-interactive-env']['properties']['llap_heap_size'], '272') self.assertEqual(configurations['hive-interactive-env']['properties']['slider_am_container_mb'], '341') - self.assertEquals(configurations['hive-interactive-env']['property_attributes']['llap_queue_capacity'], expected_visibility_true) + self.assertEquals(configurations['hive-interactive-env']['property_attributes']['llap_queue_capacity'], + expected_llap_queue_attributes_min_25_visible_true) @@ -3294,7 +3315,8 @@ class TestHDP25StackAdvisor(TestCase): self.assertEqual(configurations['hive-interactive-env']['properties']['slider_am_container_mb'], '341') - self.assertEquals(configurations['hive-interactive-env']['property_attributes']['llap_queue_capacity'], expected_visibility_true) + self.assertEquals(configurations['hive-interactive-env']['property_attributes']['llap_queue_capacity'], + expected_llap_queue_attributes_min_25_visible_true) @@ -3332,7 +3354,8 @@ class TestHDP25StackAdvisor(TestCase): self.assertEqual(configurations['hive-interactive-env']['properties']['slider_am_container_mb'], '682') - self.assertEquals(configurations['hive-interactive-env']['property_attributes']['llap_queue_capacity'], expected_visibility_true) + self.assertEquals(configurations['hive-interactive-env']['property_attributes']['llap_queue_capacity'], + expected_llap_queue_attributes_min_21_visible_true) @@ -3353,7 +3376,8 @@ class TestHDP25StackAdvisor(TestCase): expected_hive_interactive_site_empty['hive-interactive-site']['properties']) self.assertEquals(configurations['hive-interactive-env']['properties'], expected_hive_interactive_env_empty['hive-interactive-env']['properties']) - self.assertEquals(configurations['hive-interactive-env']['property_attributes']['llap_queue_capacity'], expected_visibility_true) + self.assertEquals(configurations['hive-interactive-env']['property_attributes']['llap_queue_capacity'], + expected_visibility_true)
