Repository: ambari Updated Branches: refs/heads/branch-2.4 bf0c5453b -> 7e0efa8e1
AMBARI-17103. Incorporating the inheritance logic for tez_hive2/tez-interactive-site.xml using tez/tez-site.xml. Also, removing 'hive.tez.java.opts' config from hive2/hive-site.xml as it would be got via inheritance from hive/hive-site.xml. Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7e0efa8e Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7e0efa8e Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7e0efa8e Branch: refs/heads/branch-2.4 Commit: 7e0efa8e1f87d4b0ad59ddaf9b0c7e1eacbef71b Parents: bf0c545 Author: Swapan Shridhar <[email protected]> Authored: Thu Jun 9 14:08:15 2016 -0700 Committer: Swapan Shridhar <[email protected]> Committed: Thu Jun 9 14:08:15 2016 -0700 ---------------------------------------------------------------------- .../package/scripts/hive_interactive.py | 15 +++++- .../HIVE/configuration/hive-interactive-env.xml | 4 ++ .../configuration/hive-interactive-site.xml | 17 ++++--- .../stacks/HDP/2.5/services/stack_advisor.py | 33 +++++++------ .../stacks/2.5/common/test_stack_advisor.py | 50 ++++++++++---------- 5 files changed, 69 insertions(+), 50 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/7e0efa8e/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py index 0dc5b36..7811352 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py @@ -102,10 +102,21 @@ def hive_interactive(name=None): ''' remove_atlas_hook_if_exists(merged_hive_interactive_site) - # Merge tez-interactive with tez-site + ''' + As tez_hive2/tez-site.xml only contains the new + the changed props compared to tez/tez-site.xml, + we need to merge tez/tez-site.xml and tez_hive2/tez-site.xml and store it in tez_hive2/tez-site.xml. + ''' + merged_tez_interactive_site = {} + if 'tez-site' in params.config['configurations']: + merged_tez_interactive_site.update(params.config['configurations']['tez-site']) + Logger.info("Retrieved 'tez/tez-site' for merging with 'tez_hive2/tez-interactive-site'.") + else: + Logger.error("Tez's 'tez-site' couldn't be retrieved from passed-in configurations.") + + merged_tez_interactive_site.update(params.config['configurations']['tez-interactive-site']) XmlConfig("tez-site.xml", conf_dir = params.tez_interactive_config_dir, - configurations = params.config['configurations']['tez-interactive-site'], + configurations = merged_tez_interactive_site, configuration_attributes=params.config['configuration_attributes']['tez-interactive-site'], owner = params.tez_interactive_user, group = params.user_group, http://git-wip-us.apache.org/repos/asf/ambari/blob/7e0efa8e/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml index c34351d..6f806d2 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml @@ -153,6 +153,10 @@ <type>hive-interactive-site</type> <name>hive.server2.tez.sessions.per.default.queue</name> </property> + <property> + <type>tez-site</type> + <name>tez.am.resource.memory.mb</name> + </property> </depends-on> <on-ambari-upgrade add="true"/> </property> http://git-wip-us.apache.org/repos/asf/ambari/blob/7e0efa8e/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml index 3227478..ac92b86 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml @@ -131,15 +131,6 @@ limitations under the License. <on-ambari-upgrade add="true"/> </property> <property> - <name>hive.tez.java.opts</name> - <value/> - <description>Java command line options for Tez. The -Xmx parameter value is generally 80% of hive.tez.container.size.</description> - <value-attributes> - <empty-value-valid>true</empty-value-valid> - </value-attributes> - <on-ambari-upgrade add="true"/> - </property> - <property> <name>hive.vectorized.execution.reduce.enabled</name> <value>true</value> <description> @@ -389,6 +380,10 @@ limitations under the License. <type>hive-interactive-site</type> <name>hive.server2.tez.sessions.per.default.queue</name> </property> + <property> + <type>tez-site</type> + <name>tez.am.resource.memory.mb</name> + </property> </depends-on> <on-ambari-upgrade add="true"/> </property> @@ -421,6 +416,10 @@ limitations under the License. <type>hive-interactive-site</type> <name>hive.server2.tez.sessions.per.default.queue</name> </property> + <property> + <type>tez-site</type> + <name>tez.am.resource.memory.mb</name> + </property> </depends-on> <on-ambari-upgrade add="true"/> </property> http://git-wip-us.apache.org/repos/asf/ambari/blob/7e0efa8e/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py index 6b5e2a1..df6c65c 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py @@ -816,27 +816,26 @@ class HDP25StackAdvisor(HDP24StackAdvisor): return yarn_nm_mem_in_mb """ - Gets Tez App Master container size (tez.am.resource.memory.mb). Takes into account if it has been calculated + Gets Tez App Master container size (tez.am.resource.memory.mb) from tez-site. Takes into account if it has been calculated as part of current Stack Advisor invocation. """ def get_tez_am_container_size(self, services, configurations): llap_daemon_container_size = None - # Check if 'tez.am.resource.memory.mb' is modified in current ST invocation. - if 'tez-interactive-site' in configurations and 'tez.am.resource.memory.mb' in configurations['tez-interactive-site']['properties']: - llap_daemon_container_size = float(configurations['tez-interactive-site']['properties']['tez.am.resource.memory.mb']) + if 'tez-site' in configurations and 'tez.am.resource.memory.mb' in configurations['tez-site']['properties']: + llap_daemon_container_size = float(configurations['tez-site']['properties']['tez.am.resource.memory.mb']) Logger.info("'tez.am.resource.memory.mb' read from configurations as : {0}".format(llap_daemon_container_size)) if not llap_daemon_container_size: # Check if 'tez.am.resource.memory.mb' is input in services array. - if self.HIVE_INTERACTIVE_SITE in services['configurations'] and \ - 'tez.am.resource.memory.mb' in services['configurations']['tez-interactive-site']['properties']: - llap_daemon_container_size = float(services['configurations']['tez-interactive-site']['properties']['tez.am.resource.memory.mb']) + if 'tez-site' in services['configurations'] and \ + 'tez.am.resource.memory.mb' in services['configurations']['tez-site']['properties']: + llap_daemon_container_size = float(services['configurations']['tez-site']['properties']['tez.am.resource.memory.mb']) Logger.info("'tez.am.resource.memory.mb' read from services as : {0}".format(llap_daemon_container_size)) if not llap_daemon_container_size: - raise Fail("Couldn't retrieve Hive Server interactive's 'tez.am.resource.memory.mb' config.") + raise Fail("Couldn't retrieve Hive Server's 'tez.am.resource.memory.mb' config.") assert (llap_daemon_container_size > 0), "'tez.am.resource.memory.mb' current value : {0}. " \ "Expected value : > 0".format(llap_daemon_container_size) @@ -865,9 +864,10 @@ class HDP25StackAdvisor(HDP24StackAdvisor): min_required_perc = min_required * 100 / total_cluster_cap Logger.info("Calculated min_llap_queue_perc_required_in_cluster : {0} and min_llap_queue_cap_required_in_cluster: {1} " - "using following : yarn_min_container_size : {2}, ""slider_am_size : {3}, hive_tez_container_size : {4}, " - "hive_am_container_size : {5}".format(min_required_perc, min_required, yarn_min_container_size, - slider_am_size, hive_tez_container_size, tez_am_container_size)) + "using following : yarn_min_container_size : {2}, slider_am_size : {3}, hive_tez_container_size : {4}, " + "hive_am_container_size : {5}, total_cluster_cap : {6}, yarn_rm_mem_in_mb : {7}" + "".format(min_required_perc, min_required, yarn_min_container_size, slider_am_size, hive_tez_container_size, + tez_am_container_size, total_cluster_cap, yarn_rm_mem_in_mb)) return int(math.ceil(min_required_perc)) """ @@ -903,9 +903,14 @@ class HDP25StackAdvisor(HDP24StackAdvisor): if 'llap_queue_capacity' in services['configurations']['hive-interactive-env']['properties']: llap_slider_cap_percentage = int( services['configurations']['hive-interactive-env']['properties']['llap_queue_capacity']) - llap_min_reqd_cap_percentage = self.min_llap_queue_perc_required_in_cluster(services, hosts, configurations) + llap_min_reqd_cap_percentage = math.ceil(self.min_llap_queue_perc_required_in_cluster(services, hosts, configurations)) + # Checks for upper bound for the minimum llap queue calculated capacity + if llap_min_reqd_cap_percentage > 100: + Logger.error("Minimum calculated 'llap' queue capacity ({0}%) is more than what the cluster has in totality. " + "Not creating/adjusting 'llap' queue. Increase the overall cluster capacity.".format(llap_min_reqd_cap_percentage)) + return if llap_slider_cap_percentage <= 0 or llap_slider_cap_percentage > 100: - Logger.info("Adjusting HIVE 'llap_queue_capacity' from {0}% to {1}%".format(llap_slider_cap_percentage, llap_min_reqd_cap_percentage)) + Logger.info("Adjusting HIVE 'llap_queue_capacity' from {0}% (invalid size) to {1}%".format(llap_slider_cap_percentage, llap_min_reqd_cap_percentage)) putHiveInteractiveEnvProperty('llap_queue_capacity', llap_min_reqd_cap_percentage) llap_slider_cap_percentage = llap_min_reqd_cap_percentage else: @@ -926,7 +931,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor): # Get 'llap' queue capacity currLlapQueueCap = -1 if 'yarn.scheduler.capacity.root.'+llap_queue_name+'.capacity' in cap_sched_config_keys: - currLlapQueueCap = int(capacity_scheduler_properties.get('yarn.scheduler.capacity.root.'+llap_queue_name+'.capacity')) + currLlapQueueCap = int(float(capacity_scheduler_properties.get('yarn.scheduler.capacity.root.'+llap_queue_name+'.capacity'))) if self.HIVE_INTERACTIVE_SITE in services['configurations'] and \ 'hive.llap.daemon.queue.name' in services['configurations'][self.HIVE_INTERACTIVE_SITE]['properties']: http://git-wip-us.apache.org/repos/asf/ambari/blob/7e0efa8e/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py index 22b3c52..5a54548 100644 --- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py @@ -146,7 +146,7 @@ class TestHDP25StackAdvisor(TestCase): # Expected capacity-scheduler with 'llap' (size:20) and 'default' queue at root level. self.expected_capacity_scheduler_llap_queue_size_20 = { "properties": { - "capacity-scheduler": 'yarn.scheduler.capacity.root.default.maximum-capacity=80\n' + "capacity-scheduler": 'yarn.scheduler.capacity.root.default.maximum-capacity=80.0\n' 'yarn.scheduler.capacity.root.accessible-node-labels=*\n' 'yarn.scheduler.capacity.root.capacity=100\n' 'yarn.scheduler.capacity.root.queues=default,llap\n' @@ -155,7 +155,7 @@ class TestHDP25StackAdvisor(TestCase): 'yarn.scheduler.capacity.root.default.state=RUNNING\n' 'yarn.scheduler.capacity.maximum-am-resource-percent=1\n' 'yarn.scheduler.capacity.root.default.acl_submit_applications=*\n' - 'yarn.scheduler.capacity.root.default.capacity=80\n' + 'yarn.scheduler.capacity.root.default.capacity=80.0\n' 'yarn.scheduler.capacity.root.acl_administer_queue=*\n' 'yarn.scheduler.capacity.node-locality-delay=40\n' 'yarn.scheduler.capacity.queue-mappings-override.enable=false\n' @@ -163,8 +163,8 @@ class TestHDP25StackAdvisor(TestCase): 'yarn.scheduler.capacity.root.llap.state=RUNNING\n' 'yarn.scheduler.capacity.root.llap.ordering-policy=fifo\n' 'yarn.scheduler.capacity.root.llap.minimum-user-limit-percent=100\n' - 'yarn.scheduler.capacity.root.llap.maximum-capacity=20\n' - 'yarn.scheduler.capacity.root.llap.capacity=20\n' + 'yarn.scheduler.capacity.root.llap.maximum-capacity=20.0\n' + 'yarn.scheduler.capacity.root.llap.capacity=20.0\n' 'yarn.scheduler.capacity.root.llap.acl_submit_applications=hive\n' 'yarn.scheduler.capacity.root.llap.acl_administer_queue=hive\n' 'yarn.scheduler.capacity.root.llap.maximum-am-resource-percent=1' @@ -351,7 +351,7 @@ class TestHDP25StackAdvisor(TestCase): } # Expected 'hive_interactive_env' with 'llap_queue_capacity' set to 20. - self.expected_llap_queue_capacity_20 = '20' + self.expected_llap_queue_capacity_20 = '20.0' # Expected 'hive_interactive_env' with 'llap_queue_capacity' set to 40. self.expected_llap_queue_capacity_40 = '40' @@ -614,7 +614,7 @@ class TestHDP25StackAdvisor(TestCase): 'hive.llap.daemon.queue.name':'default' } }, - "tez-interactive-site": { + "tez-site": { "properties": { "tez.am.resource.memory.mb": "341" } @@ -768,7 +768,7 @@ class TestHDP25StackAdvisor(TestCase): 'hive.llap.daemon.queue.name':'default' } }, - "tez-interactive-site": { + "tez-site": { "properties": { "tez.am.resource.memory.mb": "341" } @@ -920,7 +920,7 @@ class TestHDP25StackAdvisor(TestCase): 'hive.llap.daemon.queue.name':'default', } }, - "tez-interactive-site": { + "tez-site": { "properties": { "tez.am.resource.memory.mb": "341" } @@ -1072,7 +1072,7 @@ class TestHDP25StackAdvisor(TestCase): 'hive.llap.daemon.queue.name':'default', } }, - "tez-interactive-site": { + "tez-site": { "properties": { "tez.am.resource.memory.mb": "341" } @@ -1235,7 +1235,7 @@ class TestHDP25StackAdvisor(TestCase): 'hive.server2.tez.sessions.per.default.queue' : '1' } }, - "tez-interactive-site": { + "tez-site": { "properties": { "tez.am.resource.memory.mb": "341" } @@ -1392,7 +1392,7 @@ class TestHDP25StackAdvisor(TestCase): 'hive.llap.daemon.queue.name':'llap' } }, - "tez-interactive-site": { + "tez-site": { "properties": { "tez.am.resource.memory.mb": "341" } @@ -1556,7 +1556,7 @@ class TestHDP25StackAdvisor(TestCase): 'hive.llap.daemon.queue.name':'llap' } }, - "tez-interactive-site": { + "tez-site": { "properties": { "tez.am.resource.memory.mb": "341" } @@ -3236,7 +3236,7 @@ class TestHDP25StackAdvisor(TestCase): "yarn.nodemanager.resource.cpu-vcores": '1' } }, - "tez-interactive-site": { + "tez-site": { "properties": { "tez.am.resource.memory.mb": "512" } @@ -3421,7 +3421,7 @@ class TestHDP25StackAdvisor(TestCase): "yarn.nodemanager.resource.cpu-vcores": '1' } }, - "tez-interactive-site": { + "tez-site": { "properties": { "tez.am.resource.memory.mb": "341" } @@ -3614,7 +3614,7 @@ class TestHDP25StackAdvisor(TestCase): "yarn.nodemanager.resource.cpu-vcores": '1' } }, - "tez-interactive-site": { + "tez-site": { "properties": { "tez.am.resource.memory.mb": "1024" } @@ -3808,7 +3808,7 @@ class TestHDP25StackAdvisor(TestCase): "yarn.nodemanager.resource.cpu-vcores": '4' } }, - "tez-interactive-site": { + "tez-site": { "properties": { "tez.am.resource.memory.mb": "1024" } @@ -3997,7 +3997,7 @@ class TestHDP25StackAdvisor(TestCase): "yarn.nodemanager.resource.cpu-vcores": '3' } }, - "tez-interactive-site": { + "tez-site": { "properties": { "tez.am.resource.memory.mb": "1024" } @@ -4186,7 +4186,7 @@ class TestHDP25StackAdvisor(TestCase): "yarn.nodemanager.resource.cpu-vcores": '3' } }, - "tez-interactive-site": { + "tez-site": { "properties": { "tez.am.resource.memory.mb": "1024" } @@ -4377,7 +4377,7 @@ class TestHDP25StackAdvisor(TestCase): "yarn.nodemanager.resource.cpu-vcores": '4' } }, - "tez-interactive-site": { + "tez-site": { "properties": { "tez.am.resource.memory.mb": "1024" } @@ -4568,7 +4568,7 @@ class TestHDP25StackAdvisor(TestCase): "yarn.nodemanager.resource.cpu-vcores": '10' } }, - "tez-interactive-site": { + "tez-site": { "properties": { "tez.am.resource.memory.mb": "341" } @@ -4757,7 +4757,7 @@ class TestHDP25StackAdvisor(TestCase): "yarn.nodemanager.resource.cpu-vcores": '3' } }, - "tez-interactive-site": { + "tez-site": { "properties": { "tez.am.resource.memory.mb": "1024" } @@ -4933,7 +4933,7 @@ class TestHDP25StackAdvisor(TestCase): "yarn.nodemanager.resource.cpu-vcores": '3' } }, - "tez-interactive-site": { + "tez-site": { "properties": { "tez.am.resource.memory.mb": "1024" } @@ -5112,7 +5112,7 @@ class TestHDP25StackAdvisor(TestCase): "yarn.nodemanager.resource.cpu-vcores": '3' } }, - "tez-interactive-site": { + "tez-site": { "properties": { "tez.am.resource.memory.mb": "1024" } @@ -5465,7 +5465,7 @@ class TestHDP25StackAdvisor(TestCase): "yarn.nodemanager.resource.cpu-vcores": '3' } }, - "tez-interactive-site": { + "tez-site": { "properties": { "tez.am.resource.memory.mb": "1024", } @@ -5635,7 +5635,7 @@ class TestHDP25StackAdvisor(TestCase): "yarn.nodemanager.resource.cpu-vcores": '3' } }, - "tez-interactive-site": { + "tez-site": { "properties": { "tez.am.resource.memory.mb": "1024" }
