Repository: ambari Updated Branches: refs/heads/branch-feature-AMBARI-18901 548d4570a -> 512475f6f
AMBARI-19116. Hive2/HiveServer2 (HSI) heap size calculation to be based on LLAP concurrency. Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/512475f6 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/512475f6 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/512475f6 Branch: refs/heads/branch-feature-AMBARI-18901 Commit: 512475f6f42710bc85c031964a187f298308d9a6 Parents: 548d457 Author: Swapan Shridhar <[email protected]> Authored: Mon Dec 5 17:35:41 2016 -0800 Committer: Swapan Shridhar <[email protected]> Committed: Tue Dec 6 16:56:54 2016 -0800 ---------------------------------------------------------------------- .../0.12.0.2.0/package/scripts/params_linux.py | 1 + .../HIVE/configuration/hive-interactive-env.xml | 36 ++++++++++++++++- .../stacks/HDP/2.5/services/stack_advisor.py | 41 ++++++++++++++++---- 3 files changed, 70 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/512475f6/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py index 8df6ba7..781cf5d 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py @@ -588,6 +588,7 @@ if has_hive_interactive: hive_interactive_env_sh_template = config['configurations']['hive-interactive-env']['content'] hive_interactive_enabled = default('/configurations/hive-interactive-env/enable_hive_interactive', False) llap_app_java_opts = default('/configurations/hive-interactive-env/llap_java_opts', '-XX:+AlwaysPreTouch {% if java_version > 7 %}-XX:+UseG1GC -XX:TLABSize=8m -XX:+ResizeTLAB -XX:+UseNUMA -XX:+AggressiveOpts -XX:MetaspaceSize=1024m -XX:InitiatingHeapOccupancyPercent=80 -XX:MaxGCPauseMillis=200{% else %}-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC{% endif %}') + hive_interactive_heapsize = config['configurations']['hive-interactive-env']['hive_heapsize'] # Service check related if hive_transport_mode.lower() == "http": http://git-wip-us.apache.org/repos/asf/ambari/blob/512475f6/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml index 6e93f58..09e1741 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml @@ -212,6 +212,40 @@ <display-name>LLAP app java opts</display-name> <on-ambari-upgrade add="true"/> </property> + <property> + <name>hive_heapsize</name> + <value>512</value> + <description>Hive Java heap size</description> + <display-name>HiveServer Interactive Heap Size</display-name> + <value-attributes> + <type>int</type> + <minimum>512</minimum> + <unit>MB</unit> + </value-attributes> + <on-ambari-upgrade add="true"/> + <depends-on> + <property> + <type>hive-interactive-env</type> + <name>enable_hive_interactive</name> + </property> + <property> + <type>hive-interactive-env</type> + <name>num_llap_nodes</name> + </property> + <property> + <type>hive-interactive-site</type> + <name>hive.llap.daemon.queue.name</name> + </property> + <property> + <type>capacity-scheduler</type> + <name>yarn.scheduler.capacity.root.queues</name> + </property> + <property> + <type>hive-interactive-site</type> + <name>hive.server2.tez.sessions.per.default.queue</name> + </property> + </depends-on> + </property> <!-- hive-env.sh --> @@ -233,7 +267,7 @@ if [ "$SERVICE" = "metastore" ]; then export HADOOP_HEAPSIZE={{hive_metastore_heapsize}} # Setting for HiveMetastore else - export HADOOP_HEAPSIZE={{hive_heapsize}} # Setting for HiveServer2 and Client + export HADOOP_HEAPSIZE={{hive_interactive_heapsize}} # Setting for HiveServer2 and Client fi export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Xmx${HADOOP_HEAPSIZE}m" http://git-wip-us.apache.org/repos/asf/ambari/blob/512475f6/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py index df2faeb..edd7af1 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py @@ -820,16 +820,26 @@ class HDP25StackAdvisor(HDP24StackAdvisor): # Get all leaf queues. leafQueueNames = self.getAllYarnLeafQueues(capacity_scheduler_properties) Logger.info("YARN leaf Queues = {0}".format(leafQueueNames)) + if len(leafQueueNames) == 0: + raise Fail("Queue(s) couldn't be retrieved from capacity-scheduler.") # Check if it's 1st invocation after enabling Hive Server Interactive (config: enable_hive_interactive). changed_configs_has_enable_hive_int = self.are_config_props_in_changed_configs(services, "hive-interactive-env", set(['enable_hive_interactive']), False) - llap_named_queue_selected_in_curr_invocation = False - if changed_configs_has_enable_hive_int and services['configurations']['hive-interactive-env']['properties']['enable_hive_interactive']: - llap_named_queue_selected_in_curr_invocation = True - putHiveInteractiveSiteProperty('hive.llap.daemon.queue.name', llap_queue_name) - putHiveInteractiveSiteProperty('hive.server2.tez.default.queues', llap_queue_name) - Logger.info("'hive.llap.daemon.queue.name' and 'hive.server2.tez.default.queues' values set as : {0}".format(llap_queue_name)) + llap_named_queue_selected_in_curr_invocation = None + if changed_configs_has_enable_hive_int \ + and services['configurations']['hive-interactive-env']['properties']['enable_hive_interactive']: + if (len(leafQueueNames) == 1 or (len(leafQueueNames) == 2 and llap_queue_name in leafQueueNames)): + llap_named_queue_selected_in_curr_invocation = True + putHiveInteractiveSiteProperty('hive.llap.daemon.queue.name', llap_queue_name) + putHiveInteractiveSiteProperty('hive.server2.tez.default.queues', llap_queue_name) + Logger.info("'hive.llap.daemon.queue.name' and 'hive.server2.tez.default.queues' values set as : {0}".format(llap_queue_name)) + else: + first_leaf_queue = list(leafQueueNames)[0] # 1st invocation, pick the 1st leaf queue and set it as selected. + putHiveInteractiveSiteProperty('hive.llap.daemon.queue.name', first_leaf_queue) + putHiveInteractiveSiteProperty('hive.server2.tez.default.queues', first_leaf_queue) + llap_named_queue_selected_in_curr_invocation = False + Logger.info("'hive.llap.daemon.queue.name' and 'hive.server2.tez.default.queues' values set as : {0}".format(first_leaf_queue)) Logger.info("llap_named_queue_selected_in_curr_invocation = {0}".format(llap_named_queue_selected_in_curr_invocation)) if (len(leafQueueNames) == 2 and (llap_daemon_selected_queue_name != None and llap_daemon_selected_queue_name == llap_queue_name) or \ @@ -1078,6 +1088,19 @@ class HDP25StackAdvisor(HDP24StackAdvisor): llap_xmx = max(total_mem_for_executors_per_node * 0.8, total_mem_for_executors_per_node - self.get_llap_headroom_space(services, configurations)) Logger.info("Calculated llap_app_heap_size : {0}, using following : total_mem_for_executors : {1}".format(llap_xmx, total_mem_for_executors_per_node)) + # Calculate 'hive_heapsize' for Hive2/HiveServer2 (HSI) + hive_server_interactive_heapsize = None + hive_server_interactive_hosts = self.getHostsWithComponent("HIVE", "HIVE_SERVER_INTERACTIVE", services, hosts) + if hive_server_interactive_hosts is None: + # If its None, read the base service HDFS's DATANODE node memory, as are host are considered homogenous. + hive_server_interactive_hosts = self.getHostsWithComponent("HDFS", "DATANODE", services, hosts) + if hive_server_interactive_hosts is not None and len(hive_server_interactive_hosts) > 0: + host_mem = long(hive_server_interactive_hosts[0]["Hosts"]["total_mem"]) + hive_server_interactive_heapsize = min(max(2048.0, 400.0*llap_concurrency), 3.0/8 * host_mem) + Logger.info("Calculated 'hive_server_interactive_heapsize' : {0}, using following : llap_concurrency : {1}, host_mem : " + "{2}".format(hive_server_interactive_heapsize, llap_concurrency, host_mem)) + + Logger.info("Updating the calculations....") # Done with calculations, updating calculated configs. @@ -1149,6 +1172,10 @@ class HDP25StackAdvisor(HDP24StackAdvisor): if cache_mem_per_node >= 64: llap_io_enabled = 'true' + if hive_server_interactive_heapsize != None: + putHiveInteractiveEnvProperty("hive_heapsize", hive_server_interactive_heapsize) + Logger.info("Hive2 config 'hive_heapsize' updated. Current : {0}".format(hive_server_interactive_heapsize)) + putHiveInteractiveSiteProperty('hive.llap.io.enabled', llap_io_enabled) Logger.info("Hive2 config 'hive.llap.io.enabled' updated to '{0}' as part of " "'hive.llap.io.memory.size' calculation.".format(llap_io_enabled)) @@ -1830,7 +1857,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor): Logger.info("AM percent key got for '{0}' queue is : '{1}'".format(llap_daemon_selected_queue_name, llap_selected_queue_am_percent_key)) break; if llap_selected_queue_am_percent_key is None: - Logger.info("Returning default AM percent value : '0.1' for queue : {1}".format(llap_daemon_selected_queue_name)) + Logger.info("Returning default AM percent value : '0.1' for queue : {0}".format(llap_daemon_selected_queue_name)) return 0.1 # Default value to use if we couldn't retrieve queue's corresponding AM Percent key. else: llap_selected_queue_am_percent = capacity_scheduler_properties.get(llap_selected_queue_am_percent_key)
