Repository: ambari Updated Branches: refs/heads/trunk 7e0097501 -> c46ccc51b
AMBARI-19616. Hive Server Interactive : (1). Set 'metaspacesize' based on LLAP daemon size and (2). Fix for using --slider-placement argument only for HDP 2.6 and onwards. Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c46ccc51 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c46ccc51 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c46ccc51 Branch: refs/heads/trunk Commit: c46ccc51b3fd06acf3a830f5306a482338ef5617 Parents: 7e00975 Author: Swapan Shridhar <[email protected]> Authored: Fri Jan 20 10:45:15 2017 -0800 Committer: Swapan Shridhar <[email protected]> Committed: Fri Jan 20 10:45:15 2017 -0800 ---------------------------------------------------------------------- .../libraries/functions/constants.py | 1 + .../package/scripts/hive_server_interactive.py | 39 ++++++++++++-------- .../0.12.0.2.0/package/scripts/params_linux.py | 3 +- .../HDP/2.0.6/properties/stack_features.json | 5 +++ .../HIVE/configuration/hive-interactive-env.xml | 2 +- 5 files changed, 33 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c46ccc51/ambari-common/src/main/python/resource_management/libraries/functions/constants.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py index 6895e34..02ce194 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py @@ -114,3 +114,4 @@ class StackFeature: CORE_SITE_FOR_RANGER_PLUGINS_SUPPORT='core_site_for_ranger_plugins' ATLAS_INSTALL_HOOK_PACKAGE_SUPPORT="atlas_install_hook_package_support" ATLAS_HDFS_SITE_ON_NAMENODE_HA='atlas_hdfs_site_on_namenode_ha' + HIVE_INTERACTIVE_GA_SUPPORT='hive_interactive_ga' http://git-wip-us.apache.org/repos/asf/ambari/blob/c46ccc51/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py index d58f93a..1b6b3f2 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py @@ -273,25 +273,27 @@ class HiveServerInteractiveDefault(HiveServerInteractive): unique_name = "llap-slider%s" % datetime.utcnow().strftime('%Y-%m-%d_%H-%M-%S') - # Figure out the Slider Anti-affinity to be used. - # YARN does not support anti-affinity, and therefore Slider implements AA by the means of exclusion lists, i.e, it - # starts containers one by one and excludes the nodes it gets (adding a delay of ~2sec./machine). When the LLAP - # container memory size configuration is more than half of YARN node memory, AA is implicit and should be avoided. - slider_placement = 4 - if long(params.llap_daemon_container_size) > (0.5 * long(params.yarn_nm_mem)): - slider_placement = 0 - Logger.info("Setting slider_placement : 0, as llap_daemon_container_size : {0} > 0.5 * " - "YARN NodeManager Memory({1})".format(params.llap_daemon_container_size, params.yarn_nm_mem)) - else: - Logger.info("Setting slider_placement: 4, as llap_daemon_container_size : {0} <= 0.5 * " - "YARN NodeManager Memory({1})".format(params.llap_daemon_container_size, params.yarn_nm_mem)) - - cmd = format("{stack_root}/current/hive-server2-hive2/bin/hive --service llap --instances {params.num_llap_nodes}" " --slider-am-container-mb {params.slider_am_container_mb} --size {params.llap_daemon_container_size}m" " --cache {params.hive_llap_io_mem_size}m --xmx {params.llap_heap_size}m --loglevel {params.llap_log_level}" - " --slider-placement {slider_placement} --output {LLAP_PACKAGE_CREATION_PATH}/{unique_name}" " {params.llap_extra_slider_opts} --skiphadoopversion --skiphbasecp --output {LLAP_PACKAGE_CREATION_PATH}/{unique_name}") + + # '--slider-placement' param is supported from HDP Hive GA version. + if params.stack_supports_hive_interactive_ga: + # Figure out the Slider Anti-affinity to be used. + # YARN does not support anti-affinity, and therefore Slider implements AA by the means of exclusion lists, i.e, it + # starts containers one by one and excludes the nodes it gets (adding a delay of ~2sec./machine). When the LLAP + # container memory size configuration is more than half of YARN node memory, AA is implicit and should be avoided. + slider_placement = 4 + if long(params.llap_daemon_container_size) > (0.5 * long(params.yarn_nm_mem)): + slider_placement = 0 + Logger.info("Setting slider_placement : 0, as llap_daemon_container_size : {0} > 0.5 * " + "YARN NodeManager Memory({1})".format(params.llap_daemon_container_size, params.yarn_nm_mem)) + else: + Logger.info("Setting slider_placement: 4, as llap_daemon_container_size : {0} <= 0.5 * " + "YARN NodeManager Memory({1})".format(params.llap_daemon_container_size, params.yarn_nm_mem)) + cmd += format(" --slider-placement {slider_placement}") + if params.security_enabled: llap_keytab_splits = params.hive_llap_keytab_file.split("/") Logger.debug("llap_keytab_splits : {0}".format(llap_keytab_splits)) @@ -305,6 +307,13 @@ class HiveServerInteractiveDefault(HiveServerInteractive): # Append args. llap_java_args = InlineTemplate(params.llap_app_java_opts).get_content() cmd += format(" --args \" {llap_java_args}\"") + # Append metaspace size to args. + if params.java_version > 7 and params.llap_daemon_container_size > 4096: + if params.llap_daemon_container_size <= 32768: + metaspaceSize = "256m" + else: + metaspaceSize = "1024m" + cmd = cmd[:-1] + " -XX:MetaspaceSize="+metaspaceSize+ "\"" run_file_path = None try: http://git-wip-us.apache.org/repos/asf/ambari/blob/c46ccc51/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py index d0d8d1c..e9e6337 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py @@ -92,6 +92,7 @@ stack_supports_ranger_kerberos = check_stack_feature(StackFeature.RANGER_KERBERO stack_supports_ranger_audit_db = check_stack_feature(StackFeature.RANGER_AUDIT_DB_SUPPORT, version_for_stack_feature_checks) stack_supports_ranger_hive_jdbc_url_change = check_stack_feature(StackFeature.RANGER_HIVE_PLUGIN_JDBC_URL, version_for_stack_feature_checks) stack_supports_atlas_hook_for_hive_interactive = check_stack_feature(StackFeature.HIVE_INTERACTIVE_ATLAS_HOOK_REQUIRED, version_for_stack_feature_checks) +stack_supports_hive_interactive_ga = check_stack_feature(StackFeature.HIVE_INTERACTIVE_GA_SUPPORT, version_for_stack_feature_checks) # component ROLE directory (like hive-metastore or hive-server2-hive2) component_directory = status_params.component_directory @@ -608,7 +609,7 @@ if has_hive_interactive: start_hiveserver2_interactive_path = format("{tmp_dir}/start_hiveserver2_interactive_script") hive_interactive_env_sh_template = config['configurations']['hive-interactive-env']['content'] hive_interactive_enabled = default('/configurations/hive-interactive-env/enable_hive_interactive', False) - llap_app_java_opts = default('/configurations/hive-interactive-env/llap_java_opts', '-XX:+AlwaysPreTouch {% if java_version > 7 %}-XX:+UseG1GC -XX:TLABSize=8m -XX:+ResizeTLAB -XX:+UseNUMA -XX:+AggressiveOpts -XX:MetaspaceSize=1024m -XX:InitiatingHeapOccupancyPercent=80 -XX:MaxGCPauseMillis=200{% else %}-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC{% endif %}') + llap_app_java_opts = default('/configurations/hive-interactive-env/llap_java_opts', '-XX:+AlwaysPreTouch {% if java_version > 7 %}-XX:+UseG1GC -XX:TLABSize=8m -XX:+ResizeTLAB -XX:+UseNUMA -XX:+AggressiveOpts -XX:InitiatingHeapOccupancyPercent=80 -XX:MaxGCPauseMillis=200{% else %}-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC{% endif %}') hive_interactive_heapsize = hive_heapsize llap_app_name = config['configurations']['hive-interactive-env']['llap_app_name'] # Ambari upgrade may not add this config as it will force restart of HSI (stack upgrade should) http://git-wip-us.apache.org/repos/asf/ambari/blob/c46ccc51/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json index 6801d5a..e4a499b 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json @@ -372,6 +372,11 @@ "name": "atlas_hdfs_site_on_namenode_ha", "description": "Need to create hdfs-site under atlas-conf dir when Namenode-HA is enabled.", "min_version": "2.6.0.0" + }, + { + "name": "hive_interactive_ga", + "description": "Hive Interactive GA support", + "min_version": "2.6.0.0" } ] } http://git-wip-us.apache.org/repos/asf/ambari/blob/c46ccc51/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-env.xml index 6a68335..0c9ce62 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-env.xml @@ -44,7 +44,7 @@ </property> <property> <name>llap_java_opts</name> - <value>-XX:+AlwaysPreTouch {% if java_version > 7 %}-XX:+UseG1GC -XX:TLABSize=8m -XX:+ResizeTLAB -XX:+UseNUMA -XX:+AggressiveOpts -XX:MetaspaceSize=1024m -XX:InitiatingHeapOccupancyPercent=80 -XX:MaxGCPauseMillis=200{% else %}-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC{% endif %}{{heap_dump_opts}}</value> + <value>-XX:+AlwaysPreTouch {% if java_version > 7 %}-XX:+UseG1GC -XX:TLABSize=8m -XX:+ResizeTLAB -XX:+UseNUMA -XX:+AggressiveOpts -XX:InitiatingHeapOccupancyPercent=80 -XX:MaxGCPauseMillis=200{% else %}-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC{% endif %}{{heap_dump_opts}}</value> <description>Java opts for llap application</description> <display-name>LLAP app java opts</display-name> <on-ambari-upgrade add="false"/>
