AMBARI-18901. LLAP integration enhancements (Swapan Sridhar via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8552345d Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8552345d Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8552345d Branch: refs/heads/branch-2.5 Commit: 8552345deee7e6d366571a4c926a3093f7a7492a Parents: 61477b9 Author: Sumit Mohanty <[email protected]> Authored: Sun Dec 11 11:15:46 2016 -0800 Committer: Sumit Mohanty <[email protected]> Committed: Sun Dec 11 11:15:46 2016 -0800 ---------------------------------------------------------------------- .../server/configuration/Configuration.java | 1 - .../server/upgrade/UpgradeCatalog250.java | 74 +- .../package/scripts/hive_interactive.py | 12 + .../package/scripts/hive_server_interactive.py | 48 +- .../0.12.0.2.0/package/scripts/params_linux.py | 9 + .../0.4.0.2.1/package/scripts/params_linux.py | 7 + .../stacks/HDP/2.2/services/stack_advisor.py | 15 +- .../stacks/HDP/2.3/services/stack_advisor.py | 2 + .../HIVE/configuration/hive-interactive-env.xml | 80 +- .../configuration/hive-interactive-site.xml | 68 +- .../HIVE/configuration/tez-interactive-site.xml | 71 +- .../HDP/2.5/services/HIVE/themes/theme.json | 27 +- .../YARN/configuration/capacity-scheduler.xml | 2 +- .../stacks/HDP/2.5/services/stack_advisor.py | 700 ++-- .../services/HIVE/configuration/hive-env.xml | 119 + .../HIVE/configuration/hive-interactive-env.xml | 87 + .../2.6/services/TEZ/configuration/tez-env.xml | 56 + .../2.6/services/TEZ/configuration/tez-site.xml | 28 + .../stacks/HDP/2.6/services/stack_advisor.py | 25 +- .../server/upgrade/UpgradeCatalog250Test.java | 71 +- .../stacks/2.2/common/test_stack_advisor.py | 7 +- .../stacks/2.5/HIVE/test_hive_server_int.py | 2 + .../stacks/2.5/common/test_stack_advisor.py | 3280 ++---------------- .../app/mixins/common/configs/configs_saver.js | 2 +- 24 files changed, 1451 insertions(+), 3342 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/8552345d/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java index 8f10023..7ae4353 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java @@ -2450,7 +2450,6 @@ public class Configuration { "log4j.monitor.delay", TimeUnit.MINUTES.toMillis(5)); /** -<<<<<<< a5fdae802210ae1f8d4fed2234f1651cbe61c2b5 * Indicates whether parallel topology task creation is enabled for blueprint cluster provisioning. * Defaults to <code>false</code>. * @see #TOPOLOGY_TASK_PARALLEL_CREATION_THREAD_COUNT http://git-wip-us.apache.org/repos/asf/ambari/blob/8552345d/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java index ac97987..3f35adc 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java @@ -17,14 +17,8 @@ */ package org.apache.ambari.server.upgrade; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; - +import com.google.inject.Inject; +import com.google.inject.Injector; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.actionmanager.CommandExecutionType; import org.apache.ambari.server.controller.AmbariManagementController; @@ -38,8 +32,13 @@ import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.inject.Inject; -import com.google.inject.Injector; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; /** * Upgrade catalog for version 2.5.0. @@ -135,6 +134,8 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog { addNewConfigurationsFromXml(); updateAMSConfigs(); updateKafkaConfigs(); + updateHIVEInteractiveConfigs(); + updateTEZInteractiveConfigs(); updateHiveLlapConfigs(); updateTablesForZeppelinViewRemoval(); updateAtlasConfigs(); @@ -357,5 +358,56 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog { } } } -} + /** + * Updates Hive Interactive's config in hive-interactive-site. + * + * @throws AmbariException + */ + protected void updateHIVEInteractiveConfigs() throws AmbariException { + AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class); + Clusters clusters = ambariManagementController.getClusters(); + if (clusters != null) { + Map<String, Cluster> clusterMap = clusters.getClusters(); + + if (clusterMap != null && !clusterMap.isEmpty()) { + for (final Cluster cluster : clusterMap.values()) { + Config hiveInteractiveSite = cluster.getDesiredConfigByType("hive-interactive-site"); + if (hiveInteractiveSite != null) { + updateConfigurationProperties("hive-interactive-site", Collections.singletonMap("hive.tez.container.size", + "SET_ON_FIRST_INVOCATION"), true, true); + + updateConfigurationProperties("hive-interactive-site", Collections.singletonMap("hive.auto.convert.join.noconditionaltask.size", + "1000000000"), true, true); + } + } + } + } + } + + /** + * Updates Tez for Hive2 Interactive's config in tez-interactive-site. + * + * @throws AmbariException + */ + protected void updateTEZInteractiveConfigs() throws AmbariException { + AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class); + Clusters clusters = ambariManagementController.getClusters(); + if (clusters != null) { + Map<String, Cluster> clusterMap = clusters.getClusters(); + + if (clusterMap != null && !clusterMap.isEmpty()) { + for (final Cluster cluster : clusterMap.values()) { + Config tezInteractiveSite = cluster.getDesiredConfigByType("tez-interactive-site"); + if (tezInteractiveSite != null) { + + updateConfigurationProperties("tez-interactive-site", Collections.singletonMap("tez.runtime.io.sort.mb", "512"), true, true); + + updateConfigurationProperties("tez-interactive-site", Collections.singletonMap("tez.runtime.unordered.output.buffer.size-mb", + "100"), true, true); + } + } + } + } + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/8552345d/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py index fa6d6a0..298db2a 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py @@ -58,6 +58,7 @@ Sets up the configs, jdbc connection and tarball copy to HDFS for Hive Server In @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) def hive_interactive(name=None): import params + MB_TO_BYTES = 1048576 # Create Hive User Dir params.HdfsResource(params.hive_hdfs_user_dir, @@ -108,6 +109,17 @@ def hive_interactive(name=None): del merged_hive_interactive_site[item] ''' + Config 'hive.llap.io.memory.size' calculated value in stack_advisor is in MB as of now. We need to + convert it to bytes before we write it down to config file. + ''' + if 'hive.llap.io.memory.size' in merged_hive_interactive_site.keys(): + hive_llap_io_mem_size_in_mb = merged_hive_interactive_site.get("hive.llap.io.memory.size") + hive_llap_io_mem_size_in_bytes = long(hive_llap_io_mem_size_in_mb) * MB_TO_BYTES + merged_hive_interactive_site['hive.llap.io.memory.size'] = hive_llap_io_mem_size_in_bytes + Logger.info("Converted 'hive.llap.io.memory.size' value from '{0} MB' to '{1} Bytes' before writing " + "it to config file.".format(hive_llap_io_mem_size_in_mb, hive_llap_io_mem_size_in_bytes)) + + ''' Hive2 doesn't have support for Atlas, we need to remove the Hook 'org.apache.atlas.hive.hook.HiveHook', which would have come in config 'hive.exec.post.hooks' during the site merge logic, if Atlas is installed. ''' http://git-wip-us.apache.org/repos/asf/ambari/blob/8552345d/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py index 7335637..055c690 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py @@ -269,15 +269,29 @@ class HiveServerInteractiveDefault(HiveServerInteractive): Logger.info("LLAP app '{0}' is not running. llap will be started.".format(LLAP_APP_NAME)) pass + # Call for cleaning up the earlier run(s) LLAP package folders. + self._cleanup_past_llap_package_dirs() + Logger.info("Starting LLAP") LLAP_PACKAGE_CREATION_PATH = Script.get_tmp_dir() unique_name = "llap-slider%s" % datetime.utcnow().strftime('%Y-%m-%d_%H-%M-%S') + # Figure out the Slider Anti-affinity to be used. + # YARN does not support anti-affinity, and therefore Slider implements AA by the means of exclusion lists, i.e, it + # starts containers one by one and excludes the nodes it gets (adding a delay of ~2sec./machine). When the LLAP + # container memory size configuration is more than half of YARN node memory, AA is implicit and should be avoided. + slider_placement = 4 + if long(params.llap_daemon_container_size) > (0.5 * long(params.yarn_nm_mem)): + slider_placement = 0 + Logger.info("Setting slider_placement : 0, as llap_daemon_container_size : {0} > 0.5 * " + "YARN NodeManager Memory({1})".format(params.llap_daemon_container_size, params.yarn_nm_mem)) + cmd = format("{stack_root}/current/hive-server2-hive2/bin/hive --service llap --instances {params.num_llap_nodes}" " --slider-am-container-mb {params.slider_am_container_mb} --size {params.llap_daemon_container_size}m" " --cache {params.hive_llap_io_mem_size}m --xmx {params.llap_heap_size}m --loglevel {params.llap_log_level}" - " {params.llap_extra_slider_opts} --output {LLAP_PACKAGE_CREATION_PATH}/{unique_name}") + " --slider-placement {slider_placement} --output {LLAP_PACKAGE_CREATION_PATH}/{unique_name}" + " {params.llap_extra_slider_opts} --skiphadoopversion --skiphbasecp --output {LLAP_PACKAGE_CREATION_PATH}/{unique_name}") if params.security_enabled: llap_keytab_splits = params.hive_llap_keytab_file.split("/") Logger.debug("llap_keytab_splits : {0}".format(llap_keytab_splits)) @@ -342,6 +356,38 @@ class HiveServerInteractiveDefault(HiveServerInteractive): raise """ + Checks and deletes previous run 'LLAP package' folders, ignoring three latest packages. + Last three are are ignore for debugging/reference purposes. + Helps in keeping check on disk space used. + """ + def _cleanup_past_llap_package_dirs(self): + try: + import params + Logger.info("Determining previous run 'LLAP package' folder(s) to be deleted ....") + llap_package_folder_name_prefix = "llap-slider" # Package name is like : llap-sliderYYYY-MM-DD-HH:MM:SS + num_folders_to_retain = 3 # Hardcoding it as of now, as no considerable use was found to provide an env param. + file_names = [dir_name for dir_name in os.listdir(Script.get_tmp_dir()) + if dir_name.startswith(llap_package_folder_name_prefix)] + + file_names.sort() + del file_names[-num_folders_to_retain:] # Ignore 'num_folders_to_retain' latest package folders. + Logger.info("Previous run 'LLAP package' folder(s) to be deleted = {0}".format(file_names)) + + if file_names: + for path in file_names: + abs_path = Script.get_tmp_dir()+"/"+path + if os.path.isdir(abs_path): + shutil.rmtree(abs_path) + Logger.info("Deleted previous run 'LLAP package' folder : {0}".format(abs_path)) + else: + Logger.info("No '{0}*' folder deleted.".format(llap_package_folder_name_prefix)) + except Exception as e: + Logger.info("Exception while doing cleanup for past 'LLAP package(s)'.") + traceback.print_exc() + + + + """ Does kinit and copies keytab for Hive/LLAP to HDFS. """ def setup_security(self): http://git-wip-us.apache.org/repos/asf/ambari/blob/8552345d/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py index 496102e..e9436c1 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py @@ -134,6 +134,13 @@ if check_stack_feature(StackFeature.HIVE_SERVER_INTERACTIVE, version_for_stack_f hive_interactive_bin = format('{stack_root}/current/{component_directory_interactive}/bin') hive_interactive_lib = format('{stack_root}/current/{component_directory_interactive}/lib') +# Heap dump related +heap_dump_enabled = default('/configurations/hive-env/enable_heap_dump', None) +heap_dump_opts = "" # Empty if 'heap_dump_enabled' is False. +if heap_dump_enabled: + heap_dump_path = default('/configurations/hive-env/heap_dump_location', "/tmp") + heap_dump_opts = " -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath="+heap_dump_path + # Hive Interactive related paths hive_interactive_var_lib = '/var/lib/hive2' @@ -587,6 +594,7 @@ if has_hive_interactive: hive_interactive_env_sh_template = config['configurations']['hive-interactive-env']['content'] hive_interactive_enabled = default('/configurations/hive-interactive-env/enable_hive_interactive', False) llap_app_java_opts = default('/configurations/hive-interactive-env/llap_java_opts', '-XX:+AlwaysPreTouch {% if java_version > 7 %}-XX:+UseG1GC -XX:TLABSize=8m -XX:+ResizeTLAB -XX:+UseNUMA -XX:+AggressiveOpts -XX:MetaspaceSize=1024m -XX:InitiatingHeapOccupancyPercent=80 -XX:MaxGCPauseMillis=200{% else %}-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC{% endif %}') + hive_interactive_heapsize = config['configurations']['hive-interactive-env']['hive_heapsize'] # Service check related if hive_transport_mode.lower() == "http": @@ -598,6 +606,7 @@ if has_hive_interactive: tez_interactive_user = config['configurations']['tez-env']['tez_user'] num_retries_for_checking_llap_status = default('/configurations/hive-interactive-env/num_retries_for_checking_llap_status', 10) # Used in LLAP slider package creation + yarn_nm_mem = config['configurations']['yarn-site']['yarn.nodemanager.resource.memory-mb'] num_llap_nodes = config['configurations']['hive-interactive-env']['num_llap_nodes'] llap_daemon_container_size = config['configurations']['hive-interactive-site']['hive.llap.daemon.yarn.container.mb'] llap_log_level = config['configurations']['hive-interactive-env']['llap_log_level'] http://git-wip-us.apache.org/repos/asf/ambari/blob/8552345d/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_linux.py index 6587151..0322406 100644 --- a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_linux.py @@ -63,6 +63,13 @@ if stack_version_formatted and check_stack_feature(StackFeature.CONFIG_VERSIONIN config_path = os.path.join(stack_root, "current/tez-client/conf") config_dir = os.path.realpath(config_path) +# Heap dump related +heap_dump_enabled = default('/configurations/tez-env/enable_heap_dump', None) +heap_dump_opts = "" # Empty if 'heap_dump_enabled' is False. +if heap_dump_enabled: + heap_dump_path = default('/configurations/tez-env/heap_dump_location', "/tmp") + heap_dump_opts = " -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath="+heap_dump_path + kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) security_enabled = config['configurations']['cluster-env']['security_enabled'] smokeuser = config['configurations']['cluster-env']['smokeuser'] http://git-wip-us.apache.org/repos/asf/ambari/blob/8552345d/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py index 4802193..3789a16 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py @@ -307,6 +307,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor): putWebhcatSiteProperty = self.putProperty(configurations, "webhcat-site", services) putHiveSitePropertyAttribute = self.putPropertyAttribute(configurations, "hive-site") putHiveEnvPropertyAttributes = self.putPropertyAttribute(configurations, "hive-env") + putHiveServerPropertyAttributes = self.putPropertyAttribute(configurations, "hiveserver2-site") servicesList = [service["StackServices"]["service_name"] for service in services["services"]] # Storage @@ -527,7 +528,8 @@ class HDP22StackAdvisor(HDP21StackAdvisor): putHiveServerProperty("hive.security.authorization.enabled", "true") putHiveServerProperty("hive.security.authorization.manager", "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory") putHiveServerProperty("hive.security.authenticator.manager", "org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator") - putHiveServerProperty("hive.conf.restricted.list", "hive.security.authenticator.manager,hive.security.authorization.manager,hive.users.in.admin.role") + putHiveServerProperty("hive.conf.restricted.list", "hive.security.authenticator.manager,hive.security.authorization.manager,hive.security.metastore.authorization.manager," + "hive.security.metastore.authenticator.manager,hive.users.in.admin.role,hive.server2.xsrf.filter.enabled,hive.security.authorization.enabled") putHiveSiteProperty("hive.security.authorization.manager", "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdConfOnlyAuthorizerFactory") if sqlstdauth_class not in auth_manager_values: auth_manager_values.append(sqlstdauth_class) @@ -543,7 +545,16 @@ class HDP22StackAdvisor(HDP21StackAdvisor): putHiveServerProperty("hive.security.authorization.enabled", "true") putHiveServerProperty("hive.security.authorization.manager", "com.xasecure.authorization.hive.authorizer.XaSecureHiveAuthorizerFactory") putHiveServerProperty("hive.security.authenticator.manager", "org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator") - putHiveServerProperty("hive.conf.restricted.list", "hive.security.authorization.enabled,hive.security.authorization.manager,hive.security.authenticator.manager") + putHiveServerProperty("hive.conf.restricted.list", "hive.security.authenticator.manager,hive.security.authorization.manager,hive.security.metastore.authorization.manager," + "hive.security.metastore.authenticator.manager,hive.users.in.admin.role,hive.server2.xsrf.filter.enabled,hive.security.authorization.enabled") + + # hive_security_authorization == 'None' + if str(configurations["hive-env"]["properties"]["hive_security_authorization"]).lower() == "None": + putHiveSiteProperty("hive.server2.enable.doAs", "true") + putHiveServerProperty("hive.security.authorization.enabled", "false") + putHiveServerPropertyAttributes("hive.security.authorization.manager", 'delete', 'true') + putHiveServerPropertyAttributes("hive.security.authenticator.manager", 'delete', 'true') + putHiveServerPropertyAttributes("hive.conf.restricted.list", 'delete', 'true') putHiveSiteProperty("hive.server2.use.SSL", "false") http://git-wip-us.apache.org/repos/asf/ambari/blob/8552345d/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py index 3c34fc6..cc53ae5 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py @@ -162,6 +162,8 @@ class HDP23StackAdvisor(HDP22StackAdvisor): if versionSplits and len(versionSplits) > 1 and int(versionSplits[0]) > 0 and int(versionSplits[1]) > 7: jvmGCParams = "-XX:+UseG1GC -XX:+ResizeTLAB" putTezProperty('tez.am.launch.cmd-opts', "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA " + jvmGCParams) + # Note: Same calculation is done in 2.6/stack_advisor::recommendTezConfigurations() for 'tez.task.launch.cmd-opts', + # and along with it, are appended heap dump opts. If something changes here, make sure to change it in 2.6 stack. putTezProperty('tez.task.launch.cmd-opts', "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA " + jvmGCParams) http://git-wip-us.apache.org/repos/asf/ambari/blob/8552345d/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml index 279c7c4..7b20728 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml @@ -53,40 +53,14 @@ <on-ambari-upgrade add="true"/> </property> <property> - <name>llap_queue_capacity</name> - <value>0</value> - <description>Percentage of the cluster dedicated to interactive query.</description> - <display-name>% of Cluster Capacity</display-name> - <value-attributes> - <type>int</type> - <minimum>20</minimum> - <maximum>100</maximum> - <increment-step>1</increment-step> - </value-attributes> - <depends-on> - <property> - <type>hive-interactive-env</type> - <name>enable_hive_interactive</name> - </property> - <property> - <type>hive-interactive-site</type> - <name>hive.llap.daemon.queue.name</name> - </property> - <property> - <type>capacity-scheduler</type> - <name>yarn.scheduler.capacity.root.queues</name> - </property> - </depends-on> - <on-ambari-upgrade add="true"/> - </property> - <property> <name>num_llap_nodes</name> <value>1</value> <description>The number of Hive LLAP daemons to run.</description> - <display-name>Number of LLAP Daemons</display-name> + <display-name>Number of LLAP Nodes</display-name> <value-attributes> <type>int</type> <minimum>1</minimum> + <increment-step>1</increment-step> </value-attributes> <depends-on> <property> @@ -94,10 +68,6 @@ <name>enable_hive_interactive</name> </property> <property> - <type>hive-interactive-env</type> - <name>llap_queue_capacity</name> - </property> - <property> <type>hive-interactive-site</type> <name>hive.llap.daemon.queue.name</name> </property> @@ -105,10 +75,6 @@ <type>capacity-scheduler</type> <name>yarn.scheduler.capacity.root.queues</name> </property> - <property> - <type>hive-interactive-site</type> - <name>hive.server2.tez.sessions.per.default.queue</name> - </property> </depends-on> <on-ambari-upgrade add="true"/> </property> @@ -150,7 +116,7 @@ </property> <property> <type>hive-interactive-env</type> - <name>llap_queue_capacity</name> + <name>num_llap_nodes</name> </property> <property> <type>hive-interactive-site</type> @@ -187,7 +153,7 @@ </property> <property> <type>hive-interactive-env</type> - <name>llap_queue_capacity</name> + <name>num_llap_nodes</name> </property> <property> <type>hive-interactive-site</type> @@ -211,7 +177,7 @@ <display-name>LLAP's reserved headroom for YARN container</display-name> <value-attributes> <type>int</type> - <unit>bytes</unit> + <unit>MB</unit> </value-attributes> <on-ambari-upgrade add="true"/> </property> @@ -246,6 +212,40 @@ <display-name>LLAP app java opts</display-name> <on-ambari-upgrade add="true"/> </property> + <property> + <name>hive_heapsize</name> + <value>512</value> + <description>Hive Java heap size</description> + <display-name>HiveServer Interactive Heap Size</display-name> + <value-attributes> + <type>int</type> + <minimum>512</minimum> + <unit>MB</unit> + </value-attributes> + <on-ambari-upgrade add="true"/> + <depends-on> + <property> + <type>hive-interactive-env</type> + <name>enable_hive_interactive</name> + </property> + <property> + <type>hive-interactive-env</type> + <name>num_llap_nodes</name> + </property> + <property> + <type>hive-interactive-site</type> + <name>hive.llap.daemon.queue.name</name> + </property> + <property> + <type>capacity-scheduler</type> + <name>yarn.scheduler.capacity.root.queues</name> + </property> + <property> + <type>hive-interactive-site</type> + <name>hive.server2.tez.sessions.per.default.queue</name> + </property> + </depends-on> + </property> <!-- hive-env.sh --> @@ -267,7 +267,7 @@ if [ "$SERVICE" = "metastore" ]; then export HADOOP_HEAPSIZE={{hive_metastore_heapsize}} # Setting for HiveMetastore else - export HADOOP_HEAPSIZE={{hive_heapsize}} # Setting for HiveServer2 and Client + export HADOOP_HEAPSIZE={{hive_interactive_heapsize}} # Setting for HiveServer2 and Client fi export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Xmx${HADOOP_HEAPSIZE}m" http://git-wip-us.apache.org/repos/asf/ambari/blob/8552345d/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml index 0207e49..2fb1553 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml @@ -67,7 +67,7 @@ limitations under the License. </property> <property> <type>hive-interactive-env</type> - <name>llap_queue_capacity</name> + <name>num_llap_nodes</name> </property> <property> <type>hive-interactive-site</type> @@ -411,7 +411,7 @@ limitations under the License. </property> <property> <type>hive-interactive-env</type> - <name>llap_queue_capacity</name> + <name>num_llap_nodes</name> </property> <property> <type>hive-interactive-site</type> @@ -447,7 +447,7 @@ limitations under the License. </property> <property> <type>hive-interactive-env</type> - <name>llap_queue_capacity</name> + <name>num_llap_nodes</name> </property> <property> <type>hive-interactive-site</type> @@ -478,7 +478,7 @@ limitations under the License. <name>hive.llap.daemon.yarn.container.mb</name> <display-name>Memory per daemon</display-name> <description>Total memory used by individual LLAP daemons. This includes memory for the cache as well as for the query execution.</description> - <value>341</value> + <value>0</value> <value-attributes> <type>int</type> <unit>MB</unit> @@ -491,7 +491,7 @@ limitations under the License. </property> <property> <type>hive-interactive-env</type> - <name>llap_queue_capacity</name> + <name>num_llap_nodes</name> </property> <property> <type>hive-interactive-site</type> @@ -646,4 +646,62 @@ limitations under the License. </value-attributes> <on-ambari-upgrade add="false"/> </property> + <property> + <name>hive.tez.container.size</name> + <value>SET_ON_FIRST_INVOCATION</value> + <depends-on> + <property> + <type>hive-interactive-env</type> + <name>enable_hive_interactive</name> + </property> + <property> + <type>hive-interactive-env</type> + <name>num_llap_nodes</name> + </property> + <property> + <type>hive-interactive-site</type> + <name>hive.llap.daemon.queue.name</name> + </property> + <property> + <type>capacity-scheduler</type> + <name>yarn.scheduler.capacity.root.queues</name> + </property> + <property> + <type>hive-interactive-site</type> + <name>hive.server2.tez.sessions.per.default.queue</name> + </property> + </depends-on> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>hive.auto.convert.join.noconditionaltask.size</name> + <value>1000000000</value> + <description>If hive.auto.convert.join.noconditionaltask is off, this parameter does not take affect. However, if it + is on, and the sum of size for n-1 of the tables/partitions for a n-way join is smaller than this size, the join is directly + converted to a mapjoin(there is no conditional task). The default is 10MB. + </description> + <depends-on> + <property> + <type>hive-interactive-env</type> + <name>enable_hive_interactive</name> + </property> + <property> + <type>hive-interactive-env</type> + <name>num_llap_nodes</name> + </property> + <property> + <type>hive-interactive-site</type> + <name>hive.llap.daemon.queue.name</name> + </property> + <property> + <type>capacity-scheduler</type> + <name>yarn.scheduler.capacity.root.queues</name> + </property> + <property> + <type>hive-interactive-site</type> + <name>hive.server2.tez.sessions.per.default.queue</name> + </property> + </depends-on> + <on-ambari-upgrade add="true"/> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/8552345d/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml index 9e588e9..b331736 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml @@ -83,11 +83,8 @@ </property> <property> <name>tez.am.resource.memory.mb</name> - <value>1536</value> + <value>0</value> <description>The amount of memory to be used by the AppMaster</description> - <value-attributes> - <type>int</type> - </value-attributes> <depends-on> <property> <type>hive-interactive-env</type> @@ -95,7 +92,71 @@ </property> <property> <type>hive-interactive-env</type> - <name>llap_queue_capacity</name> + <name>num_llap_nodes</name> + </property> + <property> + <type>hive-interactive-site</type> + <name>hive.llap.daemon.queue.name</name> + </property> + <property> + <type>capacity-scheduler</type> + <name>yarn.scheduler.capacity.root.queues</name> + </property> + <property> + <type>hive-interactive-site</type> + <name>hive.server2.tez.sessions.per.default.queue</name> + </property> + </depends-on> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>tez.runtime.io.sort.mb</name> + <value>512</value> + <description>The size of the sort buffer when output needs to be sorted</description> + <depends-on> + <property> + <type>tez-site</type> + <name>tez.task.resource.memory.mb</name> + </property> + <property> + <type>hive-interactive-env</type> + <name>enable_hive_interactive</name> + </property> + <property> + <type>hive-interactive-env</type> + <name>num_llap_nodes</name> + </property> + <property> + <type>hive-interactive-site</type> + <name>hive.llap.daemon.queue.name</name> + </property> + <property> + <type>capacity-scheduler</type> + <name>yarn.scheduler.capacity.root.queues</name> + </property> + <property> + <type>hive-interactive-site</type> + <name>hive.server2.tez.sessions.per.default.queue</name> + </property> + </depends-on> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>tez.runtime.unordered.output.buffer.size-mb</name> + <value>100</value> + <description>The size of the buffer when output does not require to be sorted</description> + <depends-on> + <property> + <type>tez-site</type> + <name>tez.task.resource.memory.mb</name> + </property> + <property> + <type>hive-interactive-env</type> + <name>enable_hive_interactive</name> + </property> + <property> + <type>hive-interactive-env</type> + <name>num_llap_nodes</name> </property> <property> <type>hive-interactive-site</type> http://git-wip-us.apache.org/repos/asf/ambari/blob/8552345d/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json index 452537d..1d50b6a 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json @@ -63,11 +63,7 @@ ] }, { - "config": "hive-interactive-env/llap_queue_capacity", - "subsection-name": "interactive-query-row1-col1" - }, - { - "config": "hive-interactive-site/hive.server2.tez.sessions.per.default.queue", + "config": "hive-interactive-env/num_llap_nodes", "subsection-name": "interactive-query-row1-col1", "depends-on": [ { @@ -89,12 +85,8 @@ ] }, { - "config": "hive-interactive-env/copy_num_llap_nodes", + "config": "hive-interactive-site/hive.server2.tez.sessions.per.default.queue", "subsection-name": "interactive-query-row1-col1", - "property_value_attributes": { - "ui_only_property": true, - "copy": "hive-interactive-env/num_llap_nodes" - }, "depends-on": [ { "configs":[ @@ -226,23 +218,12 @@ } }, { - "config": "hive-interactive-env/copy_num_llap_nodes", - "widget": { - "type": "label", - "units": [ - { - "unit-name": "int" - } - ] - } - }, - { - "config": "hive-interactive-env/llap_queue_capacity", + "config": "hive-interactive-env/num_llap_nodes", "widget": { "type": "slider", "units": [ { - "unit-name": "percent" + "unit-name": "int" } ] } http://git-wip-us.apache.org/repos/asf/ambari/blob/8552345d/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/capacity-scheduler.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/capacity-scheduler.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/capacity-scheduler.xml index 9ff8484..bc0ecc4 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/capacity-scheduler.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/capacity-scheduler.xml @@ -26,7 +26,7 @@ </property> <property> <type>hive-interactive-env</type> - <name>llap_queue_capacity</name> + <name>num_llap_nodes</name> </property> </depends-on> <on-ambari-upgrade add="false"/>
