Repository: ambari Updated Branches: refs/heads/trunk 438a3cec2 -> 979e17ddf
AMBARI-17283. Display LLAP calculated configs as 'Labels' (non-editable) on 'Hive Server Interactive' Panel and Textboxes (editable) in HIVE Advanced section. Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/979e17dd Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/979e17dd Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/979e17dd Branch: refs/heads/trunk Commit: 979e17ddfaf6622e6540648b76ee31c0a1a70536 Parents: 438a3ce Author: Swapan Shridhar <sshrid...@hortonworks.com> Authored: Thu Jun 16 13:26:58 2016 -0700 Committer: Swapan Shridhar <sshrid...@hortonworks.com> Committed: Fri Jun 17 01:26:44 2016 -0700 ---------------------------------------------------------------------- .../HIVE/configuration/hive-interactive-env.xml | 31 +++++ .../configuration/hive-interactive-site.xml | 81 ++++++++++++ .../HDP/2.5/services/HIVE/themes/theme.json | 22 ++-- .../stacks/HDP/2.5/services/stack_advisor.py | 100 ++++++++++----- .../stacks/2.5/common/test_stack_advisor.py | 123 ++++++++++++++++++- 5 files changed, 311 insertions(+), 46 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/979e17dd/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml index 4d34455..b460464 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml @@ -215,6 +215,37 @@ <display-name>LLAP app java opts</display-name> <on-ambari-upgrade add="true"/> </property> + + + <!-- + Note: This config is added for displaying 'num_llap_nodes' value as a 'label' on Hive Server Interactive panel and will always + have same value as 'num_llap_nodes'. 'num_llap_nodes' will stay editable in Advanced section. + --> + + <property> + <name>num_llap_nodes_copy</name> + <value>1</value> + <description>The number of Hive LLAP daemons to run.</description> + <display-name>Number of LLAP Daemons</display-name> + <value-attributes> + <type>int</type> + <minimum>1</minimum> + </value-attributes> + <depends-on> + <property> + <type>hive-interactive-env</type> + <name>enable_hive_interactive</name> + </property> + <property> + <type>hive-interactive-env</type> + <name>num_llap_nodes</name> + </property> + </depends-on> + <on-ambari-upgrade add="true"/> + </property> + + + <!-- hive-env.sh --> <property> <name>content</name> http://git-wip-us.apache.org/repos/asf/ambari/blob/979e17dd/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml index 1b2eaf5..9c7e865 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml @@ -570,4 +570,85 @@ limitations under the License. <value>none</value> <on-ambari-upgrade add="true"/> </property> + + <!-- + Note: Properties added here are just for displaying them as 'label' on Hive Server Interactive panel. + --> + <property> + <name>hive.llap.daemon.num.executors_copy</name> + <!-- + Added for displaying 'hive.llap.daemon.num.executors' value as a 'label' on Hive Server Interactive panel and will always + have same value as 'hive.llap.daemon.num.executors'. 'hive.llap.daemon.num.executors_copy' will stay editable in Advanced section. + --> + <display-name>Maximum CPUs per Daemon</display-name> + <description>The maximum number of CPUs a single LLAP daemon will use. Usually this should be equal to the number of available CPUs.</description> + <value>1</value> + <value-attributes> + <type>int</type> + </value-attributes> + <depends-on> + <property> + <type>hive-interactive-site</type> + <name>hive.llap.daemon.num.executors</name> + </property> + <property> + <type>hive-interactive-env</type> + <name>enable_hive_interactive</name> + </property> + </depends-on> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>hive.llap.daemon.yarn.container.mb_copy</name> + <!-- + Added for displaying 'hive.llap.daemon.yarn.container.mb' value as a 'label' on Hive Server Interactive panel and will always + have same value as 'hive.llap.daemon.yarn.container.mb'. 'hive.llap.daemon.yarn.container.mb' will stay editable in Advanced section. + --> + <display-name>Memory per daemon</display-name> + <description>Total memory used by individual LLAP daemons. This includes memory for the cache as well as for the query execution.</description> + <value>{{yarn_scheduler_allocation_minimum_mb}}</value> + <value-attributes> + <type>int</type> + <unit>MB</unit> + <overridable>false</overridable> + </value-attributes> + <depends-on> + <property> + <type>hive-interactive-env</type> + <name>enable_hive_interactive</name> + </property> + <property> + <type>hive-interactive-site</type> + <name>hive.llap.daemon.yarn.container.mb</name> + </property> + </depends-on> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>hive.llap.io.memory.size_copy</name> + <!-- + Added for displaying 'hive.llap.io.memory.size' value as a 'label' on Hive Server Interactive panel and will always + have same value as 'hive.llap.io.memory.size'. 'hive.llap.io.memory.size' will stay editable in Advanced section. + --> + <display-name>In-Memory Cache per Daemon</display-name> + <description>The amount of memory reserved for Hive's optimized in-memory cache.</description> + <value>0</value> + <value-attributes> + <type>int</type> + <unit>MB</unit> + <overridable>false</overridable> + </value-attributes> + <depends-on> + <property> + <type>hive-interactive-env</type> + <name>enable_hive_interactive</name> + </property> + <property> + <type>hive-interactive-site</type> + <name>hive.llap.io.memory.size</name> + </property> + </depends-on> + <on-ambari-upgrade add="true"/> + </property> + </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/979e17dd/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json index 395ae43..81277a8 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json @@ -89,7 +89,7 @@ ] }, { - "config": "hive-interactive-env/num_llap_nodes", + "config": "hive-interactive-env/num_llap_nodes_copy", "subsection-name": "interactive-query-row1-col1", "depends-on": [ { @@ -111,7 +111,7 @@ ] }, { - "config": "hive-interactive-site/hive.llap.daemon.yarn.container.mb", + "config": "hive-interactive-site/hive.llap.daemon.yarn.container.mb_copy", "subsection-name": "interactive-query-row1-col1", "depends-on": [ { @@ -133,7 +133,7 @@ ] }, { - "config": "hive-interactive-site/hive.llap.io.memory.size", + "config": "hive-interactive-site/hive.llap.io.memory.size_copy", "subsection-name": "interactive-query-row1-col1", "depends-on": [ { @@ -155,7 +155,7 @@ ] }, { - "config": "hive-interactive-site/hive.llap.daemon.num.executors", + "config": "hive-interactive-site/hive.llap.daemon.num.executors_copy", "subsection-name": "interactive-query-row1-col1", "depends-on": [ { @@ -210,9 +210,9 @@ } }, { - "config": "hive-interactive-env/num_llap_nodes", + "config": "hive-interactive-env/num_llap_nodes_copy", "widget": { - "type": "text-field", + "type": "label", "units": [ { "unit-name": "int" @@ -232,7 +232,7 @@ } }, { - "config": "hive-interactive-site/hive.llap.daemon.yarn.container.mb", + "config": "hive-interactive-site/hive.llap.daemon.yarn.container.mb_copy", "widget": { "type": "label", "units": [ @@ -243,9 +243,9 @@ } }, { - "config": "hive-interactive-site/hive.llap.io.memory.size", + "config": "hive-interactive-site/hive.llap.io.memory.size_copy", "widget": { - "type": "text-field", + "type": "label", "units": [ { "unit-name": "MB" @@ -254,9 +254,9 @@ } }, { - "config": "hive-interactive-site/hive.llap.daemon.num.executors", + "config": "hive-interactive-site/hive.llap.daemon.num.executors_copy", "widget": { - "type": "text-field", + "type": "label", "units": [ { "unit-name": "int" http://git-wip-us.apache.org/repos/asf/ambari/blob/979e17dd/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py index 01522b2..76654c3 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py @@ -402,12 +402,32 @@ class HDP25StackAdvisor(HDP24StackAdvisor): putHiveInteractiveEnvProperty('enable_hive_interactive', 'false') putHiveInteractiveEnvPropertyAttribute("llap_queue_capacity", "visible", "false") - if self.HIVE_INTERACTIVE_SITE in services['configurations'] and \ - 'hive.llap.zk.sm.connectionString' in services['configurations'][self.HIVE_INTERACTIVE_SITE]['properties']: - # Fill the property 'hive.llap.zk.sm.connectionString' required by Hive Server Interactive (HiveServer2) - zookeeper_host_port = self.getZKHostPortString(services) - if zookeeper_host_port: - putHiveInteractiveSiteProperty("hive.llap.zk.sm.connectionString", zookeeper_host_port) + if self.HIVE_INTERACTIVE_SITE in services['configurations']: + if 'hive.llap.zk.sm.connectionString' in services['configurations'][self.HIVE_INTERACTIVE_SITE]['properties']: + # Fill the property 'hive.llap.zk.sm.connectionString' required by Hive Server Interactive (HiveServer2) + zookeeper_host_port = self.getZKHostPortString(services) + if zookeeper_host_port: + putHiveInteractiveSiteProperty("hive.llap.zk.sm.connectionString", zookeeper_host_port) + + # Copy the value of 'hive.llap.daemon.yarn.container.mb', 'hive.llap.daemon.num.executors', 'hive.llap.io.memory.size' and 'num_llap_nodes' + # to their '*_copy' configs as they are used to display the values as 'label' on Hive Server interactive panel. + if 'hive.llap.daemon.yarn.container.mb' in services['configurations'][self.HIVE_INTERACTIVE_SITE]['properties']: + hive_llap_yarn_cont_mb = services['configurations'][self.HIVE_INTERACTIVE_SITE]['properties']['hive.llap.daemon.yarn.container.mb'] + putHiveInteractiveSiteProperty("hive.llap.daemon.yarn.container.mb_copy", hive_llap_yarn_cont_mb) + Logger.info("Adjusted config 'hive.llap.daemon.yarn.container.mb_copy' value same as config 'hive.llap.daemon.yarn.container.mb' value({0})".format(hive_llap_yarn_cont_mb)) + if 'hive.llap.daemon.num.executors' in services['configurations'][self.HIVE_INTERACTIVE_SITE]['properties']: + hive_llap_daemon_num_executors = services['configurations'][self.HIVE_INTERACTIVE_SITE]['properties']['hive.llap.daemon.num.executors'] + putHiveInteractiveSiteProperty("hive.llap.daemon.num.executors_copy", hive_llap_daemon_num_executors) + Logger.info("Adjusted config 'hive.llap.daemon.num.executors_copy' value same as config 'hive.llap.daemon.num.executors' value({0})".format(hive_llap_daemon_num_executors)) + if 'hive.llap.io.memory.size' in services['configurations'][self.HIVE_INTERACTIVE_SITE]['properties']: + hive_llap_io_mem_size = services['configurations'][self.HIVE_INTERACTIVE_SITE]['properties']['hive.llap.io.memory.size'] + putHiveInteractiveSiteProperty("hive.llap.io.memory.size_copy", hive_llap_io_mem_size) + Logger.info("Adjusted config 'hive.llap.io.memory.size_copy' value same as config 'hive.llap.io.memory.size' value({0})".format(hive_llap_io_mem_size)) + if 'hive-interactive-env' in services['configurations'] and \ + 'num_llap_nodes' in services['configurations']['hive-interactive-env']['properties']: + num_llap_node = services['configurations']['hive-interactive-env']['properties']['num_llap_nodes'] + putHiveInteractiveEnvProperty("num_llap_nodes_copy", num_llap_node) + Logger.info("Adjusted config 'num_llap_nodes_copy' value same as config 'num_llap_nodes' value({0})".format(num_llap_node)) pass def recommendYARNConfigurations(self, configurations, clusterData, services, hosts): @@ -427,9 +447,10 @@ class HDP25StackAdvisor(HDP24StackAdvisor): self.checkAndStopLlapQueue(services, configurations, LLAP_QUEUE_NAME) """ - Entry point for updating Hive's 'LLAP app' configs namely : (1). num_llap_nodes (2). hive.llap.daemon.yarn.container.mb - (3). hive.llap.daemon.num.executors (4). hive.llap.io.memory.size (5). llap_heap_size (6). slider_am_container_size, - and (7). hive.server2.tez.sessions.per.default.queue + Entry point for updating Hive's 'LLAP app' configs namely : (1). num_llap_nodes, num_llap_nodes_copy + (2). hive.llap.daemon.yarn.container.mb, hive.llap.daemon.yarn.container.mb_copy (3). hive.llap.daemon.num.executors, + hive.llap.daemon.num.executors_copy (4). hive.llap.io.memory.size, hive.llap.io.memory.size_copy (5). llap_heap_size + (6). slider_am_container_size, and (7). hive.server2.tez.sessions.per.default.queue The trigger point for updating LLAP configs (mentioned above) is change in values of any of the following: (1). 'enable_hive_interactive' set to 'true' (2). 'llap_queue_capacity' (3). 'hive.server2.tez.sessions.per.default.queue' @@ -437,6 +458,10 @@ class HDP25StackAdvisor(HDP24StackAdvisor): If change in value for 'llap_queue_capacity' or 'hive.server2.tez.sessions.per.default.queue' is detected, that config value is not calulated, but read and use in calculation for dependent configs. + + Note : Configs with name as '*_copy' will have same value as their conterpart configs (eg: 'num_llap_nodes' + for 'num_llap_nodes_copy'). '*_copy' configs are added so that we can show them as label on UI in Hive Server Interactive + Panel and their counterparts shown in 'Advanced' section as editable ones. """ def updateLlapConfigs(self, configurations, services, hosts, llap_queue_name): putHiveInteractiveSiteProperty = self.putProperty(configurations, self.HIVE_INTERACTIVE_SITE, services) @@ -452,10 +477,20 @@ class HDP25StackAdvisor(HDP24StackAdvisor): # Update 'hive.llap.daemon.queue.name' prop combo entries and llap capacity slider visibility. self.setLlapDaemonQueuePropAttributesAndCapSliderVisibility(services, configurations) + read_llap_daemon_yarn_cont_mb = long(self.get_yarn_min_container_size(services, configurations)) + putHiveInteractiveSiteProperty('hive.llap.daemon.yarn.container.mb', read_llap_daemon_yarn_cont_mb) # initial memory setting to make sure hive.llap.daemon.yarn.container.mb >= yarn.scheduler.minimum-allocation-mb - Logger.debug("Setting hive.llap.daemon.yarn.container.mb to yarn min container size as initial size " + Logger.debug("Adjusted 'hive.llap.daemon.yarn.container.mb' to yarn min container size as initial size " "(" + str(self.get_yarn_min_container_size(services, configurations)) + " MB).") - putHiveInteractiveSiteProperty('hive.llap.daemon.yarn.container.mb', long(self.get_yarn_min_container_size(services, configurations))) + # This is a stopgap thing, to be sure that on 1st invocation, value for 'hive.llap.daemon.yarn.container.mb_copy' + # is same as value set by 'hive.llap.daemon.yarn.container.mb'. + if 'hive-interactive-site' not in configurations or \ + ('hive-interactive-site' in configurations and \ + 'hive.llap.daemon.yarn.container.mb_copy' not in configurations['hive-interactive-site']['properties']): + putHiveInteractiveSiteProperty('hive.llap.daemon.yarn.container.mb_copy', read_llap_daemon_yarn_cont_mb) + Logger.debug("Adjusted 'hive.llap.daemon.yarn.container.mb_copy' to yarn min container size as initial size " + "(" + str(self.get_yarn_min_container_size(services, configurations)) + " MB).") + try: if self.HIVE_INTERACTIVE_SITE in services['configurations'] and \ @@ -560,15 +595,16 @@ class HDP25StackAdvisor(HDP24StackAdvisor): # Calculate 'total memory available for llap daemons' across cluster total_am_capacity_required = tez_am_container_size * llap_concurrency + slider_am_container_size cap_available_for_daemons = total_llap_queue_size - total_am_capacity_required - if cap_available_for_daemons < yarn_min_container_size : - raise Fail("'Capacity available for LLAP daemons'({0}) < 'YARN minimum container size'({1}). Invalid configuration detected. " - "Increase LLAP queue size.".format(cap_available_for_daemons, yarn_min_container_size)) Logger.info("Calculated cap_available_for_daemons : {0}, using following : current_selected_queue_for_llap_cap : {1}, " "yarn_nm_mem_in_mb : {2}, total_cluster_capacity : {3}, total_llap_queue_size : {4}, tez_am_container_size" " : {5}, yarn_min_container_size : {6}, llap_concurrency : {7}, total_am_capacity_required : {8}" .format(cap_available_for_daemons, current_selected_queue_for_llap_cap, yarn_nm_mem_in_mb, total_cluster_capacity, total_llap_queue_size, tez_am_container_size, yarn_min_container_size, llap_concurrency, total_am_capacity_required)) + if cap_available_for_daemons < yarn_min_container_size : + raise Fail("'Capacity available for LLAP daemons'({0}) < 'YARN minimum container size'({1}). Invalid configuration detected. " + "Increase LLAP queue size.".format(cap_available_for_daemons, yarn_min_container_size)) + # Calculate value for 'num_llap_nodes', an across cluster config. @@ -639,13 +675,22 @@ class HDP25StackAdvisor(HDP24StackAdvisor): putHiveInteractiveEnvProperty('num_llap_nodes', num_llap_nodes) Logger.info("LLAP config 'num_llap_nodes' updated. Current: {0}".format(num_llap_nodes)) + # Copy the value of 'num_llap_nodes' to 'num_llap_nodes_copy' as 'num_llap_nodes_copy' config is used to display + # it as Label on Hive Server interactive panel. + putHiveInteractiveEnvProperty('num_llap_nodes_copy', num_llap_nodes) llap_container_size = long(llap_container_size) putHiveInteractiveSiteProperty('hive.llap.daemon.yarn.container.mb', llap_container_size) Logger.info("LLAP config 'hive.llap.daemon.yarn.container.mb' updated. Current: {0}".format(llap_container_size)) + # Copy the value of 'hive.llap.daemon.yarn.container.mb' to 'hive.llap.daemon.yarn.container.mb_copy' as + # 'hive.llap.daemon.yarn.container.mb_copy' config is used to display it as Label on Hive Server interactive panel. + putHiveInteractiveSiteProperty('hive.llap.daemon.yarn.container.mb_copy', llap_container_size) num_executors_per_node = long(num_executors_per_node) putHiveInteractiveSiteProperty('hive.llap.daemon.num.executors', num_executors_per_node) + # Copy the value of 'hive.llap.daemon.num.executors' to 'hive.llap.daemon.num.executors_copy' as + # 'hive.llap.daemon.num.executors_copy' config is used to display it as Label on Hive Server interactive panel. + putHiveInteractiveSiteProperty('hive.llap.daemon.num.executors_copy', num_executors_per_node) Logger.info("LLAP config 'hive.llap.daemon.num.executors' updated. Current: {0}".format(num_executors_per_node)) # 'hive.llap.io.threadpool.size' config value is to be set same as value calculated for # 'hive.llap.daemon.num.executors' at all times. @@ -655,6 +700,9 @@ class HDP25StackAdvisor(HDP24StackAdvisor): cache_size_per_node = long(cache_size_per_node) putHiveInteractiveSiteProperty('hive.llap.io.memory.size', cache_size_per_node) Logger.info("LLAP config 'hive.llap.io.memory.size' updated. Current: {0}".format(cache_size_per_node)) + # Copy the value of 'hive.llap.io.memory.size' to 'hive.llap.io.memory.size_copy' as + # 'hive.llap.io.memory.size_copy' config is used to display it as Label on Hive Server interactive panel. + putHiveInteractiveSiteProperty('hive.llap.io.memory.size_copy', cache_size_per_node) llap_io_enabled = 'false' if cache_size_per_node >= 64: llap_io_enabled = 'true' @@ -691,17 +739,26 @@ class HDP25StackAdvisor(HDP24StackAdvisor): putHiveInteractiveEnvProperty('num_llap_nodes', 0) putHiveInteractiveEnvPropertyAttribute('num_llap_nodes', "minimum", 1) putHiveInteractiveEnvPropertyAttribute('num_llap_nodes', "maximum", node_manager_cnt) + putHiveInteractiveEnvProperty('num_llap_nodes_copy', 0) + putHiveInteractiveEnvPropertyAttribute('num_llap_nodes_copy', "minimum", 1) + putHiveInteractiveEnvPropertyAttribute('num_llap_nodes_copy', "maximum", node_manager_cnt) putHiveInteractiveSiteProperty('hive.llap.daemon.yarn.container.mb', yarn_min_container_size) putHiveInteractiveSitePropertyAttribute('hive.llap.daemon.yarn.container.mb', "minimum", yarn_min_container_size) + putHiveInteractiveSiteProperty('hive.llap.daemon.yarn.container.mb_copy', yarn_min_container_size) + putHiveInteractiveSitePropertyAttribute('hive.llap.daemon.yarn.container.mb_copy', "minimum", yarn_min_container_size) putHiveInteractiveSiteProperty('hive.llap.daemon.num.executors', 0) putHiveInteractiveSitePropertyAttribute('hive.llap.daemon.num.executors', "minimum", 1) + putHiveInteractiveSiteProperty('hive.llap.daemon.num.executors_copy', 0) + putHiveInteractiveSitePropertyAttribute('hive.llap.daemon.num.executors_copy', "minimum", 1) + putHiveInteractiveSiteProperty('hive.llap.io.threadpool.size', 0) putHiveInteractiveSiteProperty('hive.llap.io.threadpool.size', 0) putHiveInteractiveSiteProperty('hive.llap.io.memory.size', 0) + putHiveInteractiveSiteProperty('hive.llap.io.memory.size_copy', 0) putHiveInteractiveEnvProperty('llap_heap_size', 0) @@ -769,21 +826,6 @@ class HDP25StackAdvisor(HDP24StackAdvisor): """ - Returns current value of cache per node for LLAP (hive.llap.io.memory.size). If value can't be retrieved, return 0. - """ - def get_cache_size_per_node_for_llap_nodes(self, services): - if 'hive.llap.io.memory.size' in services['configurations']['hive-interactive-site']['properties']: - cache_size_per_node = float( - services['configurations']['hive-interactive-site']['properties']['hive.llap.io.memory.size']) - return cache_size_per_node - else: - Logger.error("Couldn't retrieve Hive Server interactive's 'hive.llap.io.memory.size' config.") - # Not doing raise as the Exception that catches it will set all other LLAP configs related - # to LLAP package as 0, a way to tell that calulations couldn't be done. This is not the intention here. - # Just keep cache 0, if it couldn't be retrieved. - return 0 - - """ Returns current value of number of LLAP nodes in cluster (num_llap_nodes) """ def get_num_llap_nodes(self, services): http://git-wip-us.apache.org/repos/asf/ambari/blob/979e17dd/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py index f66140d..1bc53ea 100644 --- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py @@ -597,6 +597,8 @@ class TestHDP25StackAdvisor(TestCase): + + # Tests related to 'recommendYARNConfigurations()' @@ -3538,6 +3540,13 @@ class TestHDP25StackAdvisor(TestCase): self.assertEqual(configurations['hive-interactive-env']['properties']['slider_am_container_size'], '512') self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.default.queues'], 'llap') + # Check '*_copy' configs, used for displaying it as label on UI. + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb_copy'], '1024') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors_copy'], '1') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size_copy'], '512') + self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes_copy'], '1') + + # Test 20: (1). 'default' and 'llap' (State : RUNNING) queue exists at root level in capacity-scheduler, and # 'capacity-scheduler' configs are passed-in as single "/n" separated string and @@ -3729,6 +3738,12 @@ class TestHDP25StackAdvisor(TestCase): self.assertEquals(configurations['hive-interactive-env']['property_attributes']['llap_queue_capacity'], {'visible': 'false'}) + # Check '*_copy' configs, used for displaying it as label on UI. + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb_copy'], '5120') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors_copy'], '1') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size_copy'], '4779') + self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes_copy'], '1') + @@ -3925,6 +3940,12 @@ class TestHDP25StackAdvisor(TestCase): self.assertEquals(configurations['hive-interactive-env']['property_attributes']['llap_queue_capacity'], {'maximum': '100', 'minimum': '20', 'visible': 'true'}) + # Check '*_copy' configs, used for displaying it as label on UI. + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb_copy'], '9216') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors_copy'], '1') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size_copy'], '8192') + self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes_copy'], '1') + ####################### 'Three Node Managers' cluster - tests for calculating llap configs ################ @@ -4122,6 +4143,12 @@ class TestHDP25StackAdvisor(TestCase): self.assertEquals(configurations['hive-interactive-env']['property_attributes']['llap_queue_capacity'], {'maximum': '100', 'minimum': '20', 'visible': 'true'}) + # Check '*_copy' configs, used for displaying it as label on UI. + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb_copy'], '40960') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors_copy'], '4') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size_copy'], '36864') + self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes_copy'], '2') + # Test 23: (1). 'default' and 'llap' (State : RUNNING) queue exists at root level in capacity-scheduler, and @@ -4314,6 +4341,12 @@ class TestHDP25StackAdvisor(TestCase): self.assertEquals(configurations['hive-interactive-env']['property_attributes']['llap_queue_capacity'], {'maximum': '100', 'minimum': '20', 'visible': 'true'}) + # Check '*_copy' configs, used for displaying it as label on UI. + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb_copy'], '10230') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors_copy'], '3') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size_copy'], '7158') + self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes_copy'], '2') + # Test 24: (1). 'default' and 'llap' (State : RUNNING) queue exists at root level in capacity-scheduler, and @@ -4504,6 +4537,12 @@ class TestHDP25StackAdvisor(TestCase): {'maximum': '100', 'minimum': '20', 'visible': 'true'}) + # Check '*_copy' configs, used for displaying it as label on UI. + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb_copy'], '61440') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors_copy'], '3') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size_copy'], '58368') + self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes_copy'], '1') + @@ -4701,6 +4740,12 @@ class TestHDP25StackAdvisor(TestCase): {'maximum': '100', 'minimum': '20', 'visible': 'true'}) + # Check '*_copy' configs, used for displaying it as label on UI. + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb_copy'], '39936') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors_copy'], '4') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size_copy'], '35840') + self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes_copy'], '3') + # Test 26: (1). 'default' and 'llap' (State : RUNNING) queue exists at root level in capacity-scheduler, and @@ -4894,6 +4939,12 @@ class TestHDP25StackAdvisor(TestCase): {'maximum': '100', 'minimum': '20', 'visible': 'true'}) + # Check '*_copy' configs, used for displaying it as label on UI. + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb_copy'], '10230') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors_copy'], '10') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size_copy'], '6820') + self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes_copy'], '3') + # Test 27: (1). 'default' and 'llap' (State : RUNNING) queue exists at root level in capacity-scheduler, and @@ -5081,6 +5132,12 @@ class TestHDP25StackAdvisor(TestCase): {'maximum': '100', 'minimum': '25', 'visible': 'true'}) + # Check '*_copy' configs, used for displaying it as label on UI. + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb_copy'], '164480') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors_copy'], '3') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size_copy'], '161408') + self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes_copy'], '1') + @@ -5261,6 +5318,12 @@ class TestHDP25StackAdvisor(TestCase): {'maximum': '100', 'minimum': '49', 'visible': 'true'}) + # Check '*_copy' configs, used for displaying it as label on UI. + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb_copy'], '164480') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors_copy'], '3') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size_copy'], '161408') + self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes_copy'], '1') + @@ -5443,6 +5506,12 @@ class TestHDP25StackAdvisor(TestCase): {'maximum': '100', 'minimum': '20', 'visible': 'true'}) + # Check '*_copy' configs, used for displaying it as label on UI. + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb_copy'], '246720') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors_copy'], '3') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size_copy'], '243648') + self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes_copy'], '1') + # Test 30: (1). Multiple queue exist at various depths in capacity-scheduler, and 'capacity-scheduler' configs are @@ -5654,6 +5723,12 @@ class TestHDP25StackAdvisor(TestCase): + # Check '*_copy' configs, used for displaying it as label on UI. + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb_copy'], '10240') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors_copy'], '3') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size_copy'], '7168') + self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes_copy'], '4') + @@ -5865,6 +5940,12 @@ class TestHDP25StackAdvisor(TestCase): self.assertEquals(configurations['hive-interactive-env']['property_attributes']['llap_queue_capacity'], {'visible': 'false'}) + # Check '*_copy' configs, used for displaying it as label on UI. + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb_copy'], '512') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors_copy'], '0') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size_copy'], '0') + self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes_copy'], '0') + @@ -5971,14 +6052,18 @@ class TestHDP25StackAdvisor(TestCase): { 'properties': { 'enable_hive_interactive': 'true', - 'llap_queue_capacity':'50' + 'llap_queue_capacity':'50', + 'num_llap_nodes' : '1' } }, "hive-interactive-site": { 'properties': { 'hive.llap.daemon.queue.name': 'llap', - 'hive.server2.tez.sessions.per.default.queue': '1' + 'hive.server2.tez.sessions.per.default.queue': '1', + 'hive.llap.daemon.num.executors' : '1', + 'hive.llap.daemon.yarn.container.mb' : '10240', + 'hive.llap.io.memory.size' : '512' } }, "hive-env": @@ -6036,6 +6121,12 @@ class TestHDP25StackAdvisor(TestCase): self.assertEquals(configurations['hive-interactive-env']['property_attributes']['llap_queue_capacity'], {'maximum': '100', 'minimum': '100', 'visible': 'true'}) + # Check '*_copy' configs, used for displaying it as label on UI. + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb_copy'], '10240') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors_copy'], '1') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size_copy'], '512') + self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes_copy'], '1') + @@ -6162,7 +6253,8 @@ class TestHDP25StackAdvisor(TestCase): { 'properties': { 'enable_hive_interactive': 'true', - 'llap_queue_capacity':'0' + 'llap_queue_capacity':'0', + 'num_llap_nodes': '2' } }, "hive-env": @@ -6193,7 +6285,10 @@ class TestHDP25StackAdvisor(TestCase): { 'properties': { 'hive.llap.daemon.queue.name': 'llap', - 'hive.server2.tez.sessions.per.default.queue': '1' + 'hive.server2.tez.sessions.per.default.queue': '1', + 'hive.llap.daemon.num.executors' : '1', + 'hive.llap.daemon.yarn.container.mb' : '8192', + 'hive.llap.io.memory.size' : '0' } }, } @@ -6220,6 +6315,12 @@ class TestHDP25StackAdvisor(TestCase): self.assertEquals(configurations['hive-interactive-env']['property_attributes']['llap_queue_capacity'], {'visible': 'false'}) + # Check '*_copy' configs, used for displaying it as label on UI. + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb_copy'], '8192') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors_copy'], '1') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size_copy'], '0') + self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes_copy'], '2') + @@ -6327,14 +6428,18 @@ class TestHDP25StackAdvisor(TestCase): { 'properties': { 'enable_hive_interactive': 'true', - 'llap_queue_capacity':'50' + 'llap_queue_capacity':'50', + 'num_llap_nodes': '1' } }, "hive-interactive-site": { 'properties': { 'hive.llap.daemon.queue.name': 'default', - 'hive.server2.tez.sessions.per.default.queue': '1' + 'hive.server2.tez.sessions.per.default.queue': '1', + 'hive.llap.daemon.num.executors' : '1', + 'hive.llap.daemon.yarn.container.mb' : '4096', + 'hive.llap.io.memory.size' : '512' } }, "hive-env": @@ -6384,6 +6489,12 @@ class TestHDP25StackAdvisor(TestCase): self.expected_hive_interactive_site_default['hive-interactive-site']['properties']['hive.llap.daemon.queue.name']) self.assertEquals(configurations['hive-interactive-site']['properties']['hive.server2.tez.default.queues'], 'default') + # Check '*_copy' configs, used for displaying it as label on UI. + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb_copy'], '4096') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors_copy'], '1') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size_copy'], '512') + self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes_copy'], '1') +
