AMBARI-18508. Provide a configurable option for "LLAP's headroom space for YARN container".
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7ac93f3a Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7ac93f3a Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7ac93f3a Branch: refs/heads/branch-feature-AMBARI-18456 Commit: 7ac93f3a8aeeedb71d0feecc1f4ff99972f717f7 Parents: f2bcbbe Author: Swapan Shridhar <[email protected]> Authored: Fri Sep 30 14:36:17 2016 -0700 Committer: Swapan Shridhar <[email protected]> Committed: Mon Oct 3 12:39:50 2016 -0700 ---------------------------------------------------------------------- .../HIVE/configuration/hive-interactive-env.xml | 11 ++++++++++ .../stacks/HDP/2.5/services/stack_advisor.py | 23 +++++++++++++++++++- 2 files changed, 33 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/7ac93f3a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml index b14cc8c..c1a7592 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml @@ -195,6 +195,17 @@ <on-ambari-upgrade add="true"/> </property> <property> + <name>llap_headroom_space</name> + <value>6144</value> + <description>LLAP app headroom space</description> + <display-name>LLAP's reserved headroom for YARN container</display-name> + <value-attributes> + <type>int</type> + <unit>bytes</unit> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> <name>llap_log_level</name> <value>INFO</value> <description>LLAP app logging level (WARN/INFO/DEBUG/TRACE)</description> http://git-wip-us.apache.org/repos/asf/ambari/blob/7ac93f3a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py index 2a4728a..d2cc429 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py @@ -953,7 +953,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor): # Calculate value for prop 'llap_heap_size' - llap_xmx = max(total_mem_for_executors * 0.8, total_mem_for_executors - 1024) + llap_xmx = max(total_mem_for_executors * 0.8, total_mem_for_executors - self.get_llap_headroom_space(services, configurations)) Logger.info("Calculated llap_app_heap_size : {0}, using following : hive_container_size : {1}, " "total_mem_for_executors : {2}".format(llap_xmx, hive_tez_container_size, total_mem_for_executors)) @@ -1149,6 +1149,27 @@ class HDP25StackAdvisor(HDP24StackAdvisor): return hive_container_size """ + Gets HIVE Server Interactive's 'llap_headroom_space' config. (Default value set to 6144 bytes). + """ + def get_llap_headroom_space(self, services, configurations): + llap_headroom_space = None + # Check if 'llap_headroom_space' is modified in current SA invocation. + if 'hive-interactive-env' in configurations and 'llap_headroom_space' in configurations['hive-interactive-env']['properties']: + hive_container_size = float(configurations['hive-interactive-env']['properties']['llap_headroom_space']) + Logger.info("'llap_headroom_space' read from configurations as : {0}".format(llap_headroom_space)) + + if not llap_headroom_space: + # Check if 'llap_headroom_space' is input in services array. + if 'llap_headroom_space' in services['configurations']['hive-interactive-env']['properties']: + llap_headroom_space = float(services['configurations']['hive-interactive-env']['properties']['llap_headroom_space']) + Logger.info("'llap_headroom_space' read from services as : {0}".format(llap_headroom_space)) + if not llap_headroom_space or llap_headroom_space < 1: + llap_headroom_space = 6144 # 6GB + Logger.info("Couldn't read 'llap_headroom_space' from services or configurations. Returing default value : 6144 bytes") + + return llap_headroom_space + + """ Gets YARN's minimum container size (yarn.scheduler.minimum-allocation-mb). Reads from: - configurations (if changed as part of current Stack Advisor invocation (output)), and services["changed-configurations"]
