AMBARI-20033. Typecasting to 'long' from earlier 'float', before setting them the following configs for recommendation : (1). 'llap_concurrency', (2). 'llap_concurrency' max value and (3). 'hive.llap.daemon.num.executors' max value.
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/90235286 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/90235286 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/90235286 Branch: refs/heads/branch-feature-AMBARI-12556 Commit: 90235286c8a4aebb1c5159be89509f6679a6e1da Parents: 95d096d Author: Swapan Shridhar <[email protected]> Authored: Wed Feb 15 13:37:54 2017 -0800 Committer: Swapan Shridhar <[email protected]> Committed: Wed Feb 15 13:37:54 2017 -0800 ---------------------------------------------------------------------- .../common-services/YARN/3.0.0.3.0/service_advisor.py | 14 +++++++++----- .../stacks/HDP/2.5/services/stack_advisor.py | 13 ++++++++----- 2 files changed, 17 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/90235286/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py index aecf1e3..6e4e55f 100644 --- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py +++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py @@ -23,6 +23,7 @@ import os import traceback import inspect import socket +import math from math import floor, ceil # Local imports @@ -699,12 +700,15 @@ class YARNRecommender(service_advisor.ServiceAdvisor): mem_per_thread_for_llap, normalized_tez_am_container_size)) if llap_concurrency == 0: llap_concurrency = 1 + Logger.info("DBG: Readjusted 'llap_concurrency' to : 1. Earlier calculated value : 0") if llap_concurrency * normalized_tez_am_container_size > hive_tez_am_cap_available: - llap_concurrency = floor(hive_tez_am_cap_available / normalized_tez_am_container_size) + llap_concurrency = long(math.floor(hive_tez_am_cap_available / normalized_tez_am_container_size)) + Logger.info("DBG: Readjusted 'llap_concurrency' to : {0}, as llap_concurrency({1}) * normalized_tez_am_container_size({2}) > hive_tez_am_cap_available({3}))" + .format(llap_concurrency, llap_concurrency, normalized_tez_am_container_size, hive_tez_am_cap_available)) if llap_concurrency <= 0: - Logger.warning("Calculated 'LLAP Concurrent Queries' = {0}. Expected value >= 1.".format(llap_concurrency)) + Logger.warning("DBG: Calculated 'LLAP Concurrent Queries' = {0}. Expected value >= 1.".format(llap_concurrency)) self.recommendDefaultLlapConfiguration(configurations, services, hosts) return Logger.info("DBG: Adjusted 'llap_concurrency' : {0}, using following: hive_tez_am_cap_available : {1}, normalized_tez_am_container_size: " @@ -730,8 +734,8 @@ class YARNRecommender(service_advisor.ServiceAdvisor): ": {2}, MIN_EXECUTOR_TO_AM_RATIO : {3}, MAX_CONCURRENT_QUERIES : {4}".format(max_llap_concurreny_limit, max_executors_per_node, num_llap_nodes_requested, MIN_EXECUTOR_TO_AM_RATIO, MAX_CONCURRENT_QUERIES)) - max_llap_concurreny = min(max_llap_concurreny_limit, floor(llap_mem_for_tezAm_and_daemons / (MIN_EXECUTOR_TO_AM_RATIO * - mem_per_thread_for_llap + normalized_tez_am_container_size))) + max_llap_concurreny = long(min(max_llap_concurreny_limit, floor(llap_mem_for_tezAm_and_daemons / (MIN_EXECUTOR_TO_AM_RATIO * + mem_per_thread_for_llap + normalized_tez_am_container_size)))) Logger.info("DBG: Calculated 'max_llap_concurreny' : {0}, using following : max_llap_concurreny_limit : {1}, llap_mem_for_tezAm_and_daemons : " "{2}, MIN_EXECUTOR_TO_AM_RATIO : {3}, mem_per_thread_for_llap : {4}, normalized_tez_am_container_size : " "{5}".format(max_llap_concurreny, max_llap_concurreny_limit, llap_mem_for_tezAm_and_daemons, MIN_EXECUTOR_TO_AM_RATIO, @@ -873,7 +877,7 @@ class YARNRecommender(service_advisor.ServiceAdvisor): num_executors_per_node = long(num_executors_per_node) putHiveInteractiveSiteProperty('hive.llap.daemon.num.executors', num_executors_per_node) putHiveInteractiveSitePropertyAttribute('hive.llap.daemon.num.executors', "minimum", 1) - putHiveInteractiveSitePropertyAttribute('hive.llap.daemon.num.executors', "maximum", float(num_executors_per_node_max)) + putHiveInteractiveSitePropertyAttribute('hive.llap.daemon.num.executors', "maximum", long(num_executors_per_node_max)) # 'hive.llap.io.threadpool.size' config value is to be set same as value calculated for # 'hive.llap.daemon.num.executors' at all times. http://git-wip-us.apache.org/repos/asf/ambari/blob/90235286/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py index 4de9a41..52ada52 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py @@ -1031,12 +1031,15 @@ class HDP25StackAdvisor(HDP24StackAdvisor): mem_per_thread_for_llap, normalized_tez_am_container_size)) if llap_concurrency == 0: llap_concurrency = 1 + Logger.info("DBG: Readjusted 'llap_concurrency' to : 1. Earlier calculated value : 0") if llap_concurrency * normalized_tez_am_container_size > hive_tez_am_cap_available: - llap_concurrency = math.floor(hive_tez_am_cap_available / normalized_tez_am_container_size) + llap_concurrency = long(math.floor(hive_tez_am_cap_available / normalized_tez_am_container_size)) + Logger.info("DBG: Readjusted 'llap_concurrency' to : {0}, as llap_concurrency({1}) * normalized_tez_am_container_size({2}) > hive_tez_am_cap_available({3}))" + .format(llap_concurrency, llap_concurrency, normalized_tez_am_container_size, hive_tez_am_cap_available)) if llap_concurrency <= 0: - Logger.warning("Calculated 'LLAP Concurrent Queries' = {0}. Expected value >= 1.".format(llap_concurrency)) + Logger.warning("DBG: Calculated 'LLAP Concurrent Queries' = {0}. Expected value >= 1.".format(llap_concurrency)) self.recommendDefaultLlapConfiguration(configurations, services, hosts) return Logger.info("DBG: Adjusted 'llap_concurrency' : {0}, using following: hive_tez_am_cap_available : {1}, normalized_tez_am_container_size: " @@ -1062,8 +1065,8 @@ class HDP25StackAdvisor(HDP24StackAdvisor): ": {2}, MIN_EXECUTOR_TO_AM_RATIO : {3}, MAX_CONCURRENT_QUERIES : {4}".format(max_llap_concurreny_limit, max_executors_per_node, num_llap_nodes_requested, MIN_EXECUTOR_TO_AM_RATIO, MAX_CONCURRENT_QUERIES)) - max_llap_concurreny = min(max_llap_concurreny_limit, math.floor(llap_mem_for_tezAm_and_daemons / (MIN_EXECUTOR_TO_AM_RATIO * - mem_per_thread_for_llap + normalized_tez_am_container_size))) + max_llap_concurreny = long(min(max_llap_concurreny_limit, math.floor(llap_mem_for_tezAm_and_daemons / (MIN_EXECUTOR_TO_AM_RATIO * + mem_per_thread_for_llap + normalized_tez_am_container_size)))) Logger.info("DBG: Calculated 'max_llap_concurreny' : {0}, using following : max_llap_concurreny_limit : {1}, llap_mem_for_tezAm_and_daemons : " "{2}, MIN_EXECUTOR_TO_AM_RATIO : {3}, mem_per_thread_for_llap : {4}, normalized_tez_am_container_size : " "{5}".format(max_llap_concurreny, max_llap_concurreny_limit, llap_mem_for_tezAm_and_daemons, MIN_EXECUTOR_TO_AM_RATIO, @@ -1209,7 +1212,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor): Logger.info("DBG: Putting num_executors_per_node as {0}".format(num_executors_per_node)) putHiveInteractiveSiteProperty('hive.llap.daemon.num.executors', num_executors_per_node) putHiveInteractiveSitePropertyAttribute('hive.llap.daemon.num.executors', "minimum", 1) - putHiveInteractiveSitePropertyAttribute('hive.llap.daemon.num.executors', "maximum", float(num_executors_per_node_max)) + putHiveInteractiveSitePropertyAttribute('hive.llap.daemon.num.executors', "maximum", long(num_executors_per_node_max)) # 'hive.llap.io.threadpool.size' config value is to be set same as value calculated for # 'hive.llap.daemon.num.executors' at all times.
