Repository: ambari Updated Branches: refs/heads/trunk 8ccb1d330 -> 6dc2c9558
AMBARI-16197. LLAP related fix : Adding 'args' while package creation, commenting LLAP app status call, and code related to cache calculations. Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6dc2c955 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6dc2c955 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6dc2c955 Branch: refs/heads/trunk Commit: 6dc2c95582c5000f87d75ec82fb3ef812628a40c Parents: 8ccb1d3 Author: Swapan Shridhar <[email protected]> Authored: Sun May 1 13:38:29 2016 -0700 Committer: Swapan Shridhar <[email protected]> Committed: Sun May 1 16:25:48 2016 -0700 ---------------------------------------------------------------------- .../package/scripts/hive_server_interactive.py | 14 ++- .../configuration/hive-interactive-site.xml | 37 ++---- .../stacks/HDP/2.5/services/stack_advisor.py | 117 ++++++++++++----- .../stacks/2.5/common/test_stack_advisor.py | 126 ++++++++++++------- 4 files changed, 190 insertions(+), 104 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/6dc2c955/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py index 882136a..e94b22e 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py @@ -217,10 +217,17 @@ class HiveServerInteractiveDefault(HiveServerInteractive): cmd += format(" --slider-keytab-dir .slider/keytabs/{params.hive_user}/ --slider-keytab " "{llap_keytab_splits[4]} --slider-principal {hive_headless_keytab}") + # Append args. + cmd+= " --args \" -XX:+UseG1GC -XX:TLABSize=8m -XX:+ResizeTLAB -XX:+UseNUMA -XX:+AggressiveOpts" \ + " -XX:+AlwaysPreTouch -XX:MetaspaceSize=1024m -XX:InitiatingHeapOccupancyPercent=80 -XX:MaxGCPauseMillis=200\"" + # TODO: Remove adding "-XX:MaxDirectMemorySize" when Driver starts appending itself. + if params.hive_llap_io_mem_size > params.llap_heap_size: + max_dir_mem_size = long(params.hive_llap_io_mem_size) + 256 + cmd = cmd[0:len(cmd)-1] + format(" -XX:MaxDirectMemorySize={max_dir_mem_size}m\"") + run_file_path = None try: Logger.info(format("Command: {cmd}")) - cmd = cmd.split() code, output, error = shell.checked_call(cmd, user=params.hive_user, stderr=subprocess.PIPE, logoutput=True) if code != 0 or output is None: @@ -243,16 +250,19 @@ class HiveServerInteractiveDefault(HiveServerInteractive): # TODO : Sleep below is not a good idea. We need to check the status of LLAP app to figure out it got # launched properly and is in running state. Then go ahead with Hive Interactive Server start. Logger.info("Sleeping for 30 secs") - # Important to mock this sleep call during unit tests. time.sleep(30) Logger.info("Submitted LLAP app name : {0}".format(LLAP_APP_NAME)) + # TODO: Uncomment this when 'llapstatus' commands returns correct output. + ''' status = self.check_llap_app_status(LLAP_APP_NAME, params.num_retries_for_checking_llap_status) if status: Logger.info("LLAP app '{0}' deployed successfully.".format(LLAP_APP_NAME)) return True else: return False + ''' + return True else: raise Fail(format("Did not find run file {run_file_path}")) except: http://git-wip-us.apache.org/repos/asf/ambari/blob/6dc2c955/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml index 3f110dc..f07da88 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml @@ -137,30 +137,6 @@ limitations under the License. </property> <property> - <name>hive.tez.container.size</name> - <value>341</value> - <description>By default, Tez uses the java options from map tasks. Use this property to override that value.</description> - <display-name>Tez Container Size</display-name> - <value-attributes> - <type>int</type> - <minimum>341</minimum> - <maximum>6820</maximum> - <unit>MB</unit> - <increment-step>682</increment-step> - </value-attributes> - <depends-on> - <property> - <type>yarn-site</type> - <name>yarn.scheduler.minimum-allocation-mb</name> - </property> - <property> - <type>yarn-site</type> - <name>yarn.scheduler.maximum-allocation-mb</name> - </property> - </depends-on> - </property> - - <property> <name>hive.tez.java.opts</name> <value></value> <description>Java command line options for Tez. The -Xmx parameter value is generally 80% of hive.tez.container.size.</description> @@ -325,6 +301,12 @@ limitations under the License. <property> <name>hive.llap.io.enabled</name> <value>true</value> + <depends-on> + <property> + <type>hive-interactive-site</type> + <name>hive.llap.io.memory.size</name> + </property> + </depends-on> </property> <property> @@ -380,7 +362,7 @@ limitations under the License. <value-attributes> <type>int</type> <minimum>0</minimum> - <maximum>4</maximum> + <maximum>64</maximum> <unit>MB</unit> <increment-step>256</increment-step> <overridable>false</overridable> @@ -571,4 +553,9 @@ limitations under the License. </depends-on> </property> + <property> + <name>hive.llap.io.memory.mode</name> + <value>none</value> + </property> + </configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/6dc2c955/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py index ca92157..ea19676 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py @@ -158,6 +158,20 @@ class HDP25StackAdvisor(HDP24StackAdvisor): # Update 'hive.llap.daemon.queue.name' if capacity scheduler is changed. if 'hive.llap.daemon.queue.name' in services['configurations'][self.HIVE_INTERACTIVE_SITE]['properties']: self.setLlapDaemonQueueName(services, configurations) + + # Check to see if 'cache' config HS2 'hive.llap.io.memory.size' has been modified by user. + # 'cache' size >= 64m implies config 'hive.llap.io.enabled' set to true, else false + cache_size_per_node_in_changed_configs = self.are_config_props_in_changed_configs(services, + "hive-interactive-site", + set(['hive.llap.io.memory.size']), + False) + if cache_size_per_node_in_changed_configs: + cache_size_per_node = self.get_cache_size_per_node_for_llap_nodes(services) + llap_io_enabled = 'false' + if cache_size_per_node >= 64: + llap_io_enabled = 'true' + putHiveInteractiveSiteProperty('hive.llap.io.enabled', llap_io_enabled) + Logger.info("Updated 'Hive Server interactive' config 'hive.llap.io.enabled' to '{0}'.".format(llap_io_enabled)) else: putHiveInteractiveEnvProperty('enable_hive_interactive', 'false') pass @@ -203,6 +217,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor): num_llap_nodes_in_changed_configs = False llap_daemon_selected_queue_name = None llap_queue_selected_in_current_call = None + cache_size_per_node_in_changed_configs = False try: if self.HIVE_INTERACTIVE_SITE in services['configurations'] and \ @@ -276,12 +291,24 @@ class HDP25StackAdvisor(HDP24StackAdvisor): assert (num_executors_per_node >= 0), "'Number of executors per node' : {0}. Expected value : > 0".format( num_executors_per_node) - # Get calculated value for prop 'hive.llap.io.memory.size' - cache_size_per_node, cache_size_per_node_max_limit = self.calculate_llap_cache_size_per_executor(services, + + # We calculate and update 'cache' value only if user has not modified it. + cache_size_per_node_in_changed_configs = self.are_config_props_in_changed_configs(services, + "hive-interactive-site", + set(['hive.llap.io.memory.size']), + False) + if not cache_size_per_node_in_changed_configs: + # Get calculated value for prop 'hive.llap.io.memory.size' + cache_size_per_node, cache_size_per_node_max_limit = self.calculate_llap_cache_size_per_executor(services, llap_container_size, num_executors_per_node) - assert (cache_size_per_node >= 0), "'Cache size per node' : {0}. Expected value : > 0".format( - cache_size_per_node) + if cache_size_per_node < 0: + Logger.info("Calculated 'cache_size_per_node' : {0}. Setting 'cache_size_per_node' to 0.".format(cache_size_per_node)) + cache_size_per_node = 0 + if cache_size_per_node_max_limit < 0: + Logger.info("Calculated 'cache_size_per_node_max_limit' : {0}. Setting 'cache_size_per_node_max_limit' to " + "0.".format(cache_size_per_node_max_limit)) + cache_size_per_node_max_limit = 0 # Get calculated value for prop 'llap_heap_size' llap_xmx = self.calculate_llap_app_heap_size(services, num_executors_per_node) @@ -290,53 +317,61 @@ class HDP25StackAdvisor(HDP24StackAdvisor): yarn_min_container_size = self.get_yarn_min_container_size(services) slider_am_container_mb = self.calculate_slider_am_size(yarn_min_container_size) - num_llap_nodes = long(num_llap_nodes) - num_llap_nodes_max_limit= long(num_llap_nodes_max_limit) - llap_container_size = long(llap_container_size) - llap_container_size_min_limit = long(llap_container_size_min_limit) - num_executors_per_node = long(num_executors_per_node) - num_executors_per_node_max_limit = long(num_executors_per_node_max_limit) - cache_size_per_node = long(cache_size_per_node) - cache_size_per_node_max_limit = long(cache_size_per_node_max_limit) - llap_xmx = long(llap_xmx) - slider_am_container_mb = long(slider_am_container_mb) - - Logger.info("Setting following value for 'hive.llap.daemon.yarn.container.mb'. Min : {0}, Curr: {1}, Max: {2}"\ - .format(llap_container_size_min_limit, llap_container_size, llap_container_size)) - Logger.info("Setting following value for 'hive.llap.daemon.num.executors'. Min : {0}, Curr: {1}, Max: {2}"\ - .format('1', num_executors_per_node, num_executors_per_node_max_limit)) - Logger.info("Setting following value for 'hive.llap.io.memory.size'. Min : {0}, Curr: {1}, Max: {2}"\ - .format('0', cache_size_per_node, cache_size_per_node_max_limit)) - Logger.info("Setting following value for 'llap_heap_size'. Curr: {0}".format(llap_xmx)) - Logger.info("Setting following value for 'slider_am_container_mb'. Curr: {0}".format(slider_am_container_mb)) + # Updating configs. if not num_llap_nodes_in_changed_configs: - Logger.info("Setting following value for 'num_llap_nodes'. Min : {0}, Curr: {1}, Max: {2}" \ + num_llap_nodes = long(num_llap_nodes) + num_llap_nodes_max_limit= long(num_llap_nodes_max_limit) + + Logger.info("LLAP config 'num_llap_nodes' updated. Min : {0}, Curr: {1}, Max: {2}" \ .format('1', num_llap_nodes, num_llap_nodes_max_limit)) putHiveInteractiveEnvProperty('num_llap_nodes', num_llap_nodes) putHiveInteractiveEnvPropertyAttribute('num_llap_nodes', "minimum", 1) putHiveInteractiveEnvPropertyAttribute('num_llap_nodes', "maximum", int(num_llap_nodes_max_limit)) + llap_container_size = long(llap_container_size) + llap_container_size_min_limit = long(llap_container_size_min_limit) putHiveInteractiveSiteProperty('hive.llap.daemon.yarn.container.mb', llap_container_size) putHiveInteractiveSitePropertyAttribute('hive.llap.daemon.yarn.container.mb', "minimum", llap_container_size_min_limit) putHiveInteractiveSitePropertyAttribute('hive.llap.daemon.yarn.container.mb', "maximum", llap_container_size) + Logger.info("LLAP config 'hive.llap.daemon.yarn.container.mb' updated. Min : {0}, Curr: {1}, Max: {2}" \ + .format(llap_container_size_min_limit, llap_container_size, llap_container_size)) + num_executors_per_node = long(num_executors_per_node) + num_executors_per_node_max_limit = long(num_executors_per_node_max_limit) putHiveInteractiveSiteProperty('hive.llap.daemon.num.executors', num_executors_per_node) putHiveInteractiveSitePropertyAttribute('hive.llap.daemon.num.executors', "minimum", 1) putHiveInteractiveSitePropertyAttribute('hive.llap.daemon.num.executors', "maximum", num_executors_per_node_max_limit) + Logger.info("LLAP config 'hive.llap.daemon.num.executors' updated. Min : {0}, Curr: {1}, Max: {2}" \ + .format('1', num_executors_per_node, num_executors_per_node_max_limit)) - putHiveInteractiveSiteProperty('hive.llap.io.memory.size', cache_size_per_node) - putHiveInteractiveSitePropertyAttribute('hive.llap.io.memory.size', "minimum", 0) # 0 -> Disables caching. - putHiveInteractiveSitePropertyAttribute('hive.llap.io.memory.size', "maximum", - cache_size_per_node_max_limit) + if not cache_size_per_node_in_changed_configs: + cache_size_per_node = long(cache_size_per_node) + cache_size_per_node_max_limit = long(cache_size_per_node_max_limit) + putHiveInteractiveSiteProperty('hive.llap.io.memory.size', cache_size_per_node) + putHiveInteractiveSitePropertyAttribute('hive.llap.io.memory.size', "minimum", 0) # 0 -> Disables caching. + putHiveInteractiveSitePropertyAttribute('hive.llap.io.memory.size', "maximum", + cache_size_per_node_max_limit) + llap_io_enabled = 'false' + if cache_size_per_node >= 64: + llap_io_enabled = 'true' + + putHiveInteractiveSiteProperty('hive.llap.io.enabled', llap_io_enabled) + Logger.info("LLAP config 'hive.llap.io.memory.size' updated. Min : {0}, Curr: {1}, Max: {2}" \ + .format('0', cache_size_per_node, cache_size_per_node_max_limit)) + Logger.info("HiveServer2 config 'hive.llap.io.enabled' updated to '{0}' as part of " + "'hive.llap.io.memory.size' calculation.".format(llap_io_enabled)) + llap_xmx = long(llap_xmx) putHiveInteractiveEnvProperty('llap_heap_size', llap_xmx) + Logger.info("LLAP config 'llap_heap_size' updated. Curr: {0}".format(llap_xmx)) + slider_am_container_mb = long(slider_am_container_mb) putHiveInteractiveEnvProperty('slider_am_container_mb', slider_am_container_mb) - + Logger.info("LLAP config 'slider_am_container_mb' updated. Curr: {0}".format(slider_am_container_mb)) except Exception as e: Logger.info(e.message+" Skipping calculating LLAP configs. Setting them to minimum values.") @@ -429,6 +464,21 @@ class HDP25StackAdvisor(HDP24StackAdvisor): """ + Returns current value of cache per node for LLAP (hive.llap.io.memory.size) + """ + def get_cache_size_per_node_for_llap_nodes(self, services): + if 'hive.llap.io.memory.size' in services['configurations']['hive-interactive-site']['properties']: + cache_size_per_node = float( + services['configurations']['hive-interactive-site']['properties']['hive.llap.io.memory.size']) + return cache_size_per_node + else: + Logger.error("Couldn't retrieve Hive Server interactive's 'hive.llap.io.memory.size' config.") + # Not doing raise as the Exception that catches it will set all other LLAP configs related + # to LLAP package as 0, a way to tell that calulations couldn't be done. This is not the intention here. + # Just keep cache 0, if it couldn't be retrieved. + return 0 + + """ Returns current value of number of LLAP nodes in cluster (num_llap_nodes) """ def get_num_llap_nodes(self, services): @@ -470,14 +520,14 @@ class HDP25StackAdvisor(HDP24StackAdvisor): """ def get_tez_container_size(self, services): hive_container_size = 0 - if 'hive.tez.container.size' in services['configurations'][self.HIVE_INTERACTIVE_SITE]['properties']: + if 'hive.tez.container.size' in services['configurations']['hive-site']['properties']: hive_container_size = float( - services['configurations'][self.HIVE_INTERACTIVE_SITE]['properties']['hive.tez.container.size']) + services['configurations']['hive-site']['properties']['hive.tez.container.size']) assert ( hive_container_size > 0), "'hive.tez.container.size' current value : {0}. Expected value : > 0".format( hive_container_size) else: - raise Fail("Couldn't retrieve Hive Server interactive's 'hive.tez.container.size' config.") + raise Fail("Couldn't retrieve Hive Server 'hive.tez.container.size' config.") return hive_container_size @@ -738,7 +788,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor): llap_slider_cap_percentage = int( services['configurations']['hive-interactive-env']['properties']['llap_queue_capacity']) llap_min_reqd_cap_percentage = self.min_llap_queue_perc_required_in_cluster(services, hosts) - if llap_slider_cap_percentage not in range(1,101): + if llap_slider_cap_percentage <= 0 or llap_slider_cap_percentage > 100: Logger.info("Adjusting HIVE 'llap_queue_capacity' from {0}% to {1}%".format(llap_slider_cap_percentage, llap_min_reqd_cap_percentage)) putHiveInteractiveEnvProperty('llap_queue_capacity', llap_min_reqd_cap_percentage) llap_slider_cap_percentage = llap_min_reqd_cap_percentage @@ -832,6 +882,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor): # Update Hive 'hive.llap.daemon.queue.name' prop to use 'llap' queue. putHiveInteractiveSiteProperty('hive.llap.daemon.queue.name', 'llap') putHiveInteractiveEnvPropertyAttribute('llap_queue_capacity', "minimum", llap_min_reqd_cap_percentage) + putHiveInteractiveEnvPropertyAttribute('llap_queue_capacity', "maximum", 100) # Update 'hive.llap.daemon.queue.name' prop combo entries. self.setLlapDaemonQueueName(services, configurations) http://git-wip-us.apache.org/repos/asf/ambari/blob/6dc2c955/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py index 574964f..e9ddc25 100644 --- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py @@ -334,8 +334,7 @@ class TestHDP25StackAdvisor(TestCase): "hive-interactive-site": { 'properties': { - 'hive.llap.daemon.queue.name':'default', - 'hive.tez.container.size': '341' + 'hive.llap.daemon.queue.name':'default' } }, "tez-interactive-site": { @@ -348,7 +347,13 @@ class TestHDP25StackAdvisor(TestCase): 'properties': { 'hive_user': 'hive' } - } + }, + "hive-site": + { + 'properties': { + 'hive.tez.container.size': '341' + } + }, } } @@ -554,7 +559,6 @@ class TestHDP25StackAdvisor(TestCase): { 'properties': { 'hive.llap.daemon.queue.name':'default', - 'hive.tez.container.size': '341' } }, "tez-interactive-site": { @@ -562,13 +566,18 @@ class TestHDP25StackAdvisor(TestCase): "tez.am.resource.memory.mb": "341" } }, - "hive-env": { 'properties': { 'hive_user': 'hive' } - } + }, + "hive-site": + { + 'properties': { + 'hive.tez.container.size': '341' + } + }, } } @@ -1323,8 +1332,7 @@ class TestHDP25StackAdvisor(TestCase): { 'properties': { 'hive.llap.daemon.queue.name':'llap', - 'hive.server2.tez.sessions.per.default.queue' : '1', - 'hive.tez.container.size': '341' + 'hive.server2.tez.sessions.per.default.queue' : '1' } }, "tez-interactive-site": { @@ -1337,7 +1345,13 @@ class TestHDP25StackAdvisor(TestCase): 'properties': { 'hive_user': 'hive' } - } + }, + "hive-site": + { + 'properties': { + 'hive.tez.container.size': '341' + } + }, } } # Services 11: YARN service with : (1). 'capacity scheduler' having 'llap' and 'default' queue at root level and @@ -1438,8 +1452,7 @@ class TestHDP25StackAdvisor(TestCase): "hive-interactive-site": { 'properties': { - 'hive.llap.daemon.queue.name':'llap', - 'hive.tez.container.size': '341' + 'hive.llap.daemon.queue.name':'llap' } }, "tez-interactive-site": { @@ -1459,7 +1472,13 @@ class TestHDP25StackAdvisor(TestCase): 'properties': { 'hive_user': 'hive' } - } + }, + "hive-site": + { + 'properties': { + 'hive.tez.container.size': '341' + } + }, } } @@ -1711,9 +1730,7 @@ class TestHDP25StackAdvisor(TestCase): { 'properties': { 'hive.llap.daemon.queue.name': 'llap', - 'hive.server2.tez.sessions.per.default.queue': '1', - 'hive.tez.container.size': '341' - + 'hive.server2.tez.sessions.per.default.queue': '1' } }, "hive-env": @@ -1733,7 +1750,13 @@ class TestHDP25StackAdvisor(TestCase): "properties": { "tez.am.resource.memory.mb": "341" } - } + }, + "hive-site": + { + 'properties': { + 'hive.tez.container.size': '341' + } + }, } } @@ -1879,9 +1902,7 @@ class TestHDP25StackAdvisor(TestCase): { 'properties': { 'hive.llap.daemon.queue.name': 'llap', - 'hive.server2.tez.sessions.per.default.queue': '1', - 'hive.tez.container.size': '341' - + 'hive.server2.tez.sessions.per.default.queue': '1' } }, "hive-env": @@ -1901,7 +1922,13 @@ class TestHDP25StackAdvisor(TestCase): "properties": { "tez.am.resource.memory.mb": "341" } - } + }, + "hive-site": + { + 'properties': { + 'hive.tez.container.size': '341' + } + }, } } @@ -2022,9 +2049,7 @@ class TestHDP25StackAdvisor(TestCase): { 'properties': { 'hive.llap.daemon.queue.name': 'default', - 'hive.server2.tez.sessions.per.default.queue': '1', - 'hive.tez.container.size': '341' - + 'hive.server2.tez.sessions.per.default.queue': '1' } }, "hive-env": @@ -2044,7 +2069,13 @@ class TestHDP25StackAdvisor(TestCase): "properties": { "tez.am.resource.memory.mb": "341" } - } + }, + "hive-site": + { + 'properties': { + 'hive.tez.container.size': '341' + } + }, } } @@ -2165,9 +2196,7 @@ class TestHDP25StackAdvisor(TestCase): { 'properties': { 'hive.llap.daemon.queue.name': 'llap', - 'hive.server2.tez.sessions.per.default.queue': '1', - 'hive.tez.container.size': '341' - + 'hive.server2.tez.sessions.per.default.queue': '1' } }, "hive-env": @@ -2187,7 +2216,13 @@ class TestHDP25StackAdvisor(TestCase): "properties": { "tez.am.resource.memory.mb": "341" } - } + }, + "hive-site": + { + 'properties': { + 'hive.tez.container.size': '341' + } + }, } } @@ -2307,9 +2342,7 @@ class TestHDP25StackAdvisor(TestCase): { 'properties': { 'hive.llap.daemon.queue.name': 'llap', - 'hive.server2.tez.sessions.per.default.queue': '1', - 'hive.tez.container.size': '341' - + 'hive.server2.tez.sessions.per.default.queue': '1' } }, "hive-env": @@ -2443,9 +2476,7 @@ class TestHDP25StackAdvisor(TestCase): { 'properties': { 'hive.llap.daemon.queue.name': 'llap', - 'hive.server2.tez.sessions.per.default.queue': '1', - 'hive.tez.container.size': '341' - + 'hive.server2.tez.sessions.per.default.queue': '1' } }, "hive-env": @@ -2587,7 +2618,6 @@ class TestHDP25StackAdvisor(TestCase): 'properties': { 'hive.llap.daemon.queue.name': 'llap', 'hive.server2.tez.sessions.per.default.queue': '1', - 'hive.tez.container.size': '341' } }, @@ -2608,7 +2638,13 @@ class TestHDP25StackAdvisor(TestCase): "properties": { "tez.am.resource.memory.mb": "341" } - } + }, + "hive-site": + { + 'properties': { + 'hive.tez.container.size': '341' + } + }, } } @@ -2729,9 +2765,7 @@ class TestHDP25StackAdvisor(TestCase): { 'properties': { 'hive.llap.daemon.queue.name': 'llap', - 'hive.server2.tez.sessions.per.default.queue': '1', - 'hive.tez.container.size': '341' - + 'hive.server2.tez.sessions.per.default.queue': '1' } }, "hive-env": @@ -2751,7 +2785,13 @@ class TestHDP25StackAdvisor(TestCase): "properties": { "tez.am.resource.memory.mb": "341" } - } + }, + "hive-site": + { + 'properties': { + 'hive.tez.container.size': '341' + } + }, } } @@ -2944,8 +2984,8 @@ class TestHDP25StackAdvisor(TestCase): # expected vals. expected_visibility_false = {'visible': 'false'} expected_visibility_true = {'visible': 'true'} - expected_llap_queue_attributes_min_25_visible_true = {'minimum': '25', 'visible': 'true'} - expected_llap_queue_attributes_min_21_visible_true = {'minimum': '21', 'visible': 'true'} + expected_llap_queue_attributes_min_25_visible_true = {'maximum': '100', 'minimum': '25', 'visible': 'true'} + expected_llap_queue_attributes_min_21_visible_true = {'maximum': '100', 'minimum': '21', 'visible': 'true'} @@ -2959,7 +2999,6 @@ class TestHDP25StackAdvisor(TestCase): } self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services_1, hosts) # Check output - print "\n\n\n\n conf = ",configurations self.assertEquals(configurations['hive-interactive-site']['properties']['hive.llap.daemon.queue.name'], expected_hive_interactive_site_llap['hive-interactive-site']['properties']['hive.llap.daemon.queue.name']) self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name'], @@ -3184,7 +3223,6 @@ class TestHDP25StackAdvisor(TestCase): } # expected vals. - print "\n\n\n\n\n\n\n\n\n\n\n" yarn_cont_mb_attributes = {'minimum': '682', 'maximum': '682'} llap_io_mem_size_attributes = {'minimum': '0', 'maximum': '341'} llap_num_executors_attributes = {'minimum': '1', 'maximum': '1'}
