AMBARI-19971. HiveServerInteractive. (1). Use the correct value for Node(s) used for running LLAP Daemon(s). (2). Set 'num_llap_nodes' calculated value only for non Ambari managed queue.
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f4c83843 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f4c83843 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f4c83843 Branch: refs/heads/branch-feature-AMBARI-12556 Commit: f4c83843674764918bd50443705af1fce9852a2b Parents: 38a17a7 Author: Swapan Shridhar <[email protected]> Authored: Fri Feb 10 13:29:41 2017 -0800 Committer: Swapan Shridhar <[email protected]> Committed: Fri Feb 10 15:13:48 2017 -0800 ---------------------------------------------------------------------- .../package/scripts/hive_server_interactive.py | 12 ++--- .../0.12.0.2.0/package/scripts/params_linux.py | 2 + .../HIVE/configuration/hive-interactive-env.xml | 4 +- .../stacks/HDP/2.5/services/stack_advisor.py | 9 +++- .../HIVE/configuration/hive-interactive-env.xml | 31 ++++++++++++- .../stacks/2.5/common/test_stack_advisor.py | 49 ++++++++++++++------ 6 files changed, 83 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/f4c83843/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py index 1f1d9a8..2ab48ae 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py @@ -281,10 +281,9 @@ class HiveServerInteractiveDefault(HiveServerInteractive): unique_name = "llap-slider%s" % datetime.utcnow().strftime('%Y-%m-%d_%H-%M-%S') - cmd = format("{stack_root}/current/hive-server2-hive2/bin/hive --service llap --instances {params.num_llap_nodes}" - " --slider-am-container-mb {params.slider_am_container_mb} --size {params.llap_daemon_container_size}m" - " --cache {params.hive_llap_io_mem_size}m --xmx {params.llap_heap_size}m --loglevel {params.llap_log_level}" - " {params.llap_extra_slider_opts} --output {LLAP_PACKAGE_CREATION_PATH}/{unique_name}") + cmd = format("{stack_root}/current/hive-server2-hive2/bin/hive --service llap --slider-am-container-mb {params.slider_am_container_mb} " + "--size {params.llap_daemon_container_size}m --cache {params.hive_llap_io_mem_size}m --xmx {params.llap_heap_size}m " + "--loglevel {params.llap_log_level} {params.llap_extra_slider_opts} --output {LLAP_PACKAGE_CREATION_PATH}/{unique_name}") # Append params that are supported from Hive llap GA version. if params.stack_supports_hive_interactive_ga: @@ -300,11 +299,12 @@ class HiveServerInteractiveDefault(HiveServerInteractive): else: Logger.info("Setting slider_placement: 4, as llap_daemon_container_size : {0} <= 0.5 * " "YARN NodeManager Memory({1})".format(params.llap_daemon_container_size, params.yarn_nm_mem)) - cmd += format(" --slider-placement {slider_placement} --skiphadoopversion --skiphbasecp") + cmd += format(" --slider-placement {slider_placement} --skiphadoopversion --skiphbasecp --instances {params.num_llap_daemon_running_nodes}") # Setup the logger for the ga version only cmd += format(" --logger {params.llap_logger}") - + else: + cmd += format(" --instances {params.num_llap_nodes}") if params.security_enabled: llap_keytab_splits = params.hive_llap_keytab_file.split("/") Logger.debug("llap_keytab_splits : {0}".format(llap_keytab_splits)) http://git-wip-us.apache.org/repos/asf/ambari/blob/f4c83843/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py index a5618ca..936b194 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py @@ -685,6 +685,8 @@ if has_hive_interactive: num_retries_for_checking_llap_status = default('/configurations/hive-interactive-env/num_retries_for_checking_llap_status', 10) # Used in LLAP slider package creation yarn_nm_mem = config['configurations']['yarn-site']['yarn.nodemanager.resource.memory-mb'] + if stack_supports_hive_interactive_ga: + num_llap_daemon_running_nodes = config['configurations']['hive-interactive-env']['num_llap_nodes_for_llap_daemons'] num_llap_nodes = config['configurations']['hive-interactive-env']['num_llap_nodes'] llap_daemon_container_size = config['configurations']['hive-interactive-site']['hive.llap.daemon.yarn.container.mb'] llap_log_level = config['configurations']['hive-interactive-env']['llap_log_level'] http://git-wip-us.apache.org/repos/asf/ambari/blob/f4c83843/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml index e636e0c..ef9be76 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml @@ -45,8 +45,8 @@ <property> <name>num_llap_nodes</name> <value>1</value> - <description>The number of Hive LLAP daemons to run.</description> - <display-name>Number of LLAP Nodes</display-name> + <description>Number of nodes used by Hive's LLAP, which includes nodes running : LLAP daemon, Slider and Tez App Master(s).</description> + <display-name>Number of nodes used by Hive's LLAP</display-name> <value-attributes> <type>int</type> <minimum>1</minimum> http://git-wip-us.apache.org/repos/asf/ambari/blob/f4c83843/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py index 99b6776..9fe8fc3 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py @@ -951,10 +951,13 @@ class HDP25StackAdvisor(HDP24StackAdvisor): Logger.info("DBG: Calculated '{0}' queue available capacity : {1}, using following: llap_daemon_selected_queue_cap : {2}, " "yarn_min_container_size : {3}".format(llap_daemon_selected_queue_name, total_llap_mem_normalized, llap_daemon_selected_queue_cap, yarn_min_container_size)) - '''Rounding up numNodes so that we run more daemons, and utilitze more CPUs. The rest of the calcaulkations will take care of cutting this down if required''' + '''Rounding up numNodes so that we run more daemons, and utilitze more CPUs. The rest of the calcaulations will take care of cutting this down if required''' num_llap_nodes_requested = math.ceil(total_llap_mem_normalized / yarn_nm_mem_in_mb_normalized) Logger.info("DBG: Calculated 'num_llap_nodes_requested' : {0}, using following: total_llap_mem_normalized : {1}, " "yarn_nm_mem_in_mb_normalized : {2}".format(num_llap_nodes_requested, total_llap_mem_normalized, yarn_nm_mem_in_mb_normalized)) + # Pouplate the 'num_llap_nodes_requested' in config 'num_llap_nodes', a read only config for non-Ambari managed queue case. + putHiveInteractiveEnvProperty('num_llap_nodes', num_llap_nodes_requested) + Logger.info("Setting config 'num_llap_nodes' as : {0}".format(num_llap_nodes_requested)) queue_am_fraction_perc = float(self.__getQueueAmFractionFromCapacityScheduler(capacity_scheduler_properties, llap_daemon_selected_queue_name)) hive_tez_am_cap_available = queue_am_fraction_perc * total_llap_mem_normalized Logger.info("DBG: Calculated 'hive_tez_am_cap_available' : {0}, using following: queue_am_fraction_perc : {1}, " @@ -1182,7 +1185,8 @@ class HDP25StackAdvisor(HDP24StackAdvisor): Logger.info("User requested num_llap_nodes : {0}, but used/adjusted value for calculations is : {1}".format(num_llap_nodes_requested, num_llap_nodes)) else: Logger.info("Used num_llap_nodes for calculations : {0}".format(num_llap_nodes_requested)) - putHiveInteractiveEnvProperty('num_llap_nodes', num_llap_nodes) + putHiveInteractiveEnvProperty('num_llap_nodes_for_llap_daemons', num_llap_nodes) + Logger.info("Setting config 'num_llap_nodes_for_llap_daemons' as : {0}".format(num_llap_nodes)) llap_container_size = long(llap_daemon_mem_per_node) putHiveInteractiveSiteProperty('hive.llap.daemon.yarn.container.mb', llap_container_size) @@ -1243,6 +1247,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor): putHiveInteractiveSitePropertyAttribute('hive.server2.tez.sessions.per.default.queue', "minimum", 1) putHiveInteractiveSitePropertyAttribute('hive.server2.tez.sessions.per.default.queue', "maximum", 1) putHiveInteractiveEnvProperty('num_llap_nodes', 0) + putHiveInteractiveEnvProperty('num_llap_nodes_for_llap_daemons', 0) putHiveInteractiveEnvPropertyAttribute('num_llap_nodes', "minimum", 1) putHiveInteractiveEnvPropertyAttribute('num_llap_nodes', "maximum", node_manager_cnt) putHiveInteractiveSiteProperty('hive.llap.daemon.yarn.container.mb', yarn_min_container_size) http://git-wip-us.apache.org/repos/asf/ambari/blob/f4c83843/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-env.xml index af656f4..b659205 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-env.xml @@ -62,7 +62,36 @@ </value-attributes> <on-ambari-upgrade add="false"/> </property> - + <property> + <name>num_llap_nodes_for_llap_daemons</name> + <value>1</value> + <description>Number of Node(s) on which Hive LLAP daemon runs.</description> + <display-name>Number of Node(s) for running Hive LLAP daemon</display-name> + <value-attributes> + <type>int</type> + <minimum>1</minimum> + <increment-step>1</increment-step> + </value-attributes> + <depends-on> + <property> + <type>hive-interactive-env</type> + <name>num_llap_nodes</name> + </property> + <property> + <type>hive-interactive-env</type> + <name>enable_hive_interactive</name> + </property> + <property> + <type>hive-interactive-site</type> + <name>hive.llap.daemon.queue.name</name> + </property> + <property> + <type>capacity-scheduler</type> + <name>yarn.scheduler.capacity.root.queues</name> + </property> + </depends-on> + <on-ambari-upgrade add="true"/> + </property> <!-- hive-env.sh --> http://git-wip-us.apache.org/repos/asf/ambari/blob/f4c83843/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py index ec56bad..ef83bca 100644 --- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py @@ -1169,8 +1169,8 @@ class TestHDP25StackAdvisor(TestCase): self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.sessions.per.default.queue'], '1') self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.server2.tez.sessions.per.default.queue'], {'minimum': '1', 'maximum': '4'}) - self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes'], 3) - + self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons'], 3) + self.assertTrue('num_llap_nodes' not in configurations['hive-interactive-env']['properties']) self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '9216') @@ -1369,6 +1369,8 @@ class TestHDP25StackAdvisor(TestCase): self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.server2.tez.sessions.per.default.queue'], {'maximum': '3.0'}) self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes'], 3) + self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons'], 3) + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '9548') self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors'], '1') @@ -1561,7 +1563,9 @@ class TestHDP25StackAdvisor(TestCase): self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.server2.tez.sessions.per.default.queue'], {'maximum': '4'}) - self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes'], 3) + self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons'], 3) + self.assertTrue('num_llap_nodes' not in configurations['hive-interactive-env']['properties']) + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '48128') self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors'], '1') @@ -1762,7 +1766,9 @@ class TestHDP25StackAdvisor(TestCase): self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.sessions.per.default.queue'], '1') self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.server2.tez.sessions.per.default.queue'], {'maximum': '4', 'minimum': '1'}) - self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes'], 3) + self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons'], 3) + self.assertTrue('num_llap_nodes' not in configurations['hive-interactive-env']['properties']) + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '38912') self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors'], '4') @@ -1953,14 +1959,15 @@ class TestHDP25StackAdvisor(TestCase): configurations = { } - self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, self.hosts) - self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.sessions.per.default.queue'], '1') self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.server2.tez.sessions.per.default.queue'], {'minimum': '1', 'maximum': '4'}) self.assertEqual(configurations['capacity-scheduler']['properties'], {'capacity-scheduler': 'yarn.scheduler.capacity.root.accessible-node-labels=*\nyarn.scheduler.capacity.maximum-am-resource-percent=1\nyarn.scheduler.capacity.node-locality-delay=40\nyarn.scheduler.capacity.root.capacity=100\nyarn.scheduler.capacity.root.default.state=RUNNING\nyarn.scheduler.capacity.root.default.maximum-capacity=66.0\nyarn.scheduler.capacity.root.queues=default,llap\nyarn.scheduler.capacity.maximum-applications=10000\nyarn.scheduler.capacity.root.default.user-limit-factor=1\nyarn.scheduler.capacity.root.acl_administer_queue=*\nyarn.scheduler.capacity.root.default.acl_submit_applications=*\nyarn.scheduler.capacity.root.default.capacity=66.0\nyarn.scheduler.capacity.queue-mappings-override.enable=false\nyarn.scheduler.capacity.root.llap.user-limit-factor=1\nyarn.scheduler.capacity.root.llap.state=RUNNING\nyarn.scheduler.capacity.root.llap.ordering-policy=fifo\nyarn.scheduler.capacity.root.llap.mi nimum-user-limit-percent=100\nyarn.scheduler.capacity.root.llap.maximum-capacity=34.0\nyarn.scheduler.capacity.root.llap.capacity=34.0\nyarn.scheduler.capacity.root.llap.acl_submit_applications=hive\nyarn.scheduler.capacity.root.llap.acl_administer_queue=hive\nyarn.scheduler.capacity.root.llap.maximum-am-resource-percent=1'}) - self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes'], 3) + self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons'], 3) + self.assertTrue('num_llap_nodes' not in configurations['hive-interactive-env']['properties']) + + self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons'], 3) self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '10571') self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors'], '3') @@ -1988,7 +1995,7 @@ class TestHDP25StackAdvisor(TestCase): # Small configuration test with 3 nodes - 'yarn.nodemanager.resource.memory-mb' : 2046 and 'yarn.scheduler.minimum-allocation-mb' : 682, representing a small GCE cluster. # # Expected : Configurations values recommended for llap related configs. - def test_recommendYARNConfigurations_three_node_manager_llap_configs_updated_2(self): + def test_recommendYARNConfigurations_three_node_manager_llap_configs_updated_3(self): # 3 node managers and yarn.nodemanager.resource.memory-mb": "12288" services = { "services": [{ @@ -2158,7 +2165,9 @@ class TestHDP25StackAdvisor(TestCase): self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.server2.tez.sessions.per.default.queue'], {'minimum': '1', 'maximum': '3.0'}) self.assertEqual(configurations['capacity-scheduler']['properties'], {'capacity-scheduler': 'yarn.scheduler.capacity.root.accessible-node-labels=*\nyarn.scheduler.capacity.maximum-am-resource-percent=1\nyarn.scheduler.capacity.node-locality-delay=40\nyarn.scheduler.capacity.root.capacity=100\nyarn.scheduler.capacity.root.default.state=RUNNING\nyarn.scheduler.capacity.root.default.maximum-capacity=66.0\nyarn.scheduler.capacity.root.queues=default,llap\nyarn.scheduler.capacity.maximum-applications=10000\nyarn.scheduler.capacity.root.default.user-limit-factor=1\nyarn.scheduler.capacity.root.acl_administer_queue=*\nyarn.scheduler.capacity.root.default.acl_submit_applications=*\nyarn.scheduler.capacity.root.default.capacity=66.0\nyarn.scheduler.capacity.queue-mappings-override.enable=false\nyarn.scheduler.capacity.root.llap.user-limit-factor=1\nyarn.scheduler.capacity.root.llap.state=RUNNING\nyarn.scheduler.capacity.root.llap.ordering-policy=fifo\nyarn.scheduler.capacity.root.llap.mi nimum-user-limit-percent=100\nyarn.scheduler.capacity.root.llap.maximum-capacity=34.0\nyarn.scheduler.capacity.root.llap.capacity=34.0\nyarn.scheduler.capacity.root.llap.acl_submit_applications=hive\nyarn.scheduler.capacity.root.llap.acl_administer_queue=hive\nyarn.scheduler.capacity.root.llap.maximum-am-resource-percent=1'}) - self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes'], 3) + self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons'], 3) + self.assertTrue('num_llap_nodes' not in configurations['hive-interactive-env']['properties']) + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '682') self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors'], '1') @@ -2354,7 +2363,8 @@ class TestHDP25StackAdvisor(TestCase): self.assertEqual(configurations['capacity-scheduler']['properties'], {'capacity-scheduler': 'yarn.scheduler.capacity.root.accessible-node-labels=*\nyarn.scheduler.capacity.maximum-am-resource-percent=1\nyarn.scheduler.capacity.node-locality-delay=40\nyarn.scheduler.capacity.root.capacity=100\nyarn.scheduler.capacity.root.default.state=RUNNING\nyarn.scheduler.capacity.root.default.maximum-capacity=0.0\nyarn.scheduler.capacity.root.queues=default,llap\nyarn.scheduler.capacity.maximum-applications=10000\nyarn.scheduler.capacity.root.default.user-limit-factor=1\nyarn.scheduler.capacity.root.acl_administer_queue=*\nyarn.scheduler.capacity.root.default.acl_submit_applications=*\nyarn.scheduler.capacity.root.default.capacity=0.0\nyarn.scheduler.capacity.queue-mappings-override.enable=false\nyarn.scheduler.capacity.root.llap.user-limit-factor=1\nyarn.scheduler.capacity.root.llap.state=RUNNING\nyarn.scheduler.capacity.root.llap.ordering-policy=fifo\nyarn.scheduler.capacity.root.llap.mini mum-user-limit-percent=100\nyarn.scheduler.capacity.root.llap.maximum-capacity=100.0\nyarn.scheduler.capacity.root.llap.capacity=100.0\nyarn.scheduler.capacity.root.llap.acl_submit_applications=hive\nyarn.scheduler.capacity.root.llap.acl_administer_queue=hive\nyarn.scheduler.capacity.root.llap.maximum-am-resource-percent=1'}) self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.server2.tez.sessions.per.default.queue'], {'maximum': '4'}) - self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes'], 3) + self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons'], 3) + self.assertTrue('num_llap_nodes' not in configurations['hive-interactive-env']['properties']) self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '202752') self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors'], '3') @@ -2557,7 +2567,9 @@ class TestHDP25StackAdvisor(TestCase): self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.sessions.per.default.queue'], '1.0') self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.server2.tez.sessions.per.default.queue'], {'maximum': '4', 'minimum': '1'}) - self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes'], 5) + self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons'], 3) + self.assertTrue('num_llap_nodes' not in configurations['hive-interactive-env']['properties']) + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '36864') self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors'], '4') @@ -2754,7 +2766,9 @@ class TestHDP25StackAdvisor(TestCase): self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.sessions.per.default.queue'], '1') self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.server2.tez.sessions.per.default.queue'], {'maximum': '4', 'minimum': '1'}) - self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes'], 3) + self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons'], 3) + self.assertTrue('num_llap_nodes' not in configurations['hive-interactive-env']['properties']) + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '203918') self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors'], '10') @@ -2947,7 +2961,9 @@ class TestHDP25StackAdvisor(TestCase): self.assertEqual(configurations['capacity-scheduler']['properties'], {'capacity-scheduler': 'yarn.scheduler.capacity.root.accessible-node-labels=*\nyarn.scheduler.capacity.maximum-am-resource-percent=1\nyarn.scheduler.capacity.node-locality-delay=40\nyarn.scheduler.capacity.root.capacity=100\nyarn.scheduler.capacity.root.default.state=RUNNING\nyarn.scheduler.capacity.root.default.maximum-capacity=0.0\nyarn.scheduler.capacity.root.queues=default,llap\nyarn.scheduler.capacity.maximum-applications=10000\nyarn.scheduler.capacity.root.default.user-limit-factor=1\nyarn.scheduler.capacity.root.acl_administer_queue=*\nyarn.scheduler.capacity.root.default.acl_submit_applications=*\nyarn.scheduler.capacity.root.default.capacity=0.0\nyarn.scheduler.capacity.queue-mappings-override.enable=false\nyarn.scheduler.capacity.root.llap.user-limit-factor=1\nyarn.scheduler.capacity.root.llap.state=RUNNING\nyarn.scheduler.capacity.root.llap.ordering-policy=fifo\nyarn.scheduler.capacity.root.llap.mini mum-user-limit-percent=100\nyarn.scheduler.capacity.root.llap.maximum-capacity=100.0\nyarn.scheduler.capacity.root.llap.capacity=100.0\nyarn.scheduler.capacity.root.llap.acl_submit_applications=hive\nyarn.scheduler.capacity.root.llap.acl_administer_queue=hive\nyarn.scheduler.capacity.root.llap.maximum-am-resource-percent=1'}) self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.server2.tez.sessions.per.default.queue'], {'maximum': '4'}) - self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes'], 3) + self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons'], 3) + self.assertTrue('num_llap_nodes' not in configurations['hive-interactive-env']['properties']) + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '202752') self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors'], '3') @@ -3139,6 +3155,8 @@ class TestHDP25StackAdvisor(TestCase): self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.server2.tez.sessions.per.default.queue'], {'maximum': '4'}) self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes'], 3) + self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons'], 3) + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '202752') self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors'], '3') @@ -3361,6 +3379,8 @@ class TestHDP25StackAdvisor(TestCase): self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.server2.tez.sessions.per.default.queue'], {'maximum': '4', 'minimum': '1'}) self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes'], 3) + self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons'], 3) + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '204288') self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors'], '3') @@ -3580,6 +3600,7 @@ class TestHDP25StackAdvisor(TestCase): self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, self.hosts) self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes'], '0') + self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons'], 0) self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '2048') @@ -3962,6 +3983,8 @@ class TestHDP25StackAdvisor(TestCase): self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.server2.tez.sessions.per.default.queue'], {'maximum': '4', 'minimum': '1'}) self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes'], 1) + self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons'], 1) + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '200704') self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors'], '3')
