AMBARI-20279. HiveServerInteractive. Set value for num_llap_nodes_for_llap_daemons only if it exists in hive-interactive-site.
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1142b526 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1142b526 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1142b526 Branch: refs/heads/branch-feature-AMBARI-12556 Commit: 1142b526195feced7b42c82119220e065d56e24b Parents: 7e2eb29 Author: Swapan Shridhar <sshrid...@hortonworks.com> Authored: Thu Mar 2 14:25:12 2017 -0800 Committer: Swapan Shridhar <sshrid...@hortonworks.com> Committed: Thu Mar 2 17:19:33 2017 -0800 ---------------------------------------------------------------------- .../YARN/3.0.0.3.0/service_advisor.py | 15 ++++++++-- .../stacks/HDP/2.5/services/stack_advisor.py | 17 +++++++++-- .../stacks/2.5/common/test_stack_advisor.py | 30 +++++++++++++------- 3 files changed, 45 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/1142b526/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py index 3629f30..3d77a26 100644 --- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py +++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py @@ -918,8 +918,12 @@ class YARNRecommender(service_advisor.ServiceAdvisor): else: Logger.info("DBG: Used num_llap_nodes for calculations : {0}".format(num_llap_nodes_requested)) - putHiveInteractiveEnvProperty('num_llap_nodes_for_llap_daemons', num_llap_nodes) - Logger.info("DBG: Setting config 'num_llap_nodes_for_llap_daemons' as : {0}".format(num_llap_nodes)) + # Safeguard for not adding "num_llap_nodes_for_llap_daemons" if it doesnt exist in hive-interactive-site. + # This can happen if we upgrade from Ambari 2.4 (with HDP 2.5) to Ambari 2.5, as this config is from 2.6 stack onwards only. + if "hive-interactive-env" in services["configurations"] and \ + "num_llap_nodes_for_llap_daemons" in services["configurations"]["hive-interactive-env"]["properties"]: + putHiveInteractiveEnvProperty('num_llap_nodes_for_llap_daemons', num_llap_nodes) + Logger.info("DBG: Setting config 'num_llap_nodes_for_llap_daemons' as : {0}".format(num_llap_nodes)) llap_container_size = long(llap_daemon_mem_per_node) putHiveInteractiveSiteProperty('hive.llap.daemon.yarn.container.mb', llap_container_size) @@ -980,7 +984,12 @@ class YARNRecommender(service_advisor.ServiceAdvisor): putHiveInteractiveSitePropertyAttribute('hive.server2.tez.sessions.per.default.queue', "minimum", 1) putHiveInteractiveSitePropertyAttribute('hive.server2.tez.sessions.per.default.queue', "maximum", 1) putHiveInteractiveEnvProperty('num_llap_nodes', 0) - putHiveInteractiveEnvProperty('num_llap_nodes_for_llap_daemons', 0) + + # Safeguard for not adding "num_llap_nodes_for_llap_daemons" if it doesnt exist in hive-interactive-site. + # This can happen if we upgrade from Ambari 2.4 (with HDP 2.5) to Ambari 2.5, as this config is from 2.6 stack onwards only. + if "hive-interactive-env" in services["configurations"] and \ + "num_llap_nodes_for_llap_daemons" in services["configurations"]["hive-interactive-env"]["properties"]: + putHiveInteractiveEnvProperty('num_llap_nodes_for_llap_daemons', 0) putHiveInteractiveEnvPropertyAttribute('num_llap_nodes', "minimum", 1) putHiveInteractiveEnvPropertyAttribute('num_llap_nodes', "maximum", node_manager_cnt) putHiveInteractiveSiteProperty('hive.llap.daemon.yarn.container.mb', yarn_min_container_size) http://git-wip-us.apache.org/repos/asf/ambari/blob/1142b526/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py index a450d4d..85e632d 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py @@ -1210,8 +1210,13 @@ class HDP25StackAdvisor(HDP24StackAdvisor): Logger.info("DBG: User requested num_llap_nodes : {0}, but used/adjusted value for calculations is : {1}".format(num_llap_nodes_requested, num_llap_nodes)) else: Logger.info("DBG: Used num_llap_nodes for calculations : {0}".format(num_llap_nodes_requested)) - putHiveInteractiveEnvProperty('num_llap_nodes_for_llap_daemons', num_llap_nodes) - Logger.info("DBG: Setting config 'num_llap_nodes_for_llap_daemons' as : {0}".format(num_llap_nodes)) + + # Safeguard for not adding "num_llap_nodes_for_llap_daemons" if it doesnt exist in hive-interactive-site. + # This can happen if we upgrade from Ambari 2.4 (with HDP 2.5) to Ambari 2.5, as this config is from 2.6 stack onwards only. + if "hive-interactive-env" in services["configurations"] and \ + "num_llap_nodes_for_llap_daemons" in services["configurations"]["hive-interactive-env"]["properties"]: + putHiveInteractiveEnvProperty('num_llap_nodes_for_llap_daemons', num_llap_nodes) + Logger.info("DBG: Setting config 'num_llap_nodes_for_llap_daemons' as : {0}".format(num_llap_nodes)) llap_container_size = long(llap_daemon_mem_per_node) putHiveInteractiveSiteProperty('hive.llap.daemon.yarn.container.mb', llap_container_size) @@ -1274,7 +1279,13 @@ class HDP25StackAdvisor(HDP24StackAdvisor): putHiveInteractiveSitePropertyAttribute('hive.server2.tez.sessions.per.default.queue', "minimum", 1) putHiveInteractiveSitePropertyAttribute('hive.server2.tez.sessions.per.default.queue', "maximum", 1) putHiveInteractiveEnvProperty('num_llap_nodes', 0) - putHiveInteractiveEnvProperty('num_llap_nodes_for_llap_daemons', 0) + + # Safeguard for not adding "num_llap_nodes_for_llap_daemons" if it doesnt exist in hive-interactive-site. + # This can happen if we upgrade from Ambari 2.4 (with HDP 2.5) to Ambari 2.5, as this config is from 2.6 stack onwards only. + if "hive-interactive-env" in services["configurations"] and \ + "num_llap_nodes_for_llap_daemons" in services["configurations"]["hive-interactive-env"]["properties"]: + putHiveInteractiveEnvProperty('num_llap_nodes_for_llap_daemons', 0) + putHiveInteractiveEnvPropertyAttribute('num_llap_nodes', "minimum", 1) putHiveInteractiveEnvPropertyAttribute('num_llap_nodes', "maximum", node_manager_cnt) putHiveInteractiveSiteProperty('hive.llap.daemon.yarn.container.mb', yarn_min_container_size) http://git-wip-us.apache.org/repos/asf/ambari/blob/1142b526/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py index 2d7bcd3..a913b35 100644 --- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py @@ -1134,7 +1134,8 @@ class TestHDP25StackAdvisor(TestCase): { 'properties': { 'enable_hive_interactive': 'true', - 'num_llap_nodes':'1' + 'num_llap_nodes':'1', + 'num_llap_nodes_for_llap_daemons': '1' } }, "hive-interactive-site": @@ -1336,7 +1337,8 @@ class TestHDP25StackAdvisor(TestCase): { 'properties': { 'enable_hive_interactive': 'true', - 'num_llap_nodes':'1' + 'num_llap_nodes':'1', + 'num_llap_nodes_for_llap_daemons': '1' } }, "hive-interactive-site": @@ -1532,7 +1534,8 @@ class TestHDP25StackAdvisor(TestCase): { 'properties': { 'enable_hive_interactive': 'true', - 'num_llap_nodes': 1 + 'num_llap_nodes': 1, + 'num_llap_nodes_for_llap_daemons': '1' } }, "hive-interactive-site": @@ -1737,7 +1740,8 @@ class TestHDP25StackAdvisor(TestCase): { 'properties': { 'enable_hive_interactive': 'true', - 'num_llap_nodes':'3' + 'num_llap_nodes':'3', + 'num_llap_nodes_for_llap_daemons': '1' } }, "hive-interactive-site": @@ -1938,6 +1942,7 @@ class TestHDP25StackAdvisor(TestCase): 'properties': { 'enable_hive_interactive': 'true', 'num_llap_nodes':'3', + 'num_llap_nodes_for_llap_daemons': '1', } }, "hive-interactive-site": @@ -2337,7 +2342,8 @@ class TestHDP25StackAdvisor(TestCase): { 'properties': { 'enable_hive_interactive': 'true', - 'num_llap_nodes':'3' + 'num_llap_nodes':'3', + 'num_llap_nodes_for_llap_daemons': '1' } }, "hive-interactive-site": @@ -2605,7 +2611,7 @@ class TestHDP25StackAdvisor(TestCase): self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.sessions.per.default.queue'], '1.0') self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.server2.tez.sessions.per.default.queue'], {'maximum': '4', 'minimum': '1'}) - self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons'], 3) + self.assertTrue('num_llap_nodes_for_llap_daemons' not in configurations['hive-interactive-env']['properties']) self.assertTrue('num_llap_nodes' not in configurations['hive-interactive-env']['properties']) self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '36864') @@ -2806,7 +2812,7 @@ class TestHDP25StackAdvisor(TestCase): self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.sessions.per.default.queue'], '1') self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.server2.tez.sessions.per.default.queue'], {'maximum': '4', 'minimum': '1'}) - self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons'], 3) + self.assertTrue('num_llap_nodes_for_llap_daemons' not in configurations['hive-interactive-env']['properties']) self.assertTrue('num_llap_nodes' not in configurations['hive-interactive-env']['properties']) self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '202554') @@ -3003,7 +3009,7 @@ class TestHDP25StackAdvisor(TestCase): self.assertEqual(configurations['capacity-scheduler']['properties'], {'capacity-scheduler': 'yarn.scheduler.capacity.root.accessible-node-labels=*\nyarn.scheduler.capacity.maximum-am-resource-percent=1\nyarn.scheduler.capacity.node-locality-delay=40\nyarn.scheduler.capacity.root.capacity=100\nyarn.scheduler.capacity.root.default.state=RUNNING\nyarn.scheduler.capacity.root.default.maximum-capacity=0.0\nyarn.scheduler.capacity.root.queues=default,llap\nyarn.scheduler.capacity.maximum-applications=10000\nyarn.scheduler.capacity.root.default.user-limit-factor=1\nyarn.scheduler.capacity.root.acl_administer_queue=*\nyarn.scheduler.capacity.root.default.acl_submit_applications=*\nyarn.scheduler.capacity.root.default.capacity=0.0\nyarn.scheduler.capacity.queue-mappings-override.enable=false\nyarn.scheduler.capacity.root.ordering-policy=priority-utilization\nyarn.scheduler.capacity.root.llap.user-limit-factor=1\nyarn.scheduler.capacity.root.llap.state=RUNNING\nyarn.scheduler.capacity.roo t.llap.ordering-policy=fifo\nyarn.scheduler.capacity.root.llap.priority=10\nyarn.scheduler.capacity.root.llap.minimum-user-limit-percent=100\nyarn.scheduler.capacity.root.llap.maximum-capacity=100.0\nyarn.scheduler.capacity.root.llap.capacity=100.0\nyarn.scheduler.capacity.root.llap.acl_submit_applications=hive\nyarn.scheduler.capacity.root.llap.acl_administer_queue=hive\nyarn.scheduler.capacity.root.llap.maximum-am-resource-percent=1'}) self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.server2.tez.sessions.per.default.queue'], {'maximum': '4'}) - self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons'], 3) + self.assertTrue('num_llap_nodes_for_llap_daemons' not in configurations['hive-interactive-env']['properties']) self.assertTrue('num_llap_nodes' not in configurations['hive-interactive-env']['properties']) self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '202752') @@ -3197,7 +3203,7 @@ class TestHDP25StackAdvisor(TestCase): self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.server2.tez.sessions.per.default.queue'], {'maximum': '4'}) self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes'], 3) - self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons'], 3) + self.assertTrue('num_llap_nodes_for_llap_daemons' not in configurations['hive-interactive-env']['properties']) self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '202752') @@ -3421,7 +3427,7 @@ class TestHDP25StackAdvisor(TestCase): self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.server2.tez.sessions.per.default.queue'], {'maximum': '4', 'minimum': '1'}) self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes'], 3) - self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons'], 3) + self.assertTrue('num_llap_nodes_for_llap_daemons' not in configurations['hive-interactive-env']['properties']) self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '204288') @@ -3588,6 +3594,7 @@ class TestHDP25StackAdvisor(TestCase): { 'properties': { 'enable_hive_interactive': 'true', + 'num_llap_nodes_for_llap_daemons': '1' } }, "hive-interactive-site": @@ -3960,7 +3967,8 @@ class TestHDP25StackAdvisor(TestCase): { 'properties': { 'enable_hive_interactive': 'true', - 'num_llap_nodes': '1' + 'num_llap_nodes': '1', + 'num_llap_nodes_for_llap_daemons': '0' } }, "hive-interactive-site":
