Repository: ambari Updated Branches: refs/heads/branch-2.4 ae0e38026 -> 8c10a51f6
http://git-wip-us.apache.org/repos/asf/ambari/blob/8c10a51f/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py index cf82a9c..51c0d0c 100644 --- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py @@ -58,158 +58,9 @@ class TestHDP25StackAdvisor(TestCase): self.get_system_min_uid_real = self.stackAdvisor.get_system_min_uid self.stackAdvisor.get_system_min_uid = self.get_system_min_uid_magic - def load_json(self, filename): - file = os.path.join(self.testDirectory, filename) - with open(file, 'rb') as f: - data = json.load(f) - return data - - def prepareHosts(self, hostsNames): - hosts = { "items": [] } - for hostName in hostsNames: - nextHost = {"Hosts":{"host_name" : hostName}} - hosts["items"].append(nextHost) - return hosts - - @patch('__builtin__.open') - @patch('os.path.exists') - def get_system_min_uid_magic(self, exists_mock, open_mock): - class MagicFile(object): - def read(self): - return """ - #test line UID_MIN 200 - UID_MIN 500 - """ - - def __exit__(self, exc_type, exc_val, exc_tb): - pass - - def __enter__(self): - return self - - exists_mock.return_value = True - open_mock.return_value = MagicFile() - return self.get_system_min_uid_real() - - - def __getHosts(self, componentsList, componentName): - return [component["StackServiceComponents"] for component in componentsList if component["StackServiceComponents"]["component_name"] == componentName][0] - - - def test_getComponentLayoutValidations_one_hsi_host(self): - - hosts = self.load_json("host-3-hosts.json") - services = self.load_json("services-normal-his-2-hosts.json") - - validations = self.stackAdvisor.getComponentLayoutValidations(services, hosts) - expected = {'component-name': 'HIVE_SERVER_INTERACTIVE', 'message': 'Between 0 and 1 HiveServer2 Interactive components should be installed in cluster.', 'type': 'host-component', 'level': 'ERROR'} - self.assertEquals(validations[0], expected) - - - def test_validateYarnConfigurations(self): - properties = {'enable_hive_interactive': 'true', - 'hive_server_interactive_host': 'c6401.ambari.apache.org', - 'hive.tez.container.size': '2048'} - recommendedDefaults = {'enable_hive_interactive': 'true', - "hive_server_interactive_host": "c6401.ambari.apache.org"} - configurations = { - "hive-interactive-env": { - "properties": {'enable_hive_interactive': 'true', "hive_server_interactive_host": "c6401.ambari.apache.org"} - }, - "hive-site": { - "properties": {"hive.security.authorization.enabled": "true", 'hive.tez.java.opts': '-server -Djava.net.preferIPv4Stack=true'} - }, - "hive-env": { - "properties": {"hive_security_authorization": "None"} - }, - "yarn-site": { - "properties": {"yarn.resourcemanager.work-preserving-recovery.enabled": "false"} - } - } - services = self.load_json("services-normal-his-valid.json") - - res_expected = [ - {'config-type': 'yarn-site', 'message': 'While enabling HIVE_SERVER_INTERACTIVE it is recommended that you enable work preserving restart in YARN.', 'type': 'configuration', 'config-name': 'yarn.resourcemanager.work-preserving-recovery.enabled', 'level': 'WARN'} - ] - res = self.stackAdvisor.validateYarnConfigurations(properties, recommendedDefaults, configurations, services, {}) - self.assertEquals(res, res_expected) - pass - - def test_validateHiveConfigurations(self): - properties = {'enable_hive_interactive': 'true', - 'hive_server_interactive_host': 'c6401.ambari.apache.org', - 'hive.tez.container.size': '2048'} - recommendedDefaults = {'enable_hive_interactive': 'true', - "hive_server_interactive_host": "c6401.ambari.apache.org"} - configurations = { - "hive-interactive-env": { - "properties": {'enable_hive_interactive': 'true', 'hive_server_interactive_host': 'c6401.ambari.apache.org'} - }, - "hive-site": { - "properties": {"hive.security.authorization.enabled": "true", 'hive.tez.java.opts': '-server -Djava.net.preferIPv4Stack=true'} - }, - "hive-env": { - "properties": {"hive_security_authorization": "None"} - }, - "yarn-site": { - "properties": {"yarn.resourcemanager.work-preserving-recovery.enabled": "true"} - } - } - configurations2 = { - "hive-interactive-env": { - "properties": {'enable_hive_interactive': 'false'} - }, - "hive-site": { - "properties": {"hive.security.authorization.enabled": "true", 'hive.tez.java.opts': '-server -Djava.net.preferIPv4Stack=true'} - }, - "hive-env": { - "properties": {"hive_security_authorization": "None"} - }, - "yarn-site": { - "properties": {"yarn.resourcemanager.work-preserving-recovery.enabled": "true"} - } - } - configurations3 = { - "hive-interactive-env": { - "properties": {'enable_hive_interactive': 'true', "hive_server_interactive_host": "c6402.ambari.apache.org"} - }, - "hive-site": { - "properties": {"hive.security.authorization.enabled": "true", 'hive.tez.java.opts': '-server -Djava.net.preferIPv4Stack=true'} - }, - "hive-env": { - "properties": {"hive_security_authorization": "None"} - }, - "yarn-site": { - "properties": {"yarn.resourcemanager.work-preserving-recovery.enabled": "true"} - } - } - services = self.load_json("services-normal-his-valid.json") - - res_expected = [ - ] - # the above error is not what we are checking for - just to keep test happy without having to test - res = self.stackAdvisor.validateHiveInteractiveEnvConfigurations(properties, recommendedDefaults, configurations, services, {}) - self.assertEquals(res, res_expected) - - res_expected = [ - {'config-type': 'hive-interactive-env', 'message': 'HIVE_SERVER_INTERACTIVE requires enable_hive_interactive in hive-interactive-env set to true.', 'type': 'configuration', 'config-name': 'enable_hive_interactive', 'level': 'ERROR'}, - {'config-type': 'hive-interactive-env', 'message': 'HIVE_SERVER_INTERACTIVE requires hive_server_interactive_host in hive-interactive-env set to its host name.', 'type': 'configuration', 'config-name': 'hive_server_interactive_host', 'level': 'ERROR'} - ] - res = self.stackAdvisor.validateHiveInteractiveEnvConfigurations(properties, recommendedDefaults, configurations2, services, {}) - self.assertEquals(res, res_expected) - - res_expected = [ - {'config-type': 'hive-interactive-env', 'message': 'HIVE_SERVER_INTERACTIVE requires hive_server_interactive_host in hive-interactive-env set to its host name.', 'type': 'configuration', 'config-name': 'hive_server_interactive_host', 'level': 'ERROR'} - ] - res = self.stackAdvisor.validateHiveInteractiveEnvConfigurations(properties, recommendedDefaults, configurations3, services, {}) - self.assertEquals(res, res_expected) - pass - - - def test_recommendYARNConfigurations(self): - ################ Setting up Inputs. ######################### - hosts = { + # setup for 'test_recommendYARNConfigurations' + self.hosts = { "items": [ { "Hosts": { @@ -225,12 +76,72 @@ class TestHDP25StackAdvisor(TestCase): ], "public_host_name": "c6401.ambari.apache.org", "host_name": "c6401.ambari.apache.org" - } + }, + }, { + "Hosts": { + "cpu_count": 6, + "total_mem": 50331648, + "disk_info": [ + {"mountpoint": "/"}, + {"mountpoint": "/dev/shm"}, + {"mountpoint": "/vagrant"}, + {"mountpoint": "/"}, + {"mountpoint": "/dev/shm"}, + {"mountpoint": "/vagrant"} + ], + "public_host_name": "c6402.ambari.apache.org", + "host_name": "c6402.ambari.apache.org" + }, + }, { + "Hosts": { + "cpu_count": 6, + "total_mem": 50331648, + "disk_info": [ + {"mountpoint": "/"}, + {"mountpoint": "/dev/shm"}, + {"mountpoint": "/vagrant"}, + {"mountpoint": "/"}, + {"mountpoint": "/dev/shm"}, + {"mountpoint": "/vagrant"} + ], + "public_host_name": "c6403.ambari.apache.org", + "host_name": "c6403.ambari.apache.org" + }, + }, { + "Hosts": { + "cpu_count": 6, + "total_mem": 50331648, + "disk_info": [ + {"mountpoint": "/"}, + {"mountpoint": "/dev/shm"}, + {"mountpoint": "/vagrant"}, + {"mountpoint": "/"}, + {"mountpoint": "/dev/shm"}, + {"mountpoint": "/vagrant"} + ], + "public_host_name": "c6404.ambari.apache.org", + "host_name": "c6404.ambari.apache.org" + }, + }, { + "Hosts": { + "cpu_count": 6, + "total_mem": 50331648, + "disk_info": [ + {"mountpoint": "/"}, + {"mountpoint": "/dev/shm"}, + {"mountpoint": "/vagrant"}, + {"mountpoint": "/"}, + {"mountpoint": "/dev/shm"}, + {"mountpoint": "/vagrant"} + ], + "public_host_name": "c6405.ambari.apache.org", + "host_name": "c6405.ambari.apache.org" + }, } ] } - clusterData = { + self.clusterData = { "cpu": 4, "mapMemory": 3000, "amMemory": 2000, @@ -239,92 +150,439 @@ class TestHDP25StackAdvisor(TestCase): "ramPerContainer": 256 } - # Services 1: YARN service with : (1). 'capacity scheduler' having only 'default' queue, - # (2). 'enable_hive_interactive' is ON, and (3). 'llap_queue_capacity' input is 0. - services_1 = { - "services": [{ - "StackServices": { - "service_name": "YARN", - }, - "Versions": { - "stack_version": "2.5" - }, - "components": [ - { - "StackServiceComponents": { - "component_name": "NODEMANAGER", - "hostnames": ["c6401.ambari.apache.org"] - }, - } - ] - }, { - "href": "/api/v1/stacks/HDP/versions/2.5/services/HIVE", - "StackServices": { - "service_name": "HIVE", - "service_version": "1.2.1.2.5", - "stack_name": "HDP", - "stack_version": "2.5" - }, - "components": [ - { - "href": "/api/v1/stacks/HDP/versions/2.5/services/HIVE/components/HIVE_SERVER_INTERACTIVE", - "StackServiceComponents": { - "advertise_version": "true", - "bulk_commands_display_name": "", - "bulk_commands_master_component_name": "", - "cardinality": "0-1", - "component_category": "MASTER", - "component_name": "HIVE_SERVER_INTERACTIVE", - "custom_commands": ["RESTART_LLAP"], - "decommission_allowed": "false", - "display_name": "HiveServer2 Interactive", - "has_bulk_commands_definition": "false", - "is_client": "false", - "is_master": "true", - "reassign_allowed": "false", - "recovery_enabled": "false", - "service_name": "HIVE", - "stack_name": "HDP", - "stack_version": "2.5", - "hostnames": ["c6401.ambari.apache.org"] - }, - "dependencies": [] - } - ] - } - ], - "changed-configurations": [ - { - u'old_value': u'', - u'type': u'', - u'name': u'' - } - ], - "configurations": { - "capacity-scheduler": { - "properties": { - "capacity-scheduler": "yarn.scheduler.capacity.root.queues=default\n" - "yarn.scheduler.capacity.root.default.user-limit-factor=1\n" - "yarn.scheduler.capacity.root.default.state=RUNNING\n" - "yarn.scheduler.capacity.root.default.maximum-capacity=100\n" - "yarn.scheduler.capacity.root.default.capacity=100\n" - "yarn.scheduler.capacity.root.default.acl_submit_applications=*\n" - "yarn.scheduler.capacity.root.capacity=100\n" - "yarn.scheduler.capacity.root.acl_administer_queue=*\n" - "yarn.scheduler.capacity.root.accessible-node-labels=*\n" - "yarn.scheduler.capacity.node-locality-delay=40\n" - "yarn.scheduler.capacity.maximum-applications=10000\n" - "yarn.scheduler.capacity.maximum-am-resource-percent=1\n" - "yarn.scheduler.capacity.queue-mappings-override.enable=false\n" - } - }, + # Expected config outputs. + + # Expected capacity-scheduler with 'llap' (size:20) and 'default' queue at root level. + self.expected_capacity_scheduler_llap_queue_size_20 = { + "properties": { + "capacity-scheduler": 'yarn.scheduler.capacity.root.default.maximum-capacity=80\n' + 'yarn.scheduler.capacity.root.accessible-node-labels=*\n' + 'yarn.scheduler.capacity.root.capacity=100\n' + 'yarn.scheduler.capacity.root.queues=default,llap\n' + 'yarn.scheduler.capacity.maximum-applications=10000\n' + 'yarn.scheduler.capacity.root.default.user-limit-factor=1\n' + 'yarn.scheduler.capacity.root.default.state=RUNNING\n' + 'yarn.scheduler.capacity.maximum-am-resource-percent=1\n' + 'yarn.scheduler.capacity.root.default.acl_submit_applications=*\n' + 'yarn.scheduler.capacity.root.default.capacity=80\n' + 'yarn.scheduler.capacity.root.acl_administer_queue=*\n' + 'yarn.scheduler.capacity.node-locality-delay=40\n' + 'yarn.scheduler.capacity.queue-mappings-override.enable=false\n' + 'yarn.scheduler.capacity.root.llap.user-limit-factor=1\n' + 'yarn.scheduler.capacity.root.llap.state=RUNNING\n' + 'yarn.scheduler.capacity.root.llap.ordering-policy=fifo\n' + 'yarn.scheduler.capacity.root.llap.minimum-user-limit-percent=100\n' + 'yarn.scheduler.capacity.root.llap.maximum-capacity=20\n' + 'yarn.scheduler.capacity.root.llap.capacity=20\n' + 'yarn.scheduler.capacity.root.llap.acl_submit_applications=hive\n' + 'yarn.scheduler.capacity.root.llap.acl_administer_queue=hive\n' + 'yarn.scheduler.capacity.root.llap.maximum-am-resource-percent=1' + + } + } + + # Expected capacity-scheduler with 'llap' (size:40) and 'default' queue at root level. + self.expected_capacity_scheduler_llap_queue_size_40 = { + "properties": { + "capacity-scheduler": 'yarn.scheduler.capacity.root.default.maximum-capacity=60\n' + 'yarn.scheduler.capacity.root.accessible-node-labels=*\n' + 'yarn.scheduler.capacity.root.capacity=100\n' + 'yarn.scheduler.capacity.root.queues=default,llap\n' + 'yarn.scheduler.capacity.maximum-applications=10000\n' + 'yarn.scheduler.capacity.root.default.user-limit-factor=1\n' + 'yarn.scheduler.capacity.root.default.state=RUNNING\n' + 'yarn.scheduler.capacity.maximum-am-resource-percent=1\n' + 'yarn.scheduler.capacity.root.default.acl_submit_applications=*\n' + 'yarn.scheduler.capacity.root.default.capacity=60\n' + 'yarn.scheduler.capacity.root.acl_administer_queue=*\n' + 'yarn.scheduler.capacity.node-locality-delay=40\n' + 'yarn.scheduler.capacity.queue-mappings-override.enable=false\n' + 'yarn.scheduler.capacity.root.llap.user-limit-factor=1\n' + 'yarn.scheduler.capacity.root.llap.state=RUNNING\n' + 'yarn.scheduler.capacity.root.llap.ordering-policy=fifo\n' + 'yarn.scheduler.capacity.root.llap.minimum-user-limit-percent=100\n' + 'yarn.scheduler.capacity.root.llap.maximum-capacity=40\n' + 'yarn.scheduler.capacity.root.llap.capacity=40\n' + 'yarn.scheduler.capacity.root.llap.acl_submit_applications=hive\n' + 'yarn.scheduler.capacity.root.llap.acl_administer_queue=hive\n' + 'yarn.scheduler.capacity.root.llap.maximum-am-resource-percent=1' + + } + } + + # Expected capacity-scheduler with 'llap' state = STOPPED, cap = 0 % and 'default' queue cap to 100%. + self.expected_capacity_scheduler_llap_Stopped_size_0 = { + "properties": { + "capacity-scheduler": 'yarn.scheduler.capacity.root.default.maximum-capacity=100\n' + 'yarn.scheduler.capacity.root.accessible-node-labels=*\n' + 'yarn.scheduler.capacity.root.capacity=100\n' + 'yarn.scheduler.capacity.root.queues=default,llap\n' + 'yarn.scheduler.capacity.maximum-applications=10000\n' + 'yarn.scheduler.capacity.root.default.user-limit-factor=1\n' + 'yarn.scheduler.capacity.root.default.state=RUNNING\n' + 'yarn.scheduler.capacity.maximum-am-resource-percent=1\n' + 'yarn.scheduler.capacity.root.default.acl_submit_applications=*\n' + 'yarn.scheduler.capacity.root.default.capacity=100\n' + 'yarn.scheduler.capacity.root.acl_administer_queue=*\n' + 'yarn.scheduler.capacity.node-locality-delay=40\n' + 'yarn.scheduler.capacity.queue-mappings-override.enable=false\n' + 'yarn.scheduler.capacity.root.llap.user-limit-factor=1\n' + 'yarn.scheduler.capacity.root.llap.state=STOPPED\n' + 'yarn.scheduler.capacity.root.llap.ordering-policy=fifo\n' + 'yarn.scheduler.capacity.root.llap.minimum-user-limit-percent=100\n' + 'yarn.scheduler.capacity.root.llap.maximum-capacity=0\n' + 'yarn.scheduler.capacity.root.llap.capacity=0\n' + 'yarn.scheduler.capacity.root.llap.acl_submit_applications=hive\n' + 'yarn.scheduler.capacity.root.llap.acl_administer_queue=hive\n' + 'yarn.scheduler.capacity.root.llap.maximum-am-resource-percent=1' + + } + } + + # Expected capacity-scheduler with only 'default' queue. + self.expected_capacity_scheduler_with_default_queue_only = { + "properties": { + "capacity-scheduler": 'yarn.scheduler.capacity.root.accessible-node-labels=*\n' + 'yarn.scheduler.capacity.maximum-am-resource-percent=1\n' + 'yarn.scheduler.capacity.root.capacity=100\n' + 'yarn.scheduler.capacity.root.default.state=RUNNING\n' + 'yarn.scheduler.capacity.node-locality-delay=40\n' + 'yarn.scheduler.capacity.root.queues=default\n' + 'yarn.scheduler.capacity.maximum-applications=10000\n' + 'yarn.scheduler.capacity.root.default.user-limit-factor=1\n' + 'yarn.scheduler.capacity.root.acl_administer_queue=*\n' + 'yarn.scheduler.capacity.root.default.acl_submit_applications=*\n' + 'yarn.scheduler.capacity.root.default.capacity=100\n' + "yarn.scheduler.capacity.root.default.maximum-capacity=100\n" + 'yarn.scheduler.capacity.queue-mappings-override.enable=false\n' + } + } + + # Expected capacity-scheduler as empty. + self.expected_capacity_scheduler_empty = { + "properties": { + } + } + + # Expected 'hive_interactive_site' with (1). 'hive.llap.daemon.queue.name' set to 'llap' queue, and + # (2). 'hive.llap.daemon.queue.name' property_attributes set to : default and llap. + self.expected_hive_interactive_site_llap = { + "hive-interactive-site": { + "properties": { + "hive.llap.daemon.queue.name": "llap" + }, + "property_attributes": { + "hive.llap.daemon.queue.name": { + "entries": [ + { + "value": "default", + "label": "default" + }, + { + "value": "llap", + "label": "llap" + } + ] + } + } + } + } + + # Expected 'hive_interactive_site' with (1). 'hive.llap.daemon.queue.name' set to 'default' queue, and + # (2). 'hive.llap.daemon.queue.name' property_attributes set to : default. + self.expected_hive_interactive_site_default = { + "hive-interactive-site": { + "properties": { + "hive.llap.daemon.queue.name": "default" + }, + "property_attributes": { + "hive.llap.daemon.queue.name": { + "entries": [ + { + "value": "default", + "label": "default" + } + ] + } + } + } + } + + # Expected 'hive_interactive_site' when no modifications are done. + self.expected_hive_interactive_site_empty = { + "hive-interactive-site": { + "properties": { + } + } + } + + # Expected 'hive_interactive_site' when no modifications are done. + self.expected_hive_interactive_env_empty = { + "hive-interactive-env": { + "properties": { + } + } + } + + self.expected_hive_interactive_site_only_memory = { + "hive-interactive-site": { + "properties": { + 'hive.llap.daemon.yarn.container.mb': '341' + } + } + } + + # Expected 'hive_interactive_env' with 'llap_queue_capacity' set to 20. + self.expected_llap_queue_capacity_20 = '20' + + # Expected 'hive_interactive_env' with 'llap_queue_capacity' set to 40. + self.expected_llap_queue_capacity_40 = '40' + + + # expected vals. + self.expected_visibility_false = {'visible': 'false'} + self.expected_visibility_true = {'visible': 'true'} + + + def load_json(self, filename): + file = os.path.join(self.testDirectory, filename) + with open(file, 'rb') as f: + data = json.load(f) + return data + + def prepareHosts(self, hostsNames): + hosts = { "items": [] } + for hostName in hostsNames: + nextHost = {"Hosts":{"host_name" : hostName}} + hosts["items"].append(nextHost) + return hosts + + @patch('__builtin__.open') + @patch('os.path.exists') + def get_system_min_uid_magic(self, exists_mock, open_mock): + class MagicFile(object): + def read(self): + return """ + #test line UID_MIN 200 + UID_MIN 500 + """ + + def __exit__(self, exc_type, exc_val, exc_tb): + pass + + def __enter__(self): + return self + + exists_mock.return_value = True + open_mock.return_value = MagicFile() + return self.get_system_min_uid_real() + + + def __getHosts(self, componentsList, componentName): + return [component["StackServiceComponents"] for component in componentsList if component["StackServiceComponents"]["component_name"] == componentName][0] + + + def test_getComponentLayoutValidations_one_hsi_host(self): + + hosts = self.load_json("host-3-hosts.json") + services = self.load_json("services-normal-his-2-hosts.json") + + validations = self.stackAdvisor.getComponentLayoutValidations(services, hosts) + expected = {'component-name': 'HIVE_SERVER_INTERACTIVE', 'message': 'Between 0 and 1 HiveServer2 Interactive components should be installed in cluster.', 'type': 'host-component', 'level': 'ERROR'} + self.assertEquals(validations[0], expected) + + + def test_validateYarnConfigurations(self): + properties = {'enable_hive_interactive': 'true', + 'hive_server_interactive_host': 'c6401.ambari.apache.org', + 'hive.tez.container.size': '2048'} + recommendedDefaults = {'enable_hive_interactive': 'true', + "hive_server_interactive_host": "c6401.ambari.apache.org"} + configurations = { + "hive-interactive-env": { + "properties": {'enable_hive_interactive': 'true', "hive_server_interactive_host": "c6401.ambari.apache.org"} + }, + "hive-site": { + "properties": {"hive.security.authorization.enabled": "true", 'hive.tez.java.opts': '-server -Djava.net.preferIPv4Stack=true'} + }, + "hive-env": { + "properties": {"hive_security_authorization": "None"} + }, + "yarn-site": { + "properties": {"yarn.resourcemanager.work-preserving-recovery.enabled": "false"} + } + } + services = self.load_json("services-normal-his-valid.json") + + res_expected = [ + {'config-type': 'yarn-site', 'message': 'While enabling HIVE_SERVER_INTERACTIVE it is recommended that you enable work preserving restart in YARN.', 'type': 'configuration', 'config-name': 'yarn.resourcemanager.work-preserving-recovery.enabled', 'level': 'WARN'} + ] + res = self.stackAdvisor.validateYarnConfigurations(properties, recommendedDefaults, configurations, services, {}) + self.assertEquals(res, res_expected) + pass + + def test_validateHiveConfigurations(self): + properties = {'enable_hive_interactive': 'true', + 'hive_server_interactive_host': 'c6401.ambari.apache.org', + 'hive.tez.container.size': '2048'} + recommendedDefaults = {'enable_hive_interactive': 'true', + "hive_server_interactive_host": "c6401.ambari.apache.org"} + configurations = { + "hive-interactive-env": { + "properties": {'enable_hive_interactive': 'true', 'hive_server_interactive_host': 'c6401.ambari.apache.org'} + }, + "hive-site": { + "properties": {"hive.security.authorization.enabled": "true", 'hive.tez.java.opts': '-server -Djava.net.preferIPv4Stack=true'} + }, + "hive-env": { + "properties": {"hive_security_authorization": "None"} + }, + "yarn-site": { + "properties": {"yarn.resourcemanager.work-preserving-recovery.enabled": "true"} + } + } + configurations2 = { + "hive-interactive-env": { + "properties": {'enable_hive_interactive': 'false'} + }, + "hive-site": { + "properties": {"hive.security.authorization.enabled": "true", 'hive.tez.java.opts': '-server -Djava.net.preferIPv4Stack=true'} + }, + "hive-env": { + "properties": {"hive_security_authorization": "None"} + }, + "yarn-site": { + "properties": {"yarn.resourcemanager.work-preserving-recovery.enabled": "true"} + } + } + configurations3 = { + "hive-interactive-env": { + "properties": {'enable_hive_interactive': 'true', "hive_server_interactive_host": "c6402.ambari.apache.org"} + }, + "hive-site": { + "properties": {"hive.security.authorization.enabled": "true", 'hive.tez.java.opts': '-server -Djava.net.preferIPv4Stack=true'} + }, + "hive-env": { + "properties": {"hive_security_authorization": "None"} + }, + "yarn-site": { + "properties": {"yarn.resourcemanager.work-preserving-recovery.enabled": "true"} + } + } + services = self.load_json("services-normal-his-valid.json") + + res_expected = [ + ] + # the above error is not what we are checking for - just to keep test happy without having to test + res = self.stackAdvisor.validateHiveInteractiveEnvConfigurations(properties, recommendedDefaults, configurations, services, {}) + self.assertEquals(res, res_expected) + + res_expected = [ + {'config-type': 'hive-interactive-env', 'message': 'HIVE_SERVER_INTERACTIVE requires enable_hive_interactive in hive-interactive-env set to true.', 'type': 'configuration', 'config-name': 'enable_hive_interactive', 'level': 'ERROR'}, + {'config-type': 'hive-interactive-env', 'message': 'HIVE_SERVER_INTERACTIVE requires hive_server_interactive_host in hive-interactive-env set to its host name.', 'type': 'configuration', 'config-name': 'hive_server_interactive_host', 'level': 'ERROR'} + ] + res = self.stackAdvisor.validateHiveInteractiveEnvConfigurations(properties, recommendedDefaults, configurations2, services, {}) + self.assertEquals(res, res_expected) + + res_expected = [ + {'config-type': 'hive-interactive-env', 'message': 'HIVE_SERVER_INTERACTIVE requires hive_server_interactive_host in hive-interactive-env set to its host name.', 'type': 'configuration', 'config-name': 'hive_server_interactive_host', 'level': 'ERROR'} + ] + res = self.stackAdvisor.validateHiveInteractiveEnvConfigurations(properties, recommendedDefaults, configurations3, services, {}) + self.assertEquals(res, res_expected) + pass + + + # Tests related to 'recommendYARNConfigurations()' + + + # Test 1 : (1). Only default queue exists in capacity-scheduler (2). enable_hive_interactive' is 'On' and + # 'llap_queue_capacity is 0. + def test_recommendYARNConfigurations_create_llap_queue_1(self): + + services = { + "services": [{ + "StackServices": { + "service_name": "YARN", + }, + "Versions": { + "stack_version": "2.5" + }, + "components": [ + { + "StackServiceComponents": { + "component_name": "NODEMANAGER", + "hostnames": ["c6401.ambari.apache.org"] + }, + } + ] + }, { + "href": "/api/v1/stacks/HDP/versions/2.5/services/HIVE", + "StackServices": { + "service_name": "HIVE", + "service_version": "1.2.1.2.5", + "stack_name": "HDP", + "stack_version": "2.5" + }, + "components": [ + { + "href": "/api/v1/stacks/HDP/versions/2.5/services/HIVE/components/HIVE_SERVER_INTERACTIVE", + "StackServiceComponents": { + "advertise_version": "true", + "bulk_commands_display_name": "", + "bulk_commands_master_component_name": "", + "cardinality": "0-1", + "component_category": "MASTER", + "component_name": "HIVE_SERVER_INTERACTIVE", + "custom_commands": ["RESTART_LLAP"], + "decommission_allowed": "false", + "display_name": "HiveServer2 Interactive", + "has_bulk_commands_definition": "false", + "is_client": "false", + "is_master": "true", + "reassign_allowed": "false", + "recovery_enabled": "false", + "service_name": "HIVE", + "stack_name": "HDP", + "stack_version": "2.5", + "hostnames": ["c6401.ambari.apache.org"] + }, + "dependencies": [] + } + ] + } + ], + "changed-configurations": [ + { + u'old_value': u'', + u'type': u'', + u'name': u'' + } + ], + "configurations": { + "capacity-scheduler": { + "properties": { + "capacity-scheduler": "yarn.scheduler.capacity.root.queues=default\n" + "yarn.scheduler.capacity.root.default.user-limit-factor=1\n" + "yarn.scheduler.capacity.root.default.state=RUNNING\n" + "yarn.scheduler.capacity.root.default.maximum-capacity=100\n" + "yarn.scheduler.capacity.root.default.capacity=100\n" + "yarn.scheduler.capacity.root.default.acl_submit_applications=*\n" + "yarn.scheduler.capacity.root.capacity=100\n" + "yarn.scheduler.capacity.root.acl_administer_queue=*\n" + "yarn.scheduler.capacity.root.accessible-node-labels=*\n" + "yarn.scheduler.capacity.node-locality-delay=40\n" + "yarn.scheduler.capacity.maximum-applications=10000\n" + "yarn.scheduler.capacity.maximum-am-resource-percent=1\n" + "yarn.scheduler.capacity.queue-mappings-override.enable=false\n" + } + }, "hive-interactive-env": { 'properties': { 'enable_hive_interactive': 'true', 'llap_queue_capacity':'0' } - }, + }, "yarn-site": { "properties": { "yarn.scheduler.minimum-allocation-mb": "682", @@ -347,19 +605,37 @@ class TestHDP25StackAdvisor(TestCase): 'properties': { 'hive_user': 'hive' } - }, + }, "hive-site": { 'properties': { 'hive.tez.container.size': '341' } - }, + }, } } - # Services 2: YARN service with : (1). 'capacity scheduler' having only 'default' queue, - # (2). 'enable_hive_interactive' is OFF, and (3). 'llap_queue_capacity' input is 0. - services_2 = { + configurations = { + } + self.stackAdvisor.recommendYARNConfigurations(configurations, self.clusterData, services, self.hosts) + # Check output + self.assertEquals(configurations['hive-interactive-site']['properties']['hive.llap.daemon.queue.name'], + self.expected_hive_interactive_site_llap['hive-interactive-site']['properties']['hive.llap.daemon.queue.name']) + self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name'], + self.expected_hive_interactive_site_llap['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name']) + self.assertEquals(configurations['hive-interactive-env']['properties']['llap_queue_capacity'], + self.expected_llap_queue_capacity_20) + + cap_sched_output_dict = convertToDict(configurations['capacity-scheduler']['properties']['capacity-scheduler']) + cap_sched_expected_dict = convertToDict(self.expected_capacity_scheduler_llap_queue_size_20['properties']['capacity-scheduler']) + self.assertEqual(cap_sched_output_dict, cap_sched_expected_dict) + + + + # Test 2: (1). Only default queue exists in capacity-scheduler (2). enable_hive_interactive' is 'On' and + # 'llap_queue_capacity is 40. + def test_recommendYARNConfigurations_create_llap_queue_2(self): + services = { "services": [{ "StackServices": { "service_name": "YARN", @@ -439,8 +715,8 @@ class TestHDP25StackAdvisor(TestCase): "hive-interactive-env": { 'properties': { - 'enable_hive_interactive': 'false', - 'llap_queue_capacity':'0' + 'enable_hive_interactive': 'true', + 'llap_queue_capacity':'40' } }, "yarn-site": { @@ -449,6 +725,12 @@ class TestHDP25StackAdvisor(TestCase): "yarn.nodemanager.resource.memory-mb": "2048" } }, + "hive-interactive-site": + { + 'properties': { + 'hive.llap.daemon.queue.name':'default', + } + }, "tez-interactive-site": { "properties": { "tez.am.resource.memory.mb": "341" @@ -459,13 +741,39 @@ class TestHDP25StackAdvisor(TestCase): 'properties': { 'hive_user': 'hive' } - } + }, + "hive-site": + { + 'properties': { + 'hive.tez.container.size': '341' + } + }, } } - # Services 3: YARN service with : (1). 'capacity scheduler' having only 'default' queue, - # (2). 'enable_hive_interactive' is ON, and (3). 'llap_queue_capacity' input is 30. - services_3 = { + configurations = { + } + + self.stackAdvisor.recommendYARNConfigurations(configurations, self.clusterData, services, self.hosts) + + # Check output + self.assertEquals(configurations['hive-interactive-site']['properties']['hive.llap.daemon.queue.name'], + self.expected_hive_interactive_site_llap['hive-interactive-site']['properties']['hive.llap.daemon.queue.name']) + self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name'], + self.expected_hive_interactive_site_llap['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name']) + self.assertTrue('llap_queue_capacity' not in configurations['hive-interactive-env']['properties']) + + cap_sched_output_dict = convertToDict(configurations['capacity-scheduler']['properties']['capacity-scheduler']) + cap_sched_expected_dict = convertToDict(self.expected_capacity_scheduler_llap_queue_size_40['properties']['capacity-scheduler']) + self.assertEqual(cap_sched_output_dict, cap_sched_expected_dict) + + + + # Test 3: (1). 'llap' (0%) and 'default' (100%) queues exists at leaf level in capacity-scheduler + # (2). llap is state = STOPPED, (3). llap_queue_capacity = 0, and (4). enable_hive_interactive' is 'ON'. + # Expected : llap queue state = RUNNING, llap_queue_capacity = 20 + def test_recommendYARNConfigurations_update_llap_queue_1(self): + services = { "services": [{ "StackServices": { "service_name": "YARN", @@ -519,46 +827,57 @@ class TestHDP25StackAdvisor(TestCase): ], "changed-configurations": [ { - u'old_value': u'', - u'type': u'', - u'name': u'' + u'old_value': u'off', + u'type': u'hive-interactive-env', + u'name': u'enable_hive_interactive' } ], "configurations": { "capacity-scheduler": { "properties": { - "capacity-scheduler": "yarn.scheduler.capacity.root.queues=default\n" + "capacity-scheduler": "yarn.scheduler.capacity.root.accessible-node-labels=*\n" + "yarn.scheduler.capacity.root.capacity=100\n" + "yarn.scheduler.capacity.root.queues=default,llap\n" + "yarn.scheduler.capacity.maximum-applications=10000\n" "yarn.scheduler.capacity.root.default.user-limit-factor=1\n" "yarn.scheduler.capacity.root.default.state=RUNNING\n" - "yarn.scheduler.capacity.root.default.maximum-capacity=100\n" - "yarn.scheduler.capacity.root.default.capacity=100\n" + "yarn.scheduler.capacity.maximum-am-resource-percent=1\n" "yarn.scheduler.capacity.root.default.acl_submit_applications=*\n" - "yarn.scheduler.capacity.root.capacity=100\n" + "yarn.scheduler.capacity.root.default.capacity=100\n" "yarn.scheduler.capacity.root.acl_administer_queue=*\n" - "yarn.scheduler.capacity.root.accessible-node-labels=*\n" "yarn.scheduler.capacity.node-locality-delay=40\n" - "yarn.scheduler.capacity.maximum-applications=10000\n" - "yarn.scheduler.capacity.maximum-am-resource-percent=1\n" "yarn.scheduler.capacity.queue-mappings-override.enable=false\n" + "yarn.scheduler.capacity.root.llap.user-limit-factor=1\n" + "yarn.scheduler.capacity.root.llap.state=STOPPED\n" + "yarn.scheduler.capacity.root.llap.ordering-policy=fifo\n" + "yarn.scheduler.capacity.root.llap.minimum-user-limit-percent=100\n" + "yarn.scheduler.capacity.root.llap.maximum-capacity=0\n" + "yarn.scheduler.capacity.root.default.maximum-capacity=100\n" + "yarn.scheduler.capacity.root.llap.capacity=0\n" + "yarn.scheduler.capacity.root.llap.acl_submit_applications=hive\n" + "yarn.scheduler.capacity.root.llap.acl_administer_queue=hive\n" + "yarn.scheduler.capacity.root.llap.maximum-am-resource-percent=1\n" } }, "hive-interactive-env": { 'properties': { 'enable_hive_interactive': 'true', - 'llap_queue_capacity':'40' + 'llap_queue_capacity':'0' } }, "yarn-site": { "properties": { - "yarn.scheduler.minimum-allocation-mb": "682", - "yarn.nodemanager.resource.memory-mb": "2048" + "yarn.scheduler.minimum-allocation-mb": "341", + "yarn.nodemanager.resource.memory-mb": "20000", + "yarn.nodemanager.resource.cpu-vcores": '1' } }, "hive-interactive-site": { 'properties': { - 'hive.llap.daemon.queue.name':'default', + 'hive.llap.daemon.queue.name':'llap', + 'hive.server2.tez.sessions.per.default.queue' : '1' } }, "tez-interactive-site": { @@ -581,9 +900,30 @@ class TestHDP25StackAdvisor(TestCase): } } - # Services 4: YARN service with : (1). 'capacity scheduler' having 'llap' and 'default' queue at root level and - # (2). 'enable_hive_interactive' is OFF - services_4 = { + configurations = { + } + + self.stackAdvisor.recommendYARNConfigurations(configurations, self.clusterData, services, self.hosts) + + # Check output + self.assertEquals(configurations['hive-interactive-site']['properties']['hive.llap.daemon.queue.name'], + self.expected_hive_interactive_site_llap['hive-interactive-site']['properties']['hive.llap.daemon.queue.name']) + self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name'], + self.expected_hive_interactive_site_llap['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name']) + self.assertEquals(configurations['hive-interactive-env']['properties']['llap_queue_capacity'], + self.expected_llap_queue_capacity_20) + + cap_sched_output_dict = convertToDict(configurations['capacity-scheduler']['properties']['capacity-scheduler']) + cap_sched_expected_dict = convertToDict(self.expected_capacity_scheduler_llap_queue_size_20['properties']['capacity-scheduler']) + self.assertEqual(cap_sched_output_dict, cap_sched_expected_dict) + + + + # Test 4: (1). 'llap' (20%) and 'default' (80%) queues exists at leaf level in capacity-scheduler + # (2). llap is state = STOPPED, (3). llap_queue_capacity = 40, and (4). enable_hive_interactive' is 'ON'. + # Expected : llap state goes RUNNING. + def test_recommendYARNConfigurations_update_llap_queue_2(self): + services = { "services": [{ "StackServices": { "service_name": "YARN", @@ -662,7 +1002,7 @@ class TestHDP25StackAdvisor(TestCase): "yarn.scheduler.capacity.root.llap.ordering-policy=fifo\n" "yarn.scheduler.capacity.root.llap.minimum-user-limit-percent=100\n" "yarn.scheduler.capacity.root.llap.maximum-capacity=20\n" - "yarn.scheduler.capacity.root.default.maximum-capacity=100\n" + "yarn.scheduler.capacity.root.default.maximum-capacity=80\n" "yarn.scheduler.capacity.root.llap.capacity=20\n" "yarn.scheduler.capacity.root.llap.acl_submit_applications=hive\n" "yarn.scheduler.capacity.root.llap.acl_administer_queue=hive\n" @@ -672,38 +1012,67 @@ class TestHDP25StackAdvisor(TestCase): "hive-interactive-env": { 'properties': { - 'enable_hive_interactive': 'false' + 'enable_hive_interactive': 'true', + 'llap_queue_capacity':'40' } }, "hive-interactive-site": { 'properties': { - 'hive.llap.daemon.queue.name':'default' + 'hive.llap.daemon.queue.name':'llap' } }, + "tez-interactive-site": { + "properties": { + "tez.am.resource.memory.mb": "341" + } + }, "yarn-site": { "properties": { - "yarn.scheduler.minimum-allocation-mb": "682", - "yarn.nodemanager.resource.memory-mb": "2048" - }, - "tez-interactive-site": { - "properties": { - "tez.am.resource.memory.mb": "341" - } - }, + "yarn.scheduler.minimum-allocation-mb": "341", + "yarn.nodemanager.resource.memory-mb": "20000", + "yarn.nodemanager.resource.cpu-vcores": '1' + } }, "hive-env": { 'properties': { 'hive_user': 'hive' } - } + }, + "hive-site": + { + 'properties': { + 'hive.tez.container.size': '341' + } + }, } } - # Services 5: YARN service with : (1). 'capacity scheduler' having 'llap' and 'default' queue at root level and - # (2). 'enable_hive_interactive' is ON - services_5 = { + configurations = { + } + + self.stackAdvisor.recommendYARNConfigurations(configurations, self.clusterData, services, self.hosts) + + # Check output + self.assertEquals(configurations['hive-interactive-site']['properties']['hive.llap.daemon.queue.name'], + self.expected_hive_interactive_site_llap['hive-interactive-site']['properties']['hive.llap.daemon.queue.name']) + self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name'], + self.expected_hive_interactive_site_llap['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name']) + self.assertTrue('llap_queue_capacity' not in configurations['hive-interactive-env']['properties']) + + cap_sched_output_dict = convertToDict(configurations['capacity-scheduler']['properties']['capacity-scheduler']) + cap_sched_expected_dict = convertToDict(self.expected_capacity_scheduler_llap_queue_size_40['properties']['capacity-scheduler']) + self.assertEqual(cap_sched_output_dict, cap_sched_expected_dict) + + + + + # Test 5: (1). 'llap' (20%) and 'default' (60%) queues exists at leaf level in capacity-scheduler + # (2). llap is state = RUNNING, (3). llap_queue_capacity = 40, and (4). enable_hive_interactive' is 'ON'. + # Expected : Existing llap queue's capacity in capacity-scheduler set to 40. + def test_recommendYARNConfigurations_update_llap_queue_3(self): + services = { "services": [{ "StackServices": { "service_name": "YARN", @@ -757,50 +1126,49 @@ class TestHDP25StackAdvisor(TestCase): ], "changed-configurations": [ { - u'old_value': u'0', - u'type': u'hive-interactive-env', - u'name': u'llap_queue_capacity' + u'old_value': u'', + u'type': u'', + u'name': u'' } ], "configurations": { "capacity-scheduler": { "properties": { - "capacity-scheduler": 'yarn.scheduler.capacity.root.default.maximum-capacity=60\n' - 'yarn.scheduler.capacity.root.accessible-node-labels=*\n' - 'yarn.scheduler.capacity.root.capacity=100\n' - 'yarn.scheduler.capacity.root.queues=default,llap\n' - 'yarn.scheduler.capacity.maximum-applications=10000\n' - 'yarn.scheduler.capacity.root.default.user-limit-factor=1\n' - 'yarn.scheduler.capacity.root.default.state=RUNNING\n' - 'yarn.scheduler.capacity.maximum-am-resource-percent=1\n' - 'yarn.scheduler.capacity.root.default.acl_submit_applications=*\n' - 'yarn.scheduler.capacity.root.default.capacity=60\n' - 'yarn.scheduler.capacity.root.acl_administer_queue=*\n' - 'yarn.scheduler.capacity.node-locality-delay=40\n' - 'yarn.scheduler.capacity.queue-mappings-override.enable=false\n' - 'yarn.scheduler.capacity.root.llap.user-limit-factor=1\n' - 'yarn.scheduler.capacity.root.llap.state=RUNNING\n' - 'yarn.scheduler.capacity.root.llap.ordering-policy=fifo\n' - 'yarn.scheduler.capacity.root.llap.minimum-user-limit-percent=100\n' - 'yarn.scheduler.capacity.root.llap.maximum-capacity=40\n' - 'yarn.scheduler.capacity.root.llap.capacity=40\n' - 'yarn.scheduler.capacity.root.llap.acl_submit_applications=hive\n' - 'yarn.scheduler.capacity.root.llap.acl_administer_queue=hive\n' - 'yarn.scheduler.capacity.root.llap.maximum-am-resource-percent=1' - + "capacity-scheduler": "yarn.scheduler.capacity.root.accessible-node-labels=*\n" + "yarn.scheduler.capacity.root.capacity=100\n" + "yarn.scheduler.capacity.root.queues=default,llap\n" + "yarn.scheduler.capacity.maximum-applications=10000\n" + "yarn.scheduler.capacity.root.default.user-limit-factor=1\n" + "yarn.scheduler.capacity.root.default.state=RUNNING\n" + "yarn.scheduler.capacity.maximum-am-resource-percent=1\n" + "yarn.scheduler.capacity.root.default.acl_submit_applications=*\n" + "yarn.scheduler.capacity.root.default.capacity=80\n" + "yarn.scheduler.capacity.root.acl_administer_queue=*\n" + "yarn.scheduler.capacity.node-locality-delay=40\n" + "yarn.scheduler.capacity.queue-mappings-override.enable=false\n" + "yarn.scheduler.capacity.root.llap.user-limit-factor=1\n" + "yarn.scheduler.capacity.root.llap.state=RUNNING\n" + "yarn.scheduler.capacity.root.llap.ordering-policy=fifo\n" + "yarn.scheduler.capacity.root.llap.minimum-user-limit-percent=100\n" + "yarn.scheduler.capacity.root.llap.maximum-capacity=20\n" + "yarn.scheduler.capacity.root.default.maximum-capacity=80\n" + "yarn.scheduler.capacity.root.llap.capacity=20\n" + "yarn.scheduler.capacity.root.llap.acl_submit_applications=hive\n" + "yarn.scheduler.capacity.root.llap.acl_administer_queue=hive\n" + "yarn.scheduler.capacity.root.llap.maximum-am-resource-percent=1\n" } }, "hive-interactive-env": { 'properties': { 'enable_hive_interactive': 'true', - 'llap_queue_capacity':'0' + 'llap_queue_capacity':'40' } }, "hive-interactive-site": { 'properties': { - 'hive.llap.daemon.queue.name': 'llap' + 'hive.llap.daemon.queue.name':'llap' } }, "tez-interactive-site": { @@ -808,24 +1176,52 @@ class TestHDP25StackAdvisor(TestCase): "tez.am.resource.memory.mb": "341" } }, + "yarn-site": { + "properties": { + "yarn.scheduler.minimum-allocation-mb": "341", + "yarn.nodemanager.resource.memory-mb": "20000", + "yarn.nodemanager.resource.cpu-vcores": '1' + } + }, "hive-env": { 'properties': { 'hive_user': 'hive' } }, - "yarn-site": { - "properties": { - "yarn.scheduler.minimum-allocation-mb": "682", - "yarn.nodemanager.resource.memory-mb": "2048" - } - } + "hive-site": + { + 'properties': { + 'hive.tez.container.size': '341' + } + }, } } - # Services 6: YARN service with : (1). 'capacity scheduler' having more than 2 queues and - # (2). 'enable_hive_interactive' is OFF - services_6= { + configurations = { + } + + self.stackAdvisor.recommendYARNConfigurations(configurations, self.clusterData, services, self.hosts) + + # Check output + self.assertEquals(configurations['hive-interactive-site']['properties']['hive.llap.daemon.queue.name'], + self.expected_hive_interactive_site_llap['hive-interactive-site']['properties']['hive.llap.daemon.queue.name']) + self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name'], + self.expected_hive_interactive_site_llap['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name']) + self.assertTrue('llap_queue_capacity' not in configurations['hive-interactive-env']['properties']) + + cap_sched_output_dict = convertToDict(configurations['capacity-scheduler']['properties']['capacity-scheduler']) + cap_sched_expected_dict = convertToDict(self.expected_capacity_scheduler_llap_queue_size_40['properties']['capacity-scheduler']) + self.assertEqual(cap_sched_output_dict, cap_sched_expected_dict) + + + + + # Test 6: (1). Only default queue exists in capacity-scheduler (2). enable_hive_interactive' is 'Off' and + # 'llap_queue_capacity is 0. + # Expected : No changes + def test_recommendYARNConfigurations_no_update_to_llap_queue_1(self): + services = { "services": [{ "StackServices": { "service_name": "YARN", @@ -887,54 +1283,19 @@ class TestHDP25StackAdvisor(TestCase): "configurations": { "capacity-scheduler": { "properties": { - "capacity-scheduler": "yarn.scheduler.capacity.maximum-am-resource-percent=0.2\n" - "yarn.scheduler.capacity.maximum-applications=10000\n" + "capacity-scheduler": "yarn.scheduler.capacity.root.queues=default\n" + "yarn.scheduler.capacity.root.default.user-limit-factor=1\n" + "yarn.scheduler.capacity.root.default.state=RUNNING\n" + "yarn.scheduler.capacity.root.default.maximum-capacity=100\n" + "yarn.scheduler.capacity.root.default.capacity=100\n" + "yarn.scheduler.capacity.root.default.acl_submit_applications=*\n" + "yarn.scheduler.capacity.root.capacity=100\n" + "yarn.scheduler.capacity.root.acl_administer_queue=*\n" + "yarn.scheduler.capacity.root.accessible-node-labels=*\n" "yarn.scheduler.capacity.node-locality-delay=40\n" + "yarn.scheduler.capacity.maximum-applications=10000\n" + "yarn.scheduler.capacity.maximum-am-resource-percent=1\n" "yarn.scheduler.capacity.queue-mappings-override.enable=false\n" - "yarn.scheduler.capacity.resource-calculator=org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator\n" - "yarn.scheduler.capacity.root.accessible-node-labels=*\n" - "yarn.scheduler.capacity.root.acl_administer_queue=*\n" - "yarn.scheduler.capacity.root.capacity=100\n" - "yarn.scheduler.capacity.root.default.a.a1.acl_administer_queue=*\n" - "yarn.scheduler.capacity.root.default.a.a1.acl_submit_applications=*\n" - "yarn.scheduler.capacity.root.default.a.a1.capacity=75\n" - "yarn.scheduler.capacity.root.default.a.a1.maximum-capacity=100\n" - "yarn.scheduler.capacity.root.default.a.a1.minimum-user-limit-percent=100\n" - "yarn.scheduler.capacity.root.default.a.a1.ordering-policy=fifo\n" - "yarn.scheduler.capacity.root.default.a.a1.state=RUNNING\n" - "yarn.scheduler.capacity.root.default.a.a1.user-limit-factor=1\n" - "yarn.scheduler.capacity.root.default.a.a2.acl_administer_queue=*\n" - "yarn.scheduler.capacity.root.default.a.a2.acl_submit_applications=*\n" - "yarn.scheduler.capacity.root.default.a.a2.capacity=25\n" - "yarn.scheduler.capacity.root.default.a.a2.maximum-capacity=25\n" - "yarn.scheduler.capacity.root.default.a.a2.minimum-user-limit-percent=100\n" - "yarn.scheduler.capacity.root.default.a.a2.ordering-policy=fifo\n" - "yarn.scheduler.capacity.root.default.a.a2.state=RUNNING\n" - "yarn.scheduler.capacity.root.default.a.a2.user-limit-factor=1\n" - "yarn.scheduler.capacity.root.default.a.acl_administer_queue=*\n" - "yarn.scheduler.capacity.root.default.a.acl_submit_applications=*\n" - "yarn.scheduler.capacity.root.default.a.capacity=50\n" - "yarn.scheduler.capacity.root.default.a.maximum-capacity=100\n" - "yarn.scheduler.capacity.root.default.a.minimum-user-limit-percent=100\n" - "yarn.scheduler.capacity.root.default.a.ordering-policy=fifo\n" - "yarn.scheduler.capacity.root.default.a.queues=a1,a2\n" - "yarn.scheduler.capacity.root.default.a.state=RUNNING\n" - "yarn.scheduler.capacity.root.default.a.user-limit-factor=1\n" - "yarn.scheduler.capacity.root.default.acl_submit_applications=*\n" - "yarn.scheduler.capacity.root.default.b.acl_administer_queue=*\n" - "yarn.scheduler.capacity.root.default.b.acl_submit_applications=*\n" - "yarn.scheduler.capacity.root.default.b.capacity=50\n" - "yarn.scheduler.capacity.root.default.b.maximum-capacity=50\n" - "yarn.scheduler.capacity.root.default.b.minimum-user-limit-percent=100\n" - "yarn.scheduler.capacity.root.default.b.ordering-policy=fifo\n" - "yarn.scheduler.capacity.root.default.b.state=RUNNING\n" - "yarn.scheduler.capacity.root.default.b.user-limit-factor=1\n" - "yarn.scheduler.capacity.root.default.capacity=100\n" - "yarn.scheduler.capacity.root.default.maximum-capacity=100\n" - "yarn.scheduler.capacity.root.default.queues=a,b\n" - "yarn.scheduler.capacity.root.default.state=RUNNING\n" - "yarn.scheduler.capacity.root.default.user-limit-factor=1\n" - "yarn.scheduler.capacity.root.queues=default" } }, "hive-interactive-env": @@ -944,6 +1305,12 @@ class TestHDP25StackAdvisor(TestCase): 'llap_queue_capacity':'0' } }, + "yarn-site": { + "properties": { + "yarn.scheduler.minimum-allocation-mb": "682", + "yarn.nodemanager.resource.memory-mb": "2048" + } + }, "tez-interactive-site": { "properties": { "tez.am.resource.memory.mb": "341" @@ -958,8 +1325,23 @@ class TestHDP25StackAdvisor(TestCase): } } - # Services 7: YARN service with 'capacity scheduler' empty and (2). 'enable_hive_interactive' is OFF - services_7= { + configurations = { + } + + self.stackAdvisor.recommendYARNConfigurations(configurations, self.clusterData, services, self.hosts) + + # Check output + self.assertTrue('hive.llap.daemon.queue.name' not in configurations['hive-interactive-site']['properties']) + self.assertTrue('property_attributes' not in configurations['hive-interactive-site']) + self.assertTrue('hive-interactive-env' not in configurations) + self.assertEquals(configurations['capacity-scheduler']['properties'],self.expected_capacity_scheduler_empty['properties']) + + + # Test 7: (1). 'default' and 'llap' (State : RUNNING) queue exists at root level in capacity-scheduler, and + # (2). enable_hive_interactive' is 'off'. + # Expected : 'default' queue set to Size 100, 'llap' queue state set to STOPPED and sized to 0. + def test_recommendYARNConfigurations_llap_queue_set_to_stopped_1(self): + services = { "services": [{ "StackServices": { "service_name": "YARN", @@ -1021,15 +1403,53 @@ class TestHDP25StackAdvisor(TestCase): "configurations": { "capacity-scheduler": { "properties": { + "capacity-scheduler": "yarn.scheduler.capacity.root.accessible-node-labels=*\n" + "yarn.scheduler.capacity.root.capacity=100\n" + "yarn.scheduler.capacity.root.queues=default,llap\n" + "yarn.scheduler.capacity.maximum-applications=10000\n" + "yarn.scheduler.capacity.root.default.user-limit-factor=1\n" + "yarn.scheduler.capacity.root.default.state=RUNNING\n" + "yarn.scheduler.capacity.maximum-am-resource-percent=1\n" + "yarn.scheduler.capacity.root.default.acl_submit_applications=*\n" + "yarn.scheduler.capacity.root.default.capacity=80\n" + "yarn.scheduler.capacity.root.acl_administer_queue=*\n" + "yarn.scheduler.capacity.node-locality-delay=40\n" + "yarn.scheduler.capacity.queue-mappings-override.enable=false\n" + "yarn.scheduler.capacity.root.llap.user-limit-factor=1\n" + "yarn.scheduler.capacity.root.llap.state=RUNNING\n" + "yarn.scheduler.capacity.root.llap.ordering-policy=fifo\n" + "yarn.scheduler.capacity.root.llap.minimum-user-limit-percent=100\n" + "yarn.scheduler.capacity.root.llap.maximum-capacity=20\n" + "yarn.scheduler.capacity.root.default.maximum-capacity=100\n" + "yarn.scheduler.capacity.root.llap.capacity=20\n" + "yarn.scheduler.capacity.root.llap.acl_submit_applications=hive\n" + "yarn.scheduler.capacity.root.llap.acl_administer_queue=hive\n" + "yarn.scheduler.capacity.root.llap.maximum-am-resource-percent=1\n" } }, "hive-interactive-env": { 'properties': { - 'enable_hive_interactive': 'false', - 'llap_queue_capacity':'0' + 'enable_hive_interactive': 'false' + } + }, + "hive-interactive-site": + { + 'properties': { + 'hive.llap.daemon.queue.name':'default' + } + }, + "yarn-site": { + "properties": { + "yarn.scheduler.minimum-allocation-mb": "682", + "yarn.nodemanager.resource.memory-mb": "2048" + }, + "tez-interactive-site": { + "properties": { + "tez.am.resource.memory.mb": "341" } }, + }, "hive-env": { 'properties': { @@ -1039,8 +1459,28 @@ class TestHDP25StackAdvisor(TestCase): } } - # Services 8: YARN service with : (1). malformed 'capacity scheduler' and (2). 'enable_hive_interactive' is OFF - services_8= { + configurations = { + } + + self.stackAdvisor.recommendYARNConfigurations(configurations, self.clusterData, services, self.hosts) + + # Check output + self.assertEquals(configurations['hive-interactive-site']['properties']['hive.llap.daemon.queue.name'], + self.expected_hive_interactive_site_default['hive-interactive-site']['properties']['hive.llap.daemon.queue.name']) + self.assertFalse('property_attributes' in configurations['hive-interactive-site']) + self.assertFalse('hive-interactive-env' in configurations) + + cap_sched_output_dict = convertToDict(configurations['capacity-scheduler']['properties']['capacity-scheduler']) + cap_sched_expected_dict = convertToDict(self.expected_capacity_scheduler_llap_Stopped_size_0['properties']['capacity-scheduler']) + self.assertEqual(cap_sched_output_dict, cap_sched_expected_dict) + + + + # Test 8: (1). More than 2 queues at leaf level exists in capacity-scheduler (no queue is named 'llap') + # (2). enable_hive_interactive' is 'off'. + # Expected : No changes. + def test_recommendYARNConfigurations_no_update_to_llap_queue_2(self): + services= { "services": [{ "StackServices": { "service_name": "YARN", @@ -1102,9 +1542,54 @@ class TestHDP25StackAdvisor(TestCase): "configurations": { "capacity-scheduler": { "properties": { - "capacity-scheduler": "yarn.scheduler.capacity.root.default.a.a1.acl_submit_applications=*\n" + "capacity-scheduler": "yarn.scheduler.capacity.maximum-am-resource-percent=0.2\n" + "yarn.scheduler.capacity.maximum-applications=10000\n" + "yarn.scheduler.capacity.node-locality-delay=40\n" + "yarn.scheduler.capacity.queue-mappings-override.enable=false\n" + "yarn.scheduler.capacity.resource-calculator=org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator\n" + "yarn.scheduler.capacity.root.accessible-node-labels=*\n" + "yarn.scheduler.capacity.root.acl_administer_queue=*\n" + "yarn.scheduler.capacity.root.capacity=100\n" + "yarn.scheduler.capacity.root.default.a.a1.acl_administer_queue=*\n" + "yarn.scheduler.capacity.root.default.a.a1.acl_submit_applications=*\n" "yarn.scheduler.capacity.root.default.a.a1.capacity=75\n" "yarn.scheduler.capacity.root.default.a.a1.maximum-capacity=100\n" + "yarn.scheduler.capacity.root.default.a.a1.minimum-user-limit-percent=100\n" + "yarn.scheduler.capacity.root.default.a.a1.ordering-policy=fifo\n" + "yarn.scheduler.capacity.root.default.a.a1.state=RUNNING\n" + "yarn.scheduler.capacity.root.default.a.a1.user-limit-factor=1\n" + "yarn.scheduler.capacity.root.default.a.a2.acl_administer_queue=*\n" + "yarn.scheduler.capacity.root.default.a.a2.acl_submit_applications=*\n" + "yarn.scheduler.capacity.root.default.a.a2.capacity=25\n" + "yarn.scheduler.capacity.root.default.a.a2.maximum-capacity=25\n" + "yarn.scheduler.capacity.root.default.a.a2.minimum-user-limit-percent=100\n" + "yarn.scheduler.capacity.root.default.a.a2.ordering-policy=fifo\n" + "yarn.scheduler.capacity.root.default.a.a2.state=RUNNING\n" + "yarn.scheduler.capacity.root.default.a.a2.user-limit-factor=1\n" + "yarn.scheduler.capacity.root.default.a.acl_administer_queue=*\n" + "yarn.scheduler.capacity.root.default.a.acl_submit_applications=*\n" + "yarn.scheduler.capacity.root.default.a.capacity=50\n" + "yarn.scheduler.capacity.root.default.a.maximum-capacity=100\n" + "yarn.scheduler.capacity.root.default.a.minimum-user-limit-percent=100\n" + "yarn.scheduler.capacity.root.default.a.ordering-policy=fifo\n" + "yarn.scheduler.capacity.root.default.a.queues=a1,a2\n" + "yarn.scheduler.capacity.root.default.a.state=RUNNING\n" + "yarn.scheduler.capacity.root.default.a.user-limit-factor=1\n" + "yarn.scheduler.capacity.root.default.acl_submit_applications=*\n" + "yarn.scheduler.capacity.root.default.b.acl_administer_queue=*\n" + "yarn.scheduler.capacity.root.default.b.acl_submit_applications=*\n" + "yarn.scheduler.capacity.root.default.b.capacity=50\n" + "yarn.scheduler.capacity.root.default.b.maximum-capacity=50\n" + "yarn.scheduler.capacity.root.default.b.minimum-user-limit-percent=100\n" + "yarn.scheduler.capacity.root.default.b.ordering-policy=fifo\n" + "yarn.scheduler.capacity.root.default.b.state=RUNNING\n" + "yarn.scheduler.capacity.root.default.b.user-limit-factor=1\n" + "yarn.scheduler.capacity.root.default.capacity=100\n" + "yarn.scheduler.capacity.root.default.maximum-capacity=100\n" + "yarn.scheduler.capacity.root.default.queues=a,b\n" + "yarn.scheduler.capacity.root.default.state=RUNNING\n" + "yarn.scheduler.capacity.root.default.user-limit-factor=1\n" + "yarn.scheduler.capacity.root.queues=default" } }, "hive-interactive-env": @@ -1114,6 +1599,11 @@ class TestHDP25StackAdvisor(TestCase): 'llap_queue_capacity':'0' } }, + "tez-interactive-site": { + "properties": { + "tez.am.resource.memory.mb": "341" + } + }, "hive-env": { 'properties': { @@ -1123,9 +1613,27 @@ class TestHDP25StackAdvisor(TestCase): } } - # Services 9: YARN service with : (1). 'capacity scheduler' having 'llap' and 'default' queue at root level and - # (2). 'llap' queue state is STOPPED and sized 0 % and (3). 'enable_hive_interactive' is OFF - services_9 = { + + configurations = { + } + + self.stackAdvisor.recommendYARNConfigurations(configurations, self.clusterData, services, self.hosts) + + # Check output + self.assertEquals(configurations['hive-interactive-site']['properties'], + self.expected_hive_interactive_site_empty['hive-interactive-site']['properties']) + self.assertEquals(configurations['capacity-scheduler']['properties'], + self.expected_capacity_scheduler_empty['properties']) + self.assertFalse('hive-interactive-env' in configurations) + + + + + # Test 9: (1). More than 2 queues at leaf level exists in capacity-scheduler (one queue is named 'llap') + # (2). enable_hive_interactive' is 'off'. + # Expected : No changes. + def test_recommendYARNConfigurations_no_update_to_llap_queue_3(self): + services= { "services": [{ "StackServices": { "service_name": "YARN", @@ -1187,34 +1695,61 @@ class TestHDP25StackAdvisor(TestCase): "configurations": { "capacity-scheduler": { "properties": { - "capacity-scheduler": "yarn.scheduler.capacity.root.accessible-node-labels=*\n" - "yarn.scheduler.capacity.root.capacity=100\n" - "yarn.scheduler.capacity.root.queues=default,llap\n" + "capacity-scheduler": "yarn.scheduler.capacity.maximum-am-resource-percent=0.2\n" "yarn.scheduler.capacity.maximum-applications=10000\n" - "yarn.scheduler.capacity.root.default.user-limit-factor=1\n" - "yarn.scheduler.capacity.root.default.state=RUNNING\n" - "yarn.scheduler.capacity.maximum-am-resource-percent=1\n" - "yarn.scheduler.capacity.root.default.acl_submit_applications=*\n" - "yarn.scheduler.capacity.root.default.capacity=100\n" - "yarn.scheduler.capacity.root.acl_administer_queue=*\n" "yarn.scheduler.capacity.node-locality-delay=40\n" "yarn.scheduler.capacity.queue-mappings-override.enable=false\n" - "yarn.scheduler.capacity.root.llap.user-limit-factor=1\n" - "yarn.scheduler.capacity.root.llap.state=STOPPED\n" - "yarn.scheduler.capacity.root.llap.ordering-policy=fifo\n" - "yarn.scheduler.capacity.root.llap.minimum-user-limit-percent=100\n" - "yarn.scheduler.capacity.root.llap.maximum-capacity=0\n" + "yarn.scheduler.capacity.resource-calculator=org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator\n" + "yarn.scheduler.capacity.root.accessible-node-labels=*\n" + "yarn.scheduler.capacity.root.acl_administer_queue=*\n" + "yarn.scheduler.capacity.root.capacity=100\n" + "yarn.scheduler.capacity.root.default.a.a1.acl_administer_queue=*\n" + "yarn.scheduler.capacity.root.default.a.a1.acl_submit_applications=*\n" + "yarn.scheduler.capacity.root.default.a.a1.capacity=75\n" + "yarn.scheduler.capacity.root.default.a.a1.maximum-capacity=100\n" + "yarn.scheduler.capacity.root.default.a.a1.minimum-user-limit-percent=100\n" + "yarn.scheduler.capacity.root.default.a.a1.ordering-policy=fifo\n" + "yarn.scheduler.capacity.root.default.a.a1.state=RUNNING\n" + "yarn.scheduler.capacity.root.default.a.a1.user-limit-factor=1\n" + "yarn.scheduler.capacity.root.default.a.llap.acl_administer_queue=*\n" + "yarn.scheduler.capacity.root.default.a.llap.acl_submit_applications=*\n" + "yarn.scheduler.capacity.root.default.a.llap.capacity=25\n" + "yarn.scheduler.capacity.root.default.a.llap.maximum-capacity=25\n" + "yarn.scheduler.capacity.root.default.a.llap.minimum-user-limit-percent=100\n" + "yarn.scheduler.capacity.root.default.a.llap.ordering-policy=fifo\n" + "yarn.scheduler.capacity.root.default.a.llap.state=RUNNING\n" + "yarn.scheduler.capacity.root.default.a.llap.user-limit-factor=1\n" + "yarn.scheduler.capacity.root.default.a.acl_administer_queue=*\n" + "yarn.scheduler.capacity.root.default.a.acl_submit_applications=*\n" + "yarn.scheduler.capacity.root.default.a.capacity=50\n" + "yarn.scheduler.capacity.root.default.a.maximum-capacity=100\n" + "yarn.scheduler.capacity.root.default.a.minimum-user-limit-percent=100\n" + "yarn.scheduler.capacity.root.default.a.ordering-policy=fifo\n" + "yarn.scheduler.capacity.root.default.a.queues=a1,llap\n" + "yarn.scheduler.capacity.root.default.a.state=RUNNING\n" + "yarn.scheduler.capacity.root.default.a.user-limit-factor=1\n" + "yarn.scheduler.capacity.root.default.acl_submit_applications=*\n" + "yarn.scheduler.capacity.root.default.b.acl_administer_queue=*\n" + "yarn.scheduler.capacity.root.default.b.acl_submit_applications=*\n" + "yarn.scheduler.capacity.root.default.b.capacity=50\n" + "yarn.scheduler.capacity.root.default.b.maximum-capacity=50\n" + "yarn.scheduler.capacity.root.default.b.minimum-user-limit-percent=100\n" + "yarn.scheduler.capacity.root.default.b.ordering-policy=fifo\n" + "yarn.scheduler.capacity.root.default.b.state=RUNNING\n" + "yarn.scheduler.capacity.root.default.b.user-limit-factor=1\n" + "yarn.scheduler.capacity.root.default.capacity=100\n" "yarn.scheduler.capacity.root.default.maximum-capacity=100\n" - "yarn.scheduler.capacity.root.llap.capacity=0\n" - "yarn.scheduler.capacity.root.llap.acl_submit_applications=hive\n" - "yarn.scheduler.capacity.root.llap.acl_administer_queue=hive\n" - "yarn.scheduler.capacity.root.llap.maximum-am-resource-percent=1\n" + "yarn.scheduler.capacity.root.default.queues=a,b\n" + "yarn.scheduler.capacity.root.default.state=RUNNING\n" + "yarn.scheduler.capacity.root.default.user-limit-factor=1\n" + "yarn.scheduler.capacity.root.queues=default" } }, "hive-interactive-env": { 'properties': { - 'enable_hive_interactive': 'false' + 'enable_hive_interactive': 'false', + 'llap_queue_capacity':'0' } }, "hive-env": @@ -1226,9 +1761,28 @@ class TestHDP25StackAdvisor(TestCase): } } - # Services 10: YARN service with : (1). 'capacity scheduler' having 'llap' and 'default' queue at root level and - # (2). 'llap' queue state is STOPPED and sized 0 % and (3). 'enable_hive_interactive' is ON - services_10 = { + + configurations = { + } + + self.stackAdvisor.recommendYARNConfigurations(configurations, self.clusterData, services, self.hosts) + + # Check output + self.assertEquals(configurations['hive-interactive-site']['properties'], + self.expected_hive_interactive_site_empty['hive-interactive-site']['properties']) + self.assertEquals(configurations['capacity-scheduler']['properties'], + self.expected_capacity_scheduler_empty['properties']) + self.assertFalse('hive-interactive-env' in configurations) + + + + + # Test 10: (1). 'llap' (Cap: 0%, State: STOPPED) and 'default' (100%) queues exists at leaf level + # in capacity-scheduler + # (2). enable_hive_interactive' is 'off'. + # Expected : No changes. + def test_recommendYARNConfigurations_no_update_to_llap_queue_4(self): + services = { "services": [{ "StackServices": { "service_name": "YARN", @@ -1282,9 +1836,9 @@ class TestHDP25StackAdvisor(TestCase): ], "changed-configurations": [ { - u'old_value': u'off', - u'type': u'hive-interactive-env', - u'name': u'enable_hive_interactive' + u'old_value': u'', + u'type': u'', + u'name': u'' } ], "configurations": { @@ -1317,46 +1871,37 @@ class TestHDP25StackAdvisor(TestCase): "hive-interactive-env": { 'properties': { - 'enable_hive_interactive': 'true', - 'llap_queue_capacity':'0' - } - }, - "yarn-site": { - "properties": { - "yarn.scheduler.minimum-allocation-mb": "341", - "yarn.nodemanager.resource.memory-mb": "20000", - "yarn.nodemanager.resource.cpu-vcores": '1' - } - }, - "hive-interactive-site": - { - 'properties': { - 'hive.llap.daemon.queue.name':'llap', - 'hive.server2.tez.sessions.per.default.queue' : '1' + 'enable_hive_interactive': 'false' } }, - "tez-interactive-site": { - "properties": { - "tez.am.resource.memory.mb": "341" - } - }, "hive-env": { 'properties': { 'hive_user': 'hive' } - }, - "hive-site": - { - 'properties': { - 'hive.tez.container.size': '341' - } - }, + } } } - # Services 11: YARN service with : (1). 'capacity scheduler' having 'llap' and 'default' queue at root level and - # (2). 'llap' queue state is STOPPED and sized 40 % and (3). 'enable_hive_interactive' is ON - services_11 = { + + + configurations = { + } + + self.stackAdvisor.recommendYARNConfigurations(configurations, self.clusterData, services, self.hosts) + + # Check output + self.assertEquals(configurations['hive-interactive-site']['properties'], + self.expected_hive_interactive_site_empty['hive-interactive-site']['properties']) + self.assertEquals(configurations['capacity-scheduler']['properties'], + self.expected_capacity_scheduler_empty['properties']) + self.assertFalse('hive-interactive-env' in configurations) + + + + # Test 11: YARN service with : (1). 'capacity scheduler' having 'llap' (state:stopped) and 'default' queue at + # root level and (2). 'enable_hive_interactive' is ON and (3). 'hive.llap.daemon.queue.name' == 'default' + def test_recommendYARNConfigurations_no_update_to_llap_queue_5(self): + services_15 = { "services": [{ "StackServices": { "service_name": "YARN", @@ -1404,42 +1949,58 @@ class TestHDP25StackAdvisor(TestCase): "hostnames": ["c6401.ambari.apache.org"] }, "dependencies": [] - } + }, + { + "StackServiceComponents": { + "advertise_version": "true", + "cardinality": "1+", + "component_category": "SLAVE", + "component_name": "NODEMANAGER", + "display_name": "NodeManager", + "is_client": "false", + "is_master": "false", + "hostnames": [ + "c6403.ambari.apache.org" + ] + }, + "dependencies": [] + }, ] } ], "changed-configurations": [ { - u'old_value': u'', - u'type': u'', - u'name': u'' + u'old_value': u'0', + u'type': u'hive-interactive-env', + u'name': u'llap_queue_capacity' } ], "configurations": { "capacity-scheduler": { "properties": { - "capacity-scheduler": "yarn.scheduler.capacity.root.accessible-node-labels=*\n" - "yarn.scheduler.capacity.root.capacity=100\n" - "yarn.scheduler.capacity.root.queues=default,llap\n" - "yarn.scheduler.capacity.maximum-applications=10000\n" - "yarn.scheduler.capacity.root.default.user-limit-factor=1\n" - "yarn.scheduler.capacity.root.default.state=RUNNING\n" - "yarn.scheduler.capacity.maximum-am-resource-percent=1\n" - "yarn.scheduler.capacity.root.default.acl_submit_applications=*\n" - "yarn.scheduler.capacity.root.default.capacity=80\n" - "yarn.scheduler.capacity.root.acl_administer_queue=*\n" - "yarn.scheduler.capacity.node-locality-delay=40\n" - "yarn.scheduler.capacity.queue-mappings-override.enable=false\n" - "yarn.scheduler.capacity.root.llap.user-limit-factor=1\n" - "yarn.scheduler.capacity.root.llap.state=RUNNING\n" - "yarn.scheduler.capacity.root.llap.ordering-policy=fifo\n" - "yarn.scheduler.capacity.root.llap.minimum-user-limit-percent=100\n" - "yarn.scheduler.capacity.root.llap.maximum-capacity=20\n" - "yarn.scheduler.capacity.root.default.maximum <TRUNCATED>
