Repository: ambari Updated Branches: refs/heads/trunk 97dd0815c -> d64868e72
AMBARI-17996: HAWQ service advisor shows wrong recommendations on edge cases (mithmatt) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d64868e7 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d64868e7 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d64868e7 Branch: refs/heads/trunk Commit: d64868e725e7dd034f725dde062e0c0ac92c1f02 Parents: 97dd081 Author: Matt <[email protected]> Authored: Thu Aug 4 11:30:51 2016 -0700 Committer: Matt <[email protected]> Committed: Thu Aug 4 11:30:51 2016 -0700 ---------------------------------------------------------------------- .../HAWQ/2.0.0/configuration/hawq-site.xml | 20 - .../HAWQ/2.0.0/service_advisor.py | 136 ++- .../src/main/resources/stacks/stack_advisor.py | 15 +- .../HAWQ/test_service_advisor.py | 823 ++++++++++--------- 4 files changed, 514 insertions(+), 480 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/d64868e7/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml index 9baaa1c..68333a7 100644 --- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml +++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml @@ -177,14 +177,6 @@ </description> <depends-on> <property> - <type>hawq-sysctl-env</type> - <name>vm.overcommit_ratio</name> - </property> - <property> - <type>hawq-sysctl-env</type> - <name>vm.overcommit_memory</name> - </property> - <property> <type>hawq-site</type> <name>hawq_global_rm_type</name> </property> @@ -310,12 +302,6 @@ The default set during installation is the number of HAWQ Segments multiplied by 6. Adjust the value when the number of HAWQ Segments changes in the cluster. If parameter value is changed, existing Hash distributed tables must be redistributed. </description> - <depends-on> - <property> - <type>hawq-site</type> - <name>hawq_rm_memory_limit_perseg</name> - </property> - </depends-on> <value-attributes> <type>int</type> <minimum>1</minimum> @@ -333,12 +319,6 @@ This parameter is used to limit the maximum resource usage on each segment node. The default is 6. Not recommended to change this parameter. </description> - <depends-on> - <property> - <type>hawq-site</type> - <name>hawq_rm_memory_limit_perseg</name> - </property> - </depends-on> <value-attributes> <type>int</type> <minimum>1</minimum> http://git-wip-us.apache.org/repos/asf/ambari/blob/d64868e7/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py index 0777dc6..434228d 100644 --- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py +++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py @@ -129,11 +129,6 @@ class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor): return any([self.isLocalHost(host) for host in hawqMasterComponentHosts]) def getServiceConfigurationRecommendations(self, configurations, clusterData, services, hosts): - # Determine if the cluster is secured - if "cluster-env" in services["configurations"] and "security_enabled" in services["configurations"]["cluster-env"]["properties"]: - is_secured = services["configurations"]["cluster-env"]["properties"]["security_enabled"] - else: - is_secured = "false" # Update HDFS properties in hdfs-site if "hdfs-site" in services["configurations"]: @@ -154,10 +149,11 @@ class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor): putCoreSiteProperty(property, desired_value) # Process HAWQ specific properties - if any(x in services["configurations"] for x in ["hawq-site", "hdfs-client", "hawq-sysctl-env"]): + if all(x in services["configurations"] for x in ["hawq-site", "hdfs-client", "hawq-sysctl-env"]): componentsListList = [service["components"] for service in services["services"]] componentsList = [item["StackServiceComponents"] for sublist in componentsListList for item in sublist] servicesList = [service["StackServices"]["service_name"] for service in services["services"]] + hawqMasterHosts = set(self.getHosts(componentsList, "HAWQMASTER")).union(set(self.getHosts(componentsList, "HAWQSTANDBY"))) hawqSegmentHosts = set(self.getHosts(componentsList, "HAWQSEGMENT")) hawqHosts = hawqMasterHosts.union(hawqSegmentHosts) @@ -165,20 +161,47 @@ class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor): minHawqHostsMemory = min([host['Hosts']['total_mem'] for host in hosts['items'] if host['Hosts']['host_name'] in hawqHosts]) minHawqHostsCoreCount = min([host['Hosts']['cpu_count'] for host in hosts['items'] if host['Hosts']['host_name'] in hawqHosts]) - if "hawq-site" in services["configurations"]: hawq_site = services["configurations"]["hawq-site"]["properties"] - putHawqSiteProperty = self.putProperty(configurations, "hawq-site", services) - putHawqSitePropertyAttribute = self.putPropertyAttribute(configurations, "hawq-site") hawq_sysctl_env = services["configurations"]["hawq-sysctl-env"]["properties"] + + putHawqSiteProperty = self.putProperty(configurations, "hawq-site", services) putHawqSysctlEnvProperty = self.putProperty(configurations, "hawq-sysctl-env", services) - putHawqSysctlEnvPropertyAttribute = self.putPropertyAttribute(configurations, "hawq-sysctl-env") + putHdfsClientProperty = self.putProperty(configurations, "hdfs-client", services) # remove master port when master is colocated with Ambari server if self.isHawqMasterComponentOnAmbariServer(services) and "hawq_master_address_port" in hawq_site: - putHawqSiteProperty('hawq_master_address_port', '') + putHawqSiteProperty("hawq_master_address_port", "") + + putHawqSiteProperty("hawq_rm_nvcore_limit_perseg", minHawqHostsCoreCount) + + MEMORY_THRESHOLD = 33554432 # 32GB, minHawqHostsMemory is represented in kB + + # set vm.overcommit_memory to 2 if the minimum memory among all hawqHosts is greater than 32GB + vm_overcommit_memory = 2 if minHawqHostsMemory >= MEMORY_THRESHOLD else 1 + vm_overcommit_ratio = int(hawq_sysctl_env["vm.overcommit_ratio"]) if "vm.overcommit_ratio" in hawq_sysctl_env else 50 + + putHawqSysctlEnvProperty("vm.overcommit_memory", vm_overcommit_memory) + putHawqSysctlEnvProperty("vm.overcommit_ratio", vm_overcommit_ratio) + + host_ram_kb = (float(minHawqHostsMemory) * float(vm_overcommit_ratio) / 100) if vm_overcommit_memory == 2 else minHawqHostsMemory + host_ram_gb = float(host_ram_kb) / (1024 * 1024) + recommended_mem_percentage = { + host_ram_gb <= 64: .75, + 64 < host_ram_gb <= 512: .85, + host_ram_gb > 512: .95 + }[True] + + hawq_rm_memory_limit_perseg_value = math.ceil(host_ram_gb * recommended_mem_percentage) + hawq_rm_memory_limit_perseg_unit = "GB" + # If RAM on a host is very low ~ 2 GB, ceil function may result in making it equal to total mem, in that case we recommend the value in MB not GB + if hawq_rm_memory_limit_perseg_value >= host_ram_gb : + hawq_rm_memory_limit_perseg_value = math.ceil(float(host_ram_kb)/1024 * recommended_mem_percentage) + hawq_rm_memory_limit_perseg_unit = "MB" - # update query limits if segments are deployed - if numSegments and "default_hash_table_bucket_number" in hawq_site and "hawq_rm_nvseg_perquery_limit" in hawq_site: + # hawq_rm_memory_limit_perseg does not support decimal value so trim decimal using int + putHawqSiteProperty("hawq_rm_memory_limit_perseg", "{0}{1}".format(int(hawq_rm_memory_limit_perseg_value), hawq_rm_memory_limit_perseg_unit)) + + if numSegments and "hawq_rm_nvseg_perquery_limit" in hawq_site: factor_min = 1 factor_max = 6 limit = int(hawq_site["hawq_rm_nvseg_perquery_limit"]) @@ -191,8 +214,14 @@ class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor): buckets = factor_max * numSegments else: buckets = factor * numSegments + + # In low memory environment, recalculate buckets with factor of 4 + # When host_ram_kb = 2796202.66667kB, hawq_rm_memory_limit_perseg is 2GB + if host_ram_kb < 2796202.66667: + factor = 4 + buckets = min(factor * numSegments, buckets) + putHawqSiteProperty('default_hash_table_bucket_number', buckets) - putHawqSitePropertyAttribute('default_hash_table_bucket_number', "maximum", numSegments * 16 if 10000 > numSegments * 16 else 10000) # update YARN RM urls with the values from yarn-site if YARN is installed if "YARN" in servicesList and "yarn-site" in services["configurations"]: @@ -201,58 +230,16 @@ class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor): if hs_prop in hawq_site and ys_prop in yarn_site: putHawqSiteProperty(hs_prop, yarn_site[ys_prop]) - putHawqSiteProperty('hawq_rm_nvcore_limit_perseg', minHawqHostsCoreCount) + # set output.replace-datanode-on-failure in HAWQ hdfs-client depending on the cluster size + # if number of segments is greater than 3, set to true + putHdfsClientProperty("output.replace-datanode-on-failure", str(numSegments > 3).lower()) - if "vm.overcommit_memory" in hawq_sysctl_env: - MEM_THRESHOLD = 33554432 # 32GB, minHawqHostsMemory is represented in kB - # Set the value for hawq_rm_memory_limit_perseg based on vm.overcommit value and RAM available on HAWQ Hosts - # If value of hawq_rm_memory_limit_perseg is 67108864KB, it indicates hawq is being added and recommendation - # has not be made yet, since after recommendation it will be in GB in case its 67108864KB. - vm_overcommit_ratio = int(hawq_sysctl_env["vm.overcommit_ratio"]) if "vm.overcommit_ratio" in hawq_sysctl_env else 50 - if "hawq_rm_memory_limit_perseg" in hawq_site and hawq_site["hawq_rm_memory_limit_perseg"] == "65535MB": - vm_overcommit_mem_value = 2 if minHawqHostsMemory >= MEM_THRESHOLD else 1 - else: - # set vm.overcommit_memory to 2 if the minimum memory among all hawqHosts is greater than 32GB - vm_overcommit_mem_value = int(hawq_sysctl_env["vm.overcommit_memory"]) - putHawqSysctlEnvProperty("vm.overcommit_ratio", vm_overcommit_ratio) - # Show vm.overcommit_ratio on theme only if vm.overcommit_memory is set to 2 - overcommit_ratio_visibility = "true" if vm_overcommit_mem_value == 2 else "false" - putHawqSysctlEnvPropertyAttribute("vm.overcommit_ratio", "visible", overcommit_ratio_visibility) - putHawqSysctlEnvProperty("vm.overcommit_memory", vm_overcommit_mem_value) - host_ram_kb = minHawqHostsMemory * vm_overcommit_ratio / 100 if vm_overcommit_mem_value == 2 else minHawqHostsMemory - host_ram_gb = float(host_ram_kb) / (1024 * 1024) - recommended_mem_percentage = { - host_ram_gb <= 64: .75, - 64 < host_ram_gb <= 512: .85, - host_ram_gb > 512: .95 - }[True] - recommended_mem = math.ceil(host_ram_gb * recommended_mem_percentage) - unit = "GB" - # If RAM on a host is very low ~ 2 GB, ceil function may result in making it equal to total mem, - # in that case we recommend the value in MB not GB - if recommended_mem >= host_ram_gb: - recommended_mem = math.ceil(float(host_ram_kb)/1024 * recommended_mem_percentage) - unit = "MB" - # hawq_rm_memory_limit_perseg does not support decimal value so trim decimal using int - putHawqSiteProperty("hawq_rm_memory_limit_perseg", "{0}{1}".format(int(recommended_mem), unit)) - - # Set default hawq_rm_nvseg_perquery_perseg_limit to 6, only if value was less than 6 - if "hawq_rm_nvseg_perquery_perseg_limit" in hawq_site and int(hawq_site["hawq_rm_nvseg_perquery_perseg_limit"]) < 6: - putHawqSiteProperty('hawq_rm_nvseg_perquery_perseg_limit', 6) - - if "hawq_global_rm_type" in hawq_site and "hawq_rm_memory_limit_perseg" in hawq_site: - hawq_rm_memory_limit_perseg = hawq_site["hawq_rm_memory_limit_perseg"].strip() - unit = hawq_rm_memory_limit_perseg[-2:] - value = hawq_rm_memory_limit_perseg[:-2] - # For clusters running with hawq_rm_memory_limit_perseg greater than or equal to 1GB but less than 2GB - if (unit == "GB" and 1 <= int(value) < 2) or (unit == "MB" and 1024 <= int(value) < 2048): - factor = 4 # Since memory is less drop hawq_rm_nvseg_perquery_perseg_limit to 4 - buckets = min(factor * numSegments, int(hawq_site["default_hash_table_bucket_number"])) if "default_hash_table_bucket_number" in hawq_site else factor * numSegments - putHawqSiteProperty('default_hash_table_bucket_number', buckets) - putHawqSiteProperty('hawq_rm_nvseg_perquery_perseg_limit', factor) - - # Show / Hide properties based on the value of hawq_global_rm_type - YARN_MODE = True if hawq_site["hawq_global_rm_type"].lower() == "yarn" else False + putHawqSitePropertyAttribute = self.putPropertyAttribute(configurations, "hawq-site") + putHawqSysctlEnvPropertyAttribute = self.putPropertyAttribute(configurations, "hawq-sysctl-env") + + ## Set ranges and visibility of properties + # Show/hide properties based on the value of hawq_global_rm_type + YARN_MODE = hawq_site["hawq_global_rm_type"].lower() == "yarn" yarn_mode_properties_visibility = { "hawq_rm_memory_limit_perseg": False, "hawq_rm_nvcore_limit_perseg": False, @@ -261,20 +248,17 @@ class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor): "hawq_rm_yarn_scheduler_address": True, "hawq_rm_yarn_address": True } - for property, visible_status in yarn_mode_properties_visibility.iteritems(): - putHawqSitePropertyAttribute(property, "visible", str(visible_status if YARN_MODE else not visible_status).lower()) - # set output.replace-datanode-on-failure in HAWQ hdfs-client depending on the cluster size - if "hdfs-client" in services["configurations"]: - MIN_NUM_SEGMENT_THRESHOLD = 3 - hdfs_client = services["configurations"]["hdfs-client"]["properties"] - if "output.replace-datanode-on-failure" in hdfs_client: - propertyValue = "true" if numSegments > MIN_NUM_SEGMENT_THRESHOLD else "false" - putHdfsClientProperty = self.putProperty(configurations, "hdfs-client", services) - putHdfsClientProperty("output.replace-datanode-on-failure", propertyValue) + for property, visibility in yarn_mode_properties_visibility.iteritems(): + putHawqSitePropertyAttribute(property, "visible", str(visibility if YARN_MODE else not visibility).lower()) + + putHawqSitePropertyAttribute("default_hash_table_bucket_number", "maximum", numSegments * 16 if numSegments * 16 < 10000 else 10000) def getHAWQYARNPropertyMapping(self): - return { "hawq_rm_yarn_address": "yarn.resourcemanager.address", "hawq_rm_yarn_scheduler_address": "yarn.resourcemanager.scheduler.address" } + return { + "hawq_rm_yarn_address": "yarn.resourcemanager.address", + "hawq_rm_yarn_scheduler_address": "yarn.resourcemanager.scheduler.address" + } def getServiceConfigurationsValidationItems(self, configurations, recommendedDefaults, services, hosts): siteName = "hawq-site" http://git-wip-us.apache.org/repos/asf/ambari/blob/d64868e7/ambari-server/src/main/resources/stacks/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/stack_advisor.py b/ambari-server/src/main/resources/stacks/stack_advisor.py index 437fe4f..f6191f8 100644 --- a/ambari-server/src/main/resources/stacks/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/stack_advisor.py @@ -897,13 +897,14 @@ class DefaultStackAdvisor(StackAdvisor): filteredConfigs = {} for type, names in configs.items(): - for name in names['properties']: - if type in requestedProperties.keys() and \ - name in requestedProperties[type]: - if type not in filteredConfigs.keys(): - filteredConfigs[type] = {'properties': {}} - filteredConfigs[type]['properties'][name] = \ - configs[type]['properties'][name] + if 'properties' in names.keys(): + for name in names['properties']: + if type in requestedProperties.keys() and \ + name in requestedProperties[type]: + if type not in filteredConfigs.keys(): + filteredConfigs[type] = {'properties': {}} + filteredConfigs[type]['properties'][name] = \ + configs[type]['properties'][name] if 'property_attributes' in names.keys(): for name in names['property_attributes']: if type in requestedProperties.keys() and \ http://git-wip-us.apache.org/repos/asf/ambari/blob/d64868e7/ambari-server/src/test/python/common-services/HAWQ/test_service_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/common-services/HAWQ/test_service_advisor.py b/ambari-server/src/test/python/common-services/HAWQ/test_service_advisor.py index 7ececa0..e0f1c43 100644 --- a/ambari-server/src/test/python/common-services/HAWQ/test_service_advisor.py +++ b/ambari-server/src/test/python/common-services/HAWQ/test_service_advisor.py @@ -22,7 +22,6 @@ from mock.mock import patch, MagicMock class TestHAWQ200ServiceAdvisor(TestCase): - testDirectory = os.path.dirname(os.path.abspath(__file__)) serviceAdvisorPath = '../../../../main/resources/common-services/HAWQ/2.0.0/service_advisor.py' hawq200ServiceAdvisorPath = os.path.join(testDirectory, serviceAdvisorPath) @@ -34,7 +33,7 @@ class TestHAWQ200ServiceAdvisor(TestCase): self.serviceAdvisor = serviceAdvisorClass() def fqdn_mock_result(value=None): - return 'c6401.ambari.apache.org' if value is None else value + return 'c6401.ambari.apache.org' if value is None else value def load_json(self, filename): file = os.path.join(self.testDirectory, "../configs", filename) @@ -43,9 +42,9 @@ class TestHAWQ200ServiceAdvisor(TestCase): return data def prepareHosts(self, hostsNames): - hosts = { "items": [] } + hosts = {"items": []} for hostName in hostsNames: - nextHost = {"Hosts":{"host_name" : hostName}} + nextHost = {"Hosts": {"host_name": hostName}} hosts["items"].append(nextHost) return hosts @@ -81,16 +80,16 @@ class TestHAWQ200ServiceAdvisor(TestCase): def getDesiredHDFSSiteValues(self, is_secure): hdfs_site_desired_values = { - "dfs.allow.truncate" : "true", - "dfs.block.access.token.enable" : str(is_secure).lower(), - "dfs.block.local-path-access.user" : "gpadmin", - "dfs.client.read.shortcircuit" : "true", - "dfs.client.use.legacy.blockreader.local" : "false", - "dfs.datanode.data.dir.perm" : "750", - "dfs.datanode.handler.count" : "60", - "dfs.datanode.max.transfer.threads" : "40960", - "dfs.namenode.accesstime.precision" : "0", - "dfs.support.append" : "true" + "dfs.allow.truncate": "true", + "dfs.block.access.token.enable": str(is_secure).lower(), + "dfs.block.local-path-access.user": "gpadmin", + "dfs.client.read.shortcircuit": "true", + "dfs.client.use.legacy.blockreader.local": "false", + "dfs.datanode.data.dir.perm": "750", + "dfs.datanode.handler.count": "60", + "dfs.datanode.max.transfer.threads": "40960", + "dfs.namenode.accesstime.precision": "0", + "dfs.support.append": "true" } return hdfs_site_desired_values @@ -181,22 +180,33 @@ class TestHAWQ200ServiceAdvisor(TestCase): standbyHosts = self.serviceAdvisor.getHostsForMasterComponent(services, None, component, hostsList) self.assertEquals(standbyHosts, ["c6401.ambari.apache.org"]) - - def test_getServiceConfigurationRecommendations(self): - + def setupToTestConfigurationRecommendations(self): configurations = { - "hawq-sysctl-env": { - "properties": { - "vm.overcommit_memory": 1, - "vm.overcommit_ratio": 50 - } - }, "hawq-site": { "properties": { + "hawq_master_address_port": "5432", "hawq_rm_memory_limit_perseg": "65535MB", "hawq_rm_nvcore_limit_perseg": "16", "hawq_global_rm_type": "yarn", - "default_hash_table_bucket_number": 18 + "hawq_rm_nvseg_perquery_limit": "512", + "hawq_rm_nvseg_perquery_perseg_limit": "6", + "default_hash_table_bucket_number": "5" + } + }, + "hawq-sysctl-env": { + "properties": { + "vm.overcommit_memory": "0", + "vm.overcommit_ratio": "75" + }, + "property_attributes": { + "vm.overcommit_ratio": { + "visible": "false" + } + } + }, + "hdfs-client": { + "properties": { + "output.replace-datanode-on-failure": "true" } }, "hdfs-site": { @@ -251,195 +261,356 @@ class TestHAWQ200ServiceAdvisor(TestCase): { "Hosts": { "host_name": "c6401.ambari.apache.org", - "cpu_count" : 2, + "cpu_count": 2, "total_mem": 33554432 } }, { "Hosts": { "host_name": "c6402.ambari.apache.org", - "cpu_count" : 4, + "cpu_count": 4, "total_mem": 33554433 } }, { "Hosts": { "host_name": "c6403.ambari.apache.org", - "cpu_count" : 1, + "cpu_count": 1, "total_mem": 33554434 } }, { "Hosts": { "host_name": "c6404.ambari.apache.org", - "cpu_count" : 2, + "cpu_count": 2, "total_mem": 33554435 } } ] } - ## Test that HDFS parameters required by HAWQ are recommended - configurations["cluster-env"]["properties"]["security_enabled"]="false" + return services, configurations, hosts + + def isLocalHost_sideEffect(self, host): + return host == "c6401.ambari.apache.org" + + def test_getServiceConfigurationRecommendations(self): + ## Test that HDFS parameters required by HAWQ are recommended correctly + + # Case 1: Security is not enabled + services, configurations, hosts = self.setupToTestConfigurationRecommendations() + configurations["cluster-env"]["properties"]["security_enabled"] = "false" + self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts) hdfs_site_desired_values = self.getDesiredHDFSSiteValues(False) for property, value in hdfs_site_desired_values.iteritems(): self.assertEquals(configurations["hdfs-site"]["properties"][property], value) self.assertEquals(configurations["core-site"]["properties"]["ipc.server.listen.queue.size"], "3300") + # Case 2: Security is enabled # Kerberos causes 1 property to be recommended differently - configurations["cluster-env"]["properties"]["security_enabled"]="true" - self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts) - self.assertEquals(configurations["hdfs-site"]["properties"]["dfs.block.access.token.enable"], "true") - + services, configurations, hosts = self.setupToTestConfigurationRecommendations() + configurations["cluster-env"]["properties"]["security_enabled"] = "true" - ## Test if hawq_rm_nvcore_limit_perseg is set correctly + self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts) + hdfs_site_desired_values = self.getDesiredHDFSSiteValues(True) + for property, value in hdfs_site_desired_values.iteritems(): + self.assertEquals(configurations["hdfs-site"]["properties"][property], value) + self.assertEquals(configurations["core-site"]["properties"]["ipc.server.listen.queue.size"], "3300") - # Case 1: + ## Test if hawq_master_address_port is blanked out when HAWQMASTER is placed on Ambari Server host + with patch.object(self.serviceAdvisor.__class__, 'isLocalHost', side_effect=self.isLocalHost_sideEffect): + # Case 1: HAWQMASTER is placed on Ambari Server Host + # Blank out hawq_master_address_port + services, configurations, hosts = self.setupToTestConfigurationRecommendations() + self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts) + self.assertEquals(configurations["hawq-site"]["properties"]["hawq_master_address_port"], "") + + # Case 2: HAWQMASTER is not placed on Ambari Server Host + # Retain hawq_master_address_port from existing configurations + services, configurations, hosts = self.setupToTestConfigurationRecommendations() + services["services"][0]["components"][0]["StackServiceComponents"]["hostnames"] = ["c6404.ambari.apache.org"] # Set HAWQMASTER Host + original_hawq_master_address_port = configurations["hawq-site"]["properties"]["hawq_master_address_port"] + self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts) + self.assertEquals(configurations["hawq-site"]["properties"]["hawq_master_address_port"], original_hawq_master_address_port) + + ## Test if hawq_rm_nvcore_limit_perseg is set according to the hosts core count + + # Case 1: User is coming to configs page for first time # HAWQ Hosts Core Count: c6401.ambari.apache.org - 2, c6402.ambari.apache.org - 4, c6404.ambari.apache.org - 2 - # hawq_global_rm_type: yarn - # Non HAWQ Hosts Core Count: c6401.ambari.apache.org - 1 + # Non HAWQ Hosts Core Count: c6403.ambari.apache.org - 1 + # Set hawq_rm_nvcore_limit_perseg to 2 + services, configurations, hosts = self.setupToTestConfigurationRecommendations() self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts) self.assertEquals(configurations["hawq-site"]["properties"]["hawq_rm_nvcore_limit_perseg"], "2") + # Case 2: User is coming to configs page for first time + # HAWQ Hosts Core Count: c6401.ambari.apache.org - 2, c6402.ambari.apache.org - 1, c6404.ambari.apache.org - 2 + # Non HAWQ Hosts Core Count: c6403.ambari.apache.org - 1 + # Set hawq_rm_nvcore_limit_perseg to 1 + services, configurations, hosts = self.setupToTestConfigurationRecommendations() + hosts["items"][1]["Hosts"]["cpu_count"] = 1 # Set c6402 cpu_count to 1 + self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts) + self.assertEquals(configurations["hawq-site"]["properties"]["hawq_rm_nvcore_limit_perseg"], "1") + ## Test if vm.overcommit_memory is set correctly # Case 1: All machines have total_mem above 32GB (total_mem >= 33554432) + # Set vm.overcommit_memory as 2 + services, configurations, hosts = self.setupToTestConfigurationRecommendations() self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts) self.assertEquals(configurations["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"], "2") # Case 2: One machine has total_mem below 32GB + # Set vm.overcommit_memory as 1 + services, configurations, hosts = self.setupToTestConfigurationRecommendations() hosts["items"][0]["Hosts"]["total_mem"] = 33554431 - services["configurations"]["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"] = "65535MB" self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts) self.assertEquals(configurations["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"], "1") + ## Test if vm.overcommit_ratio is set correctly + + # Case 1: vm.overcommit_ratio is not present in hawq_sysctl_env + # Set vm.overcommit_ratio to 50 + services, configurations, hosts = self.setupToTestConfigurationRecommendations() + del services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_ratio"] + self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts) + self.assertEqual(configurations["hawq-sysctl-env"]["properties"]["vm.overcommit_ratio"], "50") + + # Case 2: vm.overcommit_ratio is present in hawq_sysctl_env + # Retain vm.overcommit_ratio from existing configurations + services, configurations, hosts = self.setupToTestConfigurationRecommendations() + original_vm_overcommit_ratio = services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_ratio"] + self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts) + self.assertEqual(configurations["hawq-sysctl-env"]["properties"]["vm.overcommit_ratio"], original_vm_overcommit_ratio) + ## Test if hawq_rm_memory_limit_perseg is set correctly - # Case 1: Minimum host memory is ~ 2 GB (2048MB), recommended val must be .75% of 2GB as vm.overcommit_memory = 1 and in MB + # Case 1: Minimum host memory is 2GB + # Set vm.overcommit_memory set to 1 + # Set hawq_rm_memory_limit_perseg to 75% of 2GB, 1536MB + services, configurations, hosts = self.setupToTestConfigurationRecommendations() hosts["items"][0]["Hosts"]["total_mem"] = 2097152 self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts) + self.assertEquals(configurations["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"], "1") self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"], "1536MB") - # Case 2: Minimum host memory is ~ 16 GB, recommended val must be .75% of 16GB as vm.overcommit_memory = 1 and in GB + # Case 2: Minimum host memory is 16GB + # Set vm.overcommit_memory set to 1 + # Set hawq_rm_memory_limit_perseg to 75% of 16GB, 12GB + services, configurations, hosts = self.setupToTestConfigurationRecommendations() hosts["items"][0]["Hosts"]["total_mem"] = 16777216 hosts["items"][1]["Hosts"]["total_mem"] = 26777216 hosts["items"][3]["Hosts"]["total_mem"] = 36777216 self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts) + self.assertEquals(configurations["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"], "1") self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"], "12GB") - # Case 2: Minimum host memory is ~ 64 GB, recommended val must be .75% of 32GB as vm.overcommit_memory = 2 and in GB + # Case 3: Minimum host memory is 64GB + # Set vm.overcommit_memory set to 2 + # Ensure vm.overcommit_ratio is 50 (from initial configuration) + # Set hawq_rm_memory_limit_perseg to 75% of 16GB, 12GB + services, configurations, hosts = self.setupToTestConfigurationRecommendations() hosts["items"][0]["Hosts"]["total_mem"] = 67108864 hosts["items"][1]["Hosts"]["total_mem"] = 77108864 hosts["items"][3]["Hosts"]["total_mem"] = 87108864 - services["configurations"]["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"] = "65535MB" + services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_ratio"] = "50" self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts) + self.assertEquals(configurations["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"], "2") + self.assertEquals(configurations["hawq-sysctl-env"]["properties"]["vm.overcommit_ratio"], "50") self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"], "24GB") - # Case 4: Minimum host memory is ~ 512 GB, recommended val must be .85% of 256GB as vm.overcommit_memory = 2 and in GB + # Case 4: Minimum host memory is 512GB + # Set vm.overcommit_memory set to 2 + # Ensure vm.overcommit_ratio is 50 (from initial configuration) + # Set hawq_rm_memory_limit_perseg to 85% of 0.5 * 512GB, 218GB + services, configurations, hosts = self.setupToTestConfigurationRecommendations() hosts["items"][0]["Hosts"]["total_mem"] = 536870912 hosts["items"][1]["Hosts"]["total_mem"] = 636870912 hosts["items"][3]["Hosts"]["total_mem"] = 736870912 + services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_ratio"] = "50" self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts) + self.assertEquals(configurations["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"], "2") + self.assertEquals(configurations["hawq-sysctl-env"]["properties"]["vm.overcommit_ratio"], "50") self.assertEquals(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"], "218GB") - # Case 5: Minimum host memory is ~ 1024 GB, recommended val must be .95% of 512GB as vm.overcommit_memory = 2 and in GB + # Case 5: Minimum host memory is 1024GB + # Set vm.overcommit_memory set to 2 + # Ensure vm.overcommit_ratio is 50 (from initial configuration) + # Set hawq_rm_memory_limit_perseg to 85% of 0.5 * 1024GB, 436GB + services, configurations, hosts = self.setupToTestConfigurationRecommendations() hosts["items"][0]["Hosts"]["total_mem"] = 1073741824 hosts["items"][1]["Hosts"]["total_mem"] = 2073741824 hosts["items"][3]["Hosts"]["total_mem"] = 3073741824 + services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_ratio"] = "50" self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts) + self.assertEquals(configurations["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"], "2") + self.assertEquals(configurations["hawq-sysctl-env"]["properties"]["vm.overcommit_ratio"], "50") self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"], "436GB") - # Case 6: Minimum host memory is ~ 1024 GB, vm.overcommit_ratio = 75, vm.overcommit_memory = 2 - # recommended val must be .95% of (1024*75)/100 and in GB + # Case 6: Minimum host memory is 1024GB + # Set vm.overcommit_memory set to 2 + # Ensure vm.overcommit_ratio is 75 (from initial configuration) + # Set hawq_rm_memory_limit_perseg to 95% of 0.5 * 1024GB, 436GB + services, configurations, hosts = self.setupToTestConfigurationRecommendations() hosts["items"][0]["Hosts"]["total_mem"] = 1073741824 hosts["items"][1]["Hosts"]["total_mem"] = 2073741824 hosts["items"][3]["Hosts"]["total_mem"] = 3073741824 - services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_ratio"] = 75 + services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_ratio"] = "75" self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts) + self.assertEquals(configurations["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"], "2") + self.assertEquals(configurations["hawq-sysctl-env"]["properties"]["vm.overcommit_ratio"], "75") self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"], "730GB") - ## Test if default_hash_table_bucket_number and hawq_rm_nvseg_perquery_perseg_limit are set correctly based on low hawq_rm_memory_limit_perseg + ## Test if default_hash_table_bucket_number and hawq_rm_nvseg_perquery_perseg_limit are set correctly - # Case 1: When hawq_rm_memory_limit_perseg is between 1GB and 2GB - # Set hawq_rm_nvseg_perquery_perseg_limit to 4 and default_hash_table_bucket_number as hawq_rm_nvseg_perquery_perseg_limit * numSegments - hosts["items"][0]["Hosts"]["total_mem"] = 2097152 - hosts["items"][1]["Hosts"]["total_mem"] = 2097152 - hosts["items"][3]["Hosts"]["total_mem"] = 2097152 - services["configurations"]["hawq-site"]["properties"]["hawq_global_rm_type"] = "none" - services["configurations"]["hawq-site"]["properties"]["hawq_rm_nvseg_perquery_limit"] = "512" + # Case 1: No. of HAWQSEGMENTs - 2 + # Set default_hash_table_bucket_number to 12 + services, configurations, hosts = self.setupToTestConfigurationRecommendations() + componentsList = self.getComponentsListFromServices(services) + hawqSegmentHosts = self.getHosts(componentsList, "HAWQSEGMENT") + self.assertEquals(len(hawqSegmentHosts["hostnames"]), 2) self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts) - self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"], "1152MB") - self.assertEqual(configurations["hawq-site"]["properties"]["default_hash_table_bucket_number"], "8") - self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_nvseg_perquery_perseg_limit"], "4") + self.assertEquals(configurations["hawq-site"]["properties"]["default_hash_table_bucket_number"], "12") - # Case 2: When hawq_rm_memory_limit_perseg > 2GB - # Set hawq_rm_nvseg_perquery_perseg_limit to 6 and default_hash_table_bucket_number as hawq_rm_nvseg_perquery_perseg_limit * numSegments - hosts["items"][0]["Hosts"]["total_mem"] = 1073741824 - hosts["items"][1]["Hosts"]["total_mem"] = 2073741824 - hosts["items"][3]["Hosts"]["total_mem"] = 3073741824 - services["configurations"]["hawq-site"]["properties"]["hawq_rm_nvseg_perquery_limit"] = "512" - services["configurations"]["hawq-site"]["properties"]["hawq_rm_nvseg_perquery_perseg_limit"] = "4" + # Case 2: No. of HAWQSEGMENTs - 100 + # Set default_hash_table_bucket_number to 500 + services, configurations, hosts = self.setupToTestConfigurationRecommendations() + componentsList = self.getComponentsListFromServices(services) + hawqSegmentHosts = self.getHosts(componentsList, "HAWQSEGMENT") + hawqSegmentHosts["hostnames"] = ["host" + str(i) for i in range(100)] + self.assertEquals(len(hawqSegmentHosts["hostnames"]), 100) self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts) - self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"], "730GB") - self.assertEqual(configurations["hawq-site"]["properties"]["default_hash_table_bucket_number"], "12") - self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_nvseg_perquery_perseg_limit"], "6") + self.assertEquals(configurations["hawq-site"]["properties"]["default_hash_table_bucket_number"], "500") - # Case 3: When hawq_rm_memory_limit_perseg > 2GB - # Set hawq_rm_nvseg_perquery_perseg_limit to 8 and default_hash_table_bucket_number as hawq_rm_nvseg_perquery_perseg_limit * numSegments - hosts["items"][0]["Hosts"]["total_mem"] = 1073741824 - hosts["items"][1]["Hosts"]["total_mem"] = 2073741824 - hosts["items"][3]["Hosts"]["total_mem"] = 3073741824 - services["configurations"]["hawq-site"]["properties"]["hawq_rm_nvseg_perquery_limit"] = "512" - services["configurations"]["hawq-site"]["properties"]["hawq_rm_nvseg_perquery_perseg_limit"] = "8" + # Case 3: No. of HAWQSEGMENTs - 512 + # Set default_hash_table_bucket_number to 512 + services, configurations, hosts = self.setupToTestConfigurationRecommendations() + componentsList = self.getComponentsListFromServices(services) + hawqSegmentHosts = self.getHosts(componentsList, "HAWQSEGMENT") + hawqSegmentHosts["hostnames"] = ["host" + str(i) for i in range(512)] + self.assertEquals(len(hawqSegmentHosts["hostnames"]), 512) self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts) - self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"], "730GB") - self.assertEqual(configurations["hawq-site"]["properties"]["default_hash_table_bucket_number"], "12") - self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_nvseg_perquery_perseg_limit"], "8") + self.assertEquals(configurations["hawq-site"]["properties"]["default_hash_table_bucket_number"], "512") + + # Case 4: No. of HAWQSEGMENTs - 513 + # Set default_hash_table_bucket_number to 512 + services, configurations, hosts = self.setupToTestConfigurationRecommendations() + componentsList = self.getComponentsListFromServices(services) + hawqSegmentHosts = self.getHosts(componentsList, "HAWQSEGMENT") + hawqSegmentHosts["hostnames"] = ["host" + str(i) for i in range(513)] + self.assertEquals(len(hawqSegmentHosts["hostnames"]), 513) + self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts) + self.assertEquals(configurations["hawq-site"]["properties"]["default_hash_table_bucket_number"], "512") + + # Case 5: No. of HAWQSEGMENTs - 3 and minimum host memory is 1.5GB + # Set default_hash_table_bucket_number to 12 + services, configurations, hosts = self.setupToTestConfigurationRecommendations() + hosts["items"][0]["Hosts"]["total_mem"] = 1572864 + componentsList = self.getComponentsListFromServices(services) + hawqSegmentHosts = self.getHosts(componentsList, "HAWQSEGMENT") + hawqSegmentHosts["hostnames"] = ["host" + str(i) for i in range(3)] + self.assertEquals(len(hawqSegmentHosts["hostnames"]), 3) + self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts) + self.assertEquals(configurations["hawq-site"]["properties"]["default_hash_table_bucket_number"], "12") - ## Test if the properties are set to visible / invisible based on the value of hawq_global_rm_type + # Case 6: No. of HAWQSEGMENTs - 513 and minimum host memory is 1.5GB + # Set default_hash_table_bucket_number to 12 + services, configurations, hosts = self.setupToTestConfigurationRecommendations() + hosts["items"][0]["Hosts"]["total_mem"] = 1572864 + componentsList = self.getComponentsListFromServices(services) + hawqSegmentHosts = self.getHosts(componentsList, "HAWQSEGMENT") + hawqSegmentHosts["hostnames"] = ["host" + str(i) for i in range(513)] + self.assertEquals(len(hawqSegmentHosts["hostnames"]), 513) + self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts) + self.assertEquals(configurations["hawq-site"]["properties"]["default_hash_table_bucket_number"], "512") + + # Case 7: No. of HAWQSEGMENTs - 0 + # Set default_hash_table_bucket_number to 12 + services, configurations, hosts = self.setupToTestConfigurationRecommendations() + componentsList = self.getComponentsListFromServices(services) + hawqSegmentHosts = self.getHosts(componentsList, "HAWQSEGMENT") + hawqSegmentHosts["hostnames"] = [] + original_default_hash_table_bucket_number = services["configurations"]["hawq-site"]["properties"]["default_hash_table_bucket_number"] + self.assertEquals(len(hawqSegmentHosts["hostnames"]), 0) + self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts) + self.assertEquals(configurations["hawq-site"]["properties"]["default_hash_table_bucket_number"], original_default_hash_table_bucket_number) + + ## Test if output.replace-datanode-on-failure correctly + + # Case 1: No. of HAWQSEGMENTs - 3 + # Set output.replace-datanode-on-failure to true + services, configurations, hosts = self.setupToTestConfigurationRecommendations() + componentsList = self.getComponentsListFromServices(services) + hawqSegmentHosts = self.getHosts(componentsList, "HAWQSEGMENT") + hawqSegmentHosts["hostnames"] = ["host" + str(i) for i in range(3)] + self.assertEquals(len(hawqSegmentHosts["hostnames"]), 3) + self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts) + self.assertEquals(configurations["hdfs-client"]["properties"]["output.replace-datanode-on-failure"], "false") + + # Case 2: No. of HAWQSEGMENTs - 4 + # Set output.replace-datanode-on-failure to true + services, configurations, hosts = self.setupToTestConfigurationRecommendations() + componentsList = self.getComponentsListFromServices(services) + hawqSegmentHosts = self.getHosts(componentsList, "HAWQSEGMENT") + hawqSegmentHosts["hostnames"] = ["host" + str(i) for i in range(4)] + self.assertEquals(len(hawqSegmentHosts["hostnames"]), 4) + self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts) + self.assertEquals(configurations["hdfs-client"]["properties"]["output.replace-datanode-on-failure"], "true") + + ## Test if RM properties visibility is set correctly # Case 1: When hawq_global_rm_type is yarn + services, configurations, hosts = self.setupToTestConfigurationRecommendations() services["configurations"]["hawq-site"]["properties"]["hawq_global_rm_type"] = "yarn" - properties_visible_status = {"hawq_rm_memory_limit_perseg": "false", - "hawq_rm_nvcore_limit_perseg": "false", - "hawq_rm_yarn_app_name":"true", - "hawq_rm_yarn_queue_name": "true", - "hawq_rm_yarn_scheduler_address": "true", - "hawq_rm_yarn_address": "true"} + properties_visibility = { + "hawq_rm_memory_limit_perseg": "false", + "hawq_rm_nvcore_limit_perseg": "false", + "hawq_rm_yarn_app_name": "true", + "hawq_rm_yarn_queue_name": "true", + "hawq_rm_yarn_scheduler_address": "true", + "hawq_rm_yarn_address": "true" + } self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts) - for property, status in properties_visible_status.iteritems(): + for property, status in properties_visibility.iteritems(): self.assertEqual(configurations["hawq-site"]["property_attributes"][property]["visible"], status) # Case 2: When hawq_global_rm_type is none + services, configurations, hosts = self.setupToTestConfigurationRecommendations() services["configurations"]["hawq-site"]["properties"]["hawq_global_rm_type"] = "none" - properties_visible_status = {"hawq_rm_memory_limit_perseg": "true", - "hawq_rm_nvcore_limit_perseg": "true", - "hawq_rm_yarn_app_name": "false", - "hawq_rm_yarn_queue_name": "false", - "hawq_rm_yarn_scheduler_address": "false", - "hawq_rm_yarn_address": "false"} + properties_visibility = { + "hawq_rm_memory_limit_perseg": "true", + "hawq_rm_nvcore_limit_perseg": "true", + "hawq_rm_yarn_app_name": "false", + "hawq_rm_yarn_queue_name": "false", + "hawq_rm_yarn_scheduler_address": "false", + "hawq_rm_yarn_address": "false" + } self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts) - for property, status in properties_visible_status.iteritems(): + for property, status in properties_visibility.iteritems(): self.assertEqual(configurations["hawq-site"]["property_attributes"][property]["visible"], status) - ## Test if vm.overcommit_ratio is set to visible / invisible based on the value of vm.overcommit_memory - - # Case 1: vm.overcommit_ratio should be invisible when overcommit_memory is set as 0 - services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"] = 0 - self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts) - self.assertEqual(configurations["hawq-sysctl-env"]["property_attributes"]["vm.overcommit_ratio"]["visible"], "false") + ## Test if maximum range of default_hash_table_bucket_number is set correctly - # Case 2: vm.overcommit_ratio should be invisible when overcommit_memory is set as 1 - services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"] = 1 + # Case 1: No. of HAWQSEGMENTs - 624 + # Set default_hash_table_bucket_number maximum range to 9984 + services, configurations, hosts = self.setupToTestConfigurationRecommendations() + componentsList = self.getComponentsListFromServices(services) + hawqSegmentHosts = self.getHosts(componentsList, "HAWQSEGMENT") + hawqSegmentHosts["hostnames"] = ["host" + str(i) for i in range(624)] + self.assertEquals(len(hawqSegmentHosts["hostnames"]), 624) self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts) - self.assertEqual(configurations["hawq-sysctl-env"]["property_attributes"]["vm.overcommit_ratio"]["visible"], "false") + self.assertEquals(configurations["hawq-site"]["property_attributes"]["default_hash_table_bucket_number"]["maximum"], "9984") - # Case 3: vm.overcommit_ratio should be visible when overcommit_memory is set as 2 - services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"] = 2 + # Case 2: No. of HAWQSEGMENTs - 1000 + # Set default_hash_table_bucket_number maximum range to 10000 + services, configurations, hosts = self.setupToTestConfigurationRecommendations() + componentsList = self.getComponentsListFromServices(services) + hawqSegmentHosts = self.getHosts(componentsList, "HAWQSEGMENT") + hawqSegmentHosts["hostnames"] = ["host" + str(i) for i in range(1000)] + self.assertEquals(len(hawqSegmentHosts["hostnames"]), 1000) self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts) - self.assertEqual(configurations["hawq-sysctl-env"]["property_attributes"]["vm.overcommit_ratio"]["visible"], "true") - + self.assertEquals(configurations["hawq-site"]["property_attributes"]["default_hash_table_bucket_number"]["maximum"], "10000") def test_createComponentLayoutRecommendations_hawq_3_Hosts(self): """ Test that HAWQSTANDBY is recommended on a 3-node cluster """ @@ -507,40 +678,40 @@ class TestHAWQ200ServiceAdvisor(TestCase): """ Test that HAWQSEGMENT gets recommended correctly during Cluster Install Wizard, when HAWQ is selected for installation """ hosts = self.prepareHosts(["c6401.ambari.apache.org", "c6402.ambari.apache.org", "c6403.ambari.apache.org"]) - services = { - "services" : [ - { - "StackServices" : { - "service_name" : "HDFS" - }, - "components" : [ - { - "StackServiceComponents" : { - "cardinality" : "1+", - "component_category" : "SLAVE", - "component_name" : "DATANODE", - "hostnames" : [] - } - } - ] - }, - { - "StackServices" : { - "service_name" : "HAWQ" - }, - "components" : [ - { - "StackServiceComponents" : { - "cardinality" : "1+", - "component_category" : "SLAVE", - "component_name" : "HAWQSEGMENT", - "hostnames" : [] - } - } - ] - } - ] - } + services = { + "services": [ + { + "StackServices": { + "service_name": "HDFS" + }, + "components": [ + { + "StackServiceComponents": { + "cardinality": "1+", + "component_category": "SLAVE", + "component_name": "DATANODE", + "hostnames": [] + } + } + ] + }, + { + "StackServices": { + "service_name": "HAWQ" + }, + "components": [ + { + "StackServiceComponents": { + "cardinality": "1+", + "component_category": "SLAVE", + "component_name": "HAWQSEGMENT", + "hostnames": [] + } + } + ] + } + ] + } hawqSegmentHosts = {"c6401.ambari.apache.org", "c6402.ambari.apache.org", "c6403.ambari.apache.org"} self.insertHAWQServiceAdvisorInfo(services) @@ -552,40 +723,40 @@ class TestHAWQ200ServiceAdvisor(TestCase): """ Test that HAWQSEGMENT gets recommended correctly during Add Service Wizard, when HAWQ is selected for installation """ hosts = self.prepareHosts(["c6401.ambari.apache.org", "c6402.ambari.apache.org", "c6403.ambari.apache.org"]) - services = { - "services" : [ - { - "StackServices" : { - "service_name" : "HDFS" - }, - "components" : [ - { - "StackServiceComponents" : { - "cardinality" : "1+", - "component_category" : "SLAVE", - "component_name" : "DATANODE", - "hostnames" : ["c6401.ambari.apache.org", "c6403.ambari.apache.org"] - } - } - ] - }, - { - "StackServices" : { - "service_name" : "HAWQ" - }, - "components" : [ - { - "StackServiceComponents" : { - "cardinality" : "1+", - "component_category" : "SLAVE", - "component_name" : "HAWQSEGMENT", - "hostnames" : [] - } - } - ] - } - ] - } + services = { + "services": [ + { + "StackServices": { + "service_name": "HDFS" + }, + "components": [ + { + "StackServiceComponents": { + "cardinality": "1+", + "component_category": "SLAVE", + "component_name": "DATANODE", + "hostnames": ["c6401.ambari.apache.org", "c6403.ambari.apache.org"] + } + } + ] + }, + { + "StackServices": { + "service_name": "HAWQ" + }, + "components": [ + { + "StackServiceComponents": { + "cardinality": "1+", + "component_category": "SLAVE", + "component_name": "HAWQSEGMENT", + "hostnames": [] + } + } + ] + } + ] + } hawqSegmentHosts = {"c6401.ambari.apache.org", "c6403.ambari.apache.org"} self.insertHAWQServiceAdvisorInfo(services) @@ -597,55 +768,55 @@ class TestHAWQ200ServiceAdvisor(TestCase): """ Test that HAWQSEGMENT does not get recommended during Add Service Wizard, when HAWQ has already been installed """ hosts = self.prepareHosts(["c6401.ambari.apache.org", "c6402.ambari.apache.org", "c6403.ambari.apache.org"]) - services = { - "services" : [ - { - "StackServices" : { - "service_name" : "HDFS" - }, - "components" : [ - { - "StackServiceComponents" : { - "cardinality" : "1+", - "component_category" : "SLAVE", - "component_name" : "DATANODE", - "hostnames" : ["c6401.ambari.apache.org", "c6403.ambari.apache.org"] - } - } - ] - }, - { - "StackServices" : { - "service_name" : "HAWQ" - }, - "components" : [ - { - "StackServiceComponents" : { - "cardinality" : "1+", - "component_category" : "SLAVE", - "component_name" : "HAWQSEGMENT", - "hostnames" : ["c6402.ambari.apache.org"] - } - } - ] - }, - { - "StackServices" : { - "service_name" : "PXF" - }, - "components" : [ - { - "StackServiceComponents" : { - "cardinality" : "1+", - "component_category" : "SLAVE", - "component_name" : "PXF", - "hostnames" : [] - } - } - ] - } - ] - } + services = { + "services": [ + { + "StackServices": { + "service_name": "HDFS" + }, + "components": [ + { + "StackServiceComponents": { + "cardinality": "1+", + "component_category": "SLAVE", + "component_name": "DATANODE", + "hostnames": ["c6401.ambari.apache.org", "c6403.ambari.apache.org"] + } + } + ] + }, + { + "StackServices": { + "service_name": "HAWQ" + }, + "components": [ + { + "StackServiceComponents": { + "cardinality": "1+", + "component_category": "SLAVE", + "component_name": "HAWQSEGMENT", + "hostnames": ["c6402.ambari.apache.org"] + } + } + ] + }, + { + "StackServices": { + "service_name": "PXF" + }, + "components": [ + { + "StackServiceComponents": { + "cardinality": "1+", + "component_category": "SLAVE", + "component_name": "PXF", + "hostnames": [] + } + } + ] + } + ] + } hawqSegmentHosts = {"c6402.ambari.apache.org"} self.insertHAWQServiceAdvisorInfo(services) @@ -679,7 +850,6 @@ class TestHAWQ200ServiceAdvisor(TestCase): validations = self.serviceAdvisor.getComponentLayoutValidations(services, hosts) self.assertEquals(len(validations), 0) - @patch('socket.getfqdn', side_effect=fqdn_mock_result) def test_getComponentLayoutValidations_hawq_3_Hosts(self, socket_mock): """ Test layout validations for HAWQ components on a 3-node cluster """ @@ -737,7 +907,7 @@ class TestHAWQ200ServiceAdvisor(TestCase): self.assertEquals(len(validations), 1) expected = { 'component-name': 'HAWQMASTER', - 'message': 'The default Postgres port (5432) on the Ambari Server conflicts with the default HAWQ Masters port. ' + + 'message': 'The default Postgres port (5432) on the Ambari Server conflicts with the default HAWQ Masters port. ' + 'If you are using port 5432 for Postgres, you must either deploy the HAWQ Master on a different host ' + 'or configure a different port for the HAWQ Masters in the HAWQ Configuration page.', 'type': 'host-component', @@ -761,7 +931,7 @@ class TestHAWQ200ServiceAdvisor(TestCase): self.assertEquals(len(validations), 1) expected = { 'component-name': 'HAWQSTANDBY', - 'message': 'The default Postgres port (5432) on the Ambari Server conflicts with the default HAWQ Masters port. ' + + 'message': 'The default Postgres port (5432) on the Ambari Server conflicts with the default HAWQ Masters port. ' + 'If you are using port 5432 for Postgres, you must either deploy the HAWQ Standby Master on a different host ' + 'or configure a different port for the HAWQ Masters in the HAWQ Configuration page.', 'type': 'host-component', @@ -770,7 +940,6 @@ class TestHAWQ200ServiceAdvisor(TestCase): } self.assertEquals(validations[0], expected) - @patch('socket.getfqdn', side_effect=fqdn_mock_result) def test_getComponentLayoutValidations_nohawq_3_Hosts(self, socket_mock): """ Test no failures when there are no HAWQ components on a 3-node cluster """ @@ -791,105 +960,6 @@ class TestHAWQ200ServiceAdvisor(TestCase): validations = self.serviceAdvisor.getComponentLayoutValidations(services, hosts) self.assertEquals(len(validations), 0) - def test_recommendHAWQConfigurations(self): - - hosts = { - "items": [ - { - "Hosts": { - "host_name": "c6401.ambari.apache.org", - "cpu_count" : 2, - "total_mem": 33554432 - } - }, - { - "Hosts": { - "host_name": "c6402.ambari.apache.org", - "cpu_count" : 4, - "total_mem": 33554433 - } - }, - { - "Hosts": { - "host_name": "c6403.ambari.apache.org", - "cpu_count" : 1, - "total_mem": 33554434 - } - }, - { - "Hosts": { - "host_name": "c6404.ambari.apache.org", - "cpu_count" : 2, - "total_mem": 33554435 - } - } - ] - } - - # original cluster data with 3 segments - services = self.load_json("services-normal-hawq-3-hosts.json") - componentsList = self.getComponentsListFromServices(services) - hawqSegmentComponent = self.getHosts(componentsList, "HAWQSEGMENT") - - # setup default configuration values - services["configurations"]["hawq-site"] = { - "properties": { - "default_hash_table_bucket_number": "24", - "hawq_rm_nvseg_perquery_limit": "512", - "hawq_rm_yarn_address": "localhost:8032", - "hawq_rm_yarn_scheduler_address": "localhost:8030", - "hawq_global_rm_type": "none", - "hawq_rm_nvseg_perquery_perseg_limit": "6" - } - } - - services["configurations"]["hdfs-client"] = {"properties": {"output.replace-datanode-on-failure": "true"}} - services["configurations"]["hawq-sysctl-env"] = {"properties": {}} - - services["configurations"]["yarn-site"] = {"properties": {"yarn.resourcemanager.address": "host1:8050", - "yarn.resourcemanager.scheduler.address": "host1:8030"}} - services["services"].append({"StackServices" : {"service_name" : "YARN"}, "components":[]}) - configurations = {} - clusterData = {} - self.insertHAWQServiceAdvisorInfo(services) - - # Test 1 - with 3 segments - self.assertEquals(len(hawqSegmentComponent["hostnames"]), 3) - self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, clusterData, services, hosts) - self.assertEquals(configurations["hawq-site"]["properties"]["default_hash_table_bucket_number"], str(3 * 6)) - self.assertEquals(configurations["hdfs-client"]["properties"]["output.replace-datanode-on-failure"], "false") - - # check derived properties - self.assertEquals(configurations["hawq-site"]["properties"]["hawq_rm_yarn_address"], "host1:8050") - self.assertEquals(configurations["hawq-site"]["properties"]["hawq_rm_yarn_scheduler_address"], "host1:8030") - - # Test 2 - with 100 segments - hawqSegmentComponent["hostnames"] = ["host" + str(i) for i in range(100)] - self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, clusterData, services, hosts) - self.assertEquals(configurations["hawq-site"]["properties"]["default_hash_table_bucket_number"], str(100 * 5)) - self.assertEquals(configurations["hdfs-client"]["properties"]["output.replace-datanode-on-failure"], "true") - - # Test 3 - with 512 segments - hawqSegmentComponent["hostnames"] = ["host" + str(i) for i in range(512)] - self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, clusterData, services, hosts) - self.assertEquals(configurations["hawq-site"]["properties"]["default_hash_table_bucket_number"], "512") - self.assertEquals(configurations["hdfs-client"]["properties"]["output.replace-datanode-on-failure"], "true") - - # Test 4 - with 513 segments - hawqSegmentComponent["hostnames"] = ["host" + str(i) for i in range(513)] - self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, clusterData, services, hosts) - self.assertEquals(configurations["hawq-site"]["properties"]["default_hash_table_bucket_number"], "512") - self.assertEquals(configurations["hdfs-client"]["properties"]["output.replace-datanode-on-failure"], "true") - - # Test 5 - with no segments - configurations = {} - services["configurations"]["hawq-site"] = {"properties":{"hawq_global_rm_type": "none"}} - hawqSegmentComponent["hostnames"] = [] - self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, clusterData, services, hosts) - self.assertEquals(configurations["hdfs-client"]["properties"]["output.replace-datanode-on-failure"], "false") - self.assertTrue("default_hash_table_bucket_number" not in configurations["hawq-site"]) - - def test_validateHAWQSiteConfigurations(self): services = self.load_json("services-hawq-3-hosts.json") # setup default configuration values @@ -899,7 +969,7 @@ class TestHAWQ200ServiceAdvisor(TestCase): "hawq_rm_yarn_scheduler_address": "localhost:8030"}} configurations["yarn-site"] = {"properties": {"yarn.resourcemanager.address": "host1:8050", "yarn.resourcemanager.scheduler.address": "host1:8030"}} - services["services"].append({"StackServices" : {"service_name" : "YARN"}, "components":[]}) + services["services"].append({"StackServices": {"service_name": "YARN"}, "components": []}) properties = configurations["hawq-site"]["properties"] defaults = {} hosts = {} @@ -966,22 +1036,22 @@ class TestHAWQ200ServiceAdvisor(TestCase): # Test hawq_global_rm_type validation services = { - "services" : [ - { - "StackServices" : { - "service_name" : "HAWQ" - }, - "components": [] - } ], - "configurations": - { - "hawq-site": { - "properties": { - "hawq_global_rm_type": "yarn" - } - } - } - } + "services": [ + { + "StackServices": { + "service_name": "HAWQ" + }, + "components": [] + }], + "configurations": + { + "hawq-site": { + "properties": { + "hawq_global_rm_type": "yarn" + } + } + } + } properties = services["configurations"]["hawq-site"]["properties"] # case 1: hawq_global_rm_type is set as yarn, but YARN service is not installed. Validation error expected. @@ -997,7 +1067,7 @@ class TestHAWQ200ServiceAdvisor(TestCase): self.assertEqual(problems[0], expected) # case 2: hawq_global_rm_type is set as yarn, and YARN service is installed. No validation errors expected. - services["services"].append({"StackServices" : {"service_name" : "YARN"}, "components":[]}) + services["services"].append({"StackServices": {"service_name": "YARN"}, "components": []}) problems = self.serviceAdvisor.validateHAWQSiteConfigurations(properties, defaults, services["configurations"], services, hosts) self.assertEqual(len(problems), 0) @@ -1044,14 +1114,14 @@ class TestHAWQ200ServiceAdvisor(TestCase): # -------- test query limits warning ---------- services = { - "services": [ - { "StackServices": {"service_name": "HAWQ"}, - "components": [{ - "StackServiceComponents": { - "component_name": "HAWQSEGMENT", - "hostnames": [] - }}] - }], + "services": [ + {"StackServices": {"service_name": "HAWQ"}, + "components": [{ + "StackServiceComponents": { + "component_name": "HAWQSEGMENT", + "hostnames": [] + }}] + }], "configurations": {} } # setup default configuration values @@ -1112,17 +1182,16 @@ class TestHAWQ200ServiceAdvisor(TestCase): problems = self.serviceAdvisor.validateHAWQSiteConfigurations(properties, defaults, configurations, services, hosts) self.assertEqual(len(problems), 0) - def test_validateHAWQHdfsClientConfigurations(self): services = { - "services": [ - { "StackServices": {"service_name": "HAWQ"}, - "components": [{ - "StackServiceComponents": { - "component_name": "HAWQSEGMENT", - "hostnames": [] - }}] - }], + "services": [ + {"StackServices": {"service_name": "HAWQ"}, + "components": [{ + "StackServiceComponents": { + "component_name": "HAWQSEGMENT", + "hostnames": [] + }}] + }], "configurations": {} } # setup default configuration values @@ -1134,11 +1203,11 @@ class TestHAWQ200ServiceAdvisor(TestCase): # 1. Try with no hosts expected = { - 'config-type': 'hdfs-client', - 'message': 'output.replace-datanode-on-failure should be set to false (unchecked) for clusters with 3 or less HAWQ Segments', - 'type': 'configuration', - 'config-name': 'output.replace-datanode-on-failure', - 'level': 'WARN' + 'config-type': 'hdfs-client', + 'message': 'output.replace-datanode-on-failure should be set to false (unchecked) for clusters with 3 or less HAWQ Segments', + 'type': 'configuration', + 'config-name': 'output.replace-datanode-on-failure', + 'level': 'WARN' } problems = self.serviceAdvisor.validateHAWQHdfsClientConfigurations(properties, defaults, configurations, services, hosts) @@ -1171,14 +1240,14 @@ class TestHAWQ200ServiceAdvisor(TestCase): def test_validateHDFSSiteConfigurations(self): services = { - "services": [ - { "StackServices": {"service_name": "HAWQ"}, - "components": [{ - "StackServiceComponents": { - "component_name": "HAWQSEGMENT", - "hostnames": [] - }}] - }], + "services": [ + {"StackServices": {"service_name": "HAWQ"}, + "components": [{ + "StackServiceComponents": { + "component_name": "HAWQSEGMENT", + "hostnames": [] + }}] + }], "configurations": {"hdfs-site": {}, "core-site": {}} } @@ -1223,7 +1292,7 @@ class TestHAWQ200ServiceAdvisor(TestCase): self.assertEqual(problems[0], expected) # check all properties setup correctly in core-site - configurations["core-site"]["properties"] = {"ipc.server.listen.queue.size" : "3300"} + configurations["core-site"]["properties"] = {"ipc.server.listen.queue.size": "3300"} problems = self.serviceAdvisor.validateCORESiteConfigurations(configurations["core-site"]["properties"], defaults, configurations, services, hosts) self.assertEqual(len(problems), 0) @@ -1240,7 +1309,7 @@ class TestHAWQ200ServiceAdvisor(TestCase): 'config-name': 'ipc.server.listen.queue.size', 'level': 'WARN' } - configurations["core-site"]["properties"] = {"ipc.server.listen.queue.size" : "0"} + configurations["core-site"]["properties"] = {"ipc.server.listen.queue.size": "0"} problems = self.serviceAdvisor.validateCORESiteConfigurations(configurations["core-site"]["properties"], defaults, configurations, services, hosts) self.assertEqual(len(problems), 1) self.assertEqual(problems[0], expected)
