Repository: ambari Updated Branches: refs/heads/trunk 3b8da53d6 -> 5fc282e95
AMBARI-17023: Show only relevant properties based on the status of HAWQ Resource Manager (bhuvnesh2703) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/5fc282e9 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/5fc282e9 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/5fc282e9 Branch: refs/heads/trunk Commit: 5fc282e95f3e142afe41902b21dc8c98b7145a27 Parents: 3b8da53 Author: Bhuvnesh Chaudhary <[email protected]> Authored: Mon Jun 6 15:15:42 2016 -0700 Committer: Bhuvnesh Chaudhary <[email protected]> Committed: Mon Jun 6 15:15:42 2016 -0700 ---------------------------------------------------------------------- .../HAWQ/2.0.0/configuration/hawq-site.xml | 26 ++++- .../HAWQ/2.0.0/service_advisor.py | 111 ++++++++++--------- .../stacks/2.3/HAWQ/test_service_advisor.py | 43 ++++--- .../stacks/2.3/common/test_stack_advisor.py | 10 +- 4 files changed, 116 insertions(+), 74 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/5fc282e9/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml index 3ac4e89..9bdf62d 100644 --- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml +++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml @@ -180,7 +180,7 @@ <property> <name>hawq_rm_memory_limit_perseg</name> <display-name>Segment Memory Usage Limit</display-name> - <value>67108864KB</value> + <value>65535MB</value> <description> The maximum memory that can be used by a HAWQ segment when Resource Manager ('hawq_global_rm_type') is set to Standalone ('none'). Its value depends on vm.overcommit_memory, vm.overcommit_ratio and system memory on HAWQ hosts. @@ -194,6 +194,10 @@ <type>hawq-sysctl-env</type> <name>vm.overcommit_memory</name> </property> + <property> + <type>hawq-site</type> + <name>hawq_global_rm_type</name> + </property> </depends-on> <value-attributes> <type>string</type> @@ -237,6 +241,10 @@ <type>yarn-site</type> <name>yarn.resourcemanager.address</name> </property> + <property> + <type>hawq-site</type> + <name>hawq_global_rm_type</name> + </property> </depends-on> <value-attributes> <type>string</type> @@ -258,6 +266,10 @@ <type>yarn-site</type> <name>yarn.resourcemanager.scheduler.address</name> </property> + <property> + <type>hawq-site</type> + <name>hawq_global_rm_type</name> + </property> </depends-on> <value-attributes> <type>string</type> @@ -273,6 +285,12 @@ The name of the YARN queue to register with HAWQ's resource manager. The default is 'default'. </description> + <depends-on> + <property> + <type>hawq-site</type> + <name>hawq_global_rm_type</name> + </property> + </depends-on> <value-attributes> <type>string</type> </value-attributes> @@ -286,6 +304,12 @@ The name of the YARN application to register with HAWQ's resource manager. The default is 'hawq'. </description> + <depends-on> + <property> + <type>hawq-site</type> + <name>hawq_global_rm_type</name> + </property> + </depends-on> <value-attributes> <type>string</type> </value-attributes> http://git-wip-us.apache.org/repos/asf/ambari/blob/5fc282e9/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py index 28eb82f..2c78bf1 100644 --- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py +++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py @@ -144,54 +144,56 @@ class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor): minHawqHostsMemory = min([host['Hosts']['total_mem'] for host in hosts['items'] if host['Hosts']['host_name'] in hawqHosts]) minHawqHostsCoreCount = min([host['Hosts']['cpu_count'] for host in hosts['items'] if host['Hosts']['host_name'] in hawqHosts]) - if "hawq-site" in services["configurations"]: - hawq_site = services["configurations"]["hawq-site"]["properties"] - putHawqSiteProperty = self.putProperty(configurations, "hawq-site", services) - - # remove master port when master is colocated with Ambari server - if self.isHawqMasterComponentOnAmbariServer(stackAdvisor, services) and "hawq_master_address_port" in hawq_site: - putHawqSiteProperty('hawq_master_address_port', '') - - # update query limits if segments are deployed - if numSegments and "default_hash_table_bucket_number" in hawq_site and "hawq_rm_nvseg_perquery_limit" in hawq_site: - factor_min = 1 - factor_max = 6 - limit = int(hawq_site["hawq_rm_nvseg_perquery_limit"]) - factor = limit / numSegments - # if too many segments or default limit is too low --> stick with the limit - if factor < factor_min: - buckets = limit - # if the limit is large and results in factor > max --> limit factor to max - elif factor > factor_max: - buckets = factor_max * numSegments - else: - buckets = factor * numSegments - putHawqSiteProperty('default_hash_table_bucket_number', buckets) - - if "hawq_global_rm_type" in hawq_site and hawq_site["hawq_global_rm_type"] == "none" and "hawq_rm_nvcore_limit_perseg" in hawq_site: - putHawqSiteProperty('hawq_rm_nvcore_limit_perseg', minHawqHostsCoreCount) - - # update YARN RM urls with the values from yarn-site if YARN is installed - if "YARN" in servicesList and "yarn-site" in services["configurations"]: - yarn_site = services["configurations"]["yarn-site"]["properties"] - for hs_prop, ys_prop in self.getHAWQYARNPropertyMapping().items(): - if hs_prop in hawq_site and ys_prop in yarn_site: - putHawqSiteProperty(hs_prop, yarn_site[ys_prop]) - - if "hawq-sysctl-env" in services["configurations"] and "vm.overcommit_memory" in services["configurations"]["hawq-sysctl-env"]["properties"]: + hawq_site = services["configurations"]["hawq-site"]["properties"] + putHawqSiteProperty = self.putProperty(configurations, "hawq-site", services) + putHawqSitePropertyAttribute = self.putPropertyAttribute(configurations, "hawq-site") + hawq_sysctl_env = services["configurations"]["hawq-sysctl-env"]["properties"] + putHawqSysctlEnvProperty = self.putProperty(configurations, "hawq-sysctl-env", services) + + # remove master port when master is colocated with Ambari server + if self.isHawqMasterComponentOnAmbariServer(stackAdvisor, services) and "hawq_master_address_port" in hawq_site: + putHawqSiteProperty('hawq_master_address_port', '') + + # update query limits if segments are deployed + if numSegments and "default_hash_table_bucket_number" in hawq_site and "hawq_rm_nvseg_perquery_limit" in hawq_site: + factor_min = 1 + factor_max = 6 + limit = int(hawq_site["hawq_rm_nvseg_perquery_limit"]) + factor = limit / numSegments + # if too many segments or default limit is too low --> stick with the limit + if factor < factor_min: + buckets = limit + # if the limit is large and results in factor > max --> limit factor to max + elif factor > factor_max: + buckets = factor_max * numSegments + else: + buckets = factor * numSegments + putHawqSiteProperty('default_hash_table_bucket_number', buckets) + + + # update YARN RM urls with the values from yarn-site if YARN is installed + if "YARN" in servicesList and "yarn-site" in services["configurations"]: + yarn_site = services["configurations"]["yarn-site"]["properties"] + for hs_prop, ys_prop in self.getHAWQYARNPropertyMapping().items(): + if hs_prop in hawq_site and ys_prop in yarn_site: + putHawqSiteProperty(hs_prop, yarn_site[ys_prop]) + + putHawqSiteProperty('hawq_rm_nvcore_limit_perseg', minHawqHostsCoreCount) + + + if "vm.overcommit_memory" in hawq_sysctl_env: MEM_THRESHOLD = 33554432 # 32GB, minHawqHostsMemory is represented in kB # Set the value for hawq_rm_memory_limit_perseg based on vm.overcommit value and RAM available on HAWQ Hosts # If value of hawq_rm_memory_limit_perseg is 67108864KB, it indicates hawq is being added and recommendation # has not be made yet, since after recommendation it will be in GB in case its 67108864KB. - if "vm.overcommit_ratio" in services["configurations"]["hawq-sysctl-env"]["properties"]: - vm_overcommit_ratio = int(services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_ratio"]) - else: - vm_overcommit_ratio = 50 - if "hawq_rm_memory_limit_perseg" in services["configurations"]["hawq-site"]["properties"] and services["configurations"]["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"] == "67108864KB": + vm_overcommit_ratio = int(hawq_sysctl_env["vm.overcommit_ratio"]) if "vm.overcommit_ratio" in hawq_sysctl_env else 50 + if "hawq_rm_memory_limit_perseg" in hawq_site and hawq_site["hawq_rm_memory_limit_perseg"] == "65535MB": vm_overcommit_mem_value = 2 if minHawqHostsMemory >= MEM_THRESHOLD else 1 else: # set vm.overcommit_memory to 2 if the minimum memory among all hawqHosts is greater than 32GB - vm_overcommit_mem_value = int(services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"]) + vm_overcommit_mem_value = int(hawq_sysctl_env["vm.overcommit_memory"]) + putHawqSysctlEnvProperty("vm.overcommit_ratio", vm_overcommit_ratio) + putHawqSysctlEnvProperty("vm.overcommit_memory", vm_overcommit_mem_value) host_ram_kb = minHawqHostsMemory * vm_overcommit_ratio / 100 if vm_overcommit_mem_value == 2 else minHawqHostsMemory host_ram_gb = float(host_ram_kb) / (1024 * 1024) recommended_mem_percentage = { @@ -208,9 +210,19 @@ class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor): unit = "MB" # hawq_rm_memory_limit_perseg does not support decimal value so trim decimal using int putHawqSiteProperty("hawq_rm_memory_limit_perseg", "{0}{1}".format(int(recommended_mem), unit)) - putHawqSysctlEnvProperty = self.putProperty(configurations, "hawq-sysctl-env", services) - putHawqSysctlEnvProperty("vm.overcommit_ratio", vm_overcommit_ratio) - putHawqSysctlEnvProperty("vm.overcommit_memory", vm_overcommit_mem_value) + + # Show / Hide properties based on the value of hawq_global_rm_type + YARN_MODE = True if hawq_site["hawq_global_rm_type"].lower() == "yarn" else False + yarn_mode_properties_visibility = { + "hawq_rm_memory_limit_perseg": False, + "hawq_rm_nvcore_limit_perseg": False, + "hawq_rm_yarn_app_name": True, + "hawq_rm_yarn_queue_name": True, + "hawq_rm_yarn_scheduler_address": True, + "hawq_rm_yarn_address": True + } + for property, visible_status in yarn_mode_properties_visibility.iteritems(): + putHawqSitePropertyAttribute(property, "visible", str(visible_status if YARN_MODE else not visible_status).lower()) # set output.replace-datanode-on-failure in HAWQ hdfs-client depending on the cluster size if "hdfs-client" in services["configurations"]: @@ -338,14 +350,3 @@ class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor): validationItems.append({"config-name": PROP_NAME, "item": self.getWarnItem(message.format(PROP_NAME, str(MIN_NUM_SEGMENT_THRESHOLD)))}) return stackAdvisor.toConfigurationValidationProblems(validationItems, "hdfs-client") - - def putPropertyAttribute(self, config, configType): - if configType not in config: - config[configType] = {} - def appendPropertyAttribute(key, attribute, attributeValue): - if "property_attributes" not in config[configType]: - config[configType]["property_attributes"] = {} - if key not in config[configType]["property_attributes"]: - config[configType]["property_attributes"][key] = {} - config[configType]["property_attributes"][key][attribute] = attributeValue if isinstance(attributeValue, list) else str(attributeValue) - return appendPropertyAttribute http://git-wip-us.apache.org/repos/asf/ambari/blob/5fc282e9/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py b/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py index 5750938..384b8be 100644 --- a/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py +++ b/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py @@ -141,7 +141,7 @@ class TestHAWQ200ServiceAdvisor(TestCase): }, "hawq-site": { "properties": { - "hawq_rm_memory_limit_perseg": "67108864KB", + "hawq_rm_memory_limit_perseg": "65535MB", "hawq_rm_nvcore_limit_perseg": "16", "hawq_global_rm_type": "yarn" } @@ -220,20 +220,9 @@ class TestHAWQ200ServiceAdvisor(TestCase): # HAWQ Hosts Core Count: c6401.ambari.apache.org - 2, c6402.ambari.apache.org - 4, c6404.ambari.apache.org - 2 # hawq_global_rm_type: yarn # Non HAWQ Hosts Core Count: c6401.ambari.apache.org - 1 - # Do not recommend hawq_rm_nvcore_limit_perseg when rm type is yarn - self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts) - self.assertEquals(configurations["hawq-site"]["properties"]["hawq_rm_nvcore_limit_perseg"], "16") - - # Case 2: - # HAWQ Hosts Core Count: c6401.ambari.apache.org - 2, c6402.ambari.apache.org - 4, c6404.ambari.apache.org - 2 - # hawq_global_rm_type: none - # Non HAWQ Hosts Core Count: c6401.ambari.apache.org - 1 - # Recommend hawq_rm_nvcore_limit_perseg when rm type is none - configurations["hawq-site"]["properties"]["hawq_global_rm_type"] = "none" self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts) self.assertEquals(configurations["hawq-site"]["properties"]["hawq_rm_nvcore_limit_perseg"], "2") - ## Test if vm.overcommit_memory is set correctly # Case 1: All machines have total_mem above 32GB (total_mem >= 33554432) @@ -242,7 +231,7 @@ class TestHAWQ200ServiceAdvisor(TestCase): # Case 2: One machine has total_mem below 32GB hosts["items"][0]["Hosts"]["total_mem"] = 33554431 - services["configurations"]["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"] = "67108864KB" + services["configurations"]["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"] = "65535MB" self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts) self.assertEquals(configurations["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"], "1") @@ -264,7 +253,7 @@ class TestHAWQ200ServiceAdvisor(TestCase): hosts["items"][0]["Hosts"]["total_mem"] = 67108864 hosts["items"][1]["Hosts"]["total_mem"] = 77108864 hosts["items"][3]["Hosts"]["total_mem"] = 87108864 - services["configurations"]["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"] = "67108864KB" + services["configurations"]["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"] = "65535MB" self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts) self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"], "24GB") @@ -290,3 +279,29 @@ class TestHAWQ200ServiceAdvisor(TestCase): services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_ratio"] = 75 self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts) self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"], "730GB") + + ## Test if the properties are set to visible / invisible based on the value of hawq_global_rm_type + + # Case 1: When hawq_global_rm_type is yarn + services["configurations"]["hawq-site"]["properties"]["hawq_global_rm_type"] = "yarn" + properties_visible_status = {"hawq_rm_memory_limit_perseg": "false", + "hawq_rm_nvcore_limit_perseg": "false", + "hawq_rm_yarn_app_name":"true", + "hawq_rm_yarn_queue_name": "true", + "hawq_rm_yarn_scheduler_address": "true", + "hawq_rm_yarn_address": "true"} + self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts) + for property, status in properties_visible_status.iteritems(): + self.assertEqual(configurations["hawq-site"]["property_attributes"][property]["visible"], status) + + # Case 2: When hawq_global_rm_type is none + services["configurations"]["hawq-site"]["properties"]["hawq_global_rm_type"] = "none" + properties_visible_status = {"hawq_rm_memory_limit_perseg": "true", + "hawq_rm_nvcore_limit_perseg": "true", + "hawq_rm_yarn_app_name": "false", + "hawq_rm_yarn_queue_name": "false", + "hawq_rm_yarn_scheduler_address": "false", + "hawq_rm_yarn_address": "false"} + self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts) + for property, status in properties_visible_status.iteritems(): + self.assertEqual(configurations["hawq-site"]["property_attributes"][property]["visible"], status) http://git-wip-us.apache.org/repos/asf/ambari/blob/5fc282e9/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py index 424a386..caf121c 100644 --- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py @@ -2032,9 +2032,11 @@ class TestHDP23StackAdvisor(TestCase): services["configurations"]["hawq-site"] = {"properties": {"default_hash_table_bucket_number": "24", "hawq_rm_nvseg_perquery_limit": "512", "hawq_rm_yarn_address": "localhost:8032", - "hawq_rm_yarn_scheduler_address": "localhost:8030"}} + "hawq_rm_yarn_scheduler_address": "localhost:8030", + "hawq_global_rm_type": "none"}} services["configurations"]["hdfs-client"] = {"properties": {"output.replace-datanode-on-failure": "true"}} + services["configurations"]["hawq-sysctl-env"] = {"properties": {}} services["configurations"]["yarn-site"] = {"properties": {"yarn.resourcemanager.address": "host1:8050", "yarn.resourcemanager.scheduler.address": "host1:8030"}} @@ -2073,11 +2075,11 @@ class TestHDP23StackAdvisor(TestCase): # Test 5 - with no segments configurations = {} - services["configurations"]["hawq-site"] = {"properties":{'hawq-site': {'properties': {}}}} + services["configurations"]["hawq-site"] = {"properties":{"hawq_global_rm_type": "none"}} hawqSegmentComponent["hostnames"] = [] serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, clusterData, services, hosts) - self.assertEquals(configurations, {'hdfs-client': {'properties': {'output.replace-datanode-on-failure': 'false'}}, - 'hawq-site': {'properties': {}}, 'hdfs-site': {'properties': {'dfs.allow.truncate': 'true'}}}) + self.assertEquals(configurations["hdfs-client"]["properties"]["output.replace-datanode-on-failure"], "false") + self.assertTrue("default_hash_table_bucket_number" not in configurations["hawq-site"]) def test_validateHiveConfigurations(self): properties = {"hive_security_authorization": "None",
