Repository: ambari Updated Branches: refs/heads/branch-2.4 13bf0cf4b -> bccd53646
AMBARI-16827: expose vm.overcommit_ratio on hawq-sysctl-env (bhuvnesh2703) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/bccd5364 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/bccd5364 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/bccd5364 Branch: refs/heads/branch-2.4 Commit: bccd53646950a470ece1a83375ca4fc95ebc9134 Parents: 13bf0cf Author: Bhuvnesh Chaudhary <[email protected]> Authored: Thu May 26 15:56:19 2016 -0700 Committer: Bhuvnesh Chaudhary <[email protected]> Committed: Thu May 26 15:56:19 2016 -0700 ---------------------------------------------------------------------- .../HAWQ/2.0.0/configuration/hawq-site.xml | 10 ++++++ .../2.0.0/configuration/hawq-sysctl-env.xml | 38 +++++++++++--------- .../HAWQ/2.0.0/service_advisor.py | 33 ++++++++++++----- .../HAWQ/2.0.0/themes/theme.json | 18 +++++++--- .../stacks/2.3/HAWQ/test_service_advisor.py | 20 +++++++++-- 5 files changed, 87 insertions(+), 32 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/bccd5364/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml index 5be3848..b096702 100644 --- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml +++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml @@ -176,6 +176,16 @@ The maximum memory that can be used by a HAWQ segment when Resource Manager ('hawq_global_rm_type') is set to Standalone ('none'). The default is 64GB. </description> + <depends-on> + <property> + <type>hawq-sysctl-env</type> + <name>vm.overcommit_ratio</name> + </property> + <property> + <type>hawq-sysctl-env</type> + <name>vm.overcommit_memory</name> + </property> + </depends-on> <value-attributes> <type>string</type> </value-attributes> http://git-wip-us.apache.org/repos/asf/ambari/blob/bccd5364/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-sysctl-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-sysctl-env.xml b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-sysctl-env.xml index c51b012..2ec145f 100644 --- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-sysctl-env.xml +++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-sysctl-env.xml @@ -320,27 +320,31 @@ <value>1</value> <description> Establishes the conditions for determining whether large memory requests are accepted or denied. - The three possible values for this parameter are: 0, 1 or 2. + The three possible values for this parameter are: + 0: Heuristic overcommit handling + 1: Always overcommit + 2: Don't overcommit. For production environments, the value of 2 is recommended and means that processes will not be killed but will receive errors upon memory allocation. The default value is 1. </description> <value-attributes> - <type>value-list</type> - <entries> - <entry> - <value>0</value> - <description>Heuristic overcommit handling</description> - </entry> - <entry> - <value>1</value> - <description>Always overcommit</description> - </entry> - <entry> - <value>2</value> - <description>Don't overcommit</description> - </entry> - </entries> - <selection-cardinality>1</selection-cardinality> + <type>int</type> + <minimum>0</minimum> + <maximum>2</maximum> + <increment-step>1</increment-step> + </value-attributes> + </property> + + <property> + <name>vm.overcommit_ratio</name> + <display-name>VM Overcommit Ratio</display-name> + <value></value> + <description> + When overcommit_memory is set to 2, the committed address space is not permitted to exceed swap plus this percentage of physical RAM. + Default is 50. + </description> + <value-attributes> + <type>int</type> </value-attributes> </property> http://git-wip-us.apache.org/repos/asf/ambari/blob/bccd5364/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py index 836144c..a634062 100644 --- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py +++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py @@ -177,15 +177,16 @@ class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor): # set vm.overcommit_memory to 2 if the minimum memory among all hawqHosts is greater than 32GB if "hawq-sysctl-env" in services["configurations"] and "vm.overcommit_memory" in services["configurations"]["hawq-sysctl-env"]["properties"]: MEM_THRESHOLD = 33554432 # 32GB, minHawqHostsMemory is represented in kB - vm_overcommit_mem_value = "2" if minHawqHostsMemory >= MEM_THRESHOLD else "1" - putHawqSysctlEnvProperty = self.putProperty(configurations, "hawq-sysctl-env", services) - putHawqSysctlEnvProperty("vm.overcommit_memory", vm_overcommit_mem_value) - # Set the value for hawq_rm_memory_limit_perseg based on vm.overcommit value and RAM available on HAWQ Hosts - # HAWQ Hosts with the minimum amount of RAM is considered for calculations - # Available RAM Formula = SWAP + RAM * vm.overcommit_ratio / 100 - # Assumption: vm.overcommit_ratio = 50 (default on Linux), SWAP not considered for recommendation - host_ram_kb = minHawqHostsMemory / 2 if vm_overcommit_mem_value == "2" else minHawqHostsMemory + # Available RAM Formula = SWAP + RAM * vm.overcommit_ratio / 100. (SWAP not considered for recommendation here) + # Default value of vm.overcommit_ratio from configuration is '', so first time during install it will be blank, for subsequent calls it will have the recommended value. + if "vm.overcommit_ratio" in services["configurations"]["hawq-sysctl-env"]["properties"] and len(str(services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_ratio"])) > 0: + vm_overcommit_ratio = int(services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_ratio"]) + vm_overcommit_mem_value = int(services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"]) + else: + vm_overcommit_ratio = 50 + vm_overcommit_mem_value = 2 if minHawqHostsMemory >= MEM_THRESHOLD else 1 + host_ram_kb = minHawqHostsMemory * vm_overcommit_ratio / 100 if vm_overcommit_mem_value == 2 else minHawqHostsMemory host_ram_gb = float(host_ram_kb) / (1024 * 1024) recommended_mem_percentage = { host_ram_gb <= 64: .75, @@ -201,6 +202,9 @@ class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor): unit = "MB" # hawq_rm_memory_limit_perseg does not support decimal value so trim decimal using int putHawqSiteProperty("hawq_rm_memory_limit_perseg", "{0}{1}".format(int(recommended_mem), unit)) + putHawqSysctlEnvProperty = self.putProperty(configurations, "hawq-sysctl-env", services) + putHawqSysctlEnvProperty("vm.overcommit_ratio", vm_overcommit_ratio) + putHawqSysctlEnvProperty("vm.overcommit_memory", vm_overcommit_mem_value) # set output.replace-datanode-on-failure in HAWQ hdfs-client depending on the cluster size if "hdfs-client" in services["configurations"]: @@ -327,4 +331,15 @@ class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor): if message: validationItems.append({"config-name": PROP_NAME, "item": self.getWarnItem(message.format(PROP_NAME, str(MIN_NUM_SEGMENT_THRESHOLD)))}) - return stackAdvisor.toConfigurationValidationProblems(validationItems, "hdfs-client") \ No newline at end of file + return stackAdvisor.toConfigurationValidationProblems(validationItems, "hdfs-client") + + def putPropertyAttribute(self, config, configType): + if configType not in config: + config[configType] = {} + def appendPropertyAttribute(key, attribute, attributeValue): + if "property_attributes" not in config[configType]: + config[configType]["property_attributes"] = {} + if key not in config[configType]["property_attributes"]: + config[configType]["property_attributes"][key] = {} + config[configType]["property_attributes"][key][attribute] = attributeValue if isinstance(attributeValue, list) else str(attributeValue) + return appendPropertyAttribute http://git-wip-us.apache.org/repos/asf/ambari/blob/bccd5364/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/themes/theme.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/themes/theme.json b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/themes/theme.json index e4d1c45..d7940a1 100644 --- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/themes/theme.json +++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/themes/theme.json @@ -122,13 +122,18 @@ }, { + "config":"hawq-sysctl-env/vm.overcommit_ratio", + "subsection-name":"resource-management-col1" + }, + + { "config":"hawq-site/hawq_rm_memory_limit_perseg", "subsection-name":"resource-management-col1" }, { "config":"hawq-site/hawq_rm_nvcore_limit_perseg", - "subsection-name":"resource-management-col1" + "subsection-name":"resource-management-col2" } ] }, @@ -191,15 +196,20 @@ } }, { - "config":"hawq-sysctl-env/vm.overcommit_memory", + "config":"hawq-sysctl-env/vm.overcommit_ratio", "widget":{ - "type":"combo" + "type":"text-field" } }, { "config":"hawq-sysctl-env/vm.overcommit_memory", "widget":{ - "type":"combo" + "type":"slider", + "units": [ + { + "unit-name":"int" + } + ] } }, { http://git-wip-us.apache.org/repos/asf/ambari/blob/bccd5364/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py b/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py index 53c5574..1d6a85c 100644 --- a/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py +++ b/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py @@ -131,7 +131,8 @@ class TestHAWQ200ServiceAdvisor(TestCase): configurations = { "hawq-sysctl-env": { "properties": { - "vm.overcommit_memory": "1" + "vm.overcommit_memory": 1, + "vm.overcommit_ratio": 50 } }, "hawq-site": { @@ -206,11 +207,13 @@ class TestHAWQ200ServiceAdvisor(TestCase): ## Test if vm.overcommit_memory is set correctly # Case 1: All machines have total_mem above 32GB (total_mem >= 33554432) + services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_ratio"] = '' self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts) self.assertEquals(configurations["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"], "2") # Case 2: One machine has total_mem below 32GB hosts["items"][0]["Hosts"]["total_mem"] = 33554431 + services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_ratio"] = '' self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts) self.assertEquals(configurations["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"], "1") @@ -231,6 +234,8 @@ class TestHAWQ200ServiceAdvisor(TestCase): hosts["items"][0]["Hosts"]["total_mem"] = 67108864 hosts["items"][1]["Hosts"]["total_mem"] = 77108864 hosts["items"][3]["Hosts"]["total_mem"] = 87108864 + services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_ratio"] = 50 + services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"] = 2 self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts) self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"], "24GB") @@ -246,4 +251,15 @@ class TestHAWQ200ServiceAdvisor(TestCase): hosts["items"][1]["Hosts"]["total_mem"] = 2073741824 hosts["items"][3]["Hosts"]["total_mem"] = 3073741824 self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts) - self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"], "436GB") \ No newline at end of file + self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"], "436GB") + + + # Case 6: Minimum host memory is ~ 1024 GB, vm.overcommit_ratio = 75, vm.overcommit_memory = 2 + # recommended val must be .95% of (1024*75)/100 and in GB + hosts["items"][0]["Hosts"]["total_mem"] = 1073741824 + hosts["items"][1]["Hosts"]["total_mem"] = 2073741824 + hosts["items"][3]["Hosts"]["total_mem"] = 3073741824 + services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_ratio"] = 75 + services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"] = 2 + self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts) + self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"], "730GB") \ No newline at end of file
