Repository: ambari Updated Branches: refs/heads/trunk 70e90fa3a -> 761000633
AMBARI-16970: Update derivation of hawq_rm_memory_limit_perseg (bhuvnesh2703) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/76100063 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/76100063 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/76100063 Branch: refs/heads/trunk Commit: 761000633ef42899b7f983ddca9a78ea52816759 Parents: 70e90fa Author: Bhuvnesh Chaudhary <[email protected]> Authored: Wed Jun 1 11:49:56 2016 -0700 Committer: Bhuvnesh Chaudhary <[email protected]> Committed: Wed Jun 1 11:49:56 2016 -0700 ---------------------------------------------------------------------- .../HAWQ/2.0.0/configuration/hawq-site.xml | 4 ++-- .../HAWQ/2.0.0/configuration/hawq-sysctl-env.xml | 3 +-- .../common-services/HAWQ/2.0.0/service_advisor.py | 12 +++++++----- .../test/python/stacks/2.3/HAWQ/test_service_advisor.py | 12 ++++-------- 4 files changed, 14 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/76100063/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml index f4bf191..ee71e6e 100644 --- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml +++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml @@ -180,10 +180,10 @@ <property> <name>hawq_rm_memory_limit_perseg</name> <display-name>Segment Memory Usage Limit</display-name> - <value>64GB</value> + <value>67108864KB</value> <description> The maximum memory that can be used by a HAWQ segment when Resource Manager ('hawq_global_rm_type') is set to Standalone ('none'). - The default is 64GB. + Its value depends on vm.overcommit_memory, vm.overcommit_ratio and system memory on HAWQ hosts. </description> <depends-on> <property> http://git-wip-us.apache.org/repos/asf/ambari/blob/76100063/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-sysctl-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-sysctl-env.xml b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-sysctl-env.xml index 84953ea..d054ff4 100644 --- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-sysctl-env.xml +++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-sysctl-env.xml @@ -347,7 +347,6 @@ 1: Always overcommit 2: Don't overcommit. For production environments, the value of 2 is recommended and means that processes will not be killed but will receive errors upon memory allocation. - The default value is 1. </description> <value-attributes> <type>int</type> @@ -361,7 +360,7 @@ <property> <name>vm.overcommit_ratio</name> <display-name>VM Overcommit Ratio</display-name> - <value/> + <value>50</value> <description> When overcommit_memory is set to 2, the committed address space is not permitted to exceed swap plus this percentage of physical RAM. Default is 50. http://git-wip-us.apache.org/repos/asf/ambari/blob/76100063/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py index a634062..94e1854 100644 --- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py +++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py @@ -174,18 +174,20 @@ class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor): if hs_prop in hawq_site and ys_prop in yarn_site: putHawqSiteProperty(hs_prop, yarn_site[ys_prop]) - # set vm.overcommit_memory to 2 if the minimum memory among all hawqHosts is greater than 32GB if "hawq-sysctl-env" in services["configurations"] and "vm.overcommit_memory" in services["configurations"]["hawq-sysctl-env"]["properties"]: MEM_THRESHOLD = 33554432 # 32GB, minHawqHostsMemory is represented in kB # Set the value for hawq_rm_memory_limit_perseg based on vm.overcommit value and RAM available on HAWQ Hosts - # Available RAM Formula = SWAP + RAM * vm.overcommit_ratio / 100. (SWAP not considered for recommendation here) - # Default value of vm.overcommit_ratio from configuration is '', so first time during install it will be blank, for subsequent calls it will have the recommended value. - if "vm.overcommit_ratio" in services["configurations"]["hawq-sysctl-env"]["properties"] and len(str(services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_ratio"])) > 0: + # If value of hawq_rm_memory_limit_perseg is 67108864KB, it indicates hawq is being added and recommendation + # has not be made yet, since after recommendation it will be in GB in case its 67108864KB. + if "vm.overcommit_ratio" in services["configurations"]["hawq-sysctl-env"]["properties"]: vm_overcommit_ratio = int(services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_ratio"]) - vm_overcommit_mem_value = int(services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"]) else: vm_overcommit_ratio = 50 + if "hawq_rm_memory_limit_perseg" in services["configurations"]["hawq-site"]["properties"] and services["configurations"]["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"] == "67108864KB": vm_overcommit_mem_value = 2 if minHawqHostsMemory >= MEM_THRESHOLD else 1 + else: + # set vm.overcommit_memory to 2 if the minimum memory among all hawqHosts is greater than 32GB + vm_overcommit_mem_value = int(services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"]) host_ram_kb = minHawqHostsMemory * vm_overcommit_ratio / 100 if vm_overcommit_mem_value == 2 else minHawqHostsMemory host_ram_gb = float(host_ram_kb) / (1024 * 1024) recommended_mem_percentage = { http://git-wip-us.apache.org/repos/asf/ambari/blob/76100063/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py b/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py index 7b64d04..6fbdf45 100644 --- a/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py +++ b/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py @@ -141,7 +141,7 @@ class TestHAWQ200ServiceAdvisor(TestCase): }, "hawq-site": { "properties": { - "hawq_rm_memory_limit_perseg": "64GB" + "hawq_rm_memory_limit_perseg": "67108864KB" } } } @@ -211,13 +211,12 @@ class TestHAWQ200ServiceAdvisor(TestCase): ## Test if vm.overcommit_memory is set correctly # Case 1: All machines have total_mem above 32GB (total_mem >= 33554432) - services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_ratio"] = '' self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts) self.assertEquals(configurations["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"], "2") # Case 2: One machine has total_mem below 32GB hosts["items"][0]["Hosts"]["total_mem"] = 33554431 - services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_ratio"] = '' + services["configurations"]["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"] = "67108864KB" self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts) self.assertEquals(configurations["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"], "1") @@ -238,8 +237,7 @@ class TestHAWQ200ServiceAdvisor(TestCase): hosts["items"][0]["Hosts"]["total_mem"] = 67108864 hosts["items"][1]["Hosts"]["total_mem"] = 77108864 hosts["items"][3]["Hosts"]["total_mem"] = 87108864 - services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_ratio"] = 50 - services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"] = 2 + services["configurations"]["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"] = "67108864KB" self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts) self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"], "24GB") @@ -257,13 +255,11 @@ class TestHAWQ200ServiceAdvisor(TestCase): self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts) self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"], "436GB") - # Case 6: Minimum host memory is ~ 1024 GB, vm.overcommit_ratio = 75, vm.overcommit_memory = 2 # recommended val must be .95% of (1024*75)/100 and in GB hosts["items"][0]["Hosts"]["total_mem"] = 1073741824 hosts["items"][1]["Hosts"]["total_mem"] = 2073741824 hosts["items"][3]["Hosts"]["total_mem"] = 3073741824 services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_ratio"] = 75 - services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"] = 2 self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts) - self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"], "730GB") \ No newline at end of file + self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"], "730GB")
