Repository: ambari Updated Branches: refs/heads/trunk 2c51b64c6 -> b4df6da51
AMBARI-16992: Recommend hawq_rm_nvcore_limit_perseg in HAWQ service advisor (mithmatt via bhuvnesh2703) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b4df6da5 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b4df6da5 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b4df6da5 Branch: refs/heads/trunk Commit: b4df6da51164b71543c764eb8e94cf7fa74da8ce Parents: 2c51b64 Author: Bhuvnesh Chaudhary <[email protected]> Authored: Wed Jun 1 15:36:39 2016 -0700 Committer: Bhuvnesh Chaudhary <[email protected]> Committed: Wed Jun 1 15:36:39 2016 -0700 ---------------------------------------------------------------------- .../HAWQ/2.0.0/service_advisor.py | 4 +++ .../stacks/2.3/HAWQ/test_service_advisor.py | 26 +++++++++++++++++++- .../stacks/2.3/common/test_stack_advisor.py | 12 ++++++--- 3 files changed, 37 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/b4df6da5/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py index 94e1854..32e23f9 100644 --- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py +++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py @@ -142,6 +142,7 @@ class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor): hawqHosts = hawqMasterHosts.union(hawqSegmentHosts) numSegments = len(hawqSegmentHosts) minHawqHostsMemory = min([host['Hosts']['total_mem'] for host in hosts['items'] if host['Hosts']['host_name'] in hawqHosts]) + minHawqHostsCoreCount = min([host['Hosts']['cpu_count'] for host in hosts['items'] if host['Hosts']['host_name'] in hawqHosts]) if "hawq-site" in services["configurations"]: hawq_site = services["configurations"]["hawq-site"]["properties"] @@ -167,6 +168,9 @@ class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor): buckets = factor * numSegments putHawqSiteProperty('default_hash_table_bucket_number', buckets) + if "hawq_rm_nvcore_limit_perseg" in hawq_site: + putHawqSiteProperty('hawq_rm_nvcore_limit_perseg', minHawqHostsCoreCount) + # update YARN RM urls with the values from yarn-site if YARN is installed if "YARN" in servicesList and "yarn-site" in services["configurations"]: yarn_site = services["configurations"]["yarn-site"]["properties"] http://git-wip-us.apache.org/repos/asf/ambari/blob/b4df6da5/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py b/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py index 6fbdf45..6531454 100644 --- a/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py +++ b/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py @@ -141,7 +141,8 @@ class TestHAWQ200ServiceAdvisor(TestCase): }, "hawq-site": { "properties": { - "hawq_rm_memory_limit_perseg": "67108864KB" + "hawq_rm_memory_limit_perseg": "67108864KB", + "hawq_rm_nvcore_limit_perseg": "16" } } } @@ -184,30 +185,52 @@ class TestHAWQ200ServiceAdvisor(TestCase): { "Hosts": { "host_name": "c6401.ambari.apache.org", + "cpu_count" : 2, "total_mem": 33554432 } }, { "Hosts": { "host_name": "c6402.ambari.apache.org", + "cpu_count" : 4, "total_mem": 33554433 } }, { "Hosts": { "host_name": "c6403.ambari.apache.org", + "cpu_count" : 1, "total_mem": 33554434 } }, { "Hosts": { "host_name": "c6404.ambari.apache.org", + "cpu_count" : 2, "total_mem": 33554435 } } ] } + ## Test if hawq_rm_nvcore_limit_perseg is set correctly + + # Case 1: + # HAWQ Hosts Core Count: c6401.ambari.apache.org - 2, c6402.ambari.apache.org - 4, c6404.ambari.apache.org - 2 + # Non HAWQ Hosts Core Count: c6401.ambari.apache.org - 1 + # Recommend hawq_rm_nvcore_limit_perseg as 2 + self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts) + self.assertEquals(configurations["hawq-site"]["properties"]["hawq_rm_nvcore_limit_perseg"], "2") + + # Case 2: + # HAWQ Hosts Core Count: c6401.ambari.apache.org - 2, c6402.ambari.apache.org - 2, c6404.ambari.apache.org - 2 + # Non HAWQ Hosts Core Count: c6401.ambari.apache.org - 1 + # Recommend hawq_rm_nvcore_limit_perseg as 2 + hosts["items"][1]["Hosts"]["cpu_count"] = 2 + self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts) + self.assertEquals(configurations["hawq-site"]["properties"]["hawq_rm_nvcore_limit_perseg"], "2") + + ## Test if vm.overcommit_memory is set correctly # Case 1: All machines have total_mem above 32GB (total_mem >= 33554432) @@ -221,6 +244,7 @@ class TestHAWQ200ServiceAdvisor(TestCase): self.assertEquals(configurations["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"], "1") ## Test if hawq_rm_memory_limit_perseg is set correctly + # Case 1: Minimum host memory is ~ 2 GB (2048MB), recommended val must be .75% of 2GB as vm.overcommit_memory = 1 and in MB hosts["items"][0]["Hosts"]["total_mem"] = 2097152 self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts) http://git-wip-us.apache.org/repos/asf/ambari/blob/b4df6da5/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py index b931dc6..424a386 100644 --- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py @@ -1994,25 +1994,29 @@ class TestHDP23StackAdvisor(TestCase): { "Hosts": { "host_name": "c6401.ambari.apache.org", - "total_mem": 12345678 + "cpu_count" : 2, + "total_mem": 33554432 } }, { "Hosts": { "host_name": "c6402.ambari.apache.org", - "total_mem": 12345678 + "cpu_count" : 4, + "total_mem": 33554433 } }, { "Hosts": { "host_name": "c6403.ambari.apache.org", - "total_mem": 12345678 + "cpu_count" : 1, + "total_mem": 33554434 } }, { "Hosts": { "host_name": "c6404.ambari.apache.org", - "total_mem": 12345678 + "cpu_count" : 2, + "total_mem": 33554435 } } ]
