Repository: ambari Updated Branches: refs/heads/branch-2.5 447fe8723 -> 4c96b84dc
AMBARI-20085 Confusing AMS collector heap size validation loop (dsen) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4c96b84d Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4c96b84d Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4c96b84d Branch: refs/heads/branch-2.5 Commit: 4c96b84dc8a82441723f3e192a0f2b78ea989044 Parents: 447fe87 Author: Dmytro Sen <[email protected]> Authored: Thu Feb 23 16:00:53 2017 +0200 Committer: Dmytro Sen <[email protected]> Committed: Thu Feb 23 16:00:53 2017 +0200 ---------------------------------------------------------------------- .../stacks/BIGTOP/0.8/services/stack_advisor.py | 2 +- .../stacks/HDP/2.0.6/services/stack_advisor.py | 3 ++- .../stacks/HDP/2.2/services/stack_advisor.py | 4 ++-- .../stacks/HDP/2.5/services/stack_advisor.py | 8 ++++++++ .../stacks/HDPWIN/2.1/services/stack_advisor.py | 2 +- .../stacks/HDPWIN/2.2/services/stack_advisor.py | 4 ++-- .../src/main/resources/stacks/stack_advisor.py | 12 ++++++------ .../python/stacks/2.5/common/test_stack_advisor.py | 16 ++++++++++++++++ .../src/test/resources/stacks/old_stack_advisor.py | 12 ++++++------ 9 files changed, 44 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/4c96b84d/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/stack_advisor.py index 701d0d4..1d18b44 100644 --- a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/stack_advisor.py @@ -284,7 +284,7 @@ class BaseBIGTOP08StackAdvisor(DefaultStackAdvisor): def getNotPreferableOnServerComponents(self): return ['GANGLIA_SERVER'] - def getCardinalitiesDict(self): + def getCardinalitiesDict(self, hosts): return { 'ZOOKEEPER_SERVER': {"min": 3}, 'HBASE_MASTER': {"min": 1}, http://git-wip-us.apache.org/repos/asf/ambari/blob/4c96b84d/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py index dad8106..5e04b53 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py @@ -1501,6 +1501,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor): if unusedMemory > 4*gb and collector_needs_increase: # warn user, if more than 4GB RAM is unused recommended_collector_heapsize = int((unusedMemory - 4*gb)/5) + collector_heapsize * mb + recommended_collector_heapsize = min(16*gb, recommended_collector_heapsize) #Make sure heapsize <= 16GB recommended_collector_heapsize = round_to_n(recommended_collector_heapsize/mb,128) # Round to 128m multiple if collector_heapsize < recommended_collector_heapsize: validation_msg = "Consider allocating {0} MB to metrics_collector_heapsize in ams-env to use up some " \ @@ -1856,7 +1857,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor): def getNotPreferableOnServerComponents(self): return ['GANGLIA_SERVER', 'METRICS_COLLECTOR'] - def getCardinalitiesDict(self): + def getCardinalitiesDict(self, hosts): return { 'ZOOKEEPER_SERVER': {"min": 3}, 'HBASE_MASTER': {"min": 1}, http://git-wip-us.apache.org/repos/asf/ambari/blob/4c96b84d/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py index f4583ba..d45cf96 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py @@ -1695,8 +1695,8 @@ class HDP22StackAdvisor(HDP21StackAdvisor): result.extend(['METRICS_MONITOR']) return result - def getCardinalitiesDict(self): - result = super(HDP22StackAdvisor, self).getCardinalitiesDict() + def getCardinalitiesDict(self, hosts): + result = super(HDP22StackAdvisor, self).getCardinalitiesDict(hosts) result['METRICS_COLLECTOR'] = {"min": 1} return result http://git-wip-us.apache.org/repos/asf/ambari/blob/4c96b84d/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py index 8d7aaf7..45fae53 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py @@ -142,6 +142,14 @@ class HDP25StackAdvisor(HDP24StackAdvisor): return self.toConfigurationValidationProblems(validationItems, "storm-site") + def getCardinalitiesDict(self, hosts): + result = super(HDP25StackAdvisor, self).getCardinalitiesDict(hosts) + min_val = 1 + if len(hosts["items"]) > 999: + min_val = 2 + result['METRICS_COLLECTOR'] = {"min": min_val} + return result + def validateAtlasConfigurations(self, properties, recommendedDefaults, configurations, services, hosts): application_properties = self.getSiteProperties(configurations, "application-properties") validationItems = [] http://git-wip-us.apache.org/repos/asf/ambari/blob/4c96b84d/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py index cf52ade..e1df728 100644 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py @@ -517,7 +517,7 @@ class HDPWIN21StackAdvisor(DefaultStackAdvisor): def getNotPreferableOnServerComponents(self): return ['STORM_UI_SERVER', 'DRPC_SERVER', 'STORM_REST_API', 'NIMBUS'] - def getCardinalitiesDict(self): + def getCardinalitiesDict(self, hosts): return { 'ZOOKEEPER_SERVER': {"min": 3}, 'HBASE_MASTER': {"min": 1}, http://git-wip-us.apache.org/repos/asf/ambari/blob/4c96b84d/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py index 88cc6bb..833edf1 100644 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py @@ -1092,8 +1092,8 @@ class HDPWIN22StackAdvisor(HDPWIN21StackAdvisor): result.extend(['METRICS_COLLECTOR']) return result - def getCardinalitiesDict(self): - result = super(HDPWIN22StackAdvisor, self).getCardinalitiesDict() + def getCardinalitiesDict(self, hosts): + result = super(HDPWIN22StackAdvisor, self).getCardinalitiesDict(hosts) result['METRICS_COLLECTOR'] = {"min": 1} return result http://git-wip-us.apache.org/repos/asf/ambari/blob/4c96b84d/ambari-server/src/main/resources/stacks/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/stack_advisor.py b/ambari-server/src/main/resources/stacks/stack_advisor.py index 2dce53a..e393b9c 100644 --- a/ambari-server/src/main/resources/stacks/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/stack_advisor.py @@ -478,7 +478,7 @@ class DefaultStackAdvisor(StackAdvisor): return component["StackServiceComponents"]["hostnames"] if len(hostsList) > 1 and self.isMasterComponentWithMultipleInstances(component): - hostsCount = self.getMinComponentCount(component) + hostsCount = self.getMinComponentCount(component, hosts) if hostsCount > 1: # get first 'hostsCount' available hosts hostsForComponent = [] hostIndex = 0 @@ -823,13 +823,13 @@ class DefaultStackAdvisor(StackAdvisor): service = self.getNotValuableComponents() return componentName in service - def getMinComponentCount(self, component): + def getMinComponentCount(self, component, hosts): componentName = self.getComponentName(component) - return self.getComponentCardinality(componentName)["min"] + return self.getComponentCardinality(componentName, hosts)["min"] # Helper dictionaries - def getComponentCardinality(self, componentName): - return self.getCardinalitiesDict().get(componentName, {"min": 1, "max": 1}) + def getComponentCardinality(self, componentName, hosts): + return self.getCardinalitiesDict(hosts).get(componentName, {"min": 1, "max": 1}) def getHostForComponent(self, component, hostsList): if len(hostsList) == 0: @@ -863,7 +863,7 @@ class DefaultStackAdvisor(StackAdvisor): def getNotPreferableOnServerComponents(self): return [] - def getCardinalitiesDict(self): + def getCardinalitiesDict(self, hosts): return {} def getComponentLayoutSchemes(self): http://git-wip-us.apache.org/repos/asf/ambari/blob/4c96b84d/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py index 6890ef6..07d3941 100644 --- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py @@ -404,6 +404,12 @@ class TestHDP25StackAdvisor(TestCase): data = json.load(f) return data + def prepareNHosts(self, host_count): + names = [] + for i in range(0, host_count): + names.append("hostname" + str(i)) + return self.prepareHosts(names) + def prepareHosts(self, hostsNames): hosts = { "items": [] } for hostName in hostsNames: @@ -435,6 +441,16 @@ class TestHDP25StackAdvisor(TestCase): def __getHosts(self, componentsList, componentName): return [component["StackServiceComponents"] for component in componentsList if component["StackServiceComponents"]["component_name"] == componentName][0] + def test_getCardinalitiesDict(self): + hosts = self.prepareNHosts(5) + actual = self.stackAdvisor.getCardinalitiesDict(hosts) + expected = {'ZOOKEEPER_SERVER': {'min': 3}, 'HBASE_MASTER': {'min': 1}, 'METRICS_COLLECTOR': {'min': 1}} + self.assertEquals(actual, expected) + + hosts = self.prepareNHosts(1001) + actual = self.stackAdvisor.getCardinalitiesDict(hosts) + expected = {'ZOOKEEPER_SERVER': {'min': 3}, 'HBASE_MASTER': {'min': 1}, 'METRICS_COLLECTOR': {'min': 2}} + self.assertEquals(actual, expected) def test_getComponentLayoutValidations_one_hsi_host(self): http://git-wip-us.apache.org/repos/asf/ambari/blob/4c96b84d/ambari-server/src/test/resources/stacks/old_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/old_stack_advisor.py b/ambari-server/src/test/resources/stacks/old_stack_advisor.py index 8a880e0..ee5674b 100644 --- a/ambari-server/src/test/resources/stacks/old_stack_advisor.py +++ b/ambari-server/src/test/resources/stacks/old_stack_advisor.py @@ -359,7 +359,7 @@ class DefaultStackAdvisor(StackAdvisor): availableHosts = [hostName for hostName in hostsList if not self.isLocalHost(hostName)] if self.isMasterComponentWithMultipleInstances(component): - hostsCount = self.getMinComponentCount(component) + hostsCount = self.getMinComponentCount(component, hosts) if hostsCount > 1: # get first 'hostsCount' available hosts if len(availableHosts) < hostsCount: hostsCount = len(availableHosts) @@ -524,13 +524,13 @@ class DefaultStackAdvisor(StackAdvisor): service = self.getNotValuableComponents() return componentName in service - def getMinComponentCount(self, component): + def getMinComponentCount(self, component, hosts): componentName = self.getComponentName(component) - return self.getComponentCardinality(componentName)["min"] + return self.getComponentCardinality(componentName, hosts)["min"] # Helper dictionaries - def getComponentCardinality(self, componentName): - return self.getCardinalitiesDict().get(componentName, {"min": 1, "max": 1}) + def getComponentCardinality(self, componentName, hosts): + return self.getCardinalitiesDict(hosts).get(componentName, {"min": 1, "max": 1}) def getHostForComponent(self, component, hostsList): componentName = self.getComponentName(component) @@ -568,7 +568,7 @@ class DefaultStackAdvisor(StackAdvisor): def getNotPreferableOnServerComponents(self): return [] - def getCardinalitiesDict(self): + def getCardinalitiesDict(self, hosts): return {} def getComponentLayoutSchemes(self):
