AMBARI-20085 Confusing AMS collector heap size validation loop (dsen)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/bb7b83f2 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/bb7b83f2 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/bb7b83f2 Branch: refs/heads/branch-feature-AMBARI-12556 Commit: bb7b83f2bb1b6af2dad38e98ffdcddda13e1cb2b Parents: 986e7a9 Author: Dmytro Sen <[email protected]> Authored: Thu Feb 23 15:52:02 2017 +0200 Committer: Dmytro Sen <[email protected]> Committed: Thu Feb 23 15:52:21 2017 +0200 ---------------------------------------------------------------------- .../stacks/HDP/2.0.6/services/stack_advisor.py | 1 + .../stacks/HDP/2.5/services/stack_advisor.py | 8 ++++++++ .../src/main/resources/stacks/stack_advisor.py | 12 ++++++------ .../python/stacks/2.5/common/test_stack_advisor.py | 16 ++++++++++++++++ .../src/test/resources/stacks/old_stack_advisor.py | 12 ++++++------ .../stacks/HDF/2.0/services/stack_advisor.py | 2 +- 6 files changed, 38 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/bb7b83f2/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py index 5c68f15..61960bb 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py @@ -1150,6 +1150,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor): if unusedMemory > 4*gb and collector_needs_increase: # warn user, if more than 4GB RAM is unused recommended_collector_heapsize = int((unusedMemory - 4*gb)/5) + collector_heapsize * mb + recommended_collector_heapsize = min(16*gb, recommended_collector_heapsize) #Make sure heapsize <= 16GB recommended_collector_heapsize = round_to_n(recommended_collector_heapsize/mb,128) # Round to 128m multiple if collector_heapsize < recommended_collector_heapsize: validation_msg = "Consider allocating {0} MB to metrics_collector_heapsize in ams-env to use up some " \ http://git-wip-us.apache.org/repos/asf/ambari/blob/bb7b83f2/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py index 52ada52..6f3dfa7 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py @@ -142,6 +142,14 @@ class HDP25StackAdvisor(HDP24StackAdvisor): return self.toConfigurationValidationProblems(validationItems, "storm-site") + def getCardinalitiesDict(self, hosts): + result = super(HDP25StackAdvisor, self).getCardinalitiesDict(hosts) + min_val = 1 + if len(hosts["items"]) > 999: + min_val = 2 + result['METRICS_COLLECTOR'] = {"min": min_val} + return result + def validateAtlasConfigurations(self, properties, recommendedDefaults, configurations, services, hosts): application_properties = self.getSiteProperties(configurations, "application-properties") validationItems = [] http://git-wip-us.apache.org/repos/asf/ambari/blob/bb7b83f2/ambari-server/src/main/resources/stacks/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/stack_advisor.py b/ambari-server/src/main/resources/stacks/stack_advisor.py index 04c6baf..0b81700 100644 --- a/ambari-server/src/main/resources/stacks/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/stack_advisor.py @@ -806,7 +806,7 @@ class DefaultStackAdvisor(StackAdvisor): return component["StackServiceComponents"]["hostnames"] if len(hostsList) > 1 and self.isMasterComponentWithMultipleInstances(component): - hostsCount = self.getMinComponentCount(component) + hostsCount = self.getMinComponentCount(component, hosts) if hostsCount > 1: # get first 'hostsCount' available hosts hostsForComponent = [] hostIndex = 0 @@ -1460,13 +1460,13 @@ class DefaultStackAdvisor(StackAdvisor): service = self.getNotValuableComponents() return componentName in service - def getMinComponentCount(self, component): + def getMinComponentCount(self, component, hosts): componentName = self.getComponentName(component) - return self.getComponentCardinality(componentName)["min"] + return self.getComponentCardinality(componentName, hosts)["min"] # Helper dictionaries - def getComponentCardinality(self, componentName): - dict = self.getCardinalitiesDict() + def getComponentCardinality(self, componentName, hosts): + dict = self.getCardinalitiesDict(hosts) if componentName in dict: return dict[componentName] else: @@ -1508,7 +1508,7 @@ class DefaultStackAdvisor(StackAdvisor): def getNotPreferableOnServerComponents(self): return self.notPreferableOnServerComponents - def getCardinalitiesDict(self): + def getCardinalitiesDict(self, hosts): return self.cardinalitiesDict def getComponentLayoutSchemes(self): http://git-wip-us.apache.org/repos/asf/ambari/blob/bb7b83f2/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py index 6890ef6..cf1c0ee 100644 --- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py @@ -404,6 +404,12 @@ class TestHDP25StackAdvisor(TestCase): data = json.load(f) return data + def prepareNHosts(self, host_count): + names = [] + for i in range(0, host_count): + names.append("hostname" + str(i)) + return self.prepareHosts(names) + def prepareHosts(self, hostsNames): hosts = { "items": [] } for hostName in hostsNames: @@ -435,6 +441,16 @@ class TestHDP25StackAdvisor(TestCase): def __getHosts(self, componentsList, componentName): return [component["StackServiceComponents"] for component in componentsList if component["StackServiceComponents"]["component_name"] == componentName][0] + def test_getCardinalitiesDict(self): + hosts = self.prepareNHosts(5) + actual = self.stackAdvisor.getCardinalitiesDict(hosts) + expected = {'METRICS_COLLECTOR': {'min': 1}} + self.assertEquals(actual, expected) + + hosts = self.prepareNHosts(1001) + actual = self.stackAdvisor.getCardinalitiesDict(hosts) + expected = {'METRICS_COLLECTOR': {'min': 2}} + self.assertEquals(actual, expected) def test_getComponentLayoutValidations_one_hsi_host(self): http://git-wip-us.apache.org/repos/asf/ambari/blob/bb7b83f2/ambari-server/src/test/resources/stacks/old_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/old_stack_advisor.py b/ambari-server/src/test/resources/stacks/old_stack_advisor.py index 8a880e0..ee5674b 100644 --- a/ambari-server/src/test/resources/stacks/old_stack_advisor.py +++ b/ambari-server/src/test/resources/stacks/old_stack_advisor.py @@ -359,7 +359,7 @@ class DefaultStackAdvisor(StackAdvisor): availableHosts = [hostName for hostName in hostsList if not self.isLocalHost(hostName)] if self.isMasterComponentWithMultipleInstances(component): - hostsCount = self.getMinComponentCount(component) + hostsCount = self.getMinComponentCount(component, hosts) if hostsCount > 1: # get first 'hostsCount' available hosts if len(availableHosts) < hostsCount: hostsCount = len(availableHosts) @@ -524,13 +524,13 @@ class DefaultStackAdvisor(StackAdvisor): service = self.getNotValuableComponents() return componentName in service - def getMinComponentCount(self, component): + def getMinComponentCount(self, component, hosts): componentName = self.getComponentName(component) - return self.getComponentCardinality(componentName)["min"] + return self.getComponentCardinality(componentName, hosts)["min"] # Helper dictionaries - def getComponentCardinality(self, componentName): - return self.getCardinalitiesDict().get(componentName, {"min": 1, "max": 1}) + def getComponentCardinality(self, componentName, hosts): + return self.getCardinalitiesDict(hosts).get(componentName, {"min": 1, "max": 1}) def getHostForComponent(self, component, hostsList): componentName = self.getComponentName(component) @@ -568,7 +568,7 @@ class DefaultStackAdvisor(StackAdvisor): def getNotPreferableOnServerComponents(self): return [] - def getCardinalitiesDict(self): + def getCardinalitiesDict(self, hosts): return {} def getComponentLayoutSchemes(self): http://git-wip-us.apache.org/repos/asf/ambari/blob/bb7b83f2/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/stack_advisor.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/stack_advisor.py index 40cc847..da33b95 100644 --- a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/stack_advisor.py +++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/stack_advisor.py @@ -1615,7 +1615,7 @@ class HDF20StackAdvisor(DefaultStackAdvisor): def getNotPreferableOnServerComponents(self): return ['STORM_UI_SERVER', 'DRPC_SERVER', 'STORM_REST_API', 'NIMBUS', 'METRICS_COLLECTOR'] - def getCardinalitiesDict(self): + def getCardinalitiesDict(self, hosts): return { 'ZOOKEEPER_SERVER': {"min": 3}, 'METRICS_COLLECTOR': {"min": 1}
