AMBARI-7464. Make stack_advisor code more human readable - additional changes (srimanth)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6c74b724 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6c74b724 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6c74b724 Branch: refs/heads/branch-alerts-dev Commit: 6c74b724c815e7bed7d4ff8c36bad4fd3d4dda85 Parents: cb656120 Author: Srimanth Gunturi <[email protected]> Authored: Thu Sep 25 12:24:23 2014 -0700 Committer: Srimanth Gunturi <[email protected]> Committed: Thu Sep 25 13:16:10 2014 -0700 ---------------------------------------------------------------------- .../stacks/HDP/1.3.2/services/stack_advisor.py | 10 +-- .../stacks/HDP/2.0.6/services/stack_advisor.py | 10 +-- .../stacks/HDP/2.1/services/stack_advisor.py | 39 +++------ .../src/main/resources/stacks/stack_advisor.py | 90 ++++++++++++-------- 4 files changed, 74 insertions(+), 75 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/6c74b724/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/stack_advisor.py index ba42075..ca7c1a1 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/stack_advisor.py @@ -25,7 +25,7 @@ from stack_advisor import DefaultStackAdvisor class HDP132StackAdvisor(DefaultStackAdvisor): - def getLayoutValidationItems(self, services, hosts): + def getComponentLayoutValidations(self, services, hosts): """Returns array of Validation objects about issues with hostnames components assigned to""" items = [] @@ -73,7 +73,7 @@ class HDP132StackAdvisor(DefaultStackAdvisor): items.append( { "type": 'host-component', "level": 'ERROR', "message": 'Cardinality violation, cardinality={0}, hosts count={1}'.format(cardinality, str(componentHostsCount)), "component-name": str(componentName) } ) # Validating host-usage - usedHostsListList = [component["StackServiceComponents"]["hostnames"] for component in componentsList if not self.isNotValuable(component)] + usedHostsListList = [component["StackServiceComponents"]["hostnames"] for component in componentsList if not self.isComponentNotValuable(component)] usedHostsList = [item for sublist in usedHostsListList for item in sublist] nonUsedHostsList = [item for item in hostsList if item not in usedHostsList] for host in nonUsedHostsList: @@ -81,7 +81,7 @@ class HDP132StackAdvisor(DefaultStackAdvisor): return items - def getServiceConfiguratorDict(self): + def getServiceConfigurationRecommenderDict(self): return { } @@ -91,7 +91,7 @@ class HDP132StackAdvisor(DefaultStackAdvisor): config[configType]["properties"][key] = str(value) return appendProperty - def getClusterData(self, servicesList, hosts, components): + def getConfigurationClusterSummary(self, servicesList, hosts, components): hBaseInstalled = False if 'HBASE' in servicesList: @@ -255,7 +255,7 @@ class HDP132StackAdvisor(DefaultStackAdvisor): 'HBASE_MASTER': {"min": 1}, } - def selectionSchemes(self): + def getComponentLayoutSchemes(self): return { 'NAMENODE': {"else": 0}, 'SECONDARY_NAMENODE': {"else": 1}, http://git-wip-us.apache.org/repos/asf/ambari/blob/6c74b724/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py index 8853e13..4057392 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py @@ -25,7 +25,7 @@ from stack_advisor import DefaultStackAdvisor class HDP206StackAdvisor(DefaultStackAdvisor): - def getLayoutValidationItems(self, services, hosts): + def getComponentLayoutValidations(self, services, hosts): """Returns array of Validation objects about issues with hostnames components assigned to""" items = [] @@ -70,7 +70,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor): items.append({"type": 'host-component', "level": 'ERROR', "message": message, "component-name": componentName}) # Validating host-usage - usedHostsListList = [component["StackServiceComponents"]["hostnames"] for component in componentsList if not self.isNotValuable(component)] + usedHostsListList = [component["StackServiceComponents"]["hostnames"] for component in componentsList if not self.isComponentNotValuable(component)] usedHostsList = [item for sublist in usedHostsListList for item in sublist] nonUsedHostsList = [item for item in hostsList if item not in usedHostsList] for host in nonUsedHostsList: @@ -78,7 +78,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor): return items - def getServiceConfiguratorDict(self): + def getServiceConfigurationRecommenderDict(self): return { "YARN": self.recommendYARNConfigurations, "MAPREDUCE2": self.recommendMapReduce2Configurations @@ -106,7 +106,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor): putMapredProperty('mapreduce.reduce.java.opts', "-Xmx" + str(int(round(0.8 * clusterData['reduceMemory']))) + "m") putMapredProperty('mapreduce.task.io.sort.mb', min(int(round(0.4 * clusterData['mapMemory'])), 1024)) - def getClusterData(self, servicesList, hosts, components): + def getConfigurationClusterSummary(self, servicesList, hosts, components): hBaseInstalled = False if 'HBASE' in servicesList: @@ -290,7 +290,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor): 'HBASE_MASTER': {"min": 1}, } - def selectionSchemes(self): + def getComponentLayoutSchemes(self): return { 'NAMENODE': {"else": 0}, 'SECONDARY_NAMENODE': {"else": 1}, http://git-wip-us.apache.org/repos/asf/ambari/blob/6c74b724/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py index 621e74d..f45211f 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py @@ -19,16 +19,15 @@ limitations under the License. class HDP21StackAdvisor(HDP206StackAdvisor): - def recommendServiceConfigurations(self, service): - calculator = super(HDP21StackAdvisor, self).recommendServiceConfigurations(service) - if calculator is None: - return { - "OOZIE": self.recommendOozieConfigurations, - "HIVE": self.recommendHiveConfigurations, - "TEZ": self.recommendTezConfigurations - }.get(service, None) - else: - return calculator + def getServiceConfigurationRecommenderDict(self, service): + parentRecommendConfDict = super(HDP21StackAdvisor, self).getServiceConfigurationRecommenderDict() + childRecommendConfDict = { + "OOZIE": self.recommendOozieConfigurations, + "HIVE": self.recommendHiveConfigurations, + "TEZ": self.recommendTezConfigurations + } + parentRecommendConfDict.update(childRecommendConfDict) + return parentRecommendConfDict def recommendOozieConfigurations(self, configurations, clusterData): if "FALCON_SERVER" in clusterData["components"]: @@ -60,24 +59,8 @@ class HDP21StackAdvisor(HDP206StackAdvisor): def getNotValuableComponents(self): return ['JOURNALNODE', 'ZKFC', 'GANGLIA_MONITOR', 'APP_TIMELINE_SERVER'] - def selectionSchemes(self): - return { - 'NAMENODE': {"else": 0}, - 'SECONDARY_NAMENODE': {"else": 1}, - 'HBASE_MASTER': {6: 0, 31: 2, "else": 3}, - - 'HISTORYSERVER': {31: 1, "else": 2}, - 'RESOURCEMANAGER': {31: 1, "else": 2}, - - 'OOZIE_SERVER': {6: 1, 31: 2, "else": 3}, - - 'HIVE_SERVER': {6: 1, 31: 2, "else": 4}, - 'HIVE_METASTORE': {6: 1, 31: 2, "else": 4}, - 'WEBHCAT_SERVER': {6: 1, 31: 2, "else": 4}, - } - - def selectionScheme(self): - parentSchemes = super(HDP21StackAdvisor, self).selectionSchemes() + def getComponentLayoutSchemes(self): + parentSchemes = super(HDP21StackAdvisor, self).getComponentLayoutSchemes() childSchemes = { 'APP_TIMELINE_SERVER': {31: 1, "else": 2}, 'FALCON_SERVER': {6: 1, 31: 2, "else": 3} http://git-wip-us.apache.org/repos/asf/ambari/blob/6c74b724/ambari-server/src/main/resources/stacks/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/stack_advisor.py b/ambari-server/src/main/resources/stacks/stack_advisor.py index f2b5446..28dc932 100644 --- a/ambari-server/src/main/resources/stacks/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/stack_advisor.py @@ -300,6 +300,13 @@ class StackAdvisor(object): class DefaultStackAdvisor(StackAdvisor): + """ + Default stack advisor implementation. + + This implementation is used when a stack-version, or its hierarchy does not + have an advisor. Stack-versions can extend this class to provide their own + implement + """ def recommendComponentLayout(self, services, hosts): """Returns Services object with hostnames array populated for components""" @@ -309,7 +316,7 @@ class DefaultStackAdvisor(StackAdvisor): hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]] servicesList = [service["StackServices"]["service_name"] for service in services["services"]] - layoutRecommendations = self.provideLayout(services, hosts) + layoutRecommendations = self.createComponentLayoutRecommendations(services, hosts) recommendations = { "Versions": {"stack_name": stackName, "stack_version": stackVersion}, @@ -320,7 +327,7 @@ class DefaultStackAdvisor(StackAdvisor): return recommendations - def provideLayout(self, services, hosts): + def createComponentLayoutRecommendations(self, services, hosts): recommendations = { "blueprint": { @@ -340,19 +347,19 @@ class DefaultStackAdvisor(StackAdvisor): #extend 'hostsComponentsMap' with MASTER components for service in services["services"]: - masterComponents = [component for component in service["components"] if self.isMaster(component)] + masterComponents = [component for component in service["components"] if self.isMasterComponent(component)] for component in masterComponents: componentName = component["StackServiceComponents"]["component_name"] - if self.isAlreadyPopulated(component): + if self.isComponentHostsPopulated(component): hostsForComponent = component["StackServiceComponents"]["hostnames"] else: availableHosts = hostsList - if len(hostsList) > 1 and self.isNotPreferableOnAmbariServerHost(component): + if len(hostsList) > 1 and self.isComponentNotPreferableOnAmbariServerHost(component): availableHosts = [hostName for hostName in hostsList if not self.isLocalHost(hostName)] - if self.isMasterWithMultipleInstances(component): - hostsCount = self.defaultNoOfMasterHosts(component) + if self.isMasterComponentWithMultipleInstances(component): + hostsCount = self.getMinComponentCount(component) if hostsCount > 1: # get first 'hostsCount' available hosts if len(availableHosts) < hostsCount: hostsCount = len(availableHosts) @@ -369,17 +376,17 @@ class DefaultStackAdvisor(StackAdvisor): #extend 'hostsComponentsMap' with Slave and Client Components componentsListList = [service["components"] for service in services["services"]] componentsList = [item for sublist in componentsListList for item in sublist] - usedHostsListList = [component["StackServiceComponents"]["hostnames"] for component in componentsList if not self.isNotValuable(component)] + usedHostsListList = [component["StackServiceComponents"]["hostnames"] for component in componentsList if not self.isComponentNotValuable(component)] utilizedHosts = [item for sublist in usedHostsListList for item in sublist] freeHosts = [hostName for hostName in hostsList if hostName not in utilizedHosts] for service in services["services"]: slaveClientComponents = [component for component in service["components"] - if self.isSlave(component) or self.isClient(component)] + if self.isSlaveComponent(component) or self.isClientComponent(component)] for component in slaveClientComponents: componentName = component["StackServiceComponents"]["component_name"] - if self.isAlreadyPopulated(component): + if self.isComponentHostsPopulated(component): hostsForComponent = component["StackServiceComponents"]["hostnames"] elif component["StackServiceComponents"]["cardinality"] == "ALL": hostsForComponent = hostsList @@ -388,7 +395,7 @@ class DefaultStackAdvisor(StackAdvisor): hostsForComponent = hostsList[-1:] else: # len(freeHosts) >= 1 hostsForComponent = freeHosts - if self.isClient(component): + if self.isClientComponent(component): hostsForComponent = freeHosts[0:1] #extend 'hostsComponentsMap' with 'hostsForComponent' @@ -410,7 +417,7 @@ class DefaultStackAdvisor(StackAdvisor): return recommendations pass - def prepareValidationResponse(self, services, validationItems): + def createValidationResponse(self, services, validationItems): """Returns array of Validation objects about issues with hostnames components assigned to""" stackName = services["Versions"]["stack_name"] stackVersion = services["Versions"]["stack_version"] @@ -424,18 +431,18 @@ class DefaultStackAdvisor(StackAdvisor): def validateComponentLayout(self, services, hosts): """Returns array of Validation objects about issues with hostnames components assigned to""" - validationItems = self.getLayoutValidationItems(services, hosts) - return self.prepareValidationResponse(services, validationItems) + validationItems = self.getComponentLayoutValidations(services, hosts) + return self.createValidationResponse(services, validationItems) def validateConfigurations(self, services, hosts): """Returns array of Validation objects about issues with hostnames components assigned to""" validationItems = self.getConfigurationsValidationItems(services, hosts) - return self.prepareValidationResponse(services, validationItems) + return self.createValidationResponse(services, validationItems) - def getLayoutValidationItems(self, services, hosts): + def getComponentLayoutValidations(self, services, hosts): return [] - def getClusterData(self, servicesList, hosts, components): + def getConfigurationClusterSummary(self, servicesList, hosts, components): pass def getConfigurationsValidationItems(self, services, hosts): @@ -450,7 +457,7 @@ class DefaultStackAdvisor(StackAdvisor): for service in services["services"] for component in service["components"]] - clusterData = self.getClusterData(servicesList, hosts, components) + clusterSummary = self.getConfigurationClusterSummary(servicesList, hosts, components) recommendations = { "Versions": {"stack_name": stackName, "stack_version": stackVersion}, @@ -470,32 +477,32 @@ class DefaultStackAdvisor(StackAdvisor): configurations = recommendations["recommendations"]["blueprint"]["configurations"] for service in servicesList: - calculation = self.recommendServiceConfigurations(service) + calculation = self.getServiceConfigurationRecommender(service) if calculation is not None: - calculation(configurations, clusterData) + calculation(configurations, clusterSummary) return recommendations - def recommendServiceConfigurations(self, service): - return self.getServiceConfiguratorDict().get(service, None) + def getServiceConfigurationRecommender(self, service): + return self.getServiceConfigurationRecommenderDict().get(service, None) - def getServiceConfiguratorDict(self): + def getServiceConfigurationRecommenderDict(self): return {} # Recommendation helper methods - def isAlreadyPopulated(self, component): + def isComponentHostsPopulated(self, component): hostnames = self.getComponentAttribute(component, "hostnames") if hostnames is not None: return len(hostnames) > 0 return False - def isClient(self, component): + def isClientComponent(self, component): return self.getComponentAttribute(component, "component_category") == 'CLIENT' - def isSlave(self, component): + def isSlaveComponent(self, component): return self.getComponentAttribute(component, "component_category") == 'SLAVE' - def isMaster(self, component): + def isMasterComponent(self, component): return self.getComponentAttribute(component, "is_master") def getComponentAttribute(self, component, attribute): @@ -507,22 +514,22 @@ class DefaultStackAdvisor(StackAdvisor): def isLocalHost(self, hostName): return socket.getfqdn(hostName) == socket.getfqdn() - def isMasterWithMultipleInstances(self, component): + def isMasterComponentWithMultipleInstances(self, component): componentName = self.getComponentName(component) masters = self.getMastersWithMultipleInstances() return componentName in masters - def isNotValuable(self, component): + def isComponentNotValuable(self, component): componentName = self.getComponentName(component) service = self.getNotValuableComponents() return componentName in service - def defaultNoOfMasterHosts(self, component): + def getMinComponentCount(self, component): componentName = self.getComponentName(component) - return self.cardinality(componentName)["min"] + return self.getComponentCardinality(componentName)["min"] # Helper dictionaries - def cardinality(self, componentName): + def getComponentCardinality(self, componentName): return self.getCardinalitiesDict().get(componentName, {"min": 1, "max": 1}) def getHostForComponent(self, component, hostsList, hostsComponentsMap): @@ -531,7 +538,7 @@ class DefaultStackAdvisor(StackAdvisor): if len(hostsList) == 1: return hostsList[0] else: - scheme = self.defineSelectionScheme(componentName) + scheme = self.getComponentLayoutScheme(componentName) if scheme is not None: for key in scheme.keys(): if isinstance(key, ( int, long )): @@ -543,13 +550,16 @@ class DefaultStackAdvisor(StackAdvisor): hostOccupations = dict((host, len(components)) for host, components in hostComponentsMap.iteritems() if host in hostsList) return min(hostOccupations, key=hostOccupations.get) - def defineSelectionScheme(self, componentName): - return self.selectionSchemes().get(componentName, None) + def getComponentLayoutScheme(self, componentName): + """ + Provides a scheme for laying out given component on different number of hosts. + """ + return self.getComponentLayoutSchemes().get(componentName, None) def getComponentName(self, component): return self.getComponentAttribute(component, "component_name") - def isNotPreferableOnAmbariServerHost(self, component): + def isComponentNotPreferableOnAmbariServerHost(self, component): componentName = self.getComponentName(component) service = self.getNotPreferableOnServerComponents() return componentName in service @@ -566,5 +576,11 @@ class DefaultStackAdvisor(StackAdvisor): def getCardinalitiesDict(self): return {} - def selectionSchemes(self): + def getComponentLayoutSchemes(self): + """ + Provides layout scheme dictionaries for components. + + The scheme dictionary basically maps the number of hosts to + host index where component should exist. + """ return {}
