AMBARI-6885. stack_advisor.py from one stack-version contains another's logic
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/32caa435 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/32caa435 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/32caa435 Branch: refs/heads/branch-alerts-dev Commit: 32caa4359b7fd777fc7ca9b74e6d8a1212d6f6a0 Parents: eefa2fd Author: Srimanth Gunturi <sgunt...@hortonworks.com> Authored: Fri Aug 15 17:14:51 2014 -0700 Committer: Srimanth Gunturi <sgunt...@hortonworks.com> Committed: Mon Aug 18 10:36:04 2014 -0700 ---------------------------------------------------------------------- .../stacks/HDP/2.0.6/services/stack_advisor.py | 274 ++++++++----------- .../stacks/HDP/2.1/services/stack_advisor.py | 68 +++++ .../stacks/2.0.6/common/test_stack_advisor.py | 49 ++-- 3 files changed, 211 insertions(+), 180 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/32caa435/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py index 025caa2..77c57e2 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py @@ -41,15 +41,6 @@ class HDP206StackAdvisor(StackAdvisor): "services": servicesList, "recommendations": { "blueprint": { - "configurations": { - "global": { - "properties": { } - }, - "core-site": { }, - "hdfs-site": { }, - "yarn-site": { }, - "hbase-site": { } - }, "host_groups": [ ] }, "blueprint_cluster_binding": { @@ -71,7 +62,7 @@ class HDP206StackAdvisor(StackAdvisor): hostsForComponent = component["StackServiceComponents"]["hostnames"] else: availableHosts = hostsList - if len(hostsList) > 1 and isNotPreferableOnAmbariServerHost(component): + if len(hostsList) > 1 and self.isNotPreferableOnAmbariServerHost(component): availableHosts = [hostName for hostName in hostsList if not isLocalHost(hostName)] if isMasterWithMultipleInstances(component): @@ -81,9 +72,9 @@ class HDP206StackAdvisor(StackAdvisor): hostsCount = len(availableHosts) hostsForComponent = availableHosts[:hostsCount] else: - hostsForComponent = [getHostForComponent(component, availableHosts)] + hostsForComponent = [self.getHostForComponent(component, availableHosts)] else: - hostsForComponent = [getHostForComponent(component, availableHosts)] + hostsForComponent = [self.getHostForComponent(component, availableHosts)] #extend 'hostsComponentsMap' with 'hostsForComponent' for hostName in hostsForComponent: @@ -94,7 +85,7 @@ class HDP206StackAdvisor(StackAdvisor): #extend 'hostsComponentsMap' with Slave and Client Components componentsListList = [service["components"] for service in services["services"]] componentsList = [item for sublist in componentsListList for item in sublist] - usedHostsListList = [component["StackServiceComponents"]["hostnames"] for component in componentsList if not isNotValuable(component)] + usedHostsListList = [component["StackServiceComponents"]["hostnames"] for component in componentsList if not self.isNotValuable(component)] utilizedHosts = [item for sublist in usedHostsListList for item in sublist] freeHosts = [hostName for hostName in hostsList if hostName not in utilizedHosts] @@ -135,6 +126,46 @@ class HDP206StackAdvisor(StackAdvisor): return recommendations pass + def getHostForComponent(self, component, hostsList): + componentName = component["StackServiceComponents"]["component_name"] + scheme = self.defineSelectionScheme(componentName) + + if len(hostsList) == 1: + return hostsList[0] + else: + for key in scheme.keys(): + if isinstance(key, ( int, long )): + if len(hostsList) < key: + return hostsList[scheme[key]] + return hostsList[scheme['else']] + + def defineSelectionScheme(self, componentName): + scheme = self.selectionScheme(componentName) + if scheme is None: + scheme = {"else": 0} + return scheme + + def selectionScheme(self, componentName): + return { + 'NAMENODE': {"else": 0}, + 'SECONDARY_NAMENODE': {"else": 1}, + 'HBASE_MASTER': {6: 0, 31: 2, "else": 3}, + + 'HISTORYSERVER': {31: 1, "else": 2}, + 'RESOURCEMANAGER': {31: 1, "else": 2}, + + 'OOZIE_SERVER': {6: 1, 31: 2, "else": 3}, + + 'HIVE_SERVER': {6: 1, 31: 2, "else": 4}, + 'HIVE_METASTORE': {6: 1, 31: 2, "else": 4}, + 'WEBHCAT_SERVER': {6: 1, 31: 2, "else": 4}, + }.get(componentName, None) + + def isNotPreferableOnAmbariServerHost(self, component): + componentName = component["StackServiceComponents"]["component_name"] + service = ['GANGLIA_SERVER', 'NAGIOS_SERVER'] + return componentName in service + def validateComponentLayout(self, services, hosts): """Returns array of Validation objects about issues with hostnames components assigned to""" stackName = services["Versions"]["stack_name"] @@ -195,7 +226,7 @@ class HDP206StackAdvisor(StackAdvisor): items.append( { "type": 'host-component', "level": 'ERROR', "message": 'Cardinality violation, cardinality={0}, hosts count={1}'.format(cardinality, str(componentHostsCount)), "component-name": str(componentName) } ) # Validating host-usage - usedHostsListList = [component["StackServiceComponents"]["hostnames"] for component in componentsList if not isNotValuable(component)] + usedHostsListList = [component["StackServiceComponents"]["hostnames"] for component in componentsList if not self.isNotValuable(component)] usedHostsList = [item for sublist in usedHostsListList for item in sublist] nonUsedHostsList = [item for item in hostsList if item not in usedHostsList] for host in nonUsedHostsList: @@ -204,6 +235,11 @@ class HDP206StackAdvisor(StackAdvisor): return validations pass + def isNotValuable(self, component): + componentName = component["StackServiceComponents"]["component_name"] + service = ['JOURNALNODE', 'ZKFC', 'GANGLIA_MONITOR'] + return componentName in service + def recommendConfigurations(self, services, hosts): stackName = services["Versions"]["stack_name"] stackVersion = services["Versions"]["stack_version"] @@ -242,9 +278,7 @@ class HDP206StackAdvisor(StackAdvisor): def recommendServiceConfigurations(self, service): return { "YARN": self.recommendYARNConfigurations, - "MAPREDUCE2": self.recommendMapReduce2Configurations, - "HIVE": self.recommendHiveConfigurations, - "OOZIE": self.recommendOozieConfigurations + "MAPREDUCE2": self.recommendMapReduce2Configurations }.get(service, None) def putProperty(self, config, configType): @@ -259,15 +293,6 @@ class HDP206StackAdvisor(StackAdvisor): putYarnProperty('yarn.scheduler.minimum-allocation-mb', clusterData['ramPerContainer']) putYarnProperty('yarn.scheduler.maximum-allocation-mb', clusterData['containers'] * clusterData['ramPerContainer']) - def recommendHiveConfigurations(self, configurations, clusterData): - containerSize = clusterData['mapMemory'] if clusterData['mapMemory'] > 2048 else clusterData['reduceMemory'] - containerSize = min(clusterData['containers'] * clusterData['ramPerContainer'], containerSize) - putHiveProperty = self.putProperty(configurations, "hive-site") - putHiveProperty('hive.auto.convert.join.noconditionaltask.size', int(containerSize / 3) * 1048576) - putHiveProperty('hive.tez.java.opts', "-server -Xmx" + str(int(0.8 * containerSize)) - + "m -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseParallelGC") - putHiveProperty('hive.tez.container.size', containerSize) - def recommendMapReduce2Configurations(self, configurations, clusterData): putMapredProperty = self.putProperty(configurations, "mapred-site") putMapredProperty('yarn.app.mapreduce.am.resource.mb', clusterData['amMemory']) @@ -278,14 +303,6 @@ class HDP206StackAdvisor(StackAdvisor): putMapredProperty('mapreduce.reduce.java.opts', "-Xmx" + str(int(0.8 * clusterData['reduceMemory'])) + "m") putMapredProperty('mapreduce.task.io.sort.mb', int(min(0.4 * clusterData['mapMemory'], 1024))) - def recommendOozieConfigurations(self, configurations, clusterData): - if "FALCON_SERVER" in clusterData["components"]: - putMapredProperty = self.putProperty(configurations, "oozie-site") - putMapredProperty("oozie.services.ext", - "org.apache.oozie.service.JMSAccessorService," + - "org.apache.oozie.service.PartitionDependencyManagerService," + - "org.apache.oozie.service.HCatAccessorService") - def getClusterData(self, servicesList, hosts, components): hBaseInstalled = False @@ -380,26 +397,72 @@ class HDP206StackAdvisor(StackAdvisor): configurations = services["configurations"] for service in services["services"]: serviceName = service["StackServices"]["service_name"] - if serviceName == "MAPREDUCE2": - mapReduceErrors = validateMapReduce2Configurations(getSiteProperties(configurations, "mapred-site"), recommendedDefaults["mapred-site"]["properties"]) - items.extend(mapReduceErrors) - elif serviceName == "HIVE": - hiveErrors = validateHiveConfigurations(getSiteProperties(configurations, "hive-site"), recommendedDefaults["hive-site"]["properties"]) - items.extend(hiveErrors) - elif serviceName == "STORM": - oozieErrors = [] #validateStormConfigurations(getSiteProperties(configurations, "storm-site"), recommendedDefaults["storm-site"]["properties"]) - items.extend(oozieErrors) - elif serviceName == "TEZ": - tezErrors = validateTezConfigurations(getSiteProperties(configurations, "tez-site"), recommendedDefaults["tez-site"]["properties"]) - items.extend(tezErrors) - elif serviceName == "YARN": - yarnErrors = validateYARNConfigurations(getSiteProperties(configurations, "yarn-site"), recommendedDefaults["yarn-site"]["properties"]) - items.extend(yarnErrors) - else: - pass + validator = self.validateServiceConfigurations(serviceName) + if validator is not None: + siteName = validator[0] + method = validator[1] + resultItems = method(getSiteProperties(configurations, siteName), recommendedDefaults[siteName]["properties"]) + items.extend(resultItems) return validations pass + def validateServiceConfigurations(self, serviceName): + return { + "MAPREDUCE2": ["mapred-site", self.validateMapReduce2Configurations], + "YARN": ["yarn-site", self.validateYARNConfigurations] + }.get(serviceName, None) + + def toConfigurationValidationErrors(self, items, siteName): + result = [] + for item in items: + if item["message"] is not None: + error = { "type": 'configuration', "level": 'ERROR', "message": item["message"], "config-type": siteName, "config-name": item["config-name"] } + result.append(error) + return result + + def validatorLessThenDefaultValue(self, properties, recommendedDefaults, propertyName): + value = to_number(properties[propertyName]) + if value is None: + return "Value should be integer" + defaultValue = to_number(recommendedDefaults[propertyName]) + if defaultValue is None: + return None + if value < defaultValue: + return "Value is less than the recommended default of {0}".format(defaultValue) + return None + + def validateXmxValue(self, properties, recommendedDefaults, propertyName): + value = properties[propertyName] + defaultValue = recommendedDefaults[propertyName] + if defaultValue is None: + return "Config's default value can't be null or undefined" + if not checkXmxValueFormat(value): + return 'Invalid value format' + valueInt = formatXmxSizeToBytes(getXmxSize(value)) + defaultValueXmx = getXmxSize(defaultValue) + defaultValueInt = formatXmxSizeToBytes(defaultValueXmx) + if valueInt < defaultValueInt: + return "Value is less than the recommended default of -Xmx" + defaultValueXmx + return None + + def validateMapReduce2Configurations(self, properties, recommendedDefaults): + validationItems = [ {"config-name": 'mapreduce.map.java.opts', "message": self.validateXmxValue(properties, recommendedDefaults, 'mapreduce.map.java.opts')}, + {"config-name": 'mapreduce.reduce.java.opts', "message": self.validateXmxValue(properties, recommendedDefaults, 'mapreduce.reduce.java.opts')}, + {"config-name": 'mapreduce.task.io.sort.mb', "message": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'mapreduce.task.io.sort.mb')}, + {"config-name": 'mapreduce.map.memory.mb', "message": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'mapreduce.map.memory.mb')}, + {"config-name": 'mapreduce.reduce.memory.mb', "message": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'mapreduce.reduce.memory.mb')}, + {"config-name": 'yarn.app.mapreduce.am.resource.mb', "message": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.app.mapreduce.am.resource.mb')}, + {"config-name": 'yarn.app.mapreduce.am.command-opts', "message": self.validateXmxValue(properties, recommendedDefaults, 'yarn.app.mapreduce.am.command-opts')} ] + return self.toConfigurationValidationErrors(validationItems, "mapred-site") + + def validateYARNConfigurations(self, properties, recommendedDefaults): + validationItems = [ {"config-name": 'yarn.nodemanager.resource.memory-mb', "message": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.nodemanager.resource.memory-mb')}, + {"config-name": 'yarn.scheduler.minimum-allocation-mb', "message": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.scheduler.minimum-allocation-mb')}, + {"config-name": 'yarn.scheduler.maximum-allocation-mb', "message": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.scheduler.maximum-allocation-mb')} ] + return self.toConfigurationValidationErrors(validationItems, "yarn-site") + + +# Validation helper methods def getSiteProperties(configurations, siteName): if configurations[siteName] is None: return {} @@ -413,39 +476,6 @@ def to_number(s): except ValueError: return None -def toConfigurationValidationErrors(items, siteName): - result = [] - for item in items: - if item["message"] is not None: - error = { "type": 'configuration', "level": 'ERROR', "message": item["message"], "config-type": siteName, "config-name": item["config-name"] } - result.append(error) - return result - -def validatorLessThenDefaultValue(properties, recommendedDefaults, propertyName): - value = to_number(properties[propertyName]) - if value is None: - return "Value should be integer" - defaultValue = to_number(recommendedDefaults[propertyName]) - if defaultValue is None: - return None - if value < defaultValue: - return "Value is less than the recommended default of {0}".format(defaultValue) - return None - -def validateXmxValue(properties, recommendedDefaults, propertyName): - value = properties[propertyName] - defaultValue = recommendedDefaults[propertyName] - if defaultValue is None: - return "Config's default value can't be null or undefined" - if not checkXmxValueFormat(value): - return 'Invalid value format' - valueInt = formatXmxSizeToBytes(getXmxSize(value)) - defaultValueXmx = getXmxSize(defaultValue) - defaultValueInt = formatXmxSizeToBytes(defaultValueXmx) - if valueInt < defaultValueInt: - return "Value is less than the recommended default of -Xmx" + defaultValueXmx - return None - def checkXmxValueFormat(value): p = re.compile('-Xmx(\d+)(b|k|m|g|p|t|B|K|M|G|P|T)?') matches = p.findall(value) @@ -477,58 +507,8 @@ def formatXmxSizeToBytes(value): }[1] return to_number(value) * m -def validateMapReduce2Configurations(properties, recommendedDefaults): - validationItems = [ {"config-name": 'mapreduce.map.java.opts', "message": validateXmxValue(properties, recommendedDefaults, 'mapreduce.map.java.opts')}, - {"config-name": 'mapreduce.reduce.java.opts', "message": validateXmxValue(properties, recommendedDefaults, 'mapreduce.reduce.java.opts')}, - {"config-name": 'mapreduce.task.io.sort.mb', "message": validatorLessThenDefaultValue(properties, recommendedDefaults, 'mapreduce.task.io.sort.mb')}, - {"config-name": 'mapreduce.map.memory.mb', "message": validatorLessThenDefaultValue(properties, recommendedDefaults, 'mapreduce.map.memory.mb')}, - {"config-name": 'mapreduce.reduce.memory.mb', "message": validatorLessThenDefaultValue(properties, recommendedDefaults, 'mapreduce.reduce.memory.mb')}, - {"config-name": 'yarn.app.mapreduce.am.resource.mb', "message": validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.app.mapreduce.am.resource.mb')}, - {"config-name": 'yarn.app.mapreduce.am.command-opts', "message": validateXmxValue(properties, recommendedDefaults, 'yarn.app.mapreduce.am.command-opts')} ] - return toConfigurationValidationErrors(validationItems, "mapred-site") - -def validateHiveConfigurations(properties, recommendedDefaults): - validationItems = [ {"config-name": 'hive.tez.container.size', "message": validatorLessThenDefaultValue(properties, recommendedDefaults, 'hive.tez.container.size')}, - {"config-name": 'hive.tez.java.opts', "message": validateXmxValue(properties, recommendedDefaults, 'hive.tez.java.opts')}, - {"config-name": 'hive.auto.convert.join.noconditionaltask.size', "message": validatorLessThenDefaultValue(properties, recommendedDefaults, 'hive.auto.convert.join.noconditionaltask.size')} ] - return toConfigurationValidationErrors(validationItems, "hive-site") - -def validateStormConfigurations(properties, recommendedDefaults): - validationItems = [ {"config-name": 'drpc.childopts', "message": validateXmxValue(properties, recommendedDefaults, 'drpc.childopts')}, - {"config-name": 'ui.childopts', "message": validateXmxValue(properties, recommendedDefaults, 'ui.childopts')}, - {"config-name": 'logviewer.childopts', "message": validateXmxValue(properties, recommendedDefaults, 'logviewer.childopts')} ] - return toConfigurationValidationErrors(validationItems, "storm-site") - -def validateTezConfigurations(properties, recommendedDefaults): - validationItems = [ {"config-name": 'tez.am.resource.memory.mb', "message": validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.am.resource.memory.mb')}, - {"config-name": 'tez.am.java.opts', "message": validateXmxValue(properties, recommendedDefaults, 'tez.am.java.opts')} ] - return toConfigurationValidationErrors(validationItems, "tez-site") - -def validateYARNConfigurations(properties, recommendedDefaults): - validationItems = [ {"config-name": 'yarn.nodemanager.resource.memory-mb', "message": validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.nodemanager.resource.memory-mb')}, - {"config-name": 'yarn.scheduler.minimum-allocation-mb', "message": validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.scheduler.minimum-allocation-mb')}, - {"config-name": 'yarn.scheduler.maximum-allocation-mb', "message": validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.scheduler.maximum-allocation-mb')} ] - return toConfigurationValidationErrors(validationItems, "yarn-site") - -# Helper methods -def getHostForComponent(component, hostsList): - componentName = component["StackServiceComponents"]["component_name"] - scheme = selectionScheme(componentName) - - if len(hostsList) == 1: - return hostsList[0] - else: - for key in scheme.keys(): - if isinstance(key, ( int, long )): - if len(hostsList) < key: - return hostsList[scheme[key]] - return hostsList[scheme['else']] - -def isNotValuable(component): - componentName = component["StackServiceComponents"]["component_name"] - service = ['JOURNALNODE', 'ZKFC', 'APP_TIMELINE_SERVER', 'GANGLIA_MONITOR'] - return componentName in service +# Recommendation helper methods def isAlreadyPopulated(component): if component["StackServiceComponents"]["hostnames"] is not None: return len(component["StackServiceComponents"]["hostnames"]) > 0 @@ -550,11 +530,6 @@ def isMaster(component): def isLocalHost(hostName): return socket.getfqdn(hostName) == socket.getfqdn() -def isNotPreferableOnAmbariServerHost(component): - componentName = component["StackServiceComponents"]["component_name"] - service = ['STORM_UI_SERVER', 'DRPC_SERVER', 'STORM_REST_API', 'NIMBUS', 'GANGLIA_SERVER', 'NAGIOS_SERVER', 'HUE_SERVER'] - return componentName in service - def isMasterWithMultipleInstances(component): componentName = component["StackServiceComponents"]["component_name"] masters = ['ZOOKEEPER_SERVER', 'HBASE_MASTER'] @@ -572,22 +547,3 @@ def cardinality(componentName): 'HBASE_MASTER': {min: 1}, }.get(componentName, {min:1, max:1}) -def selectionScheme(componentName): - return { - 'NAMENODE': {"else": 0}, - 'SECONDARY_NAMENODE': {"else": 1}, - 'HBASE_MASTER': {6: 0, 31: 2, "else": 3}, - - 'JOBTRACKER': {31: 1, "else": 2}, - 'HISTORYSERVER': {31: 1, "else": 2}, - 'RESOURCEMANAGER': {31: 1, "else": 2}, - 'APP_TIMELINE_SERVER': {31: 1, "else": 2}, - - 'OOZIE_SERVER': {6: 1, 31: 2, "else": 3}, - 'FALCON_SERVER': {6: 1, 31: 2, "else": 3}, - - 'HIVE_SERVER': {6: 1, 31: 2, "else": 4}, - 'HIVE_METASTORE': {6: 1, 31: 2, "else": 4}, - 'WEBHCAT_SERVER': {6: 1, 31: 2, "else": 4}, - }.get(componentName, {"else": 0}) - http://git-wip-us.apache.org/repos/asf/ambari/blob/32caa435/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py index eb2faa1..5d7a3bc 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py @@ -27,11 +27,30 @@ class HDP21StackAdvisor(HDP206StackAdvisor): calculator = super(HDP21StackAdvisor, self).recommendServiceConfigurations(service) if calculator is None: return { + "OOZIE": self.recommendOozieConfigurations, + "HIVE": self.recommendHiveConfigurations, "TEZ": self.recommendTezConfigurations }.get(service, None) else: return calculator + def recommendOozieConfigurations(self, configurations, clusterData): + if "FALCON_SERVER" in clusterData["components"]: + putMapredProperty = self.putProperty(configurations, "oozie-site") + putMapredProperty("oozie.services.ext", + "org.apache.oozie.service.JMSAccessorService," + + "org.apache.oozie.service.PartitionDependencyManagerService," + + "org.apache.oozie.service.HCatAccessorService") + + def recommendHiveConfigurations(self, configurations, clusterData): + containerSize = clusterData['mapMemory'] if clusterData['mapMemory'] > 2048 else clusterData['reduceMemory'] + containerSize = min(clusterData['containers'] * clusterData['ramPerContainer'], containerSize) + putHiveProperty = self.putProperty(configurations, "hive-site") + putHiveProperty('hive.auto.convert.join.noconditionaltask.size', int(containerSize / 3) * 1048576) + putHiveProperty('hive.tez.java.opts', "-server -Xmx" + str(int(0.8 * containerSize)) + + "m -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseParallelGC") + putHiveProperty('hive.tez.container.size', containerSize) + def recommendTezConfigurations(self, configurations, clusterData): putTezProperty = self.putProperty(configurations, "tez-site") putTezProperty("tez.am.resource.memory.mb", clusterData['amMemory']) @@ -39,3 +58,52 @@ class HDP21StackAdvisor(HDP206StackAdvisor): "-server -Xmx" + str(int(0.8 * clusterData["amMemory"])) + "m -Djava.net.preferIPv4Stack=true -XX:+UseNUMA -XX:+UseParallelGC") + def isNotPreferableOnAmbariServerHost(self, component): + componentName = component["StackServiceComponents"]["component_name"] + service = ['STORM_UI_SERVER', 'DRPC_SERVER', 'STORM_REST_API', 'NIMBUS', 'GANGLIA_SERVER', 'NAGIOS_SERVER'] + return componentName in service + + def isNotValuable(self, component): + componentName = component["StackServiceComponents"]["component_name"] + service = ['JOURNALNODE', 'ZKFC', 'GANGLIA_MONITOR', 'APP_TIMELINE_SERVER'] + return componentName in service + + def selectionScheme(self, componentName): + scheme = super(HDP21StackAdvisor, self).selectionScheme(componentName) + if scheme is None: + return { + 'APP_TIMELINE_SERVER': {31: 1, "else": 2}, + 'FALCON_SERVER': {6: 1, 31: 2, "else": 3} + }.get(componentName, None) + else: + return scheme + + def validateServiceConfigurations(self, serviceName): + validator = super(HDP21StackAdvisor, self).validateServiceConfigurations(serviceName) + if validator is None: + return { + "STORM": ["storm-site", self.validateStormConfigurations], + "HIVE": ["hive-site", self.validateHiveConfigurations], + "TEZ": ["tez-site", self.validateTezConfigurations] + }.get(serviceName, None) + else: + return validator + + def validateHiveConfigurations(self, properties, recommendedDefaults): + validationItems = [ {"config-name": 'hive.tez.container.size', "message": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'hive.tez.container.size')}, + {"config-name": 'hive.tez.java.opts', "message": self.validateXmxValue(properties, recommendedDefaults, 'hive.tez.java.opts')}, + {"config-name": 'hive.auto.convert.join.noconditionaltask.size', "message": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'hive.auto.convert.join.noconditionaltask.size')} ] + return self.toConfigurationValidationErrors(validationItems, "hive-site") + + def validateStormConfigurations(self, properties, recommendedDefaults): + validationItems = [ {"config-name": 'drpc.childopts', "message": self.validateXmxValue(properties, recommendedDefaults, 'drpc.childopts')}, + {"config-name": 'ui.childopts', "message": self.validateXmxValue(properties, recommendedDefaults, 'ui.childopts')}, + {"config-name": 'logviewer.childopts', "message": self.validateXmxValue(properties, recommendedDefaults, 'logviewer.childopts')} ] + return self.toConfigurationValidationErrors(validationItems, "storm-site") + + def validateTezConfigurations(self, properties, recommendedDefaults): + validationItems = [ {"config-name": 'tez.am.resource.memory.mb', "message": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.am.resource.memory.mb')}, + {"config-name": 'tez.am.java.opts', "message": self.validateXmxValue(properties, recommendedDefaults, 'tez.am.java.opts')} ] + return self.toConfigurationValidationErrors(validationItems, "tez-site") + + http://git-wip-us.apache.org/repos/asf/ambari/blob/32caa435/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py index 3139a3e..b5bdba6 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py @@ -35,7 +35,7 @@ class TestHDP206StackAdvisor(TestCase): stack_advisor_impl = imp.load_module('stack_advisor_impl', fp, hdp206StackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE)) clazz = getattr(stack_advisor_impl, hdp206StackAdvisorClassName) self.stackAdvisor = clazz() - + def test_recommendationCardinalityALL(self): servicesInfo = [ { @@ -98,12 +98,12 @@ class TestHDP206StackAdvisor(TestCase): hosts = self.prepareHosts(["host1", "host2"]) result = self.stackAdvisor.validateComponentLayout(services, hosts) - expectedMessages = [ - "NameNode and Secondary NameNode cannot be hosted on same machine", - "NameNode and Secondary NameNode cannot be hosted on same machine", - "Host is not used" + expectedItems = [ + {"message": "NameNode and Secondary NameNode cannot be hosted on same machine", "host": "host1"}, + {"message": "NameNode and Secondary NameNode cannot be hosted on same machine", "host": "host1"}, + {"message": "Host is not used", "host": "host2"} ] - self.assertValidationMessages(expectedMessages, result) + self.assertValidationResult(expectedItems, result) def test_validationCardinalityALL(self): servicesInfo = [ @@ -119,10 +119,10 @@ class TestHDP206StackAdvisor(TestCase): hosts = self.prepareHosts(["host1", "host2"]) result = self.stackAdvisor.validateComponentLayout(services, hosts) - expectedMessages = [ - "Cardinality violation, cardinality=ALL, hosts count=1" + expectedItems = [ + {"message": "Cardinality violation, cardinality=ALL, hosts count=1"} ] - self.assertValidationMessages(expectedMessages, result) + self.assertValidationResult(expectedItems, result) def test_validationHostIsNotUsedForNonValuableComponent(self): servicesInfo = [ @@ -138,10 +138,10 @@ class TestHDP206StackAdvisor(TestCase): hosts = self.prepareHosts(["host1", "host2"]) result = self.stackAdvisor.validateComponentLayout(services, hosts) - expectedMessages = [ - "Host is not used" + expectedItems = [ + {"message": "Host is not used", "host": "host1"} ] - self.assertValidationMessages(expectedMessages, result) + self.assertValidationResult(expectedItems, result) def test_validationCardinality01TwoHostsAssigned(self): servicesInfo = [ @@ -156,10 +156,10 @@ class TestHDP206StackAdvisor(TestCase): hosts = self.prepareHosts(["host1", "host2"]) result = self.stackAdvisor.validateComponentLayout(services, hosts) - expectedMessages = [ - "Cardinality violation, cardinality=0-1, hosts count=2" + expectedItems = [ + {"message": "Cardinality violation, cardinality=0-1, hosts count=2"} ] - self.assertValidationMessages(expectedMessages, result) + self.assertValidationResult(expectedItems, result) def test_validationHostIsNotUsed(self): servicesInfo = [ @@ -174,10 +174,10 @@ class TestHDP206StackAdvisor(TestCase): hosts = self.prepareHosts(["host1", "host2"]) result = self.stackAdvisor.validateComponentLayout(services, hosts) - expectedMessages = [ - "Host is not used" + expectedItems = [ + {"message": "Host is not used", "host": "host2"} ] - self.assertValidationMessages(expectedMessages, result) + self.assertValidationResult(expectedItems, result) def prepareHosts(self, hostsNames): @@ -241,8 +241,15 @@ class TestHDP206StackAdvisor(TestCase): if not len(l1) == len(l2) or not sorted(l1) == sorted(l2): raise AssertionError("list1={0}, list2={1}".format(l1, l2)) - def assertValidationMessages(self, expectedMessages, result): - realMessages = [item["message"] for item in result["items"]] - self.checkEqual(expectedMessages, realMessages) + def assertValidationResult(self, expectedItems, result): + actualItems = [] + for item in result["items"]: + next = { "message": item["message"] } + try: + next["host"] = item["host"] + except KeyError, err: + pass + actualItems.append(next) + self.checkEqual(expectedItems, actualItems)