Repository: ambari Updated Branches: refs/heads/trunk 6e9b4be25 -> 8258cf893
AMBARI-19096. HDP 3.0 TP - create Service Advisor for ZK (alejandro) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8258cf89 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8258cf89 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8258cf89 Branch: refs/heads/trunk Commit: 8258cf893175342142523ec0001b9f4d3f09ce3f Parents: 6e9b4be Author: Alejandro Fernandez <afernan...@hortonworks.com> Authored: Mon Dec 19 14:59:40 2016 -0800 Committer: Alejandro Fernandez <afernan...@hortonworks.com> Committed: Mon Jan 9 11:46:36 2017 -0800 ---------------------------------------------------------------------- .../ZOOKEEPER/3.4.9/service_advisor.py | 143 ++++++++ .../src/main/resources/scripts/stack_advisor.py | 9 +- .../stacks/BIGTOP/0.8/services/stack_advisor.py | 125 +++++-- .../stacks/HDP/2.0.6/services/stack_advisor.py | 192 ++++------ .../stacks/HDP/2.1/services/stack_advisor.py | 63 +++- .../stacks/HDP/2.2/services/stack_advisor.py | 69 +++- .../stacks/HDPWIN/2.1/services/stack_advisor.py | 68 +++- .../stacks/HDPWIN/2.2/services/stack_advisor.py | 74 +++- .../main/resources/stacks/service_advisor.py | 65 ++-- .../src/main/resources/stacks/stack_advisor.py | 367 ++++++++++++++++++- .../2.2/common/test_stack_advisor_perf.py | 22 +- .../stacks/2.3/common/test_stack_advisor.py | 30 +- 12 files changed, 965 insertions(+), 262 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/8258cf89/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.9/service_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.9/service_advisor.py b/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.9/service_advisor.py new file mode 100644 index 0000000..82316f4 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.9/service_advisor.py @@ -0,0 +1,143 @@ +#!/usr/bin/env ambari-python-wrap +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +# Python imports +import imp +import os +import traceback +import inspect + +# Local imports +from resource_management.core.logger import Logger + + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +STACKS_DIR = os.path.join(SCRIPT_DIR, '../../../stacks/') +PARENT_FILE = os.path.join(STACKS_DIR, 'service_advisor.py') + +try: + with open(PARENT_FILE, 'rb') as fp: + service_advisor = imp.load_module('service_advisor', fp, PARENT_FILE, ('.py', 'rb', imp.PY_SOURCE)) +except Exception as e: + traceback.print_exc() + print "Failed to load parent" + + +class ZookeeperServiceAdvisor(service_advisor.ServiceAdvisor): + + def __init__(self, *args, **kwargs): + self.as_super = super(ZookeeperServiceAdvisor, self) + self.as_super.__init__(*args, **kwargs) + + self.modifyMastersWithMultipleInstances() + self.modifyCardinalitiesDict() + self.modifyHeapSizeProperties() + + def modifyMastersWithMultipleInstances(self): + """ + Modify the set of masters with multiple instances. + Must be overriden in child class. + """ + self.mastersWithMultipleInstances.add("ZOOKEEPER_SERVER") + + def modifyCardinalitiesDict(self): + """ + Modify the dictionary of cardinalities. + Must be overriden in child class. + """ + self.cardinalitiesDict["ZOOKEEPER_SERVER"] = {"min": 3} + + def modifyHeapSizeProperties(self): + """ + Modify the dictionary of heap size properties. + Must be overriden in child class. + """ + self.heap_size_properties = {"ZOOKEEPER_SERVER": [{"config-name": "zookeeper-env", + "property": "zk_server_heapsize", + "default": "1024m"}]} + + def getServiceComponentLayoutValidations(self, services, hosts): + """ + Get a list of errors. Zookeeper does not have any validations in this version. + """ + service_name = services["services"][0]["StackServices"]["service_name"] + Logger.info("Class: %s, Method: %s. Validating Service Component Layout for Service: %s." % + (self.__class__.__name__, inspect.stack()[0][3], service_name)) + return self.as_super.getServiceComponentLayoutValidations(services, hosts) + + def getServiceConfigurationRecommendations(self, configurations, clusterData, services, hosts): + """ + Recommend configurations to set. Zookeeper does not have any recommendations in this version. + """ + service_name = services["services"][0]["StackServices"]["service_name"] + Logger.info("Class: %s, Method: %s. Recommending Service Configurations for Service: %s." % + (self.__class__.__name__, inspect.stack()[0][3], service_name)) + + self.recommendConfigurations(configurations, clusterData, services, hosts) + + def recommendConfigurations(self, configurations, clusterData, services, hosts): + """ + Recommend configurations for this service. + """ + service_name = services["services"][0]["StackServices"]["service_name"] + Logger.info("Class: %s, Method: %s. Recommending Service Configurations for Service: %s." % + (self.__class__.__name__, inspect.stack()[0][3], service_name)) + + Logger.info("Setting zoo.cfg to default dataDir to /hadoop/zookeeper on the best matching mount") + + zk_mount_properties = [ + ("dataDir", "ZOOKEEPER_SERVER", "/hadoop/zookeeper", "single"), + ] + self.updateMountProperties("zoo.cfg", zk_mount_properties, configurations, services, hosts) + + def getServiceConfigurationsValidationItems(self, configurations, recommendedDefaults, services, hosts): + """ + Validate configurations for the service. Return a list of errors. + """ + service_name = services["services"][0]["StackServices"]["service_name"] + Logger.info("Class: %s, Method: %s. Validating Configurations for Service: %s." % + (self.__class__.__name__, inspect.stack()[0][3], service_name)) + + items = [] + + # Example of validating by calling helper methods + ''' + configType = "zookeeper-env" + method = self.someMethodInThisClass + resultItems = self.validateConfigurationsForSite(configurations, recommendedDefaults, services, hosts, configType, method) + items.extend(resultItems) + + method = self.anotherMethodInThisClass + resultItems = self.validateConfigurationsForSite(configurations, recommendedDefaults, services, hosts, configType, method) + items.extend(resultItems) + ''' + + return items + + ''' + def someMethodInThisClass(self, properties, recommendedDefaults, configurations, services, hosts): + validationItems = [] + validationItems.append({"config-name": "zookeeper-env", "item": self.getErrorItem("My custom message 1")}) + return self.toConfigurationValidationProblems(validationItems, "zookeeper-env") + + def anotherMethodInThisClass(self, properties, recommendedDefaults, configurations, services, hosts): + validationItems = [] + validationItems.append({"config-name": "zookeeper-env", "item": self.getErrorItem("My custom message 2")}) + return self.toConfigurationValidationProblems(validationItems, "zookeeper-env") + ''' \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/8258cf89/ambari-server/src/main/resources/scripts/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/scripts/stack_advisor.py b/ambari-server/src/main/resources/scripts/stack_advisor.py index abfab87..1d9b4a2 100755 --- a/ambari-server/src/main/resources/scripts/stack_advisor.py +++ b/ambari-server/src/main/resources/scripts/stack_advisor.py @@ -134,10 +134,11 @@ def instantiateStackAdvisor(stackName, stackVersion, parentVersions): try: path = STACK_ADVISOR_IMPL_PATH_TEMPLATE.format(stackName, version) - with open(path, 'rb') as fp: - stack_advisor = imp.load_module('stack_advisor_impl', fp, path, ('.py', 'rb', imp.PY_SOURCE)) - className = STACK_ADVISOR_IMPL_CLASS_TEMPLATE.format(stackName, version.replace('.', '')) - print "StackAdvisor implementation for stack {0}, version {1} was loaded".format(stackName, version) + if os.path.isfile(path): + with open(path, 'rb') as fp: + stack_advisor = imp.load_module('stack_advisor_impl', fp, path, ('.py', 'rb', imp.PY_SOURCE)) + className = STACK_ADVISOR_IMPL_CLASS_TEMPLATE.format(stackName, version.replace('.', '')) + print "StackAdvisor implementation for stack {0}, version {1} was loaded".format(stackName, version) except IOError: # file not found traceback.print_exc() print "StackAdvisor implementation for stack {0}, version {1} was not found".format(stackName, version) http://git-wip-us.apache.org/repos/asf/ambari/blob/8258cf89/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/stack_advisor.py index 701d0d4..5172042 100644 --- a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/stack_advisor.py @@ -17,13 +17,68 @@ See the License for the specific language governing permissions and limitations under the License. """ +# Python Imports import re from math import ceil +# Local Imports +from resource_management.core.logger import Logger from stack_advisor import DefaultStackAdvisor class BaseBIGTOP08StackAdvisor(DefaultStackAdvisor): + + def __init__(self): + super(BaseBIGTOP08StackAdvisor, self).__init__() + Logger.initialize_logger() + + self.modifyMastersWithMultipleInstances() + self.modifyCardinalitiesDict() + self.modifyHeapSizeProperties() + self.modifyNotValuableComponents() + self.modifyComponentsNotPreferableOnServer() + + def modifyMastersWithMultipleInstances(self): + """ + Modify the set of masters with multiple instances. + Must be overriden in child class. + """ + self.mastersWithMultipleInstances |= set(['ZOOKEEPER_SERVER', 'HBASE_MASTER']) + + def modifyCardinalitiesDict(self): + """ + Modify the dictionary of cardinalities. + Must be overriden in child class. + """ + self.cardinalitiesDict.update( + { + 'ZOOKEEPER_SERVER': {"min": 3}, + 'HBASE_MASTER': {"min": 1} + } + ) + + def modifyHeapSizeProperties(self): + """ + Modify the dictionary of heap size properties. + Must be overriden in child class. + """ + # Nothing to do + pass + + def modifyNotValuableComponents(self): + """ + Modify the set of components whose host assignment is based on other services. + Must be overriden in child class. + """ + self.notValuableComponents |= set(['JOURNALNODE', 'ZKFC', 'GANGLIA_MONITOR']) + + def modifyComponentsNotPreferableOnServer(self): + """ + Modify the set of components that are not preferable on the server. + Must be overriden in child class. + """ + self.notPreferableOnServerComponents |= set(['GANGLIA_SERVER']) + def getComponentLayoutValidations(self, services, hosts): """Returns array of Validation objects about issues with hostnames components assigned to""" items = [] @@ -275,21 +330,7 @@ class BaseBIGTOP08StackAdvisor(DefaultStackAdvisor): {"config-name": 'yarn.scheduler.maximum-allocation-mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.scheduler.maximum-allocation-mb')} ] return self.toConfigurationValidationProblems(validationItems, "yarn-site") - def getMastersWithMultipleInstances(self): - return ['ZOOKEEPER_SERVER', 'HBASE_MASTER'] - - def getNotValuableComponents(self): - return ['JOURNALNODE', 'ZKFC', 'GANGLIA_MONITOR'] - - def getNotPreferableOnServerComponents(self): - return ['GANGLIA_SERVER'] - - def getCardinalitiesDict(self): - return { - 'ZOOKEEPER_SERVER': {"min": 3}, - 'HBASE_MASTER': {"min": 1}, - } - + # TODO, move to Service Advisors. def getComponentLayoutSchemes(self): return { 'NAMENODE': {"else": 0}, @@ -308,6 +349,53 @@ class BaseBIGTOP08StackAdvisor(DefaultStackAdvisor): class BIGTOP08StackAdvisor(BaseBIGTOP08StackAdvisor): + def __init__(self): + super(BIGTOP08StackAdvisor, self).__init__() + + self.modifyMastersWithMultipleInstances() + self.modifyCardinalitiesDict() + self.modifyHeapSizeProperties() + self.modifyNotValuableComponents() + self.modifyComponentsNotPreferableOnServer() + + def modifyMastersWithMultipleInstances(self): + """ + Modify the set of masters with multiple instances. + Must be overriden in child class. + """ + # Nothing to do + pass + + def modifyCardinalitiesDict(self): + """ + Modify the dictionary of cardinalities. + Must be overriden in child class. + """ + # Nothing to do + pass + + def modifyHeapSizeProperties(self): + """ + Modify the dictionary of heap size properties. + Must be overriden in child class. + """ + # Nothing to do + pass + + def modifyNotValuableComponents(self): + """ + Modify the set of components whose host assignment is based on other services. + Must be overriden in child class. + """ + self.notValuableComponents |= set(['APP_TIMELINE_SERVER']) + + def modifyComponentsNotPreferableOnServer(self): + """ + Modify the set of components that are not preferable on the server. + Must be overriden in child class. + """ + self.notPreferableOnServerComponents |= set(['STORM_UI_SERVER', 'DRPC_SERVER', 'STORM_REST_API', 'NIMBUS']) + def getServiceConfigurationRecommenderDict(self): parentRecommendConfDict = super(BIGTOP08StackAdvisor, self).getServiceConfigurationRecommenderDict() childRecommendConfDict = { @@ -342,12 +430,7 @@ class BIGTOP08StackAdvisor(BaseBIGTOP08StackAdvisor): "-server -Xmx" + str(int(0.8 * clusterData["amMemory"])) + "m -Djava.net.preferIPv4Stack=true -XX:+UseNUMA -XX:+UseParallelGC") - def getNotPreferableOnServerComponents(self): - return ['STORM_UI_SERVER', 'DRPC_SERVER', 'STORM_REST_API', 'NIMBUS', 'GANGLIA_SERVER'] - - def getNotValuableComponents(self): - return ['JOURNALNODE', 'ZKFC', 'GANGLIA_MONITOR', 'APP_TIMELINE_SERVER'] - + # TODO, move to Service Advisors. def getComponentLayoutSchemes(self): parentSchemes = super(BIGTOP08StackAdvisor, self).getComponentLayoutSchemes() childSchemes = { http://git-wip-us.apache.org/repos/asf/ambari/blob/8258cf89/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py index 0c02eab..9b217b3 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py @@ -17,17 +17,17 @@ See the License for the specific language governing permissions and limitations under the License. """ +# Python Imports import re import os import sys import socket - from math import ceil, floor, log +# Local Imports from resource_management.core.logger import Logger from resource_management.libraries.functions.mounted_dirs_helper import get_mounts_with_multiple_data_dirs from resource_management.libraries.functions.data_structure_utils import get_from_dict - from stack_advisor import DefaultStackAdvisor @@ -37,6 +37,53 @@ class HDP206StackAdvisor(DefaultStackAdvisor): super(HDP206StackAdvisor, self).__init__() Logger.initialize_logger() + self.modifyMastersWithMultipleInstances() + self.modifyCardinalitiesDict() + self.modifyHeapSizeProperties() + self.modifyNotValuableComponents() + self.modifyComponentsNotPreferableOnServer() + + def modifyMastersWithMultipleInstances(self): + """ + Modify the set of masters with multiple instances. + Must be overriden in child class. + """ + self.mastersWithMultipleInstances |= set(['ZOOKEEPER_SERVER', 'HBASE_MASTER']) + + def modifyCardinalitiesDict(self): + """ + Modify the dictionary of cardinalities. + Must be overriden in child class. + """ + self.cardinalitiesDict.update( + { + 'ZOOKEEPER_SERVER': {"min": 3}, + 'HBASE_MASTER': {"min": 1}, + } + ) + + def modifyHeapSizeProperties(self): + """ + Modify the dictionary of heap size properties. + Must be overriden in child class. + """ + # Nothing to do + pass + + def modifyNotValuableComponents(self): + """ + Modify the set of components whose host assignment is based on other services. + Must be overriden in child class. + """ + self.notValuableComponents |= set(['JOURNALNODE', 'ZKFC', 'GANGLIA_MONITOR']) + + def modifyComponentsNotPreferableOnServer(self): + """ + Modify the set of components that are not preferable on the server. + Must be overriden in child class. + """ + self.notPreferableOnServerComponents |= set(['GANGLIA_SERVER', 'METRICS_COLLECTOR']) + def getComponentLayoutValidations(self, services, hosts): """Returns array of Validation objects about issues with hostnames components assigned to""" items = super(HDP206StackAdvisor, self).getComponentLayoutValidations(services, hosts) @@ -953,6 +1000,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor): and hostname in componentEntry["StackServiceComponents"]["hostnames"]]) return components + # TODO, move this to a generic stack advisor. def getZKHostPortString(self, services, include_port=True): """ Returns the comma delimited string of zookeeper server host with the configure port installed in a cluster @@ -978,6 +1026,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor): zookeeper_host_port = ",".join(zookeeper_host_port_arr) return zookeeper_host_port + # TODO, move this to a generic stack advisor def getZKPort(self, services): zookeeper_port = '2181' #default port if 'zoo.cfg' in services['configurations'] and ('clientPort' in services['configurations']['zoo.cfg']['properties']): @@ -1414,7 +1463,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor): if collectorHostName in component["StackServiceComponents"]["hostnames"]: hostComponents.append(component["StackServiceComponents"]["component_name"]) - requiredMemory = getMemorySizeRequired(hostComponents, configurations) + requiredMemory = self.getMemorySizeRequired(services, hostComponents, configurations) unusedMemory = host["Hosts"]["total_mem"] * 1024 - requiredMemory # in bytes collector_needs_increase = collector_heapsize * mb < 16 * gb @@ -1429,6 +1478,33 @@ class HDP206StackAdvisor(DefaultStackAdvisor): pass return self.toConfigurationValidationProblems(validationItems, "ams-env") + def getMemorySizeRequired(self, services, components, configurations): + totalMemoryRequired = 512*1024*1024 # 512Mb for OS needs + # Dictionary from component name to list of dictionary with keys: config-name, property, default. + heapSizeProperties = self.get_heap_size_properties(services) + for component in components: + if component in heapSizeProperties.keys(): + heapSizePropertiesForComp = heapSizeProperties[component] + for heapSizeProperty in heapSizePropertiesForComp: + try: + properties = configurations[heapSizeProperty["config-name"]]["properties"] + heapsize = properties[heapSizeProperty["property"]] + except KeyError: + heapsize = heapSizeProperty["default"] + + # Assume Mb if no modifier + if len(heapsize) > 1 and heapsize[-1] in '0123456789': + heapsize = str(heapsize) + "m" + + totalMemoryRequired += formatXmxSizeToBytes(heapsize) + else: + if component == "METRICS_MONITOR" or "CLIENT" in component: + heapsize = '512m' + else: + heapsize = '1024m' + totalMemoryRequired += formatXmxSizeToBytes(heapsize) + return totalMemoryRequired + def getPreferredMountPoints(self, hostInfo): # '/etc/resolv.conf', '/etc/hostname', '/etc/hosts' are docker specific mount points @@ -1707,21 +1783,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor): return dataNodeHosts return [] - def getMastersWithMultipleInstances(self): - return ['ZOOKEEPER_SERVER', 'HBASE_MASTER'] - - def getNotValuableComponents(self): - return ['JOURNALNODE', 'ZKFC', 'GANGLIA_MONITOR'] - - def getNotPreferableOnServerComponents(self): - return ['GANGLIA_SERVER', 'METRICS_COLLECTOR'] - - def getCardinalitiesDict(self): - return { - 'ZOOKEEPER_SERVER': {"min": 3}, - 'HBASE_MASTER': {"min": 1}, - } - + # TODO, move to Service Advisors. def getComponentLayoutSchemes(self): return { 'NAMENODE': {"else": 0}, @@ -2063,99 +2125,5 @@ def getMountPointForDir(dir, mountPoints): return bestMountFound -def getHeapsizeProperties(): - return { "NAMENODE": [{"config-name": "hadoop-env", - "property": "namenode_heapsize", - "default": "1024m"}], - "SECONDARY_NAMENODE": [{"config-name": "hadoop-env", - "property": "namenode_heapsize", - "default": "1024m"}], - "DATANODE": [{"config-name": "hadoop-env", - "property": "dtnode_heapsize", - "default": "1024m"}], - "REGIONSERVER": [{"config-name": "hbase-env", - "property": "hbase_regionserver_heapsize", - "default": "1024m"}], - "HBASE_MASTER": [{"config-name": "hbase-env", - "property": "hbase_master_heapsize", - "default": "1024m"}], - "HIVE_CLIENT": [{"config-name": "hive-env", - "property": "hive.client.heapsize", - "default": "1024"}], - "HIVE_METASTORE": [{"config-name": "hive-env", - "property": "hive.metastore.heapsize", - "default": "1024"}], - "HIVE_SERVER": [{"config-name": "hive-env", - "property": "hive.heapsize", - "default": "1024"}], - "HISTORYSERVER": [{"config-name": "mapred-env", - "property": "jobhistory_heapsize", - "default": "1024m"}], - "OOZIE_SERVER": [{"config-name": "oozie-env", - "property": "oozie_heapsize", - "default": "1024m"}], - "RESOURCEMANAGER": [{"config-name": "yarn-env", - "property": "resourcemanager_heapsize", - "default": "1024m"}], - "NODEMANAGER": [{"config-name": "yarn-env", - "property": "nodemanager_heapsize", - "default": "1024m"}], - "APP_TIMELINE_SERVER": [{"config-name": "yarn-env", - "property": "apptimelineserver_heapsize", - "default": "1024m"}], - "ZOOKEEPER_SERVER": [{"config-name": "zookeeper-env", - "property": "zk_server_heapsize", - "default": "1024m"}], - "METRICS_COLLECTOR": [{"config-name": "ams-hbase-env", - "property": "hbase_master_heapsize", - "default": "1024"}, - {"config-name": "ams-hbase-env", - "property": "hbase_regionserver_heapsize", - "default": "1024"}, - {"config-name": "ams-env", - "property": "metrics_collector_heapsize", - "default": "512"}], - "ATLAS_SERVER": [{"config-name": "atlas-env", - "property": "atlas_server_xmx", - "default": "2048"}], - "LOGSEARCH_SERVER": [{"config-name": "logsearch-env", - "property": "logsearch_app_max_memory", - "default": "1024"}], - "LOGSEARCH_LOGFEEDER": [{"config-name": "logfeeder-env", - "property": "logfeeder_max_mem", - "default": "512"}], - "SPARK_JOBHISTORYSERVER": [{"config-name": "spark-env", - "property": "spark_daemon_memory", - "default": "1024"}], - "SPARK2_JOBHISTORYSERVER": [{"config-name": "spark2-env", - "property": "spark_daemon_memory", - "default": "1024"}] - } - -def getMemorySizeRequired(components, configurations): - totalMemoryRequired = 512*1024*1024 # 512Mb for OS needs - for component in components: - if component in getHeapsizeProperties().keys(): - heapSizeProperties = getHeapsizeProperties()[component] - for heapSizeProperty in heapSizeProperties: - try: - properties = configurations[heapSizeProperty["config-name"]]["properties"] - heapsize = properties[heapSizeProperty["property"]] - except KeyError: - heapsize = heapSizeProperty["default"] - - # Assume Mb if no modifier - if len(heapsize) > 1 and heapsize[-1] in '0123456789': - heapsize = str(heapsize) + "m" - - totalMemoryRequired += formatXmxSizeToBytes(heapsize) - else: - if component == "METRICS_MONITOR" or "CLIENT" in component: - heapsize = '512m' - else: - heapsize = '1024m' - totalMemoryRequired += formatXmxSizeToBytes(heapsize) - return totalMemoryRequired - def round_to_n(mem_size, n=128): return int(round(mem_size / float(n))) * int(n) http://git-wip-us.apache.org/repos/asf/ambari/blob/8258cf89/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py index f34f5a1..6ae29cb 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py @@ -17,8 +17,63 @@ See the License for the specific language governing permissions and limitations under the License. """ +# Local Imports +from resource_management.core.logger import Logger + + class HDP21StackAdvisor(HDP206StackAdvisor): + def __init__(self): + super(HDP21StackAdvisor, self).__init__() + Logger.initialize_logger() + + self.modifyMastersWithMultipleInstances() + self.modifyCardinalitiesDict() + self.modifyHeapSizeProperties() + self.modifyNotValuableComponents() + self.modifyComponentsNotPreferableOnServer() + + def modifyMastersWithMultipleInstances(self): + """ + Modify the set of masters with multiple instances. + Must be overriden in child class. + """ + self.mastersWithMultipleInstances |= set(['ZOOKEEPER_SERVER', 'HBASE_MASTER']) + + def modifyCardinalitiesDict(self): + """ + Modify the dictionary of cardinalities. + Must be overriden in child class. + """ + self.cardinalitiesDict.update( + { + 'ZOOKEEPER_SERVER': {"min": 3}, + 'HBASE_MASTER': {"min": 1}, + } + ) + + def modifyHeapSizeProperties(self): + """ + Modify the dictionary of heap size properties. + Must be overriden in child class. + """ + # Nothing to do + pass + + def modifyNotValuableComponents(self): + """ + Modify the set of components whose host assignment is based on other services. + Must be overriden in child class. + """ + self.notValuableComponents |= set(['JOURNALNODE', 'ZKFC', 'GANGLIA_MONITOR', 'APP_TIMELINE_SERVER']) + + def modifyComponentsNotPreferableOnServer(self): + """ + Modify the set of components that are not preferable on the server. + Must be overriden in child class. + """ + self.notPreferableOnServerComponents |= set(['STORM_UI_SERVER', 'DRPC_SERVER', 'STORM_REST_API', 'NIMBUS', 'GANGLIA_SERVER', 'METRICS_COLLECTOR']) + def getServiceConfigurationRecommenderDict(self): parentRecommendConfDict = super(HDP21StackAdvisor, self).getServiceConfigurationRecommenderDict() childRecommendConfDict = { @@ -155,13 +210,7 @@ class HDP21StackAdvisor(HDP206StackAdvisor): if recommended_tez_queue is not None: putTezProperty("tez.queue.name", recommended_tez_queue) - - def getNotPreferableOnServerComponents(self): - return ['STORM_UI_SERVER', 'DRPC_SERVER', 'STORM_REST_API', 'NIMBUS', 'GANGLIA_SERVER', 'METRICS_COLLECTOR'] - - def getNotValuableComponents(self): - return ['JOURNALNODE', 'ZKFC', 'GANGLIA_MONITOR', 'APP_TIMELINE_SERVER'] - + # TODO, move to Service Advisors. def getComponentLayoutSchemes(self): parentSchemes = super(HDP21StackAdvisor, self).getComponentLayoutSchemes() childSchemes = { http://git-wip-us.apache.org/repos/asf/ambari/blob/8258cf89/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py index cf7134e..b601179 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py @@ -17,6 +17,7 @@ See the License for the specific language governing permissions and limitations under the License. """ +# Python Imports import math from math import floor from urlparse import urlparse @@ -26,9 +27,62 @@ import socket import re import xml.etree.ElementTree as ET +# Local Imports +from resource_management.core.logger import Logger class HDP22StackAdvisor(HDP21StackAdvisor): + def __init__(self): + super(HDP22StackAdvisor, self).__init__() + Logger.initialize_logger() + + self.modifyMastersWithMultipleInstances() + self.modifyCardinalitiesDict() + self.modifyHeapSizeProperties() + self.modifyNotValuableComponents() + self.modifyComponentsNotPreferableOnServer() + + def modifyMastersWithMultipleInstances(self): + """ + Modify the set of masters with multiple instances. + Must be overriden in child class. + """ + self.mastersWithMultipleInstances |= set(['METRICS_COLLECTOR']) + + def modifyCardinalitiesDict(self): + """ + Modify the dictionary of cardinalities. + Must be overriden in child class. + """ + self.cardinalitiesDict.update( + { + 'METRICS_COLLECTOR': {"min": 1} + } + ) + + def modifyHeapSizeProperties(self): + """ + Modify the dictionary of heap size properties. + Must be overriden in child class. + """ + # Nothing to do + pass + + def modifyNotValuableComponents(self): + """ + Modify the set of components whose host assignment is based on other services. + Must be overriden in child class. + """ + self.notValuableComponents |= set(['METRICS_MONITOR']) + + def modifyComponentsNotPreferableOnServer(self): + """ + Modify the set of components that are not preferable on the server. + Must be overriden in child class. + """ + # Nothing to do + pass + def getServiceConfigurationRecommenderDict(self): parentRecommendConfDict = super(HDP22StackAdvisor, self).getServiceConfigurationRecommenderDict() childRecommendConfDict = { @@ -1674,21 +1728,6 @@ class HDP22StackAdvisor(HDP21StackAdvisor): "item": self.getWarnItem("Ranger Storm plugin should not be enabled in non-kerberos environment.")}) return self.toConfigurationValidationProblems(validationItems, "ranger-env") - def getMastersWithMultipleInstances(self): - result = super(HDP22StackAdvisor, self).getMastersWithMultipleInstances() - result.extend(['METRICS_COLLECTOR']) - return result - - def getNotValuableComponents(self): - result = super(HDP22StackAdvisor, self).getNotValuableComponents() - result.extend(['METRICS_MONITOR']) - return result - - def getCardinalitiesDict(self): - result = super(HDP22StackAdvisor, self).getCardinalitiesDict() - result['METRICS_COLLECTOR'] = {"min": 1} - return result - def getAffectedConfigs(self, services): affectedConfigs = super(HDP22StackAdvisor, self).getAffectedConfigs(services) http://git-wip-us.apache.org/repos/asf/ambari/blob/8258cf89/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py index cf52ade..4e4ef51 100644 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py @@ -17,15 +17,68 @@ See the License for the specific language governing permissions and limitations under the License. """ +# Python Imports import re import sys import os from math import ceil +# Local Imports +from resource_management.core.logger import Logger from stack_advisor import DefaultStackAdvisor class HDPWIN21StackAdvisor(DefaultStackAdvisor): + def __init__(self): + super(HDPWIN21StackAdvisor, self).__init__() + + self.modifyMastersWithMultipleInstances() + self.modifyCardinalitiesDict() + self.modifyHeapSizeProperties() + self.modifyNotValuableComponents() + self.modifyComponentsNotPreferableOnServer() + + def modifyMastersWithMultipleInstances(self): + """ + Modify the set of masters with multiple instances. + Must be overriden in child class. + """ + self.mastersWithMultipleInstances |= set(['ZOOKEEPER_SERVER', 'HBASE_MASTER']) + + def modifyCardinalitiesDict(self): + """ + Modify the dictionary of cardinalities. + Must be overriden in child class. + """ + self.cardinalitiesDict.update( + { + 'ZOOKEEPER_SERVER': {"min": 3}, + 'HBASE_MASTER': {"min": 1}, + } + ) + + def modifyHeapSizeProperties(self): + """ + Modify the dictionary of heap size properties. + Must be overriden in child class. + """ + # Nothing to do + pass + + def modifyNotValuableComponents(self): + """ + Modify the set of components whose host assignment is based on other services. + Must be overriden in child class. + """ + self.notValuableComponents |= set(['JOURNALNODE', 'ZKFC', 'APP_TIMELINE_SERVER']) + + def modifyComponentsNotPreferableOnServer(self): + """ + Modify the set of components that are not preferable on the server. + Must be overriden in child class. + """ + self.notPreferableOnServerComponents |= set(['STORM_UI_SERVER', 'DRPC_SERVER', 'STORM_REST_API', 'NIMBUS']) + def getComponentLayoutValidations(self, services, hosts): """Returns array of Validation objects about issues with hostnames components assigned to""" items = [] @@ -508,21 +561,8 @@ class HDPWIN21StackAdvisor(DefaultStackAdvisor): {"config-name": 'hbase_master_heapsize', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'hbase_master_heapsize')}] return self.toConfigurationValidationProblems(validationItems, "hbase-env") - def getMastersWithMultipleInstances(self): - return ['ZOOKEEPER_SERVER', 'HBASE_MASTER'] - - def getNotValuableComponents(self): - return ['JOURNALNODE', 'ZKFC', 'APP_TIMELINE_SERVER'] - - def getNotPreferableOnServerComponents(self): - return ['STORM_UI_SERVER', 'DRPC_SERVER', 'STORM_REST_API', 'NIMBUS'] - - def getCardinalitiesDict(self): - return { - 'ZOOKEEPER_SERVER': {"min": 3}, - 'HBASE_MASTER': {"min": 1}, - } + # TODO, move to Service Advisors. def getComponentLayoutSchemes(self): return { 'NAMENODE': {"else": 0}, http://git-wip-us.apache.org/repos/asf/ambari/blob/8258cf89/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py index d560146..26292d9 100644 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py @@ -17,12 +17,16 @@ See the License for the specific language governing permissions and limitations under the License. """ +# Python Imports import math import os import re import sys from urlparse import urlparse +# Local Imports +from resource_management.core.logger import Logger + def getSiteProperties(configurations, siteName): siteConfig = configurations.get(siteName) if siteConfig is None: @@ -52,6 +56,56 @@ def isSecurePort(port): class HDPWIN22StackAdvisor(HDPWIN21StackAdvisor): + def __init__(self): + super(HDPWIN22StackAdvisor, self).__init__() + Logger.initialize_logger() + + self.modifyMastersWithMultipleInstances() + self.modifyCardinalitiesDict() + self.modifyHeapSizeProperties() + self.modifyNotValuableComponents() + self.modifyComponentsNotPreferableOnServer() + + def modifyMastersWithMultipleInstances(self): + """ + Modify the set of masters with multiple instances. + Must be overriden in child class. + """ + self.mastersWithMultipleInstances |= set(['METRICS_COLLECTOR']) + + def modifyCardinalitiesDict(self): + """ + Modify the dictionary of cardinalities. + Must be overriden in child class. + """ + self.cardinalitiesDict.update( + { + 'METRICS_COLLECTOR': {"min": 1} + } + ) + + def modifyHeapSizeProperties(self): + """ + Modify the dictionary of heap size properties. + Must be overriden in child class. + """ + # Nothing to do + pass + + def modifyNotValuableComponents(self): + """ + Modify the set of components whose host assignment is based on other services. + Must be overriden in child class. + """ + self.notValuableComponents |= set(['METRICS_MONITOR']) + + def modifyComponentsNotPreferableOnServer(self): + """ + Modify the set of components that are not preferable on the server. + Must be overriden in child class. + """ + self.notPreferableOnServerComponents |= set(['METRICS_COLLECTOR']) + def getServiceConfigurationRecommenderDict(self): parentRecommendConfDict = super(HDPWIN22StackAdvisor, self).getServiceConfigurationRecommenderDict() childRecommendConfDict = { @@ -1077,26 +1131,6 @@ class HDPWIN22StackAdvisor(HDPWIN21StackAdvisor): } return driverDict.get(databaseType.upper()) - def getMastersWithMultipleInstances(self): - result = super(HDPWIN22StackAdvisor, self).getMastersWithMultipleInstances() - result.extend(['METRICS_COLLECTOR']) - return result - - def getNotValuableComponents(self): - result = super(HDPWIN22StackAdvisor, self).getNotValuableComponents() - result.extend(['METRICS_MONITOR']) - return result - - def getNotPreferableOnServerComponents(self): - result = super(HDPWIN22StackAdvisor, self).getNotPreferableOnServerComponents() - result.extend(['METRICS_COLLECTOR']) - return result - - def getCardinalitiesDict(self): - result = super(HDPWIN22StackAdvisor, self).getCardinalitiesDict() - result['METRICS_COLLECTOR'] = {"min": 1} - return result - def getAffectedConfigs(self, services): affectedConfigs = super(HDPWIN22StackAdvisor, self).getAffectedConfigs(services) http://git-wip-us.apache.org/repos/asf/ambari/blob/8258cf89/ambari-server/src/main/resources/stacks/service_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/service_advisor.py b/ambari-server/src/main/resources/stacks/service_advisor.py index 7a3c7e9..4b32926 100644 --- a/ambari-server/src/main/resources/stacks/service_advisor.py +++ b/ambari-server/src/main/resources/stacks/service_advisor.py @@ -53,43 +53,50 @@ class ServiceAdvisor(DefaultStackAdvisor): Abstract class implemented by all service advisors. """ - """ - If any components of the service should be colocated with other services, - this is where you should set up that layout. Example: - - # colocate HAWQSEGMENT with DATANODE, if no hosts have been allocated for HAWQSEGMENT - hawqSegment = [component for component in serviceComponents if component["StackServiceComponents"]["component_name"] == "HAWQSEGMENT"][0] - if not self.isComponentHostsPopulated(hawqSegment): - for hostName in hostsComponentsMap.keys(): - hostComponents = hostsComponentsMap[hostName] - if {"name": "DATANODE"} in hostComponents and {"name": "HAWQSEGMENT"} not in hostComponents: - hostsComponentsMap[hostName].append( { "name": "HAWQSEGMENT" } ) - if {"name": "DATANODE"} not in hostComponents and {"name": "HAWQSEGMENT"} in hostComponents: - hostComponents.remove({"name": "HAWQSEGMENT"}) - """ + def colocateService(self, hostsComponentsMap, serviceComponents): + """ + Populate hostsComponentsMap with key = hostname and value = [{"name": "COMP_NAME_1"}, {"name": "COMP_NAME_2"}, ...] + of services that must be co-hosted and on which host they should be present. + :param hostsComponentsMap: Map from hostname to list of [{"name": "COMP_NAME_1"}, {"name": "COMP_NAME_2"}, ...] + present on on that host. + :param serviceComponents: Mapping of components + + If any components of the service should be colocated with other services, + this is where you should set up that layout. Example: + + # colocate HAWQSEGMENT with DATANODE, if no hosts have been allocated for HAWQSEGMENT + hawqSegment = [component for component in serviceComponents if component["StackServiceComponents"]["component_name"] == "HAWQSEGMENT"][0] + if not self.isComponentHostsPopulated(hawqSegment): + for hostName in hostsComponentsMap.keys(): + hostComponents = hostsComponentsMap[hostName] + if {"name": "DATANODE"} in hostComponents and {"name": "HAWQSEGMENT"} not in hostComponents: + hostsComponentsMap[hostName].append( { "name": "HAWQSEGMENT" } ) + if {"name": "DATANODE"} not in hostComponents and {"name": "HAWQSEGMENT"} in hostComponents: + hostComponents.remove({"name": "HAWQSEGMENT"}) + """ pass - """ - Any configuration recommendations for the service should be defined in this function. - This should be similar to any of the recommendXXXXConfigurations functions in the stack_advisor.py - such as recommendYARNConfigurations(). - """ def getServiceConfigurationRecommendations(self, configurations, clusterSummary, services, hosts): + """ + Any configuration recommendations for the service should be defined in this function. + This should be similar to any of the recommendXXXXConfigurations functions in the stack_advisor.py + such as recommendYARNConfigurations(). + """ pass - """ - Returns an array of Validation objects about issues with the hostnames to which components are assigned. - This should detect validation issues which are different than those the stack_advisor.py detects. - The default validations are in stack_advisor.py getComponentLayoutValidations function. - """ def getServiceComponentLayoutValidations(self, services, hosts): + """ + Returns an array of Validation objects about issues with the hostnames to which components are assigned. + This should detect validation issues which are different than those the stack_advisor.py detects. + The default validations are in stack_advisor.py getComponentLayoutValidations function. + """ return [] - """ - Any configuration validations for the service should be defined in this function. - This should be similar to any of the validateXXXXConfigurations functions in the stack_advisor.py - such as validateHDFSConfigurations. - """ def getServiceConfigurationsValidationItems(self, configurations, recommendedDefaults, services, hosts): + """ + Any configuration validations for the service should be defined in this function. + This should be similar to any of the validateXXXXConfigurations functions in the stack_advisor.py + such as validateHDFSConfigurations. + """ return [] http://git-wip-us.apache.org/repos/asf/ambari/blob/8258cf89/ambari-server/src/main/resources/stacks/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/stack_advisor.py b/ambari-server/src/main/resources/stacks/stack_advisor.py index 8865b70..d648a53 100644 --- a/ambari-server/src/main/resources/stacks/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/stack_advisor.py @@ -17,12 +17,18 @@ See the License for the specific language governing permissions and limitations under the License. """ +# Python Imports import imp import os import re import socket import traceback +# Local imports +from resource_management.core.exceptions import Fail +from resource_management.core.logger import Logger + + class StackAdvisor(object): """ Abstract class implemented by all stack advisors. Stack advisors advise on stack specific questions. @@ -310,6 +316,9 @@ class DefaultStackAdvisor(StackAdvisor): def __init__(self): self.services = None + + Logger.initialize_logger() + # Dictionary that maps serviceName or componentName to serviceAdvisor self.serviceAdvisorsDict = {} @@ -318,6 +327,169 @@ class DefaultStackAdvisor(StackAdvisor): self.allRequestedProperties = None + # Data structures that may be extended by Service Advisors with information specific to each Service + self.mastersWithMultipleInstances = set() + self.notValuableComponents = set() + self.notPreferableOnServerComponents = set() + self.cardinalitiesDict = {} + self.loaded_service_advisors = False + + + def getServiceComponentLayoutValidations(self, services, hosts): + """ + Get a list of errors. + + :param services: Dictionary of the form: + { + 'changed-configurations': [], + 'Versions": { + 'parent_stack_version': '9.0', + 'stack_name': 'HDP', + 'stack_version': '9.0', + 'stack_hierarchy': { + 'stack_name': 'HDP', + 'stack_versions': ['8.0', '7.0', ..., '1.0'] + } + }, + 'ambari-server-properties': {'key': 'value', ...}, + 'services': [ + {'StackServices': { + 'advisor_path': '/var/lib/ambari-server/resources/common-services/MYSERVICE/1.2.3/service_advisor.py', + 'service_version': '1.2.3', + 'stack_name': 'HDP', + 'service_name': 'MYSERVICE', + 'stack_version': '9.0', + 'advisor_name': 'MYSERVICEServiceAdvisor' + }, + 'components': [ + {'StackServiceComponents': { + 'stack_version': '9.0', + 'decommission_allowed': True|False, + 'display_name': 'My Service Display Name', + 'stack_name': 'HDP', + 'custom_commands': [], + 'component_category': 'CLIENT|MASTER|SLAVE', + 'advertise_version': True|False, + 'is_client': True|False, + 'is_master': False|False, + 'bulk_commands_display_name': '', + 'bulk_commands_master_component_name': '', + 'service_name': 'MYSERVICE', + 'has_bulk_commands_definition': True|False, + 'reassign_allowed': True|False, + 'recovery_enabled': True|False, + 'cardinality': '0+|1|1+', + 'hostnames': ['c6401.ambari.apache.org'], + 'component_name': 'MY_COMPONENT_NAME' + }, + 'dependencies': [] + }, + ... + }], + 'configurations': [ + { + 'StackConfigurations': + { + 'stack_name': 'HDP', + 'service_name': 'MYSERVICE', + 'stack_version': '9.0', + 'property_depends_on': [], + 'type': 'myservice-config.xml', + 'property_name': 'foo' + }, + 'dependencies': [] + }, + { + 'StackConfigurations': { + 'stack_name': 'HDP', + 'service_name': 'ZOOKEEPER', + 'stack_version': + '2.6', + 'property_depends_on': [], + 'type': 'zoo.cfg.xml', + 'property_name': 'autopurge.snapRetainCount' + }, + 'dependencies': [] + } + ... + ] + } + ], + 'configurations': {} + } + + :param hosts: Dictionary where hosts["items"] contains list of hosts on the cluster. + E.g. of the form, + { + 'items': [ + { + 'Hosts': + { + 'host_name': 'c6401.ambari.apache.org', + 'public_host_name': 'c6401.ambari.apache.org', + 'ip': '192.168.64.101', + 'rack_info': '/default-rack', + 'os_type': 'centos6', + 'os_arch': 'x86_64', + 'cpu_count': 1, + 'ph_cpu_count': 1 + 'host_state': 'HEALTHY', + 'total_mem': 2926196, + 'host_status': 'HEALTHY', + 'last_registration_time': 1481833146522, + 'os_family': 'redhat6', + 'last_heartbeat_time': 1481835051067, + 'recovery_summary': 'DISABLED', + 'host_health_report': '', + 'desired_configs': None, + 'disk_info': [ + { + 'available': '483608892', + 'used': '3304964', + 'percent': '1%', + 'device': '/dev/mapper/VolGroup-lv_root', + 'mountpoint': '/', + 'type': 'ext4', + 'size': '512971376' + }, + ... + ], + 'recovery_report': { + 'component_reports': [], + 'summary': 'DISABLED' + }, + 'last_agent_env': { + 'transparentHugePage': 'always', + 'hostHealth': { + 'agentTimeStampAtReporting': 1481835031135, + 'activeJavaProcs': [], + 'serverTimeStampAtReporting': 1481835031180, + 'liveServices': [{ + 'status': 'Healthy', + 'name': 'ntpd', + 'desc': '' + }] + }, + 'umask': 18, + 'reverseLookup': True, + 'alternatives': [], + 'existingUsers': [], + 'firewallName': 'iptables', + 'stackFoldersAndFiles': [], + 'existingRepos': [], + 'installedPackages': [], + 'firewallRunning': False + } + } + } + ] + } + + :return: List of errors + """ + # To be overriden by subclass or Service Advisor + raise Fail("Must be overriden by subclass or Service Advisor") + def getActiveHosts(self, hosts): """ Filters the list of specified hosts object and returns a list of hosts which are not in maintenance mode. """ @@ -328,37 +500,69 @@ class DefaultStackAdvisor(StackAdvisor): return hostsList def getServiceAdvisor(self, key): - if len(self.serviceAdvisorsDict) == 0: + """ + Get the class name for the Service Advisor with the given name if it exists, or None otherwise. + :param key: Service Name + :return: Class name if it exists, or None otherwise. + """ + if not self.loaded_service_advisors: self.loadServiceAdvisors() - return self.serviceAdvisorsDict[key] + + return self.serviceAdvisorsDict[key] if key in self.serviceAdvisorsDict else None def loadServiceAdvisors(self): + """ + If not loaded, for all of the services requested load the Service Advisor into the in-memory dictionary. + """ + self.loaded_service_advisors = True + if self.services is None or "services" not in self.services: + return + for service in self.services["services"]: serviceName = service["StackServices"]["service_name"] - self.serviceAdvisorsDict[serviceName] = self.instantiateServiceAdvisor(service) + serviceAdvisor = self.instantiateServiceAdvisor(service) + # This may store None for that service advisor. + self.serviceAdvisorsDict[serviceName] = serviceAdvisor for component in service["components"]: componentName = self.getComponentName(component) self.serviceAdvisorsDict[componentName] = self.serviceAdvisorsDict[serviceName] def instantiateServiceAdvisor(self, service): - - serviceName = service["StackServices"]["service_name"] - className = service["StackServices"]["advisor_name"] if "advisor_name" in service["StackServices"] else None + """ + Load the Service Advisor for the given services by finding the best class in the given file. + :param service: Service object that contains a path to the advisor being requested. + :return: The class name for the Service Advisor requested, or None if one could not be found. + """ + service_name = service["StackServices"]["service_name"] + class_name = service["StackServices"]["advisor_name"] if "advisor_name" in service["StackServices"] else None path = service["StackServices"]["advisor_path"] if "advisor_path" in service["StackServices"] else None - if path is not None and os.path.exists(path) and className is not None: + class_name_pattern = re.compile("%s.*?ServiceAdvisor" % service_name, re.IGNORECASE) + + if path is not None and os.path.exists(path) and class_name is not None: try: with open(path, 'rb') as fp: - serviceAdvisor = imp.load_module('service_advisor_impl', fp, path, ('.py', 'rb', imp.PY_SOURCE)) - if hasattr(serviceAdvisor, className): - print "ServiceAdvisor implementation for service {0} was loaded".format(serviceName) - return getattr(serviceAdvisor, className)() - else: - print "Failed to load or create ServiceAdvisor implementation for service {0}: " \ - "Expecting class name {1} but it was not found.".format(serviceName, className) + service_advisor = imp.load_module('service_advisor_impl', fp, path, ('.py', 'rb', imp.PY_SOURCE)) + + # Find the class name by reading from all of the available attributes of the python file. + attributes = dir(service_advisor) + best_class_name = class_name + for potential_class_name in attributes: + if not potential_class_name.startswith("__"): + m = class_name_pattern.match(potential_class_name) + if m: + best_class_name = potential_class_name + break + + if hasattr(service_advisor, best_class_name): + Logger.info("ServiceAdvisor implementation for service {0} was loaded".format(service_name)) + return getattr(service_advisor, best_class_name)() + else: + Logger.error("Failed to load or create ServiceAdvisor implementation for service {0}: " \ + "Expecting class name {1} but it was not found.".format(service_name, best_class_name)) except Exception as e: traceback.print_exc() - print "Failed to load or create ServiceAdvisor implementation for service {0}".format(serviceName) + Logger.error("Failed to load or create ServiceAdvisor implementation for service {0}".format(service_name)) return None @@ -381,6 +585,124 @@ class DefaultStackAdvisor(StackAdvisor): return recommendations + def get_heap_size_properties(self, services): + """ + Get dictionary of all of the components and a mapping to the heap-size configs, along with default values + if the heap-size config could not be found. This is used in calculations for the total memory needed to run + the cluster. + :param services: Dictionary that contains all of the services being requested. This is used to find heap-size + configs that have been delegated to Service Advisors to define. + :return: Dictionary of mappings from component name to another dictionary of the heap-size configs. + """ + default = { + "NAMENODE": + [{"config-name": "hadoop-env", + "property": "namenode_heapsize", + "default": "1024m"}], + "SECONDARY_NAMENODE": + [{"config-name": "hadoop-env", + "property": "namenode_heapsize", + "default": "1024m"}], + "DATANODE": + [{"config-name": "hadoop-env", + "property": "dtnode_heapsize", + "default": "1024m"}], + "REGIONSERVER": + [{"config-name": "hbase-env", + "property": "hbase_regionserver_heapsize", + "default": "1024m"}], + "HBASE_MASTER": + [{"config-name": "hbase-env", + "property": "hbase_master_heapsize", + "default": "1024m"}], + "HIVE_CLIENT": + [{"config-name": "hive-env", + "property": "hive.client.heapsize", + "default": "1024m"}], + "HIVE_METASTORE": + [{"config-name": "hive-env", + "property": "hive.metastore.heapsize", + "default": "1024m"}], + "HIVE_SERVER": + [{"config-name": "hive-env", + "property": "hive.heapsize", + "default": "1024m"}], + "HISTORYSERVER": + [{"config-name": "mapred-env", + "property": "jobhistory_heapsize", + "default": "1024m"}], + "OOZIE_SERVER": + [{"config-name": "oozie-env", + "property": "oozie_heapsize", + "default": "1024m"}], + "RESOURCEMANAGER": + [{"config-name": "yarn-env", + "property": "resourcemanager_heapsize", + "default": "1024m"}], + "NODEMANAGER": + [{"config-name": "yarn-env", + "property": "nodemanager_heapsize", + "default": "1024m"}], + "APP_TIMELINE_SERVER": + [{"config-name": "yarn-env", + "property": "apptimelineserver_heapsize", + "default": "1024m"}], + "ZOOKEEPER_SERVER": + [{"config-name": "zookeeper-env", + "property": "zk_server_heapsize", + "default": "1024m"}], + "METRICS_COLLECTOR": + [{"config-name": "ams-hbase-env", + "property": "hbase_master_heapsize", + "default": "1024m"}, + {"config-name": "ams-hbase-env", + "property": "hbase_regionserver_heapsize", + "default": "1024m"}, + {"config-name": "ams-env", + "property": "metrics_collector_heapsize", + "default": "512m"}], + "ATLAS_SERVER": + [{"config-name": "atlas-env", + "property": "atlas_server_xmx", + "default": "2048m"}], + "LOGSEARCH_SERVER": + [{"config-name": "logsearch-env", + "property": "logsearch_app_max_memory", + "default": "1024m"}], + "LOGSEARCH_LOGFEEDER": + [{"config-name": "logfeeder-env", + "property": "logfeeder_max_mem", + "default": "512m"}], + "SPARK_JOBHISTORYSERVER": + [{"config-name": "spark-env", + "property": "spark_daemon_memory", + "default": "1024m"}], + "SPARK2_JOBHISTORYSERVER": + [{"config-name": "spark2-env", + "property": "spark_daemon_memory", + "default": "1024m"}] + } + + + try: + # Override any by reading from the Service Advisors + + for service in services["services"]: + serviceName = service["StackServices"]["service_name"] + serviceAdvisor = self.getServiceAdvisor(serviceName) + + # This seems confusing, but "self" may actually refer to the actual Service Advisor class that was loaded + # as opposed to this class. + advisor = serviceAdvisor if serviceAdvisor is not None else self + + # TODO, switch this to a function instead of a property. + if hasattr(advisor, "heap_size_properties"): + # Override the values in "default" with those from the service advisor + default.update(advisor.heap_size_properties) + except Exception, e: + traceback.print_exc() + return default + def createComponentLayoutRecommendations(self, services, hosts): self.services = services @@ -683,6 +1005,7 @@ class DefaultStackAdvisor(StackAdvisor): serviceAdvisors = [] for service in services["services"]: serviceName = service["StackServices"]["service_name"] + # "calculation" is a function pointer calculation = self.getServiceConfigurationRecommender(serviceName) if calculation is not None: calculation(configurations, cgClusterSummary, cgServices, cgHosts) @@ -818,7 +1141,11 @@ class DefaultStackAdvisor(StackAdvisor): # Helper dictionaries def getComponentCardinality(self, componentName): - return self.getCardinalitiesDict().get(componentName, {"min": 1, "max": 1}) + dict = self.getCardinalitiesDict() + if componentName in dict: + return dict[componentName] + else: + return {"min": 1, "max": 1} def getHostForComponent(self, component, hostsList): if len(hostsList) == 0: @@ -844,16 +1171,16 @@ class DefaultStackAdvisor(StackAdvisor): return not (self.getComponentName(component) in self.getNotPreferableOnServerComponents() and self.isLocalHost(host)) def getMastersWithMultipleInstances(self): - return [] + return self.mastersWithMultipleInstances def getNotValuableComponents(self): - return [] + return self.notValuableComponents def getNotPreferableOnServerComponents(self): - return [] + return self.notPreferableOnServerComponents def getCardinalitiesDict(self): - return {} + return self.cardinalitiesDict def getComponentLayoutSchemes(self): """ http://git-wip-us.apache.org/repos/asf/ambari/blob/8258cf89/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor_perf.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor_perf.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor_perf.py index 5b8c30c..82cef1b 100644 --- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor_perf.py +++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor_perf.py @@ -25,13 +25,15 @@ from mock.mock import patch class TestHDP22StackAdvisor(TestCase): - def instantiate_stack_advisor(self, testDirectory, base_stack_advisor_path): + def instantiate_stack_advisor(self, testDirectory): + default_stack_advisor_path = os.path.join(testDirectory, '../../../../../main/resources/stacks/stack_advisor.py') hdp_206_stack_advisor_path = os.path.join(testDirectory, '../../../../../main/resources/stacks/HDP/2.0.6/services/stack_advisor.py') hdp_21_stack_advisor_path = os.path.join(testDirectory, '../../../../../main/resources/stacks/HDP/2.1/services/stack_advisor.py') hdp_22_stack_advisor_path = os.path.join(testDirectory, '../../../../../main/resources/stacks/HDP/2.2/services/stack_advisor.py') - hdp_206_stack_advisor_classname = 'HDP22StackAdvisor' - with open(base_stack_advisor_path, 'rb') as fp: - imp.load_module('stack_advisor', fp, base_stack_advisor_path, ('.py', 'rb', imp.PY_SOURCE)) + hdp_206_stack_advisor_classname = 'HDP206StackAdvisor' + + with open(default_stack_advisor_path, 'rb') as fp: + imp.load_module('stack_advisor', fp, default_stack_advisor_path, ('.py', 'rb', imp.PY_SOURCE)) with open(hdp_206_stack_advisor_path, 'rb') as fp: imp.load_module('stack_advisor_impl', fp, hdp_206_stack_advisor_path, ('.py', 'rb', imp.PY_SOURCE)) with open(hdp_21_stack_advisor_path, 'rb') as fp: @@ -45,25 +47,19 @@ class TestHDP22StackAdvisor(TestCase): def test_performance(self, getfqdn_method): getfqdn_method.side_effect = lambda host='perf400-a-1.c.pramod-thangali.internal': host testDirectory = os.path.dirname(os.path.abspath(__file__)) - old_stack_advisor_path = os.path.join(testDirectory, '../../../../../test/resources/stacks/old_stack_advisor.py') current_stack_advisor_path = os.path.join(testDirectory, '../../../../../main/resources/stacks/stack_advisor.py') for folder_name in ['1', '2']: - stack_advisor_old = self.instantiate_stack_advisor(testDirectory, old_stack_advisor_path) services = json.load(open(os.path.join(testDirectory, folder_name + '/services.json'))) hosts = json.load(open(os.path.join(testDirectory, folder_name + '/hosts.json'))) - start = time.time() - recommendation_old = stack_advisor_old.recommendComponentLayout(services, hosts) - time_taken = time.time() - start - print "time taken by old stack_advisor.py = " + str(time_taken) - stack_advisor = self.instantiate_stack_advisor(testDirectory, current_stack_advisor_path) + stack_advisor = self.instantiate_stack_advisor(testDirectory) start = time.time() recommendation = stack_advisor.recommendComponentLayout(services, hosts) time_taken = time.time() - start print "time taken by current stack_advisor.py = " + str(time_taken) - self.assertEquals(recommendation, recommendation_old, - "current stack_advisor gives different results running on folder '" + folder_name + "'") + self.assertTrue(time_taken < 0.1) + http://git-wip-us.apache.org/repos/asf/ambari/blob/8258cf89/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py index 378522e..c89eee4 100644 --- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py @@ -21,13 +21,14 @@ import os import socket from unittest import TestCase from mock.mock import patch - +import unittest class TestHDP23StackAdvisor(TestCase): def setUp(self): import imp self.maxDiff = None + unittest.util._MAX_LENGTH=2000 self.testDirectory = os.path.dirname(os.path.abspath(__file__)) stackAdvisorPath = os.path.join(self.testDirectory, '../../../../../main/resources/stacks/stack_advisor.py') hdp206StackAdvisorPath = os.path.join(self.testDirectory, '../../../../../main/resources/stacks/HDP/2.0.6/services/stack_advisor.py') @@ -1966,12 +1967,27 @@ class TestHDP23StackAdvisor(TestCase): "ramPerContainer": 256 } expected = { - 'logfeeder-env': {'property_attributes': {'logfeeder_external_solr_kerberos_keytab': {'visible': 'false'}, - 'logfeeder_external_solr_kerberos_principal': {'visible': 'false'}}}, - 'logsearch-common-env': {'properties': {'logsearch_external_solr_kerberos_enabled': 'false'}, - 'property_attributes': {'logsearch_external_solr_kerberos_enabled': {'visible': 'false'}}}, - 'logsearch-env': {'property_attributes': {'logsearch_external_solr_kerberos_keytab': {'visible': 'false'}, - 'logsearch_external_solr_kerberos_principal': {'visible': 'false'}}}, + 'logfeeder-env': { + 'property_attributes': { + 'logfeeder_external_solr_kerberos_keytab': {'visible': 'false' + }, + 'logfeeder_external_solr_kerberos_principal': {'visible': 'false'} + } + }, + 'logsearch-common-env': { + 'properties': { + 'logsearch_external_solr_kerberos_enabled': 'false' + }, + 'property_attributes': { + 'logsearch_external_solr_kerberos_enabled': {'visible': 'false'} + } + }, + 'logsearch-env': { + 'property_attributes': { + 'logsearch_external_solr_kerberos_keytab': {'visible': 'false'}, + 'logsearch_external_solr_kerberos_principal': {'visible': 'false'} + } + }, 'logsearch-properties': { 'properties': { "logsearch.collection.service.logs.numshards" : "2",
