Repository: ambari Updated Branches: refs/heads/trunk 86bf4dcaa -> cee5c4215
AMBARI-16663: Refactor service_advisor apis to remove passing of stack_advisor (Lav Jain via mithmatt) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/cee5c421 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/cee5c421 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/cee5c421 Branch: refs/heads/trunk Commit: cee5c421599a8e21bccd95f538186ee651d3b90d Parents: 86bf4dc Author: Matt <[email protected]> Authored: Tue Jun 7 00:18:27 2016 -0700 Committer: Matt <[email protected]> Committed: Tue Jun 7 00:18:27 2016 -0700 ---------------------------------------------------------------------- .../HAWQ/2.0.0/service_advisor.py | 58 ++--- .../PXF/3.0.0/service_advisor.py | 17 +- .../stacks/HDP/2.0.6/services/stack_advisor.py | 75 ------ .../main/resources/stacks/service_advisor.py | 169 +------------ .../src/main/resources/stacks/stack_advisor.py | 241 +++++++++++-------- .../stacks/2.3/HAWQ/test_service_advisor.py | 44 ++-- .../stacks/2.3/PXF/test_service_advisor.py | 19 +- .../stacks/2.3/common/test_stack_advisor.py | 40 +-- 8 files changed, 225 insertions(+), 438 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/cee5c421/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py index 2c78bf1..f0e8b33 100644 --- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py +++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py @@ -36,7 +36,7 @@ except Exception as e: class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor): - def getHostsForMasterComponent(self, stackAdvisor, services, hosts, component, hostsList, hostsComponentsMap): + def getHostsForMasterComponent(self, services, hosts, component, hostsList): if component["StackServiceComponents"]["component_name"] == 'HAWQSTANDBY': # Do not recommend HAWQSTANDBY on single node cluster, or cluster with no active hosts if len(hostsList) <= 1: @@ -56,24 +56,21 @@ class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor): return availableHosts[:1] return [ambariServerHost] - return stackAdvisor.getHostsForMasterComponent(services, hosts, component, hostsList, hostsComponentsMap) + return super(HAWQ200ServiceAdvisor, self).getHostsForMasterComponent(services, hosts, component, hostsList) - def isComponentNotPreferableOnAmbariServerHost(self, componentName): - return componentName in ('HAWQMASTER', 'HAWQSTANDBY') + def getNotPreferableOnServerComponents(self): + return ['HAWQMASTER', 'HAWQSTANDBY'] - def getComponentLayoutScheme(self, componentName): - if componentName == 'HAWQMASTER': - return {6: 2, 31: 1, "else": 5} - - if componentName == 'HAWQSTANDBY': - return {6: 1, 31: 2, "else": 3} - - return None + def getComponentLayoutSchemes(self): + return { + 'HAWQMASTER': {6: 2, 31: 1, "else": 5}, + 'HAWQSTANDBY': {6: 1, 31: 2, "else": 3} + } - def colocateService(self, stackAdvisor, hostsComponentsMap, serviceComponents): + def colocateService(self, hostsComponentsMap, serviceComponents): # colocate HAWQSEGMENT with DATANODE, if no hosts have been allocated for HAWQSEGMENT hawqSegment = [component for component in serviceComponents if component["StackServiceComponents"]["component_name"] == "HAWQSEGMENT"][0] - if not stackAdvisor.isComponentHostsPopulated(hawqSegment): + if not self.isComponentHostsPopulated(hawqSegment): for hostName in hostsComponentsMap.keys(): hostComponents = hostsComponentsMap[hostName] if {"name": "DATANODE"} in hostComponents and {"name": "HAWQSEGMENT"} not in hostComponents: @@ -81,7 +78,7 @@ class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor): if {"name": "DATANODE"} not in hostComponents and {"name": "HAWQSEGMENT"} in hostComponents: hostComponents.remove({"name": "HAWQSEGMENT"}) - def getComponentLayoutValidations(self, stackAdvisor, services, hosts): + def getServiceComponentLayoutValidations(self, services, hosts): componentsListList = [service["components"] for service in services["services"]] componentsList = [item["StackServiceComponents"] for sublist in componentsListList for item in sublist] hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]] @@ -107,13 +104,13 @@ class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor): message = "HAWQ Master and HAWQ Standby Master cannot be deployed on the same host." items.append( { "type": 'host-component', "level": 'ERROR', "message": message, "component-name": 'HAWQSTANDBY', "host": hawqStandbyHosts[0] } ) - if len(hawqMasterHosts) == 1 and hostsCount > 1 and stackAdvisor.isLocalHost(hawqMasterHosts[0]): + if len(hawqMasterHosts) == 1 and hostsCount > 1 and self.isLocalHost(hawqMasterHosts[0]): message = "The default Postgres port (5432) on the Ambari Server conflicts with the default HAWQ Masters port. " \ "If you are using port 5432 for Postgres, you must either deploy the HAWQ Master on a different host " \ "or configure a different port for the HAWQ Masters in the HAWQ Configuration page." items.append( { "type": 'host-component', "level": 'WARN', "message": message, "component-name": 'HAWQMASTER', "host": hawqMasterHosts[0] } ) - if len(hawqStandbyHosts) == 1 and hostsCount > 1 and stackAdvisor.isLocalHost(hawqStandbyHosts[0]): + if len(hawqStandbyHosts) == 1 and hostsCount > 1 and self.isLocalHost(hawqStandbyHosts[0]): message = "The default Postgres port (5432) on the Ambari Server conflicts with the default HAWQ Masters port. " \ "If you are using port 5432 for Postgres, you must either deploy the HAWQ Standby Master on a different host " \ "or configure a different port for the HAWQ Masters in the HAWQ Configuration page." @@ -121,13 +118,13 @@ class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor): return items - def isHawqMasterComponentOnAmbariServer(self, stackAdvisor, services): + def isHawqMasterComponentOnAmbariServer(self, services): componentsListList = [service["components"] for service in services["services"]] componentsList = [item for sublist in componentsListList for item in sublist] hawqMasterComponentHosts = [hostname for component in componentsList if component["StackServiceComponents"]["component_name"] in ("HAWQMASTER", "HAWQSTANDBY") for hostname in component["StackServiceComponents"]["hostnames"]] - return any([stackAdvisor.isLocalHost(host) for host in hawqMasterComponentHosts]) + return any([self.isLocalHost(host) for host in hawqMasterComponentHosts]) - def getServiceConfigurationRecommendations(self, stackAdvisor, configurations, clusterData, services, hosts): + def getServiceConfigurationRecommendations(self, configurations, clusterData, services, hosts): putHdfsSiteProperty = self.putProperty(configurations, "hdfs-site", services) # Set dfs.allow.truncate to true @@ -151,7 +148,7 @@ class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor): putHawqSysctlEnvProperty = self.putProperty(configurations, "hawq-sysctl-env", services) # remove master port when master is colocated with Ambari server - if self.isHawqMasterComponentOnAmbariServer(stackAdvisor, services) and "hawq_master_address_port" in hawq_site: + if self.isHawqMasterComponentOnAmbariServer(services) and "hawq_master_address_port" in hawq_site: putHawqSiteProperty('hawq_master_address_port', '') # update query limits if segments are deployed @@ -170,7 +167,6 @@ class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor): buckets = factor * numSegments putHawqSiteProperty('default_hash_table_bucket_number', buckets) - # update YARN RM urls with the values from yarn-site if YARN is installed if "YARN" in servicesList and "yarn-site" in services["configurations"]: yarn_site = services["configurations"]["yarn-site"]["properties"] @@ -180,7 +176,6 @@ class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor): putHawqSiteProperty('hawq_rm_nvcore_limit_perseg', minHawqHostsCoreCount) - if "vm.overcommit_memory" in hawq_sysctl_env: MEM_THRESHOLD = 33554432 # 32GB, minHawqHostsMemory is represented in kB # Set the value for hawq_rm_memory_limit_perseg based on vm.overcommit value and RAM available on HAWQ Hosts @@ -236,14 +231,14 @@ class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor): def getHAWQYARNPropertyMapping(self): return { "hawq_rm_yarn_address": "yarn.resourcemanager.address", "hawq_rm_yarn_scheduler_address": "yarn.resourcemanager.scheduler.address" } - def getConfigurationsValidationItems(self, stackAdvisor, configurations, recommendedDefaults, services, hosts): + def getConfigurationsValidationItems(self, configurations, recommendedDefaults, services, hosts): siteName = "hawq-site" method = self.validateHAWQSiteConfigurations - items = self.validateConfigurationsForSite(stackAdvisor, configurations, recommendedDefaults, services, hosts, siteName, method) + items = self.validateConfigurationsForSite(configurations, recommendedDefaults, services, hosts, siteName, method) siteName = "hdfs-client" method = self.validateHAWQHdfsClientConfigurations - resultItems = self.validateConfigurationsForSite(stackAdvisor, configurations, recommendedDefaults, services, hosts, siteName, method) + resultItems = self.validateConfigurationsForSite(configurations, recommendedDefaults, services, hosts, siteName, method) items.extend(resultItems) return items @@ -270,13 +265,13 @@ class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor): "item": self.getErrorItem( "Multiple directories for " + display_name + " are not allowed.")}) - def validateHAWQSiteConfigurations(self, stackAdvisor, properties, recommendedDefaults, configurations, services, hosts): + def validateHAWQSiteConfigurations(self, properties, recommendedDefaults, configurations, services, hosts): hawq_site = properties validationItems = [] # 1. Check if HAWQ master/standby port numbers don't conflict with Ambari ports. Both Ambari and HAWQ use postgres DB and 5432 port. - if self.isHawqMasterComponentOnAmbariServer(stackAdvisor, services) and self.isHawqMasterPortConflict(configurations): + if self.isHawqMasterComponentOnAmbariServer(services) and self.isHawqMasterPortConflict(configurations): prop_name = 'hawq_master_address_port' validationItems.append({"config-name": prop_name, "item": self.getWarnItem( @@ -325,9 +320,9 @@ class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor): message = "Default buckets for Hash Distributed tables parameter value should not be greater than the value of Virtual Segments Limit per Query (Total) parameter, currently set to {0}.".format(hawq_site["hawq_rm_nvseg_perquery_limit"]) validationItems.append({"config-name": "default_hash_table_bucket_number", "item": self.getErrorItem(message)}) - return stackAdvisor.toConfigurationValidationProblems(validationItems, "hawq-site") + return self.toConfigurationValidationProblems(validationItems, "hawq-site") - def validateHAWQHdfsClientConfigurations(self, stackAdvisor, properties, recommendedDefaults, configurations, services, hosts): + def validateHAWQHdfsClientConfigurations(self, properties, recommendedDefaults, configurations, services, hosts): hdfs_client = properties validationItems = [] @@ -345,8 +340,7 @@ class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor): message = "{0} should be set to true (checked) for clusters with more than {1} HAWQ Segments" elif numSegments <= MIN_NUM_SEGMENT_THRESHOLD and value != 'FALSE': message = "{0} should be set to false (unchecked) for clusters with {1} or less HAWQ Segments" - if message: validationItems.append({"config-name": PROP_NAME, "item": self.getWarnItem(message.format(PROP_NAME, str(MIN_NUM_SEGMENT_THRESHOLD)))}) - return stackAdvisor.toConfigurationValidationProblems(validationItems, "hdfs-client") + return self.toConfigurationValidationProblems(validationItems, "hdfs-client") http://git-wip-us.apache.org/repos/asf/ambari/blob/cee5c421/ambari-server/src/main/resources/common-services/PXF/3.0.0/service_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/PXF/3.0.0/service_advisor.py b/ambari-server/src/main/resources/common-services/PXF/3.0.0/service_advisor.py index 9b34171..4089b75 100644 --- a/ambari-server/src/main/resources/common-services/PXF/3.0.0/service_advisor.py +++ b/ambari-server/src/main/resources/common-services/PXF/3.0.0/service_advisor.py @@ -33,10 +33,10 @@ except Exception as e: class PXF300ServiceAdvisor(service_advisor.ServiceAdvisor): - def colocateService(self, stackAdvisor, hostsComponentsMap, serviceComponents): + def colocateService(self, hostsComponentsMap, serviceComponents): # colocate PXF with NAMENODE and DATANODE, if no hosts have been allocated for PXF pxf = [component for component in serviceComponents if component["StackServiceComponents"]["component_name"] == "PXF"][0] - if not stackAdvisor.isComponentHostsPopulated(pxf): + if not self.isComponentHostsPopulated(pxf): for hostName in hostsComponentsMap.keys(): hostComponents = hostsComponentsMap[hostName] if ({"name": "NAMENODE"} in hostComponents or {"name": "DATANODE"} in hostComponents) \ @@ -46,13 +46,12 @@ class PXF300ServiceAdvisor(service_advisor.ServiceAdvisor): and {"name": "PXF"} in hostComponents: hostsComponentsMap[hostName].remove({"name": "PXF"}) - def getComponentLayoutValidations(self, stackAdvisor, services, hosts): + def getServiceComponentLayoutValidations(self, services, hosts): componentsListList = [service["components"] for service in services["services"]] componentsList = [item["StackServiceComponents"] for sublist in componentsListList for item in sublist] pxfHosts = self.getHosts(componentsList, "PXF") expectedPxfHosts = set(self.getHosts(componentsList, "NAMENODE") + self.getHosts(componentsList, "DATANODE")) - items = [] # Generate WARNING if any PXF is not colocated with NAMENODE or DATANODE @@ -65,7 +64,7 @@ class PXF300ServiceAdvisor(service_advisor.ServiceAdvisor): return items - def getServiceConfigurationRecommendations(self, stackAdvisor, configurations, clusterData, services, hosts): + def getServiceConfigurationRecommendations(self, configurations, clusterData, services, hosts): if "hbase-env" in services["configurations"]: hbase_env = services["configurations"]["hbase-env"]["properties"] if "content" in hbase_env: @@ -77,7 +76,7 @@ class PXF300ServiceAdvisor(service_advisor.ServiceAdvisor): putHbaseEnvProperty = self.putProperty(configurations, "hbase-env", services) putHbaseEnvProperty("content", content) - def validatePXFHBaseEnvConfigurations(self, stackAdvisor, properties, recommendedDefaults, configurations, services, hosts): + def validatePXFHBaseEnvConfigurations(self, properties, recommendedDefaults, configurations, services, hosts): # Check if HBASE_CLASSPATH should has the location of pxf-hbase.jar hbase_env = properties validationItems = [] @@ -85,10 +84,10 @@ class PXF300ServiceAdvisor(service_advisor.ServiceAdvisor): message = "HBASE_CLASSPATH must contain the location of pxf-hbase.jar" validationItems.append({"config-name": "content", "item": self.getWarnItem(message)}) - return stackAdvisor.toConfigurationValidationProblems(validationItems, "hbase-env") + return self.toConfigurationValidationProblems(validationItems, "hbase-env") - def getConfigurationsValidationItems(self, stackAdvisor, configurations, recommendedDefaults, services, hosts): + def getConfigurationsValidationItems(self, configurations, recommendedDefaults, services, hosts): siteName = "hbase-env" method = self.validatePXFHBaseEnvConfigurations - items = self.validateConfigurationsForSite(stackAdvisor, configurations, recommendedDefaults, services, hosts, siteName, method) + items = self.validateConfigurationsForSite(configurations, recommendedDefaults, services, hosts, siteName, method) return items \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/cee5c421/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py index 1dedf3d..aa62d1e 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py @@ -93,72 +93,6 @@ class HDP206StackAdvisor(DefaultStackAdvisor): "RANGER": self.recommendRangerConfigurations } - def putProperty(self, config, configType, services=None): - userConfigs = {} - changedConfigs = [] - # if services parameter, prefer values, set by user - if services: - if 'configurations' in services.keys(): - userConfigs = services['configurations'] - if 'changed-configurations' in services.keys(): - changedConfigs = services["changed-configurations"] - - if configType not in config: - config[configType] = {} - if"properties" not in config[configType]: - config[configType]["properties"] = {} - def appendProperty(key, value): - # If property exists in changedConfigs, do not override, use user defined property - if self.__isPropertyInChangedConfigs(configType, key, changedConfigs): - config[configType]["properties"][key] = userConfigs[configType]['properties'][key] - else: - config[configType]["properties"][key] = str(value) - return appendProperty - - def updateProperty(self, config, configType, services=None): - userConfigs = {} - changedConfigs = [] - # if services parameter, prefer values, set by user - if services: - if 'configurations' in services.keys(): - userConfigs = services['configurations'] - if 'changed-configurations' in services.keys(): - changedConfigs = services["changed-configurations"] - - if configType not in config: - config[configType] = {} - if"properties" not in config[configType]: - config[configType]["properties"] = {} - def updatePropertyWithCallback(key, value, callback): - # If property exists in changedConfigs, do not override, use user defined property - if self.__isPropertyInChangedConfigs(configType, key, changedConfigs): - config[configType]["properties"][key] = userConfigs[configType]['properties'][key] - else: - # Give the callback an empty string if the mapping doesn't exist - current_value = "" - if key in config[configType]["properties"]: - current_value = config[configType]["properties"][key] - - config[configType]["properties"][key] = callback(current_value, value) - return updatePropertyWithCallback - - def __isPropertyInChangedConfigs(self, configType, propertyName, changedConfigs): - for changedConfig in changedConfigs: - if changedConfig['type']==configType and changedConfig['name']==propertyName: - return True - return False - - def putPropertyAttribute(self, config, configType): - if configType not in config: - config[configType] = {} - def appendPropertyAttribute(key, attribute, attributeValue): - if "property_attributes" not in config[configType]: - config[configType]["property_attributes"] = {} - if key not in config[configType]["property_attributes"]: - config[configType]["property_attributes"][key] = {} - config[configType]["property_attributes"][key][attribute] = attributeValue if isinstance(attributeValue, list) else str(attributeValue) - return appendPropertyAttribute - def recommendYARNConfigurations(self, configurations, clusterData, services, hosts): putYarnProperty = self.putProperty(configurations, "yarn-site", services) putYarnPropertyAttribute = self.putPropertyAttribute(configurations, "yarn-site") @@ -1280,15 +1214,6 @@ class HDP206StackAdvisor(DefaultStackAdvisor): return self.toConfigurationValidationProblems(validationItems, "ams-hbase-env") - def validateServiceConfigurations(self, serviceName): - return self.getServiceConfigurationValidators().get(serviceName, None) - - def getWarnItem(self, message): - return {"level": "WARN", "message": message} - - def getErrorItem(self, message): - return {"level": "ERROR", "message": message} - def getPreferredMountPoints(self, hostInfo): # '/etc/resolv.conf', '/etc/hostname', '/etc/hosts' are docker specific mount points http://git-wip-us.apache.org/repos/asf/ambari/blob/cee5c421/ambari-server/src/main/resources/stacks/service_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/service_advisor.py b/ambari-server/src/main/resources/stacks/service_advisor.py index 3d6c293..7a3c7e9 100644 --- a/ambari-server/src/main/resources/stacks/service_advisor.py +++ b/ambari-server/src/main/resources/stacks/service_advisor.py @@ -46,55 +46,12 @@ convention listed above. For examples see: common-services/HAWQ/2.0.0/service_advisor.py and common-services/PXF/3.0.0/service_advisor.py """ -class ServiceAdvisor(object): - """ - Abstract class implemented by all service advisors. - """ - - """ - Provides a scheme for laying out a given component on different number of hosts. - It should return a json object such as {6: 2, 31: 1, "else": 5}. If a service - contains more than one component, it may have a statement such as: - - if componentName == 'HAWQMASTER': - return {6: 2, 31: 1, "else": 5} - - return None - """ - def getComponentLayoutScheme(self, componentName): - return None +from stack_advisor import DefaultStackAdvisor +class ServiceAdvisor(DefaultStackAdvisor): """ - Returns the cardinality of the component as a json object with min and max attributes. - Example: {"min": 1, "max": 1} - """ - def getComponentCardinality(self, componentName): - return {"min": 1, "max": 1} - - """ - Returns True if the component should avoid being configured on the ambari server host. - """ - def isComponentNotPreferableOnAmbariServerHost(self, componentName): - return False - - """ - Return True if the component is not considered valuable, otherwise returns False. - Hosts which are only running non valuable components are considered to be not used. - """ - def isComponentNotValuable(self, component): - return False - - """ - Returns True if the component should use its cardinatity to determine its layout. - """ - def isComponentUsingCardinalityForLayout(self, componentName): - return False - - """ - Returns True if the component is a master component with multiple instances. + Abstract class implemented by all service advisors. """ - def isMasterComponentWithMultipleInstances(self, component): - return False """ If any components of the service should be colocated with other services, @@ -102,7 +59,7 @@ class ServiceAdvisor(object): # colocate HAWQSEGMENT with DATANODE, if no hosts have been allocated for HAWQSEGMENT hawqSegment = [component for component in serviceComponents if component["StackServiceComponents"]["component_name"] == "HAWQSEGMENT"][0] - if not stackAdvisor.isComponentHostsPopulated(hawqSegment): + if not self.isComponentHostsPopulated(hawqSegment): for hostName in hostsComponentsMap.keys(): hostComponents = hostsComponentsMap[hostName] if {"name": "DATANODE"} in hostComponents and {"name": "HAWQSEGMENT"} not in hostComponents: @@ -110,7 +67,7 @@ class ServiceAdvisor(object): if {"name": "DATANODE"} not in hostComponents and {"name": "HAWQSEGMENT"} in hostComponents: hostComponents.remove({"name": "HAWQSEGMENT"}) """ - def colocateService(self, stackAdvisor, hostsComponentsMap, serviceComponents): + def colocateService(self, hostsComponentsMap, serviceComponents): pass """ @@ -118,7 +75,7 @@ class ServiceAdvisor(object): This should be similar to any of the recommendXXXXConfigurations functions in the stack_advisor.py such as recommendYARNConfigurations(). """ - def getServiceConfigurationRecommendations(self, stackAdvisor, configurations, clusterSummary, services, hosts): + def getServiceConfigurationRecommendations(self, configurations, clusterSummary, services, hosts): pass """ @@ -126,7 +83,7 @@ class ServiceAdvisor(object): This should detect validation issues which are different than those the stack_advisor.py detects. The default validations are in stack_advisor.py getComponentLayoutValidations function. """ - def getComponentLayoutValidations(self, stackAdvisor, services, hosts): + def getServiceComponentLayoutValidations(self, services, hosts): return [] """ @@ -134,115 +91,5 @@ class ServiceAdvisor(object): This should be similar to any of the validateXXXXConfigurations functions in the stack_advisor.py such as validateHDFSConfigurations. """ - def getConfigurationsValidationItems(self, stackAdvisor, configurations, recommendedDefaults, services, hosts): + def getServiceConfigurationsValidationItems(self, configurations, recommendedDefaults, services, hosts): return [] - - """ - If the service needs to any processing differently than the stack_advisor.py getHostsForMasterComponents - function, then this logic should be added in this function. - """ - def getHostsForMasterComponent(self, stackAdvisor, services, hosts, component, hostsList, hostsComponentsMap): - return stackAdvisor.getHostsForMasterComponent(services, hosts, component, hostsList, hostsComponentsMap) - - """ - If the service needs to any processing differently than the stack_advisor.py getHostsForSlaveComponents - function, then this logic should be added in this function. - """ - def getHostsForSlaveComponent(self, stackAdvisor, services, hosts, component, hostsList, hostsComponentsMap, freeHosts): - return stackAdvisor.getHostsForSlaveComponent(services, hosts, component, hostsList, hostsComponentsMap, freeHosts) - - - """ - Utility methods - """ - - """ - Utility method used for validation warnings. - """ - def getWarnItem(self, message): - return {"level": "WARN", "message": message} - - """ - Utility method used for validation errors. - """ - def getErrorItem(self, message): - return {"level": "ERROR", "message": message} - - """ - Returns the hosts which are running the given component. - """ - def getHosts(self, componentsList, componentName): - hostNamesList = [component["hostnames"] for component in componentsList if component["component_name"] == componentName] - return hostNamesList[0] if len(hostNamesList) > 0 else [] - - """ - Utility method for setting a configuration property value. - """ - def putProperty(self, config, configType, services=None): - userConfigs = {} - changedConfigs = [] - # if services parameter, prefer values, set by user - if services: - if 'configurations' in services.keys(): - userConfigs = services['configurations'] - if 'changed-configurations' in services.keys(): - changedConfigs = services["changed-configurations"] - - if configType not in config: - config[configType] = {} - if"properties" not in config[configType]: - config[configType]["properties"] = {} - def appendProperty(key, value): - # If property exists in changedConfigs, do not override, use user defined property - if self.__isPropertyInChangedConfigs(configType, key, changedConfigs): - config[configType]["properties"][key] = userConfigs[configType]['properties'][key] - else: - config[configType]["properties"][key] = str(value) - return appendProperty - - """ - Utility method to determine if the configuration property value has been changed. - """ - def __isPropertyInChangedConfigs(self, configType, propertyName, changedConfigs): - for changedConfig in changedConfigs: - if changedConfig['type']==configType and changedConfig['name']==propertyName: - return True - return False - - """ - Utility method for setting a configuration property attribute. - """ - def putPropertyAttribute(self, config, configType): - if configType not in config: - config[configType] = {} - def appendPropertyAttribute(key, attribute, attributeValue): - if "property_attributes" not in config[configType]: - config[configType]["property_attributes"] = {} - if key not in config[configType]["property_attributes"]: - config[configType]["property_attributes"][key] = {} - config[configType]["property_attributes"][key][attribute] = attributeValue if isinstance(attributeValue, list) else str(attributeValue) - return appendPropertyAttribute - - """ - Utility method to validate configuration settings for a given configuration file. - This function will call the method provided. - """ - def validateConfigurationsForSite(self, stackAdvisor, configurations, recommendedDefaults, services, hosts, siteName, method): - if siteName in recommendedDefaults: - siteProperties = self.getSiteProperties(configurations, siteName) - if siteProperties is not None: - siteRecommendations = recommendedDefaults[siteName]["properties"] - print("SiteName: %s, method: %s\n" % (siteName, method.__name__)) - print("Site properties: %s\n" % str(siteProperties)) - print("Recommendations: %s\n********\n" % str(siteRecommendations)) - return method(stackAdvisor, siteProperties, siteRecommendations, configurations, services, hosts) - return [] - - """ - Utility method used to return all the properties from a given configuration file. - """ - def getSiteProperties(self, configurations, siteName): - siteConfig = configurations.get(siteName) - if siteConfig is None: - return None - return siteConfig.get("properties") http://git-wip-us.apache.org/repos/asf/ambari/blob/cee5c421/ambari-server/src/main/resources/stacks/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/stack_advisor.py b/ambari-server/src/main/resources/stacks/stack_advisor.py index beba225..d8685c3 100644 --- a/ambari-server/src/main/resources/stacks/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/stack_advisor.py @@ -17,8 +17,11 @@ See the License for the specific language governing permissions and limitations under the License. """ -import socket +import imp +import os import re +import socket +import traceback class StackAdvisor(object): """ @@ -296,10 +299,6 @@ class StackAdvisor(object): pass - - - - class DefaultStackAdvisor(StackAdvisor): """ Default stack advisor implementation. @@ -343,32 +342,22 @@ class DefaultStackAdvisor(StackAdvisor): self.serviceAdvisorsDict[componentName] = self.serviceAdvisorsDict[serviceName] def instantiateServiceAdvisor(self, service): - import imp - import os - import traceback serviceName = service["StackServices"]["service_name"] className = service["StackServices"]["advisor_name"] if "advisor_name" in service["StackServices"] else None path = service["StackServices"]["advisor_path"] if "advisor_path" in service["StackServices"] else None - if path is None or className is None: - return None - - if not os.path.exists(path): - return None - - try: - serviceAdvisor = None - with open(path, 'rb') as fp: - serviceAdvisor = imp.load_module('service_advisor_impl', fp, path, ('.py', 'rb', imp.PY_SOURCE)) - if hasattr(serviceAdvisor, className): - print "ServiceAdvisor implementation for service {0} was loaded".format(serviceName) - clazz = getattr(serviceAdvisor, className) - return clazz() + if path is not None and os.path.exists(path) and className is not None: + try: + with open(path, 'rb') as fp: + serviceAdvisor = imp.load_module('service_advisor_impl', fp, path, ('.py', 'rb', imp.PY_SOURCE)) + if hasattr(serviceAdvisor, className): + print "ServiceAdvisor implementation for service {0} was loaded".format(serviceName) + return getattr(serviceAdvisor, className)() - except Exception as e: - traceback.print_exc() - print "Failed to load or create ServiceAdvisor implementation for service {0}".format(serviceName) + except Exception as e: + traceback.print_exc() + print "Failed to load or create ServiceAdvisor implementation for service {0}".format(serviceName) return None @@ -420,11 +409,8 @@ class DefaultStackAdvisor(StackAdvisor): serviceAdvisor = self.getServiceAdvisor(serviceName) for component in masterComponents: componentName = component["StackServiceComponents"]["component_name"] - hostsForComponent = [] - if serviceAdvisor is None: - hostsForComponent = self.getHostsForMasterComponent(services, hosts, component, hostsList, hostsComponentsMap) - else: - hostsForComponent = serviceAdvisor.getHostsForMasterComponent(self, services, hosts, component, hostsList, hostsComponentsMap) + advisor = serviceAdvisor if serviceAdvisor is not None else self + hostsForComponent = advisor.getHostsForMasterComponent(services, hosts, component, hostsList) #extend 'hostsComponentsMap' with 'hostsForComponent' for hostName in hostsForComponent: @@ -445,11 +431,8 @@ class DefaultStackAdvisor(StackAdvisor): serviceAdvisor = self.getServiceAdvisor(serviceName) for component in slaveClientComponents: componentName = component["StackServiceComponents"]["component_name"] - hostsForComponent = [] - if serviceAdvisor is None: - hostsForComponent = self.getHostsForSlaveComponent(services, hosts, component, hostsList, hostsComponentsMap, freeHosts) - else: - hostsForComponent = serviceAdvisor.getHostsForSlaveComponent(self, services, hosts, component, hostsList, hostsComponentsMap, freeHosts) + advisor = serviceAdvisor if serviceAdvisor is not None else self + hostsForComponent = advisor.getHostsForSlaveComponent(services, hosts, component, hostsList, freeHosts) #extend 'hostsComponentsMap' with 'hostsForComponent' for hostName in hostsForComponent: @@ -464,7 +447,7 @@ class DefaultStackAdvisor(StackAdvisor): serviceAdvisor = self.getServiceAdvisor(serviceName) if serviceAdvisor is not None: serviceComponents = [component for component in service["components"]] - serviceAdvisor.colocateService(self, hostsComponentsMap, serviceComponents) + serviceAdvisor.colocateService(hostsComponentsMap, serviceComponents) #prepare 'host-group's from 'hostsComponentsMap' host_groups = recommendations["blueprint"]["host_groups"] @@ -477,11 +460,8 @@ class DefaultStackAdvisor(StackAdvisor): bindings.append( { "name": host_group_name, "hosts": [{ "fqdn": key }] } ) return recommendations - pass - - def getHostsForMasterComponent(self, services, hosts, component, hostsList, hostsComponentsMap): - componentName = component["StackServiceComponents"]["component_name"] + def getHostsForMasterComponent(self, services, hosts, component, hostsList): if self.isComponentHostsPopulated(component): return component["StackServiceComponents"]["hostnames"] @@ -499,18 +479,15 @@ class DefaultStackAdvisor(StackAdvisor): return [self.getHostForComponent(component, hostsList)] - def getHostsForSlaveComponent(self, services, hosts, component, hostsList, hostsComponentsMap, freeHosts): - componentName = component["StackServiceComponents"]["component_name"] - + def getHostsForSlaveComponent(self, services, hosts, component, hostsList, freeHosts): if component["StackServiceComponents"]["cardinality"] == "ALL": return hostsList - componentIsPopulated = self.isComponentHostsPopulated(component) - if componentIsPopulated: + if self.isComponentHostsPopulated(component): return component["StackServiceComponents"]["hostnames"] hostsForComponent = [] - + componentName = component["StackServiceComponents"]["component_name"] if self.isSlaveComponent(component): cardinality = str(component["StackServiceComponents"]["cardinality"]) if self.isComponentUsingCardinalityForLayout(componentName) and cardinality: @@ -538,10 +515,6 @@ class DefaultStackAdvisor(StackAdvisor): return hostsForComponent def isComponentUsingCardinalityForLayout(self, componentName): - serviceAdvisor = self.getServiceAdvisor(componentName) - if serviceAdvisor is not None: - return serviceAdvisor.isComponentUsingCardinalityForLayout(componentName) - return False def createValidationResponse(self, services, validationItems): @@ -579,12 +552,12 @@ class DefaultStackAdvisor(StackAdvisor): serviceName = service["StackServices"]["service_name"] serviceAdvisor = self.getServiceAdvisor(serviceName) if serviceAdvisor is not None: - items.extend(serviceAdvisor.getComponentLayoutValidations(self, services, hosts)) + items.extend(serviceAdvisor.getServiceComponentLayoutValidations(services, hosts)) return items def getConfigurationClusterSummary(self, servicesList, hosts, components, services): - pass + return {} def validateClusterConfigurations(self, configurations, services, hosts): validationItems = [] @@ -653,25 +626,29 @@ class DefaultStackAdvisor(StackAdvisor): self.validateMinMax(items, recommendedDefaults, configurations) return items + def validateConfigurationsForSite(self, configurations, recommendedDefaults, services, hosts, siteName, method): + if siteName in recommendedDefaults: + siteProperties = self.getSiteProperties(configurations, siteName) + if siteProperties is not None: + siteRecommendations = recommendedDefaults[siteName]["properties"] + print("SiteName: %s, method: %s" % (siteName, method.__name__)) + print("Site properties: %s" % str(siteProperties)) + print("Recommendations: %s" % str(siteRecommendations)) + return method(siteProperties, siteRecommendations, configurations, services, hosts) + return [] + def getConfigurationsValidationItemsForService(self, configurations, recommendedDefaults, service, services, hosts): items = [] serviceName = service["StackServices"]["service_name"] validator = self.validateServiceConfigurations(serviceName) if validator is not None: for siteName, method in validator.items(): - if siteName in recommendedDefaults: - siteProperties = getSiteProperties(configurations, siteName) - if siteProperties is not None: - siteRecommendations = recommendedDefaults[siteName]["properties"] - print("SiteName: %s, method: %s\n" % (siteName, method.__name__)) - print("Site properties: %s\n" % str(siteProperties)) - print("Recommendations: %s\n********\n" % str(siteRecommendations)) - resultItems = method(siteProperties, siteRecommendations, configurations, services, hosts) - items.extend(resultItems) + resultItems = self.validateConfigurationsForSite(configurations, recommendedDefaults, services, hosts, siteName, method) + items.extend(resultItems) serviceAdvisor = self.getServiceAdvisor(serviceName) if serviceAdvisor is not None: - items.extend(serviceAdvisor.getConfigurationsValidationItems(self, configurations, recommendedDefaults, services, hosts)) + items.extend(serviceAdvisor.getServiceConfigurationsValidationItems(configurations, recommendedDefaults, services, hosts)) return items @@ -710,7 +687,7 @@ class DefaultStackAdvisor(StackAdvisor): else: serviceAdvisor = self.getServiceAdvisor(serviceName) if serviceAdvisor is not None: - serviceAdvisor.getServiceConfigurationRecommendations(self, configurations, cgClusterSummary, cgServices, cgHosts) + serviceAdvisor.getServiceConfigurationRecommendations(configurations, cgClusterSummary, cgServices, cgHosts) cgRecommendation = { "configurations": {}, @@ -781,7 +758,7 @@ class DefaultStackAdvisor(StackAdvisor): else: serviceAdvisor = self.getServiceAdvisor(serviceName) if serviceAdvisor is not None: - serviceAdvisor.getServiceConfigurationRecommendations(self, configurations, clusterSummary, services, hosts) + serviceAdvisor.getServiceConfigurationRecommendations(configurations, clusterSummary, services, hosts) return recommendations @@ -818,19 +795,11 @@ class DefaultStackAdvisor(StackAdvisor): def isMasterComponentWithMultipleInstances(self, component): componentName = self.getComponentName(component) - serviceAdvisor = self.getServiceAdvisor(componentName) - if serviceAdvisor is not None: - return serviceAdvisor.isMasterComponentWithMultipleInstances(componentName) - masters = self.getMastersWithMultipleInstances() return componentName in masters def isComponentNotValuable(self, component): componentName = self.getComponentName(component) - serviceAdvisor = self.getServiceAdvisor(componentName) - if serviceAdvisor is not None: - return serviceAdvisor.isComponentNotValuable(componentName) - service = self.getNotValuableComponents() return componentName in service @@ -840,20 +809,16 @@ class DefaultStackAdvisor(StackAdvisor): # Helper dictionaries def getComponentCardinality(self, componentName): - serviceAdvisor = self.getServiceAdvisor(componentName) - if serviceAdvisor is not None: - return serviceAdvisor.getComponentCardinality(componentName) - return self.getCardinalitiesDict().get(componentName, {"min": 1, "max": 1}) def getHostForComponent(self, component, hostsList): - if (len(hostsList) == 0): + if len(hostsList) == 0: return None componentName = self.getComponentName(component) if len(hostsList) != 1: - scheme = self.getComponentLayoutScheme(componentName) + scheme = self.getComponentLayoutSchemes().get(componentName, None) if scheme is not None: hostIndex = next((index for key, index in scheme.iteritems() if isinstance(key, ( int, long )) and len(hostsList) < key), scheme['else']) else: @@ -863,30 +828,11 @@ class DefaultStackAdvisor(StackAdvisor): return host return hostsList[0] - def getComponentLayoutScheme(self, componentName): - """ - Provides a scheme for laying out given component on different number of hosts. - """ - serviceAdvisor = self.getServiceAdvisor(componentName) - if serviceAdvisor is not None: - return serviceAdvisor.getComponentLayoutScheme(componentName) - - return self.getComponentLayoutSchemes().get(componentName, None) - def getComponentName(self, component): return self.getComponentAttribute(component, "component_name") - def isComponentNotPreferableOnAmbariServerHost(self, component): - componentName = self.getComponentName(component) - serviceAdvisor = self.getServiceAdvisor(componentName) - if serviceAdvisor is not None: - return serviceAdvisor.isComponentNotPreferableOnAmbariServerHost(componentName) - - service = self.getNotPreferableOnServerComponents() - return componentName in service - def isHostSuitableForComponent(self, host, component): - return not (self.isComponentNotPreferableOnAmbariServerHost(component) and self.isLocalHost(host)) + return not (self.getComponentName(component) in self.getNotPreferableOnServerComponents() and self.isLocalHost(host)) def getMastersWithMultipleInstances(self): return [] @@ -909,13 +855,24 @@ class DefaultStackAdvisor(StackAdvisor): """ return {} + """ + Utility method used for validation warnings. + """ + def getWarnItem(self, message): + return {"level": "WARN", "message": message} + + """ + Utility method used for validation errors. + """ + def getErrorItem(self, message): + return {"level": "ERROR", "message": message} + def getComponentHostNames(self, servicesDict, serviceName, componentName): for service in servicesDict["services"]: if service["StackServices"]["service_name"] == serviceName: for component in service['components']: if component["StackServiceComponents"]["component_name"] == componentName: return component["StackServiceComponents"]["hostnames"] - pass def recommendConfigurationDependencies(self, services, hosts): result = self.recommendConfigurations(services, hosts) @@ -1003,8 +960,84 @@ class DefaultStackAdvisor(StackAdvisor): return cmp(normalize(version1), normalize(version2)) pass -def getSiteProperties(configurations, siteName): - siteConfig = configurations.get(siteName) - if siteConfig is None: - return None - return siteConfig.get("properties") + def getSiteProperties(self, configurations, siteName): + siteConfig = configurations.get(siteName) + if siteConfig is None: + return None + return siteConfig.get("properties") + + def putProperty(self, config, configType, services=None): + userConfigs = {} + changedConfigs = [] + # if services parameter, prefer values, set by user + if services: + if 'configurations' in services.keys(): + userConfigs = services['configurations'] + if 'changed-configurations' in services.keys(): + changedConfigs = services["changed-configurations"] + + if configType not in config: + config[configType] = {} + if"properties" not in config[configType]: + config[configType]["properties"] = {} + def appendProperty(key, value): + # If property exists in changedConfigs, do not override, use user defined property + if self.__isPropertyInChangedConfigs(configType, key, changedConfigs): + config[configType]["properties"][key] = userConfigs[configType]['properties'][key] + else: + config[configType]["properties"][key] = str(value) + return appendProperty + + def __isPropertyInChangedConfigs(self, configType, propertyName, changedConfigs): + for changedConfig in changedConfigs: + if changedConfig['type']==configType and changedConfig['name']==propertyName: + return True + return False + + def updateProperty(self, config, configType, services=None): + userConfigs = {} + changedConfigs = [] + # if services parameter, prefer values, set by user + if services: + if 'configurations' in services.keys(): + userConfigs = services['configurations'] + if 'changed-configurations' in services.keys(): + changedConfigs = services["changed-configurations"] + + if configType not in config: + config[configType] = {} + if "properties" not in config[configType]: + config[configType]["properties"] = {} + + def updatePropertyWithCallback(key, value, callback): + # If property exists in changedConfigs, do not override, use user defined property + if self.__isPropertyInChangedConfigs(configType, key, changedConfigs): + config[configType]["properties"][key] = userConfigs[configType]['properties'][key] + else: + # Give the callback an empty string if the mapping doesn't exist + current_value = "" + if key in config[configType]["properties"]: + current_value = config[configType]["properties"][key] + + config[configType]["properties"][key] = callback(current_value, value) + + return updatePropertyWithCallback + + def putPropertyAttribute(self, config, configType): + if configType not in config: + config[configType] = {} + def appendPropertyAttribute(key, attribute, attributeValue): + if "property_attributes" not in config[configType]: + config[configType]["property_attributes"] = {} + if key not in config[configType]["property_attributes"]: + config[configType]["property_attributes"][key] = {} + config[configType]["property_attributes"][key][attribute] = attributeValue if isinstance(attributeValue, list) else str(attributeValue) + return appendPropertyAttribute + + + """ + Returns the hosts which are running the given component. + """ + def getHosts(self, componentsList, componentName): + hostNamesList = [component["hostnames"] for component in componentsList if component["component_name"] == componentName] + return hostNamesList[0] if len(hostNamesList) > 0 else [] http://git-wip-us.apache.org/repos/asf/ambari/blob/cee5c421/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py b/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py index 384b8be..12f4fa1 100644 --- a/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py +++ b/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py @@ -28,16 +28,11 @@ class TestHAWQ200ServiceAdvisor(TestCase): self.testDirectory = os.path.dirname(os.path.abspath(__file__)) stackAdvisorPath = os.path.join(self.testDirectory, '../../../../../main/resources/stacks/stack_advisor.py') hawq200ServiceAdvisorPath = os.path.join(self.testDirectory, '../../../../../main/resources/common-services/HAWQ/2.0.0/service_advisor.py') - with open(stackAdvisorPath, 'rb') as fp: - stack_advisor = imp.load_module('stack_advisor', fp, stackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE)) + imp.load_module('stack_advisor', fp, stackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE)) with open(hawq200ServiceAdvisorPath, 'rb') as fp: - service_advisor = imp.load_module('stack_advisor_impl', fp, hawq200ServiceAdvisorPath, ('.py', 'rb', imp.PY_SOURCE)) - - stackAdvisorClass = getattr(stack_advisor, 'DefaultStackAdvisor') - self.stackAdvisor = stackAdvisorClass() - - serviceAdvisorClass = getattr(service_advisor, 'HAWQ200ServiceAdvisor') + service_advisor_impl = imp.load_module('service_advisor_impl', fp, hawq200ServiceAdvisorPath, ('.py', 'rb', imp.PY_SOURCE)) + serviceAdvisorClass = getattr(service_advisor_impl, 'HAWQ200ServiceAdvisor') self.serviceAdvisor = serviceAdvisorClass() @@ -85,8 +80,7 @@ class TestHAWQ200ServiceAdvisor(TestCase): # HAWQMASTER is placed on c6403.ambari.apache.org # There are 4 available hosts in the cluster # Recommend HAWQSTANDBY on next available host, c6402.ambari.apache.org - self.stackAdvisor.loadServiceAdvisors(services) - standbyHosts = self.serviceAdvisor.getHostsForMasterComponent(self.stackAdvisor, services, None, component, hostsList, None) + standbyHosts = self.serviceAdvisor.getHostsForMasterComponent(services, None, component, hostsList) self.assertEquals(standbyHosts, ["c6402.ambari.apache.org"]) # Case 2: @@ -95,7 +89,7 @@ class TestHAWQ200ServiceAdvisor(TestCase): # There are 4 available hosts in the cluster # Recommend HAWQSTANDBY on next available host, c6403.ambari.apache.org services["services"][0]["components"][0]["StackServiceComponents"]["hostnames"] = ["c6402.ambari.apache.org"] - standbyHosts = self.serviceAdvisor.getHostsForMasterComponent(self.stackAdvisor, services, None, component, hostsList, None) + standbyHosts = self.serviceAdvisor.getHostsForMasterComponent(services, None, component, hostsList) self.assertEquals(standbyHosts, ["c6403.ambari.apache.org"]) # Case 3: @@ -104,7 +98,7 @@ class TestHAWQ200ServiceAdvisor(TestCase): # There are 2 available hosts in the cluster # Recommend HAWQSTANDBY on a host which does not have HAWQMASTER, c6401.ambari.apache.org hostsList = ["c6401.ambari.apache.org", "c6402.ambari.apache.org"] - standbyHosts = self.serviceAdvisor.getHostsForMasterComponent(self.stackAdvisor, services, None, component, hostsList, None) + standbyHosts = self.serviceAdvisor.getHostsForMasterComponent(services, None, component, hostsList) self.assertEquals(standbyHosts, ["c6401.ambari.apache.org"]) # Case 4: @@ -114,7 +108,7 @@ class TestHAWQ200ServiceAdvisor(TestCase): # Do not recommend HAWQSTANDBY on a single node cluster hostsList = ["c6401.ambari.apache.org"] services["services"][0]["components"][0]["StackServiceComponents"]["hostnames"] = ["c6401.ambari.apache.org"] - standbyHosts = self.serviceAdvisor.getHostsForMasterComponent(self.stackAdvisor, services, None, component, hostsList, None) + standbyHosts = self.serviceAdvisor.getHostsForMasterComponent(services, None, component, hostsList) self.assertEquals(standbyHosts, []) # Case 5: @@ -126,7 +120,7 @@ class TestHAWQ200ServiceAdvisor(TestCase): hostsList = ["c6401.ambari.apache.org", "c6402.ambari.apache.org", "c6403.ambari.apache.org"] services["services"][0]["components"][0]["StackServiceComponents"]["hostnames"] = ["c6402.ambari.apache.org"] services["services"][0]["components"][1]["StackServiceComponents"]["hostnames"] = ["c6401.ambari.apache.org"] - standbyHosts = self.serviceAdvisor.getHostsForMasterComponent(self.stackAdvisor, services, None, component, hostsList, None) + standbyHosts = self.serviceAdvisor.getHostsForMasterComponent(services, None, component, hostsList) self.assertEquals(standbyHosts, ["c6401.ambari.apache.org"]) @@ -220,33 +214,33 @@ class TestHAWQ200ServiceAdvisor(TestCase): # HAWQ Hosts Core Count: c6401.ambari.apache.org - 2, c6402.ambari.apache.org - 4, c6404.ambari.apache.org - 2 # hawq_global_rm_type: yarn # Non HAWQ Hosts Core Count: c6401.ambari.apache.org - 1 - self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts) + self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts) self.assertEquals(configurations["hawq-site"]["properties"]["hawq_rm_nvcore_limit_perseg"], "2") ## Test if vm.overcommit_memory is set correctly # Case 1: All machines have total_mem above 32GB (total_mem >= 33554432) - self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts) + self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts) self.assertEquals(configurations["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"], "2") # Case 2: One machine has total_mem below 32GB hosts["items"][0]["Hosts"]["total_mem"] = 33554431 services["configurations"]["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"] = "65535MB" - self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts) + self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts) self.assertEquals(configurations["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"], "1") ## Test if hawq_rm_memory_limit_perseg is set correctly # Case 1: Minimum host memory is ~ 2 GB (2048MB), recommended val must be .75% of 2GB as vm.overcommit_memory = 1 and in MB hosts["items"][0]["Hosts"]["total_mem"] = 2097152 - self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts) + self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts) self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"], "1536MB") # Case 2: Minimum host memory is ~ 16 GB, recommended val must be .75% of 16GB as vm.overcommit_memory = 1 and in GB hosts["items"][0]["Hosts"]["total_mem"] = 16777216 hosts["items"][1]["Hosts"]["total_mem"] = 26777216 hosts["items"][3]["Hosts"]["total_mem"] = 36777216 - self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts) + self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts) self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"], "12GB") # Case 2: Minimum host memory is ~ 64 GB, recommended val must be .75% of 32GB as vm.overcommit_memory = 2 and in GB @@ -254,21 +248,21 @@ class TestHAWQ200ServiceAdvisor(TestCase): hosts["items"][1]["Hosts"]["total_mem"] = 77108864 hosts["items"][3]["Hosts"]["total_mem"] = 87108864 services["configurations"]["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"] = "65535MB" - self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts) + self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts) self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"], "24GB") # Case 4: Minimum host memory is ~ 512 GB, recommended val must be .85% of 256GB as vm.overcommit_memory = 2 and in GB hosts["items"][0]["Hosts"]["total_mem"] = 536870912 hosts["items"][1]["Hosts"]["total_mem"] = 636870912 hosts["items"][3]["Hosts"]["total_mem"] = 736870912 - self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts) + self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts) self.assertEquals(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"], "218GB") # Case 5: Minimum host memory is ~ 1024 GB, recommended val must be .95% of 512GB as vm.overcommit_memory = 2 and in GB hosts["items"][0]["Hosts"]["total_mem"] = 1073741824 hosts["items"][1]["Hosts"]["total_mem"] = 2073741824 hosts["items"][3]["Hosts"]["total_mem"] = 3073741824 - self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts) + self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts) self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"], "436GB") # Case 6: Minimum host memory is ~ 1024 GB, vm.overcommit_ratio = 75, vm.overcommit_memory = 2 @@ -277,7 +271,7 @@ class TestHAWQ200ServiceAdvisor(TestCase): hosts["items"][1]["Hosts"]["total_mem"] = 2073741824 hosts["items"][3]["Hosts"]["total_mem"] = 3073741824 services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_ratio"] = 75 - self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts) + self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts) self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"], "730GB") ## Test if the properties are set to visible / invisible based on the value of hawq_global_rm_type @@ -290,7 +284,7 @@ class TestHAWQ200ServiceAdvisor(TestCase): "hawq_rm_yarn_queue_name": "true", "hawq_rm_yarn_scheduler_address": "true", "hawq_rm_yarn_address": "true"} - self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts) + self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts) for property, status in properties_visible_status.iteritems(): self.assertEqual(configurations["hawq-site"]["property_attributes"][property]["visible"], status) @@ -302,6 +296,6 @@ class TestHAWQ200ServiceAdvisor(TestCase): "hawq_rm_yarn_queue_name": "false", "hawq_rm_yarn_scheduler_address": "false", "hawq_rm_yarn_address": "false"} - self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts) + self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts) for property, status in properties_visible_status.iteritems(): self.assertEqual(configurations["hawq-site"]["property_attributes"][property]["visible"], status) http://git-wip-us.apache.org/repos/asf/ambari/blob/cee5c421/ambari-server/src/test/python/stacks/2.3/PXF/test_service_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/PXF/test_service_advisor.py b/ambari-server/src/test/python/stacks/2.3/PXF/test_service_advisor.py index c3a63cc..4ea3bfb 100644 --- a/ambari-server/src/test/python/stacks/2.3/PXF/test_service_advisor.py +++ b/ambari-server/src/test/python/stacks/2.3/PXF/test_service_advisor.py @@ -27,16 +27,11 @@ class TestPXF300ServiceAdvisor(TestCase): self.testDirectory = os.path.dirname(os.path.abspath(__file__)) stackAdvisorPath = os.path.join(self.testDirectory, '../../../../../main/resources/stacks/stack_advisor.py') pxf300ServiceAdvisorPath = os.path.join(self.testDirectory, '../../../../../main/resources/common-services/PXF/3.0.0/service_advisor.py') - with open(stackAdvisorPath, 'rb') as fp: - stack_advisor = imp.load_module('stack_advisor', fp, stackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE)) + imp.load_module('stack_advisor', fp, stackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE)) with open(pxf300ServiceAdvisorPath, 'rb') as fp: - service_advisor = imp.load_module('stack_advisor_impl', fp, pxf300ServiceAdvisorPath, ('.py', 'rb', imp.PY_SOURCE)) - - stackAdvisorClass = getattr(stack_advisor, 'DefaultStackAdvisor') - self.stackAdvisor = stackAdvisorClass() - - serviceAdvisorClass = getattr(service_advisor, 'PXF300ServiceAdvisor') + service_advisor_impl = imp.load_module('service_advisor_impl', fp, pxf300ServiceAdvisorPath, ('.py', 'rb', imp.PY_SOURCE)) + serviceAdvisorClass = getattr(service_advisor_impl, 'PXF300ServiceAdvisor') self.serviceAdvisor = serviceAdvisorClass() self.PXF_PATH = "export HBASE_CLASSPATH=${HBASE_CLASSPATH}:/usr/lib/pxf/pxf-hbase.jar" @@ -56,13 +51,13 @@ class TestPXF300ServiceAdvisor(TestCase): # Case 1: Test pxf-hbase.jar classpath line was added to content expected = "# Some hbase-env content text\n\n#Add pxf-hbase.jar to HBASE_CLASSPATH\n" + self.PXF_PATH - self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, services["configurations"], None, services, None) + self.serviceAdvisor.getServiceConfigurationRecommendations(services["configurations"], None, services, None) self.assertEquals(services["configurations"]["hbase-env"]["properties"]["content"], expected) # Case 2: Test pxf-hbase.jar classpath line is not added again if content already has it services["configurations"]["hbase-env"]["properties"]["content"] = self.PXF_PATH expected = self.PXF_PATH - self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, services["configurations"], None, services, None) + self.serviceAdvisor.getServiceConfigurationRecommendations(services["configurations"], None, services, None) self.assertEquals(services["configurations"]["hbase-env"]["properties"]["content"], expected) def test_getConfigurationsValidationItems(self): @@ -107,10 +102,10 @@ class TestPXF300ServiceAdvisor(TestCase): "level": "WARN" } ] - items = self.serviceAdvisor.getConfigurationsValidationItems(self.stackAdvisor, properties, properties, services, None) + items = self.serviceAdvisor.getConfigurationsValidationItems(properties, properties, services, None) self.assertEquals(items, expected) # Case 2: No warning should be generated if PXF_PATH is present in hbase-env properties = services["configurations"]["hbase-env"]["properties"]["content"] = self.PXF_PATH - items = self.serviceAdvisor.getConfigurationsValidationItems(self.stackAdvisor, properties, properties, services, None) + items = self.serviceAdvisor.getConfigurationsValidationItems(properties, properties, services, None) self.assertEquals(items, []) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/cee5c421/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py index caf121c..c077a9f 100644 --- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py @@ -2047,7 +2047,7 @@ class TestHDP23StackAdvisor(TestCase): # Test 1 - with 3 segments self.assertEquals(len(hawqSegmentComponent["hostnames"]), 3) serviceAdvisor = self.createHAWQServiceAdvisor() - serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, clusterData, services, hosts) + serviceAdvisor.getServiceConfigurationRecommendations(configurations, clusterData, services, hosts) self.assertEquals(configurations["hawq-site"]["properties"]["default_hash_table_bucket_number"], str(3 * 6)) self.assertEquals(configurations["hdfs-client"]["properties"]["output.replace-datanode-on-failure"], "false") @@ -2057,19 +2057,19 @@ class TestHDP23StackAdvisor(TestCase): # Test 2 - with 100 segments hawqSegmentComponent["hostnames"] = ["host" + str(i) for i in range(100)] - serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, clusterData, services, hosts) + serviceAdvisor.getServiceConfigurationRecommendations(configurations, clusterData, services, hosts) self.assertEquals(configurations["hawq-site"]["properties"]["default_hash_table_bucket_number"], str(100 * 5)) self.assertEquals(configurations["hdfs-client"]["properties"]["output.replace-datanode-on-failure"], "true") # Test 3 - with 512 segments hawqSegmentComponent["hostnames"] = ["host" + str(i) for i in range(512)] - serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, clusterData, services, hosts) + serviceAdvisor.getServiceConfigurationRecommendations(configurations, clusterData, services, hosts) self.assertEquals(configurations["hawq-site"]["properties"]["default_hash_table_bucket_number"], "512") self.assertEquals(configurations["hdfs-client"]["properties"]["output.replace-datanode-on-failure"], "true") # Test 4 - with 513 segments hawqSegmentComponent["hostnames"] = ["host" + str(i) for i in range(513)] - serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, clusterData, services, hosts) + serviceAdvisor.getServiceConfigurationRecommendations(configurations, clusterData, services, hosts) self.assertEquals(configurations["hawq-site"]["properties"]["default_hash_table_bucket_number"], "512") self.assertEquals(configurations["hdfs-client"]["properties"]["output.replace-datanode-on-failure"], "true") @@ -2077,7 +2077,7 @@ class TestHDP23StackAdvisor(TestCase): configurations = {} services["configurations"]["hawq-site"] = {"properties":{"hawq_global_rm_type": "none"}} hawqSegmentComponent["hostnames"] = [] - serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, clusterData, services, hosts) + serviceAdvisor.getServiceConfigurationRecommendations(configurations, clusterData, services, hosts) self.assertEquals(configurations["hdfs-client"]["properties"]["output.replace-datanode-on-failure"], "false") self.assertTrue("default_hash_table_bucket_number" not in configurations["hawq-site"]) @@ -2380,7 +2380,7 @@ class TestHDP23StackAdvisor(TestCase): } serviceAdvisor = self.createHAWQServiceAdvisor() - problems = serviceAdvisor.validateHAWQSiteConfigurations(self.stackAdvisor, properties, defaults, configurations, services, hosts) + problems = serviceAdvisor.validateHAWQSiteConfigurations(properties, defaults, configurations, services, hosts) problems_dict = {} for problem in problems: problems_dict[problem['config-name']] = problem @@ -2391,7 +2391,7 @@ class TestHDP23StackAdvisor(TestCase): configurations["hawq-site"] = {"properties": {"hawq_master_directory": "/data/hawq/master", "hawq_segment_directory": "/data/hawq/segment"}} properties = configurations["hawq-site"]["properties"] - problems = serviceAdvisor.validateHAWQSiteConfigurations(self.stackAdvisor, properties, defaults, configurations, services, hosts) + problems = serviceAdvisor.validateHAWQSiteConfigurations(properties, defaults, configurations, services, hosts) problems_dict = {} self.assertEqual(len(problems), 0) expected_warnings = {} @@ -2400,7 +2400,7 @@ class TestHDP23StackAdvisor(TestCase): configurations["hawq-site"] = {"properties": {"hawq_master_directory": "/data/hawq/master1,/data/hawq/master2", "hawq_segment_directory": "/data/hawq/segment1 /data/hawq/segment2"}} properties = configurations["hawq-site"]["properties"] - problems = serviceAdvisor.validateHAWQSiteConfigurations(self.stackAdvisor, properties, defaults, configurations, services, hosts) + problems = serviceAdvisor.validateHAWQSiteConfigurations(properties, defaults, configurations, services, hosts) problems_dict = {} for problem in problems: problems_dict[problem['config-name']] = problem @@ -2454,7 +2454,7 @@ class TestHDP23StackAdvisor(TestCase): "level": "ERROR" } ] """ - problems = serviceAdvisor.validateHAWQSiteConfigurations(self.stackAdvisor, properties, defaults, services["configurations"], services, hosts) + problems = serviceAdvisor.validateHAWQSiteConfigurations(properties, defaults, services["configurations"], services, hosts) self.assertEqual(len(problems), 1) expected = { "config-type": "hawq-site", @@ -2468,7 +2468,7 @@ class TestHDP23StackAdvisor(TestCase): # case 2: hawq_global_rm_type is set as yarn, and YARN service is installed. No validation errors expected. services["services"].append({"StackServices" : {"service_name" : "YARN"}, "components":[]}) - problems = serviceAdvisor.validateHAWQSiteConfigurations(self.stackAdvisor, properties, defaults, services["configurations"], services, hosts) + problems = serviceAdvisor.validateHAWQSiteConfigurations(properties, defaults, services["configurations"], services, hosts) self.assertEqual(len(problems), 0) # Test HAWQ Master port conflict with Ambari Server Postgres port @@ -2481,7 +2481,7 @@ class TestHDP23StackAdvisor(TestCase): {"hawq_master_address_port": "5432"} } } - problems = serviceAdvisor.validateHAWQSiteConfigurations(self.stackAdvisor, properties, defaults, configurations, services, hosts) + problems = serviceAdvisor.validateHAWQSiteConfigurations(properties, defaults, configurations, services, hosts) self.assertEqual(len(problems), 1) expected = { "config-name": "hawq_master_address_port", @@ -2496,19 +2496,19 @@ class TestHDP23StackAdvisor(TestCase): # case 2: HAWQ Master is placed on Ambari Server and HAWQ Master port is different from Ambari Server Postgres Port serviceAdvisor.isHawqMasterComponentOnAmbariServer = MagicMock(return_value=True) configurations["hawq-site"]["properties"]["hawq_master_address_port"] = "10432" - problems = serviceAdvisor.validateHAWQSiteConfigurations(self.stackAdvisor, properties, defaults, configurations, services, hosts) + problems = serviceAdvisor.validateHAWQSiteConfigurations(properties, defaults, configurations, services, hosts) self.assertEqual(len(problems), 0) # case 3: HAWQ Master is not placed on Ambari Server and HAWQ Master port is same as Ambari Server Postgres Port serviceAdvisor.isHawqMasterComponentOnAmbariServer = MagicMock(return_value=False) configurations["hawq-site"]["properties"]["hawq_master_address_port"] = "5432" - problems = serviceAdvisor.validateHAWQSiteConfigurations(self.stackAdvisor, properties, defaults, configurations, services, hosts) + problems = serviceAdvisor.validateHAWQSiteConfigurations(properties, defaults, configurations, services, hosts) self.assertEqual(len(problems), 0) # case 4: HAWQ Master is not placed on Ambari Server and HAWQ Master port is different from Ambari Server Postgres Port serviceAdvisor.isHawqMasterComponentOnAmbariServer = MagicMock(return_value=False) configurations["hawq-site"]["properties"]["hawq_master_address_port"] = "10432" - problems = serviceAdvisor.validateHAWQSiteConfigurations(self.stackAdvisor, properties, defaults, configurations, services, hosts) + problems = serviceAdvisor.validateHAWQSiteConfigurations(properties, defaults, configurations, services, hosts) self.assertEqual(len(problems), 0) # -------- test query limits warning ---------- @@ -2538,14 +2538,14 @@ class TestHDP23StackAdvisor(TestCase): 'config-name': 'default_hash_table_bucket_number', 'level': 'ERROR' } - problems = serviceAdvisor.validateHAWQSiteConfigurations(self.stackAdvisor, properties, defaults, configurations, services, hosts) + problems = serviceAdvisor.validateHAWQSiteConfigurations(properties, defaults, configurations, services, hosts) self.assertEqual(len(problems), 1) self.assertEqual(problems[0], expected) configurations["hawq-site"] = {"properties": {"default_hash_table_bucket_number": "500", "hawq_rm_nvseg_perquery_limit": "500"}} properties = configurations["hawq-site"]["properties"] - problems = serviceAdvisor.validateHAWQSiteConfigurations(self.stackAdvisor, properties, defaults, configurations, services, hosts) + problems = serviceAdvisor.validateHAWQSiteConfigurations(properties, defaults, configurations, services, hosts) self.assertEqual(len(problems), 0) @@ -2578,19 +2578,19 @@ class TestHDP23StackAdvisor(TestCase): } serviceAdvisor = self.createHAWQServiceAdvisor() - problems = serviceAdvisor.validateHAWQHdfsClientConfigurations(self.stackAdvisor, properties, defaults, configurations, services, hosts) + problems = serviceAdvisor.validateHAWQHdfsClientConfigurations(properties, defaults, configurations, services, hosts) self.assertEqual(len(problems), 1) self.assertEqual(problems[0], expected) # 2. Try with 3 hosts services["services"][0]["components"][0]["StackServiceComponents"]["hostnames"] = ["host1", "host2", "host3"] - problems = serviceAdvisor.validateHAWQHdfsClientConfigurations(self.stackAdvisor, properties, defaults, configurations, services, hosts) + problems = serviceAdvisor.validateHAWQHdfsClientConfigurations(properties, defaults, configurations, services, hosts) self.assertEqual(len(problems), 1) self.assertEqual(problems[0], expected) # 3. Try with 4 hosts - default value services["services"][0]["components"][0]["StackServiceComponents"]["hostnames"] = ["host1", "host2", "host3", "host4"] - problems = serviceAdvisor.validateHAWQHdfsClientConfigurations(self.stackAdvisor, properties, defaults, configurations, services, hosts) + problems = serviceAdvisor.validateHAWQHdfsClientConfigurations(properties, defaults, configurations, services, hosts) self.assertEqual(len(problems), 0) # 4. Try with 4 hosts @@ -2602,7 +2602,7 @@ class TestHDP23StackAdvisor(TestCase): 'config-name': 'output.replace-datanode-on-failure', 'level': 'WARN' } - problems = serviceAdvisor.validateHAWQHdfsClientConfigurations(self.stackAdvisor, properties, defaults, configurations, services, hosts) + problems = serviceAdvisor.validateHAWQHdfsClientConfigurations(properties, defaults, configurations, services, hosts) self.assertEqual(len(problems), 1) self.assertEqual(problems[0], expected)
