Repository: ambari Updated Branches: refs/heads/trunk 5da97da85 -> e508a5fbf
AMBARI-10316 Stack-advisor mechanism when recommending configuration-dependencies has incorrect services.json content (dsen) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e508a5fb Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e508a5fb Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e508a5fb Branch: refs/heads/trunk Commit: e508a5fbf8209ca119e56167c6269b1b6cb0f898 Parents: 5da97da Author: Dmytro Sen <[email protected]> Authored: Thu Apr 2 00:07:08 2015 +0300 Committer: Dmytro Sen <[email protected]> Committed: Thu Apr 2 00:07:33 2015 +0300 ---------------------------------------------------------------------- .../server/api/services/AmbariMetaInfo.java | 38 ----------- .../commands/StackAdvisorCommand.java | 12 ++-- ...StackLevelConfigurationResourceProvider.java | 6 +- .../src/main/resources/scripts/stack_advisor.py | 44 +----------- .../src/main/resources/stacks/stack_advisor.py | 71 +++++++++++++++++++- .../server/api/services/AmbariMetaInfoTest.java | 12 ---- .../stacks/2.2/common/test_stack_advisor.py | 38 +---------- 7 files changed, 80 insertions(+), 141 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/e508a5fb/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java index 3a7fb6a..3e37c3c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java @@ -1224,42 +1224,4 @@ public class AmbariMetaInfo { return kerberosServiceDescriptors; } - /** - * Get set of all the depended-by properties down to directed acyclic graph(DAG) - * of dependencies between configuration properties - * @param stackName the stack name - * @param stackVersion the stack version - * @param changedConfigs the list of changed configurations - * @return set of all depended-by properties including all changedConfigs - */ - public Set<PropertyDependencyInfo> getDependedByProperties(String stackName, - String stackVersion, - List<PropertyDependencyInfo> changedConfigs) { - StackInfo stack = getStackManager().getStack(stackName, stackVersion); - - if (changedConfigs == null) { - return Collections.emptySet(); - } - int size = 0; - Set<PropertyDependencyInfo> configs = - new HashSet<PropertyDependencyInfo>(); - - configs.addAll(changedConfigs); - - while (size != configs.size()) { - size = configs.size(); - for (ServiceInfo service: stack.getServices()) { - for (PropertyInfo pi: service.getProperties()) { - String type = ConfigHelper.fileNameToConfigType(pi.getFilename()); - String name = pi.getName(); - PropertyDependencyInfo dep = - new PropertyDependencyInfo(type, name); - if (configs.contains(dep) && !configs.containsAll(pi.getDependedByProperties())) { - configs.addAll(pi.getDependedByProperties()); - } - } - } - } - return configs; - } } http://git-wip-us.apache.org/repos/asf/ambari/blob/e508a5fb/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java index a7646a0..ce67002 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java @@ -71,10 +71,13 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse> extend private static final String GET_HOSTS_INFO_URI = "/api/v1/hosts" + "?fields=Hosts&Hosts/host_name.in(%s)"; - private static final String GET_SERVICES_INFO_URI = "/api/v1/stacks/%s/versions/%s" + private static final String GET_SERVICES_INFO_URI = "/api/v1/stacks/%s/versions/%s/" + "?fields=Versions/stack_name,Versions/stack_version,Versions/parent_stack_version" + ",services/StackServices/service_name,services/StackServices/service_version" + ",services/components/StackServiceComponents,services/components/dependencies,services/components/auto_deploy" + + ",services/configurations/StackConfigurations/property_depends_on" + + ",services/configurations/StackConfigurations/property_depended_by" + + ",services/configurations/StackConfigurations/type" + "&services/StackServices/service_name.in(%s)"; private static final String SERVICES_PROPERTY = "services"; private static final String SERVICES_COMPONENTS_PROPERTY = "components"; @@ -83,7 +86,6 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse> extend private static final String COMPONENT_HOSTNAMES_PROPERTY = "hostnames"; private static final String CONFIGURATIONS_PROPERTY = "configurations"; private static final String CHANGED_CONFIGURATIONS_PROPERTY = "changed-configurations"; - private static final String DEPENDED_CONFIGURATIONS_PROPERTY = "depended-configurations"; private File recommendationsDir; private String stackAdvisorScript; @@ -174,15 +176,9 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse> extend } } } - // Populating changed configs info and depended by configs info - Set<PropertyDependencyInfo> dependedProperties = - metaInfo.getDependedByProperties(request.getStackName(), - request.getStackVersion(), request.getChangedConfigurations()); JsonNode changedConfigs = mapper.valueToTree(request.getChangedConfigurations()); - JsonNode dependendConfigs = mapper.valueToTree(dependedProperties); root.put(CHANGED_CONFIGURATIONS_PROPERTY, changedConfigs); - root.put(DEPENDED_CONFIGURATIONS_PROPERTY, dependendConfigs); } protected void populateStackHierarchy(ObjectNode root) { http://git-wip-us.apache.org/repos/asf/ambari/blob/e508a5fb/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackLevelConfigurationResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackLevelConfigurationResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackLevelConfigurationResourceProvider.java index f010b37..1e5109c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackLevelConfigurationResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackLevelConfigurationResourceProvider.java @@ -55,13 +55,13 @@ public class StackLevelConfigurationResourceProvider extends .getPropertyId("StackLevelConfigurations", "property_value"); public static final String PROPERTY_VALUE_ATTRIBUTES_PROPERTY_ID = PropertyHelper - .getPropertyId("StackConfigurations", "property_value_attributes"); + .getPropertyId("StackLevelConfigurations", "property_value_attributes"); public static final String DEPENDS_ON_PROPERTY_ID = PropertyHelper - .getPropertyId("StackConfigurations", "property_depends_on"); + .getPropertyId("StackLevelConfigurations", "property_depends_on"); public static final String DEPENDED_BY_PROPERTY_ID = PropertyHelper - .getPropertyId("StackConfigurations", "property_depended_by"); + .getPropertyId("StackLevelConfigurations", "property_depended_by"); public static final String PROPERTY_DESCRIPTION_PROPERTY_ID = PropertyHelper .getPropertyId("StackLevelConfigurations", "property_description"); http://git-wip-us.apache.org/repos/asf/ambari/blob/e508a5fb/ambari-server/src/main/resources/scripts/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/scripts/stack_advisor.py b/ambari-server/src/main/resources/scripts/stack_advisor.py index 1d67872..446086c 100755 --- a/ambari-server/src/main/resources/scripts/stack_advisor.py +++ b/ambari-server/src/main/resources/scripts/stack_advisor.py @@ -51,7 +51,7 @@ def loadJson(path): with open(path, 'r') as f: return json.load(f) except Exception, err: - raise StackAdvisorException("File not found at: {0}".format(hostsFile)) + raise StackAdvisorException("File not found at: {0}".format(path)) def dumpJson(json_object, dump_file): @@ -107,9 +107,8 @@ def main(argv=None): result = stackAdvisor.recommendConfigurations(services, hosts) result_file = os.path.join(actionDir, "configurations.json") elif action == RECOMMEND_CONFIGURATION_DEPENDENCIES: - result = stackAdvisor.recommendConfigurations(services, hosts) + result = stackAdvisor.recommendConfigurationDependencies(services, hosts) result_file = os.path.join(actionDir, "configurations.json") - result = filterResult(result, services) else: # action == VALIDATE_CONFIGURATIONS result = stackAdvisor.validateConfigurations(services, hosts) result_file = os.path.join(actionDir, "configurations-validation.json") @@ -117,45 +116,6 @@ def main(argv=None): dumpJson(result, result_file) pass -# returns recommendations only for changed and depended properties -def filterResult(result, services): - allRequestedProperties = getAllRequestedProperties(services) - - configs = result['recommendations']['blueprint']['configurations'] - filteredConfigs = {} - for type, names in configs.items(): - for name in names['properties']: - if type in allRequestedProperties.keys() and \ - name in allRequestedProperties[type]: - if type not in filteredConfigs.keys(): - filteredConfigs[type] = {'properties': {}} - filteredConfigs[type]['properties'][name] = \ - configs[type]['properties'][name] - if 'property_attributes' in names.keys(): - for name in names['property_attributes']: - if type in allRequestedProperties.keys() and \ - name in allRequestedProperties[type]: - if type not in filteredConfigs.keys(): - filteredConfigs[type] = {'property_Attributes': {}} - elif 'property_attributes' not in filteredConfigs[type].keys(): - filteredConfigs[type]['property_attributes'] = {} - filteredConfigs[type]['property_attributes'][name] = \ - configs[type]['property_attributes'][name] - - result['recommendations']['blueprint']['configurations'] = filteredConfigs - return result - -def getAllRequestedProperties(services): - changedConfigs = [] - changedConfigs.extend(services['changed-configurations']) - changedConfigs.extend(services['depended-configurations']) - allRequestedProperties = {} - for config in changedConfigs: - if config['type'] in allRequestedProperties: - allRequestedProperties[config['type']].append(config['name']) - else: - allRequestedProperties[config['type']] = [config['name']] - return allRequestedProperties def instantiateStackAdvisor(stackName, stackVersion, parentVersions): """Instantiates StackAdvisor implementation for the specified Stack""" http://git-wip-us.apache.org/repos/asf/ambari/blob/e508a5fb/ambari-server/src/main/resources/stacks/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/stack_advisor.py b/ambari-server/src/main/resources/stacks/stack_advisor.py index c19a7ce..4da8abb 100644 --- a/ambari-server/src/main/resources/stacks/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/stack_advisor.py @@ -18,6 +18,8 @@ limitations under the License. """ import socket +import re + class StackAdvisor(object): """ @@ -613,4 +615,71 @@ class DefaultStackAdvisor(StackAdvisor): for component in service['components']: if component["StackServiceComponents"]["component_name"] == componentName: return component["StackServiceComponents"]["hostnames"] - pass \ No newline at end of file + pass + + def recommendConfigurationDependencies(self, services, hosts): + result = self.recommendConfigurations(services, hosts) + return self.filterResult(result, services) + + # returns recommendations only for changed and depended properties + def filterResult(self, result, services): + allRequestedProperties = self.getAllRequestedProperties(services) + + configs = result['recommendations']['blueprint']['configurations'] + filteredConfigs = {} + for type, names in configs.items(): + for name in names['properties']: + if type in allRequestedProperties.keys() and \ + name in allRequestedProperties[type]: + if type not in filteredConfigs.keys(): + filteredConfigs[type] = {'properties': {}} + filteredConfigs[type]['properties'][name] = \ + configs[type]['properties'][name] + if 'property_attributes' in names.keys(): + for name in names['property_attributes']: + if type in allRequestedProperties.keys() and \ + name in allRequestedProperties[type]: + if type not in filteredConfigs.keys(): + filteredConfigs[type] = {'property_Attributes': {}} + elif 'property_attributes' not in filteredConfigs[type].keys(): + filteredConfigs[type]['property_attributes'] = {} + filteredConfigs[type]['property_attributes'][name] = \ + configs[type]['property_attributes'][name] + + result['recommendations']['blueprint']['configurations'] = filteredConfigs + return result + + def getAllRequestedProperties(self, services): + affectedConfigs = self.getAffectedConfigs(services) + allRequestedProperties = {} + for config in affectedConfigs: + if config['type'] in allRequestedProperties: + allRequestedProperties[config['type']].append(config['name']) + else: + allRequestedProperties[config['type']] = [config['name']] + return allRequestedProperties + + def getAffectedConfigs(self, services): + """returns properties dict including changed-configurations and depended-by configs""" + changedConfigs = services['changed-configurations'] + allDependencies = [] + + for item in services['services']: + allDependencies.extend(item['configurations']) + + dependencies = [] + dependencies.extend(changedConfigs) + + size = 0 + while size != len(dependencies): + size = len(dependencies) + for config in allDependencies: + type = re.sub('\.xml$', '', config['StackConfigurations']['type']) + name = config['StackConfigurations']['property_name'] + + if {"type": type, "name": name} in dependencies: + for dependedConfig in config['StackConfigurations']['property_depended_by']: + if dependedConfig not in dependencies: + dependencies.append(dependedConfig) + + return dependencies http://git-wip-us.apache.org/repos/asf/ambari/blob/e508a5fb/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java index 22bece8..ec61af8 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java @@ -818,18 +818,6 @@ public class AmbariMetaInfoTest { originalProperty.getDescription()); Assert.assertEquals(6, redefinedService.getConfigDependencies().size()); Assert.assertEquals(7, redefinedService.getConfigDependenciesWithComponents().size()); - - // Test directed-acyclic-graph (DAG) of dependencies between configurations - List<PropertyDependencyInfo> changedConfigs = new LinkedList<PropertyDependencyInfo>(); - String type = ConfigHelper.fileNameToConfigType(newProperty.getFilename()); - String name = newProperty.getName(); - changedConfigs.add(new PropertyDependencyInfo(type, name)); - Set<PropertyDependencyInfo> dependedByProperties = metaInfo.getDependedByProperties(stackInfo.getName(), stackInfo.getVersion(), changedConfigs); - Assert.assertEquals(3, dependedByProperties.size()); - Assert.assertTrue(dependedByProperties.contains(new PropertyDependencyInfo("yarn-site", "new-enhanced-yarn-property2"))); - Assert.assertTrue(dependedByProperties.contains(new PropertyDependencyInfo("yarn-site", "new-enhanced-yarn-property"))); - Assert.assertTrue(dependedByProperties.contains(new PropertyDependencyInfo("yarn-site", "new-yarn-property"))); - } @Test http://git-wip-us.apache.org/repos/asf/ambari/blob/e508a5fb/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py index 76ecc96..d12c067 100644 --- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py @@ -778,45 +778,9 @@ class TestHDP22StackAdvisor(TestCase): "type": "yarn-env", "name": "min_user_id" }, - ], - "depended-configurations": [ - { - "type" : "mapred-site", - "name" : "yarn.app.mapreduce.am.admin-command-opts" - }, { - "type" : "yarn-site", - "name" : "yarn.scheduler.maximum-allocation-mb" - }, { - "type" : "yarn-site", - "name" : "yarn.scheduler.minimum-allocation-mb" - }, { - "type" : "mapred-site", - "name" : "mapreduce.reduce.java.opts" - }, { - "type" : "mapred-site", - "name" : "mapreduce.map.java.opts" - }, { - "type" : "mapred-site", - "name" : "yarn.app.mapreduce.am.command-opts" - }, { - "type" : "mapred-site", - "name" : "yarn.app.mapreduce.am.resource.mb" - }, { - "type" : "yarn-site", - "name" : "yarn.nodemanager.resource.memory-mb" - }, { - "type" : "mapred-site", - "name" : "mapreduce.task.io.sort.mb" - }, { - "type" : "mapred-site", - "name" : "mapreduce.reduce.memory.mb" - }, { - "type" : "mapred-site", - "name" : "mapreduce.map.memory.mb" - } ] - } + } hosts = { "items" : [ {
