AMBARI-10207. BE: Extend stack-advisor to recommend property_value_attributes (mpapyrkovskyy via srimanth)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d7fcd258 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d7fcd258 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d7fcd258 Branch: refs/heads/trunk Commit: d7fcd258d9788923a03d0d121320e0d39636a2cd Parents: b95fe44 Author: Srimanth Gunturi <[email protected]> Authored: Tue Mar 31 16:31:37 2015 -0700 Committer: Srimanth Gunturi <[email protected]> Committed: Tue Mar 31 16:31:44 2015 -0700 ---------------------------------------------------------------------- .../server/api/services/AmbariMetaInfo.java | 41 ++++ .../stackadvisor/StackAdvisorHelper.java | 20 +- .../stackadvisor/StackAdvisorRequest.java | 21 +- .../ComponentLayoutRecommendationCommand.java | 67 ++++++ .../ComponentLayoutValidationCommand.java | 63 ++++++ ...rationDependenciesRecommendationCommand.java | 119 ++++++++++ .../ConfigurationRecommendationCommand.java | 113 ++++++++++ .../ConfigurationValidationCommand.java | 62 ++++++ ...GetComponentLayoutRecommnedationCommand.java | 64 ------ .../GetComponentLayoutValidationCommand.java | 63 ------ .../GetConfigurationRecommnedationCommand.java | 113 ---------- .../GetConfigurationValidationCommand.java | 62 ------ .../commands/StackAdvisorCommand.java | 39 ++-- .../commands/StackAdvisorCommandType.java | 2 + .../recommendations/RecommendationResponse.java | 36 ++- .../internal/StackAdvisorResourceProvider.java | 34 ++- .../apache/ambari/server/stack/StackModule.java | 14 +- .../configuration-mapred/mapred-site.xml | 16 +- .../YARN/2.1.0.2.0/configuration/yarn-site.xml | 4 +- .../src/main/resources/properties.json | 1 + .../src/main/resources/scripts/stack_advisor.py | 50 ++++- .../stacks/BIGTOP/0.8/services/stack_advisor.py | 2 +- .../stacks/HDP/2.0.6/services/stack_advisor.py | 58 ++++- .../services/YARN/configuration/yarn-site.xml | 4 +- .../stacks/HDP/2.2/services/stack_advisor.py | 28 ++- .../stacks/HDPWIN/2.1/services/stack_advisor.py | 2 +- .../PHD/3.0.0.0/services/stack_advisor.py | 2 +- .../src/main/resources/stacks/stack_advisor.py | 4 +- .../server/api/services/AmbariMetaInfoTest.java | 20 +- .../stackadvisor/StackAdvisorHelperTest.java | 33 ++- .../ConfigurationRecommendationCommandTest.java | 103 +++++++++ ...tConfigurationRecommnedationCommandTest.java | 103 --------- .../stacks/2.0.6/common/test_stack_advisor.py | 8 +- .../stacks/2.1/common/test_stack_advisor.py | 2 +- .../stacks/2.2/common/test_stack_advisor.py | 222 ++++++++++++++++++- .../services/YARN/configuration/yarn-site.xml | 17 +- 36 files changed, 1123 insertions(+), 489 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java index 0cb8fa4..3a7fb6a 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java @@ -59,8 +59,10 @@ import org.apache.ambari.server.stack.StackManager; import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Clusters; import org.apache.ambari.server.state.ComponentInfo; +import org.apache.ambari.server.state.ConfigHelper; import org.apache.ambari.server.state.DependencyInfo; import org.apache.ambari.server.state.OperatingSystemInfo; +import org.apache.ambari.server.state.PropertyDependencyInfo; import org.apache.ambari.server.state.PropertyInfo; import org.apache.ambari.server.state.RepositoryInfo; import org.apache.ambari.server.state.Service; @@ -1221,4 +1223,43 @@ public class AmbariMetaInfo { return kerberosServiceDescriptors; } + + /** + * Get set of all the depended-by properties down to directed acyclic graph(DAG) + * of dependencies between configuration properties + * @param stackName the stack name + * @param stackVersion the stack version + * @param changedConfigs the list of changed configurations + * @return set of all depended-by properties including all changedConfigs + */ + public Set<PropertyDependencyInfo> getDependedByProperties(String stackName, + String stackVersion, + List<PropertyDependencyInfo> changedConfigs) { + StackInfo stack = getStackManager().getStack(stackName, stackVersion); + + if (changedConfigs == null) { + return Collections.emptySet(); + } + int size = 0; + Set<PropertyDependencyInfo> configs = + new HashSet<PropertyDependencyInfo>(); + + configs.addAll(changedConfigs); + + while (size != configs.size()) { + size = configs.size(); + for (ServiceInfo service: stack.getServices()) { + for (PropertyInfo pi: service.getProperties()) { + String type = ConfigHelper.fileNameToConfigType(pi.getFilename()); + String name = pi.getName(); + PropertyDependencyInfo dep = + new PropertyDependencyInfo(type, name); + if (configs.contains(dep) && !configs.containsAll(pi.getDependedByProperties())) { + configs.addAll(pi.getDependedByProperties()); + } + } + } + } + return configs; + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelper.java index 9e683f0..a925d7d 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelper.java @@ -23,10 +23,11 @@ import java.io.IOException; import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest.StackAdvisorRequestType; -import org.apache.ambari.server.api.services.stackadvisor.commands.GetComponentLayoutRecommnedationCommand; -import org.apache.ambari.server.api.services.stackadvisor.commands.GetComponentLayoutValidationCommand; -import org.apache.ambari.server.api.services.stackadvisor.commands.GetConfigurationRecommnedationCommand; -import org.apache.ambari.server.api.services.stackadvisor.commands.GetConfigurationValidationCommand; +import org.apache.ambari.server.api.services.stackadvisor.commands.ComponentLayoutRecommendationCommand; +import org.apache.ambari.server.api.services.stackadvisor.commands.ComponentLayoutValidationCommand; +import org.apache.ambari.server.api.services.stackadvisor.commands.ConfigurationDependenciesRecommendationCommand; +import org.apache.ambari.server.api.services.stackadvisor.commands.ConfigurationRecommendationCommand; +import org.apache.ambari.server.api.services.stackadvisor.commands.ConfigurationValidationCommand; import org.apache.ambari.server.api.services.stackadvisor.commands.StackAdvisorCommand; import org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse; import org.apache.ambari.server.api.services.stackadvisor.validations.ValidationResponse; @@ -77,10 +78,10 @@ public class StackAdvisorHelper { StackAdvisorRequestType requestType) throws StackAdvisorException { StackAdvisorCommand<ValidationResponse> command; if (requestType == StackAdvisorRequestType.HOST_GROUPS) { - command = new GetComponentLayoutValidationCommand(recommendationsDir, stackAdvisorScript, + command = new ComponentLayoutValidationCommand(recommendationsDir, stackAdvisorScript, requestId, saRunner, metaInfo); } else if (requestType == StackAdvisorRequestType.CONFIGURATIONS) { - command = new GetConfigurationValidationCommand(recommendationsDir, stackAdvisorScript, + command = new ConfigurationValidationCommand(recommendationsDir, stackAdvisorScript, requestId, saRunner, metaInfo); } else { throw new StackAdvisorRequestException(String.format("Unsupported request type, type=%s", @@ -112,10 +113,13 @@ public class StackAdvisorHelper { StackAdvisorRequestType requestType) throws StackAdvisorException { StackAdvisorCommand<RecommendationResponse> command; if (requestType == StackAdvisorRequestType.HOST_GROUPS) { - command = new GetComponentLayoutRecommnedationCommand(recommendationsDir, stackAdvisorScript, + command = new ComponentLayoutRecommendationCommand(recommendationsDir, stackAdvisorScript, requestId, saRunner, metaInfo); } else if (requestType == StackAdvisorRequestType.CONFIGURATIONS) { - command = new GetConfigurationRecommnedationCommand(recommendationsDir, stackAdvisorScript, + command = new ConfigurationRecommendationCommand(recommendationsDir, stackAdvisorScript, + requestId, saRunner, metaInfo); + } else if (requestType == StackAdvisorRequestType.CONFIGURATION_DEPENDENCIES) { + command = new ConfigurationDependenciesRecommendationCommand(recommendationsDir, stackAdvisorScript, requestId, saRunner, metaInfo); } else { throw new StackAdvisorRequestException(String.format("Unsupported request type, type=%s", http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java index 991f198..89a3570 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java @@ -21,10 +21,12 @@ package org.apache.ambari.server.api.services.stackadvisor; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; +import org.apache.ambari.server.state.PropertyDependencyInfo; import org.apache.commons.lang.StringUtils; /** @@ -41,6 +43,7 @@ public class StackAdvisorRequest { private Map<String, Set<String>> hostComponents = new HashMap<String, Set<String>>(); private Map<String, Set<String>> hostGroupBindings = new HashMap<String, Set<String>>(); private Map<String, Map<String, Map<String, String>>> configurations = new HashMap<String, Map<String, Map<String, String>>>(); + private List<PropertyDependencyInfo> changedConfigurations = new LinkedList<PropertyDependencyInfo>(); public String getStackName() { return stackName; @@ -86,6 +89,14 @@ public class StackAdvisorRequest { return configurations; } + public List<PropertyDependencyInfo> getChangedConfigurations() { + return changedConfigurations; + } + + public void setChangedConfigurations(List<PropertyDependencyInfo> changedConfigurations) { + this.changedConfigurations = changedConfigurations; + } + private StackAdvisorRequest(String stackName, String stackVersion) { this.stackName = stackName; this.stackVersion = stackVersion; @@ -140,13 +151,21 @@ public class StackAdvisorRequest { return this; } + public StackAdvisorRequestBuilder withChangedConfigurations( + List<PropertyDependencyInfo> changedConfigurations) { + this.instance.changedConfigurations = changedConfigurations; + return this; + } + public StackAdvisorRequest build() { return this.instance; } } public enum StackAdvisorRequestType { - HOST_GROUPS("host_groups"), CONFIGURATIONS("configurations"); + HOST_GROUPS("host_groups"), + CONFIGURATIONS("configurations"), + CONFIGURATION_DEPENDENCIES("configuration-dependencies"); private String type; http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/ComponentLayoutRecommendationCommand.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/ComponentLayoutRecommendationCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/ComponentLayoutRecommendationCommand.java new file mode 100644 index 0000000..0dff92b --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/ComponentLayoutRecommendationCommand.java @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.api.services.stackadvisor.commands; + +import java.io.File; + +import org.apache.ambari.server.api.services.AmbariMetaInfo; +import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorException; +import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest; +import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRunner; +import org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse; + +/** + * {@link StackAdvisorCommand} implementation for component-layout + * recommendation. + */ +public class ComponentLayoutRecommendationCommand extends + StackAdvisorCommand<RecommendationResponse> { + + public ComponentLayoutRecommendationCommand(File recommendationsDir, + String stackAdvisorScript, + int requestId, + StackAdvisorRunner saRunner, + AmbariMetaInfo metaInfo) { + super(recommendationsDir, stackAdvisorScript, requestId, saRunner, metaInfo); + } + + @Override + protected StackAdvisorCommandType getCommandType() { + return StackAdvisorCommandType.RECOMMEND_COMPONENT_LAYOUT; + } + + @Override + protected void validate(StackAdvisorRequest request) throws StackAdvisorException { + if (request.getHosts() == null || request.getHosts().isEmpty() + || request.getServices() == null || request.getServices().isEmpty()) { + throw new StackAdvisorException("Hosts and services must not be empty"); + } + } + + @Override + protected RecommendationResponse updateResponse(StackAdvisorRequest request, RecommendationResponse response) { + return response; + } + + @Override + protected String getResultFileName() { + return "component-layout.json"; + } + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/ComponentLayoutValidationCommand.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/ComponentLayoutValidationCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/ComponentLayoutValidationCommand.java new file mode 100644 index 0000000..757ebee --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/ComponentLayoutValidationCommand.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.api.services.stackadvisor.commands; + +import java.io.File; + +import org.apache.ambari.server.api.services.AmbariMetaInfo; +import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorException; +import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest; +import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRunner; +import org.apache.ambari.server.api.services.stackadvisor.validations.ValidationResponse; + +/** + * {@link StackAdvisorCommand} implementation for component-layout validation. + */ +public class ComponentLayoutValidationCommand extends StackAdvisorCommand<ValidationResponse> { + + public ComponentLayoutValidationCommand(File recommendationsDir, String stackAdvisorScript, + int requestId, StackAdvisorRunner saRunner, AmbariMetaInfo metaInfo) { + super(recommendationsDir, stackAdvisorScript, requestId, saRunner, metaInfo); + } + + @Override + protected StackAdvisorCommandType getCommandType() { + return StackAdvisorCommandType.VALIDATE_COMPONENT_LAYOUT; + } + + @Override + protected void validate(StackAdvisorRequest request) throws StackAdvisorException { + if (request.getHosts() == null || request.getHosts().isEmpty() || request.getServices() == null + || request.getServices().isEmpty() || request.getComponentHostsMap() == null + || request.getComponentHostsMap().isEmpty()) { + throw new StackAdvisorException("Hosts, services and recommendations must not be empty"); + } + } + + @Override + protected ValidationResponse updateResponse(StackAdvisorRequest request, ValidationResponse response) { + return response; + } + + @Override + protected String getResultFileName() { + return "component-layout-validation.json"; + } + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/ConfigurationDependenciesRecommendationCommand.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/ConfigurationDependenciesRecommendationCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/ConfigurationDependenciesRecommendationCommand.java new file mode 100644 index 0000000..ae86548 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/ConfigurationDependenciesRecommendationCommand.java @@ -0,0 +1,119 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.api.services.stackadvisor.commands; + +import org.apache.ambari.server.api.services.AmbariMetaInfo; +import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorException; +import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest; +import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRunner; +import org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse; + +import java.io.File; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import static org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse.BindingHostGroup; +import static org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse.HostGroup; + +/** + * {@link StackAdvisorCommand} implementation for + * configuration dependencies recommendation. + */ +public class ConfigurationDependenciesRecommendationCommand extends + StackAdvisorCommand<RecommendationResponse> { + + public ConfigurationDependenciesRecommendationCommand(File recommendationsDir, String stackAdvisorScript, int requestId, + StackAdvisorRunner saRunner, AmbariMetaInfo metaInfo) { + super(recommendationsDir, stackAdvisorScript, requestId, saRunner, metaInfo); + } + + @Override + protected StackAdvisorCommandType getCommandType() { + return StackAdvisorCommandType.RECOMMEND_CONFIGURATION_DEPENDENCIES; + } + + @Override + protected void validate(StackAdvisorRequest request) + throws StackAdvisorException { + if (request.getHosts() == null || request.getHosts().isEmpty() + || request.getServices() == null || request.getServices().isEmpty() + || request.getChangedConfigurations() == null + || request.getChangedConfigurations().isEmpty()) { + throw new StackAdvisorException("Hosts, services and changed-configurations must not be empty"); + } + } + + @Override + protected RecommendationResponse updateResponse(StackAdvisorRequest request, + RecommendationResponse response) { + response.getRecommendations().getBlueprint().setHostGroups(processHostGroups(request)); + response.getRecommendations().getBlueprintClusterBinding().setHostGroups(processHostGroupBindings(request)); + return response; + } + + protected Set<HostGroup> processHostGroups(StackAdvisorRequest request) { + Set<HostGroup> resultSet = new HashSet<HostGroup>(); + for (Map.Entry<String, Set<String>> componentHost : request.getHostComponents().entrySet()) { + String hostGroupName = componentHost.getKey(); + Set<String> components = componentHost.getValue(); + if (hostGroupName != null && components != null) { + HostGroup hostGroup = new HostGroup(); + Set<Map<String, String>> componentsSet = new HashSet<Map<String, String>>(); + for (String component : components) { + Map<String, String> componentMap = new HashMap<String, String>(); + componentMap.put("name", component); + componentsSet.add(componentMap); + } + hostGroup.setComponents(componentsSet); + hostGroup.setName(hostGroupName); + resultSet.add(hostGroup); + } + } + return resultSet; + } + + private Set<BindingHostGroup> processHostGroupBindings(StackAdvisorRequest request) { + Set<BindingHostGroup> resultSet = new HashSet<BindingHostGroup>(); + for (Map.Entry<String, Set<String>> hostBinding : request.getHostGroupBindings().entrySet()) { + String hostGroupName = hostBinding.getKey(); + Set<String> hosts = hostBinding.getValue(); + if (hostGroupName != null && hosts != null) { + BindingHostGroup bindingHostGroup = new BindingHostGroup(); + Set<Map<String, String>> hostsSet = new HashSet<Map<String, String>>(); + for (String host : hosts) { + Map<String, String> hostMap = new HashMap<String, String>(); + hostMap.put("name", host); + hostsSet.add(hostMap); + } + bindingHostGroup.setHosts(hostsSet); + bindingHostGroup.setName(hostGroupName); + resultSet.add(bindingHostGroup); + } + } + return resultSet; + } + + @Override + protected String getResultFileName() { + return "configurations.json"; + } + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/ConfigurationRecommendationCommand.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/ConfigurationRecommendationCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/ConfigurationRecommendationCommand.java new file mode 100644 index 0000000..ad01b40 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/ConfigurationRecommendationCommand.java @@ -0,0 +1,113 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.api.services.stackadvisor.commands; + +import org.apache.ambari.server.api.services.AmbariMetaInfo; +import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorException; +import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest; +import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRunner; +import org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse; +import static org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse.*; + +import java.io.File; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +/** + * {@link org.apache.ambari.server.api.services.stackadvisor.commands.StackAdvisorCommand} implementation for + * configuration recommendation. + */ +public class ConfigurationRecommendationCommand extends + StackAdvisorCommand<RecommendationResponse> { + + public ConfigurationRecommendationCommand(File recommendationsDir, String stackAdvisorScript, int requestId, + StackAdvisorRunner saRunner, AmbariMetaInfo metaInfo) { + super(recommendationsDir, stackAdvisorScript, requestId, saRunner, metaInfo); + } + + @Override + protected StackAdvisorCommandType getCommandType() { + return StackAdvisorCommandType.RECOMMEND_CONFIGURATIONS; + } + + @Override + protected void validate(StackAdvisorRequest request) throws StackAdvisorException { + if (request.getHosts() == null || request.getHosts().isEmpty() || request.getServices() == null + || request.getServices().isEmpty()) { + throw new StackAdvisorException("Hosts and services must not be empty"); + } + } + + @Override + protected RecommendationResponse updateResponse(StackAdvisorRequest request, RecommendationResponse response) { + response.getRecommendations().getBlueprint().setHostGroups(processHostGroups(request)); + response.getRecommendations().getBlueprintClusterBinding().setHostGroups(processHostGroupBindings(request)); + return response; + } + + protected Set<HostGroup> processHostGroups(StackAdvisorRequest request) { + Set<HostGroup> resultSet = new HashSet<HostGroup>(); + for (Map.Entry<String, Set<String>> componentHost : request.getHostComponents().entrySet()) { + String hostGroupName = componentHost.getKey(); + Set<String> components = componentHost.getValue(); + if (hostGroupName != null && components != null) { + HostGroup hostGroup = new HostGroup(); + Set<Map<String, String>> componentsSet = new HashSet<Map<String, String>>(); + for (String component : components) { + Map<String, String> componentMap = new HashMap<String, String>(); + componentMap.put("name", component); + componentsSet.add(componentMap); + } + hostGroup.setComponents(componentsSet); + hostGroup.setName(hostGroupName); + resultSet.add(hostGroup); + } + } + return resultSet; + } + + private Set<BindingHostGroup> processHostGroupBindings(StackAdvisorRequest request) { + Set<BindingHostGroup> resultSet = new HashSet<BindingHostGroup>(); + for (Map.Entry<String, Set<String>> hostBinding : request.getHostGroupBindings().entrySet()) { + String hostGroupName = hostBinding.getKey(); + Set<String> hosts = hostBinding.getValue(); + if (hostGroupName != null && hosts != null) { + BindingHostGroup bindingHostGroup = new BindingHostGroup(); + Set<Map<String, String>> hostsSet = new HashSet<Map<String, String>>(); + for (String host : hosts) { + Map<String, String> hostMap = new HashMap<String, String>(); + hostMap.put("name", host); + hostsSet.add(hostMap); + } + bindingHostGroup.setHosts(hostsSet); + bindingHostGroup.setName(hostGroupName); + resultSet.add(bindingHostGroup); + } + } + return resultSet; + } + + @Override + protected String getResultFileName() { + return "configurations.json"; + } + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/ConfigurationValidationCommand.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/ConfigurationValidationCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/ConfigurationValidationCommand.java new file mode 100644 index 0000000..c234947 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/ConfigurationValidationCommand.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.api.services.stackadvisor.commands; + +import java.io.File; + +import org.apache.ambari.server.api.services.AmbariMetaInfo; +import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorException; +import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest; +import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRunner; +import org.apache.ambari.server.api.services.stackadvisor.validations.ValidationResponse; + +/** + * {@link StackAdvisorCommand} implementation for configuration validation. + */ +public class ConfigurationValidationCommand extends StackAdvisorCommand<ValidationResponse> { + + public ConfigurationValidationCommand(File recommendationsDir, String stackAdvisorScript, + int requestId, StackAdvisorRunner saRunner, AmbariMetaInfo metaInfo) { + super(recommendationsDir, stackAdvisorScript, requestId, saRunner, metaInfo); + } + + @Override + protected StackAdvisorCommandType getCommandType() { + return StackAdvisorCommandType.VALIDATE_CONFIGURATIONS; + } + + @Override + protected void validate(StackAdvisorRequest request) throws StackAdvisorException { + if (request.getHosts() == null || request.getHosts().isEmpty() || request.getServices() == null + || request.getServices().isEmpty()) { + throw new StackAdvisorException("Hosts, services and configurations must not be empty"); + } + } + + @Override + protected ValidationResponse updateResponse(StackAdvisorRequest request, + ValidationResponse response) { + return response; + } + + @Override + protected String getResultFileName() { + return "configurations-validation.json"; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetComponentLayoutRecommnedationCommand.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetComponentLayoutRecommnedationCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetComponentLayoutRecommnedationCommand.java deleted file mode 100644 index b91f912..0000000 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetComponentLayoutRecommnedationCommand.java +++ /dev/null @@ -1,64 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ambari.server.api.services.stackadvisor.commands; - -import java.io.File; - -import org.apache.ambari.server.api.services.AmbariMetaInfo; -import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorException; -import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest; -import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRunner; -import org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse; - -/** - * {@link StackAdvisorCommand} implementation for component-layout - * recommendation. - */ -public class GetComponentLayoutRecommnedationCommand extends - StackAdvisorCommand<RecommendationResponse> { - - public GetComponentLayoutRecommnedationCommand(File recommendationsDir, - String stackAdvisorScript, int requestId, StackAdvisorRunner saRunner, AmbariMetaInfo metaInfo) { - super(recommendationsDir, stackAdvisorScript, requestId, saRunner, metaInfo); - } - - @Override - protected StackAdvisorCommandType getCommandType() { - return StackAdvisorCommandType.RECOMMEND_COMPONENT_LAYOUT; - } - - @Override - protected void validate(StackAdvisorRequest request) throws StackAdvisorException { - if (request.getHosts() == null || request.getHosts().isEmpty() || request.getServices() == null - || request.getServices().isEmpty()) { - throw new StackAdvisorException("Hosts and services must not be empty"); - } - } - - @Override - protected RecommendationResponse updateResponse(StackAdvisorRequest request, RecommendationResponse response) { - return response; - } - - @Override - protected String getResultFileName() { - return "component-layout.json"; - } - -} http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetComponentLayoutValidationCommand.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetComponentLayoutValidationCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetComponentLayoutValidationCommand.java deleted file mode 100644 index 1a1fc98..0000000 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetComponentLayoutValidationCommand.java +++ /dev/null @@ -1,63 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ambari.server.api.services.stackadvisor.commands; - -import java.io.File; - -import org.apache.ambari.server.api.services.AmbariMetaInfo; -import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorException; -import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest; -import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRunner; -import org.apache.ambari.server.api.services.stackadvisor.validations.ValidationResponse; - -/** - * {@link StackAdvisorCommand} implementation for component-layout validation. - */ -public class GetComponentLayoutValidationCommand extends StackAdvisorCommand<ValidationResponse> { - - public GetComponentLayoutValidationCommand(File recommendationsDir, String stackAdvisorScript, - int requestId, StackAdvisorRunner saRunner, AmbariMetaInfo metaInfo) { - super(recommendationsDir, stackAdvisorScript, requestId, saRunner, metaInfo); - } - - @Override - protected StackAdvisorCommandType getCommandType() { - return StackAdvisorCommandType.VALIDATE_COMPONENT_LAYOUT; - } - - @Override - protected void validate(StackAdvisorRequest request) throws StackAdvisorException { - if (request.getHosts() == null || request.getHosts().isEmpty() || request.getServices() == null - || request.getServices().isEmpty() || request.getComponentHostsMap() == null - || request.getComponentHostsMap().isEmpty()) { - throw new StackAdvisorException("Hosts, services and recommendations must not be empty"); - } - } - - @Override - protected ValidationResponse updateResponse(StackAdvisorRequest request, ValidationResponse response) { - return response; - } - - @Override - protected String getResultFileName() { - return "component-layout-validation.json"; - } - -} http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetConfigurationRecommnedationCommand.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetConfigurationRecommnedationCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetConfigurationRecommnedationCommand.java deleted file mode 100644 index b20c966..0000000 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetConfigurationRecommnedationCommand.java +++ /dev/null @@ -1,113 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ambari.server.api.services.stackadvisor.commands; - -import org.apache.ambari.server.api.services.AmbariMetaInfo; -import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorException; -import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest; -import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRunner; -import org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse; -import static org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse.*; - -import java.io.File; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -/** - * {@link org.apache.ambari.server.api.services.stackadvisor.commands.StackAdvisorCommand} implementation for - * configuration recommendation. - */ -public class GetConfigurationRecommnedationCommand extends - StackAdvisorCommand<RecommendationResponse> { - - public GetConfigurationRecommnedationCommand(File recommendationsDir, String stackAdvisorScript, int requestId, - StackAdvisorRunner saRunner, AmbariMetaInfo metaInfo) { - super(recommendationsDir, stackAdvisorScript, requestId, saRunner, metaInfo); - } - - @Override - protected StackAdvisorCommandType getCommandType() { - return StackAdvisorCommandType.RECOMMEND_CONFIGURATIONS; - } - - @Override - protected void validate(StackAdvisorRequest request) throws StackAdvisorException { - if (request.getHosts() == null || request.getHosts().isEmpty() || request.getServices() == null - || request.getServices().isEmpty()) { - throw new StackAdvisorException("Hosts and services must not be empty"); - } - } - - @Override - protected RecommendationResponse updateResponse(StackAdvisorRequest request, RecommendationResponse response) { - response.getRecommendations().getBlueprint().setHostGroups(processHostGroups(request)); - response.getRecommendations().getBlueprintClusterBinding().setHostGroups(processHostGroupBindings(request)); - return response; - } - - protected Set<HostGroup> processHostGroups(StackAdvisorRequest request) { - Set<HostGroup> resultSet = new HashSet<HostGroup>(); - for (Map.Entry<String, Set<String>> componentHost : request.getHostComponents().entrySet()) { - String hostGroupName = componentHost.getKey(); - Set<String> components = componentHost.getValue(); - if (hostGroupName != null && components != null) { - HostGroup hostGroup = new HostGroup(); - Set<Map<String, String>> componentsSet = new HashSet<Map<String, String>>(); - for (String component : components) { - Map<String, String> componentMap = new HashMap<String, String>(); - componentMap.put("name", component); - componentsSet.add(componentMap); - } - hostGroup.setComponents(componentsSet); - hostGroup.setName(hostGroupName); - resultSet.add(hostGroup); - } - } - return resultSet; - } - - private Set<BindingHostGroup> processHostGroupBindings(StackAdvisorRequest request) { - Set<BindingHostGroup> resultSet = new HashSet<BindingHostGroup>(); - for (Map.Entry<String, Set<String>> hostBinding : request.getHostGroupBindings().entrySet()) { - String hostGroupName = hostBinding.getKey(); - Set<String> hosts = hostBinding.getValue(); - if (hostGroupName != null && hosts != null) { - BindingHostGroup bindingHostGroup = new BindingHostGroup(); - Set<Map<String, String>> hostsSet = new HashSet<Map<String, String>>(); - for (String host : hosts) { - Map<String, String> hostMap = new HashMap<String, String>(); - hostMap.put("name", host); - hostsSet.add(hostMap); - } - bindingHostGroup.setHosts(hostsSet); - bindingHostGroup.setName(hostGroupName); - resultSet.add(bindingHostGroup); - } - } - return resultSet; - } - - @Override - protected String getResultFileName() { - return "configurations.json"; - } - -} http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetConfigurationValidationCommand.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetConfigurationValidationCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetConfigurationValidationCommand.java deleted file mode 100644 index 36fe6cb..0000000 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetConfigurationValidationCommand.java +++ /dev/null @@ -1,62 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ambari.server.api.services.stackadvisor.commands; - -import java.io.File; - -import org.apache.ambari.server.api.services.AmbariMetaInfo; -import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorException; -import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest; -import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRunner; -import org.apache.ambari.server.api.services.stackadvisor.validations.ValidationResponse; - -/** - * {@link StackAdvisorCommand} implementation for configuration validation. - */ -public class GetConfigurationValidationCommand extends StackAdvisorCommand<ValidationResponse> { - - public GetConfigurationValidationCommand(File recommendationsDir, String stackAdvisorScript, - int requestId, StackAdvisorRunner saRunner, AmbariMetaInfo metaInfo) { - super(recommendationsDir, stackAdvisorScript, requestId, saRunner, metaInfo); - } - - @Override - protected StackAdvisorCommandType getCommandType() { - return StackAdvisorCommandType.VALIDATE_CONFIGURATIONS; - } - - @Override - protected void validate(StackAdvisorRequest request) throws StackAdvisorException { - if (request.getHosts() == null || request.getHosts().isEmpty() || request.getServices() == null - || request.getServices().isEmpty()) { - throw new StackAdvisorException("Hosts, services and configurations must not be empty"); - } - } - - @Override - protected ValidationResponse updateResponse(StackAdvisorRequest request, - ValidationResponse response) { - return response; - } - - @Override - protected String getResultFileName() { - return "configurations-validation.json"; - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java index 2e20452..a7646a0 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java @@ -44,6 +44,7 @@ import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest; import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorResponse; import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRunner; import org.apache.ambari.server.controller.spi.Resource; +import org.apache.ambari.server.state.PropertyDependencyInfo; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.io.FileUtils; import org.apache.commons.logging.Log; @@ -75,12 +76,14 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse> extend + ",services/StackServices/service_name,services/StackServices/service_version" + ",services/components/StackServiceComponents,services/components/dependencies,services/components/auto_deploy" + "&services/StackServices/service_name.in(%s)"; - private static final String SERVICES_PROPETRY = "services"; - private static final String SERVICES_COMPONENTS_PROPETRY = "components"; - private static final String COMPONENT_INFO_PROPETRY = "StackServiceComponents"; + private static final String SERVICES_PROPERTY = "services"; + private static final String SERVICES_COMPONENTS_PROPERTY = "components"; + private static final String COMPONENT_INFO_PROPERTY = "StackServiceComponents"; private static final String COMPONENT_NAME_PROPERTY = "component_name"; - private static final String COMPONENT_HOSTNAMES_PROPETRY = "hostnames"; - private static final String CONFIGURATIONS_PROPETRY = "configurations"; + private static final String COMPONENT_HOSTNAMES_PROPERTY = "hostnames"; + private static final String CONFIGURATIONS_PROPERTY = "configurations"; + private static final String CHANGED_CONFIGURATIONS_PROPERTY = "changed-configurations"; + private static final String DEPENDED_CONFIGURATIONS_PROPERTY = "depended-configurations"; private File recommendationsDir; private String stackAdvisorScript; @@ -140,8 +143,7 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse> extend populateStackHierarchy(root); populateComponentHostsMap(root, request.getComponentHostsMap()); - populateConfigurations(root, request.getConfigurations()); - + populateConfigurations(root, request); data.servicesJSON = mapper.writeValueAsString(root); } catch (Exception e) { // should not happen @@ -154,8 +156,10 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse> extend } private void populateConfigurations(ObjectNode root, - Map<String, Map<String, Map<String, String>>> configurations) { - ObjectNode configurationsNode = root.putObject(CONFIGURATIONS_PROPETRY); + StackAdvisorRequest request) { + Map<String, Map<String, Map<String, String>>> configurations = + request.getConfigurations(); + ObjectNode configurationsNode = root.putObject(CONFIGURATIONS_PROPERTY); for (String siteName : configurations.keySet()) { ObjectNode siteNode = configurationsNode.putObject(siteName); @@ -170,6 +174,15 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse> extend } } } + // Populating changed configs info and depended by configs info + Set<PropertyDependencyInfo> dependedProperties = + metaInfo.getDependedByProperties(request.getStackName(), + request.getStackVersion(), request.getChangedConfigurations()); + + JsonNode changedConfigs = mapper.valueToTree(request.getChangedConfigurations()); + JsonNode dependendConfigs = mapper.valueToTree(dependedProperties); + root.put(CHANGED_CONFIGURATIONS_PROPERTY, changedConfigs); + root.put(DEPENDED_CONFIGURATIONS_PROPERTY, dependendConfigs); } protected void populateStackHierarchy(ObjectNode root) { @@ -185,21 +198,21 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse> extend } private void populateComponentHostsMap(ObjectNode root, Map<String, Set<String>> componentHostsMap) { - ArrayNode services = (ArrayNode) root.get(SERVICES_PROPETRY); + ArrayNode services = (ArrayNode) root.get(SERVICES_PROPERTY); Iterator<JsonNode> servicesIter = services.getElements(); while (servicesIter.hasNext()) { JsonNode service = servicesIter.next(); - ArrayNode components = (ArrayNode) service.get(SERVICES_COMPONENTS_PROPETRY); + ArrayNode components = (ArrayNode) service.get(SERVICES_COMPONENTS_PROPERTY); Iterator<JsonNode> componentsIter = components.getElements(); while (componentsIter.hasNext()) { JsonNode component = componentsIter.next(); - ObjectNode componentInfo = (ObjectNode) component.get(COMPONENT_INFO_PROPETRY); + ObjectNode componentInfo = (ObjectNode) component.get(COMPONENT_INFO_PROPERTY); String componentName = componentInfo.get(COMPONENT_NAME_PROPERTY).getTextValue(); Set<String> componentHosts = componentHostsMap.get(componentName); - ArrayNode hostnames = componentInfo.putArray(COMPONENT_HOSTNAMES_PROPETRY); + ArrayNode hostnames = componentInfo.putArray(COMPONENT_HOSTNAMES_PROPERTY); if (null != componentHosts) { for (String hostName : componentHosts) { hostnames.add(hostName); http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommandType.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommandType.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommandType.java index 380b81d..5aa1ea9 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommandType.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommandType.java @@ -29,6 +29,8 @@ public enum StackAdvisorCommandType { RECOMMEND_CONFIGURATIONS("recommend-configurations"), + RECOMMEND_CONFIGURATION_DEPENDENCIES("recommend-configuration-dependencies"), + VALIDATE_CONFIGURATIONS("validate-configurations"); private final String name; http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/recommendations/RecommendationResponse.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/recommendations/RecommendationResponse.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/recommendations/RecommendationResponse.java index 1796f9a..b81320d 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/recommendations/RecommendationResponse.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/recommendations/RecommendationResponse.java @@ -23,6 +23,7 @@ import java.util.Set; import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorResponse; import org.codehaus.jackson.annotate.JsonProperty; +import org.codehaus.jackson.map.annotate.JsonSerialize; /** * Recommendation response POJO. @@ -88,16 +89,16 @@ public class RecommendationResponse extends StackAdvisorResponse { public static class Blueprint { @JsonProperty - private Map<String, Map<String, Map<String, String>>> configurations; + private Map<String, BlueprintConfigurations> configurations; @JsonProperty("host_groups") private Set<HostGroup> hostGroups; - public Map<String, Map<String, Map<String, String>>> getConfigurations() { + public Map<String, BlueprintConfigurations> getConfigurations() { return configurations; } - public void setConfigurations(Map<String, Map<String, Map<String, String>>> configurations) { + public void setConfigurations(Map<String, BlueprintConfigurations> configurations) { this.configurations = configurations; } @@ -110,6 +111,35 @@ public class RecommendationResponse extends StackAdvisorResponse { } } + public static class BlueprintConfigurations { + @JsonProperty + private Map<String, String> properties; + + @JsonProperty("property_attributes") + @JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) + private Map<String, Map<String, String>> propertyAttributes; + + public BlueprintConfigurations() { + System.out.println(this); + } + + public Map<String, String> getProperties() { + return properties; + } + + public void setProperties(Map<String, String> properties) { + this.properties = properties; + } + + public Map<String, Map<String, String>> getPropertyAttributes() { + return propertyAttributes; + } + + public void setPropertyAttributes(Map<String, Map<String, String>> propertyAttributes) { + this.propertyAttributes = propertyAttributes; + } + } + public static class HostGroup { @JsonProperty private String name; http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java index 6ae557d..620251b 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java @@ -18,8 +18,10 @@ package org.apache.ambari.server.controller.internal; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; @@ -38,6 +40,7 @@ import org.apache.ambari.server.controller.spi.Resource.Type; import org.apache.ambari.server.controller.utilities.PropertyHelper; import com.google.inject.Inject; +import org.apache.ambari.server.state.PropertyDependencyInfo; /** * Abstract superclass for recommendations and validations. @@ -52,6 +55,8 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi private static final String HOST_PROPERTY = "hosts"; private static final String SERVICES_PROPERTY = "services"; + private static final String CHANGED_CONFIGURATIONS_PROPERTY = "changed_configurations"; + private static final String BLUEPRINT_HOST_GROUPS_PROPERTY = "recommendations/blueprint/host_groups"; private static final String BINDING_HOST_GROUPS_PROPERTY = "recommendations/blueprint_cluster_binding/host_groups"; @@ -99,12 +104,17 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi hgHostsMap); Map<String, Map<String, Map<String, String>>> configurations = calculateConfigurations(request); - StackAdvisorRequest saRequest = StackAdvisorRequestBuilder.forStack(stackName, stackVersion) - .ofType(requestType).forHosts(hosts).forServices(services) - .forHostComponents(hgComponentsMap).forHostsGroupBindings(hgHostsMap) - .withComponentHostsMap(componentHostsMap).withConfigurations(configurations).build(); - - return saRequest; + List<PropertyDependencyInfo> changedConfigurations = + requestType == StackAdvisorRequestType.CONFIGURATION_DEPENDENCIES ? + calculateChangedConfigurations(request) : Collections.<PropertyDependencyInfo>emptyList(); + + return StackAdvisorRequestBuilder. + forStack(stackName, stackVersion).ofType(requestType).forHosts(hosts). + forServices(services).forHostComponents(hgComponentsMap). + forHostsGroupBindings(hgHostsMap). + withComponentHostsMap(componentHostsMap). + withConfigurations(configurations). + withChangedConfigurations(changedConfigurations).build(); } catch (Exception e) { LOG.warn("Error occured during preparation of stack advisor request", e); Response response = Response.status(Status.BAD_REQUEST) @@ -176,6 +186,18 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi return map; } + protected List<PropertyDependencyInfo> calculateChangedConfigurations(Request request) { + List<PropertyDependencyInfo> configs = + new LinkedList<PropertyDependencyInfo>(); + HashSet<HashMap<String, String>> changedConfigs = + (HashSet<HashMap<String, String>>) getRequestProperty(request, CHANGED_CONFIGURATIONS_PROPERTY); + for (HashMap<String, String> props: changedConfigs) { + configs.add(new PropertyDependencyInfo(props.get("type"), props.get("name"))); + } + + return configs; + } + protected static final String CONFIGURATIONS_PROPERTY_ID = "recommendations/blueprint/configurations/"; protected Map<String, Map<String, Map<String, String>>> calculateConfigurations(Request request) { http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java index 0b27ae5..4b88aff 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java @@ -29,6 +29,7 @@ import java.util.Set; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.api.services.AmbariMetaInfo; +import org.apache.ambari.server.state.ConfigHelper; import org.apache.ambari.server.state.PropertyDependencyInfo; import org.apache.ambari.server.state.PropertyInfo; import org.apache.ambari.server.state.RepositoryInfo; @@ -597,11 +598,15 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V for (ServiceModule serviceModule : serviceModules.values()) { for (PropertyInfo pi : serviceModule.getModuleInfo().getProperties()) { for (PropertyDependencyInfo pdi : pi.getDependsOnProperties()) { - PropertyDependencyInfo propertyDependency = new PropertyDependencyInfo(pi.getFilename(), pi.getName()); + String type = ConfigHelper.fileNameToConfigType(pi.getFilename()); + String name = pi.getName(); + PropertyDependencyInfo propertyDependency = + new PropertyDependencyInfo(type, name); if (dependedByMap.keySet().contains(pdi)) { dependedByMap.get(pdi).add(propertyDependency); } else { - Set<PropertyDependencyInfo> newDependenciesSet = new HashSet<PropertyDependencyInfo>(); + Set<PropertyDependencyInfo> newDependenciesSet = + new HashSet<PropertyDependencyInfo>(); newDependenciesSet.add(propertyDependency); dependedByMap.put(pdi, newDependenciesSet); } @@ -612,7 +617,10 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V // Go through all service-configs again and set their 'depended-by' if necessary. for (ServiceModule serviceModule : serviceModules.values()) { for (PropertyInfo pi : serviceModule.getModuleInfo().getProperties()) { - Set<PropertyDependencyInfo> set = dependedByMap.remove(new PropertyDependencyInfo(pi.getFilename(), pi.getName())); + String type = ConfigHelper.fileNameToConfigType(pi.getFilename()); + String name = pi.getName(); + Set<PropertyDependencyInfo> set = + dependedByMap.remove(new PropertyDependencyInfo(type, name)); if (set != null) { pi.getDependedByProperties().addAll(set); } http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml index a99a132..bf20d6e 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml @@ -41,7 +41,7 @@ </value-attributes> <depends-on> <property> - <type>mapred-site.xml</type> + <type>mapred-site</type> <name>mapreduce.map.memory.mb</name> </property> </depends-on> @@ -186,7 +186,7 @@ </value-attributes> <depends-on> <property> - <type>yarn-site.xml</type> + <type>yarn-site</type> <name>yarn.scheduler.maximum-allocation-mb</name> </property> </depends-on> @@ -206,7 +206,7 @@ </value-attributes> <depends-on> <property> - <type>yarn-site.xml</type> + <type>yarn-site</type> <name>yarn.scheduler.maximum-allocation-mb</name> </property> </depends-on> @@ -281,7 +281,7 @@ </value-attributes> <depends-on> <property> - <type>yarn-site.xml</type> + <type>yarn-site</type> <name>yarn.scheduler.maximum-allocation-mb</name> </property> </depends-on> @@ -306,7 +306,7 @@ <display-name>MR AppMaster Java Heap Size</display-name> <depends-on> <property> - <type>mapred-site.xml</type> + <type>mapred-site</type> <name>yarn.app.mapreduce.am.resource.mb</name> </property> </depends-on> @@ -328,7 +328,7 @@ <display-name>MR AppMaster Java Heap Size</display-name> <depends-on> <property> - <type>mapred-site.xml</type> + <type>mapred-site</type> <name>yarn.app.mapreduce.am.resource.mb</name> </property> </depends-on> @@ -383,7 +383,7 @@ <display-name>MR Map Java Heap Size</display-name> <depends-on> <property> - <type>mapred-site.xml</type> + <type>mapred-site</type> <name>mapreduce.map.memory.mb</name> </property> </depends-on> @@ -399,7 +399,7 @@ <display-name>MR Reduce Java Heap Size</display-name> <depends-on> <property> - <type>mapred-site.xml</type> + <type>mapred-site</type> <name>mapreduce.reduce.memory.mb</name> </property> </depends-on> http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-site.xml b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-site.xml index e6407ef..e71dfc9 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-site.xml +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-site.xml @@ -80,7 +80,7 @@ </value-attributes> <depends-on> <property> - <type>yarn-site.xml</type> + <type>yarn-site</type> <name>yarn.nodemanager.resource.memory-mb</name> </property> </depends-on> @@ -104,7 +104,7 @@ </value-attributes> <depends-on> <property> - <type>yarn-site.xml</type> + <type>yarn-site</type> <name>yarn.nodemanager.resource.memory-mb</name> </property> </depends-on> http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/resources/properties.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/properties.json b/ambari-server/src/main/resources/properties.json index b230cb7..94374bc 100644 --- a/ambari-server/src/main/resources/properties.json +++ b/ambari-server/src/main/resources/properties.json @@ -376,6 +376,7 @@ "recommend", "hosts", "services", + "changed_configurations", "recommendations", "recommendations/blueprint", "recommendations/blueprint/configurations", http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/resources/scripts/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/scripts/stack_advisor.py b/ambari-server/src/main/resources/scripts/stack_advisor.py index 60f87f9..1d67872 100755 --- a/ambari-server/src/main/resources/scripts/stack_advisor.py +++ b/ambari-server/src/main/resources/scripts/stack_advisor.py @@ -26,9 +26,14 @@ import traceback RECOMMEND_COMPONENT_LAYOUT_ACTION = 'recommend-component-layout' VALIDATE_COMPONENT_LAYOUT_ACTION = 'validate-component-layout' RECOMMEND_CONFIGURATIONS = 'recommend-configurations' +RECOMMEND_CONFIGURATION_DEPENDENCIES = 'recommend-configuration-dependencies' VALIDATE_CONFIGURATIONS = 'validate-configurations' -ALL_ACTIONS = [ RECOMMEND_COMPONENT_LAYOUT_ACTION, VALIDATE_COMPONENT_LAYOUT_ACTION, RECOMMEND_CONFIGURATIONS, VALIDATE_CONFIGURATIONS ] +ALL_ACTIONS = [RECOMMEND_COMPONENT_LAYOUT_ACTION, + VALIDATE_COMPONENT_LAYOUT_ACTION, + RECOMMEND_CONFIGURATIONS, + RECOMMEND_CONFIGURATION_DEPENDENCIES, + VALIDATE_CONFIGURATIONS] USAGE = "Usage: <action> <hosts_file> <services_file>\nPossible actions are: {0}\n".format( str(ALL_ACTIONS) ) SCRIPT_DIRECTORY = os.path.dirname(os.path.abspath(__file__)) @@ -101,6 +106,10 @@ def main(argv=None): elif action == RECOMMEND_CONFIGURATIONS: result = stackAdvisor.recommendConfigurations(services, hosts) result_file = os.path.join(actionDir, "configurations.json") + elif action == RECOMMEND_CONFIGURATION_DEPENDENCIES: + result = stackAdvisor.recommendConfigurations(services, hosts) + result_file = os.path.join(actionDir, "configurations.json") + result = filterResult(result, services) else: # action == VALIDATE_CONFIGURATIONS result = stackAdvisor.validateConfigurations(services, hosts) result_file = os.path.join(actionDir, "configurations-validation.json") @@ -108,6 +117,45 @@ def main(argv=None): dumpJson(result, result_file) pass +# returns recommendations only for changed and depended properties +def filterResult(result, services): + allRequestedProperties = getAllRequestedProperties(services) + + configs = result['recommendations']['blueprint']['configurations'] + filteredConfigs = {} + for type, names in configs.items(): + for name in names['properties']: + if type in allRequestedProperties.keys() and \ + name in allRequestedProperties[type]: + if type not in filteredConfigs.keys(): + filteredConfigs[type] = {'properties': {}} + filteredConfigs[type]['properties'][name] = \ + configs[type]['properties'][name] + if 'property_attributes' in names.keys(): + for name in names['property_attributes']: + if type in allRequestedProperties.keys() and \ + name in allRequestedProperties[type]: + if type not in filteredConfigs.keys(): + filteredConfigs[type] = {'property_Attributes': {}} + elif 'property_attributes' not in filteredConfigs[type].keys(): + filteredConfigs[type]['property_attributes'] = {} + filteredConfigs[type]['property_attributes'][name] = \ + configs[type]['property_attributes'][name] + + result['recommendations']['blueprint']['configurations'] = filteredConfigs + return result + +def getAllRequestedProperties(services): + changedConfigs = [] + changedConfigs.extend(services['changed-configurations']) + changedConfigs.extend(services['depended-configurations']) + allRequestedProperties = {} + for config in changedConfigs: + if config['type'] in allRequestedProperties: + allRequestedProperties[config['type']].append(config['name']) + else: + allRequestedProperties[config['type']] = [config['name']] + return allRequestedProperties def instantiateStackAdvisor(stackName, stackVersion, parentVersions): """Instantiates StackAdvisor implementation for the specified Stack""" http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/stack_advisor.py index 853a732..53591cd 100644 --- a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/stack_advisor.py @@ -105,7 +105,7 @@ class BaseBIGTOP08StackAdvisor(DefaultStackAdvisor): putMapredProperty('mapreduce.reduce.java.opts', "-Xmx" + str(int(round(0.8 * clusterData['reduceMemory']))) + "m") putMapredProperty('mapreduce.task.io.sort.mb', min(int(round(0.4 * clusterData['mapMemory'])), 1024)) - def getConfigurationClusterSummary(self, servicesList, hosts, components): + def getConfigurationClusterSummary(self, servicesList, hosts, components, services): hBaseInstalled = False if 'HBASE' in servicesList: http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py index c9ded5e..2405954 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py @@ -88,23 +88,48 @@ class HDP206StackAdvisor(DefaultStackAdvisor): "AMBARI_METRICS": self.recommendAmsConfigurations } - def putProperty(self, config, configType): - if configType not in config or "properties" not in config[configType]: - config[configType] = {"properties": {}} + def putProperty(self, config, configType, services=None): + userConfigs = {} + changedConfigs = [] + # if services parameter, prefer values, set by user + if services: + if 'configurations' in services.keys(): + userConfigs = services['configurations'] + if 'changed-configurations' in services.keys(): + changedConfigs = services["changed-configurations"] + + if configType not in config: + config[configType] = {} + if"properties" not in config[configType]: + config[configType]["properties"] = {} def appendProperty(key, value): - config[configType]["properties"][key] = str(value) + if {'type': configType, 'name': key} in changedConfigs: + config[configType]["properties"][key] = userConfigs[configType]['properties'][key] + else: + config[configType]["properties"][key] = str(value) return appendProperty + def putPropertyAttribute(self, config, configType): + if configType not in config: + config[configType] = {} + def appendPropertyAttribute(key, attribute, attributeValue): + if"property_attributes" not in config[configType]: + config[configType]["property_attributes"] = {} + if key not in config[configType]["property_attributes"]: + config[configType]["property_attributes"][key] = {} + config[configType]["property_attributes"][key][attribute] = str(attributeValue) + return appendPropertyAttribute + def recommendYARNConfigurations(self, configurations, clusterData, services, hosts): - putYarnProperty = self.putProperty(configurations, "yarn-site") - putYarnEnvProperty = self.putProperty(configurations, "yarn-env") + putYarnProperty = self.putProperty(configurations, "yarn-site", services) + putYarnEnvProperty = self.putProperty(configurations, "yarn-env", services) putYarnProperty('yarn.nodemanager.resource.memory-mb', int(round(clusterData['containers'] * clusterData['ramPerContainer']))) putYarnProperty('yarn.scheduler.minimum-allocation-mb', int(clusterData['ramPerContainer'])) putYarnProperty('yarn.scheduler.maximum-allocation-mb', int(round(clusterData['containers'] * clusterData['ramPerContainer']))) putYarnEnvProperty('min_user_id', self.get_system_min_uid()) def recommendMapReduce2Configurations(self, configurations, clusterData, services, hosts): - putMapredProperty = self.putProperty(configurations, "mapred-site") + putMapredProperty = self.putProperty(configurations, "mapred-site", services) putMapredProperty('yarn.app.mapreduce.am.resource.mb', int(clusterData['amMemory'])) putMapredProperty('yarn.app.mapreduce.am.command-opts', "-Xmx" + str(int(round(0.8 * clusterData['amMemory']))) + "m") putMapredProperty('mapreduce.map.memory.mb', clusterData['mapMemory']) @@ -184,7 +209,18 @@ class HDP206StackAdvisor(DefaultStackAdvisor): - def getConfigurationClusterSummary(self, servicesList, hosts, components): + def getHostWithComponent(self, serviceName, componentName, services, hosts): + if services is not None and hosts is not None: + service = [serviceEntry for serviceEntry in services["services"] if serviceEntry["StackServices"]["service_name"] == serviceName][0] + components = [componentEntry for componentEntry in service["components"] if componentEntry["StackServiceComponents"]["component_name"] == "NODEMANAGER"] + if (len(components) > 0 and len(components[0]["StackServiceComponents"]["hostnames"]) > 0): + # NodeManager available - determine hosts and memory + componentHostname = components[0]["StackServiceComponents"]["hostnames"][0] + componentHosts = [host for host in hosts["items"] if host["Hosts"]["host_name"] == componentHostname] + if (componentHosts is not None and len(componentHosts) > 0): + return componentHosts[0] + + def getConfigurationClusterSummary(self, servicesList, hosts, components, services): hBaseInstalled = False if 'HBASE' in servicesList: @@ -199,7 +235,11 @@ class HDP206StackAdvisor(DefaultStackAdvisor): } if len(hosts["items"]) > 0: - host = hosts["items"][0]["Hosts"] + nodeManagerHost = self.getHostWithComponent("YARN", "NODEMANAGER", services, hosts) + if nodeManagerHost is not None: + host = nodeManagerHost["Hosts"] + else: + host = hosts["items"][0]["Hosts"] cluster["cpu"] = host["cpu_count"] cluster["disk"] = len(host["disk_info"]) cluster["ram"] = int(host["total_mem"] / (1024 * 1024)) http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration/yarn-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration/yarn-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration/yarn-site.xml index 826f458..9137bdb 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration/yarn-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration/yarn-site.xml @@ -349,7 +349,7 @@ </value-attributes> <depends-on> <property> - <type>yarn-site.xml</type> + <type>yarn-site</type> <name>yarn.nodemanager.resource.cpu-vcores</name> </property> </depends-on> @@ -367,7 +367,7 @@ </value-attributes> <depends-on> <property> - <type>yarn-site.xml</type> + <type>yarn-site</type> <name>yarn.nodemanager.resource.cpu-vcores</name> </property> </depends-on> http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py index b8ddc49..87748d2 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py @@ -38,8 +38,19 @@ class HDP22StackAdvisor(HDP21StackAdvisor): def recommendYARNConfigurations(self, configurations, clusterData, services, hosts): super(HDP22StackAdvisor, self).recommendYARNConfigurations(configurations, clusterData, services, hosts) - putYarnProperty = self.putProperty(configurations, "yarn-site") + putYarnProperty = self.putProperty(configurations, "yarn-site", services) putYarnProperty('yarn.nodemanager.resource.cpu-vcores', clusterData['cpu']) + # Property Attributes + putYarnPropertyAttribute = self.putPropertyAttribute(configurations, "yarn-site") + nodeManagerHost = self.getHostWithComponent("YARN", "NODEMANAGER", services, hosts) + if (nodeManagerHost is not None): + putYarnProperty('yarn.nodemanager.resource.cpu-vcores', nodeManagerHost["Hosts"]["cpu_count"] * 2) + putYarnPropertyAttribute('yarn.nodemanager.resource.memory-mb', 'max', int(nodeManagerHost["Hosts"]["total_mem"] / 1024)) # total_mem in kb + putYarnPropertyAttribute('yarn.nodemanager.resource.cpu-vcores', 'max', nodeManagerHost["Hosts"]["cpu_count"] * 4) + putYarnPropertyAttribute('yarn.scheduler.minimum-allocation-vcores', 'max', configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.cpu-vcores"]) + putYarnPropertyAttribute('yarn.scheduler.maximum-allocation-vcores', 'max', configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.cpu-vcores"]) + putYarnPropertyAttribute('yarn.scheduler.minimum-allocation-mb', 'max', configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"]) + putYarnPropertyAttribute('yarn.scheduler.maximum-allocation-mb', 'max', configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"]) def recommendHDFSConfigurations(self, configurations, clusterData, services, hosts): putHdfsProperty = self.putProperty(configurations, "hdfs-site") @@ -156,14 +167,25 @@ class HDP22StackAdvisor(HDP21StackAdvisor): return self.toConfigurationValidationProblems(validationItems, "tez-site") def recommendMapReduce2Configurations(self, configurations, clusterData, services, hosts): - putMapredProperty = self.putProperty(configurations, "mapred-site") + self.recommendYARNConfigurations(configurations, clusterData, services, hosts) + putMapredProperty = self.putProperty(configurations, "mapred-site", services) putMapredProperty('yarn.app.mapreduce.am.resource.mb', int(clusterData['amMemory'])) putMapredProperty('yarn.app.mapreduce.am.command-opts', "-Xmx" + str(int(round(0.8 * clusterData['amMemory']))) + "m" + " -Dhdp.version=${hdp.version}") - putMapredProperty('mapreduce.map.memory.mb', clusterData['mapMemory']) + putMapredProperty('mapreduce.map.memory.mb', configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]) putMapredProperty('mapreduce.reduce.memory.mb', int(clusterData['reduceMemory'])) putMapredProperty('mapreduce.map.java.opts', "-Xmx" + str(int(round(0.8 * clusterData['mapMemory']))) + "m") putMapredProperty('mapreduce.reduce.java.opts', "-Xmx" + str(int(round(0.8 * clusterData['reduceMemory']))) + "m") putMapredProperty('mapreduce.task.io.sort.mb', min(int(round(0.4 * clusterData['mapMemory'])), 1024)) + # Property Attributes + putMapredPropertyAttribute = self.putPropertyAttribute(configurations, "mapred-site") + yarnMinAllocationSize = int(clusterData['ramPerContainer']) + yarnMaxAllocationSize = max(30 * int(clusterData['ramPerContainer']), configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]) + putMapredPropertyAttribute("mapreduce.map.memory.mb", "max", yarnMaxAllocationSize) + putMapredPropertyAttribute("mapreduce.map.memory.mb", "min", yarnMinAllocationSize) + putMapredPropertyAttribute("mapreduce.reduce.memory.mb", "max", yarnMaxAllocationSize) + putMapredPropertyAttribute("mapreduce.reduce.memory.mb", "min", yarnMinAllocationSize) + putMapredPropertyAttribute("yarn.app.mapreduce.am.resource.mb", "max", yarnMaxAllocationSize) + putMapredPropertyAttribute("yarn.app.mapreduce.am.resource.mb", "min", yarnMinAllocationSize) def validateMapReduce2Configurations(self, properties, recommendedDefaults, configurations, services, hosts): validationItems = [ {"config-name": 'mapreduce.map.java.opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'mapreduce.map.java.opts')}, http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py index cf35e47..3353af1 100644 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py @@ -133,7 +133,7 @@ class HDPWIN21StackAdvisor(DefaultStackAdvisor): "-server -Xmx" + str(int(0.8 * clusterData["amMemory"])) + "m -Djava.net.preferIPv4Stack=true -XX:+UseNUMA -XX:+UseParallelGC") - def getConfigurationClusterSummary(self, servicesList, hosts, components): + def getConfigurationClusterSummary(self, servicesList, hosts, components, services): hBaseInstalled = False if 'HBASE' in servicesList:
