Repository: ambari Updated Branches: refs/heads/trunk e8b355412 -> b13bb71dc
AMBARI-6781. BE: Provide configurations recommendations via /recommendations endpoint on stack-version Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b13bb71d Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b13bb71d Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b13bb71d Branch: refs/heads/trunk Commit: b13bb71dc27494f220f97f1b5c6ba8d05354a55e Parents: e8b3554 Author: Srimanth Gunturi <[email protected]> Authored: Thu Aug 7 14:23:50 2014 -0700 Committer: Srimanth Gunturi <[email protected]> Committed: Thu Aug 7 16:47:08 2014 -0700 ---------------------------------------------------------------------- .../stackadvisor/StackAdvisorHelper.java | 21 +++ .../stackadvisor/StackAdvisorHelper.java.orig | 116 ++++++++++++++ .../stackadvisor/StackAdvisorRequest.java | 22 +++ ...GetComponentLayoutRecommnedationCommand.java | 5 + .../GetComponentLayoutValidationCommand.java | 5 + .../GetConfigurationRecommnedationCommand.java | 118 ++++++++++++++ .../commands/StackAdvisorCommand.java | 5 +- .../internal/StackAdvisorResourceProvider.java | 75 ++++----- .../src/main/resources/scripts/stack_advisor.py | 20 +-- .../stacks/HDP/2.0.6/services/stack_advisor.py | 159 ++++++++++++++++++- .../stacks/HDP/2.1/services/stack_advisor.py | 41 +++++ .../main/resources/stacks/HDP/stack_advisor.py | 2 +- .../commands/StackAdvisorCommandTest.java | 5 + 13 files changed, 544 insertions(+), 50 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/b13bb71d/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelper.java index 213b0f0..156b637 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelper.java @@ -24,6 +24,7 @@ import java.io.IOException; import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest.StackAdvisorRequestType; import org.apache.ambari.server.api.services.stackadvisor.commands.GetComponentLayoutRecommnedationCommand; import org.apache.ambari.server.api.services.stackadvisor.commands.GetComponentLayoutValidationCommand; +import org.apache.ambari.server.api.services.stackadvisor.commands.GetConfigurationRecommnedationCommand; import org.apache.ambari.server.api.services.stackadvisor.commands.StackAdvisorCommand; import org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse; import org.apache.ambari.server.api.services.stackadvisor.validations.ValidationResponse; @@ -105,6 +106,9 @@ public class StackAdvisorHelper { if (requestType == StackAdvisorRequestType.HOST_GROUPS) { command = new GetComponentLayoutRecommnedationCommand(recommendationsDir, stackAdvisorScript, requestId, saRunner); + } else if (requestType == StackAdvisorRequestType.CONFIGURATIONS) { + command = new GetConfigurationRecommnedationCommand(recommendationsDir, stackAdvisorScript, + requestId, saRunner); } else { throw new StackAdvisorException(String.format("Unsupported request type, type=%s", requestType)); @@ -113,4 +117,21 @@ public class StackAdvisorHelper { return command; } + /** + * Return configurations recommendation based on hosts and services + * information. + * + * @param request the recommendation request + * @return {@link RecommendationResponse} instance + * @throws StackAdvisorException in case of stack advisor script errors + */ + public synchronized RecommendationResponse getConfigurationRecommnedation( + StackAdvisorRequest request) throws StackAdvisorException { + requestId += 1; + + GetConfigurationRecommnedationCommand command = new GetConfigurationRecommnedationCommand( + recommendationsDir, stackAdvisorScript, requestId, saRunner); + return command.invoke(request); + } + } http://git-wip-us.apache.org/repos/asf/ambari/blob/b13bb71d/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelper.java.orig ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelper.java.orig b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelper.java.orig new file mode 100644 index 0000000..213b0f0 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelper.java.orig @@ -0,0 +1,116 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.api.services.stackadvisor; + +import java.io.File; +import java.io.IOException; + +import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest.StackAdvisorRequestType; +import org.apache.ambari.server.api.services.stackadvisor.commands.GetComponentLayoutRecommnedationCommand; +import org.apache.ambari.server.api.services.stackadvisor.commands.GetComponentLayoutValidationCommand; +import org.apache.ambari.server.api.services.stackadvisor.commands.StackAdvisorCommand; +import org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse; +import org.apache.ambari.server.api.services.stackadvisor.validations.ValidationResponse; +import org.apache.ambari.server.configuration.Configuration; + +import com.google.inject.Inject; +import com.google.inject.Singleton; + +@Singleton +public class StackAdvisorHelper { + + private File recommendationsDir; + private String stackAdvisorScript; + + /* Monotonically increasing requestid */ + private int requestId = 0; + private StackAdvisorRunner saRunner; + + @Inject + public StackAdvisorHelper(Configuration conf, StackAdvisorRunner saRunner) throws IOException { + this.recommendationsDir = conf.getRecommendationsDir(); + this.stackAdvisorScript = conf.getStackAdvisorScript(); + this.saRunner = saRunner; + } + + /** + * Returns validation (component-layout or configurations) result for the + * request. + * + * @param validationRequest the validation request + * @return {@link ValidationResponse} instance + * @throws StackAdvisorException in case of stack advisor script errors + */ + public synchronized ValidationResponse validate(StackAdvisorRequest request) + throws StackAdvisorException { + requestId += 1; + + StackAdvisorCommand<ValidationResponse> command = createValidationCommand(request + .getRequestType()); + + return command.invoke(request); + } + + StackAdvisorCommand<ValidationResponse> createValidationCommand( + StackAdvisorRequestType requestType) throws StackAdvisorException { + StackAdvisorCommand<ValidationResponse> command; + if (requestType == StackAdvisorRequestType.HOST_GROUPS) { + command = new GetComponentLayoutValidationCommand(recommendationsDir, stackAdvisorScript, + requestId, saRunner); + } else { + throw new StackAdvisorException(String.format("Unsupported request type, type=%s", + requestType)); + } + + return command; + } + + /** + * Returns recommendation (component-layout or configurations) based on the + * request. + * + * @param request the recommendation request + * @return {@link RecommendationResponse} instance + * @throws StackAdvisorException in case of stack advisor script errors + */ + public synchronized RecommendationResponse recommend(StackAdvisorRequest request) + throws StackAdvisorException { + requestId += 1; + + StackAdvisorCommand<RecommendationResponse> command = createRecommendationCommand(request + .getRequestType()); + + return command.invoke(request); + } + + StackAdvisorCommand<RecommendationResponse> createRecommendationCommand( + StackAdvisorRequestType requestType) throws StackAdvisorException { + StackAdvisorCommand<RecommendationResponse> command; + if (requestType == StackAdvisorRequestType.HOST_GROUPS) { + command = new GetComponentLayoutRecommnedationCommand(recommendationsDir, stackAdvisorScript, + requestId, saRunner); + } else { + throw new StackAdvisorException(String.format("Unsupported request type, type=%s", + requestType)); + } + + return command; + } + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/b13bb71d/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java index b82047e..bc1678e 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java @@ -27,6 +27,8 @@ import java.util.Set; import org.apache.commons.lang.StringUtils; +import static org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse.*; + /** * Stack advisor request. */ @@ -38,6 +40,8 @@ public class StackAdvisorRequest { private List<String> hosts = new ArrayList<String>(); private List<String> services = new ArrayList<String>(); private Map<String, Set<String>> componentHostsMap = new HashMap<String, Set<String>>(); + private Map<String, Set<String>> hostComponents = new HashMap<String, Set<String>>(); + private Map<String, Set<String>> hostGroupBindings = new HashMap<String, Set<String>>(); public String getStackName() { return stackName; @@ -71,6 +75,14 @@ public class StackAdvisorRequest { return StringUtils.join(services, ","); } + public Map<String, Set<String>> getHostComponents() { + return hostComponents; + } + + public Map<String, Set<String>> getHostGroupBindings() { + return hostGroupBindings; + } + private StackAdvisorRequest(String stackName, String stackVersion) { this.stackName = stackName; this.stackVersion = stackVersion; @@ -108,6 +120,16 @@ public class StackAdvisorRequest { return this; } + public StackAdvisorRequestBuilder forHostComponents(Map<String, Set<String>> hostComponents) { + this.instance.hostComponents = hostComponents; + return this; + } + + public StackAdvisorRequestBuilder forHostsGroupBindings(Map<String, Set<String>> hostGroupBindings) { + this.instance.hostGroupBindings = hostGroupBindings; + return this; + } + public StackAdvisorRequest build() { return this.instance; } http://git-wip-us.apache.org/repos/asf/ambari/blob/b13bb71d/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetComponentLayoutRecommnedationCommand.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetComponentLayoutRecommnedationCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetComponentLayoutRecommnedationCommand.java index b3b5057..6e7533a 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetComponentLayoutRecommnedationCommand.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetComponentLayoutRecommnedationCommand.java @@ -51,6 +51,11 @@ public class GetComponentLayoutRecommnedationCommand extends } @Override + protected RecommendationResponse updateResponse(StackAdvisorRequest request, RecommendationResponse response) { + return response; + } + + @Override protected String getResultFileName() { return "component-layout.json"; } http://git-wip-us.apache.org/repos/asf/ambari/blob/b13bb71d/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetComponentLayoutValidationCommand.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetComponentLayoutValidationCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetComponentLayoutValidationCommand.java index ac24d07..a5453f6 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetComponentLayoutValidationCommand.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetComponentLayoutValidationCommand.java @@ -50,6 +50,11 @@ public class GetComponentLayoutValidationCommand extends StackAdvisorCommand<Val } @Override + protected ValidationResponse updateResponse(StackAdvisorRequest request, ValidationResponse response) { + return response; + } + + @Override protected String getResultFileName() { return "component-layout-validation.json"; } http://git-wip-us.apache.org/repos/asf/ambari/blob/b13bb71d/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetConfigurationRecommnedationCommand.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetConfigurationRecommnedationCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetConfigurationRecommnedationCommand.java new file mode 100644 index 0000000..48946aa --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetConfigurationRecommnedationCommand.java @@ -0,0 +1,118 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.api.services.stackadvisor.commands; + +import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorException; +import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest; +import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRunner; +import org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse; +import static org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse.*; + +import java.io.File; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +/** + * {@link org.apache.ambari.server.api.services.stackadvisor.commands.StackAdvisorCommand} implementation for + * configuration recommendation. + */ +public class GetConfigurationRecommnedationCommand extends + StackAdvisorCommand<RecommendationResponse> { + + public GetConfigurationRecommnedationCommand(File recommendationsDir, + String stackAdvisorScript, int requestId, StackAdvisorRunner saRunner) { + super(recommendationsDir, stackAdvisorScript, requestId, saRunner); + } + + @Override + protected StackAdvisorCommandType getCommandType() { + return StackAdvisorCommandType.RECOMMEND_CONFIGURATIONS; + } + + @Override + protected void validate(StackAdvisorRequest request) throws StackAdvisorException { + if (request.getHosts().isEmpty() || request.getServices().isEmpty()) { + throw new StackAdvisorException("Hosts and services must not be empty"); + } + } + + @Override + protected StackAdvisorData adjust(StackAdvisorData data, StackAdvisorRequest request) { + // do nothing + return data; + } + + @Override + protected RecommendationResponse updateResponse(StackAdvisorRequest request, RecommendationResponse response) { + response.getRecommendations().getBlueprint().setHostGroups(processHostGroups(request)); + response.getRecommendations().getBlueprintClusterBinding().setHostGroups(processHostGroupBindings(request)); + return response; + } + + private Set<HostGroup> processHostGroups(StackAdvisorRequest request) { + + Set<HostGroup> resultSet = new HashSet<HostGroup>(); + for (Map.Entry<String, Set<String>> componentHost : request.getComponentHostsMap().entrySet()) { + String hostGroupName = componentHost.getKey(); + Set<String> components = componentHost.getValue(); + if (hostGroupName != null && components != null) { + HostGroup hostGroup = new HostGroup(); + Set<Map<String, String>> componentsSet = new HashSet<Map<String, String>>(); + for (String component : components) { + Map<String, String> componentMap = new HashMap<String, String>(); + componentMap.put("name", component); + componentsSet.add(componentMap); + } + hostGroup.setComponents(componentsSet); + hostGroup.setName(hostGroupName); + resultSet.add(hostGroup); + } + } + return resultSet; + } + + private Set<BindingHostGroup> processHostGroupBindings(StackAdvisorRequest request) { + Set<BindingHostGroup> resultSet = new HashSet<BindingHostGroup>(); + for (Map.Entry<String, Set<String>> hostBinding : request.getHostGroupBindings().entrySet()) { + String hostGroupName = hostBinding.getKey(); + Set<String> hosts = hostBinding.getValue(); + if (hostGroupName != null && hosts != null) { + BindingHostGroup bindingHostGroup = new BindingHostGroup(); + Set<Map<String, String>> hostsSet = new HashSet<Map<String, String>>(); + for (String host : hosts) { + Map<String, String> hostMap = new HashMap<String, String>(); + hostMap.put("name", host); + hostsSet.add(hostMap); + } + bindingHostGroup.setHosts(hostsSet); + bindingHostGroup.setName(hostGroupName); + resultSet.add(bindingHostGroup); + } + } + return resultSet; + } + + @Override + protected String getResultFileName() { + return "configurations.json"; + } + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/b13bb71d/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java index a9ff24b..1a30d38 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java @@ -193,7 +193,8 @@ public abstract class StackAdvisorCommand<T> extends BaseService { String result = FileUtils.readFileToString(new File(requestDirectory, getResultFileName())); - return this.mapper.readValue(result, this.type); + T response = this.mapper.readValue(result, this.type); + return updateResponse(request, response); } catch (Exception e) { String message = "Error occured during stack advisor command invocation"; LOG.warn(message, e); @@ -201,6 +202,8 @@ public abstract class StackAdvisorCommand<T> extends BaseService { } } + protected abstract T updateResponse(StackAdvisorRequest request, T response); + /** * Create request id directory for each call */ http://git-wip-us.apache.org/repos/asf/ambari/blob/b13bb71d/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java index 9cf387a..95024d2 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java @@ -93,11 +93,13 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi */ List<String> hosts = (List<String>) getRequestProperty(request, HOST_PROPERTY); List<String> services = (List<String>) getRequestProperty(request, SERVICES_PROPERTY); - Map<String, Set<String>> componentHostsMap = calculateComponentHostsMap(request); + Map<String, Set<String>> hgComponentsMap = calculateHostGroupComponentsMap(request); + Map<String, Set<String>> hgHostsMap = calculateHostGroupHostsMap(request); + Map<String, Set<String>> componentHostsMap = calculateComponentHostsMap(hgComponentsMap, hgHostsMap); StackAdvisorRequest saRequest = StackAdvisorRequestBuilder.forStack(stackName, stackVersion) - .ofType(requestType).forHosts(hosts).forServices(services) - .withComponentHostsMap(componentHostsMap).build(); + .ofType(requestType).forHosts(hosts).forServices(services).forHostComponents(hgComponentsMap) + .forHostsGroupBindings(hgHostsMap).withComponentHostsMap(componentHostsMap).build(); return saRequest; } catch (Exception e) { @@ -113,25 +115,28 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi * Will prepare host-group names to components names map from the * recommendation blueprint host groups. * - * @param hostGroups the blueprint host groups + * @param request stack advisor request * @return host-group to components map */ @SuppressWarnings("unchecked") - private Map<String, Set<String>> calculateHostGroupComponentsMap( - Set<Map<String, Object>> hostGroups) { + private Map<String, Set<String>> calculateHostGroupComponentsMap(Request request) { + Set<Map<String, Object>> hostGroups = + (Set<Map<String, Object>>) getRequestProperty(request, BLUEPRINT_HOST_GROUPS_PROPERTY); Map<String, Set<String>> map = new HashMap<String, Set<String>>(); - for (Map<String, Object> hostGroup : hostGroups) { - String hostGroupName = (String) hostGroup.get(BLUEPRINT_HOST_GROUPS_NAME_PROPERTY); + if (hostGroups != null) { + for (Map<String, Object> hostGroup : hostGroups) { + String hostGroupName = (String) hostGroup.get(BLUEPRINT_HOST_GROUPS_NAME_PROPERTY); - Set<Map<String, Object>> componentsSet = (Set<Map<String, Object>>) hostGroup - .get(BLUEPRINT_HOST_GROUPS_COMPONENTS_PROPERTY); + Set<Map<String, Object>> componentsSet = (Set<Map<String, Object>>) hostGroup + .get(BLUEPRINT_HOST_GROUPS_COMPONENTS_PROPERTY); - Set<String> components = new HashSet<String>(); - for (Map<String, Object> component : componentsSet) { - components.add((String) component.get(BLUEPRINT_HOST_GROUPS_COMPONENTS_NAME_PROPERTY)); - } + Set<String> components = new HashSet<String>(); + for (Map<String, Object> component : componentsSet) { + components.add((String) component.get(BLUEPRINT_HOST_GROUPS_COMPONENTS_NAME_PROPERTY)); + } - map.put(hostGroupName, components); + map.put(hostGroupName, components); + } } return map; @@ -141,52 +146,48 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi * Will prepare host-group names to hosts names map from the recommendation * binding host groups. * - * @param bindingHostGroups the binding host groups + * @param request stack advisor request * @return host-group to hosts map */ @SuppressWarnings("unchecked") - private Map<String, Set<String>> calculateHostGroupHostsMap( - Set<Map<String, Object>> bindingHostGroups) { + private Map<String, Set<String>> calculateHostGroupHostsMap(Request request) { + Set<Map<String, Object>> bindingHostGroups = + (Set<Map<String, Object>>) getRequestProperty(request, BINDING_HOST_GROUPS_PROPERTY); Map<String, Set<String>> map = new HashMap<String, Set<String>>(); + if (bindingHostGroups != null) { + for (Map<String, Object> hostGroup : bindingHostGroups) { + String hostGroupName = (String) hostGroup.get(BINDING_HOST_GROUPS_NAME_PROPERTY); - for (Map<String, Object> hostGroup : bindingHostGroups) { - String hostGroupName = (String) hostGroup.get(BINDING_HOST_GROUPS_NAME_PROPERTY); + Set<Map<String, Object>> hostsSet = (Set<Map<String, Object>>) hostGroup + .get(BINDING_HOST_GROUPS_HOSTS_PROPERTY); - Set<Map<String, Object>> hostsSet = (Set<Map<String, Object>>) hostGroup - .get(BINDING_HOST_GROUPS_HOSTS_PROPERTY); + Set<String> hosts = new HashSet<String>(); + for (Map<String, Object> host : hostsSet) { + hosts.add((String) host.get(BINDING_HOST_GROUPS_HOSTS_NAME_PROPERTY)); + } - Set<String> hosts = new HashSet<String>(); - for (Map<String, Object> host : hostsSet) { - hosts.add((String) host.get(BINDING_HOST_GROUPS_HOSTS_NAME_PROPERTY)); + map.put(hostGroupName, hosts); } - - map.put(hostGroupName, hosts); } return map; } @SuppressWarnings("unchecked") - private Map<String, Set<String>> calculateComponentHostsMap(Request request) { + private Map<String, Set<String>> calculateComponentHostsMap(Map<String, Set<String>> hostGroups, + Map<String, Set<String>> bindingHostGroups) { /* * ClassCastException may occur in case of body inconsistency: property * missed, etc. */ - Set<Map<String, Object>> hostGroups = (Set<Map<String, Object>>) getRequestProperty(request, - BLUEPRINT_HOST_GROUPS_PROPERTY); - Set<Map<String, Object>> bindingHostGroups = (Set<Map<String, Object>>) getRequestProperty( - request, BINDING_HOST_GROUPS_PROPERTY); Map<String, Set<String>> componentHostsMap = new HashMap<String, Set<String>>(); if (null != bindingHostGroups && null != hostGroups) { - Map<String, Set<String>> hgComponentsMap = calculateHostGroupComponentsMap(hostGroups); - Map<String, Set<String>> hgHostsMap = calculateHostGroupHostsMap(bindingHostGroups); - - for (Map.Entry<String, Set<String>> hgComponents : hgComponentsMap.entrySet()) { + for (Map.Entry<String, Set<String>> hgComponents : hostGroups.entrySet()) { String hgName = hgComponents.getKey(); Set<String> components = hgComponents.getValue(); - Set<String> hosts = hgHostsMap.get(hgName); + Set<String> hosts = bindingHostGroups.get(hgName); for (String component : components) { Set<String> componentHosts = componentHostsMap.get(component); if (componentHosts == null) { // if was not initialized http://git-wip-us.apache.org/repos/asf/ambari/blob/b13bb71d/ambari-server/src/main/resources/scripts/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/scripts/stack_advisor.py b/ambari-server/src/main/resources/scripts/stack_advisor.py index 1eb5e4d..05d3a20 100755 --- a/ambari-server/src/main/resources/scripts/stack_advisor.py +++ b/ambari-server/src/main/resources/scripts/stack_advisor.py @@ -82,7 +82,8 @@ def main(argv=None): stackName = services["Versions"]["stack_name"] stackVersion = services["Versions"]["stack_version"] parentVersions = [] - if "parent_stack_version" in services["Versions"]: + if "parent_stack_version" in services["Versions"] and \ + services["Versions"]["parent_stack_version"] is not None: parentVersions = [ services["Versions"]["parent_stack_version"] ] stackAdvisor = instantiateStackAdvisor(stackName, stackVersion, parentVersions) @@ -120,22 +121,23 @@ def instantiateStackAdvisor(stackName, stackVersion, parentVersions): versions = [stackVersion] versions.extend(parentVersions) - for version in versions: + for version in reversed(versions): try: path = STACK_ADVISOR_IMPL_PATH_TEMPLATE.format(stackName, version) className = STACK_ADVISOR_IMPL_CLASS_TEMPLATE.format(stackName, version.replace('.', '')) with open(path, 'rb') as fp: - stack_advisor_impl = imp.load_module( 'stack_advisor_impl', fp, path, ('.py', 'rb', imp.PY_SOURCE) ) - clazz = getattr(stack_advisor_impl, className) - - print "StackAdvisor for stack {0}, version {1} will be used".format(stackName, version) - return clazz(); + stack_advisor_impl = imp.load_module('stack_advisor_impl', fp, path, ('.py', 'rb', imp.PY_SOURCE)) + print "StackAdvisor implementation for stack {0}, version {1} was loaded".format(stackName, version) except Exception, e: print "StackAdvisor implementation for stack {0}, version {1} was not found".format(stackName, version) - print "StackAdvisor default implementation will be used!" - return stack_advisor.StackAdvisor() + try: + clazz = getattr(stack_advisor_impl, className) + return clazz() + except Exception, e: + print "Returning default implementation" + return stack_advisor.StackAdvisor() if __name__ == '__main__': http://git-wip-us.apache.org/repos/asf/ambari/blob/b13bb71d/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py index 85c3a8e..402d95b 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py @@ -204,8 +204,163 @@ class HDP206StackAdvisor(StackAdvisor): pass def recommendConfigurations(self, services, hosts): - """Returns Services object with configurations object populated""" - pass + stackName = services["Versions"]["stack_name"] + stackVersion = services["Versions"]["stack_version"] + hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]] + servicesList = [service["StackServices"]["service_name"] for service in services["services"]] + components = [component["StackServiceComponents"]["component_name"] + for service in services["services"] + for component in service["components"]] + + clusterData = self.getClusterData(servicesList, hosts, components) + + recommendations = { + "Versions": {"stack_name": stackName, "stack_version": stackVersion}, + "hosts": hostsList, + "services": servicesList, + "recommendations": { + "blueprint": { + "configurations": {}, + "host_groups": [] + }, + "blueprint_cluster_binding": { + "host_groups": [] + } + } + } + + configurations = recommendations["recommendations"]["blueprint"]["configurations"] + + for service in servicesList: + calculation = self.recommendServiceConfigurations(service) + if calculation is not None: + calculation(configurations, clusterData) + + return recommendations + + def recommendServiceConfigurations(self, service): + return { + "YARN": self.recommendYARNConfigurations, + "MAPREDUCE2": self.recommendMapReduce2Configurations, + "HIVE": self.recommendHiveConfigurations, + "OOZIE": self.recommendOozieConfigurations + }.get(service, None) + + def putProperty(self, config, configType): + config[configType] = {"properties": {}} + def appendProperty(key, value): + config[configType]["properties"][key] = str(value) + return appendProperty + + def recommendYARNConfigurations(self, configurations, clusterData): + putYarnProperty = self.putProperty(configurations, "yarn-site") + putYarnProperty('yarn.nodemanager.resource.memory-mb', clusterData['containers'] * clusterData['ramPerContainer']) + putYarnProperty('yarn.scheduler.minimum-allocation-mb', clusterData['ramPerContainer']) + putYarnProperty('yarn.scheduler.maximum-allocation-mb', clusterData['containers'] * clusterData['ramPerContainer']) + + def recommendHiveConfigurations(self, configurations, clusterData): + containerSize = clusterData['mapMemory'] if clusterData['mapMemory'] > 2048 else clusterData['reduceMemory'] + containerSize = min(clusterData['containers'] * clusterData['ramPerContainer'], containerSize) + putHiveProperty = self.putProperty(configurations, "hive-site") + putHiveProperty('hive.auto.convert.join.noconditionaltask.size', int(containerSize / 3) * 1048576) + putHiveProperty('hive.tez.java.opts', "-server -Xmx" + str(int(0.8 * containerSize)) + + "m -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseParallelGC") + putHiveProperty('hive.tez.container.size', containerSize) + + def recommendMapReduce2Configurations(self, configurations, clusterData): + putMapredProperty = self.putProperty(configurations, "mapred-site") + putMapredProperty('yarn.app.mapreduce.am.resource.mb', clusterData['amMemory']) + putMapredProperty('yarn.app.mapreduce.am.command-opts', "-Xmx" + str(int(0.8 * clusterData['amMemory'])) + "m") + putMapredProperty('mapreduce.map.memory.mb', clusterData['mapMemory']) + putMapredProperty('mapreduce.reduce.memory.mb', clusterData['reduceMemory']) + putMapredProperty('mapreduce.map.java.opts', "-Xmx" + str(int(0.8 * clusterData['mapMemory'])) + "m") + putMapredProperty('mapreduce.reduce.java.opts', "-Xmx" + str(int(0.8 * clusterData['reduceMemory'])) + "m") + putMapredProperty('mapreduce.task.io.sort.mb', int(min(0.4 * clusterData['mapMemory'], 1024))) + + def recommendOozieConfigurations(self, configurations, clusterData): + if "FALCON_SERVER" in clusterData["components"]: + putMapredProperty = self.putProperty(configurations, "oozie-site") + putMapredProperty("oozie.services.ext", + "org.apache.oozie.service.JMSAccessorService," + + "org.apache.oozie.service.PartitionDependencyManagerService," + + "org.apache.oozie.service.HCatAccessorService") + + def getClusterData(self, servicesList, hosts, components): + + hBaseInstalled = False + if 'HBASE' in servicesList: + hBaseInstalled = True + + cluster = { + "cpu": 0, + "disk": 0, + "ram": 0, + "hBaseInstalled": hBaseInstalled, + "components": components + } + + if len(hosts["items"]) > 0: + host = hosts["items"][0]["Hosts"] + cluster["cpu"] = host["cpu_count"] + cluster["disk"] = len(host["disk_info"]) + cluster["ram"] = int(host["total_mem"] / (1024 * 1024)) + + ramRecommendations = [ + {"os":1, "hbase":1}, + {"os":2, "hbase":1}, + {"os":2, "hbase":2}, + {"os":4, "hbase":4}, + {"os":6, "hbase":8}, + {"os":8, "hbase":8}, + {"os":8, "hbase":8}, + {"os":12, "hbase":16}, + {"os":24, "hbase":24}, + {"os":32, "hbase":32}, + {"os":64, "hbase":64} + ] + index = { + cluster["ram"] <= 4: 0, + 4 < cluster["ram"] <= 8: 1, + 8 < cluster["ram"] <= 16: 2, + 16 < cluster["ram"] <= 24: 3, + 24 < cluster["ram"] <= 48: 4, + 48 < cluster["ram"] <= 64: 5, + 64 < cluster["ram"] <= 72: 6, + 72 < cluster["ram"] <= 96: 7, + 96 < cluster["ram"] <= 128: 8, + 128 < cluster["ram"] <= 256: 9, + 256 < cluster["ram"]: 10 + }[1] + cluster["reservedRam"] = ramRecommendations[index]["os"] + cluster["hbaseRam"] = ramRecommendations[index]["hbase"] + + cluster["minContainerSize"] = { + cluster["ram"] <= 4: 256, + 4 < cluster["ram"] <= 8: 512, + 8 < cluster["ram"] <= 24: 1024, + 24 < cluster["ram"]: 2048 + }[1] + + '''containers = max(3, min (2*cores,min (1.8*DISKS,(Total available RAM) / MIN_CONTAINER_SIZE))))''' + cluster["containers"] = max(3, + min(2 * cluster["cpu"], + int(min(1.8 * cluster["disk"], + cluster["ram"] / cluster["minContainerSize"])))) + + '''ramPerContainers = max(2GB, RAM - reservedRam - hBaseRam) / containers''' + cluster["ramPerContainer"] = max(2048, + cluster["ram"] - cluster["reservedRam"] - cluster["hbaseRam"]) + cluster["ramPerContainer"] /= cluster["containers"] + '''If greater than 1GB, value will be in multiples of 512.''' + if cluster["ramPerContainer"] > 1024: + cluster["ramPerContainer"] = ceil(cluster["ramPerContainer"] / 512) * 512 + + cluster["mapMemory"] = int(cluster["ramPerContainer"]) + cluster["reduceMemory"] = cluster["ramPerContainer"] + cluster["amMemory"] = max(cluster["mapMemory"], cluster["reduceMemory"]) + + return cluster + def validateConfigurations(self, services, hosts): """Returns array of Validation objects about issues with configuration values provided in services""" http://git-wip-us.apache.org/repos/asf/ambari/blob/b13bb71d/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py new file mode 100644 index 0000000..eb2faa1 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py @@ -0,0 +1,41 @@ +#!/usr/bin/env ambari-python-wrap +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import socket + +from stack_advisor import StackAdvisor + +class HDP21StackAdvisor(HDP206StackAdvisor): + + def recommendServiceConfigurations(self, service): + calculator = super(HDP21StackAdvisor, self).recommendServiceConfigurations(service) + if calculator is None: + return { + "TEZ": self.recommendTezConfigurations + }.get(service, None) + else: + return calculator + + def recommendTezConfigurations(self, configurations, clusterData): + putTezProperty = self.putProperty(configurations, "tez-site") + putTezProperty("tez.am.resource.memory.mb", clusterData['amMemory']) + putTezProperty("tez.am.java.opts", + "-server -Xmx" + str(int(0.8 * clusterData["amMemory"])) + + "m -Djava.net.preferIPv4Stack=true -XX:+UseNUMA -XX:+UseParallelGC") + http://git-wip-us.apache.org/repos/asf/ambari/blob/b13bb71d/ambari-server/src/main/resources/stacks/HDP/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/stack_advisor.py index fdef482..5f66fff 100644 --- a/ambari-server/src/main/resources/stacks/HDP/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/stack_advisor.py @@ -17,7 +17,7 @@ See the License for the specific language governing permissions and limitations under the License. """ -class StackAdvisor(): +class StackAdvisor(object): def recommendComponentLayout(self, services, hosts): """Returns Services object with hostnames array populated for components""" http://git-wip-us.apache.org/repos/asf/ambari/blob/b13bb71d/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommandTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommandTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommandTest.java index 90dafcb..5e4e3d0 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommandTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommandTest.java @@ -185,6 +185,11 @@ public class StackAdvisorCommandTest { protected StackAdvisorCommandType getCommandType() { return StackAdvisorCommandType.RECOMMEND_COMPONENT_LAYOUT; } + + @Override + protected TestResource updateResponse(StackAdvisorRequest request, TestResource response) { + return response; + } } public static class TestResource {
