Repository: ambari Updated Branches: refs/heads/trunk ee94c2f5b -> b0b58189c
AMBARI-6631. Provide host-layout recommendations via /recommendations endpoint on stack-version Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b0b58189 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b0b58189 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b0b58189 Branch: refs/heads/trunk Commit: b0b58189cc8713642102eadb46e4bfabfaf84c21 Parents: ee94c2f Author: Srimanth Gunturi <[email protected]> Authored: Mon Jul 28 12:10:42 2014 -0700 Committer: Srimanth Gunturi <[email protected]> Committed: Mon Jul 28 13:19:40 2014 -0700 ---------------------------------------------------------------------- ambari-server/conf/unix/ambari.properties | 3 + ambari-server/pom.xml | 4 + .../RecommendationResourceDefinition.java | 44 ++++ .../resources/ResourceInstanceFactoryImpl.java | 4 + .../ambari/server/api/services/BaseService.java | 2 +- .../server/api/services/LocalUriInfo.java | 144 +++++++++++ .../api/services/RecommendationService.java | 71 ++++++ .../server/api/services/StacksService.java | 4 +- .../services/parsers/JsonRequestBodyParser.java | 19 +- .../stackadvisor/StackAdvisorException.java | 31 +++ .../stackadvisor/StackAdvisorHelper.java | 241 +++++++++++++++++++ .../stackadvisor/StackAdvisorRunner.java | 125 ++++++++++ .../recommendations/RecommendationRequest.java | 91 +++++++ .../recommendations/RecommendationResponse.java | 208 ++++++++++++++++ .../server/configuration/Configuration.java | 13 + .../ambari/server/controller/AmbariServer.java | 3 + .../AbstractControllerResourceProvider.java | 2 + .../RecommendationResourceProvider.java | 199 +++++++++++++++ .../ambari/server/controller/spi/Resource.java | 2 + .../src/main/resources/key_properties.json | 5 + .../src/main/resources/properties.json | 17 ++ .../src/main/resources/scripts/stack_advisor.py | 148 ++++++++++++ .../stacks/HDP/2.0.6/services/stack_advisor.py | 202 ++++++++++++++++ .../main/resources/stacks/HDP/stack_advisor.py | 37 +++ .../server/api/services/GroupServiceTest.java | 2 +- .../server/api/services/MemberServiceTest.java | 2 +- .../api/services/RecommendationServiceTest.java | 85 +++++++ 27 files changed, 1699 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/b0b58189/ambari-server/conf/unix/ambari.properties ---------------------------------------------------------------------- diff --git a/ambari-server/conf/unix/ambari.properties b/ambari-server/conf/unix/ambari.properties index b5816af..b77ae32 100644 --- a/ambari-server/conf/unix/ambari.properties +++ b/ambari-server/conf/unix/ambari.properties @@ -29,6 +29,9 @@ webapp.dir=/usr/lib/ambari-server/web bootstrap.dir=/var/run/ambari-server/bootstrap bootstrap.script=/usr/lib/python2.6/site-packages/ambari_server/bootstrap.py bootstrap.setup_agent.script=/usr/lib/python2.6/site-packages/ambari_server/setupAgent.py +recommendations.dir=/var/run/ambari-server/stack-recommendations +stackadvisor.script=/var/lib/ambari-server/resources/scripts/stack_advisor.py + api.authenticate=true server.connection.max.idle.millis=900000 server.fqdn.service.url=http://169.254.169.254/latest/meta-data/public-hostname http://git-wip-us.apache.org/repos/asf/ambari/blob/b0b58189/ambari-server/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-server/pom.xml b/ambari-server/pom.xml index 789de18..a6d03ab 100644 --- a/ambari-server/pom.xml +++ b/ambari-server/pom.xml @@ -352,6 +352,9 @@ <directory>/var/run/ambari-server/bootstrap</directory> </mapping> <mapping> + <directory>/var/run/ambari-server/stack-recommendations</directory> + </mapping> + <mapping> <directory>/var/log/ambari-server</directory> </mapping> <mapping> @@ -515,6 +518,7 @@ <path>/usr/lib/ambari-server</path> <path>/var/run/ambari-server</path> <path>/var/run/ambari-server/bootstrap</path> + <path>/var/run/ambari-server/stack-recommendations</path> <path>/var/log/ambari-server</path> <path>/var/lib/ambari-server/resources/upgrade</path> </paths> http://git-wip-us.apache.org/repos/asf/ambari/blob/b0b58189/ambari-server/src/main/java/org/apache/ambari/server/api/resources/RecommendationResourceDefinition.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/resources/RecommendationResourceDefinition.java b/ambari-server/src/main/java/org/apache/ambari/server/api/resources/RecommendationResourceDefinition.java new file mode 100644 index 0000000..2c9e1f2 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/resources/RecommendationResourceDefinition.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.api.resources; + +import org.apache.ambari.server.controller.spi.Resource; + +/** + * Recommendation resource definition. + */ +public class RecommendationResourceDefinition extends BaseResourceDefinition { + /** + * Constructor. + */ + public RecommendationResourceDefinition() { + super(Resource.Type.Recommendation); + } + + @Override + public String getPluralName() { + return "recommendations"; + } + + @Override + public String getSingularName() { + return "recommendation"; + } + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/b0b58189/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ResourceInstanceFactoryImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ResourceInstanceFactoryImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ResourceInstanceFactoryImpl.java index cdc3cd0..892ac22 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ResourceInstanceFactoryImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ResourceInstanceFactoryImpl.java @@ -226,6 +226,10 @@ public class ResourceInstanceFactoryImpl implements ResourceInstanceFactory { resourceDefinition = new BlueprintResourceDefinition(); break; + case Recommendation: + resourceDefinition = new RecommendationResourceDefinition(); + break; + case HostComponentProcess: resourceDefinition = new HostComponentProcessResourceDefinition(); break; http://git-wip-us.apache.org/repos/asf/ambari/blob/b0b58189/ambari-server/src/main/java/org/apache/ambari/server/api/services/BaseService.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/BaseService.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/BaseService.java index fa3b3ba..3afc23d 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/BaseService.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/BaseService.java @@ -135,7 +135,7 @@ public abstract class BaseService { * * @return a newly created resource instance */ - ResourceInstance createResource(Resource.Type type, Map<Resource.Type, String> mapIds) { + protected ResourceInstance createResource(Resource.Type type, Map<Resource.Type, String> mapIds) { return m_resourceFactory.createResource(type, mapIds); } http://git-wip-us.apache.org/repos/asf/ambari/blob/b0b58189/ambari-server/src/main/java/org/apache/ambari/server/api/services/LocalUriInfo.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/LocalUriInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/LocalUriInfo.java new file mode 100644 index 0000000..9e927f6 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/LocalUriInfo.java @@ -0,0 +1,144 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.api.services; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.List; + +import javax.ws.rs.core.MultivaluedMap; +import javax.ws.rs.core.PathSegment; +import javax.ws.rs.core.UriBuilder; +import javax.ws.rs.core.UriInfo; + +import org.apache.http.NameValuePair; +import org.apache.http.client.utils.URLEncodedUtils; + +import com.sun.jersey.core.util.MultivaluedMapImpl; + +/** + * Internal {@link UriInfo} implementation. Most of the methods are not + * currently supported. + * + * Can be used for internal REST API calls with avoiding REST layer. + */ +public class LocalUriInfo implements UriInfo { + + private final URI uri; + + public LocalUriInfo(String uri) { + try { + this.uri = new URI(uri); + } catch (URISyntaxException e) { + throw new RuntimeException("URI syntax is not correct", e); + } + } + + @Override + public URI getAbsolutePath() { + throw new UnsupportedOperationException("Method is not supported"); + } + + @Override + public UriBuilder getAbsolutePathBuilder() { + throw new UnsupportedOperationException("Method is not supported"); + } + + @Override + public URI getBaseUri() { + throw new UnsupportedOperationException("Method is not supported"); + } + + @Override + public UriBuilder getBaseUriBuilder() { + throw new UnsupportedOperationException("Method is not supported"); + } + + @Override + public List<Object> getMatchedResources() { + throw new UnsupportedOperationException("Method is not supported"); + } + + @Override + public List<String> getMatchedURIs() { + throw new UnsupportedOperationException("Method is not supported"); + } + + @Override + public List<String> getMatchedURIs(boolean arg0) { + throw new UnsupportedOperationException("Method is not supported"); + } + + @Override + public String getPath() { + throw new UnsupportedOperationException("Method is not supported"); + } + + @Override + public String getPath(boolean arg0) { + throw new UnsupportedOperationException("Method is not supported"); + } + + @Override + public MultivaluedMap<String, String> getPathParameters() { + throw new UnsupportedOperationException("Method is not supported"); + } + + @Override + public MultivaluedMap<String, String> getPathParameters(boolean arg0) { + throw new UnsupportedOperationException("Method is not supported"); + } + + @Override + public List<PathSegment> getPathSegments() { + throw new UnsupportedOperationException("Method is not supported"); + } + + @Override + public List<PathSegment> getPathSegments(boolean arg0) { + throw new UnsupportedOperationException("Method is not supported"); + } + + @Override + public MultivaluedMap<String, String> getQueryParameters() { + List<NameValuePair> parametersList = URLEncodedUtils.parse(uri, "UTF-8"); + + MultivaluedMap<String, String> parameters = new MultivaluedMapImpl(); + for (NameValuePair pair : parametersList) { + parameters.add(pair.getName(), pair.getValue()); + } + return parameters; + } + + @Override + public MultivaluedMap<String, String> getQueryParameters(boolean arg0) { + throw new UnsupportedOperationException("Method is not supported"); + } + + @Override + public URI getRequestUri() { + return uri; + } + + @Override + public UriBuilder getRequestUriBuilder() { + throw new UnsupportedOperationException("Method is not supported"); + } + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/b0b58189/ambari-server/src/main/java/org/apache/ambari/server/api/services/RecommendationService.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/RecommendationService.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/RecommendationService.java new file mode 100644 index 0000000..23248b3 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/RecommendationService.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.api.services; + +import java.util.HashMap; +import java.util.Map; + +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.HttpHeaders; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.UriInfo; + +import org.apache.ambari.server.api.resources.ResourceInstance; +import org.apache.ambari.server.controller.spi.Resource; + +/** + * Service responsible for preparing recommendations for host-layout and + * configurations. + */ +@Path("/stacks/{stackName}/versions/{stackVersion}/recommendations") +public class RecommendationService extends BaseService { + + /** + * Returns host-layout recommendations for list of hosts and services. + * + * @param body http body + * @param headers http headers + * @param ui uri info + * @param stackName stack name + * @param stackVersion stack version + * @return recommendations for host-layout + */ + @POST + @Produces(MediaType.TEXT_PLAIN) + public Response getRecommendation(String body, @Context HttpHeaders headers, @Context UriInfo ui, + @PathParam("stackName") String stackName, @PathParam("stackVersion") String stackVersion) { + + return handleRequest(headers, body, ui, Request.Type.POST, + createRecommendationResource(stackName, stackVersion)); + } + + ResourceInstance createRecommendationResource(String stackName, String stackVersion) { + Map<Resource.Type, String> mapIds = new HashMap<Resource.Type, String>(); + mapIds.put(Resource.Type.Stack, stackName); + mapIds.put(Resource.Type.StackVersion, stackVersion); + + return createResource(Resource.Type.Recommendation, mapIds); + } + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/b0b58189/ambari-server/src/main/java/org/apache/ambari/server/api/services/StacksService.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/StacksService.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/StacksService.java index fc6e821..822199c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/StacksService.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/StacksService.java @@ -358,10 +358,10 @@ public class StacksService extends BaseService { * This should be removed when /stacks2 is removed and we can change the property names * in the resource definitions to the new form. */ - private static class StackUriInfo implements UriInfo { + public static class StackUriInfo implements UriInfo { private UriInfo m_delegate; - private StackUriInfo(UriInfo delegate) { + public StackUriInfo(UriInfo delegate) { m_delegate = delegate; } @Override http://git-wip-us.apache.org/repos/asf/ambari/blob/b0b58189/ambari-server/src/main/java/org/apache/ambari/server/api/services/parsers/JsonRequestBodyParser.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/parsers/JsonRequestBodyParser.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/parsers/JsonRequestBodyParser.java index 0d75a8e..59dd1af 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/parsers/JsonRequestBodyParser.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/parsers/JsonRequestBodyParser.java @@ -113,13 +113,24 @@ public class JsonRequestBodyParser implements RequestBodyParser { //array Iterator<JsonNode> arrayIter = child.getElements(); Set<Map<String, Object>> arraySet = new HashSet<Map<String, Object>>(); + List<String> primitives = new ArrayList<String>(); while (arrayIter.hasNext()) { - NamedPropertySet arrayPropertySet = new NamedPropertySet(name, new HashMap<String, Object>()); - processNode(arrayIter.next(), "", arrayPropertySet, requestInfoProps); - arraySet.add(arrayPropertySet.getProperties()); + JsonNode next = arrayIter.next(); + + if (next.isValueNode()) { + // All remain nodes will be also primitives + primitives.add(next.getTextValue()); + } else { + NamedPropertySet arrayPropertySet = new NamedPropertySet(name, + new HashMap<String, Object>()); + processNode(next, "", arrayPropertySet, requestInfoProps); + arraySet.add(arrayPropertySet.getProperties()); + } } - propertySet.getProperties().put(PropertyHelper.getPropertyId(path, name), arraySet); + + Object properties = primitives.isEmpty() ? arraySet : primitives; + propertySet.getProperties().put(PropertyHelper.getPropertyId(path, name), properties); } else if (child.isContainerNode()) { // object if (name.equals(BODY_TITLE)) { http://git-wip-us.apache.org/repos/asf/ambari/blob/b0b58189/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorException.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorException.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorException.java new file mode 100644 index 0000000..b13e1e5 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorException.java @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.api.services.stackadvisor; + +@SuppressWarnings("serial") +public class StackAdvisorException extends Exception { + + public StackAdvisorException(String message) { + super(message); + } + + public StackAdvisorException(String message, Throwable cause) { + super(message, cause); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/b0b58189/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelper.java new file mode 100644 index 0000000..c0799c9 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelper.java @@ -0,0 +1,241 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.api.services.stackadvisor; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import javax.ws.rs.core.Response; +import javax.ws.rs.core.Response.Status; + +import org.apache.ambari.server.api.resources.ResourceInstance; +import org.apache.ambari.server.api.services.BaseService; +import org.apache.ambari.server.api.services.LocalUriInfo; +import org.apache.ambari.server.api.services.RecommendationService; +import org.apache.ambari.server.api.services.Request; +import org.apache.ambari.server.api.services.StacksService.StackUriInfo; +import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRunner.StackAdvisorCommand; +import org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationRequest; +import org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse; +import org.apache.ambari.server.configuration.Configuration; +import org.apache.ambari.server.controller.spi.Resource; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.io.FileUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.codehaus.jackson.JsonNode; +import org.codehaus.jackson.map.ObjectMapper; + +import com.google.inject.Inject; +import com.google.inject.Singleton; + +@Singleton +public class StackAdvisorHelper extends BaseService { + + private static Log LOG = LogFactory.getLog(RecommendationService.class); + + private static final String GET_HOSTS_INFO_URI = "/api/v1/hosts" + + "?fields=*&Hosts/host_name.in(%s)"; + private static final String GET_SERVICES_INFO_URI = "/api/v1/stacks/%s/versions/%s" + + "?fields=Versions/stack_name,Versions/stack_version,Versions/parent_stack_version" + + ",services/StackServices/service_name,services/StackServices/service_version" + + ",services/components/StackServiceComponents,services/components/dependencies,services/components/auto_deploy" + + "&services/StackServices/service_name.in(%s)"; + + private File recommendationsDir; + private String stackAdvisorScript; + + /* Monotonically increasing requestid */ + private int requestId = 0; + private File requestDirectory; + private StackAdvisorRunner saRunner; + + @Inject + public StackAdvisorHelper(Configuration conf, StackAdvisorRunner saRunner) throws IOException { + this.recommendationsDir = conf.getRecommendationsDir(); + this.stackAdvisorScript = conf.getStackAdvisorScript(); + this.saRunner = saRunner; + } + + /** + * Return component-layout recommendation based on hosts and services + * information. + * + * @param hosts list of hosts + * @param services list of services + * @return {@link String} representation of recommendations + * @throws StackAdvisorException + */ + public synchronized RecommendationResponse getComponentLayoutRecommnedation( + RecommendationRequest request) throws StackAdvisorException { + + validateRecommendationRequest(request); + String hostsJSON = getHostsInformation(request); + String servicesJSON = getServicesInformation(request); + + try { + createRequestDirectory(); + + FileUtils.writeStringToFile(new File(requestDirectory, "hosts.json"), hostsJSON); + FileUtils.writeStringToFile(new File(requestDirectory, "services.json"), servicesJSON); + + boolean success = saRunner.runScript(stackAdvisorScript, + StackAdvisorCommand.RECOMMEND_COMPONENT_LAYOUT, requestDirectory); + if (!success) { + String message = "Stack advisor script finished with errors"; + LOG.warn(message); + throw new StackAdvisorException(message); + } + + String result = FileUtils + .readFileToString(new File(requestDirectory, "component-layout.json")); + + ObjectMapper mapper = new ObjectMapper(); + return mapper.readValue(result, RecommendationResponse.class); + } catch (Exception e) { + String message = "Error occured during preparing component-layout recommendations"; + LOG.warn(message, e); + throw new StackAdvisorException(message, e); + } + } + + /** + * Create request id directory for each call + */ + private void createRequestDirectory() throws IOException { + if (!recommendationsDir.exists()) { + if (!recommendationsDir.mkdirs()) { + throw new IOException("Cannot create " + recommendationsDir); + } + } + + requestId += 1; + requestDirectory = new File(recommendationsDir, Integer.toString(requestId)); + + if (requestDirectory.exists()) { + FileUtils.deleteDirectory(requestDirectory); + } + if (!requestDirectory.mkdirs()) { + throw new IOException("Cannot create " + requestDirectory); + } + } + + private String getHostsInformation(RecommendationRequest request) throws StackAdvisorException { + String hostsURI = String.format(GET_HOSTS_INFO_URI, request.getHostsCommaSeparated()); + + Response response = handleRequest(null, null, new LocalUriInfo(hostsURI), Request.Type.GET, + createHostResource()); + + if (response.getStatus() != Status.OK.getStatusCode()) { + String message = String.format( + "Error occured during hosts information retrieving, status=%s, response=%s", + response.getStatus(), (String) response.getEntity()); + LOG.warn(message); + throw new StackAdvisorException(message); + } + + String hostsJSON = (String) response.getEntity(); + if (LOG.isDebugEnabled()) { + LOG.debug("Hosts information: " + hostsJSON); + } + + Collection<String> unregistered = getUnregisteredHosts(hostsJSON, request.getHosts()); + if (unregistered.size() > 0) { + String message = String.format("There are unregistered hosts in the request, %s", + Arrays.toString(unregistered.toArray())); + LOG.warn(message); + throw new StackAdvisorException(message); + } + + return hostsJSON; + } + + @SuppressWarnings("unchecked") + private Collection<String> getUnregisteredHosts(String hostsJSON, List<String> hosts) + throws StackAdvisorException { + ObjectMapper mapper = new ObjectMapper(); + List<String> registeredHosts = new ArrayList<String>(); + + try { + JsonNode root = mapper.readTree(hostsJSON); + Iterator<JsonNode> iterator = root.get("items").getElements(); + while (iterator.hasNext()) { + JsonNode next = iterator.next(); + String hostName = next.get("Hosts").get("host_name").getTextValue(); + registeredHosts.add(hostName); + } + + return CollectionUtils.subtract(hosts, registeredHosts); + } catch (Exception e) { + throw new StackAdvisorException("Error occured during calculating unregistered hosts", e); + } + } + + private String getServicesInformation(RecommendationRequest request) throws StackAdvisorException { + String stackName = request.getStackName(); + String stackVersion = request.getStackVersion(); + String servicesURI = String.format(GET_SERVICES_INFO_URI, stackName, stackVersion, + request.getServicesCommaSeparated()); + + Response response = handleRequest(null, null, new StackUriInfo(new LocalUriInfo(servicesURI)), + Request.Type.GET, createStackVersionResource(stackName, stackVersion)); + + if (response.getStatus() != Status.OK.getStatusCode()) { + String message = String.format( + "Error occured during services information retrieving, status=%s, response=%s", + response.getStatus(), (String) response.getEntity()); + LOG.warn(message); + throw new StackAdvisorException(message); + } + + String servicesJSON = (String) response.getEntity(); + if (LOG.isDebugEnabled()) { + LOG.debug("Services information: " + servicesJSON); + } + return servicesJSON; + } + + private ResourceInstance createHostResource() { + Map<Resource.Type, String> mapIds = new HashMap<Resource.Type, String>(); + return createResource(Resource.Type.Host, mapIds); + } + + private ResourceInstance createStackVersionResource(String stackName, String stackVersion) { + Map<Resource.Type, String> mapIds = new HashMap<Resource.Type, String>(); + mapIds.put(Resource.Type.Stack, stackName); + mapIds.put(Resource.Type.StackVersion, stackVersion); + + return createResource(Resource.Type.StackVersion, mapIds); + } + + private void validateRecommendationRequest(RecommendationRequest request) + throws StackAdvisorException { + if (request.getHosts().isEmpty() || request.getServices().isEmpty()) { + throw new StackAdvisorException("Hosts and services must not be empty"); + } + } + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/b0b58189/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRunner.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRunner.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRunner.java new file mode 100644 index 0000000..57feaa0 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRunner.java @@ -0,0 +1,125 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.api.services.stackadvisor; + +import java.io.File; +import java.io.IOException; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import com.google.inject.Singleton; + +@Singleton +public class StackAdvisorRunner { + + private static Log LOG = LogFactory.getLog(StackAdvisorRunner.class); + + /** + * Runs stack_advisor.py script in the specified {@code actionDirectory}. + * + * @param script stack advisor script + * @param saCommand {@link StackAdvisorCommand} to run. + * @param actionDirectory directory for the action + * @return {@code true} if script completed successfully, {@code false} + * otherwise. + */ + public boolean runScript(String script, StackAdvisorCommand saCommand, File actionDirectory) { + LOG.info(String.format("Script=%s, actionDirectory=%s, command=%s", script, actionDirectory, + saCommand)); + + String outputFile = actionDirectory + File.separator + "stackadvisor.out"; + String errorFile = actionDirectory + File.separator + "stackadvisor.err"; + + String shellCommand[] = prepareShellCommand(script, saCommand, actionDirectory, outputFile, + errorFile); + String[] env = new String[] {}; + + try { + Process process = Runtime.getRuntime().exec(shellCommand, env); + + try { + LOG.info(String.format("Stack-advisor output=%s, error=%s", outputFile, errorFile)); + + int exitCode = process.waitFor(); + try { + String outMessage = FileUtils.readFileToString(new File(outputFile)); + String errMessage = FileUtils.readFileToString(new File(errorFile)); + LOG.info("Script log message: " + outMessage + "\n\n" + errMessage); + } catch (IOException io) { + LOG.info("Error in reading script log files", io); + } + + return exitCode == 0; + } finally { + process.destroy(); + } + } catch (Exception io) { + LOG.info("Error executing stack advisor " + io.getMessage()); + return false; + } + } + + private String[] prepareShellCommand(String script, StackAdvisorCommand saCommand, + File actionDirectory, String outputFile, String errorFile) { + String hostsFile = actionDirectory + File.separator + "hosts.json"; + String servicesFile = actionDirectory + File.separator + "services.json"; + + String shellCommand[] = new String[] { "sh", "-c", null /* to be calculated */}; + String commands[] = new String[] { script, saCommand.toString(), hostsFile, servicesFile }; + + StringBuilder commandString = new StringBuilder(); + for (String command : commands) { + commandString.append(" " + command); + } + + if (LOG.isDebugEnabled()) { + LOG.debug(commandString); + } + + commandString.append(" 1> " + outputFile + " 2>" + errorFile); + shellCommand[2] = commandString.toString(); + + return shellCommand; + } + + public enum StackAdvisorCommand { + + RECOMMEND_COMPONENT_LAYOUT("recommend-component-layout"), + + VALIDATE_COMPONENT_LAYOUT("validate-component-layout"), + + RECOMMEND_CONFIGURATIONS("recommend-configurations"), + + VALIDATE_CONFIGURATIONS("validate-configurations"); + + private final String name; + + private StackAdvisorCommand(String name) { + this.name = name; + } + + @Override + public String toString() { + return name; + } + } + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/b0b58189/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/recommendations/RecommendationRequest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/recommendations/RecommendationRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/recommendations/RecommendationRequest.java new file mode 100644 index 0000000..5ab6b7a --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/recommendations/RecommendationRequest.java @@ -0,0 +1,91 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.api.services.stackadvisor.recommendations; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.lang.StringUtils; + +/** + * Recommendations request. + */ +public class RecommendationRequest { + + private String stackName; + private String stackVersion; + private List<String> hosts = new ArrayList<String>(); + private List<String> services = new ArrayList<String>(); + + + public String getStackName() { + return stackName; + } + + public String getStackVersion() { + return stackVersion; + } + + public List<String> getHosts() { + return hosts; + } + + public List<String> getServices() { + return services; + } + + public String getHostsCommaSeparated() { + return StringUtils.join(hosts, ","); + } + + public String getServicesCommaSeparated() { + return StringUtils.join(services, ","); + } + + private RecommendationRequest(String stackName, String stackVersion) { + this.stackName = stackName; + this.stackVersion = stackVersion; + } + + public static class RecommendationRequestBuilder { + RecommendationRequest instance; + + private RecommendationRequestBuilder(String stackName, String stackVersion) { + this.instance = new RecommendationRequest(stackName, stackVersion); + } + + public static RecommendationRequestBuilder forStack(String stackName, String stackVersion) { + return new RecommendationRequestBuilder(stackName, stackVersion); + } + + public RecommendationRequestBuilder forHosts(List<String> hosts) { + this.instance.hosts = hosts; + return this; + } + + public RecommendationRequestBuilder forServices(List<String> services) { + this.instance.services = services; + return this; + } + + public RecommendationRequest build() { + return this.instance; + } + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/b0b58189/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/recommendations/RecommendationResponse.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/recommendations/RecommendationResponse.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/recommendations/RecommendationResponse.java new file mode 100644 index 0000000..a34291c --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/recommendations/RecommendationResponse.java @@ -0,0 +1,208 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.api.services.stackadvisor.recommendations; + +import java.util.Map; +import java.util.Set; + +import org.codehaus.jackson.annotate.JsonProperty; + +/** + * Recommendation response POJO. + */ +public class RecommendationResponse { + + @JsonProperty("Versions") + private Version version; + + @JsonProperty + private Set<String> hosts; + + @JsonProperty + private Set<String> services; + + @JsonProperty + private Recommendation recommendations; + + public Version getVersion() { + return version; + } + + public void setVersion(Version version) { + this.version = version; + } + + public Set<String> getHosts() { + return hosts; + } + + public void setHosts(Set<String> hosts) { + this.hosts = hosts; + } + + public Set<String> getServices() { + return services; + } + + public void setServices(Set<String> services) { + this.services = services; + } + + public Recommendation getRecommendations() { + return recommendations; + } + + public void setRecommendations(Recommendation recommendations) { + this.recommendations = recommendations; + } + + public static class Version { + @JsonProperty("stack_name") + private String stackName; + + @JsonProperty("stack_version") + private String stackVersion; + + public String getStackName() { + return stackName; + } + + public void setStackName(String stackName) { + this.stackName = stackName; + } + + public String getStackVersion() { + return stackVersion; + } + + public void setStackVersion(String stackVersion) { + this.stackVersion = stackVersion; + } + } + + public static class Recommendation { + @JsonProperty + private Blueprint blueprint; + + @JsonProperty("blueprint_cluster_binding") + private BlueprintClusterBinding blueprintClusterBinding; + + public Blueprint getBlueprint() { + return blueprint; + } + + public void setBlueprint(Blueprint blueprint) { + this.blueprint = blueprint; + } + + public BlueprintClusterBinding getBlueprintClusterBinding() { + return blueprintClusterBinding; + } + + public void setBlueprintClusterBinding(BlueprintClusterBinding blueprintClusterBinding) { + this.blueprintClusterBinding = blueprintClusterBinding; + } + } + + public static class Blueprint { + @JsonProperty + private Map<String, Map<String, Map<String, String>>> configurations; + + @JsonProperty("host_groups") + private Set<HostGroup> hostGroups; + + public Map<String, Map<String, Map<String, String>>> getConfigurations() { + return configurations; + } + + public void setConfigurations(Map<String, Map<String, Map<String, String>>> configurations) { + this.configurations = configurations; + } + + public Set<HostGroup> getHostGroups() { + return hostGroups; + } + + public void setHostGroups(Set<HostGroup> hostGroups) { + this.hostGroups = hostGroups; + } + } + + public static class HostGroup { + @JsonProperty + private String name; + + @JsonProperty + private Set<Map<String, String>> components; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Set<Map<String, String>> getComponents() { + return components; + } + + public void setComponents(Set<Map<String, String>> components) { + this.components = components; + } + } + + public static class BlueprintClusterBinding { + @JsonProperty("host_groups") + private Set<BindingHostGroup> hostGroups; + + public Set<BindingHostGroup> getHostGroups() { + return hostGroups; + } + + public void setHostGroups(Set<BindingHostGroup> hostGroups) { + this.hostGroups = hostGroups; + } + } + + public static class BindingHostGroup { + @JsonProperty + private String name; + + @JsonProperty + private Set<Map<String, String>> hosts; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Set<Map<String, String>> getHosts() { + return hosts; + } + + public void setHosts(Set<Map<String, String>> hosts) { + this.hosts = hosts; + } + } + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/b0b58189/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java index c585402..68df120 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java @@ -67,6 +67,10 @@ public class Configuration { public static final String BOOTSTRAP_SETUP_AGENT_SCRIPT = "bootstrap.setup_agent.script"; public static final String BOOTSTRAP_SETUP_AGENT_PASSWORD = "bootstrap.setup_agent.password"; public static final String BOOTSTRAP_MASTER_HOSTNAME = "bootstrap.master_host_name"; + public static final String RECOMMENDATIONS_DIR = "recommendations.dir"; + public static final String RECOMMENDATIONS_DIR_DEFAULT = "/var/run/ambari-server/stack-recommendations"; + public static final String STACK_ADVISOR_SCRIPT = "stackadvisor.script"; + public static final String STACK_ADVISOR_SCRIPT_DEFAULT = "/var/lib/ambari-server/resources/scripts/stack_advisor.py"; public static final String API_AUTHENTICATE = "api.authenticate"; public static final String API_USE_SSL = "api.ssl"; public static final String API_CSRF_PREVENTION_KEY = "api.csrfPrevention.enabled"; @@ -526,6 +530,15 @@ public class Configuration { return properties.getProperty(BOOTSTRAP_SETUP_AGENT_PASSWORD, "password"); } + public File getRecommendationsDir() { + String fileName = properties.getProperty(RECOMMENDATIONS_DIR, RECOMMENDATIONS_DIR_DEFAULT); + return new File(fileName); + } + + public String getStackAdvisorScript() { + return properties.getProperty(STACK_ADVISOR_SCRIPT, STACK_ADVISOR_SCRIPT_DEFAULT); + } + /** * Get the map with server config parameters. * Keys - public constants of this class http://git-wip-us.apache.org/repos/asf/ambari/blob/b0b58189/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java index 1ea8311..6803a01 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java @@ -40,6 +40,7 @@ import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.api.services.KeyService; import org.apache.ambari.server.api.services.PersistKeyValueImpl; import org.apache.ambari.server.api.services.PersistKeyValueService; +import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorHelper; import org.apache.ambari.server.bootstrap.BootStrapImpl; import org.apache.ambari.server.configuration.ComponentSSLConfiguration; import org.apache.ambari.server.configuration.Configuration; @@ -49,6 +50,7 @@ import org.apache.ambari.server.controller.internal.BlueprintResourceProvider; import org.apache.ambari.server.controller.internal.ClusterResourceProvider; import org.apache.ambari.server.controller.internal.PermissionResourceProvider; import org.apache.ambari.server.controller.internal.PrivilegeResourceProvider; +import org.apache.ambari.server.controller.internal.RecommendationResourceProvider; import org.apache.ambari.server.controller.internal.StackDefinedPropertyProvider; import org.apache.ambari.server.controller.internal.StackDependencyResourceProvider; import org.apache.ambari.server.controller.nagios.NagiosPropertyProvider; @@ -525,6 +527,7 @@ public class AmbariServer { PersistKeyValueService.init(injector.getInstance(PersistKeyValueImpl.class)); KeyService.init(injector.getInstance(PersistKeyValueImpl.class)); BootStrapResource.init(injector.getInstance(BootStrapImpl.class)); + RecommendationResourceProvider.init(injector.getInstance(StackAdvisorHelper.class)); StageUtils.setGson(injector.getInstance(Gson.class)); WorkflowJsonService.setDBProperties( injector.getInstance(Configuration.class)); http://git-wip-us.apache.org/repos/asf/ambari/blob/b0b58189/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractControllerResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractControllerResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractControllerResourceProvider.java index 5ba926c..028bf41 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractControllerResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractControllerResourceProvider.java @@ -144,6 +144,8 @@ public abstract class AbstractControllerResourceProvider extends AbstractResourc return new HostComponentProcessResourceProvider(propertyIds, keyPropertyIds, managementController); case Blueprint: return new BlueprintResourceProvider(propertyIds, keyPropertyIds, managementController); + case Recommendation: + return new RecommendationResourceProvider(propertyIds, keyPropertyIds, managementController); case AlertDefinition: return new AlertDefinitionResourceProvider(propertyIds, keyPropertyIds, managementController); default: http://git-wip-us.apache.org/repos/asf/ambari/blob/b0b58189/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RecommendationResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RecommendationResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RecommendationResourceProvider.java new file mode 100644 index 0000000..d4fd1ce --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RecommendationResourceProvider.java @@ -0,0 +1,199 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.controller.internal; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.Response.Status; + +import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorException; +import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorHelper; +import org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationRequest; +import org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationRequest.RecommendationRequestBuilder; +import org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse; +import org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse.BindingHostGroup; +import org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse.HostGroup; +import org.apache.ambari.server.controller.AmbariManagementController; +import org.apache.ambari.server.controller.spi.NoSuchParentResourceException; +import org.apache.ambari.server.controller.spi.Request; +import org.apache.ambari.server.controller.spi.RequestStatus; +import org.apache.ambari.server.controller.spi.Resource; +import org.apache.ambari.server.controller.spi.Resource.Type; +import org.apache.ambari.server.controller.spi.ResourceAlreadyExistsException; +import org.apache.ambari.server.controller.spi.SystemException; +import org.apache.ambari.server.controller.spi.UnsupportedPropertyException; +import org.apache.ambari.server.controller.utilities.PropertyHelper; + +import com.google.inject.Inject; + +public class RecommendationResourceProvider extends ReadOnlyResourceProvider { + + private static StackAdvisorHelper saHelper; + + @Inject + public static void init(StackAdvisorHelper instance) { + saHelper = instance; + } + + protected static final String RECOMMENDATION_ID_PROPERTY_ID = PropertyHelper.getPropertyId( + "Recommendations", "id"); + protected static final String STACK_NAME_PROPERTY_ID = PropertyHelper.getPropertyId("Versions", + "stack_name"); + protected static final String STACK_VERSION_PROPERTY_ID = PropertyHelper.getPropertyId( + "Versions", "stack_version"); + + protected static final String HOSTS_PROPERTY_ID = "hosts"; + protected static final String SERVICES_PROPERTY_ID = "services"; + + protected static final String BLUEPRINT_CONFIGURATIONS_PROPERTY_ID = PropertyHelper + .getPropertyId("recommendations/blueprint", "configurations"); + + protected static final String BLUEPRINT_HOST_GROUPS_PROPERTY_ID = PropertyHelper.getPropertyId( + "recommendations/blueprint", "host_groups"); + protected static final String BLUEPRINT_HOST_GROUPS_NAME_PROPERTY_ID = "name"; + protected static final String BLUEPRINT_HOST_GROUPS_COMPONENTS_PROPERTY_ID = "components"; + + protected static final String BINDING_HOST_GROUPS_PROPERTY_ID = PropertyHelper.getPropertyId( + "recommendations/blueprint_cluster_binding", "host_groups"); + protected static final String BINDING_HOST_GROUPS_NAME_PROPERTY_ID = "name"; + protected static final String BINDING_HOST_GROUPS_HOSTS_PROPERTY_ID = "hosts"; + + private static Set<String> pkPropertyIds = new HashSet<String>( + Arrays.asList(new String[] { RECOMMENDATION_ID_PROPERTY_ID })); + + protected RecommendationResourceProvider(Set<String> propertyIds, + Map<Type, String> keyPropertyIds, AmbariManagementController managementController) { + super(propertyIds, keyPropertyIds, managementController); + } + + @Override + public RequestStatus createResources(final Request request) throws SystemException, + UnsupportedPropertyException, ResourceAlreadyExistsException, NoSuchParentResourceException { + RecommendationRequest recommendationRequest = prepareRecommendationRequest(request); + + final RecommendationResponse response; + try { + response = saHelper.getComponentLayoutRecommnedation(recommendationRequest); + } catch (StackAdvisorException e) { + LOG.warn("Error occured during component-layout recommnedation", e); + throw new WebApplicationException(Response.status(Status.BAD_REQUEST).entity(e.getMessage()) + .build()); + } + + Resource recommendation = createResources(new Command<Resource>() { + @Override + public Resource invoke() throws AmbariException { + + Resource resource = new ResourceImpl(Resource.Type.Recommendation); + setResourceProperty(resource, RECOMMENDATION_ID_PROPERTY_ID, "1", getPropertyIds()); + setResourceProperty(resource, STACK_NAME_PROPERTY_ID, response.getVersion().getStackName(), + getPropertyIds()); + setResourceProperty(resource, STACK_VERSION_PROPERTY_ID, response.getVersion() + .getStackVersion(), getPropertyIds()); + setResourceProperty(resource, HOSTS_PROPERTY_ID, response.getHosts(), getPropertyIds()); + setResourceProperty(resource, SERVICES_PROPERTY_ID, response.getServices(), + getPropertyIds()); + setResourceProperty(resource, BLUEPRINT_CONFIGURATIONS_PROPERTY_ID, response + .getRecommendations().getBlueprint().getConfigurations(), getPropertyIds()); + + Set<HostGroup> hostGroups = response.getRecommendations().getBlueprint().getHostGroups(); + List<Map<String, Object>> listGroupProps = new ArrayList<Map<String, Object>>(); + for (HostGroup hostGroup : hostGroups) { + Map<String, Object> mapGroupProps = new HashMap<String, Object>(); + mapGroupProps.put(BLUEPRINT_HOST_GROUPS_NAME_PROPERTY_ID, hostGroup.getName()); + mapGroupProps + .put(BLUEPRINT_HOST_GROUPS_COMPONENTS_PROPERTY_ID, hostGroup.getComponents()); + listGroupProps.add(mapGroupProps); + } + setResourceProperty(resource, BLUEPRINT_HOST_GROUPS_PROPERTY_ID, listGroupProps, + getPropertyIds()); + + Set<BindingHostGroup> bindingHostGroups = response.getRecommendations() + .getBlueprintClusterBinding().getHostGroups(); + List<Map<String, Object>> listBindingGroupProps = new ArrayList<Map<String, Object>>(); + for (BindingHostGroup hostGroup : bindingHostGroups) { + Map<String, Object> mapGroupProps = new HashMap<String, Object>(); + mapGroupProps.put(BINDING_HOST_GROUPS_NAME_PROPERTY_ID, hostGroup.getName()); + mapGroupProps.put(BINDING_HOST_GROUPS_HOSTS_PROPERTY_ID, hostGroup.getHosts()); + listBindingGroupProps.add(mapGroupProps); + } + setResourceProperty(resource, BINDING_HOST_GROUPS_PROPERTY_ID, listBindingGroupProps, + getPropertyIds()); + + return resource; + } + }); + notifyCreate(Resource.Type.Recommendation, request); + + Set<Resource> resources = new HashSet<Resource>(Arrays.asList(recommendation)); + return new RequestStatusImpl(null, resources); + } + + @SuppressWarnings("unchecked") + private RecommendationRequest prepareRecommendationRequest(Request request) { + try { + String stackName = (String) getRequestProperty(request, STACK_NAME_PROPERTY_ID); + String stackVersion = (String) getRequestProperty(request, STACK_VERSION_PROPERTY_ID); + + /* + * ClassCastException will occur if hosts or services are empty in the + * request. + * + * @see JsonRequestBodyParser for arrays parsing + */ + List<String> hosts = (List<String>) getRequestProperty(request, "hosts"); + List<String> services = (List<String>) getRequestProperty(request, "services"); + + RecommendationRequest recommendationRequest = RecommendationRequestBuilder + .forStack(stackName, stackVersion).forHosts(hosts).forServices(services).build(); + + return recommendationRequest; + } catch (Exception e) { + LOG.warn("Error occured during preparation of recommendation request", e); + + Response response = Response.status(Status.BAD_REQUEST) + .entity("Hosts and services must not be empty").build(); + throw new WebApplicationException(response); + } + } + + private Object getRequestProperty(Request request, String propertyName) { + for (Map<String, Object> propertyMap : request.getProperties()) { + if (propertyMap.containsKey(propertyName)) { + return propertyMap.get(propertyName); + } + } + return null; + } + + @Override + protected Set<String> getPKPropertyIds() { + return pkPropertyIds; + } + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/b0b58189/ambari-server/src/main/java/org/apache/ambari/server/controller/spi/Resource.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/spi/Resource.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/spi/Resource.java index 53f8a9c..50519cc 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/spi/Resource.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/spi/Resource.java @@ -109,6 +109,7 @@ public interface Resource { ViewVersion, ViewInstance, Blueprint, + Recommendation, HostComponentProcess, Permission, AlertDefinition, @@ -184,6 +185,7 @@ public interface Resource { public static final Type ViewVersion = InternalType.ViewVersion.getType(); public static final Type ViewInstance = InternalType.ViewInstance.getType(); public static final Type Blueprint = InternalType.Blueprint.getType(); + public static final Type Recommendation = InternalType.Recommendation.getType(); public static final Type HostComponentProcess = InternalType.HostComponentProcess.getType(); public static final Type Permission = InternalType.Permission.getType(); public static final Type AlertDefinition = InternalType.AlertDefinition.getType(); http://git-wip-us.apache.org/repos/asf/ambari/blob/b0b58189/ambari-server/src/main/resources/key_properties.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/key_properties.json b/ambari-server/src/main/resources/key_properties.json index 13adde8..4e9c379 100644 --- a/ambari-server/src/main/resources/key_properties.json +++ b/ambari-server/src/main/resources/key_properties.json @@ -124,6 +124,11 @@ "Blueprint": { "Blueprint": "Blueprints/blueprint_name" }, + "Recommendation": { + "Recommendation": "Recommendation/id", + "Stack": "Versions/stack_name", + "StackVersion": "Versions/stack_version" + }, "HostComponentProcess": { "Cluster": "HostComponentProcess/cluster_name", "Host": "HostComponentProcess/host_name", http://git-wip-us.apache.org/repos/asf/ambari/blob/b0b58189/ambari-server/src/main/resources/properties.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/properties.json b/ambari-server/src/main/resources/properties.json index 543773e..170fa4b 100644 --- a/ambari-server/src/main/resources/properties.json +++ b/ambari-server/src/main/resources/properties.json @@ -352,6 +352,23 @@ "configurations", "validate_topology" ], + "Recommendation":[ + "Recommendation/id", + "Versions/stack_name", + "Versions/stack_version", + "hosts", + "services", + "recommendations", + "recommendations/blueprint", + "recommendations/blueprint/configurations", + "recommendations/blueprint/host_groups", + "recommendations/blueprint/host_groups/name", + "recommendations/blueprint/host_groups/components", + "recommendations/blueprint_cluster_binding", + "recommendations/blueprint_cluster_binding/host_groups", + "recommendations/blueprint_cluster_binding/host_groups/name", + "recommendations/blueprint_cluster_binding/host_groups/hosts" + ], "HostComponentProcess": [ "HostComponentProcess/cluster_name", "HostComponentProcess/host_name", http://git-wip-us.apache.org/repos/asf/ambari/blob/b0b58189/ambari-server/src/main/resources/scripts/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/scripts/stack_advisor.py b/ambari-server/src/main/resources/scripts/stack_advisor.py new file mode 100644 index 0000000..1eb5e4d --- /dev/null +++ b/ambari-server/src/main/resources/scripts/stack_advisor.py @@ -0,0 +1,148 @@ +#!/usr/bin/env ambari-python-wrap + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' +import StringIO + +import json +import os +import sys +import traceback + +RECOMMEND_COMPONENT_LAYOUT_ACTION = 'recommend-component-layout' +VALIDATE_COMPONENT_LAYOUT_ACTION = 'validate-component-layout' +RECOMMEND_CONFIGURATIONS = 'recommend-configurations' +VALIDATE_CONFIGURATIONS = 'validate-configurations' + +ALL_ACTIONS = [ RECOMMEND_COMPONENT_LAYOUT_ACTION, VALIDATE_COMPONENT_LAYOUT_ACTION, RECOMMEND_CONFIGURATIONS, VALIDATE_CONFIGURATIONS ] +USAGE = "Usage: <action> <hosts_file> <services_file>\nPossible actions are: {0}\n".format( str(ALL_ACTIONS) ) + +SCRIPT_DIRECTORY = os.path.dirname(os.path.abspath(__file__)) +STACK_ADVISOR_PATH_TEMPLATE = os.path.join(SCRIPT_DIRECTORY, '../stacks/{0}/stack_advisor.py') +STACK_ADVISOR_IMPL_PATH_TEMPLATE = os.path.join(SCRIPT_DIRECTORY, './../stacks/{0}/{1}/services/stack_advisor.py') +STACK_ADVISOR_IMPL_CLASS_TEMPLATE = '{0}{1}StackAdvisor' + + +class StackAdvisorException(Exception): + pass + +def loadJson(path): + try: + with open(path, 'r') as f: + return json.load(f) + except Exception, err: + raise StackAdvisorException("File not found at: {0}".format(hostsFile)) + + +def dumpJson(json_object, dump_file): + try: + with open(dump_file, 'w') as out: + json.dump(json_object, out, indent=1) + except Exception, err: + raise StackAdvisorException("Can not write to file {0} : {1}".format(dump_file, str(err))) + + +def main(argv=None): + args = argv[1:] + + if len(args) < 3: + sys.stderr.write(USAGE) + sys.exit(2) + pass + + action = args[0] + if action not in ALL_ACTIONS: + sys.stderr.write(USAGE) + sys.exit(2) + pass + + hostsFile = args[1] + servicesFile = args[2] + + # Parse hostsFile and servicesFile + hosts = loadJson(hostsFile) + services = loadJson(servicesFile) + + # Instantiate StackAdvisor and call action related method + stackName = services["Versions"]["stack_name"] + stackVersion = services["Versions"]["stack_version"] + parentVersions = [] + if "parent_stack_version" in services["Versions"]: + parentVersions = [ services["Versions"]["parent_stack_version"] ] + + stackAdvisor = instantiateStackAdvisor(stackName, stackVersion, parentVersions) + + # Perform action + actionDir = os.path.realpath(os.path.dirname(args[1])) + result = {} + result_file = "non_valid_result_file.json" + + if action == RECOMMEND_COMPONENT_LAYOUT_ACTION: + result = stackAdvisor.recommendComponentLayout(services, hosts) + result_file = os.path.join(actionDir, "component-layout.json") + elif action == VALIDATE_COMPONENT_LAYOUT_ACTION: + result = stackAdvisor.validateComponentLayout(services, hosts) + result_file = os.path.join(actionDir, "component-layout-validation.json") + elif action == RECOMMEND_CONFIGURATIONS: + result = stackAdvisor.recommendConfigurations(services, hosts) + result_file = os.path.join(actionDir, "configurations.json") + else: # action == VALIDATE_CONFIGURATIONS + result = stackAdvisor.validateConfigurations(services, hosts) + result_file = os.path.join(actionDir, "configurations-validation.json") + + dumpJson(result, result_file) + pass + + +def instantiateStackAdvisor(stackName, stackVersion, parentVersions): + """Instantiates StackAdvisor implementation for the specified Stack""" + import imp + + stackAdvisorPath = STACK_ADVISOR_PATH_TEMPLATE.format(stackName) + with open(stackAdvisorPath, 'rb') as fp: + stack_advisor = imp.load_module( 'stack_advisor', fp, stackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE) ) + + versions = [stackVersion] + versions.extend(parentVersions) + + for version in versions: + try: + path = STACK_ADVISOR_IMPL_PATH_TEMPLATE.format(stackName, version) + className = STACK_ADVISOR_IMPL_CLASS_TEMPLATE.format(stackName, version.replace('.', '')) + + with open(path, 'rb') as fp: + stack_advisor_impl = imp.load_module( 'stack_advisor_impl', fp, path, ('.py', 'rb', imp.PY_SOURCE) ) + clazz = getattr(stack_advisor_impl, className) + + print "StackAdvisor for stack {0}, version {1} will be used".format(stackName, version) + return clazz(); + except Exception, e: + print "StackAdvisor implementation for stack {0}, version {1} was not found".format(stackName, version) + + print "StackAdvisor default implementation will be used!" + return stack_advisor.StackAdvisor() + + +if __name__ == '__main__': + try: + main(sys.argv) + except Exception, e: + traceback.print_exc() + print "Error occured in stack advisor.\nError details: {0}".format(str(e)) + sys.exit(1) + http://git-wip-us.apache.org/repos/asf/ambari/blob/b0b58189/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py new file mode 100644 index 0000000..01a38cf --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py @@ -0,0 +1,202 @@ +#!/usr/bin/env ambari-python-wrap +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import socket + +from stack_advisor import StackAdvisor + +class HDP206StackAdvisor(StackAdvisor): + + def recommendComponentLayout(self, services, hosts): + """Returns Services object with hostnames array populated for components""" + stackName = services["Versions"]["stack_name"] + stackVersion = services["Versions"]["stack_version"] + hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]] + servicesList = [service["StackServices"]["service_name"] for service in services["services"]] + + recommendations = { + "Versions": {"stack_name": stackName, "stack_version": stackVersion}, + "hosts": hostsList, + "services": servicesList, + "recommendations": { + "blueprint": { + "configurations": { + "global": { + "properties": { } + }, + "core-site": { }, + "hdfs-site": { }, + "yarn-site": { }, + "hbase-site": { } + }, + "host_groups": [ ] + }, + "blueprint_cluster_binding": { + "host_groups": [ ] + } + } + } + + hostsComponentsMap = {} + for service in services["services"]: + masterComponents = [component for component in service["components"] if isMaster(component)] + for component in masterComponents: + componentName = component["StackServiceComponents"]["component_name"] + hostsForComponent = [] + + availableHosts = hostsList + if len(hostsList) > 1 and isNotPreferableOnAmbariServerHost(component): + availableHosts = [hostName for hostName in hostsList if not isLocalHost(hostName)] + + if isMasterWithMultipleInstances(component): + hostsCount = defaultNoOfMasterHosts(component) + if hostsCount > 1: # get first 'hostsCount' available hosts + if len(availableHosts) < hostsCount: + hostsCount = len(availableHosts) + hostsForComponent = availableHosts[:hostsCount] + else: + hostsForComponent = [getHostForComponent(component, availableHosts)] + else: + hostsForComponent = [getHostForComponent(component, availableHosts)] + + #extend 'hostsComponentsMap' with 'hostsForComponent' + for hostName in hostsForComponent: + if hostName not in hostsComponentsMap: + hostsComponentsMap[hostName] = [] + hostsComponentsMap[hostName].append( { "name":componentName } ) + + #extend 'hostsComponentsMap' with Slave and Client Components + utilizedHosts = hostsComponentsMap.keys() + freeHosts = [hostName for hostName in hostsList if hostName not in utilizedHosts] + + for service in services["services"]: + slaveClientComponents = [component for component in service["components"] if isSlave(component) or isClient(component)] + for component in slaveClientComponents: + componentName = component["StackServiceComponents"]["component_name"] + hostsForComponent = [] + if len(freeHosts) == 0: + hostsForComponent = hostsList[-1:] + else: # len(freeHosts) >= 1 + hostsForComponent = freeHosts + if isClient(component): + hostsForComponent = freeHosts[0:1] + + #extend 'hostsComponentsMap' with 'hostsForComponent' + for hostName in hostsForComponent: + if hostName not in hostsComponentsMap: + hostsComponentsMap[hostName] = [] + hostsComponentsMap[hostName].append( { "name": componentName } ) + + #prepare 'host-group's from 'hostsComponentsMap' + host_groups = recommendations["recommendations"]["blueprint"]["host_groups"] + bindings = recommendations["recommendations"]["blueprint_cluster_binding"]["host_groups"] + index = 0 + for key in hostsComponentsMap.keys(): + index += 1 + host_group_name = "host-group-{0}".format(index) + host_groups.append( { "name": host_group_name, "components": hostsComponentsMap[key] } ) + bindings.append( { "name": host_group_name, "hosts": [{ "fqdn": socket.getfqdn(key) }] } ) + + return recommendations + pass + + def validateComponentLayout(self, services, hosts): + """Returns array of Validation objects about issues with hostnames components assigned to""" + pass + + def recommendConfigurations(self, services, hosts): + """Returns Services object with configurations object populated""" + pass + + def validateConfigurations(self, services, hosts): + """Returns array of Validation objects about issues with configuration values provided in services""" + pass + + +# Helper methods +def getHostForComponent(component, hostsList): + componentName = component["StackServiceComponents"]["component_name"] + scheme = selectionScheme(componentName) + + if len(hostsList) == 1: + return hostsList[0] + else: + for key in scheme.keys(): + if isinstance(key, ( int, long )): + if len(hostsList) < key: + return hostsList[scheme[key]] + return hostsList[scheme['else']] + +def isClient(component): + return component["StackServiceComponents"]["component_category"] == 'CLIENT' + +def isSlave(component): + componentName = component["StackServiceComponents"]["component_name"] + isSlave = component["StackServiceComponents"]["component_category"] == 'SLAVE' + return isSlave and componentName != 'APP_TIMELINE_SERVER' + +def isMaster(component): + componentName = component["StackServiceComponents"]["component_name"] + isMaster = component["StackServiceComponents"]["is_master"] + return isMaster or componentName == 'APP_TIMELINE_SERVER' + +def isLocalHost(hostName): + return socket.getfqdn(hostName) == socket.getfqdn() + +def isNotPreferableOnAmbariServerHost(component): + componentName = component["StackServiceComponents"]["component_name"] + service = ['STORM_UI_SERVER', 'DRPC_SERVER', 'STORM_REST_API', 'NIMBUS', 'GANGLIA_SERVER', 'NAGIOS_SERVER', 'HUE_SERVER'] + return componentName in service + +def isMasterWithMultipleInstances(component): + componentName = component["StackServiceComponents"]["component_name"] + masters = ['ZOOKEEPER_SERVER', 'HBASE_MASTER'] + return componentName in masters + +def defaultNoOfMasterHosts(component): + componentName = component["StackServiceComponents"]["component_name"] + return cardinality(componentName)[min] + + +# Helper dictionaries +def cardinality(componentName): + return { + 'ZOOKEEPER_SERVER': {min: 3}, + 'HBASE_MASTER': {min: 1}, + }.get(componentName, {min:1, max:1}) + +def selectionScheme(componentName): + return { + 'NAMENODE': {"else": 0}, + 'SECONDARY_NAMENODE': {"else": 1}, + 'HBASE_MASTER': {6: 0, 31: 2, "else": 3}, + + 'JOBTRACKER': {31: 1, "else": 2}, + 'HISTORYSERVER': {31: 1, "else": 2}, + 'RESOURCEMANAGER': {31: 1, "else": 2}, + 'APP_TIMELINE_SERVER': {31: 1, "else": 2}, + + 'OOZIE_SERVER': {6: 1, 31: 2, "else": 3}, + 'FALCON_SERVER': {6: 1, 31: 2, "else": 3}, + + 'HIVE_SERVER': {6: 1, 31: 2, "else": 4}, + 'HIVE_METASTORE': {6: 1, 31: 2, "else": 4}, + 'WEBHCAT_SERVER': {6: 1, 31: 2, "else": 4}, + }.get(componentName, {"else": 0}) + http://git-wip-us.apache.org/repos/asf/ambari/blob/b0b58189/ambari-server/src/main/resources/stacks/HDP/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/stack_advisor.py new file mode 100644 index 0000000..fdef482 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/stack_advisor.py @@ -0,0 +1,37 @@ +#!/usr/bin/env ambari-python-wrap +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +class StackAdvisor(): + + def recommendComponentLayout(self, services, hosts): + """Returns Services object with hostnames array populated for components""" + pass + + def validateComponentLayout(self, services, hosts): + """Returns array of Validation objects about issues with hostnames components assigned to""" + pass + + def recommendConfigurations(self, services, hosts): + """Returns Services object with configurations object populated""" + pass + + def validateConfigurations(self, services, hosts): + """Returns array of Validation objects about issues with configuration values provided in services""" + pass + http://git-wip-us.apache.org/repos/asf/ambari/blob/b0b58189/ambari-server/src/test/java/org/apache/ambari/server/api/services/GroupServiceTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/services/GroupServiceTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/services/GroupServiceTest.java index 2458920..68ee4a5 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/api/services/GroupServiceTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/api/services/GroupServiceTest.java @@ -78,7 +78,7 @@ public class GroupServiceTest extends BaseServiceTest { private class TestGroupService extends GroupService { @Override - ResourceInstance createResource(Type type, Map<Type, String> mapIds) { + protected ResourceInstance createResource(Type type, Map<Type, String> mapIds) { return getTestResource(); } http://git-wip-us.apache.org/repos/asf/ambari/blob/b0b58189/ambari-server/src/test/java/org/apache/ambari/server/api/services/MemberServiceTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/services/MemberServiceTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/services/MemberServiceTest.java index 9c5c860..b3bf3fb 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/api/services/MemberServiceTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/api/services/MemberServiceTest.java @@ -88,7 +88,7 @@ public class MemberServiceTest extends BaseServiceTest { } @Override - ResourceInstance createResource(Type type, Map<Type, String> mapIds) { + protected ResourceInstance createResource(Type type, Map<Type, String> mapIds) { return getTestResource(); } http://git-wip-us.apache.org/repos/asf/ambari/blob/b0b58189/ambari-server/src/test/java/org/apache/ambari/server/api/services/RecommendationServiceTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/services/RecommendationServiceTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/services/RecommendationServiceTest.java new file mode 100644 index 0000000..a1a0908 --- /dev/null +++ b/ambari-server/src/test/java/org/apache/ambari/server/api/services/RecommendationServiceTest.java @@ -0,0 +1,85 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.api.services; + +import static org.junit.Assert.assertEquals; + +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.List; + +import javax.ws.rs.core.HttpHeaders; +import javax.ws.rs.core.UriInfo; + +import org.apache.ambari.server.api.resources.ResourceInstance; +import org.apache.ambari.server.api.services.parsers.RequestBodyParser; +import org.apache.ambari.server.api.services.serializers.ResultSerializer; + +/** + * Unit tests for RecommendationService. + */ +public class RecommendationServiceTest extends BaseServiceTest { + + @Override + public List<ServiceTestInvocation> getTestInvocations() throws Exception { + List<ServiceTestInvocation> listInvocations = new ArrayList<ServiceTestInvocation>(); + + //getRecommendation + RecommendationService service = new TestRecommendationService("stackName", "stackVersion"); + Method m = service.getClass().getMethod("getRecommendation", String.class, HttpHeaders.class, UriInfo.class, String.class, String.class); + Object[] args = new Object[] {"body", getHttpHeaders(), getUriInfo(), "stackName", "stackVersion"}; + listInvocations.add(new ServiceTestInvocation(Request.Type.POST, service, m, args, "body")); + + return listInvocations; + } + + private class TestRecommendationService extends RecommendationService { + private String stackName; + private String stackVersion; + + private TestRecommendationService(String stackName, String stackVersion) { + super(); + this.stackName = stackName; + this.stackVersion = stackVersion; + } + + @Override + ResourceInstance createRecommendationResource(String stackName, String stackVersion) { + assertEquals(this.stackName, stackName); + assertEquals(this.stackVersion, stackVersion); + return getTestResource(); + } + + @Override + RequestFactory getRequestFactory() { + return getTestRequestFactory(); + } + + @Override + protected RequestBodyParser getBodyParser() { + return getTestBodyParser(); + } + + @Override + protected ResultSerializer getResultSerializer() { + return getTestResultSerializer(); + } + } + +}
