AMBARI-15538. Support service-specific repo for add-on services (Balazs bence Sari via magyari_sandor)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7961cd11 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7961cd11 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7961cd11 Branch: refs/heads/branch-2.5 Commit: 7961cd114566d035ee844098865195667d1cb4f5 Parents: 6a947a9 Author: Balazs Bence Sari <bs...@hortonworks.com> Authored: Thu Sep 15 15:08:16 2016 +0200 Committer: Sandor Magyari <smagy...@hortonworks.com> Committed: Thu Sep 15 15:20:27 2016 +0200 ---------------------------------------------------------------------- .../checks/DatabaseConsistencyCheckHelper.java | 1 + .../AmbariManagementControllerImpl.java | 13 +- .../ambari/server/controller/AmbariServer.java | 2 + .../VersionDefinitionResourceProvider.java | 18 +- .../apache/ambari/server/stack/RepoUtil.java | 208 +++++++++++++++++++ .../ambari/server/stack/ServiceModule.java | 7 + .../ambari/server/stack/StackDirectory.java | 23 +- .../apache/ambari/server/stack/StackModule.java | 135 ++++++++++-- .../server/stack/StackServiceDirectory.java | 70 ++++++- .../stack/UpdateActiveRepoVersionOnStartup.java | 118 +++++++++++ .../ambari/server/state/RepositoryInfo.java | 57 +++++ .../apache/ambari/server/state/StackInfo.java | 13 +- .../stack/upgrade/RepositoryVersionHelper.java | 28 ++- .../src/main/resources/version_definition.xsd | 24 +-- .../ambari/server/stack/RepoUtilTest.java | 166 +++++++++++++++ .../stack/StackManagerCommonServicesTest.java | 20 ++ .../ambari/server/stack/StackModuleTest.java | 188 +++++++++++++++++ .../UpdateActiveRepoVersionOnStartupTest.java | 143 +++++++++++++ .../ADDON/1.0/configuration/addon-env.xml | 35 ++++ .../common-services/ADDON/1.0/metainfo.xml | 35 ++++ ...veRepoVersionOnStartupTest_initialRepos.json | 32 +++ .../HDP/0.2/services/ADDON/metainfo.xml | 28 +++ .../HDP/0.2/services/ADDON/repos/repoinfo.xml | 26 +++ .../8.0.0/configuration/microsoft-r-env.xml | 35 ++++ .../8.0.0/package/scripts/microsoft_r.py | 22 +- .../MICROSOFT_R/8.0.0/repos/repoinfo.xml | 33 +++ 26 files changed, 1400 insertions(+), 80 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/7961cd11/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java index f302b8b..2d91eca 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java @@ -545,6 +545,7 @@ public class DatabaseConsistencyCheckHelper { String stackVersion = stackInfo.get(stackName); Map<String, ServiceInfo> serviceInfoMap = ambariMetaInfo.getServices(stackName, stackVersion); for (String serviceName : serviceNames) { + LOG.info("Processing {}-{} / {}", stackName, stackVersion, serviceName); ServiceInfo serviceInfo = serviceInfoMap.get(serviceName); if (serviceInfo != null) { Set<String> configTypes = serviceInfo.getConfigTypeAttributes().keySet(); http://git-wip-us.apache.org/repos/asf/ambari/blob/7961cd11/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java index 3acf490..a35b0e9 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java @@ -140,6 +140,7 @@ import org.apache.ambari.server.security.ldap.LdapSyncDto; import org.apache.ambari.server.serveraction.kerberos.KerberosInvalidConfigurationException; import org.apache.ambari.server.serveraction.kerberos.KerberosOperationException; import org.apache.ambari.server.stack.ExtensionHelper; +import org.apache.ambari.server.stack.RepoUtil; import org.apache.ambari.server.stageplanner.RoleGraph; import org.apache.ambari.server.stageplanner.RoleGraphFactory; import org.apache.ambari.server.state.Cluster; @@ -206,6 +207,7 @@ import com.google.inject.Inject; import com.google.inject.Injector; import com.google.inject.Singleton; import com.google.inject.persist.Transactional; +import com.google.common.collect.ListMultimap; @Singleton public class AmbariManagementControllerImpl implements AmbariManagementController { @@ -2286,7 +2288,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle hostParams.put(PACKAGE_LIST, packageList); Map<String, DesiredConfig> desiredConfigs = cluster.getDesiredConfigs(); - + Set<String> userSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.USER, cluster, desiredConfigs); String userList = gson.toJson(userSet); hostParams.put(USER_LIST, userList); @@ -2326,7 +2328,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle } execCmd.setRoleParams(roleParams); - + execCmd.setAvailableServicesFromServiceInfoMap(ambariMetaInfo.getServices(stackId.getStackName(), stackId.getStackVersion())); if ((execCmd != null) && (execCmd.getConfigurationTags().containsKey("cluster-env"))) { @@ -4082,7 +4084,9 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle } StackId stackId = new StackId(xml.release.stackId); + ListMultimap<String, RepositoryInfo> stackRepositoriesByOs = ambariMetaInfo.getStackManager().getStack(stackName, stackVersion).getRepositoriesByOs(); for (RepositoryXml.Os os : xml.repositoryInfo.getOses()) { + for (RepositoryXml.Repo repo : os.getRepos()) { RepositoryResponse resp = new RepositoryResponse(repo.getBaseUrl(), os.getFamily(), repo.getRepoId(), repo.getRepoName(), repo.getMirrorsList(), @@ -4096,6 +4100,11 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle } } + // Add service repos to the response. (These are not contained by the VDF but are present in the stack model) + List<RepositoryInfo> serviceRepos = + RepoUtil.getServiceRepos(xml.repositoryInfo.getRepositories(), stackRepositoriesByOs); + responses.addAll(RepoUtil.asResponses(serviceRepos, versionDefinitionId, stackName, stackVersion)); + } else { if (repoId == null) { List<RepositoryInfo> repositories = ambariMetaInfo.getRepositories(stackName, stackVersion, osType); http://git-wip-us.apache.org/repos/asf/ambari/blob/7961cd11/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java index 097f01c..89cdb93 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java @@ -108,6 +108,7 @@ import org.apache.ambari.server.security.ldap.AmbariLdapDataPopulator; import org.apache.ambari.server.security.unsecured.rest.CertificateDownload; import org.apache.ambari.server.security.unsecured.rest.CertificateSign; import org.apache.ambari.server.security.unsecured.rest.ConnectionInfo; +import org.apache.ambari.server.stack.UpdateActiveRepoVersionOnStartup; import org.apache.ambari.server.state.Clusters; import org.apache.ambari.server.topology.AmbariContext; import org.apache.ambari.server.topology.BlueprintFactory; @@ -918,6 +919,7 @@ public class AmbariServer { injector.getInstance(GuiceJpaInitializer.class); DatabaseConsistencyCheckHelper.checkDBVersionCompatible(); server = injector.getInstance(AmbariServer.class); + injector.getInstance(UpdateActiveRepoVersionOnStartup.class).process(); CertificateManager certMan = injector.getInstance(CertificateManager.class); certMan.initRootCert(); KerberosChecker.checkJaasConfiguration(); http://git-wip-us.apache.org/repos/asf/ambari/blob/7961cd11/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProvider.java index 02fc2ec..629f3cd 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProvider.java @@ -54,6 +54,8 @@ import org.apache.ambari.server.orm.entities.RepositoryVersionEntity; import org.apache.ambari.server.orm.entities.StackEntity; import org.apache.ambari.server.security.authorization.ResourceType; import org.apache.ambari.server.security.authorization.RoleAuthorization; +import org.apache.ambari.server.stack.RepoUtil; +import org.apache.ambari.server.state.RepositoryInfo; import org.apache.ambari.server.state.RepositoryType; import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.state.StackInfo; @@ -72,6 +74,7 @@ import org.codehaus.jackson.node.ObjectNode; import com.google.common.collect.Sets; import com.google.inject.Inject; import com.google.inject.Provider; +import com.google.common.collect.ListMultimap; /** * The {@link VersionDefinitionResourceProvider} class deals with managing Version Definition @@ -237,7 +240,7 @@ public class VersionDefinitionResourceProvider extends AbstractAuthorizedResourc try { holder.xmlString = xml.toXml(); } catch (Exception e) { - throw new AmbariException(String.format("The available repository %s does not serialize", definitionName)); + throw new AmbariException(String.format("The available repository %s does not serialize", definitionName), e); } } else { @@ -559,8 +562,16 @@ public class VersionDefinitionResourceProvider extends AbstractAuthorizedResourc StackEntity stackEntity = s_stackDAO.find(stackId.getStackName(), stackId.getStackVersion()); entity.setStack(stackEntity); - entity.setOperatingSystems(s_repoVersionHelper.get().serializeOperatingSystems( - holder.xml.repositoryInfo.getRepositories())); + + List<RepositoryInfo> repos = holder.xml.repositoryInfo.getRepositories(); + + // Add service repositories (these are not contained by the VDF but are there in the stack model) + ListMultimap<String, RepositoryInfo> stackReposByOs = + s_metaInfo.get().getStack(stackId.getStackName(), stackId.getStackVersion()).getRepositoriesByOs(); + repos.addAll(RepoUtil.getServiceRepos(repos, stackReposByOs)); + + entity.setOperatingSystems(s_repoVersionHelper.get().serializeOperatingSystems(repos)); + entity.setVersion(holder.xml.release.getFullVersion()); entity.setDisplayName(stackId, holder.xml.release); entity.setType(holder.xml.release.repositoryType); @@ -723,7 +734,6 @@ public class VersionDefinitionResourceProvider extends AbstractAuthorizedResourc entity.getStackName()); repoElement.put(PropertyHelper.getPropertyName(RepositoryResourceProvider.REPOSITORY_STACK_VERSION_PROPERTY_ID), entity.getStackVersion()); - repoBase.put(PropertyHelper.getPropertyCategory(RepositoryResourceProvider.REPOSITORY_BASE_URL_PROPERTY_ID), repoElement); http://git-wip-us.apache.org/repos/asf/ambari/blob/7961cd11/ambari-server/src/main/java/org/apache/ambari/server/stack/RepoUtil.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/RepoUtil.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/RepoUtil.java new file mode 100644 index 0000000..07b845a --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/RepoUtil.java @@ -0,0 +1,208 @@ + /** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.stack; + +import java.io.File; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import javax.annotation.Nullable; +import javax.xml.bind.JAXBException; + +import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.controller.RepositoryResponse; +import org.apache.ambari.server.orm.entities.OperatingSystemEntity; +import org.apache.ambari.server.orm.entities.RepositoryEntity; +import org.apache.ambari.server.state.RepositoryInfo; +import org.apache.ambari.server.state.stack.RepositoryXml; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.base.Function; +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.ListMultimap; +import com.google.common.collect.Lists; +import com.google.common.collect.Multimaps; +import com.google.common.collect.Sets; + +/** + * Utility functions for repository replated tasks. + */ +public class RepoUtil { + + /** + * logger instance + */ + private final static Logger LOG = LoggerFactory.getLogger(RepoUtil.class); + + + /** + * repository directory name + */ + final static String REPOSITORY_FOLDER_NAME = "repos"; + + /** + * repository file name + */ + final static String REPOSITORY_FILE_NAME = "repoinfo.xml"; + + private static final Function<RepositoryEntity, String> REPO_ENTITY_TO_NAME = new Function<RepositoryEntity, String>() { + @Override public String apply(@Nullable RepositoryEntity input) { return input.getName(); } + }; + + + /** + * Parses the repository file for a stack/service if exists. + * + * @param directory stack/service base directory + * @param subDirs stack/service directory sub directories + * @param unmarshaller {@link ModuleFileUnmarshaller}, needed to parse repo XML + * @throws AmbariException if unable to parse the repository file + * @return The directory containing the repo file and the parsed repo file (if exists) + */ + public static RepositoryFolderAndXml parseRepoFile(File directory, + Collection<String> subDirs, + ModuleFileUnmarshaller unmarshaller) { + File repositoryFile = null; + String repoDir = null; + RepositoryXml repoFile = null; + + if (subDirs.contains(REPOSITORY_FOLDER_NAME)) { + repoDir = directory.getAbsolutePath() + File.separator + REPOSITORY_FOLDER_NAME; + repositoryFile = new File(directory.getPath()+ File.separator + + REPOSITORY_FOLDER_NAME + File.separator + REPOSITORY_FILE_NAME); + + if (repositoryFile.exists()) { + try { + repoFile = unmarshaller.unmarshal(RepositoryXml.class, repositoryFile); + } catch (JAXBException e) { + repoFile = new RepositoryXml(); + repoFile.setValid(false); + String msg = "Unable to parse repo file at location: " + + repositoryFile.getAbsolutePath(); + repoFile.addError(msg); + LOG.warn(msg); + } + } + } + + return new RepositoryFolderAndXml(Optional.fromNullable(repoDir), Optional.fromNullable(repoFile)); + } + + /** + * Checks the passed {@code operatingSystems} parameter if it contains all repositories from the stack model. If a + * repository is present in the stack model but missing in the operating system entity list, it is considered a + * service repository and will be added. + * @param operatingSystems - A list of OperatingSystemEntity objects extracted from a RepositoryVersionEntity + * @param stackReposByOs - Stack repositories loaded from the disk (including service repositories), grouped by os. + */ + public static void addServiceReposToOperatingSystemEntities(List<OperatingSystemEntity> operatingSystems, + ListMultimap<String, RepositoryInfo> stackReposByOs) { + Set<String> addedRepos = new HashSet<>(); + for (OperatingSystemEntity os : operatingSystems) { + List<RepositoryInfo> serviceReposForOs = stackReposByOs.get(os.getOsType()); + ImmutableSet<String> repoNames = ImmutableSet.copyOf(Lists.transform(os.getRepositories(), REPO_ENTITY_TO_NAME)); + for (RepositoryInfo repoInfo : serviceReposForOs) + if (!repoNames.contains(repoInfo.getRepoName())) { + os.getRepositories().add(toRepositoryEntity(repoInfo)); + addedRepos.add(String.format("%s (%s)", repoInfo.getRepoId(), os.getOsType())); + } + } + LOG.info("Added {} service repos: {}", addedRepos.size(),Iterables.toString(addedRepos)); + } + + /** + * Given a list of VDF repositorie and stack repositories (grouped by os) returns the service repositories. + * A repository is considered a service repo if present in the stack model but missing in the VDF (check is performed + * by repository name, per operating system). + * @param vdfRepos the repositories coming from a version definition + * @param stackReposByOs the repositories in the stack model (loaded from disks) + * @return A list of service repositories + */ + public static List<RepositoryInfo> getServiceRepos(List<RepositoryInfo> vdfRepos, + ListMultimap<String, RepositoryInfo> stackReposByOs) { + Set<String> serviceRepoIds = new HashSet<>(); + List<RepositoryInfo> serviceRepos = new ArrayList<>(); + ListMultimap<String, RepositoryInfo> vdfReposByOs = Multimaps.index(vdfRepos, RepositoryInfo.GET_OSTYPE_FUNCTION); + for(String os: vdfReposByOs.keySet()) { + Set<String> vdfRepoNames = Sets.newHashSet( + Lists.transform(vdfReposByOs.get(os), RepositoryInfo.GET_REPO_NAME_FUNCTION)); + for (RepositoryInfo repo: stackReposByOs.get(os)) { + if (!vdfRepoNames.contains(repo.getRepoName())) { + serviceRepos.add(repo); + serviceRepoIds.add(repo.getRepoId()); + } + } + } + LOG.info("Found {} service repos: {}", serviceRepoIds.size(),Iterables.toString(serviceRepoIds)); + return serviceRepos; + } + + /** + * Convert a list of {@link RepositoryInfo} objects to a lost of {@link RepositoryResponse} objects + * @param repositoryInfos the list of repository infos + * @param versionDefinitionId the version definition id + * @param stackName the stack name + * @param stackVersion the stack version + * @return a list of repository responses + */ + public static List<RepositoryResponse> asResponses(List<RepositoryInfo> repositoryInfos, + @Nullable String versionDefinitionId, + @Nullable String stackName, + @Nullable String stackVersion) { + List<RepositoryResponse> responses = new ArrayList<>(repositoryInfos.size()); + for (RepositoryInfo repoInfo: repositoryInfos) { + RepositoryResponse response = repoInfo.convertToResponse(); + response.setVersionDefinitionId(versionDefinitionId); + response.setStackName(stackName); + response.setStackVersion(stackVersion); + responses.add(response); + } + return responses; + } + + private static RepositoryEntity toRepositoryEntity(RepositoryInfo repoInfo) { + RepositoryEntity re = new RepositoryEntity(); + re.setBaseUrl(repoInfo.getBaseUrl()); + re.setName(repoInfo.getRepoName()); + re.setRepositoryId(repoInfo.getRepoId()); + return re; + } + +} + +/** + * Value class for a pair of repository folder and parsed repository XML. + */ +class RepositoryFolderAndXml { + final Optional<String> repoDir; + final Optional<RepositoryXml> repoXml; + + /** + * @param repoDir Path to the repository directory (optional) + * @param repoXml Parsed repository XML (optional) + */ + public RepositoryFolderAndXml(Optional<String> repoDir, Optional<RepositoryXml> repoXml) { + this.repoDir = repoDir; + this.repoXml = repoXml; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/7961cd11/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java index bc94104..a77a22f 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java @@ -621,6 +621,13 @@ public class ServiceModule extends BaseModule<ServiceModule, ServiceInfo> implem return errorSet; } + /** + * @return The service's directory + */ + public ServiceDirectory getServiceDirectory() { + return serviceDirectory; + } + @Override public void addErrors(Collection<String> errors) { this.errorSet.addAll(errors); http://git-wip-us.apache.org/repos/asf/ambari/blob/7961cd11/ambari-server/src/main/java/org/apache/ambari/server/stack/StackDirectory.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackDirectory.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackDirectory.java index bfba021..c2c8a9e 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackDirectory.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackDirectory.java @@ -321,26 +321,9 @@ public class StackDirectory extends StackDefinitionDirectory { * @throws AmbariException if unable to parse the repository file */ private void parseRepoFile(Collection<String> subDirs) throws AmbariException { - File repositoryFile; - - if (subDirs.contains(REPOSITORY_FOLDER_NAME)) { - repoDir = getAbsolutePath() + File.separator + REPOSITORY_FOLDER_NAME; - repositoryFile = new File(getPath()+ File.separator + - REPOSITORY_FOLDER_NAME + File.separator + REPOSITORY_FILE_NAME); - - if (repositoryFile.exists()) { - try { - repoFile = unmarshaller.unmarshal(RepositoryXml.class, repositoryFile); - } catch (JAXBException e) { - repoFile = new RepositoryXml(); - repoFile.setValid(false); - String msg = "Unable to parse repo file at location: " + - repositoryFile.getAbsolutePath(); - repoFile.addError(msg); - LOG.warn(msg); - } - } - } + RepositoryFolderAndXml repoDirAndXml = RepoUtil.parseRepoFile(directory, subDirs, unmarshaller); + repoDir = repoDirAndXml.repoDir.orNull(); + repoFile = repoDirAndXml.repoXml.orNull(); if (repoFile == null || !repoFile.isValid()) { LOG.warn("No repository information defined for " http://git-wip-us.apache.org/repos/asf/ambari/blob/7961cd11/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java index 0606f2a..bb8d740 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java @@ -22,6 +22,7 @@ import java.io.File; import java.io.FilenameFilter; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -55,6 +56,13 @@ import org.apache.ambari.server.state.stack.upgrade.ClusterGrouping.ExecuteStage import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.base.Function; +import com.google.common.collect.ListMultimap; +import com.google.common.collect.ImmutableListMultimap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Lists; +import com.google.common.collect.Multimaps; + /** * Stack module which provides all functionality related to parsing and fully * resolving stacks from the stack definition. @@ -290,7 +298,6 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V * * @param allStacks all stacks in stack definition * @param commonServices all common services specified in the stack definition - * @param parentVersion version of the stacks parent * * @throws AmbariException if an exception occurs merging with the parent */ @@ -1021,38 +1028,130 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V * @throws AmbariException if unable to fully process the stack repositories */ private void processRepositories() throws AmbariException { - + List<RepositoryInfo> stackRepos = Collections.emptyList(); RepositoryXml rxml = stackDirectory.getRepoFile(); - if (rxml == null) { - return; - } - stackInfo.setRepositoryXml(rxml); + if (null != rxml) { + stackInfo.setRepositoryXml(rxml); - LOG.debug("Adding repositories to stack" + - ", stackName=" + stackInfo.getName() + - ", stackVersion=" + stackInfo.getVersion() + - ", repoFolder=" + stackDirectory.getRepoDir()); + LOG.debug("Adding repositories to stack" + + ", stackName=" + stackInfo.getName() + + ", stackVersion=" + stackInfo.getVersion() + + ", repoFolder=" + stackDirectory.getRepoDir()); - List<RepositoryInfo> repos = rxml.getRepositories(); + stackRepos = rxml.getRepositories(); - for (RepositoryInfo ri : repos) { - processRepository(ri); + for (RepositoryInfo ri : stackRepos) { + processRepository(ri); + } + + stackInfo.getRepositories().addAll(stackRepos); } - stackInfo.getRepositories().addAll(repos); + LOG.debug("Process service custom repositories"); + Set<RepositoryInfo> serviceRepos = getUniqueServiceRepos(stackRepos); + stackInfo.getRepositories().addAll(serviceRepos); - if (null != rxml.getLatestURI() && repos.size() > 0) { + if (null != rxml && null != rxml.getLatestURI() && stackRepos.size() > 0) { stackContext.registerRepoUpdateTask(rxml.getLatestURI(), this); } } /** + * Gets the service repos with duplicates filtered out. A service repo is considered duplicate if: + * <ul> + * <li>It has the same name as a stack repo</li> + * <li>It has the same id as another service repo</li> + * </ul> + * Duplicate repo url's only results in warnings in the log. Duplicates are checked per os type, so e.g. the same repo + * can exsist for centos5 and centos6. + * @param stackRepos the list of stack repositories + * @return the service repos with duplicates filtered out. + */ + private Set<RepositoryInfo> getUniqueServiceRepos(List<RepositoryInfo> stackRepos) { + List<RepositoryInfo> serviceRepos = getAllServiceRepos(); + ImmutableListMultimap<String, RepositoryInfo> serviceReposByOsType = Multimaps.index(serviceRepos, RepositoryInfo.GET_OSTYPE_FUNCTION); + ImmutableListMultimap<String, RepositoryInfo> stackReposByOsType = Multimaps.index(stackRepos, RepositoryInfo.GET_OSTYPE_FUNCTION); + + Set<RepositoryInfo> uniqueServiceRepos = new HashSet<>(); + + // Uniqueness is checked for each os type + for (String osType: serviceReposByOsType.keySet()) { + List<RepositoryInfo> stackReposForOsType = stackReposByOsType.containsKey(osType) ? stackReposByOsType.get(osType) : Collections.<RepositoryInfo>emptyList(); + List<RepositoryInfo> serviceReposForOsType = serviceReposByOsType.get(osType); + Set<String> stackRepoNames = ImmutableSet.copyOf(Lists.transform(stackReposForOsType, RepositoryInfo.GET_REPO_NAME_FUNCTION)); + Set<String> stackRepoUrls = ImmutableSet.copyOf(Lists.transform(stackReposForOsType, RepositoryInfo.SAFE_GET_BASE_URL_FUNCTION)); + Set<String> duplicateServiceRepoNames = findDuplicates(serviceReposForOsType, RepositoryInfo.GET_REPO_NAME_FUNCTION); + Set<String> duplicateServiceRepoUrls = findDuplicates(serviceReposForOsType, RepositoryInfo.SAFE_GET_BASE_URL_FUNCTION); + + for (RepositoryInfo repo: serviceReposForOsType) { + // These cases only generate warnings + if (stackRepoUrls.contains(repo.getBaseUrl())) { + LOG.warn("Service repo has a base url that is identical to that of a stack repo: {}", repo); + } + else if (duplicateServiceRepoUrls.contains(repo.getBaseUrl())) { + LOG.warn("Service repo has a base url that is identical to that of another service repo: {}", repo); + } + // These cases cause the repo to be disregarded + if (stackRepoNames.contains(repo.getRepoName())) { + LOG.warn("Discarding service repository with the same name as one of the stack repos: {}", repo); + } + else if (duplicateServiceRepoNames.contains(repo.getRepoName())) { + LOG.warn("Discarding service repository with duplicate name and different content: {}", repo); + } + else { + uniqueServiceRepos.add(repo); + } + } + } + return uniqueServiceRepos; + } + + /** + * Finds duplicate repository infos. Duplicateness is checked on the property specified in the keyExtractor. + * Items that are equal don't count as duplicate, only differing items with the same key + * @param input the input list + * @param keyExtractor a function to that returns the property to be checked + * @return a set containing the keys of duplicates + */ + private static Set<String> findDuplicates(List<RepositoryInfo> input, Function<RepositoryInfo, String> keyExtractor) { + ListMultimap<String, RepositoryInfo> itemsByKey = Multimaps.index(input, keyExtractor); + Set<String> duplicates = new HashSet<>(); + for (Map.Entry<String, Collection<RepositoryInfo>> entry: itemsByKey.asMap().entrySet()) { + if (entry.getValue().size() > 1) { + Set<RepositoryInfo> differingItems = new HashSet<>(); + differingItems.addAll(entry.getValue()); + if (differingItems.size() > 1) { + duplicates.add(entry.getKey()); + } + } + } + return duplicates; + } + + /** + * Returns all service repositories for a given stack + * @return a list of service repo definitions + */ + private List<RepositoryInfo> getAllServiceRepos() { + List<RepositoryInfo> repos = new ArrayList<>(); + for (ServiceModule sm: serviceModules.values()) { + ServiceDirectory sd = sm.getServiceDirectory(); + if (sd instanceof StackServiceDirectory) { + StackServiceDirectory ssd = (StackServiceDirectory) sd; + RepositoryXml serviceRepoXml = ssd.getRepoFile(); + if (null != serviceRepoXml) { + repos.addAll(serviceRepoXml.getRepositories()); + } + } + } + return repos; + } + + /** * Process a repository associated with the stack. * - * @param osFamily OS family - * @param osType OS type - * @param r repo + * @param ri The RespositoryInfo to process */ private RepositoryInfo processRepository(RepositoryInfo ri) { http://git-wip-us.apache.org/repos/asf/ambari/blob/7961cd11/ambari-server/src/main/java/org/apache/ambari/server/stack/StackServiceDirectory.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackServiceDirectory.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackServiceDirectory.java index 7bcd08b..a8b4632 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackServiceDirectory.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackServiceDirectory.java @@ -18,19 +18,35 @@ package org.apache.ambari.server.stack; +import java.io.File; +import java.util.Arrays; +import java.util.Collection; +import javax.annotation.Nullable; + import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.state.stack.RepositoryXml; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; - - /** * Encapsulates IO operations on a stack service directory. */ public class StackServiceDirectory extends ServiceDirectory { /** + * repository file + */ + @Nullable + private RepositoryXml repoFile; + + /** + * repository directory + */ + @Nullable + private String repoDir; + + + /** * logger instance */ private static final Logger LOG = LoggerFactory.getLogger(StackServiceDirectory.class); @@ -45,6 +61,28 @@ public class StackServiceDirectory extends ServiceDirectory { super(servicePath); } + + /** + * Obtain the repository xml file if exists or null + * + * @return the repository xml file if exists or null + */ + @Nullable + public RepositoryXml getRepoFile() { + return repoFile; + } + + /** + * Obtain the repository directory if exists or null + * + * @return the repository directory if exists or null + */ + @Nullable + public String getRepoDir() { + return repoDir; + } + + @Override /** * Obtain the advisor name. @@ -65,6 +103,30 @@ public class StackServiceDirectory extends ServiceDirectory { return stackName + versionString + serviceName + "ServiceAdvisor"; } + /** + * Parse the repository file. + * + * @param subDirs service directory sub directories + */ + private void parseRepoFile(Collection<String> subDirs) { + RepositoryFolderAndXml repoDirAndXml = RepoUtil.parseRepoFile(directory, subDirs, unmarshaller); + repoDir = repoDirAndXml.repoDir.orNull(); + repoFile = repoDirAndXml.repoXml.orNull(); + + if (repoFile == null || !repoFile.isValid()) { + LOG.info("No repository information defined for " + + ", serviceName=" + getName() + + ", repoFolder=" + getPath() + File.separator + RepoUtil.REPOSITORY_FOLDER_NAME); + } + } + + @Override + protected void parsePath() throws AmbariException { + super.parsePath(); + Collection<String> subDirs = Arrays.asList(directory.list()); + parseRepoFile(subDirs); + } + @Override /** * Calculate the stack service directories. @@ -116,4 +178,6 @@ public class StackServiceDirectory extends ServiceDirectory { absUpgradesDir, serviceDir.getName(), stackId); } } + + } http://git-wip-us.apache.org/repos/asf/ambari/blob/7961cd11/ambari-server/src/main/java/org/apache/ambari/server/stack/UpdateActiveRepoVersionOnStartup.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/UpdateActiveRepoVersionOnStartup.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/UpdateActiveRepoVersionOnStartup.java new file mode 100644 index 0000000..1413c66 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/UpdateActiveRepoVersionOnStartup.java @@ -0,0 +1,118 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.stack; + +import java.util.List; +import javax.annotation.Nullable; + +import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.api.services.AmbariMetaInfo; +import org.apache.ambari.server.orm.dao.ClusterDAO; +import org.apache.ambari.server.orm.dao.ClusterVersionDAO; +import org.apache.ambari.server.orm.dao.RepositoryVersionDAO; +import org.apache.ambari.server.orm.entities.ClusterEntity; +import org.apache.ambari.server.orm.entities.ClusterVersionEntity; +import org.apache.ambari.server.orm.entities.OperatingSystemEntity; +import org.apache.ambari.server.orm.entities.RepositoryEntity; +import org.apache.ambari.server.orm.entities.RepositoryVersionEntity; +import org.apache.ambari.server.state.RepositoryInfo; +import org.apache.ambari.server.state.StackInfo; +import org.apache.ambari.server.state.stack.upgrade.RepositoryVersionHelper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.base.Function; +import com.google.common.collect.ListMultimap; +import com.google.inject.Inject; +import com.google.inject.persist.Transactional; + + +/** + * This class should be instantiated on server startup and its {@link #process()} method invoked. + * The class is part of management pack support. Management packs can contain services which define + * their own (yum/apt/ect) repositories. If a management pack is installed on an Ambari with an existing + * cluster, the cluster's repository version entity must be updated with the custom repos provided by the + * management pack. The class takes care of this. + */ +public class UpdateActiveRepoVersionOnStartup { + + private static final Logger LOG = LoggerFactory.getLogger(UpdateActiveRepoVersionOnStartup.class); + + + ClusterDAO clusterDao; + ClusterVersionDAO clusterVersionDao; + RepositoryVersionDAO repositoryVersionDao; + RepositoryVersionHelper repositoryVersionHelper; + StackManager stackManager; + + + private static final Function<RepositoryEntity, String> REPO_TO_ID = new Function<RepositoryEntity, String>() { + @Override public String apply(@Nullable RepositoryEntity input) { return input.getRepositoryId(); } + }; + + @Inject + public UpdateActiveRepoVersionOnStartup(ClusterDAO clusterDao, + ClusterVersionDAO clusterVersionDao, + RepositoryVersionDAO repositoryVersionDao, + RepositoryVersionHelper repositoryVersionHelper, + AmbariMetaInfo metaInfo) { + this.clusterDao = clusterDao; + this.clusterVersionDao = clusterVersionDao; + this.repositoryVersionDao = repositoryVersionDao; + this.repositoryVersionHelper = repositoryVersionHelper; + this.stackManager = metaInfo.getStackManager(); + } + + /** + * Updates the active {@link RepositoryVersionEntity} for clusters with add-on services defined in management packs. + * @throws AmbariException + */ + @Transactional + public void process() throws AmbariException { + LOG.info("Updating existing repo versions with service repos."); + try { + List<ClusterEntity> clusters = clusterDao.findAll(); + for (ClusterEntity cluster: clusters) { + StackInfo stack = + stackManager.getStack(cluster.getDesiredStack().getStackName(), cluster.getDesiredStack().getStackVersion()); + LOG.info("Updating existing repo versions for cluster {} on stack {}-{}", + cluster.getClusterName(), stack.getName(), stack.getVersion()); + ClusterVersionEntity clusterVersion = clusterVersionDao.findByClusterAndStateCurrent(cluster.getClusterName()); + RepositoryVersionEntity repoVersion = clusterVersion.getRepositoryVersion(); + updateRepoVersion(stack, repoVersion); + repositoryVersionDao.merge(repoVersion); + } + } + catch(Exception ex) { + throw new AmbariException( + "An error occured during updating current repository versions with stack repositories.", + ex); + } + } + + private void updateRepoVersion(StackInfo stackInfo, RepositoryVersionEntity repoVersion) throws Exception { + ListMultimap<String, RepositoryInfo> serviceReposByOs = stackInfo.getRepositoriesByOs(); + + // Update repos in the JSON representation + List<OperatingSystemEntity> operatingSystems = repoVersion.getOperatingSystems(); + RepoUtil.addServiceReposToOperatingSystemEntities(operatingSystems, serviceReposByOs); + repoVersion.setOperatingSystems(repositoryVersionHelper.serializeOperatingSystemEntities(operatingSystems)); + } + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/7961cd11/ambari-server/src/main/java/org/apache/ambari/server/state/RepositoryInfo.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/RepositoryInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/state/RepositoryInfo.java index 252592f..0b8cab8 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/RepositoryInfo.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/RepositoryInfo.java @@ -18,8 +18,12 @@ package org.apache.ambari.server.state; +import com.google.common.base.Objects; import org.apache.ambari.server.controller.RepositoryResponse; +import com.google.common.base.Function; +import com.google.common.base.Strings; + public class RepositoryInfo { private String baseUrl; private String osType; @@ -155,6 +159,24 @@ public class RepositoryInfo { + " ]"; } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + RepositoryInfo that = (RepositoryInfo) o; + return Objects.equal(baseUrl, that.baseUrl) && + Objects.equal(osType, that.osType) && + Objects.equal(repoId, that.repoId) && + Objects.equal(repoName, that.repoName) && + Objects.equal(mirrorsList, that.mirrorsList) && + Objects.equal(defaultBaseUrl, that.defaultBaseUrl) && + Objects.equal(latestBaseUrl, that.latestBaseUrl); + } + + @Override + public int hashCode() { + return Objects.hashCode(baseUrl, osType, repoId, repoName, mirrorsList, defaultBaseUrl, latestBaseUrl); + } public RepositoryResponse convertToResponse() { @@ -162,6 +184,41 @@ public class RepositoryInfo { getRepoName(), getMirrorsList(), getDefaultBaseUrl(), getLatestBaseUrl()); } + /** + * A function that returns the repo name of any RepositoryInfo + */ + public static final Function<RepositoryInfo, String> GET_REPO_NAME_FUNCTION = new Function<RepositoryInfo, String>() { + @Override public String apply(RepositoryInfo input) { + return input.repoName; + } + }; + + /** + * A function that returns the repoId of any RepositoryInfo + */ + public static final Function<RepositoryInfo, String> GET_REPO_ID_FUNCTION = new Function<RepositoryInfo, String>() { + @Override public String apply(RepositoryInfo input) { + return input.repoId; + } + }; + + /** + * A function that returns the baseUrl of any RepositoryInfo + */ + public static final Function<RepositoryInfo, String> SAFE_GET_BASE_URL_FUNCTION = new Function<RepositoryInfo, String>() { + @Override public String apply(RepositoryInfo input) { + return Strings.nullToEmpty(input.baseUrl); + } + }; + + /** + * A function that returns the osType of any RepositoryInfo + */ + public static final Function<RepositoryInfo, String> GET_OSTYPE_FUNCTION = new Function<RepositoryInfo, String>() { + @Override public String apply(RepositoryInfo input) { + return input.osType; + } + }; http://git-wip-us.apache.org/repos/asf/ambari/blob/7961cd11/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java index 14ff9de..ba5cb42 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java @@ -29,7 +29,6 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import com.google.common.io.Files; import org.apache.ambari.server.controller.StackVersionResponse; import org.apache.ambari.server.stack.Validable; import org.apache.ambari.server.state.repository.VersionDefinitionXml; @@ -38,6 +37,12 @@ import org.apache.ambari.server.state.stack.RepositoryXml; import org.apache.ambari.server.state.stack.StackRoleCommandOrder; import org.apache.ambari.server.state.stack.UpgradePack; +import com.google.common.collect.Collections2; +import com.google.common.collect.ListMultimap; +import com.google.common.collect.Lists; +import com.google.common.collect.Multimaps; +import com.google.common.io.Files; + public class StackInfo implements Comparable<StackInfo>, Validable{ private String minJdk; private String maxJdk; @@ -143,6 +148,12 @@ public class StackInfo implements Comparable<StackInfo>, Validable{ return repositories; } + /** + * @return A list containing all repos for this stack, grouped by os + */ + public ListMultimap<String, RepositoryInfo> getRepositoriesByOs() { + return Multimaps.index(getRepositories(), RepositoryInfo.GET_OSTYPE_FUNCTION); + } public synchronized Collection<ServiceInfo> getServices() { if (services == null) services = new ArrayList<ServiceInfo>(); http://git-wip-us.apache.org/repos/asf/ambari/blob/7961cd11/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java index 6cec6b0..9ca6cf4 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java @@ -100,13 +100,14 @@ public class RepositoryVersionHelper { OperatingSystemResourceProvider.OPERATING_SYSTEM_AMBARI_MANAGED_REPOS).getAsBoolean()); } - for (JsonElement repositoryJson: osObj.get(RepositoryVersionResourceProvider.SUBRESOURCE_REPOSITORIES_PROPERTY_ID).getAsJsonArray()) { + for (JsonElement repositoryElement: osObj.get(RepositoryVersionResourceProvider.SUBRESOURCE_REPOSITORIES_PROPERTY_ID).getAsJsonArray()) { final RepositoryEntity repositoryEntity = new RepositoryEntity(); - repositoryEntity.setBaseUrl(repositoryJson.getAsJsonObject().get(RepositoryResourceProvider.REPOSITORY_BASE_URL_PROPERTY_ID).getAsString()); - repositoryEntity.setName(repositoryJson.getAsJsonObject().get(RepositoryResourceProvider.REPOSITORY_REPO_NAME_PROPERTY_ID).getAsString()); - repositoryEntity.setRepositoryId(repositoryJson.getAsJsonObject().get(RepositoryResourceProvider.REPOSITORY_REPO_ID_PROPERTY_ID).getAsString()); - if (repositoryJson.getAsJsonObject().get(RepositoryResourceProvider.REPOSITORY_UNIQUE_PROPERTY_ID) != null) { - repositoryEntity.setUnique(repositoryJson.getAsJsonObject().get(RepositoryResourceProvider.REPOSITORY_UNIQUE_PROPERTY_ID).getAsBoolean()); + final JsonObject repositoryJson = repositoryElement.getAsJsonObject(); + repositoryEntity.setBaseUrl(repositoryJson.get(RepositoryResourceProvider.REPOSITORY_BASE_URL_PROPERTY_ID).getAsString()); + repositoryEntity.setName(repositoryJson.get(RepositoryResourceProvider.REPOSITORY_REPO_NAME_PROPERTY_ID).getAsString()); + repositoryEntity.setRepositoryId(repositoryJson.get(RepositoryResourceProvider.REPOSITORY_REPO_ID_PROPERTY_ID).getAsString()); + if (repositoryJson.get(RepositoryResourceProvider.REPOSITORY_UNIQUE_PROPERTY_ID) != null) { + repositoryEntity.setUnique(repositoryJson.get(RepositoryResourceProvider.REPOSITORY_UNIQUE_PROPERTY_ID).getAsBoolean()); } operatingSystemEntity.getRepositories().add(repositoryEntity); } @@ -165,6 +166,21 @@ public class RepositoryVersionHelper { return gson.toJson(rootJson); } + public String serializeOperatingSystemEntities(List<OperatingSystemEntity> operatingSystems) { + List<RepositoryInfo> repositoryInfos = new ArrayList<>(); + for (OperatingSystemEntity os: operatingSystems) { + for (RepositoryEntity repositoryEntity: os.getRepositories()) { + RepositoryInfo repositoryInfo = new RepositoryInfo(); + repositoryInfo.setRepoId(repositoryEntity.getRepositoryId()); + repositoryInfo.setRepoName(repositoryEntity.getName()); + repositoryInfo.setBaseUrl(repositoryEntity.getBaseUrl()); + repositoryInfo.setOsType(os.getOsType()); + repositoryInfos.add(repositoryInfo); + } + } + return serializeOperatingSystems(repositoryInfos); + } + /** * Scans the given stack for upgrade packages which can be applied to update the cluster to given repository version. * http://git-wip-us.apache.org/repos/asf/ambari/blob/7961cd11/ambari-server/src/main/resources/version_definition.xsd ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/version_definition.xsd b/ambari-server/src/main/resources/version_definition.xsd index 35deb6e..bef3739 100644 --- a/ambari-server/src/main/resources/version_definition.xsd +++ b/ambari-server/src/main/resources/version_definition.xsd @@ -24,7 +24,7 @@ xmllint --noout --load-trace --schema [path-to-this-file] [path-to-xml] </xs:documentation> </xs:annotation> - + <xs:complexType name="release-type"> <xs:all> <xs:element name="type" type="repo-type" /> @@ -38,7 +38,7 @@ <xs:element name="package-version" type="xs:string" minOccurs="0" /> </xs:all> </xs:complexType> - + <xs:simpleType name="repo-type"> <xs:restriction base="xs:string"> <xs:enumeration value="STANDARD" /> @@ -46,7 +46,7 @@ <xs:enumeration value="PATCH" /> </xs:restriction> </xs:simpleType> - + <xs:simpleType name="family-type"> <xs:restriction base="xs:string"> <xs:enumeration value="redhat6" /> @@ -60,7 +60,7 @@ <xs:enumeration value="suse12" /> </xs:restriction> </xs:simpleType> - + <xs:complexType name="manifest-service-type"> <xs:annotation> <xs:documentation> @@ -86,16 +86,16 @@ </xs:element> </xs:sequence> </xs:complexType> - + <xs:complexType name="available-services-type"> <xs:annotation> <xs:documentation> Provides a list of services that are available to upgrade out of this repository. A service may include a list of components that can be upgraded. These are specified (generally) for patch upgrades only. - + A service must have an 'idref' attribute to tie it back to a service and version from - the 'manifest' element. + the 'manifest' element. </xs:documentation> </xs:annotation> <xs:sequence> @@ -109,7 +109,7 @@ </xs:element> </xs:sequence> </xs:complexType> - + <xs:complexType name="repository-info-type"> <xs:sequence> <xs:element name="os" maxOccurs="unbounded"> @@ -132,7 +132,7 @@ </xs:element> </xs:sequence> </xs:complexType> - + <xs:complexType name="upgrade-type"> <xs:sequence> <xs:element name="configuration" maxOccurs="unbounded"> @@ -150,7 +150,7 @@ </xs:element> </xs:sequence> </xs:complexType> - + <xs:element name="repository-version"> <xs:annotation> <xs:documentation> @@ -177,7 +177,7 @@ <xs:selector xpath="./manifest/service" /> <xs:field xpath="@id" /> </xs:key> - + <xs:keyref name="available-services-id-keyref" refer="service-id-key"> <xs:annotation> <xs:documentation> @@ -189,5 +189,5 @@ </xs:keyref> </xs:element> - + </xs:schema> http://git-wip-us.apache.org/repos/asf/ambari/blob/7961cd11/ambari-server/src/test/java/org/apache/ambari/server/stack/RepoUtilTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/stack/RepoUtilTest.java b/ambari-server/src/test/java/org/apache/ambari/server/stack/RepoUtilTest.java new file mode 100644 index 0000000..99a34f4 --- /dev/null +++ b/ambari-server/src/test/java/org/apache/ambari/server/stack/RepoUtilTest.java @@ -0,0 +1,166 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.stack; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +import com.google.common.collect.ImmutableListMultimap; +import com.google.common.collect.ImmutableMultimap; +import com.google.common.collect.Multimaps; +import org.apache.ambari.server.controller.RepositoryResponse; +import org.apache.ambari.server.orm.entities.OperatingSystemEntity; +import org.apache.ambari.server.orm.entities.RepositoryEntity; +import org.apache.ambari.server.state.RepositoryInfo; +import org.apache.ambari.server.state.stack.RepositoryXml; +import org.junit.Assert; +import org.junit.Test; + +import com.google.common.base.Optional; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ListMultimap; +import com.google.common.collect.Lists; + +public class RepoUtilTest { + + private static final List<String> OPERATING_SYSTEMS = ImmutableList.of("redhat6", "sles11", "ubuntu12"); + + + @Test public void testAddServiceReposToOperatingSystemEntities_SimpleCase() { + List<OperatingSystemEntity> operatingSystems = new ArrayList<>(); + for (String os: OPERATING_SYSTEMS) { + RepositoryEntity repo1 = repoEntity("HDP", "HDP-2.3", "http://hdp.org/2.3"); + RepositoryEntity repo2 = repoEntity("HDP-UTILS", "HDP-UTILS-1.1.0", "http://hdp.org/utils/1.1.0"); + operatingSystems.add(osEntity(os, repo1, repo2)); + } + ListMultimap<String, RepositoryInfo> serviceRepos = serviceRepos(ImmutableList.of("redhat5", "redhat6", "sles11"), + "MSFT_R", "MSFT_R-8.1", "http://msft.r"); + + RepoUtil.addServiceReposToOperatingSystemEntities(operatingSystems, serviceRepos); + + // Verify results. Service repos should be added only to redhat6 and sles11 + for (OperatingSystemEntity os: operatingSystems) { + Assert.assertNotSame("Redhat5 should not be added as new operating system.", "redhat5", os.getOsType()); + Optional<RepositoryEntity> msft_r = findRepoEntityById(os.getRepositories(), "MSFT_R-8.1"); + Assert.assertTrue( + String.format("Only redhat6 and sles11 should contain the service repo. os: %s, repo: %s", os.getOsType(), msft_r), + findRepoEntityById(os.getRepositories(), "MSFT_R-8.1").isPresent() == ImmutableList.of("redhat6", "sles11").contains(os.getOsType())) ; + } + } + + @Test public void testAddServiceReposToOperatingSystemEntities_RepoAlreadExists() { + List<OperatingSystemEntity> operatingSystems = new ArrayList<>(); + for (String os: OPERATING_SYSTEMS) { + RepositoryEntity repo1 = repoEntity("HDP", "HDP-2.3", "http://hdp.org/2.3"); + RepositoryEntity repo2 = repoEntity("HDP-UTILS", "HDP-UTILS-1.1.0", "http://hdp.org/utils/1.1.0"); + RepositoryEntity repo3 = repoEntity("MSFT_R", "MSFT_R-8.1", "http://msft.r.ORIGINAL"); + operatingSystems.add(osEntity(os, repo1, repo2, repo3)); + } + ListMultimap<String, RepositoryInfo> serviceRepos = serviceRepos(ImmutableList.of("redhat6"), + "MSFT_R", "MSFT_R-8.2", "http://msft.r.NEW"); + + RepoUtil.addServiceReposToOperatingSystemEntities(operatingSystems, serviceRepos); + + // Verify results. Service repo should not be added second time. + for (OperatingSystemEntity os: operatingSystems) { + Optional<RepositoryEntity> msft_r_orig = findRepoEntityById(os.getRepositories(), "MSFT_R-8.1"); + Optional<RepositoryEntity> msft_r_new = findRepoEntityById(os.getRepositories(), "MSFT_R-8.2"); + Assert.assertTrue("Original repo is missing", msft_r_orig.isPresent()); + Assert.assertTrue("Service repo with duplicate name should not have been added", !msft_r_new.isPresent()); + } + } + + @Test public void testGetServiceRepos() { + List<RepositoryInfo> vdfRepos = Lists.newArrayList(repoInfo("HDP", "HDP-2.3", "redhat6"), + repoInfo("HDP-UTILS", "HDP-UTILS-1.1.0.20", "redhat6"), + repoInfo("HDP", "HDP-2.3", "redhat5"), + repoInfo("HDP-UTILS", "HDP-UTILS-1.1.0.20", "redhat5")); + List<RepositoryInfo> stackRepos = Lists.newArrayList(vdfRepos); + stackRepos.add(repoInfo("MSFT_R", "MSFT_R-8.1", "redhat6")); + + ImmutableListMultimap<String, RepositoryInfo> stackReposByOs = + Multimaps.index(stackRepos, RepositoryInfo.GET_OSTYPE_FUNCTION); + + List<RepositoryInfo> serviceRepos = RepoUtil.getServiceRepos(vdfRepos, stackReposByOs); + Assert.assertEquals("Expected 1 service repo", 1, serviceRepos.size()); + Assert.assertEquals("Expected MSFT_R service repo", "MSFT_R", serviceRepos.get(0).getRepoName()); + } + + @Test public void testAsRepositoryResponses() { + List<RepositoryInfo> repos = Lists.newArrayList(repoInfo("HDP", "HDP-2.3", "redhat6"), + repoInfo("HDP-UTILS", "HDP-UTILS-1.1.0.20", "redhat6"), + repoInfo("HDP", "HDP-2.3", "redhat5"), + repoInfo("HDP-UTILS", "HDP-UTILS-1.1.0.20", "redhat5")); + List<RepositoryResponse> responses = RepoUtil.asResponses(repos, "HDP-2.3", "HDP", "2.3"); + + Assert.assertEquals("Wrong number of responses", repos.size(), responses.size()); + for (RepositoryResponse response: responses) { + Assert.assertEquals("Unexpected version definition id", "HDP-2.3", response.getVersionDefinitionId()); + Assert.assertEquals("Unexpected stack name", "HDP", response.getStackName()); + Assert.assertEquals("Unexpected stack version", "2.3", response.getStackVersion()); + } + } + + private static Optional<RepositoryEntity> findRepoEntityById(Iterable<RepositoryEntity> repos, String repoId) { + for (RepositoryEntity repo: repos) if (Objects.equals(repo.getRepositoryId(), repoId)) { + return Optional.of(repo); + } + return Optional.absent(); + } + + private static OperatingSystemEntity osEntity(String os, RepositoryEntity... repoEntities) { + OperatingSystemEntity entity = new OperatingSystemEntity(); + entity.setOsType(os); + for (RepositoryEntity repo: repoEntities) { + entity.getRepositories().add(repo); + } + return entity; + } + + private static RepositoryEntity repoEntity(String name, String repoId, String baseUrl) { + RepositoryEntity repo = new RepositoryEntity(); + repo.setName(name); + repo.setRepositoryId(repoId); + repo.setBaseUrl(baseUrl); + return repo; + } + + private static RepositoryInfo repoInfo(String name, String repoId, String osType) { + RepositoryInfo repo = new RepositoryInfo(); + repo.setRepoName(name); + repo.setRepoId(repoId); + repo.setOsType(osType); + return repo; + } + + private static ListMultimap<String, RepositoryInfo> serviceRepos(List<String> operatingSystems, + String repoName, String repoId, String baseUrl) { + ArrayListMultimap multimap = ArrayListMultimap.create(); + for (String os: operatingSystems) { + RepositoryInfo repoInfo = new RepositoryInfo(); + repoInfo.setOsType(os); + repoInfo.setRepoId(repoId); + repoInfo.setRepoName(repoName); + repoInfo.setBaseUrl(baseUrl); + multimap.put(os, repoInfo); + } + return multimap; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/7961cd11/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerCommonServicesTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerCommonServicesTest.java b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerCommonServicesTest.java index 1d73ff3..6503e7f 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerCommonServicesTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerCommonServicesTest.java @@ -46,6 +46,7 @@ import org.apache.ambari.server.orm.entities.StackEntity; import org.apache.ambari.server.state.CommandScriptDefinition; import org.apache.ambari.server.state.ComponentInfo; import org.apache.ambari.server.state.PropertyInfo; +import org.apache.ambari.server.state.RepositoryInfo; import org.apache.ambari.server.state.ServiceInfo; import org.apache.ambari.server.state.ServiceOsSpecific; import org.apache.ambari.server.state.StackInfo; @@ -55,6 +56,10 @@ import org.easymock.EasyMock; import org.junit.BeforeClass; import org.junit.Test; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Lists; + + /** * StackManager unit tests. */ @@ -141,6 +146,21 @@ public class StackManagerCommonServicesTest { } @Test + public void testAddOnServiceRepoIsLoaded() { + Collection<StackInfo> stacks = stackManager.getStacks("HDP"); + StackInfo stack = null; + for(StackInfo stackInfo: stackManager.getStacks()) { + if ("0.2".equals(stackInfo.getVersion())) { + stack = stackInfo; + break; + } + } + List<RepositoryInfo> repos = stack.getRepositoriesByOs().get("redhat6"); + ImmutableSet<String> repoIds = ImmutableSet.copyOf(Lists.transform(repos, RepositoryInfo.GET_REPO_ID_FUNCTION)); + assertTrue("Repos are expected to contain MSFT_R-8.1", repoIds.contains("ADDON_REPO-1.0")); + } + + @Test public void testGetStack() { StackInfo stack = stackManager.getStack("HDP", "0.1"); assertNotNull(stack); http://git-wip-us.apache.org/repos/asf/ambari/blob/7961cd11/ambari-server/src/test/java/org/apache/ambari/server/stack/StackModuleTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackModuleTest.java b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackModuleTest.java new file mode 100644 index 0000000..0b7d0ff --- /dev/null +++ b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackModuleTest.java @@ -0,0 +1,188 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.stack; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.UUID; + +import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.state.RepositoryInfo; +import org.apache.ambari.server.state.ServiceInfo; +import org.apache.ambari.server.state.stack.RepositoryXml; +import org.apache.ambari.server.state.stack.ServiceMetainfoXml; +import org.junit.Test; + +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMultiset; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Lists; +import com.google.common.collect.Multiset; + + +/** + * Tests for StackModule + */ +public class StackModuleTest { + + @Test + public void stackServiceReposAreRead() throws Exception { + StackModule sm = createStackModule("FooBar", + "2.4", + Optional.of(Lists.newArrayList(repoInfo("foo", "1.0.1", "http://foo.org"))), + Lists.newArrayList(repoInfo("bar", "2.0.1", "http://bar.org"))); + Set<String> repoIds = getIds(sm.getModuleInfo().getRepositories()); + assertEquals(ImmutableSet.of("foo:1.0.1", "bar:2.0.1"), repoIds); + } + + /** + * If more add-on services define the same repo, the duplicate repo definitions should be disregarded. + * @throws Exception + */ + @Test + public void duplicateStackServiceReposAreDiscarded() throws Exception { + StackModule sm = createStackModule("FooBar", + "2.4", + // stack repos + Optional.of(Lists.newArrayList(repoInfo("StackRepoA", "1.1.1", "http://repos.org/stackrepoA"), + repoInfo("StackRepoB", "2.2.2", "http://repos.org/stackrepoB"))), + + // stack service repos + // These two should be preserved. even though duplicates, the contents are the same + Lists.newArrayList(repoInfo("serviceRepoA", "1.0.0", "http://bar.org/1_0_0")), + Lists.newArrayList(repoInfo("serviceRepoA", "1.0.0", "http://bar.org/1_0_0")), + // These should be dropped as the names are the same but contents are different + Lists.newArrayList(repoInfo("serviceRepoB", "1.2.1", "http://bar.org/1_1_1")), + Lists.newArrayList(repoInfo("serviceRepoB", "1.2.3", "http://bar.org/1_1_1")), + // The first one should be dropped (overrides a stack repo), the rest only generates warnings (duplicate urls) + Lists.newArrayList(repoInfo("StackRepoA", "2.0.0", "http://repos.org/stackrepoA_200"), + repoInfo("ShouldBeJustAWarning1", "3.1.1", "http://repos.org/stackrepoA"), + repoInfo("ShouldBeJustAWarning2", "1.0.0", "http://bar.org/1_0_0"))); + List<RepositoryInfo> repos = sm.getModuleInfo().getRepositories(); + + Set<String> repoIds = getIds(repos); + assertEquals("Unexpected number of repos. Each repo should be added only once", repoIds.size(), repos.size()); + assertEquals("Unexpected repositories", + ImmutableSet.of("StackRepoA:1.1.1", + "StackRepoB:2.2.2", + "serviceRepoA:1.0.0", + "ShouldBeJustAWarning1:3.1.1", + "ShouldBeJustAWarning2:1.0.0"), repoIds); + } + + @Test + public void serviceReposAreProcessedEvenIfNoStackRepo() throws Exception { + StackModule sm = createStackModule("FooBar", + "2.4", + Optional.<List<RepositoryInfo>>absent(), + Lists.newArrayList(repoInfo("bar", "2.0.1", "http://bar.org"))); + Set<String> repoIds = getIds(sm.getModuleInfo().getRepositories()); + assertEquals(ImmutableSet.of("bar:2.0.1"), repoIds); + } + + /** + * If two add-on services define the same repo, the repo should be disregarded. + * This applies per os, so the same repo can be defined for multiple os'es (e.g redhat5 and redhat6) + * @throws Exception + */ + @Test + public void duplicateStackServiceReposAreCheckedPerOs() throws Exception { + StackModule sm = createStackModule("FooBar", + "2.4", + Optional.<List<RepositoryInfo>>absent(), + Lists.newArrayList(repoInfo("bar", "2.0.1", "http://bar.org", "centos6")), + Lists.newArrayList(repoInfo("bar", "2.0.1", "http://bar.org", "centos7"))); + Multiset<String> repoIds = getIdsMultiple(sm.getModuleInfo().getRepositories()); + assertEquals("Repo should be occur exactly twice, once for each os type.", + ImmutableMultiset.of("bar:2.0.1", "bar:2.0.1"), repoIds); + } + + private StackModule createStackModule(String stackName, String stackVersion, Optional<? extends List<RepositoryInfo>> stackRepos, + List<RepositoryInfo>... serviceRepoLists) throws AmbariException { + StackDirectory sd = mock(StackDirectory.class); + List<ServiceDirectory> serviceDirectories = Lists.newArrayList(); + for (List<RepositoryInfo> serviceRepoList: serviceRepoLists) { + StackServiceDirectory svd = mock(StackServiceDirectory.class); + RepositoryXml serviceRepoXml = mock(RepositoryXml.class); + when(svd.getRepoFile()).thenReturn(serviceRepoXml); + when(serviceRepoXml.getRepositories()).thenReturn(serviceRepoList); + ServiceMetainfoXml serviceMetainfoXml = mock(ServiceMetainfoXml.class); + when(serviceMetainfoXml.isValid()).thenReturn(true); + ServiceInfo serviceInfo = mock(ServiceInfo.class); + when(serviceInfo.isValid()).thenReturn(true); + when(serviceInfo.getName()).thenReturn(UUID.randomUUID().toString()); // unique service names + when(serviceMetainfoXml.getServices()).thenReturn(Lists.<ServiceInfo>newArrayList(serviceInfo)); + when(svd.getMetaInfoFile()).thenReturn(serviceMetainfoXml); + serviceDirectories.add(svd); + } + if (stackRepos.isPresent()) { + RepositoryXml stackRepoXml = mock(RepositoryXml.class); + when(sd.getRepoFile()).thenReturn(stackRepoXml); + when(stackRepoXml.getRepositories()).thenReturn(stackRepos.get()); + } + when(sd.getServiceDirectories()).thenReturn(serviceDirectories); + when(sd.getStackDirName()).thenReturn(stackName); + when(sd.getDirectory()).thenReturn(new File(stackVersion)); + StackContext ctx = mock(StackContext.class); + StackModule sm = new StackModule(sd, ctx); + sm.resolve(null, + ImmutableMap.of(String.format("%s:%s", stackName, stackVersion), sm), + ImmutableMap.<String, ServiceModule>of(), ImmutableMap.<String, ExtensionModule>of()); + return sm; + } + + private RepositoryInfo repoInfo(String repoName, String repoVersion, String url) { + return repoInfo(repoName, repoVersion, url, "centos6"); + } + + private List<RepositoryInfo> repoInfosForAllOs(String repoName, String repoVersion, String url) { + List<RepositoryInfo> repos = new ArrayList<>(3); + for (String os: new String[]{ "centos5", "centos6", "centos7"}) { + repos.add(repoInfo(repoName, repoVersion, url, os)); + } + return repos; + } + + + private RepositoryInfo repoInfo(String repoName, String repoVersion, String url, String osType) { + RepositoryInfo info = new RepositoryInfo(); + info.setRepoId(String.format("%s:%s", repoName, repoVersion)); + info.setRepoName(repoName); + info.setBaseUrl(url); + info.setOsType(osType); + return info; + } + + private Set<String> getIds(List<RepositoryInfo> repoInfos) { + return ImmutableSet.copyOf(Lists.transform(repoInfos, RepositoryInfo.GET_REPO_ID_FUNCTION)); + } + + private Multiset<String> getIdsMultiple(List<RepositoryInfo> repoInfos) { + return ImmutableMultiset.copyOf(Lists.transform(repoInfos, RepositoryInfo.GET_REPO_ID_FUNCTION)); + } + + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/7961cd11/ambari-server/src/test/java/org/apache/ambari/server/stack/UpdateActiveRepoVersionOnStartupTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/stack/UpdateActiveRepoVersionOnStartupTest.java b/ambari-server/src/test/java/org/apache/ambari/server/stack/UpdateActiveRepoVersionOnStartupTest.java new file mode 100644 index 0000000..9c54a88 --- /dev/null +++ b/ambari-server/src/test/java/org/apache/ambari/server/stack/UpdateActiveRepoVersionOnStartupTest.java @@ -0,0 +1,143 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.stack; + + +import static org.mockito.Mockito.*; + +import java.io.*; + +import org.apache.ambari.server.api.services.AmbariMetaInfo; +import org.apache.ambari.server.orm.InMemoryDefaultTestModule; +import org.apache.ambari.server.orm.dao.ClusterDAO; +import org.apache.ambari.server.orm.dao.ClusterVersionDAO; +import org.apache.ambari.server.orm.dao.RepositoryVersionDAO; +import org.apache.ambari.server.orm.entities.ClusterEntity; +import org.apache.ambari.server.orm.entities.ClusterVersionEntity; +import org.apache.ambari.server.orm.entities.OperatingSystemEntity; +import org.apache.ambari.server.orm.entities.RepositoryEntity; +import org.apache.ambari.server.orm.entities.RepositoryVersionEntity; +import org.apache.ambari.server.orm.entities.StackEntity; +import org.apache.ambari.server.state.RepositoryInfo; +import org.apache.ambari.server.state.StackInfo; +import org.apache.ambari.server.state.stack.upgrade.RepositoryVersionHelper; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import com.google.common.base.Charsets; +import com.google.common.collect.ImmutableList; +import com.google.common.io.Resources; +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.Provider; + +/** + * Unit test for {@link UpdateActiveRepoVersionOnStartup} + */ +public class UpdateActiveRepoVersionOnStartupTest { + + private static String CLUSTER_NAME = "c1"; + private static String ADD_ON_REPO_ID = "MSFT_R-8.0"; + + private RepositoryVersionDAO repositoryVersionDao; + private RepositoryVersionEntity repoVersion; + private UpdateActiveRepoVersionOnStartup activeRepoUpdater; + + @Test + public void addAServiceRepoToExistingRepoVersion() throws Exception { + activeRepoUpdater.process(); + verifyRepoIsAdded(); + } + + /** + * Verifies if the add-on service repo is added to the repo version entity, both json and xml representations. + * + * @throws Exception + */ + private void verifyRepoIsAdded() throws Exception { + verify(repositoryVersionDao, times(1)).merge(repoVersion); + + boolean serviceRepoAddedToJson = false; + outer: + for (OperatingSystemEntity os: repoVersion.getOperatingSystems()) if (os.getOsType().equals("redhat6")) { + for (RepositoryEntity repo: os.getRepositories()) if (repo.getRepositoryId().equals(ADD_ON_REPO_ID)) { + serviceRepoAddedToJson = true; + break outer; + } + } + Assert.assertTrue(ADD_ON_REPO_ID + " is add-on repo was not added to JSON representation", serviceRepoAddedToJson); + } + + @Before + public void init() throws Exception { + ClusterDAO clusterDao = mock(ClusterDAO.class); + ClusterVersionDAO clusterVersionDAO = mock(ClusterVersionDAO.class); + repositoryVersionDao = mock(RepositoryVersionDAO.class); + final RepositoryVersionHelper repositoryVersionHelper = new RepositoryVersionHelper(); + AmbariMetaInfo metaInfo = mock(AmbariMetaInfo.class); + + StackManager stackManager = mock(StackManager.class); + when(metaInfo.getStackManager()).thenReturn(stackManager); + + ClusterEntity cluster = new ClusterEntity(); + cluster.setClusterName(CLUSTER_NAME); + when(clusterDao.findAll()).thenReturn(ImmutableList.of(cluster)); + + StackEntity stackEntity = new StackEntity(); + stackEntity.setStackName("HDP"); + stackEntity.setStackVersion("2.3"); + cluster.setDesiredStack(stackEntity); + + StackInfo stackInfo = new StackInfo(); + stackInfo.setName("HDP"); + stackInfo.setVersion("2.3"); + RepositoryInfo repositoryInfo = new RepositoryInfo(); + repositoryInfo.setBaseUrl("http://msft.r"); + repositoryInfo.setRepoId(ADD_ON_REPO_ID); + repositoryInfo.setRepoName("MSFT_R"); + repositoryInfo.setOsType("redhat6"); + stackInfo.getRepositories().add(repositoryInfo); + when(stackManager.getStack("HDP", "2.3")).thenReturn(stackInfo); + + Provider<RepositoryVersionHelper> repositoryVersionHelperProvider = mock(Provider.class); + when(repositoryVersionHelperProvider.get()).thenReturn(repositoryVersionHelper); + InMemoryDefaultTestModule testModule = new InMemoryDefaultTestModule() { + @Override + protected void configure() { + bind(RepositoryVersionHelper.class).toInstance(repositoryVersionHelper); + requestStaticInjection(RepositoryVersionEntity.class); + } + }; + Injector injector = Guice.createInjector(testModule); + repoVersion = new RepositoryVersionEntity(); + repoVersion.setStack(stackEntity); + repoVersion.setOperatingSystems(resourceAsString("org/apache/ambari/server/stack/UpdateActiveRepoVersionOnStartupTest_initialRepos.json")); + ClusterVersionEntity clusterVersion = new ClusterVersionEntity(); + clusterVersion.setRepositoryVersion(repoVersion); + when(clusterVersionDAO.findByClusterAndStateCurrent(CLUSTER_NAME)).thenReturn(clusterVersion); + + activeRepoUpdater = new UpdateActiveRepoVersionOnStartup(clusterDao, + clusterVersionDAO, repositoryVersionDao, repositoryVersionHelper, metaInfo); + } + + private static String resourceAsString(String resourceName) throws IOException { + return Resources.toString(Resources.getResource(resourceName), Charsets.UTF_8); + } + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/7961cd11/ambari-server/src/test/resources/common-services/ADDON/1.0/configuration/addon-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/common-services/ADDON/1.0/configuration/addon-env.xml b/ambari-server/src/test/resources/common-services/ADDON/1.0/configuration/addon-env.xml new file mode 100644 index 0000000..7005e68 --- /dev/null +++ b/ambari-server/src/test/resources/common-services/ADDON/1.0/configuration/addon-env.xml @@ -0,0 +1,35 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<!-- This is a special config file for properties used to monitor status of the service --> +<configuration supports_adding_forbidden="true"> + <property> + <name>Foo</name> + <display-name>Foo property</display-name> + <description>Foo property</description> + <value>bar</value> + <value-attributes> + <type>string</type> + <overridable>false</overridable> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/7961cd11/ambari-server/src/test/resources/common-services/ADDON/1.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/common-services/ADDON/1.0/metainfo.xml b/ambari-server/src/test/resources/common-services/ADDON/1.0/metainfo.xml new file mode 100644 index 0000000..e7fe568 --- /dev/null +++ b/ambari-server/src/test/resources/common-services/ADDON/1.0/metainfo.xml @@ -0,0 +1,35 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<metainfo> + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>ADDON</name> + <version>1.0</version> + </service> + <components> + <component> + <name>ADDON_CLIENT</name> + <displayName>Add-on service client</displayName> + <category>CLIENT</category> + <cardinality>1+</cardinality> + </component> + </components> + </services> +</metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/7961cd11/ambari-server/src/test/resources/org/apache/ambari/server/stack/UpdateActiveRepoVersionOnStartupTest_initialRepos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/org/apache/ambari/server/stack/UpdateActiveRepoVersionOnStartupTest_initialRepos.json b/ambari-server/src/test/resources/org/apache/ambari/server/stack/UpdateActiveRepoVersionOnStartupTest_initialRepos.json new file mode 100644 index 0000000..f59544f --- /dev/null +++ b/ambari-server/src/test/resources/org/apache/ambari/server/stack/UpdateActiveRepoVersionOnStartupTest_initialRepos.json @@ -0,0 +1,32 @@ +[ + { + "repositories": [ + { + "Repositories/base_url": "http://192.168.99.100/repos/HDP-2.4.0.0/", + "Repositories/repo_name": "HDP", + "Repositories/repo_id": "HDP-2.4" + }, + { + "Repositories/base_url": "http://192.168.99.100/repos/HDP-UTILS-1.1.0.20/", + "Repositories/repo_name": "HDP-UTILS", + "Repositories/repo_id": "HDP-UTILS-1.1.0.20" + } + ], + "OperatingSystems/os_type": "redhat6" + }, + { + "repositories": [ + { + "Repositories/base_url": "http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos7/2.x/BUILDS/2.4.3.0-207", + "Repositories/repo_name": "HDP", + "Repositories/repo_id": "HDP-2.4" + }, + { + "Repositories/base_url": "http://s3.amazonaws.com/dev.hortonworks.com/HDP-UTILS-1.1.0.20/repos/centos7", + "Repositories/repo_name": "HDP-UTILS", + "Repositories/repo_id": "HDP-UTILS-1.1.0.20" + } + ], + "OperatingSystems/os_type": "redhat7" + } +] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/7961cd11/ambari-server/src/test/resources/stacks_with_common_services/HDP/0.2/services/ADDON/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks_with_common_services/HDP/0.2/services/ADDON/metainfo.xml b/ambari-server/src/test/resources/stacks_with_common_services/HDP/0.2/services/ADDON/metainfo.xml new file mode 100644 index 0000000..07242db --- /dev/null +++ b/ambari-server/src/test/resources/stacks_with_common_services/HDP/0.2/services/ADDON/metainfo.xml @@ -0,0 +1,28 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<metainfo> + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>ADDON</name> + <version>1.0</version> + <extends>common-services/ADDON/1.0</extends> + </service> + </services> +</metainfo>