AMBARI-15388 - Upgrade XML should be pushed down as much as possible to the services (Tim Thorpe via jluniya)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/cec0b240 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/cec0b240 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/cec0b240 Branch: refs/heads/branch-2.4 Commit: cec0b24018517d47e686eba77156810c64cd89e3 Parents: 8ddb427 Author: Jayush Luniya <[email protected]> Authored: Thu May 19 14:21:22 2016 -0700 Committer: Jayush Luniya <[email protected]> Committed: Thu May 19 14:22:24 2016 -0700 ---------------------------------------------------------------------- .../server/stack/CommonServiceDirectory.java | 10 + .../ambari/server/stack/ServiceDirectory.java | 19 ++ .../ambari/server/stack/ServiceModule.java | 4 + .../ambari/server/stack/StackDirectory.java | 66 ++++-- .../apache/ambari/server/stack/StackModule.java | 233 +++++++++++++++++++ .../server/stack/StackServiceDirectory.java | 30 ++- .../apache/ambari/server/state/ServiceInfo.java | 14 ++ .../ambari/server/state/stack/UpgradePack.java | 72 ++++++ .../state/stack/upgrade/ClusterGrouping.java | 70 ++++++ .../server/state/stack/upgrade/Grouping.java | 69 ++++++ .../stack/upgrade/ServiceCheckGrouping.java | 78 +++++++ .../stacks/HDP/2.3/upgrades/upgrade-2.3.xml | 6 +- .../stacks/HDP/2.3/upgrades/upgrade-2.4.xml | 6 +- .../stacks/HDP/2.3/upgrades/upgrade-2.5.xml | 6 +- .../stacks/HDP/2.4/upgrades/upgrade-2.4.xml | 6 +- .../stacks/HDP/2.4/upgrades/upgrade-2.5.xml | 6 +- .../stacks/HDP/2.5/upgrades/upgrade-2.5.xml | 6 +- .../server/stack/StackManagerMiscTest.java | 31 +++ .../server/state/stack/UpgradePackTest.java | 66 ++++++ .../upgrades/HDP/2.2.0/upgrade_test_15388.xml | 94 ++++++++ .../HDP/2.2.0/upgrades/upgrade_test_15388.xml | 232 ++++++++++++++++++ .../HDP/2.2.0/metainfo.xml | 22 ++ .../HDP/2.2.0/repos/hdp.json | 15 ++ .../HDP/2.2.0/repos/repoinfo.xml | 34 +++ .../HDP/2.2.0/repos/version-2.2.0.4-123.xml | 49 ++++ .../HDP/2.2.0/role_command_order.json | 80 +++++++ .../HDP/2.2.0/services/HDFS/metainfo.xml | 203 ++++++++++++++++ .../upgrades/HDP/2.2.0/upgrade_test_15388.xml | 93 ++++++++ .../HDP/2.2.0/upgrades/config-upgrade.xml | 101 ++++++++ .../HDP/2.2.0/upgrades/upgrade_test_15388.xml | 232 ++++++++++++++++++ 30 files changed, 1905 insertions(+), 48 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/cec0b240/ambari-server/src/main/java/org/apache/ambari/server/stack/CommonServiceDirectory.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/CommonServiceDirectory.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/CommonServiceDirectory.java index 7f7a49e..cbbdb91 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/stack/CommonServiceDirectory.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/CommonServiceDirectory.java @@ -85,6 +85,16 @@ public class CommonServiceDirectory extends ServiceDirectory { LOG.debug(String.format("Service package folder %s for common service %s does not exist.", absPackageDir, serviceId )); } + + File absUpgradesDir = new File(getAbsolutePath() + File.separator + UPGRADES_FOLDER_NAME); + if(absUpgradesDir.isDirectory()) { + upgradesDir = absUpgradesDir; + LOG.debug(String.format("Service upgrades folder for common service %s has been resolved to %s", + serviceId, upgradesDir)); + } else { + LOG.debug(String.format("Service upgrades folder %s for common service %s does not exist.", + absUpgradesDir, serviceId )); + } parseMetaInfoFile(); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/cec0b240/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceDirectory.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceDirectory.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceDirectory.java index 8a7b42b..9ed2c24 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceDirectory.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceDirectory.java @@ -70,6 +70,11 @@ public abstract class ServiceDirectory extends StackDefinitionDirectory { protected String packageDir; /** + * upgrades directory path + */ + protected File upgradesDir; + + /** * service metainfo file object representation */ private ServiceMetainfoXml metaInfoXml; @@ -85,6 +90,11 @@ public abstract class ServiceDirectory extends StackDefinitionDirectory { protected static final String PACKAGE_FOLDER_NAME = "package"; /** + * upgrades directory name + */ + protected static final String UPGRADES_FOLDER_NAME = "upgrades"; + + /** * service metainfo file name */ private static final String SERVICE_METAINFO_FILE_NAME = "metainfo.xml"; @@ -148,6 +158,15 @@ public abstract class ServiceDirectory extends StackDefinitionDirectory { } /** + * Obtain the upgrades directory path. + * + * @return upgrades directory path + */ + public File getUpgradesDir() { + return upgradesDir; + } + + /** * Obtain the metrics file. * * @return metrics file http://git-wip-us.apache.org/repos/asf/ambari/blob/cec0b240/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java index f781574..17a2a93 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java @@ -133,6 +133,7 @@ public class ServiceModule extends BaseModule<ServiceModule, ServiceInfo> implem serviceInfo.setWidgetsDescriptorFile(serviceDirectory.getWidgetsDescriptorFile(serviceInfo.getName())); serviceInfo.setSchemaVersion(AmbariMetaInfo.SCHEMA_VERSION_2); serviceInfo.setServicePackageFolder(serviceDirectory.getPackageDir()); + serviceInfo.setServiceUpgradesFolder(serviceDirectory.getUpgradesDir()); serviceInfo.setAdvisorFile(serviceDirectory.getAdvisorFile()); serviceInfo.setAdvisorName(serviceDirectory.getAdvisorName(serviceInfo.getName())); @@ -213,6 +214,9 @@ public class ServiceModule extends BaseModule<ServiceModule, ServiceInfo> implem if (serviceInfo.getServicePackageFolder() == null) { serviceInfo.setServicePackageFolder(parent.getServicePackageFolder()); } + if (serviceInfo.getServiceUpgradesFolder() == null) { + serviceInfo.setServiceUpgradesFolder(parent.getServiceUpgradesFolder()); + } if (serviceInfo.getMetricsFile() == null) { serviceInfo.setMetricsFile(parent.getMetricsFile()); } http://git-wip-us.apache.org/repos/asf/ambari/blob/cec0b240/ambari-server/src/main/java/org/apache/ambari/server/stack/StackDirectory.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackDirectory.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackDirectory.java index 13d5047..ee9e383 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackDirectory.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackDirectory.java @@ -34,15 +34,13 @@ import org.slf4j.LoggerFactory; import javax.xml.bind.JAXBException; import java.io.File; +import java.io.FilenameFilter; import java.io.IOException; -import java.io.InputStream; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.Map; -import java.util.Set; /** * Encapsulates IO operations on a stack definition stack directory. @@ -417,10 +415,11 @@ public class StackDirectory extends StackDefinitionDirectory { serviceDirectories = dirs; } + /** * Parse all stack upgrade files for the stack. * - * @param subDirs stack sub directories + * @param subDirs stack sub directories * @throws AmbariException if unable to parse stack upgrade file */ private void parseUpgradePacks(Collection<String> subDirs) throws AmbariException { @@ -432,28 +431,19 @@ public class StackDirectory extends StackDefinitionDirectory { upgradesDir = f.getAbsolutePath(); for (File upgradeFile : f.listFiles(XML_FILENAME_FILTER)) { if (upgradeFile.getName().toLowerCase().startsWith(CONFIG_UPGRADE_XML_FILENAME_PREFIX)) { - try { // Parse config upgrade pack - if (configUpgradePack == null) { - configUpgradePack = unmarshaller.unmarshal(ConfigUpgradePack.class, upgradeFile); - } else { // If user messed things up with lower/upper case filenames - throw new AmbariException(String.format("There are multiple files with name like %s" + - upgradeFile.getAbsolutePath())); - } - } catch (JAXBException e) { - throw new AmbariException("Unable to parse stack upgrade file at location: " + - upgradeFile.getAbsolutePath(), e); + if (configUpgradePack == null) { + configUpgradePack = parseConfigUpgradePack(upgradeFile); } - } else { - try { - String upgradePackName = FilenameUtils.removeExtension(upgradeFile.getName()); - UpgradePack pack = unmarshaller.unmarshal(UpgradePack.class, upgradeFile); - pack.setName(upgradePackName); - upgradeMap.put(upgradePackName, pack); - } catch (JAXBException e) { - throw new AmbariException("Unable to parse stack upgrade file at location: " + - upgradeFile.getAbsolutePath(), e); + else { // If user messed things up with lower/upper case filenames + throw new AmbariException(String.format("There are multiple files with name like %s" + upgradeFile.getAbsolutePath())); } } + else { + String upgradePackName = FilenameUtils.removeExtension(upgradeFile.getName()); + UpgradePack pack = parseUpgradePack(upgradePackName, upgradeFile); + pack.setName(upgradePackName); + upgradeMap.put(upgradePackName, pack); + } } } } @@ -462,7 +452,7 @@ public class StackDirectory extends StackDefinitionDirectory { LOG.info("Stack '{}' doesn't contain an upgrade directory ", getPath()); } - if (! upgradeMap.isEmpty()) { + if (!upgradeMap.isEmpty()) { upgradePacks = upgradeMap; } @@ -471,7 +461,35 @@ public class StackDirectory extends StackDefinitionDirectory { } else { LOG.info("Stack '{}' doesn't contain config upgrade pack file", getPath()); } + } + private UpgradePack parseUpgradePack(final String packName, File upgradeFile) throws AmbariException { + UpgradePack pack = null; + try { + pack = unmarshaller.unmarshal(UpgradePack.class, upgradeFile); + pack.setName(packName); + } + catch (JAXBException e) { + if (upgradeFile == null) { + throw new AmbariException("Null upgrade pack"); + } + throw new AmbariException("Unable to parse stack upgrade file at location: " + upgradeFile.getAbsolutePath(), e); + } + return pack; + } + + private ConfigUpgradePack parseConfigUpgradePack(File upgradeFile) throws AmbariException { + ConfigUpgradePack pack = null; + try { + pack = unmarshaller.unmarshal(ConfigUpgradePack.class, upgradeFile); + } + catch (JAXBException e) { + if (upgradeFile == null) { + throw new AmbariException("Null config upgrade pack"); + } + throw new AmbariException("Unable to parse stack upgrade file at location: " + upgradeFile.getAbsolutePath(), e); + } + return pack; } /** http://git-wip-us.apache.org/repos/asf/ambari/blob/cec0b240/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java index 5a18b3f..d819a52 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java @@ -18,25 +18,39 @@ package org.apache.ambari.server.stack; +import java.io.File; +import java.io.FilenameFilter; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Set; +import javax.xml.bind.JAXBException; + import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.api.services.AmbariMetaInfo; +import org.apache.ambari.server.stack.StackDefinitionDirectory; import org.apache.ambari.server.state.ConfigHelper; import org.apache.ambari.server.state.PropertyDependencyInfo; import org.apache.ambari.server.state.PropertyInfo; import org.apache.ambari.server.state.RepositoryInfo; import org.apache.ambari.server.state.ServiceInfo; import org.apache.ambari.server.state.StackInfo; +import org.apache.ambari.server.state.stack.ConfigUpgradePack; import org.apache.ambari.server.state.stack.RepositoryXml; import org.apache.ambari.server.state.stack.ServiceMetainfoXml; import org.apache.ambari.server.state.stack.StackMetainfoXml; +import org.apache.ambari.server.state.stack.UpgradePack; +import org.apache.ambari.server.state.stack.UpgradePack.OrderService; +import org.apache.ambari.server.state.stack.upgrade.ClusterGrouping; +import org.apache.ambari.server.state.stack.upgrade.Grouping; +import org.apache.ambari.server.state.stack.upgrade.ServiceCheckGrouping; +import org.apache.ambari.server.state.stack.upgrade.ClusterGrouping.ExecuteStage; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -113,6 +127,11 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V protected boolean valid = true; /** + * file unmarshaller + */ + ModuleFileUnmarshaller unmarshaller = new ModuleFileUnmarshaller(); + + /** * Logger */ private final static Logger LOG = LoggerFactory.getLogger(StackModule.class); @@ -155,6 +174,7 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V if (parentVersion != null) { mergeStackWithParent(parentVersion, allStacks, commonServices); } + processUpgradePacks(); processRepositories(); processPropertyDependencies(); moduleState = ModuleState.RESOLVED; @@ -644,6 +664,219 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V } /** + * Process upgrade packs associated with the stack. + * @throws AmbariException if unable to fully process the upgrade packs + */ + private void processUpgradePacks() throws AmbariException { + if (stackInfo.getUpgradePacks() == null) { + return; + } + + for (UpgradePack pack : stackInfo.getUpgradePacks().values()) { + List<UpgradePack> servicePacks = new ArrayList<>(); + for (ServiceModule module : serviceModules.values()) { + File upgradesFolder = module.getModuleInfo().getServiceUpgradesFolder(); + if (upgradesFolder != null) { + UpgradePack servicePack = getServiceUpgradePack(pack, upgradesFolder); + if (servicePack != null) { + servicePacks.add(servicePack); + } + } + } + if (servicePacks.size() > 0) { + LOG.info("Merging service specific upgrades for pack: " + pack.getName()); + mergeUpgradePack(pack, servicePacks); + } + } + + ConfigUpgradePack configPack = stackInfo.getConfigUpgradePack(); + if (configPack == null) { + return; + } + for (ServiceModule module : serviceModules.values()) { + File upgradesFolder = module.getModuleInfo().getServiceUpgradesFolder(); + if (upgradesFolder != null) { + mergeConfigUpgradePack(configPack, upgradesFolder); + } + } + } + + /** + * Attempts to merge, into the stack config upgrade, all the config upgrades + * for any service which specifies its own upgrade. + */ + private void mergeConfigUpgradePack(ConfigUpgradePack pack, File upgradesFolder) throws AmbariException { + File stackFolder = new File(upgradesFolder, stackInfo.getName()); + File versionFolder = new File(stackFolder, stackInfo.getVersion()); + File serviceConfig = new File(versionFolder, StackDefinitionDirectory.CONFIG_UPGRADE_XML_FILENAME_PREFIX); + if (!serviceConfig.exists()) { + return; + } + + try { + ConfigUpgradePack serviceConfigPack = unmarshaller.unmarshal(ConfigUpgradePack.class, serviceConfig); + pack.services.addAll(serviceConfigPack.services); + } + catch (JAXBException e) { + throw new AmbariException("Unable to parse service config upgrade file at location: " + serviceConfig.getAbsolutePath(), e); + } + } + + /** + * Returns the upgrade pack for a service if it exists, otherwise returns null + */ + private UpgradePack getServiceUpgradePack(UpgradePack pack, File upgradesFolder) throws AmbariException { + File stackFolder = new File(upgradesFolder, stackInfo.getName()); + File versionFolder = new File(stackFolder, stackInfo.getVersion()); + File servicePackFile = new File(versionFolder, pack.getName() + ".xml"); + LOG.info("Service folder: " + servicePackFile.getAbsolutePath()); + if (!servicePackFile.exists()) { + return null; + } + return parseServiceUpgradePack(pack, servicePackFile); + } + + /** + * Attempts to merge, into the stack upgrade, all the upgrades + * for any service which specifies its own upgrade. + */ + private void mergeUpgradePack(UpgradePack pack, List<UpgradePack> servicePacks) throws AmbariException { + List<Grouping> originalGroups = pack.getAllGroups(); + Map<String, List<Grouping>> allGroupMap = new HashMap<>(); + for (Grouping group : originalGroups) { + List<Grouping> list = new ArrayList<>(); + list.add(group); + allGroupMap.put(group.name, list); + } + for (UpgradePack servicePack : servicePacks) { + for (Grouping group : servicePack.getAllGroups()) { + if (allGroupMap.containsKey(group.name)) { + List<Grouping> list = allGroupMap.get(group.name); + Grouping first = list.get(0); + if (!first.getClass().equals(group.getClass())) { + throw new AmbariException("Expected class: " + first.getClass() + " instead of " + group.getClass()); + } + /* If the current group doesn't specify an "after entry" and the first group does + then the current group should be added first. The first group in the list should + never be ordered relative to any other group. */ + if (group.addAfterGroupEntry == null && first.addAfterGroupEntry != null) { + list.add(0, group); + } + else { + list.add(group); + } + } + else { + List<Grouping> list = new ArrayList<>(); + list.add(group); + allGroupMap.put(group.name, list); + } + } + } + + Map<String, Grouping> mergedGroupMap = new HashMap<>(); + for (String key : allGroupMap.keySet()) { + Iterator<Grouping> iterator = allGroupMap.get(key).iterator(); + Grouping group = iterator.next(); + if (iterator.hasNext()) { + group.merge(iterator); + } + mergedGroupMap.put(key, group); + } + + orderGroups(originalGroups, mergedGroupMap); + } + + /** + * Orders the upgrade groups. All new groups specified in a service's upgrade file must + * specify after which group they should be placed in the upgrade order. + */ + private void orderGroups(List<Grouping> groups, Map<String, Grouping> mergedGroupMap) throws AmbariException { + Map<String, List<Grouping>> skippedGroups = new HashMap<>(); + for (Map.Entry<String, Grouping> entry : mergedGroupMap.entrySet()) { + String key = entry.getKey(); + Grouping group = entry.getValue(); + if (!groups.contains(group)) { + boolean added = addGrouping(groups, group); + if (added) { + addSkippedGroup(groups, skippedGroups, group); + } else { + List<Grouping> tmp = null; + // store the group until later + if (skippedGroups.containsKey(group.addAfterGroup)) { + tmp = skippedGroups.get(group.addAfterGroup); + } else { + tmp = new ArrayList<Grouping>(); + skippedGroups.put(group.addAfterGroup, tmp); + } + tmp.add(group); + } + } + } + if (!skippedGroups.isEmpty()) { + throw new AmbariException("Missing groups: " + skippedGroups.keySet()); + } + } + + /** + * Adds the group provided if the group which it should come after has been added. + */ + private boolean addGrouping(List<Grouping> groups, Grouping group) throws AmbariException { + if (group.addAfterGroup == null) { + throw new AmbariException("Group " + group.name + " needs to specify which group it should come after"); + } + else { + // Check the current services, if the "after" service is there then add these + for (int index = groups.size() - 1; index >= 0; index--) { + String name = groups.get(index).name; + if (name.equals(group.addAfterGroup)) { + groups.add(index + 1, group); + LOG.debug("Added group/after: " + group.name + "/" + group.addAfterGroup); + return true; + } + } + } + return false; + } + + /** + * Adds any groups which have been previously skipped if the group which they should come + * after have been added. + */ + private void addSkippedGroup(List<Grouping> groups, Map<String, List<Grouping>> skippedGroups, Grouping groupJustAdded) throws AmbariException { + if (skippedGroups.containsKey(groupJustAdded.name)) { + List<Grouping> groupsToAdd = skippedGroups.remove(groupJustAdded.name); + for (Grouping group : groupsToAdd) { + boolean added = addGrouping(groups, group); + if (added) { + addSkippedGroup(groups, skippedGroups, group); + } else { + throw new AmbariException("Failed to add group " + group.name); + } + } + } + } + + /** + * Parses the service specific upgrade file and merges the none order elements + * (prerequisite check and processing sections). + */ + private UpgradePack parseServiceUpgradePack(UpgradePack parent, File serviceFile) throws AmbariException { + UpgradePack pack = null; + try { + pack = unmarshaller.unmarshal(UpgradePack.class, serviceFile); + } + catch (JAXBException e) { + throw new AmbariException("Unable to parse service upgrade file at location: " + serviceFile.getAbsolutePath(), e); + } + + parent.mergePrerequisiteChecks(pack); + parent.mergeProcessing(pack); + + return pack; + } + + /** * Process repositories associated with the stack. * @throws AmbariException if unable to fully process the stack repositories */ http://git-wip-us.apache.org/repos/asf/ambari/blob/cec0b240/ambari-server/src/main/java/org/apache/ambari/server/stack/StackServiceDirectory.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackServiceDirectory.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackServiceDirectory.java index 88f6e19..68c1dd6 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackServiceDirectory.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackServiceDirectory.java @@ -87,16 +87,34 @@ public class StackServiceDirectory extends ServiceDirectory { int fileCount = files.length; if (fileCount > 0) { packageDir = absPackageDir.getPath().substring(stackDir.getParentFile().getParentFile().getPath().length() + 1); - LOG.debug(String.format("Service package folder for service %s for stack %s has been resolved to %s", - serviceDir.getName(), stackId, packageDir)); + LOG.debug("Service package folder for service %s for stack %s has been resolved to %s", + serviceDir.getName(), stackId, packageDir); } else { - LOG.debug(String.format("Service package folder %s for service %s for stack %s is empty.", - absPackageDir, serviceDir.getName(), stackId)); + LOG.debug("Service package folder %s for service %s for stack %s is empty.", + absPackageDir, serviceDir.getName(), stackId); } } else { - LOG.debug(String.format("Service package folder %s for service %s for stack %s does not exist.", - absPackageDir, serviceDir.getName(), stackId)); + LOG.debug("Service package folder %s for service %s for stack %s does not exist.", + absPackageDir, serviceDir.getName(), stackId); + } + + File absUpgradesDir = new File(getAbsolutePath() + File.separator + UPGRADES_FOLDER_NAME); + if (absUpgradesDir.isDirectory()) { + String[] files = absUpgradesDir.list(); + int fileCount = files.length; + if (fileCount > 0) { + upgradesDir = absUpgradesDir; + LOG.debug("Service upgrades folder for service %s for stack %s has been resolved to %s", + serviceDir.getName(), stackId, packageDir); + } + else { + LOG.debug("Service upgrades folder %s for service %s for stack %s is empty.", + absUpgradesDir, serviceDir.getName(), stackId); + } + } else { + LOG.debug("Service upgrades folder %s for service %s for stack %s does not exist.", + absUpgradesDir, serviceDir.getName(), stackId); } parseMetaInfoFile(); } http://git-wip-us.apache.org/repos/asf/ambari/blob/cec0b240/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java index 43cefb9..76840ea 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java @@ -261,6 +261,12 @@ public class ServiceInfo implements Validable{ @XmlTransient private String servicePackageFolder; + /** + * Stores the path to the upgrades folder which contains the upgrade xmls for the given service. + */ + @XmlTransient + private File serviceUpgradesFolder; + public boolean isDeleted() { return isDeleted; } @@ -583,6 +589,14 @@ public String getVersion() { this.servicePackageFolder = servicePackageFolder; } + public File getServiceUpgradesFolder() { + return serviceUpgradesFolder; + } + + public void setServiceUpgradesFolder(File serviceUpgradesFolder) { + this.serviceUpgradesFolder = serviceUpgradesFolder; + } + /** * Exposes (and initializes on first use) map of os-specific details. * @return map of OS specific details keyed by family http://git-wip-us.apache.org/repos/asf/ambari/blob/cec0b240/ambari-server/src/main/java/org/apache/ambari/server/state/stack/UpgradePack.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/UpgradePack.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/UpgradePack.java index b860731..486d432 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/UpgradePack.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/UpgradePack.java @@ -154,6 +154,74 @@ public class UpgradePack { } /** + * Merges the prerequisite checks section of the upgrade xml with + * the prerequisite checks from a service's upgrade xml. + * These are added to the end of the current list of checks. + * + * @param pack + * the service's upgrade pack + */ + public void mergePrerequisiteChecks(UpgradePack pack) { + PrerequisiteChecks newPrereqChecks = pack.prerequisiteChecks; + if (prerequisiteChecks == null) { + prerequisiteChecks = newPrereqChecks; + return; + } + + if (newPrereqChecks == null) { + return; + } + + if (prerequisiteChecks.checks == null) { + prerequisiteChecks.checks = new ArrayList<String>(); + } + if (newPrereqChecks.checks != null) { + prerequisiteChecks.checks.addAll(newPrereqChecks.checks); + } + + if (newPrereqChecks.configuration == null) { + return; + } + + if (prerequisiteChecks.configuration == null) { + prerequisiteChecks.configuration = newPrereqChecks.configuration; + return; + } + if (prerequisiteChecks.configuration.globalProperties == null) { + prerequisiteChecks.configuration.globalProperties = new ArrayList<PrerequisiteProperty>(); + } + if (prerequisiteChecks.configuration.prerequisiteCheckProperties == null) { + prerequisiteChecks.configuration.prerequisiteCheckProperties = new ArrayList<PrerequisiteCheckProperties>(); + } + if (newPrereqChecks.configuration.globalProperties != null) { + prerequisiteChecks.configuration.globalProperties.addAll(newPrereqChecks.configuration.globalProperties); + } + if (newPrereqChecks.configuration.prerequisiteCheckProperties != null) { + prerequisiteChecks.configuration.prerequisiteCheckProperties.addAll(newPrereqChecks.configuration.prerequisiteCheckProperties); + } + } + +/** + * Merges the processing section of the upgrade xml with + * the processing section from a service's upgrade xml. + * These are added to the end of the current list of services. + * + * @param pack + * the service's upgrade pack + */ + public void mergeProcessing(UpgradePack pack) { + List<ProcessingService> list = pack.processing; + if (list == null) { + return; + } + if (processing == null) { + processing = list; + return; + } + processing.addAll(list); + } + + /** * Gets a list of stacks which are between the current stack version and the * target stack version inclusive. For example, if upgrading from HDP-2.2 to * HDP-2.4, this should include HDP-2.3 and HDP-2.4. @@ -195,6 +263,10 @@ public class UpgradePack { return skipServiceCheckFailures; } + public List<Grouping> getAllGroups() { + return groups; + } + /** * Gets the groups defined for the upgrade pack. If a direction is defined for * a group, it must match the supplied direction to be returned http://git-wip-us.apache.org/repos/asf/ambari/blob/cec0b240/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ClusterGrouping.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ClusterGrouping.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ClusterGrouping.java index 3325469..80bb26c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ClusterGrouping.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ClusterGrouping.java @@ -19,7 +19,9 @@ package org.apache.ambari.server.state.stack.upgrade; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -33,6 +35,7 @@ import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; import javax.xml.bind.annotation.XmlType; +import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.stack.HostsType; import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Host; @@ -309,4 +312,71 @@ public class ClusterGrouping extends Grouping { mt.structuredOut = obj.toString(); } + + /** + * Attempts to merge the given cluster groupings. This merges the execute stages + * in an order specific manner. + */ + public void merge(Iterator<Grouping> iterator) throws AmbariException { + if (executionStages == null) { + executionStages = new ArrayList<ExecuteStage>(); + } + Map<String, List<ExecuteStage>> skippedStages = new HashMap<>(); + while (iterator.hasNext()) { + Grouping next = iterator.next(); + if (!(next instanceof ClusterGrouping)) { + throw new AmbariException("Invalid group type " + next.getClass().getSimpleName() + " expected cluster group"); + } + ClusterGrouping clusterGroup = (ClusterGrouping) next; + + boolean added = addGroupingStages(clusterGroup.executionStages, clusterGroup.addAfterGroupEntry); + if (added) { + addSkippedStages(skippedStages, clusterGroup.executionStages); + } + else { + // store these services until later + if (skippedStages.containsKey(next.addAfterGroupEntry)) { + List<ExecuteStage> tmp = skippedStages.get(clusterGroup.addAfterGroupEntry); + tmp.addAll(clusterGroup.executionStages); + } + else { + skippedStages.put(clusterGroup.addAfterGroupEntry, clusterGroup.executionStages); + } + } + } + } + + /** + * Adds the given stages if the stage they are supposed to come after has been added. + */ + private boolean addGroupingStages(List<ExecuteStage> stagesToAdd, String after) { + if (after == null) { + executionStages.addAll(stagesToAdd); + return true; + } + else { + // Check the current stages, if the "after" stage is there then add these + for (int index = executionStages.size() - 1; index >= 0; index--) { + ExecuteStage stage = executionStages.get(index); + if ((stage.service != null && stage.service.equals(after)) || stage.title.equals(after)) { + executionStages.addAll(index + 1, stagesToAdd); + return true; + } + } + } + return false; + } + + /** + * Adds the skipped stages if the stage they are supposed to come after has been added. + */ + private void addSkippedStages(Map<String, List<ExecuteStage>> skippedStages, List<ExecuteStage> stagesJustAdded) { + for (ExecuteStage stage : stagesJustAdded) { + if (skippedStages.containsKey(stage.service)) { + List<ExecuteStage> stagesToAdd = skippedStages.remove(stage.service); + addGroupingStages(stagesToAdd, stage.service); + addSkippedStages(skippedStages, stagesToAdd); + } + } + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/cec0b240/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java index 67d7fdb..be1f469 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java @@ -19,7 +19,9 @@ package org.apache.ambari.server.state.stack.upgrade; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -28,9 +30,11 @@ import javax.xml.bind.annotation.XmlAttribute; import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlSeeAlso; +import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.stack.HostsType; import org.apache.ambari.server.state.UpgradeContext; import org.apache.ambari.server.state.stack.UpgradePack; +import org.apache.ambari.server.state.stack.UpgradePack.OrderService; import org.apache.ambari.server.state.stack.UpgradePack.ProcessingComponent; import org.apache.ambari.server.utils.SetUtils; import org.apache.commons.lang.StringUtils; @@ -47,6 +51,12 @@ public class Grouping { @XmlAttribute(name="title") public String title; + @XmlElement(name="add-after-group") + public String addAfterGroup; + + @XmlElement(name="add-after-group-entry") + public String addAfterGroupEntry; + @XmlElement(name="skippable", defaultValue="false") public boolean skippable = false; @@ -310,4 +320,63 @@ public class Grouping { tasks.add(initial); } } + + /** + * Merge the services of all the child groups, with the current services. + * Keeping the order specified by the group. + */ + public void merge(Iterator<Grouping> iterator) throws AmbariException { + Map<String, List<OrderService>> skippedServices = new HashMap<>(); + while (iterator.hasNext()) { + Grouping group = iterator.next(); + + boolean added = addGroupingServices(group.services, group.addAfterGroupEntry); + if (added) { + addSkippedServices(skippedServices, group.services); + } else { + // store these services until later + if (skippedServices.containsKey(group.addAfterGroupEntry)) { + List<OrderService> tmp = skippedServices.get(group.addAfterGroupEntry); + tmp.addAll(group.services); + } else { + skippedServices.put(group.addAfterGroupEntry, group.services); + } + } + } + } + + /** + * Merge the services to add after a particular service name + */ + private boolean addGroupingServices(List<OrderService> servicesToAdd, String after) { + if (after == null) { + services.addAll(servicesToAdd); + return true; + } + else { + // Check the current services, if the "after" service is there then add these + for (int index = services.size() - 1; index >= 0; index--) { + OrderService service = services.get(index); + if (service.serviceName.equals(after)) { + services.addAll(index + 1, servicesToAdd); + return true; + } + } + } + return false; + } + + /** + * Adds services which were previously skipped, if the service they are supposed + * to come after has been added. + */ + private void addSkippedServices(Map<String, List<OrderService>> skippedServices, List<OrderService> servicesJustAdded) { + for (OrderService service : servicesJustAdded) { + if (skippedServices.containsKey(service.serviceName)) { + List<OrderService> servicesToAdd = skippedServices.remove(service.serviceName); + addGroupingServices(servicesToAdd, service.serviceName); + addSkippedServices(skippedServices, servicesToAdd); + } + } + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/cec0b240/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ServiceCheckGrouping.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ServiceCheckGrouping.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ServiceCheckGrouping.java index 5cda422..6a085f3 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ServiceCheckGrouping.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ServiceCheckGrouping.java @@ -19,7 +19,9 @@ package org.apache.ambari.server.state.stack.upgrade; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -81,6 +83,13 @@ public class ServiceCheckGrouping extends Grouping { } /** + * @return the set of service names that should be excluded + */ + public Set<String> getExcluded() { + return excludeServices; + } + + /** * Used to build stages for service check groupings. */ public class ServiceCheckBuilder extends StageWrapperBuilder { @@ -187,4 +196,73 @@ public class ServiceCheckGrouping extends Grouping { return false; } } + + /** + * Attempts to merge all the service check groupings. This merges the excluded list and + * the priorities. The priorities are merged in an order specific manner. + */ + public void merge(Iterator<Grouping> iterator) throws AmbariException { + List<String> priorities = new ArrayList<>(); + priorities.addAll(getPriorities()); + Map<String, Set<String>> skippedPriorities = new HashMap<>(); + while (iterator.hasNext()) { + Grouping next = iterator.next(); + if (!(next instanceof ServiceCheckGrouping)) { + throw new AmbariException("Invalid group type " + next.getClass().getSimpleName() + " expected service check group"); + } + ServiceCheckGrouping checkGroup = (ServiceCheckGrouping) next; + getExcluded().addAll(checkGroup.getExcluded()); + + boolean added = addPriorities(priorities, checkGroup.getPriorities(), checkGroup.addAfterGroupEntry); + if (added) { + addSkippedPriorities(priorities, skippedPriorities, checkGroup.getPriorities()); + } + else { + // store these services until later + if (skippedPriorities.containsKey(checkGroup.addAfterGroupEntry)) { + Set<String> tmp = skippedPriorities.get(checkGroup.addAfterGroupEntry); + tmp.addAll(checkGroup.getPriorities()); + } + else { + skippedPriorities.put(checkGroup.addAfterGroupEntry, checkGroup.getPriorities()); + } + } + } + getPriorities().clear(); + getPriorities().addAll(priorities); + } + + /** + * Add the given child priorities if the service they are supposed to come after have been added. + */ + private boolean addPriorities(List<String> priorities, Set<String> childPriorities, String after) { + if (after == null) { + priorities.addAll(childPriorities); + return true; + } + else { + // Check the current priorities, if the "after" priority is there then add these + for (int index = priorities.size() - 1; index >= 0; index--) { + String priority = priorities.get(index); + if (after.equals(priority)) { + priorities.addAll(index + 1, childPriorities); + return true; + } + } + } + return false; + } + + /** + * Add the skipped priorities if the services they are supposed to come after have been added + */ + private void addSkippedPriorities(List<String> priorities, Map<String, Set<String>> skippedPriorites, Set<String> prioritiesJustAdded) { + for (String priority : prioritiesJustAdded) { + if (skippedPriorites.containsKey(priority)) { + Set<String> prioritiesToAdd = skippedPriorites.remove(priority); + addPriorities(priorities, prioritiesToAdd, priority); + addSkippedPriorities(priorities, skippedPriorites, prioritiesToAdd); + } + } + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/cec0b240/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml index 614487e..712241b 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml @@ -157,7 +157,7 @@ </service> </group> - <group name="SERVICE_CHECK" title="All Service Checks" xsi:type="service-check"> + <group name="SERVICE_CHECK_1" title="All Service Checks" xsi:type="service-check"> <skippable>true</skippable> <direction>UPGRADE</direction> <priority> @@ -197,7 +197,7 @@ </batch> </group> - <group name="SERVICE_CHECK" title="All Service Checks" xsi:type="service-check"> + <group name="SERVICE_CHECK_2" title="All Service Checks" xsi:type="service-check"> <skippable>true</skippable> <direction>UPGRADE</direction> <priority> @@ -324,7 +324,7 @@ </service> </group> - <group name="SERVICE_CHECK" title="All Service Checks" xsi:type="service-check"> + <group name="SERVICE_CHECK_3" title="All Service Checks" xsi:type="service-check"> <skippable>true</skippable> <direction>UPGRADE</direction> <priority> http://git-wip-us.apache.org/repos/asf/ambari/blob/cec0b240/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml index 9fb2bba..525a356 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml @@ -154,7 +154,7 @@ </service> </group> - <group name="SERVICE_CHECK" title="All Service Checks" xsi:type="service-check"> + <group name="SERVICE_CHECK_1" title="All Service Checks" xsi:type="service-check"> <skippable>true</skippable> <direction>UPGRADE</direction> <priority> @@ -193,7 +193,7 @@ </batch> </group> - <group name="SERVICE_CHECK" title="All Service Checks" xsi:type="service-check"> + <group name="SERVICE_CHECK_2" title="All Service Checks" xsi:type="service-check"> <skippable>true</skippable> <direction>UPGRADE</direction> <priority> @@ -311,7 +311,7 @@ </service> </group> - <group name="SERVICE_CHECK" title="All Service Checks" xsi:type="service-check"> + <group name="SERVICE_CHECK_3" title="All Service Checks" xsi:type="service-check"> <skippable>true</skippable> <direction>UPGRADE</direction> <priority> http://git-wip-us.apache.org/repos/asf/ambari/blob/cec0b240/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml index 1e040e6..9ee3e88 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml @@ -160,7 +160,7 @@ </service> </group> - <group name="SERVICE_CHECK" title="All Service Checks" xsi:type="service-check"> + <group name="SERVICE_CHECK_1" title="All Service Checks" xsi:type="service-check"> <skippable>true</skippable> <direction>UPGRADE</direction> <priority> @@ -199,7 +199,7 @@ </batch> </group> - <group name="SERVICE_CHECK" title="All Service Checks" xsi:type="service-check"> + <group name="SERVICE_CHECK_2" title="All Service Checks" xsi:type="service-check"> <skippable>true</skippable> <direction>UPGRADE</direction> <priority> @@ -317,7 +317,7 @@ </service> </group> - <group name="SERVICE_CHECK" title="All Service Checks" xsi:type="service-check"> + <group name="SERVICE_CHECK_3" title="All Service Checks" xsi:type="service-check"> <skippable>true</skippable> <direction>UPGRADE</direction> <priority> http://git-wip-us.apache.org/repos/asf/ambari/blob/cec0b240/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml index a9ed58d..58aa95f 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml @@ -155,7 +155,7 @@ </service> </group> - <group name="SERVICE_CHECK" title="All Service Checks" xsi:type="service-check"> + <group name="SERVICE_CHECK_1" title="All Service Checks" xsi:type="service-check"> <skippable>true</skippable> <direction>UPGRADE</direction> <priority> @@ -195,7 +195,7 @@ </batch> </group> - <group name="SERVICE_CHECK" title="All Service Checks" xsi:type="service-check"> + <group name="SERVICE_CHECK_2" title="All Service Checks" xsi:type="service-check"> <skippable>true</skippable> <direction>UPGRADE</direction> <priority> @@ -322,7 +322,7 @@ </service> </group> - <group name="SERVICE_CHECK" title="All Service Checks" xsi:type="service-check"> + <group name="SERVICE_CHECK_3" title="All Service Checks" xsi:type="service-check"> <skippable>true</skippable> <direction>UPGRADE</direction> <priority> http://git-wip-us.apache.org/repos/asf/ambari/blob/cec0b240/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml index 6e27da6..cdb8634 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml @@ -155,7 +155,7 @@ </service> </group> - <group name="SERVICE_CHECK" title="All Service Checks" xsi:type="service-check"> + <group name="SERVICE_CHECK_1" title="All Service Checks" xsi:type="service-check"> <skippable>true</skippable> <direction>UPGRADE</direction> <priority> @@ -194,7 +194,7 @@ </batch> </group> - <group name="SERVICE_CHECK" title="All Service Checks" xsi:type="service-check"> + <group name="SERVICE_CHECK_2" title="All Service Checks" xsi:type="service-check"> <skippable>true</skippable> <direction>UPGRADE</direction> <priority> @@ -312,7 +312,7 @@ </service> </group> - <group name="SERVICE_CHECK" title="All Service Checks" xsi:type="service-check"> + <group name="SERVICE_CHECK_3" title="All Service Checks" xsi:type="service-check"> <skippable>true</skippable> <direction>UPGRADE</direction> <priority> http://git-wip-us.apache.org/repos/asf/ambari/blob/cec0b240/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.5.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.5.xml index 23b4ca7..d90d76d 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.5.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.5.xml @@ -156,7 +156,7 @@ </service> </group> - <group name="SERVICE_CHECK" title="All Service Checks" xsi:type="service-check"> + <group name="SERVICE_CHECK_1" title="All Service Checks" xsi:type="service-check"> <skippable>true</skippable> <direction>UPGRADE</direction> <priority> @@ -196,7 +196,7 @@ </batch> </group> - <group name="SERVICE_CHECK" title="All Service Checks" xsi:type="service-check"> + <group name="SERVICE_CHECK_2" title="All Service Checks" xsi:type="service-check"> <skippable>true</skippable> <direction>UPGRADE</direction> <priority> @@ -332,7 +332,7 @@ </service> </group> - <group name="SERVICE_CHECK" title="All Service Checks" xsi:type="service-check"> + <group name="SERVICE_CHECK_3" title="All Service Checks" xsi:type="service-check"> <skippable>true</skippable> <direction>UPGRADE</direction> <priority> http://git-wip-us.apache.org/repos/asf/ambari/blob/cec0b240/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerMiscTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerMiscTest.java b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerMiscTest.java index dda1e7a..850c716 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerMiscTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerMiscTest.java @@ -117,4 +117,35 @@ public class StackManagerMiscTest { verify(metaInfoDao, stackDao, actionMetadata, osFamily); } + + /** + * This test ensures that service upgrade xml that creates circular dependencies + * will throw an exception. + */ + @Test + public void testCircularDependencyForServiceUpgrade() throws Exception { + MetainfoDAO metaInfoDao = createNiceMock(MetainfoDAO.class); + StackDAO stackDao = createNiceMock(StackDAO.class); + ActionMetadata actionMetadata = createNiceMock(ActionMetadata.class); + OsFamily osFamily = createNiceMock(OsFamily.class); + StackEntity stackEntity = createNiceMock(StackEntity.class); + + expect( + stackDao.find(EasyMock.anyObject(String.class), + EasyMock.anyObject(String.class))).andReturn(stackEntity).atLeastOnce(); + + replay(actionMetadata, stackDao, metaInfoDao, osFamily); + + try { + String upgradeCycle = ClassLoader.getSystemClassLoader().getResource("stacks_with_upgrade_cycle").getPath(); + + StackManager stackManager = new StackManager(new File(upgradeCycle), + null, osFamily, metaInfoDao, actionMetadata, stackDao); + + fail("Expected exception due to cyclic service upgrade xml"); + } catch (AmbariException e) { + // expected + assertEquals("Missing groups: [BAR, FOO]", e.getMessage()); + } + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/cec0b240/ambari-server/src/test/java/org/apache/ambari/server/state/stack/UpgradePackTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/stack/UpgradePackTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/stack/UpgradePackTest.java index 15be8b4..c6701b6 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/stack/UpgradePackTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/state/stack/UpgradePackTest.java @@ -441,6 +441,72 @@ public class UpgradePackTest { assertNull(clusterGroup.parallelScheduler); } + + /** + * Tests that the service level XML merges correctly for 2.0.5/HDFS/HDP/2.2.0. + * + * @throws Exception + */ + @Test + public void testServiceLevelUpgradePackMerge() throws Exception { + Map<String, UpgradePack> upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.2.0"); + assertTrue(upgrades.containsKey("upgrade_test_15388")); + + UpgradePack upgradePack = upgrades.get("upgrade_test_15388"); + + List<String> checks = upgradePack.getPrerequisiteChecks(); + assertEquals(11, checks.size()); + assertTrue(checks.contains("org.apache.ambari.server.checks.FooCheck")); + + List<Grouping> groups = upgradePack.getGroups(Direction.UPGRADE); + assertEquals(8, groups.size()); + Grouping group = groups.get(0); + assertEquals(ClusterGrouping.class, group.getClass()); + ClusterGrouping cluster_group = (ClusterGrouping) group; + assertEquals("Pre {{direction.text.proper}}", cluster_group.title); + + List<ExecuteStage> stages = cluster_group.executionStages; + assertEquals(5, stages.size()); + ExecuteStage stage = stages.get(3); + assertEquals("Backup FOO", stage.title); + + group = groups.get(2); + assertEquals("Core Masters", group.title); + List<UpgradePack.OrderService> services = group.services; + assertEquals(3, services.size()); + UpgradePack.OrderService service = services.get(2); + assertEquals("HBASE", service.serviceName); + + group = groups.get(3); + assertEquals("Core Slaves", group.title); + services = group.services; + assertEquals(3, services.size()); + service = services.get(1); + assertEquals("HBASE", service.serviceName); + + group = groups.get(4); + assertEquals(ServiceCheckGrouping.class, group.getClass()); + ServiceCheckGrouping scGroup = (ServiceCheckGrouping) group; + Set<String> priorityServices = scGroup.getPriorities(); + assertEquals(4, priorityServices.size()); + Iterator serviceIterator = priorityServices.iterator(); + assertEquals("ZOOKEEPER", serviceIterator.next()); + assertEquals("HBASE", serviceIterator.next()); + + group = groups.get(5); + assertEquals("Hive", group.title); + + group = groups.get(6); + assertEquals("Foo", group.title); + services = group.services; + assertEquals(2, services.size()); + service = services.get(1); + assertEquals("FOO2", service.serviceName); + + Map<String, Map<String, ProcessingComponent>> tasks = upgradePack.getTasks(); + assertTrue(tasks.containsKey("HBASE")); + } + private int indexOf(Map<String, ?> map, String keyToFind) { int result = -1; http://git-wip-us.apache.org/repos/asf/ambari/blob/cec0b240/ambari-server/src/test/resources/stacks/HDP/2.0.5/services/HDFS/upgrades/HDP/2.2.0/upgrade_test_15388.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.0.5/services/HDFS/upgrades/HDP/2.2.0/upgrade_test_15388.xml b/ambari-server/src/test/resources/stacks/HDP/2.0.5/services/HDFS/upgrades/HDP/2.2.0/upgrade_test_15388.xml new file mode 100644 index 0000000..fd798da --- /dev/null +++ b/ambari-server/src/test/resources/stacks/HDP/2.0.5/services/HDFS/upgrades/HDP/2.2.0/upgrade_test_15388.xml @@ -0,0 +1,94 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<upgrade xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";> + <target>2.4.*</target> + <target-stack>HDP-2.4.0</target-stack> + <type>ROLLING</type> + <prerequisite-checks> + <check>org.apache.ambari.server.checks.FooCheck</check> + </prerequisite-checks> + <order> + <group xsi:type="cluster" name="PRE_CLUSTER" title="Pre {{direction.text.proper}}"> + <add-after-group-entry>HDFS</add-after-group-entry> + <execute-stage service="FOO" component="BAR" title="Backup FOO"> + <task xsi:type="manual"> + <message>Back FOO up.</message> + </task> + </execute-stage> + </group> + + <group name="SERVICE_CHECK1" title="All Service Checks" xsi:type="service-check"> + <add-after-group-entry>ZOOKEEPER</add-after-group-entry> + <priority> + <service>HBASE</service> + </priority> + </group> + + <group name="FOO" title="Foo"> + <add-after-group>HIVE</add-after-group> + <add-after-group-entry>FOO</add-after-group-entry> + <skippable>true</skippable> + <allow-retry>false</allow-retry> + <service name="FOO2"> + <component>BAR2</component> + </service> + </group> + + <group name="FOO" title="Foo"> + <add-after-group>HIVE</add-after-group> + <skippable>true</skippable> + <allow-retry>false</allow-retry> + <service name="FOO"> + <component>BAR</component> + </service> + </group> + + <group name="CORE_MASTER" title="Core Masters"> + <add-after-group-entry>YARN</add-after-group-entry> + <service name="HBASE"> + <component>HBASE_MASTER</component> + </service> + </group> + + <group name="CORE_SLAVES" title="Core Slaves" xsi:type="colocated"> + <add-after-group-entry>HDFS</add-after-group-entry> + <service name="HBASE"> + <component>REGIONSERVER</component> + </service> + </group> + </order> + + <processing> + <service name="HBASE"> + <component name="HBASE_MASTER"> + <pre-upgrade> + <task xsi:type="configure" id="hdp_2_4_0_0_hbase_remove_local_indexing"/> + </pre-upgrade> + <pre-downgrade /> <!-- no-op to prevent config changes on downgrade --> + <upgrade> + <task xsi:type="restart-task" /> + </upgrade> + </component> + <component name="HBASE_REGIONSERVER"> + <upgrade> + <task xsi:type="restart-task" /> + </upgrade> + </component> + </service> + </processing> +</upgrade> http://git-wip-us.apache.org/repos/asf/ambari/blob/cec0b240/ambari-server/src/test/resources/stacks/HDP/2.2.0/upgrades/upgrade_test_15388.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.2.0/upgrades/upgrade_test_15388.xml b/ambari-server/src/test/resources/stacks/HDP/2.2.0/upgrades/upgrade_test_15388.xml new file mode 100644 index 0000000..4e8e2ea --- /dev/null +++ b/ambari-server/src/test/resources/stacks/HDP/2.2.0/upgrades/upgrade_test_15388.xml @@ -0,0 +1,232 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<upgrade xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";> + <target>2.4.*</target> + <target-stack>HDP-2.4.0</target-stack> + <type>ROLLING</type> + <prerequisite-checks> + <check>org.apache.ambari.server.checks.HiveMultipleMetastoreCheck</check> + <check>org.apache.ambari.server.checks.MapReduce2JobHistoryStatePreservingCheck</check> + <check>org.apache.ambari.server.checks.SecondaryNamenodeDeletedCheck</check> + <check>org.apache.ambari.server.checks.ServicesMapReduceDistributedCacheCheck</check> + <check>org.apache.ambari.server.checks.ServicesNamenodeHighAvailabilityCheck</check> + <check>org.apache.ambari.server.checks.ServicesNamenodeTruncateCheck</check> + <check>org.apache.ambari.server.checks.ServicesTezDistributedCacheCheck</check> + <check>org.apache.ambari.server.checks.ServicesYarnWorkPreservingCheck</check> + <check>org.apache.ambari.server.checks.YarnRMHighAvailabilityCheck</check> + <check>org.apache.ambari.server.checks.YarnTimelineServerStatePreservingCheck</check> + </prerequisite-checks> + <order> + <group xsi:type="cluster" name="PRE_CLUSTER" title="Pre {{direction.text.proper}}"> + <execute-stage title="Confirm 1"> + <task xsi:type="manual"> + <message>Foo</message> + </task> + </execute-stage> + <execute-stage service="HIVE" component="HIVE_SERVER" title="Pre Upgrade HIVE"> + <task xsi:type="manual"> + <message>Back stuff up.</message> + </task> + </execute-stage> + <execute-stage service="HDFS" component="NAMENODE" title="Finalize HDFS"> + <task xsi:type="execute"> + <command>ls</command> + </task> + </execute-stage> + <execute-stage title="Confirm 2"> + <task xsi:type="manual"> + <message>Goo</message> + </task> + </execute-stage> + </group> + + <group name="ZOOKEEPER" title="Zookeeper"> + <skippable>true</skippable> + <allow-retry>false</allow-retry> + <service name="ZOOKEEPER"> + <component>ZOOKEEPER_SERVER</component> + <component>ZOOKEEPER_CLIENT</component> + </service> + </group> + + <group name="CORE_MASTER" title="Core Masters"> + <service name="HDFS"> + <component>JOURNALNODE</component> + <component>NAMENODE</component> + </service> + <service name="YARN"> + <component>RESOURCEMANAGER</component> + </service> + </group> + + <group name="CORE_SLAVES" title="Core Slaves" xsi:type="colocated"> + <skippable>true</skippable> <!-- set skippable for test --> + <allow-retry>false</allow-retry> <!-- set no retry for test --> + <service name="HDFS"> + <component>DATANODE</component> + </service> + <service name="YARN"> + <component>NODEMANAGER</component> + </service> + + <batch> + <percent>20</percent> + <message>Please run additional tests on {{components}}</message> + </batch> + </group> + + <group name="SERVICE_CHECK1" title="All Service Checks" xsi:type="service-check"> + <skippable>true</skippable> + <direction>UPGRADE</direction> + <priority> + <service>ZOOKEEPER</service> + <service>HDFS</service> + <service>YARN</service> + </priority> + <exclude> + <service>AMBARI_METRICS</service> + <service>LOGSEARCH</service> + </exclude> + </group> + + <group name="HIVE" title="Hive"> + <skippable>true</skippable> + <service name="HIVE"> + <component>HIVE_METASTORE</component> + <component>HIVE_SERVER</component> + <component>WEBHCAT_SERVER</component> + </service> + </group> + + <group xsi:type="cluster" name="POST_CLUSTER" title="Finalize {{direction.text.proper}}"> + <execute-stage title="Confirm Finalize"> + <task xsi:type="manual"> + <message>Please confirm you are ready to finalize</message> + </task> + </execute-stage> + <execute-stage title="Update remaining HDP stack to {{version}}"> + <task xsi:type="execute"> + <script>scripts/ru_set_all.py</script> + <function>actionexecute</function> + </task> + </execute-stage> + <execute-stage title="Save Cluster State" service="" component=""> + <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.FinalizeUpgradeAction"> + </task> + </execute-stage> + </group> + + </order> + + + <processing> + <service name="ZOOKEEPER"> + <component name="ZOOKEEPER_SERVER"> + <pre-upgrade> + <task xsi:type="manual"> + <summary>SUMMARY OF PREPARE</summary> + <message>This is a manual task with a placeholder of {{foo/bar}}</message> + </task> + </pre-upgrade> + <upgrade> + <task xsi:type="restart-task" /> + </upgrade> + <post-upgrade> + <task xsi:type="configure" /> + </post-upgrade> + </component> + </service> + + <service name="HDFS"> + <component name="NAMENODE"> + <pre-upgrade> + <task xsi:type="execute" hosts="master"> + <command>su - {hdfs-user} -c 'dosomething'</command> + </task> + <task xsi:type="configure"> + <type>hdfs-site</type> + <set key="myproperty" value="mynewvalue"/> + </task> + <task xsi:type="manual"> + <message>{{direction.verb.proper}} your database</message> + </task> + </pre-upgrade> + <upgrade> + <task xsi:type="restart-task" /> + </upgrade> + <post-upgrade> + <task xsi:type="execute"> + <command>ls</command> + </task> + </post-upgrade> + </component> + <component name="DATANODE"> + <pre-downgrade /> + <upgrade> + <task xsi:type="restart-task" /> + </upgrade> + <post-downgrade> + <task xsi:type="manual"> + <message>Manual Downgrade</message> + </task> + </post-downgrade> + </component> + </service> + + <service name="YARN"> + <component name="RESOURCEMANAGER"> + <pre-upgrade> + <task xsi:type="execute"> + <command>ls</command> + </task> + </pre-upgrade> + </component> + <component name="NODEMANAGER"> + <pre-upgrade> + <task xsi:type="execute"> + <command>ls</command> + </task> + </pre-upgrade> + </component> + </service> + + <service name="HIVE"> + <component name="HIVE_SERVER"> + <pre-upgrade> + <task xsi:type="manual"> + <summary>HiveServer Port Availability</summary> + <message>The HiveServer port will now change to 10010 if hive is using a binary transfer mode or 10011 if hive is using an http transport mode. You can use "netstat -anp | grep 1001[01]" to determine if the port is available on each of following HiveServer host(s): {{hosts.all}}. If the port is not available, the process using it must be terminated.</message> + </task> + + <task xsi:type="configure"> + <condition type="hive-site" key="hive.server2.transport.mode" value="binary"> + <type>hive-site</type> + <key>hive.server2.thrift.port</key> + <value>10010</value> + </condition> + <condition type="hive-site" key="hive.server2.transport.mode" value="http"> + <type>hive-site</type> + <key>hive.server2.http.port</key> + <value>10011</value> + </condition> + </task> + </pre-upgrade> + </component> + </service> + </processing> +</upgrade> http://git-wip-us.apache.org/repos/asf/ambari/blob/cec0b240/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/metainfo.xml b/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/metainfo.xml new file mode 100644 index 0000000..5ffea3a --- /dev/null +++ b/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/metainfo.xml @@ -0,0 +1,22 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<metainfo> + <versions> + <active>true</active> + </versions> +</metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/cec0b240/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/repos/hdp.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/repos/hdp.json b/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/repos/hdp.json new file mode 100644 index 0000000..f80fc0a --- /dev/null +++ b/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/repos/hdp.json @@ -0,0 +1,15 @@ +{ + "HDP-2.2.0": { + "latest": { + "centos6": "http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos6/2.x/BUILDS/2.2.0.0-123";, + "redhat6": "http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos6/2.x/BUILDS/2.2.2.0-123";, + "oraclelinux6": "http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos6/2.x/BUILDS/2.2.0.0-123";, + "suse11": "http://s3.amazonaws.com/dev.hortonworks.com/HDP/suse11/2.x/BUILDS/2.2.0.0-123/hdp.repo"; + }, + "manifests": { + "2.2.1.0": { + "centos6": "./version-2.2.0.4-123.xml" + } + } + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/cec0b240/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/repos/repoinfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/repos/repoinfo.xml b/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/repos/repoinfo.xml new file mode 100644 index 0000000..207d258 --- /dev/null +++ b/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/repos/repoinfo.xml @@ -0,0 +1,34 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<reposinfo> + <latest>./hdp.json</latest> + <os family="redhat6"> + <repo> + <baseurl>http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.2.0.0</baseurl> + <repoid>HDP-2.2.0</repoid> + <reponame>HDP</reponame> + </repo> + </os> + <os family="suse11"> + <repo> + <baseurl>http://public-repo-1.hortonworks.com/HDP/suse11/2.x/updates/2.2.0.0</baseurl> + <repoid>HDP-2.2.0</repoid> + <reponame>HDP</reponame> + </repo> + </os> +</reposinfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/cec0b240/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/repos/version-2.2.0.4-123.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/repos/version-2.2.0.4-123.xml b/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/repos/version-2.2.0.4-123.xml new file mode 100644 index 0000000..0b027ee --- /dev/null +++ b/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/repos/version-2.2.0.4-123.xml @@ -0,0 +1,49 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<repository-version xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:noNamespaceSchemaLocation="version_definition.xsd"> + <release> + <type>STANDARD</type> + <stack-id>HDP-2.2.0</stack-id> + <version>2.2.0.4</version> + <build>123</build> + <release-notes>http://example.com</release-notes> + <display>HDP-2.2.0.4-1234</display> + <compatible-with>2.2.0.0</compatible-with> + </release> + <manifest> + <service id="HDFS-271" name="HDFS" version="2.7.1.2.4"/> + <service id="HBASE-132" name="HBASE" version="1.3.2.4.3"/> + </manifest> + <available-services> + <service idref="HDFS-271"/> + </available-services> + <repository-info> + <os family="redhat6"> + <repo> + <baseurl>http://baseurl1</baseurl> + <repoid>HDP-2.4</repoid> + <reponame>HDP</reponame> + </repo> + <repo> + <baseurl>http://baseurl2</baseurl> + <repoid>HDP-UTILS-1.1.0.20</repoid> + <reponame>HDP-UTILS</reponame> + </repo> + </os> + </repository-info> +</repository-version> http://git-wip-us.apache.org/repos/asf/ambari/blob/cec0b240/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/role_command_order.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/role_command_order.json b/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/role_command_order.json new file mode 100644 index 0000000..ebac7cf --- /dev/null +++ b/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/role_command_order.json @@ -0,0 +1,80 @@ +{ + "_comment" : "Record format:", + "_comment" : "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]", + "general_deps" : { + "_comment" : "dependencies for all cases", + "HBASE_MASTER-START": ["ZOOKEEPER_SERVER-START"], + "OOZIE_SERVER-START": ["JOBTRACKER-START", "TASKTRACKER-START"], + "WEBHCAT_SERVER-START": ["TASKTRACKER-START", "HIVE_SERVER-START"], + "HIVE_METASTORE-START": ["MYSQL_SERVER-START"], + "HIVE_SERVER-START": ["TASKTRACKER-START", "MYSQL_SERVER-START"], + "HUE_SERVER-START": ["HIVE_SERVER-START", "HCAT-START", "OOZIE_SERVER-START"], + "FLUME_HANDLER-START": ["OOZIE_SERVER-START"], + "MAPREDUCE_SERVICE_CHECK-SERVICE_CHECK": ["JOBTRACKER-START", "TASKTRACKER-START"], + "OOZIE_SERVICE_CHECK-SERVICE_CHECK": ["OOZIE_SERVER-START"], + "WEBHCAT_SERVICE_CHECK-SERVICE_CHECK": ["WEBHCAT_SERVER-START"], + "HBASE_SERVICE_CHECK-SERVICE_CHECK": ["HBASE_MASTER-START", "HBASE_REGIONSERVER-START"], + "HIVE_SERVICE_CHECK-SERVICE_CHECK": ["HIVE_SERVER-START", "HIVE_METASTORE-START"], + "HCAT_SERVICE_CHECK-SERVICE_CHECK": ["HIVE_SERVER-START"], + "PIG_SERVICE_CHECK-SERVICE_CHECK": ["JOBTRACKER-START", "TASKTRACKER-START"], + "SQOOP_SERVICE_CHECK-SERVICE_CHECK": ["JOBTRACKER-START", "TASKTRACKER-START"], + "ZOOKEEPER_SERVICE_CHECK-SERVICE_CHECK": ["ZOOKEEPER_SERVER-START"], + "ZOOKEEPER_QUORUM_SERVICE_CHECK-SERVICE_CHECK": ["ZOOKEEPER_SERVER-START"], + "ZOOKEEPER_SERVER-STOP" : ["HBASE_MASTER-STOP", "HBASE_REGIONSERVER-STOP"], + "HBASE_MASTER-STOP": ["HBASE_REGIONSERVER-STOP"], + "TASKTRACKER-UPGRADE": ["JOBTRACKER-UPGRADE"], + "MAPREDUCE_CLIENT-UPGRADE": ["TASKTRACKER-UPGRADE", "JOBTRACKER-UPGRADE"], + "ZOOKEEPER_SERVER-UPGRADE": ["MAPREDUCE_CLIENT-UPGRADE"], + "ZOOKEEPER_CLIENT-UPGRADE": ["ZOOKEEPER_SERVER-UPGRADE"], + "HBASE_MASTER-UPGRADE": ["ZOOKEEPER_CLIENT-UPGRADE"], + "HBASE_REGIONSERVER-UPGRADE": ["HBASE_MASTER-UPGRADE"], + "HBASE_CLIENT-UPGRADE": ["HBASE_REGIONSERVER-UPGRADE"], + "HIVE_SERVER-UPGRADE" : ["HBASE_CLIENT-UPGRADE"], + "HIVE_METASTORE-UPGRADE" : ["HIVE_SERVER-UPGRADE"], + "MYSQL_SERVER-UPGRADE": ["HIVE_METASTORE-UPGRADE"], + "HIVE_CLIENT-UPGRADE": ["MYSQL_SERVER-UPGRADE"], + "HCAT-UPGRADE": ["HIVE_CLIENT-UPGRADE"], + "OOZIE_SERVER-UPGRADE" : ["HCAT-UPGRADE"], + "OOZIE_CLIENT-UPGRADE" : ["OOZIE_SERVER-UPGRADE"], + "WEBHCAT_SERVER-UPGRADE" : ["OOZIE_CLIENT-UPGRADE"], + "PIG-UPGRADE" : ["WEBHCAT_SERVER-UPGRADE"], + "SQOOP-UPGRADE" : ["PIG-UPGRADE"], + "GANGLIA_SERVER-UPGRADE" : ["SQOOP-UPGRADE"], + "GANGLIA_MONITOR-UPGRADE" : ["GANGLIA_SERVER-UPGRADE"] + }, + "_comment" : "GLUSTERFS-specific dependencies", + "optional_glusterfs": { + "HBASE_MASTER-START": ["PEERSTATUS-START"], + "JOBTRACKER-START": ["PEERSTATUS-START"], + "TASKTRACKER-START": ["PEERSTATUS-START"], + "GLUSTERFS_SERVICE_CHECK-SERVICE_CHECK": ["PEERSTATUS-START"], + "JOBTRACKER-UPGRADE": ["GLUSTERFS_CLIENT-UPGRADE"] + }, + "_comment" : "Dependencies that are used when GLUSTERFS is not present in cluster", + "optional_no_glusterfs": { + "SECONDARY_NAMENODE-START": ["NAMENODE-START"], + "RESOURCEMANAGER-START": ["NAMENODE-START", "DATANODE-START"], + "NODEMANAGER-START": ["NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START"], + "HISTORYSERVER-START": ["NAMENODE-START", "DATANODE-START"], + "HBASE_MASTER-START": ["NAMENODE-START", "DATANODE-START"], + "JOBTRACKER-START": ["NAMENODE-START", "DATANODE-START"], + "TASKTRACKER-START": ["NAMENODE-START", "DATANODE-START"], + "HIVE_SERVER-START": ["DATANODE-START"], + "WEBHCAT_SERVER-START": ["DATANODE-START"], + "HDFS_SERVICE_CHECK-SERVICE_CHECK": ["NAMENODE-START", "DATANODE-START", + "SECONDARY_NAMENODE-START"], + "MAPREDUCE2_SERVICE_CHECK-SERVICE_CHECK": ["NODEMANAGER-START", + "RESOURCEMANAGER-START", "HISTORYSERVER-START", "YARN_SERVICE_CHECK-SERVICE_CHECK"], + "YARN_SERVICE_CHECK-SERVICE_CHECK": ["NODEMANAGER-START", "RESOURCEMANAGER-START"], + "RESOURCEMANAGER_SERVICE_CHECK-SERVICE_CHECK": ["RESOURCEMANAGER-START"], + "PIG_SERVICE_CHECK-SERVICE_CHECK": ["RESOURCEMANAGER-START", "NODEMANAGER-START"], + "NAMENODE-STOP": ["JOBTRACKER-STOP", "TASKTRACKER-STOP", "RESOURCEMANAGER-STOP", + "NODEMANAGER-STOP", "HISTORYSERVER-STOP", "HBASE_MASTER-STOP"], + "DATANODE-STOP": ["JOBTRACKER-STOP", "TASKTRACKER-STOP", "RESOURCEMANAGER-STOP", + "NODEMANAGER-STOP", "HISTORYSERVER-STOP", "HBASE_MASTER-STOP"], + "SECONDARY_NAMENODE-UPGRADE": ["NAMENODE-UPGRADE"], + "DATANODE-UPGRADE": ["SECONDARY_NAMENODE-UPGRADE"], + "HDFS_CLIENT-UPGRADE": ["DATANODE-UPGRADE"], + "JOBTRACKER-UPGRADE": ["HDFS_CLIENT-UPGRADE"] + } +}
