Repository: ambari Updated Branches: refs/heads/branch-dev-patch-upgrade efcfced54 -> 1f8b527f9
AMBARI-18937. Allow generic Service Upgrade Packs for all targets (ncole) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7d07b71d Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7d07b71d Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7d07b71d Branch: refs/heads/branch-dev-patch-upgrade Commit: 7d07b71de7019efaaa04fe8c09548cdc604cea60 Parents: ef91365 Author: Nate Cole <[email protected]> Authored: Mon Nov 21 15:24:59 2016 -0500 Committer: Nate Cole <[email protected]> Committed: Mon Nov 21 16:11:21 2016 -0500 ---------------------------------------------------------------------- .../apache/ambari/server/stack/StackModule.java | 101 +++++++++++++++++-- .../ambari/server/state/stack/UpgradePack.java | 12 ++- .../stacks/HDP/2.3/upgrades/upgrade-2.5.xml | 4 +- .../stacks/HDP/2.3/upgrades/upgrade-2.6.xml | 4 +- .../stacks/HDP/2.4/upgrades/upgrade-2.5.xml | 4 +- .../stacks/HDP/2.4/upgrades/upgrade-2.6.xml | 4 +- .../src/main/resources/upgrade-pack.xsd | 10 +- .../server/state/stack/UpgradePackTest.java | 22 ++++ .../GANGLIA/upgrades/HDP/rolling-upgrade.xml | 27 +++++ 9 files changed, 168 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/7d07b71d/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java index d9eaf27..ce606de 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java @@ -46,16 +46,19 @@ import org.apache.ambari.server.state.stack.ServiceMetainfoXml; import org.apache.ambari.server.state.stack.StackMetainfoXml; import org.apache.ambari.server.state.stack.UpgradePack; import org.apache.ambari.server.state.stack.upgrade.Grouping; +import org.apache.ambari.server.state.stack.upgrade.UpgradeType; +import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.base.Function; -import com.google.common.collect.ListMultimap; import com.google.common.collect.ImmutableListMultimap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ListMultimap; import com.google.common.collect.Lists; import com.google.common.collect.Multimaps; + /** * Stack module which provides all functionality related to parsing and fully * resolving stacks from the stack definition. @@ -834,6 +837,7 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V if (configPack == null) { return; } + for (ServiceModule module : serviceModules.values()) { File upgradesFolder = module.getModuleInfo().getServiceUpgradesFolder(); if (upgradesFolder != null) { @@ -869,12 +873,17 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V private UpgradePack getServiceUpgradePack(UpgradePack pack, File upgradesFolder) throws AmbariException { File stackFolder = new File(upgradesFolder, stackInfo.getName()); File versionFolder = new File(stackFolder, stackInfo.getVersion()); + // !!! relies on the service upgrade pack filename being named the exact same File servicePackFile = new File(versionFolder, pack.getName() + ".xml"); + LOG.info("Service folder: " + servicePackFile.getAbsolutePath()); - if (!servicePackFile.exists()) { - return null; + if (servicePackFile.exists()) { + return parseServiceUpgradePack(pack, servicePackFile); + } else { + UpgradePack child = findServiceUpgradePack(pack, stackFolder); + + return null == child ? null : parseServiceUpgradePack(pack, child); } - return parseServiceUpgradePack(pack, servicePackFile); } /** @@ -882,6 +891,7 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V * for any service which specifies its own upgrade. */ private void mergeUpgradePack(UpgradePack pack, List<UpgradePack> servicePacks) throws AmbariException { + List<Grouping> originalGroups = pack.getAllGroups(); Map<String, List<Grouping>> allGroupMap = new HashMap<>(); for (Grouping group : originalGroups) { @@ -889,8 +899,21 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V list.add(group); allGroupMap.put(group.name, list); } + for (UpgradePack servicePack : servicePacks) { for (Grouping group : servicePack.getAllGroups()) { + + /* + !!! special case where the service pack is targeted for any version. When + a service UP targets to run after another group, check to make sure that the + base UP contains the group. + */ + if (servicePack.isAllTarget() && !allGroupMap.keySet().contains(group.addAfterGroup)) { + LOG.warn("Service Upgrade Pack specified after-group of {}, but that is not found in {}", + group.addAfterGroup, StringUtils.join(allGroupMap.keySet(), ',')); + continue; + } + if (allGroupMap.containsKey(group.name)) { List<Grouping> list = allGroupMap.get(group.name); Grouping first = list.get(0); @@ -906,8 +929,7 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V else { list.add(group); } - } - else { + } else { List<Grouping> list = new ArrayList<>(); list.add(group); allGroupMap.put(group.name, list); @@ -934,15 +956,17 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V */ private void orderGroups(List<Grouping> groups, Map<String, Grouping> mergedGroupMap) throws AmbariException { Map<String, List<Grouping>> skippedGroups = new HashMap<>(); + for (Map.Entry<String, Grouping> entry : mergedGroupMap.entrySet()) { - String key = entry.getKey(); Grouping group = entry.getValue(); + if (!groups.contains(group)) { boolean added = addGrouping(groups, group); if (added) { addSkippedGroup(groups, skippedGroups, group); } else { List<Grouping> tmp = null; + // store the group until later if (skippedGroups.containsKey(group.addAfterGroup)) { tmp = skippedGroups.get(group.addAfterGroup); @@ -954,6 +978,7 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V } } } + if (!skippedGroups.isEmpty()) { throw new AmbariException("Missing groups: " + skippedGroups.keySet()); } @@ -999,6 +1024,50 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V } /** + * Finds an upgrade pack that: + * <ul> + * <li>Is found in the $SERVICENAME/upgrades/$STACKNAME folder</li> + * <li>Matches the same {@link UpgradeType#getType()}as the {@code base} upgrade pack</li> + * <li>Has the {@link UpgradePack#getTarget()} value equals to "*"</li> + * <li>Has the {@link UpgradePack#getTargetStack()} value equals to "*"</li> + * </ul> + * This method will not attempt to resolve the "most correct" upgrade pack. For this + * feature to work, there should be only one upgrade pack per type. If more specificity + * is required, then follow the convention of $SERVICENAME/upgrades/$STACKNAME/$STACKVERSION/$BASE_FILE_NAME.xml + * + * @param base the base upgrade pack for a stack + * @param upgradeStackDirectory service directory that contains stack upgrade files. + * @return an upgrade pack that matches {@code base} + */ + private UpgradePack findServiceUpgradePack(UpgradePack base, File upgradeStackDirectory) { + if (!upgradeStackDirectory.exists() || !upgradeStackDirectory.isDirectory()) { + return null; + } + + File[] upgradeFiles = upgradeStackDirectory.listFiles(StackDirectory.XML_FILENAME_FILTER); + if (0 == upgradeFiles.length) { + return null; + } + + for (File f : upgradeFiles) { + try { + UpgradePack upgradePack = unmarshaller.unmarshal(UpgradePack.class, f); + + // !!! if the type is the same and the target is "*", then it's good to merge + if (upgradePack.isAllTarget() && upgradePack.getType() == base.getType()) { + return upgradePack; + } + + } catch (Exception e) { + LOG.warn("File {} does not appear to be an upgrade pack and will be skipped ({})", + f.getAbsolutePath(), e.getMessage()); + } + } + + return null; + } + + /** * Parses the service specific upgrade file and merges the none order elements * (prerequisite check and processing sections). */ @@ -1011,12 +1080,24 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V throw new AmbariException("Unable to parse service upgrade file at location: " + serviceFile.getAbsolutePath(), e); } - parent.mergePrerequisiteChecks(pack); - parent.mergeProcessing(pack); + return parseServiceUpgradePack(parent, pack); + } - return pack; + /** + * Places prerequisite checks and processing objects onto the parent upgrade pack. + * + * @param parent the parent upgrade pack + * @param child the parsed child upgrade pack + * @return the child upgrade pack + */ + private UpgradePack parseServiceUpgradePack(UpgradePack parent, UpgradePack child) { + parent.mergePrerequisiteChecks(child); + parent.mergeProcessing(child); + + return child; } + /** * Process repositories associated with the stack. * @throws AmbariException if unable to fully process the stack repositories http://git-wip-us.apache.org/repos/asf/ambari/blob/7d07b71d/ambari-server/src/main/java/org/apache/ambari/server/state/stack/UpgradePack.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/UpgradePack.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/UpgradePack.java index edf5c89..aa97fa3 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/UpgradePack.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/UpgradePack.java @@ -52,6 +52,8 @@ import org.slf4j.LoggerFactory; @XmlAccessorType(XmlAccessType.FIELD) public class UpgradePack { + private static final String ALL_VERSIONS = "*"; + private static Logger LOG = LoggerFactory.getLogger(UpgradePack.class); /** @@ -315,7 +317,6 @@ public class UpgradePack { public boolean canBeApplied(String targetVersion){ // check that upgrade pack can be applied to selected stack // converting 2.2.*.* -> 2\.2(\.\d+)?(\.\d+)?(-\d+)? - String regexPattern = getTarget().replaceAll("\\.", "\\\\."); // . -> \. regexPattern = regexPattern.replaceAll("\\\\\\.\\*", "(\\\\\\.\\\\d+)?"); // \.* -> (\.\d+)? regexPattern = regexPattern.concat("(-\\d+)?"); @@ -453,6 +454,15 @@ public class UpgradePack { } /** + * @return {@code true} if the upgrade targets any version or stack. Both + * {@link #target} and {@link #targetStack} must equal "*" + */ + public boolean isAllTarget() { + return ALL_VERSIONS.equals(target) && ALL_VERSIONS.equals(targetStack); + } + + + /** * A service definition that holds a list of components in the 'order' element. */ public static class OrderService { http://git-wip-us.apache.org/repos/asf/ambari/blob/7d07b71d/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml index a838fc7..e675633 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml @@ -421,7 +421,7 @@ </execute-stage> </group> - <group xsi:type="stop" name="STOP_STORM" title="Stop Storm Services"> + <group xsi:type="stop" name="STOP_STORM_UPGRADE" title="Stop Storm Services"> <direction>UPGRADE</direction> <service-check>false</service-check> <skippable>true</skippable> @@ -443,7 +443,7 @@ </service> </group> - <group xsi:type="stop" name="STOP_STORM" title="Stop Storm Services"> + <group xsi:type="stop" name="STOP_STORM_DOWNGRADE" title="Stop Storm Services"> <direction>DOWNGRADE</direction> <service-check>false</service-check> <skippable>true</skippable> http://git-wip-us.apache.org/repos/asf/ambari/blob/7d07b71d/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml index a01996a..7d4cfc9 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml @@ -422,7 +422,7 @@ </execute-stage> </group> - <group xsi:type="stop" name="STOP_STORM" title="Stop Storm Services"> + <group xsi:type="stop" name="STOP_STORM_UPGRADE" title="Stop Storm Services"> <direction>UPGRADE</direction> <service-check>false</service-check> <skippable>true</skippable> @@ -444,7 +444,7 @@ </service> </group> - <group xsi:type="stop" name="STOP_STORM" title="Stop Storm Services"> + <group xsi:type="stop" name="STOP_STORM_DOWNGRADE" title="Stop Storm Services"> <direction>DOWNGRADE</direction> <service-check>false</service-check> <skippable>true</skippable> http://git-wip-us.apache.org/repos/asf/ambari/blob/7d07b71d/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml index 17be4a8..effdda2 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml @@ -406,7 +406,7 @@ </execute-stage> </group> - <group xsi:type="stop" name="STOP_STORM" title="Stop Storm Services"> + <group xsi:type="stop" name="STOP_STORM_UPGRADE" title="Stop Storm Services"> <direction>UPGRADE</direction> <service-check>false</service-check> <skippable>true</skippable> @@ -428,7 +428,7 @@ </service> </group> - <group xsi:type="stop" name="STOP_STORM" title="Stop Storm Services"> + <group xsi:type="stop" name="STOP_STORM_DOWNGRADE" title="Stop Storm Services"> <direction>DOWNGRADE</direction> <service-check>false</service-check> <skippable>true</skippable> http://git-wip-us.apache.org/repos/asf/ambari/blob/7d07b71d/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml index f520faf..31e72a1 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml @@ -411,7 +411,7 @@ </execute-stage> </group> - <group xsi:type="stop" name="STOP_STORM" title="Stop Storm Services"> + <group xsi:type="stop" name="STOP_STORM_UPGRADE" title="Stop Storm Services"> <direction>UPGRADE</direction> <service-check>false</service-check> <skippable>true</skippable> @@ -433,7 +433,7 @@ </service> </group> - <group xsi:type="stop" name="STOP_STORM" title="Stop Storm Services"> + <group xsi:type="stop" name="STOP_STORM_DOWNGRADE" title="Stop Storm Services"> <direction>DOWNGRADE</direction> <service-check>false</service-check> <skippable>true</skippable> http://git-wip-us.apache.org/repos/asf/ambari/blob/7d07b71d/ambari-server/src/main/resources/upgrade-pack.xsd ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/upgrade-pack.xsd b/ambari-server/src/main/resources/upgrade-pack.xsd index 2871f05..45cf5fa 100644 --- a/ambari-server/src/main/resources/upgrade-pack.xsd +++ b/ambari-server/src/main/resources/upgrade-pack.xsd @@ -392,7 +392,15 @@ <xs:element name="type" type="upgrade-kind-type" /> <xs:element name="prerequisite-checks" type="prerequisite-check-type" minOccurs="0" /> <xs:element name="upgrade-path" type="upgrade-path-type" minOccurs="0" /> - <xs:element name="order" type="order-type" /> + <xs:element name="order" type="order-type"> + <xs:unique name="unique-by-group"> + <xs:annotation> + <xs:documentation>Ensures that the element "processing" does not have duplicate services</xs:documentation> + </xs:annotation> + <xs:selector xpath="group" /> + <xs:field xpath="@name" /> + </xs:unique> + </xs:element> <xs:element name="processing" type="processing-type"> <xs:unique name="unique-by-service"> <xs:annotation> http://git-wip-us.apache.org/repos/asf/ambari/blob/7d07b71d/ambari-server/src/test/java/org/apache/ambari/server/state/stack/UpgradePackTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/stack/UpgradePackTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/stack/UpgradePackTest.java index dde7ffa..8f9f791 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/stack/UpgradePackTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/state/stack/UpgradePackTest.java @@ -18,6 +18,7 @@ package org.apache.ambari.server.state.stack; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; @@ -570,6 +571,27 @@ public class UpgradePackTest { Map<String, Map<String, ProcessingComponent>> tasks = upgradePack.getTasks(); assertTrue(tasks.containsKey("HBASE")); + + // !!! generalized upgrade pack shouldn't be in this + boolean found = false; + for (Grouping grouping : upgradePack.getAllGroups()) { + if (grouping.name.equals("GANGLIA_UPGRADE")) { + found = true; + break; + } + } + assertFalse(found); + + // !!! test merge of a generalized upgrade pack + upgradePack = upgrades.get("upgrade_test_conditions"); + assertNotNull(upgradePack); + for (Grouping grouping : upgradePack.getAllGroups()) { + if (grouping.name.equals("GANGLIA_UPGRADE")) { + found = true; + break; + } + } + assertTrue(found); } http://git-wip-us.apache.org/repos/asf/ambari/blob/7d07b71d/ambari-server/src/test/resources/stacks/HDP/2.2.0/services/GANGLIA/upgrades/HDP/rolling-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.2.0/services/GANGLIA/upgrades/HDP/rolling-upgrade.xml b/ambari-server/src/test/resources/stacks/HDP/2.2.0/services/GANGLIA/upgrades/HDP/rolling-upgrade.xml new file mode 100644 index 0000000..c2224dd --- /dev/null +++ b/ambari-server/src/test/resources/stacks/HDP/2.2.0/services/GANGLIA/upgrades/HDP/rolling-upgrade.xml @@ -0,0 +1,27 @@ +<?xml version="1.0"?> + +<upgrade xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:noNamespaceSchemaLocation="upgrade-pack.xsd"> + <target>*</target> + <target-stack>*</target-stack> + <type>ROLLING</type> + <order> + <group name="GANGLIA_UPGRADE" title="Ganglia"> + <add-after-group>ZOOKEEPER_CONFIG_CONDITION_TEST</add-after-group> + <skippable>true</skippable> + <allow-retry>true</allow-retry> + <service name="GANGLIA"> + <component>HST_SERVER</component> + </service> + </group> + </order> + + <processing> + <service name="GANGLIA"> + <component name="GANGLIA_SERVER"> + <upgrade> + <task xsi:type="restart-task" /> + </upgrade> + </component> + </service> + </processing> +</upgrade> \ No newline at end of file
