AMBARI-19596. ZKFC version shows incorrect in DB tables upon downgrade (ncole)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b2ba7ddc Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b2ba7ddc Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b2ba7ddc Branch: refs/heads/branch-dev-patch-upgrade Commit: b2ba7ddc8fe7380c49a39285f68918492cacf6f1 Parents: 4d44269 Author: Nate Cole <[email protected]> Authored: Tue Jan 17 14:15:08 2017 -0500 Committer: Nate Cole <[email protected]> Committed: Tue Jan 17 16:53:39 2017 -0500 ---------------------------------------------------------------------- .../internal/UpgradeResourceProvider.java | 11 +++- .../listeners/upgrade/StackVersionListener.java | 2 +- .../ambari/server/state/UpgradeHelper.java | 44 +++++++++++-- .../internal/UpgradeResourceProviderTest.java | 69 +++++++++++++++++++- .../HDP/2.1.1/services/STORM/metainfo.xml | 1 + .../HDP/2.2.0/services/STORM/metainfo.xml | 37 +++++++++++ 6 files changed, 153 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/b2ba7ddc/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java index 6f8ebb7..2ec43cf 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java @@ -37,6 +37,8 @@ import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.ambari.annotations.Experimental; +import org.apache.ambari.annotations.ExperimentalFeature; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.Role; import org.apache.ambari.server.RoleCommand; @@ -859,6 +861,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider // TODO: for now, all service components are transitioned to upgrading state // TODO: When performing patch upgrade, we should only target supported services/components // from upgrade pack + @Experimental(feature=ExperimentalFeature.PATCH_UPGRADES) Set<Service> services = new HashSet<>(cluster.getServices().values()); Map<Service, Set<ServiceComponent>> targetComponents = new HashMap<>(); for (Service service: services) { @@ -866,9 +869,11 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider new HashSet<>(service.getServiceComponents().values()); targetComponents.put(service, serviceComponents); } - // TODO: is there any extreme case when we need to set component upgrade state back to NONE - // from IN_PROGRESS (e.g. canceled downgrade) - s_upgradeHelper.putComponentsToUpgradingState(version, targetComponents); + + // !!! determine which stack to check for component isAdvertised + StackId componentStack = upgradeContext.getDirection() == Direction.UPGRADE ? + upgradeContext.getTargetStackId() : upgradeContext.getOriginalStackId(); + s_upgradeHelper.putComponentsToUpgradingState(version, targetComponents, componentStack); for (UpgradeGroupHolder group : groups) { boolean skippable = group.skippable; http://git-wip-us.apache.org/repos/asf/ambari/blob/b2ba7ddc/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListener.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListener.java b/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListener.java index f5a5b0c..3179bd4 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListener.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListener.java @@ -58,7 +58,7 @@ public class StackVersionListener { * Logger. */ private final static Logger LOG = LoggerFactory.getLogger(StackVersionListener.class); - private static final String UNKNOWN_VERSION = State.UNKNOWN.toString(); + public static final String UNKNOWN_VERSION = State.UNKNOWN.toString(); /** * Used to prevent multiple threads from trying to create host alerts http://git-wip-us.apache.org/repos/asf/ambari/blob/b2ba7ddc/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java index 52bf428..ee5451f 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java @@ -45,6 +45,7 @@ import org.apache.ambari.server.controller.spi.UnsupportedPropertyException; import org.apache.ambari.server.controller.utilities.ClusterControllerHelper; import org.apache.ambari.server.controller.utilities.PredicateBuilder; import org.apache.ambari.server.controller.utilities.PropertyHelper; +import org.apache.ambari.server.events.listeners.upgrade.StackVersionListener; import org.apache.ambari.server.orm.dao.RepositoryVersionDAO; import org.apache.ambari.server.orm.entities.RepositoryVersionEntity; import org.apache.ambari.server.stack.HostsType; @@ -773,19 +774,50 @@ public class UpgradeHelper { * desired version (like 2.2.1.0-1234) for upgrade * @param targetServices * targets for upgrade + * @param targetStack + * the target stack for the components. Express and Rolling upgrades determine + * the "correct" stack differently, so the component's desired stack id is not + * a reliable indicator. */ @Transactional public void putComponentsToUpgradingState(String version, - Map<Service, Set<ServiceComponent>> targetServices) throws AmbariException { - // TODO: generalize method? + Map<Service, Set<ServiceComponent>> targetServices, StackId targetStack) throws AmbariException { + for (Map.Entry<Service, Set<ServiceComponent>> entry: targetServices.entrySet()) { for (ServiceComponent serviceComponent: entry.getValue()) { - if (serviceComponent.isVersionAdvertised()) { - for (ServiceComponentHost serviceComponentHost: serviceComponent.getServiceComponentHosts().values()) { - serviceComponentHost.setUpgradeState(UpgradeState.IN_PROGRESS); + + boolean versionAdvertised = false; + try { + ComponentInfo ci = m_ambariMetaInfo.get().getComponent(targetStack.getStackName(), + targetStack.getStackVersion(), serviceComponent.getServiceName(), + serviceComponent.getName()); + + versionAdvertised = ci.isVersionAdvertised(); + } catch (AmbariException e) { + LOG.warn("Component {}/{} doesn't exist for stack {}. Setting version to {}", + serviceComponent.getServiceName(), serviceComponent.getName(), targetStack, + StackVersionListener.UNKNOWN_VERSION); + } + + UpgradeState upgradeState = UpgradeState.IN_PROGRESS; + String desiredVersion = version; + + if (!versionAdvertised) { + upgradeState = UpgradeState.NONE; + desiredVersion = StackVersionListener.UNKNOWN_VERSION; + } + + for (ServiceComponentHost serviceComponentHost: serviceComponent.getServiceComponentHosts().values()) { + serviceComponentHost.setUpgradeState(upgradeState); + + // !!! if we aren't version advertised, but there IS a version, set it. + if (!versionAdvertised && + !serviceComponentHost.getVersion().equals(StackVersionListener.UNKNOWN_VERSION)) { + serviceComponentHost.setVersion(StackVersionListener.UNKNOWN_VERSION); } - serviceComponent.setDesiredVersion(version); } + serviceComponent.setDesiredVersion(desiredVersion); + } } } http://git-wip-us.apache.org/repos/asf/ambari/blob/b2ba7ddc/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeResourceProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeResourceProviderTest.java index a702e6f..4d95e2b 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeResourceProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeResourceProviderTest.java @@ -96,6 +96,8 @@ import org.apache.ambari.server.state.Service; import org.apache.ambari.server.state.ServiceComponent; import org.apache.ambari.server.state.ServiceComponentHost; import org.apache.ambari.server.state.StackId; +import org.apache.ambari.server.state.UpgradeHelper; +import org.apache.ambari.server.state.UpgradeState; import org.apache.ambari.server.state.stack.UpgradePack; import org.apache.ambari.server.state.stack.upgrade.Direction; import org.apache.ambari.server.state.stack.upgrade.UpgradeType; @@ -1464,7 +1466,6 @@ public class UpgradeResourceProviderTest { */ @Test() public void testCreateHostOrderedUpgradeThrowsExceptions() throws Exception { - Cluster cluster = clusters.getCluster("c1"); Map<String, Object> requestProps = new HashMap<String, Object>(); requestProps.put(UpgradeResourceProvider.UPGRADE_CLUSTER_NAME, "c1"); @@ -1509,6 +1510,72 @@ public class UpgradeResourceProviderTest { upgradeResourceProvider.createResources(request); } + /** + * Exercises that a component that goes from upgrade->downgrade that switches + * {@code versionAdvertised} between will go to UKNOWN. This exercises + * {@link UpgradeHelper#putComponentsToUpgradingState(String, Map, StackId)} + * @throws Exception + */ + @Test + public void testCreateUpgradeDowngradeCycleAdvertisingVersion() throws Exception { + Cluster cluster = clusters.getCluster("c1"); + Service service = cluster.addService("STORM"); + service.setDesiredStackVersion(cluster.getDesiredStackVersion()); + + ServiceComponent component = service.addServiceComponent("DRPC_SERVER"); + ServiceComponentHost sch = component.addServiceComponentHost("h1"); + sch.setVersion("2.1.1.0"); + + ResourceProvider upgradeResourceProvider = createProvider(amc); + + Map<String, Object> requestProps = new HashMap<>(); + requestProps.put(UpgradeResourceProvider.UPGRADE_CLUSTER_NAME, "c1"); + requestProps.put(UpgradeResourceProvider.UPGRADE_VERSION, "2.2.0.0"); + requestProps.put(UpgradeResourceProvider.UPGRADE_PACK, "upgrade_test"); + requestProps.put(UpgradeResourceProvider.UPGRADE_SKIP_PREREQUISITE_CHECKS, "true"); + requestProps.put(UpgradeResourceProvider.UPGRADE_FROM_VERSION, "2.1.1.0"); + requestProps.put(UpgradeResourceProvider.UPGRADE_DIRECTION, Direction.UPGRADE.name()); + + Map<String, String> requestInfoProperties = new HashMap<>(); + + Request request = PropertyHelper.getCreateRequest(Collections.singleton(requestProps), requestInfoProperties); + + RequestStatus status = upgradeResourceProvider.createResources(request); + assertEquals(1, status.getAssociatedResources().size()); + + Resource r = status.getAssociatedResources().iterator().next(); + String id = r.getPropertyValue("Upgrade/request_id").toString(); + + component = service.getServiceComponent("DRPC_SERVER"); + assertNotNull(component); + assertEquals("2.2.0.0", component.getDesiredVersion()); + + ServiceComponentHost hostComponent = component.getServiceComponentHost("h1"); + assertEquals(UpgradeState.IN_PROGRESS, hostComponent.getUpgradeState()); + + // !!! can't start a downgrade until cancelling the previous upgrade + abortUpgrade(Long.parseLong(id)); + + requestProps.clear(); + requestProps.put(UpgradeResourceProvider.UPGRADE_CLUSTER_NAME, "c1"); + requestProps.put(UpgradeResourceProvider.UPGRADE_VERSION, "2.1.1.0"); + requestProps.put(UpgradeResourceProvider.UPGRADE_PACK, "upgrade_test"); + requestProps.put(UpgradeResourceProvider.UPGRADE_SKIP_PREREQUISITE_CHECKS, "true"); + requestProps.put(UpgradeResourceProvider.UPGRADE_FROM_VERSION, "2.2.0.0"); + requestProps.put(UpgradeResourceProvider.UPGRADE_DIRECTION, Direction.DOWNGRADE.name()); + + request = PropertyHelper.getCreateRequest(Collections.singleton(requestProps), requestInfoProperties); + status = upgradeResourceProvider.createResources(request); + + component = service.getServiceComponent("DRPC_SERVER"); + assertNotNull(component); + assertEquals("UNKNOWN", component.getDesiredVersion()); + + hostComponent = component.getServiceComponentHost("h1"); + assertEquals(UpgradeState.NONE, hostComponent.getUpgradeState()); + assertEquals("UNKNOWN", hostComponent.getVersion()); + } + private String parseSingleMessage(String msgStr){ JsonParser parser = new JsonParser(); http://git-wip-us.apache.org/repos/asf/ambari/blob/b2ba7ddc/ambari-server/src/test/resources/stacks/HDP/2.1.1/services/STORM/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.1.1/services/STORM/metainfo.xml b/ambari-server/src/test/resources/stacks/HDP/2.1.1/services/STORM/metainfo.xml index 79a3130..d63aa1b 100644 --- a/ambari-server/src/test/resources/stacks/HDP/2.1.1/services/STORM/metainfo.xml +++ b/ambari-server/src/test/resources/stacks/HDP/2.1.1/services/STORM/metainfo.xml @@ -72,6 +72,7 @@ <name>DRPC_SERVER</name> <displayName>DRPC Server</displayName> <category>MASTER</category> + <versionAdvertised>false</versionAdvertised> <commandScript> <script>scripts/drpc_server.py</script> <scriptType>PYTHON</scriptType> http://git-wip-us.apache.org/repos/asf/ambari/blob/b2ba7ddc/ambari-server/src/test/resources/stacks/HDP/2.2.0/services/STORM/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.2.0/services/STORM/metainfo.xml b/ambari-server/src/test/resources/stacks/HDP/2.2.0/services/STORM/metainfo.xml new file mode 100644 index 0000000..6075ba3 --- /dev/null +++ b/ambari-server/src/test/resources/stacks/HDP/2.2.0/services/STORM/metainfo.xml @@ -0,0 +1,37 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<metainfo> + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>STORM</name> + <comment>Apache Hadoop Stream processing framework</comment> + <version>0.9.0.1</version> + <components> + + <component> + <name>DRPC_SERVER</name> + <displayName>DRPC Server</displayName> + <category>MASTER</category> + <versionAdvertised>true</versionAdvertised> + </component> + </components> + </service> + </services> +</metainfo> \ No newline at end of file
