AMBARI-21626 - Configs For Target Stack Are Not Created During Upgrade (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/32216f75 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/32216f75 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/32216f75 Branch: refs/heads/branch-2.6 Commit: 32216f7598aa64b1e621da8b05391251986e4364 Parents: 0535ca6 Author: Jonathan Hurley <[email protected]> Authored: Tue Aug 1 20:16:21 2017 -0400 Committer: Jonathan Hurley <[email protected]> Committed: Wed Aug 2 11:01:52 2017 -0400 ---------------------------------------------------------------------- .../upgrades/UpdateDesiredStackAction.java | 11 ++- .../org/apache/ambari/server/state/Cluster.java | 12 ++++ .../ambari/server/state/ConfigHelper.java | 6 ++ .../ambari/server/state/UpgradeHelper.java | 22 +++--- .../server/state/cluster/ClusterImpl.java | 70 ++++++++++++++++++++ .../state/stack/upgrade/ClusterGrouping.java | 42 +++--------- .../ambari/server/state/UpgradeHelperTest.java | 2 +- .../server/state/cluster/ClusterTest.java | 6 +- .../HDP/2.2.0/upgrades/upgrade_test_checks.xml | 23 ++++++- 9 files changed, 149 insertions(+), 45 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/32216f75/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/UpdateDesiredStackAction.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/UpdateDesiredStackAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/UpdateDesiredStackAction.java index ff0becc..40f238c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/UpdateDesiredStackAction.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/UpdateDesiredStackAction.java @@ -37,6 +37,7 @@ import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Clusters; import org.apache.ambari.server.state.RepositoryType; import org.apache.ambari.server.state.RepositoryVersionState; +import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.state.UpgradeContext; import org.apache.ambari.server.state.stack.upgrade.Direction; import org.apache.commons.lang.StringUtils; @@ -142,6 +143,10 @@ public class UpdateDesiredStackAction extends AbstractUpgradeServerAction { } out.append(message).append(System.lineSeparator()); + + // move the cluster's desired stack as well + StackId targetStackId = targetRepositoryVersion.getStackId(); + cluster.setDesiredStackVersion(targetStackId); } if( upgradeContext.getDirection() == Direction.DOWNGRADE ){ @@ -154,7 +159,7 @@ public class UpdateDesiredStackAction extends AbstractUpgradeServerAction { message = String.format(" %s to %s", serviceName, repositoryVersion.getVersion()); out.append(message).append(System.lineSeparator()); - } + } } // move repositories to the right version and create/revert configs @@ -175,6 +180,10 @@ public class UpdateDesiredStackAction extends AbstractUpgradeServerAction { hostVersion.setState(RepositoryVersionState.INSTALLED); } } + + // move the cluster's desired stack back to it's current stack on downgrade + StackId targetStackId = cluster.getCurrentStackVersion(); + cluster.setDesiredStackVersion(targetStackId); } return createCommandReport(0, HostRoleStatus.COMPLETED, "{}", out.toString(), err.toString()); http://git-wip-us.apache.org/repos/asf/ambari/blob/32216f75/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java b/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java index b4f7120..ee18fdf 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java @@ -548,6 +548,18 @@ public interface Cluster { Map<String, Object> getSessionAttributes(); /** + * Makes the most recent configurations for the specified stack current. + * <p/> + * When completed, all other configurations for any other stack will remain, + * but will not be marked as selected. + * + * @param stackId + * the stack to use when finding the latest configurations (not + * {@code null}). + */ + void applyLatestConfigurations(StackId stackId); + + /** * Makes the most recent configurations for the specified stack current. This * will only modify configurations for the given service. * <p/> http://git-wip-us.apache.org/repos/asf/ambari/blob/32216f75/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java index 4b2e0f1..812fb62 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java @@ -1237,6 +1237,9 @@ public class ConfigHelper { if (!defaultPropertiesByType.containsKey(type)) { defaultPropertiesByType.put(type, new HashMap<String, String>()); } + + defaultPropertiesByType.get(type).put(stackDefaultProperty.getName(), + stackDefaultProperty.getValue()); } // for every installed service, populate the default service properties @@ -1250,6 +1253,9 @@ public class ConfigHelper { if (!defaultPropertiesByType.containsKey(type)) { defaultPropertiesByType.put(type, new HashMap<String, String>()); } + + defaultPropertiesByType.get(type).put(serviceDefaultProperty.getName(), + serviceDefaultProperty.getValue()); } return defaultPropertiesByType; http://git-wip-us.apache.org/repos/asf/ambari/blob/32216f75/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java index a5b40ff..0701296 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java @@ -869,6 +869,20 @@ public class UpgradeHelper { Set<String> clusterConfigTypes = new HashSet<>(); Set<String> processedClusterConfigTypes = new HashSet<>(); + // downgrades are easy - either do nothing or simply apply configurations + // from the target stack + if (direction == Direction.DOWNGRADE) { + StackId currentStackId = cluster.getCurrentStackVersion(); + StackId desiredStackId = cluster.getDesiredStackVersion(); + + if (!currentStackId.equals(desiredStackId)) { + cluster.applyLatestConfigurations(currentStackId); + } + + return; + } + + // upgrade is a bit harder - we have to merge new stack configurations in // merge or revert configurations for any service that needs it for (String serviceName : servicesInUpgrade) { RepositoryVersionEntity sourceRepositoryVersion = upgradeContext.getSourceRepositoryVersion(serviceName); @@ -889,14 +903,6 @@ public class UpgradeHelper { ConfigHelper configHelper = m_configHelperProvider.get(); - // downgrade is easy - just remove the new and make the old current - if (direction == Direction.DOWNGRADE) { - cluster.applyLatestConfigurations(targetStackId, serviceName); - return; - } - - // upgrade is a bit harder - we have to merge new stack configurations in - // populate a map of default configurations for the service on the old // stack (this is used when determining if a property has been // customized and should be overriden with the new stack value) http://git-wip-us.apache.org/repos/asf/ambari/blob/32216f75/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java index 06b6217..3fb45ab 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java @@ -2333,6 +2333,76 @@ public class ClusterImpl implements Cluster { */ @Override @Transactional + public void applyLatestConfigurations(StackId stackId) { + clusterGlobalLock.writeLock().lock(); + + try { + // grab all of the configurations and hash them so we can easily update + // them when picking and choosing only those from the service + ClusterEntity clusterEntity = getClusterEntity(); + Collection<ClusterConfigEntity> configEntities = clusterEntity.getClusterConfigEntities(); + ImmutableMap<Object, ClusterConfigEntity> clusterConfigEntityMap = Maps.uniqueIndex( + configEntities, Functions.identity()); + + // disable any configurations which are currently selected + for (ClusterConfigEntity clusterConfig : configEntities) { + if (clusterConfig.isSelected()) { + // un-select the latest configuration for the service + clusterConfig.setSelected(false); + + LOG.debug("Disabling configuration {} with tag {}", clusterConfig.getType(), + clusterConfig.getTag()); + } + } + + // get the latest configurations for the stack so they can be selected + Collection<ClusterConfigEntity> latestConfigMappingByStack = clusterDAO.getLatestConfigurations( + clusterEntity.getClusterId(), stackId); + + Set<String> configTypesUpdated = new HashSet<>(); + + for (ClusterConfigEntity clusterConfig : latestConfigMappingByStack) { + // grab the hash'd entity from the map so we're working with the right + // one + clusterConfig = clusterConfigEntityMap.get(clusterConfig); + + clusterConfig.setSelected(true); + + configTypesUpdated.add(clusterConfig.getType()); + + LOG.info("Setting {} with version tag {} created on {} to selected for stack {}", + clusterConfig.getType(), clusterConfig.getTag(), new Date(clusterConfig.getTimestamp()), + stackId); + } + + // since the entities which were modified came from the cluster entity's + // list to begin with, we can just save them right back - no need for a + // new collection since the entity instances were modified directly + clusterEntity = clusterDAO.merge(clusterEntity); + + cacheConfigurations(); + + LOG.info( + "Applied latest configurations for stack {}. The the following types were modified: {}", + stackId, StringUtils.join(configTypesUpdated, ',')); + + } finally { + clusterGlobalLock.writeLock().unlock(); + } + + // publish an event to instruct entity managers to clear cached instances of + // ClusterEntity immediately - it takes EclipseLink about 1000ms to update + // the L1 caches of other threads and the action scheduler could act upon + // stale data + EntityManagerCacheInvalidationEvent event = new EntityManagerCacheInvalidationEvent(); + jpaEventPublisher.publish(event); + } + + /** + * {@inheritDoc} + */ + @Override + @Transactional public void applyLatestConfigurations(StackId stackId, String serviceName) { clusterGlobalLock.writeLock().lock(); http://git-wip-us.apache.org/repos/asf/ambari/blob/32216f75/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ClusterGrouping.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ClusterGrouping.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ClusterGrouping.java index 5ac7752..3deb7c8 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ClusterGrouping.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ClusterGrouping.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -25,7 +25,6 @@ import java.util.Iterator; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Set; import javax.xml.bind.annotation.XmlAccessType; @@ -47,9 +46,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.base.Objects; -import com.google.gson.JsonArray; -import com.google.gson.JsonObject; -import com.google.gson.JsonPrimitive; /** * Used to represent cluster-based operations. @@ -175,6 +171,13 @@ public class ClusterGrouping extends Grouping { continue; } + // only schedule this stage if its service is part of the upgrade + if (StringUtils.isNotBlank(execution.service)) { + if (!upgradeContext.isServiceSupported(execution.service)) { + continue; + } + } + Task task = execution.task; StageWrapper wrapper = null; @@ -264,6 +267,7 @@ public class ClusterGrouping extends Grouping { return null; } + // !!! FUTURE: check for component HostsType hosts = ctx.getResolver().getMasterAndHosts(service, component); @@ -317,32 +321,6 @@ public class ClusterGrouping extends Grouping { } /** - * Populates the manual task, mt, with information about the list of hosts. - * @param mt Manual Task - * @param hostToComponents Map from host name to list of components - */ - private void fillHostDetails(ManualTask mt, Map<String, List<String>> hostToComponents) { - JsonArray arr = new JsonArray(); - for (Entry<String, List<String>> entry : hostToComponents.entrySet()) { - JsonObject hostObj = new JsonObject(); - hostObj.addProperty("host", entry.getKey()); - - JsonArray componentArr = new JsonArray(); - for (String comp : entry.getValue()) { - componentArr.add(new JsonPrimitive(comp)); - } - hostObj.add("components", componentArr); - - arr.add(hostObj); - } - - JsonObject obj = new JsonObject(); - obj.add("unhealthy", arr); - - mt.structuredOut = obj.toString(); - } - - /** * Attempts to merge the given cluster groupings. This merges the execute stages * in an order specific manner. */ @@ -409,4 +387,4 @@ public class ClusterGrouping extends Grouping { } } } -} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/32216f75/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java index 9b85242..24a3fa2 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java @@ -335,7 +335,7 @@ public class UpgradeHelperTest extends EasyMockSupport { assertEquals("Save Cluster State", postGroup.items.get(1).getText()); assertEquals(StageWrapper.Type.SERVER_SIDE_ACTION, postGroup.items.get(1).getType()); - assertEquals(3, groups.get(0).items.size()); + assertEquals(2, groups.get(0).items.size()); assertEquals(7, groups.get(1).items.size()); assertEquals(2, groups.get(2).items.size()); http://git-wip-us.apache.org/repos/asf/ambari/blob/32216f75/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterTest.java index 3a29298..e8e7206 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterTest.java @@ -2178,6 +2178,9 @@ public class ClusterTest { createDefaultCluster(Sets.newHashSet("host-1"), stackId); Cluster cluster = clusters.getCluster("c1"); + cluster.setCurrentStackVersion(stackId); + cluster.setDesiredStackVersion(stackId); + RepositoryVersionEntity repoVersion220 = helper.getOrCreateRepositoryVersion(newStackId, "2.2.0-1234"); ConfigHelper configHelper = injector.getInstance(ConfigHelper.class); @@ -2201,8 +2204,9 @@ public class ClusterTest { // make v1 "current" cluster.addDesiredConfig("admin", Sets.newHashSet(c1), "note-1"); - // bump the repo version + // bump the repo version and the desired stack service.setDesiredRepositoryVersion(repoVersion220); + cluster.setDesiredStackVersion(newStackId); // save v2 // config for v2 on new stack http://git-wip-us.apache.org/repos/asf/ambari/blob/32216f75/ambari-server/src/test/resources/stacks/HDP/2.2.0/upgrades/upgrade_test_checks.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.2.0/upgrades/upgrade_test_checks.xml b/ambari-server/src/test/resources/stacks/HDP/2.2.0/upgrades/upgrade_test_checks.xml index 4d4d972..ef8ce70 100644 --- a/ambari-server/src/test/resources/stacks/HDP/2.2.0/upgrades/upgrade_test_checks.xml +++ b/ambari-server/src/test/resources/stacks/HDP/2.2.0/upgrades/upgrade_test_checks.xml @@ -78,7 +78,16 @@ </service> <service-check>false</service-check> </group> - + + <group xsi:type="cluster" name="HBASE" title="Update HBase Configuration"> + <skippable>true</skippable> + <execute-stage service="HBASE" component="HBASE_MASTER" title="Update HBase Configuration"> + <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.HBaseEnvMaxDirectMemorySizeAction"> + <summary>Update HBase Env Configuration</summary> + </task> + </execute-stage> + </group> + <group name="SERVICE_CHECK_1" title="Post-Master Service Checks" xsi:type="service-check"> <priority> <service>HDFS</service> @@ -108,6 +117,16 @@ <message>Please run additional tests</message> </batch> </group> + + <!-- This group will be ignored because it is an invalid syntax. It is not a cluster type but contains an execute-stage --> + <group name="HBASE" title="Update HBase Configuration"> + <skippable>true</skippable> + <execute-stage service="HBASE" component="HBASE_MASTER" title="Update HBase Configuration"> + <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.HBaseEnvMaxDirectMemorySizeAction"> + <summary>Update HBase Env Configuration</summary> + </task> + </execute-stage> + </group> <group name="SERVICE_CHECK_2" title="Post-Slave Service Checks" xsi:type="service-check"> <priority> @@ -220,4 +239,4 @@ </component> </service> </processing> -</upgrade> +</upgrade> \ No newline at end of file
