AMBARI-21626 - Configs For Target Stack Are Not Created During Upgrade 
(jonathanhurley)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/32216f75
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/32216f75
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/32216f75

Branch: refs/heads/branch-2.6
Commit: 32216f7598aa64b1e621da8b05391251986e4364
Parents: 0535ca6
Author: Jonathan Hurley <jhur...@hortonworks.com>
Authored: Tue Aug 1 20:16:21 2017 -0400
Committer: Jonathan Hurley <jhur...@hortonworks.com>
Committed: Wed Aug 2 11:01:52 2017 -0400

----------------------------------------------------------------------
 .../upgrades/UpdateDesiredStackAction.java      | 11 ++-
 .../org/apache/ambari/server/state/Cluster.java | 12 ++++
 .../ambari/server/state/ConfigHelper.java       |  6 ++
 .../ambari/server/state/UpgradeHelper.java      | 22 +++---
 .../server/state/cluster/ClusterImpl.java       | 70 ++++++++++++++++++++
 .../state/stack/upgrade/ClusterGrouping.java    | 42 +++---------
 .../ambari/server/state/UpgradeHelperTest.java  |  2 +-
 .../server/state/cluster/ClusterTest.java       |  6 +-
 .../HDP/2.2.0/upgrades/upgrade_test_checks.xml  | 23 ++++++-
 9 files changed, 149 insertions(+), 45 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/32216f75/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/UpdateDesiredStackAction.java
----------------------------------------------------------------------
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/UpdateDesiredStackAction.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/UpdateDesiredStackAction.java
index ff0becc..40f238c 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/UpdateDesiredStackAction.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/UpdateDesiredStackAction.java
@@ -37,6 +37,7 @@ import org.apache.ambari.server.state.Cluster;
 import org.apache.ambari.server.state.Clusters;
 import org.apache.ambari.server.state.RepositoryType;
 import org.apache.ambari.server.state.RepositoryVersionState;
+import org.apache.ambari.server.state.StackId;
 import org.apache.ambari.server.state.UpgradeContext;
 import org.apache.ambari.server.state.stack.upgrade.Direction;
 import org.apache.commons.lang.StringUtils;
@@ -142,6 +143,10 @@ public class UpdateDesiredStackAction extends 
AbstractUpgradeServerAction {
         }
 
         out.append(message).append(System.lineSeparator());
+
+        // move the cluster's desired stack as well
+        StackId targetStackId = targetRepositoryVersion.getStackId();
+        cluster.setDesiredStackVersion(targetStackId);
       }
 
       if( upgradeContext.getDirection() == Direction.DOWNGRADE ){
@@ -154,7 +159,7 @@ public class UpdateDesiredStackAction extends 
AbstractUpgradeServerAction {
 
           message = String.format("  %s to %s", serviceName, 
repositoryVersion.getVersion());
           out.append(message).append(System.lineSeparator());
-        }
+        }        
       }
 
       // move repositories to the right version and create/revert configs
@@ -175,6 +180,10 @@ public class UpdateDesiredStackAction extends 
AbstractUpgradeServerAction {
             hostVersion.setState(RepositoryVersionState.INSTALLED);
           }
         }
+        
+        // move the cluster's desired stack back to it's current stack on 
downgrade
+        StackId targetStackId = cluster.getCurrentStackVersion();
+        cluster.setDesiredStackVersion(targetStackId);        
       }
 
       return createCommandReport(0, HostRoleStatus.COMPLETED, "{}", 
out.toString(), err.toString());

http://git-wip-us.apache.org/repos/asf/ambari/blob/32216f75/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java
----------------------------------------------------------------------
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java 
b/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java
index b4f7120..ee18fdf 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java
@@ -548,6 +548,18 @@ public interface Cluster {
   Map<String, Object> getSessionAttributes();
 
   /**
+   * Makes the most recent configurations for the specified stack current.
+   * <p/>
+   * When completed, all other configurations for any other stack will remain,
+   * but will not be marked as selected.
+   *
+   * @param stackId
+   *          the stack to use when finding the latest configurations (not
+   *          {@code null}).
+   */
+  void applyLatestConfigurations(StackId stackId);
+
+  /**
    * Makes the most recent configurations for the specified stack current. This
    * will only modify configurations for the given service.
    * <p/>

http://git-wip-us.apache.org/repos/asf/ambari/blob/32216f75/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
----------------------------------------------------------------------
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java 
b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
index 4b2e0f1..812fb62 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
@@ -1237,6 +1237,9 @@ public class ConfigHelper {
       if (!defaultPropertiesByType.containsKey(type)) {
         defaultPropertiesByType.put(type, new HashMap<String, String>());
       }
+
+      defaultPropertiesByType.get(type).put(stackDefaultProperty.getName(),
+          stackDefaultProperty.getValue());
     }
 
     // for every installed service, populate the default service properties
@@ -1250,6 +1253,9 @@ public class ConfigHelper {
       if (!defaultPropertiesByType.containsKey(type)) {
         defaultPropertiesByType.put(type, new HashMap<String, String>());
       }
+
+      defaultPropertiesByType.get(type).put(serviceDefaultProperty.getName(),
+          serviceDefaultProperty.getValue());
     }
 
     return defaultPropertiesByType;

http://git-wip-us.apache.org/repos/asf/ambari/blob/32216f75/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java
----------------------------------------------------------------------
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java 
b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java
index a5b40ff..0701296 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java
@@ -869,6 +869,20 @@ public class UpgradeHelper {
     Set<String> clusterConfigTypes = new HashSet<>();
     Set<String> processedClusterConfigTypes = new HashSet<>();
 
+    // downgrades are easy - either do nothing or simply apply configurations
+    // from the target stack
+    if (direction == Direction.DOWNGRADE) {
+      StackId currentStackId = cluster.getCurrentStackVersion();
+      StackId desiredStackId = cluster.getDesiredStackVersion();
+
+      if (!currentStackId.equals(desiredStackId)) {
+        cluster.applyLatestConfigurations(currentStackId);
+      }
+
+      return;
+    }
+
+    // upgrade is a bit harder - we have to merge new stack configurations in
     // merge or revert configurations for any service that needs it
     for (String serviceName : servicesInUpgrade) {
       RepositoryVersionEntity sourceRepositoryVersion = 
upgradeContext.getSourceRepositoryVersion(serviceName);
@@ -889,14 +903,6 @@ public class UpgradeHelper {
 
       ConfigHelper configHelper = m_configHelperProvider.get();
 
-      // downgrade is easy - just remove the new and make the old current
-      if (direction == Direction.DOWNGRADE) {
-        cluster.applyLatestConfigurations(targetStackId, serviceName);
-        return;
-      }
-
-      // upgrade is a bit harder - we have to merge new stack configurations in
-
       // populate a map of default configurations for the service on the old
       // stack (this is used when determining if a property has been
       // customized and should be overriden with the new stack value)

http://git-wip-us.apache.org/repos/asf/ambari/blob/32216f75/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
----------------------------------------------------------------------
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
index 06b6217..3fb45ab 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
@@ -2333,6 +2333,76 @@ public class ClusterImpl implements Cluster {
    */
   @Override
   @Transactional
+  public void applyLatestConfigurations(StackId stackId) {
+    clusterGlobalLock.writeLock().lock();
+
+    try {
+      // grab all of the configurations and hash them so we can easily update
+      // them when picking and choosing only those from the service
+      ClusterEntity clusterEntity = getClusterEntity();
+      Collection<ClusterConfigEntity> configEntities = 
clusterEntity.getClusterConfigEntities();
+      ImmutableMap<Object, ClusterConfigEntity> clusterConfigEntityMap = 
Maps.uniqueIndex(
+          configEntities, Functions.identity());
+
+      // disable any configurations which are currently selected
+      for (ClusterConfigEntity clusterConfig : configEntities) {
+        if (clusterConfig.isSelected()) {
+          // un-select the latest configuration for the service
+          clusterConfig.setSelected(false);
+
+          LOG.debug("Disabling configuration {} with tag {}", 
clusterConfig.getType(),
+              clusterConfig.getTag());
+        }
+      }
+
+      // get the latest configurations for the stack so they can be selected
+      Collection<ClusterConfigEntity> latestConfigMappingByStack = 
clusterDAO.getLatestConfigurations(
+          clusterEntity.getClusterId(), stackId);
+
+      Set<String> configTypesUpdated = new HashSet<>();
+
+      for (ClusterConfigEntity clusterConfig : latestConfigMappingByStack) {
+        // grab the hash'd entity from the map so we're working with the right
+        // one
+        clusterConfig = clusterConfigEntityMap.get(clusterConfig);
+
+        clusterConfig.setSelected(true);
+
+        configTypesUpdated.add(clusterConfig.getType());
+
+        LOG.info("Setting {} with version tag {} created on {} to selected for 
stack {}",
+            clusterConfig.getType(), clusterConfig.getTag(), new 
Date(clusterConfig.getTimestamp()),
+            stackId);
+      }
+
+      // since the entities which were modified came from the cluster entity's
+      // list to begin with, we can just save them right back - no need for a
+      // new collection since the entity instances were modified directly
+      clusterEntity = clusterDAO.merge(clusterEntity);
+
+      cacheConfigurations();
+
+      LOG.info(
+          "Applied latest configurations for stack {}. The the following types 
were modified: {}",
+          stackId, StringUtils.join(configTypesUpdated, ','));
+
+    } finally {
+      clusterGlobalLock.writeLock().unlock();
+    }
+
+    // publish an event to instruct entity managers to clear cached instances 
of
+    // ClusterEntity immediately - it takes EclipseLink about 1000ms to update
+    // the L1 caches of other threads and the action scheduler could act upon
+    // stale data
+    EntityManagerCacheInvalidationEvent event = new 
EntityManagerCacheInvalidationEvent();
+    jpaEventPublisher.publish(event);
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  @Transactional
   public void applyLatestConfigurations(StackId stackId, String serviceName) {
     clusterGlobalLock.writeLock().lock();
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/32216f75/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ClusterGrouping.java
----------------------------------------------------------------------
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ClusterGrouping.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ClusterGrouping.java
index 5ac7752..3deb7c8 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ClusterGrouping.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ClusterGrouping.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -25,7 +25,6 @@ import java.util.Iterator;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Map.Entry;
 import java.util.Set;
 
 import javax.xml.bind.annotation.XmlAccessType;
@@ -47,9 +46,6 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.google.common.base.Objects;
-import com.google.gson.JsonArray;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonPrimitive;
 
 /**
  * Used to represent cluster-based operations.
@@ -175,6 +171,13 @@ public class ClusterGrouping extends Grouping {
             continue;
           }
 
+          // only schedule this stage if its service is part of the upgrade
+          if (StringUtils.isNotBlank(execution.service)) {
+            if (!upgradeContext.isServiceSupported(execution.service)) {
+              continue;
+            }
+          }
+
           Task task = execution.task;
 
           StageWrapper wrapper = null;
@@ -264,6 +267,7 @@ public class ClusterGrouping extends Grouping {
         return null;
       }
 
+      // !!! FUTURE: check for component
 
       HostsType hosts = ctx.getResolver().getMasterAndHosts(service, 
component);
 
@@ -317,32 +321,6 @@ public class ClusterGrouping extends Grouping {
   }
 
   /**
-   * Populates the manual task, mt, with information about the list of hosts.
-   * @param mt Manual Task
-   * @param hostToComponents Map from host name to list of components
-   */
-  private void fillHostDetails(ManualTask mt, Map<String, List<String>> 
hostToComponents) {
-    JsonArray arr = new JsonArray();
-    for (Entry<String, List<String>> entry : hostToComponents.entrySet()) {
-      JsonObject hostObj = new JsonObject();
-      hostObj.addProperty("host", entry.getKey());
-
-      JsonArray componentArr = new JsonArray();
-      for (String comp : entry.getValue()) {
-        componentArr.add(new JsonPrimitive(comp));
-      }
-      hostObj.add("components", componentArr);
-
-      arr.add(hostObj);
-    }
-
-    JsonObject obj = new JsonObject();
-    obj.add("unhealthy", arr);
-
-    mt.structuredOut = obj.toString();
-  }
-
-  /**
    * Attempts to merge the given cluster groupings.  This merges the execute 
stages
    * in an order specific manner.
    */
@@ -409,4 +387,4 @@ public class ClusterGrouping extends Grouping {
       }
     }
   }
-}
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/32216f75/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
----------------------------------------------------------------------
diff --git 
a/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
 
b/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
index 9b85242..24a3fa2 100644
--- 
a/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
+++ 
b/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
@@ -335,7 +335,7 @@ public class UpgradeHelperTest extends EasyMockSupport {
     assertEquals("Save Cluster State", postGroup.items.get(1).getText());
     assertEquals(StageWrapper.Type.SERVER_SIDE_ACTION, 
postGroup.items.get(1).getType());
 
-    assertEquals(3, groups.get(0).items.size());
+    assertEquals(2, groups.get(0).items.size());
     assertEquals(7, groups.get(1).items.size());
     assertEquals(2, groups.get(2).items.size());
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/32216f75/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterTest.java
----------------------------------------------------------------------
diff --git 
a/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterTest.java
 
b/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterTest.java
index 3a29298..e8e7206 100644
--- 
a/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterTest.java
+++ 
b/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterTest.java
@@ -2178,6 +2178,9 @@ public class ClusterTest {
     createDefaultCluster(Sets.newHashSet("host-1"), stackId);
 
     Cluster cluster = clusters.getCluster("c1");
+    cluster.setCurrentStackVersion(stackId);
+    cluster.setDesiredStackVersion(stackId);
+
     RepositoryVersionEntity repoVersion220 = 
helper.getOrCreateRepositoryVersion(newStackId, "2.2.0-1234");
 
     ConfigHelper configHelper = injector.getInstance(ConfigHelper.class);
@@ -2201,8 +2204,9 @@ public class ClusterTest {
     // make v1 "current"
     cluster.addDesiredConfig("admin", Sets.newHashSet(c1), "note-1");
 
-    // bump the repo version
+    // bump the repo version and the desired stack
     service.setDesiredRepositoryVersion(repoVersion220);
+    cluster.setDesiredStackVersion(newStackId);
 
     // save v2
     // config for v2 on new stack

http://git-wip-us.apache.org/repos/asf/ambari/blob/32216f75/ambari-server/src/test/resources/stacks/HDP/2.2.0/upgrades/upgrade_test_checks.xml
----------------------------------------------------------------------
diff --git 
a/ambari-server/src/test/resources/stacks/HDP/2.2.0/upgrades/upgrade_test_checks.xml
 
b/ambari-server/src/test/resources/stacks/HDP/2.2.0/upgrades/upgrade_test_checks.xml
index 4d4d972..ef8ce70 100644
--- 
a/ambari-server/src/test/resources/stacks/HDP/2.2.0/upgrades/upgrade_test_checks.xml
+++ 
b/ambari-server/src/test/resources/stacks/HDP/2.2.0/upgrades/upgrade_test_checks.xml
@@ -78,7 +78,16 @@
       </service>
       <service-check>false</service-check>
     </group>
-    
+
+    <group xsi:type="cluster" name="HBASE" title="Update HBase Configuration">
+      <skippable>true</skippable>
+      <execute-stage service="HBASE" component="HBASE_MASTER" title="Update 
HBase Configuration">
+        <task xsi:type="server_action" 
class="org.apache.ambari.server.serveraction.upgrades.HBaseEnvMaxDirectMemorySizeAction">
+          <summary>Update HBase Env Configuration</summary>
+        </task>
+      </execute-stage>
+    </group>
+
     <group name="SERVICE_CHECK_1" title="Post-Master Service Checks" 
xsi:type="service-check">
       <priority>
         <service>HDFS</service>
@@ -108,6 +117,16 @@
         <message>Please run additional tests</message>
       </batch>
     </group>
+
+    <!-- This group will be ignored because it is an invalid syntax. It is not 
a cluster type but contains an execute-stage -->
+    <group name="HBASE" title="Update HBase Configuration">
+      <skippable>true</skippable>
+      <execute-stage service="HBASE" component="HBASE_MASTER" title="Update 
HBase Configuration">
+        <task xsi:type="server_action" 
class="org.apache.ambari.server.serveraction.upgrades.HBaseEnvMaxDirectMemorySizeAction">
+          <summary>Update HBase Env Configuration</summary>
+        </task>
+      </execute-stage>
+    </group>
     
     <group name="SERVICE_CHECK_2" title="Post-Slave Service Checks" 
xsi:type="service-check">
       <priority>
@@ -220,4 +239,4 @@
       </component>
     </service>
   </processing>
-</upgrade>
+</upgrade>
\ No newline at end of file

Reply via email to