This is an automated email from the ASF dual-hosted git repository.
jonathanhurley pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new 709b43b [AMBARI-24078] Components Added During Upgrade Are Not
Scheduled for Restart (#1512)
709b43b is described below
commit 709b43baacb9df32268d586f3c8682e7215141cf
Author: Jonathan Hurley <jonathanhur...@apache.org>
AuthorDate: Tue Jun 12 09:49:29 2018 -0400
[AMBARI-24078] Components Added During Upgrade Are Not Scheduled for
Restart (#1512)
---
.../apache/ambari/server/state/UpgradeHelper.java | 91 ++++++++++++----------
.../ambari/server/state/UpgradeHelperTest.java | 39 ++++++++++
.../2.1.1/upgrades/upgrade_test_add_component.xml | 61 +++++++++++++++
3 files changed, 150 insertions(+), 41 deletions(-)
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java
b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java
index a48d98c..922b11f 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java
@@ -35,8 +35,6 @@ import java.util.stream.Collectors;
import org.apache.ambari.annotations.Experimental;
import org.apache.ambari.annotations.ExperimentalFeature;
import org.apache.ambari.server.AmbariException;
-import org.apache.ambari.server.agent.stomp.AgentConfigsHolder;
-import org.apache.ambari.server.agent.stomp.MetadataHolder;
import org.apache.ambari.server.api.services.AmbariMetaInfo;
import org.apache.ambari.server.controller.AmbariManagementController;
import org.apache.ambari.server.controller.AmbariManagementControllerImpl;
@@ -206,12 +204,6 @@ public class UpgradeHelper {
@Inject
private Provider<AmbariManagementControllerImpl> m_controllerProvider;
- @Inject
- private Provider<MetadataHolder> m_metadataHolder;
-
- @Inject
- private Provider<AgentConfigsHolder> m_agentConfigsHolder;
-
/**
* Used to get configurations by service name.
*/
@@ -382,47 +374,14 @@ public class UpgradeHelper {
continue;
}
-
for (String component : service.components) {
// Rolling Upgrade has exactly one task for a Component.
// NonRolling Upgrade has several tasks for the same component,
since it must first call Stop, perform several
// other tasks, and then Start on that Component.
-
if (upgradePack.getType() == UpgradeType.ROLLING &&
!allTasks.get(service.serviceName).containsKey(component)) {
continue;
}
- HostsType hostsType = mhr.getMasterAndHosts(service.serviceName,
component);
- if (null == hostsType) {
- // a null hosts type usually means that the component is not
- // installed in the cluster - but it's possible that it's going to
- // be added as part of the upgrade. If this is the case, then we
- // need to schedule tasks assuming the add works
- String id = service.serviceName + "/" + component;
- if (addedComponentsDuringUpgrade.containsKey(id)) {
- AddComponentTask task = addedComponentsDuringUpgrade.get(id);
- Collection<Host> candidateHosts =
MasterHostResolver.getCandidateHosts(cluster,
- task.hosts, task.hostService, task.hostComponent);
-
- if (!candidateHosts.isEmpty()) {
- hostsType = HostsType.normal(
- candidateHosts.stream().map(host ->
host.getHostName()).collect(
- Collectors.toCollection(LinkedHashSet::new)));
- }
- }
-
- // if we still have no hosts, then truly skip this component
- if (null == hostsType) {
- continue;
- }
- }
-
- if (!hostsType.unhealthy.isEmpty()) {
- context.addUnhealthy(hostsType.unhealthy);
- }
-
- Service svc = cluster.getService(service.serviceName);
-
// if a function name is present, build the tasks dynamically;
// otherwise use the tasks defined in the upgrade pack processing
ProcessingComponent pc = null;
@@ -464,6 +423,56 @@ public class UpgradeHelper {
continue;
}
+ HostsType hostsType = mhr.getMasterAndHosts(service.serviceName,
component);
+
+ // only worry about adding future commands if this is a
start/restart task
+ boolean taskIsRestartOrStart = functionName == null || functionName
== Type.START
+ || functionName == Type.RESTART;
+
+ // see if this component has an add component task which will
indicate
+ // we need to dynamically schedule some more tasks by predicting
where
+ // the components will be installed
+ String serviceAndComponentHash = service.serviceName + "/" +
component;
+ if (taskIsRestartOrStart &&
addedComponentsDuringUpgrade.containsKey(serviceAndComponentHash)) {
+ AddComponentTask task =
addedComponentsDuringUpgrade.get(serviceAndComponentHash);
+
+ Collection<Host> candidateHosts =
MasterHostResolver.getCandidateHosts(cluster,
+ task.hosts, task.hostService, task.hostComponent);
+
+ // if we have candidate hosts, then we can create a structure to be
+ // scheduled
+ if (!candidateHosts.isEmpty()) {
+ if (null == hostsType) {
+ // a null hosts type usually means that the component is not
+ // installed in the cluster - but it's possible that it's going
+ // to be added as part of the upgrade. If this is the case,
then
+ // we need to schedule tasks assuming the add works
+ hostsType = HostsType.normal(
+ candidateHosts.stream().map(host ->
host.getHostName()).collect(
+ Collectors.toCollection(LinkedHashSet::new)));
+ } else {
+ // it's possible that we're adding components that may already
+ // exist in the cluster - in this case, we must append the
+ // structure instead of creating a new one
+ Set<String> hostsForTask = hostsType.getHosts();
+ for (Host host : candidateHosts) {
+ hostsForTask.add(host.getHostName());
+ }
+ }
+ }
+ }
+
+ // if we still have no hosts, then truly skip this component
+ if (null == hostsType) {
+ continue;
+ }
+
+ if (!hostsType.unhealthy.isEmpty()) {
+ context.addUnhealthy(hostsType.unhealthy);
+ }
+
+ Service svc = cluster.getService(service.serviceName);
+
setDisplayNames(context, service.serviceName, component);
// Special case for NAMENODE when there are multiple
diff --git
a/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
index a23c451..2cc2010 100644
---
a/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
+++
b/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
@@ -2737,6 +2737,45 @@ public class UpgradeHelperTest extends EasyMockSupport {
ambariMetaInfo.init();
}
+ /**
+ * Tests that components added during the upgrade are scheduled for restart
on
+ * their future hosts.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testAddComponentsDuringUpgrade() throws Exception {
+ Map<String, UpgradePack> upgrades = ambariMetaInfo.getUpgradePacks("HDP",
"2.1.1");
+
+ assertTrue(upgrades.containsKey("upgrade_test_add_component"));
+ UpgradePack upgrade = upgrades.get("upgrade_test_add_component");
+ assertNotNull(upgrade);
+
+ Cluster cluster = makeCluster();
+
+ UpgradeContext context = getMockUpgradeContext(cluster, Direction.UPGRADE,
UpgradeType.NON_ROLLING);
+
+ List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade,
context);
+
+ // 3 groups - stop, add component, restart
+ assertEquals(3, groups.size());
+
+ // ensure that the stop did not have extra hosts added to it
+ UpgradeGroupHolder group = groups.get(0);
+ assertEquals("STOP_HIVE", group.name);
+ List<StageWrapper> stageWrappers = group.items;
+ assertEquals(2, stageWrappers.size());
+
+ // ensure that the restart has the future hosts
+ group = groups.get(2);
+ assertEquals("RESTART_HIVE", group.name);
+ stageWrappers = group.items;
+ assertEquals(4, stageWrappers.size());
+
+ // Do stacks cleanup
+ stackManagerMock.invalidateCurrentPaths();
+ ambariMetaInfo.init();
+ }
/**
* @param cluster
diff --git
a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test_add_component.xml
b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test_add_component.xml
new file mode 100644
index 0000000..de31b85
--- /dev/null
+++
b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test_add_component.xml
@@ -0,0 +1,61 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<upgrade xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:noNamespaceSchemaLocation="upgrade-pack.xsd">
+ <target>2.2.*.*</target>
+ <target-stack>HDP-2.2.0</target-stack>
+ <type>NON_ROLLING</type>
+ <prerequisite-checks/>
+
+ <order>
+ <group xsi:type="stop" name="STOP_HIVE" title="Stop Hive Server">
+ <service-check>false</service-check>
+ <skippable>true</skippable>
+ <service name="HIVE">
+ <component>HIVE_SERVER</component>
+ </service>
+ </group>
+
+ <group xsi:type="cluster" name="TEST_ADD_COMPONENT" title="Add New Hive
Servers Where There Are DataNodes">
+ <direction>UPGRADE</direction>
+ <skippable>true</skippable>
+ <execute-stage service="HIVE" title="Hive Server">
+ <task xsi:type="add_component" service="HIVE" component="HIVE_SERVER"
host-service="HDFS" host-component="DATANODE" hosts="all">
+ <summary>Add New Hive Servers Where There Are DataNodes</summary>
+ </task>
+ </execute-stage>
+ </group>
+
+ <group xsi:type="restart" name="RESTART_HIVE" title="Restart Hive Server">
+ <service-check>false</service-check>
+ <skippable>true</skippable>
+ <service name="HIVE">
+ <component>HIVE_SERVER</component>
+ </service>
+ </group>
+ </order>
+
+ <processing>
+ <service name="HIVE">
+ <component name="HIVE_SERVER">
+ <upgrade>
+ <task xsi:type="restart-task"/>
+ </upgrade>
+ </component>
+ </service>
+ </processing>
+</upgrade>
--
To stop receiving notification emails like this one, please contact
jonathanhur...@apache.org.
