This is an automated email from the ASF dual-hosted git repository.
jonathanhurley pushed a commit to branch branch-feature-AMBARI-14714
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/branch-feature-AMBARI-14714 by
this push:
new 953043c [AMBARI-24467] - Report Component Instance On Agent
Registration (#2071)
953043c is described below
commit 953043c3f3d459662435a0d32620f8a402397560
Author: Jonathan Hurley <[email protected]>
AuthorDate: Wed Aug 15 08:07:07 2018 -0400
[AMBARI-24467] - Report Component Instance On Agent Registration (#2071)
---
.../ambari_agent/ComponentVersionReporter.py | 11 +-
.../resource_management/libraries/script/script.py | 18 +-
.../ambari/server/agent/HeartbeatProcessor.java | 38 ++-
.../ambari/server/agent/stomp/TopologyHolder.java | 44 ++-
.../agent/stomp/dto/ComponentVersionReport.java | 32 +-
.../server/agent/stomp/dto/TopologyComponent.java | 52 +++-
.../controller/AmbariManagementControllerImpl.java | 1 +
.../DeleteHostComponentStatusMetaData.java | 16 +-
.../svccomphost/ServiceComponentHostImpl.java | 1 +
.../topology/STOMPComponentsDeleteHandler.java | 1 +
.../custom_actions/scripts/stack_select_set_all.py | 116 -------
.../resources/scripts/mpack_advisor_wrapper.py | 0
.../custom_actions/test_stack_select_set_all.py | 343 ---------------------
13 files changed, 156 insertions(+), 517 deletions(-)
diff --git
a/ambari-agent/src/main/python/ambari_agent/ComponentVersionReporter.py
b/ambari-agent/src/main/python/ambari_agent/ComponentVersionReporter.py
index eb4358b..7e23ec8 100644
--- a/ambari-agent/src/main/python/ambari_agent/ComponentVersionReporter.py
+++ b/ambari-agent/src/main/python/ambari_agent/ComponentVersionReporter.py
@@ -58,10 +58,11 @@ class ComponentVersionReporter(threading.Thread):
if current_host_id not in component_dict.hostIds:
continue
+ service_group_name = component_dict.serviceGroupName
service_name = component_dict.serviceName
component_name = component_dict.componentName
- result = self.check_component_version(cluster_id, service_name,
component_name)
+ result = self.check_component_version(cluster_id,
service_group_name, service_name, component_name)
if result:
cluster_reports[cluster_id].append(result)
@@ -70,7 +71,7 @@ class ComponentVersionReporter(threading.Thread):
except:
logger.exception("Exception in ComponentVersionReporter")
- def check_component_version(self, cluster_id, service_name, component_name):
+ def check_component_version(self, cluster_id, service_group_name,
service_name, component_name):
"""
Returns components version
"""
@@ -79,6 +80,7 @@ class ComponentVersionReporter(threading.Thread):
return None
command_dict = {
+ 'serviceGroupName': service_group_name,
'serviceName': service_name,
'role': component_name,
'clusterId': cluster_id,
@@ -87,16 +89,17 @@ class ComponentVersionReporter(threading.Thread):
version_result =
self.customServiceOrchestrator.requestComponentStatus(command_dict,
command_name=AgentCommand.get_version)
- if version_result['exitcode'] or not 'structuredOut' in version_result or
not 'version' in version_result['structuredOut']:
+ if version_result['exitcode'] or not 'structuredOut' in version_result or
not 'version_reporting' in version_result['structuredOut']:
logger.error("Could not get version for component {0} of {1} service
cluster_id={2}. Command returned: {3}".format(component_name, service_name,
cluster_id, version_result))
return None
# TODO: check if no strout or version if not there
result = {
+ 'serviceGroupName': service_group_name,
'serviceName': service_name,
'componentName': component_name,
- 'version': version_result['structuredOut']['version'],
+ 'version_reporting':
version_result['structuredOut']['version_reporting'],
'clusterId': cluster_id,
}
diff --git
a/ambari-common/src/main/python/resource_management/libraries/script/script.py
b/ambari-common/src/main/python/resource_management/libraries/script/script.py
index daf6175..275e4c5 100644
---
a/ambari-common/src/main/python/resource_management/libraries/script/script.py
+++
b/ambari-common/src/main/python/resource_management/libraries/script/script.py
@@ -25,20 +25,18 @@ import re
import os
import sys
import logging
-import platform
import inspect
import tarfile
import traceback
import time
from optparse import OptionParser
import resource_management
-from ambari_commons import OSCheck, OSConst
+from ambari_commons import OSCheck
from ambari_commons.constants import UPGRADE_TYPE_EXPRESS
from ambari_commons.constants import UPGRADE_TYPE_ROLLING
from ambari_commons.constants import UPGRADE_TYPE_HOST_ORDERED
from ambari_commons.network import reconfigure_urllib2_opener
-from ambari_commons.inet_utils import resolve_address,
ensure_ssl_using_protocol
-from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
+from ambari_commons.inet_utils import ensure_ssl_using_protocol
from resource_management.libraries.resources import XmlConfig
from resource_management.libraries.resources import PropertiesFile
from resource_management.core import sudo
@@ -54,14 +52,11 @@ from resource_management.libraries.functions import
stack_tools
from resource_management.libraries.functions.constants import Direction
from resource_management.libraries.script.config_dictionary import
ConfigDictionary, UnknownConfiguration
from resource_management.libraries.functions.repository_util import
CommandRepository, RepositoryUtil
-from resource_management.core.resources.system import Execute
from contextlib import closing
from resource_management.libraries.functions.stack_features import
check_stack_feature
from resource_management.libraries.functions.constants import StackFeature
from resource_management.libraries.functions.show_logs import show_logs
-from resource_management.libraries.functions.fcntl_based_process_lock import
FcntlBasedProcessLock
from resource_management.libraries.execution_command.execution_command import
ExecutionCommand
-from resource_management.libraries.execution_command.module_configs import
ModuleConfigs
import ambari_simplejson as json # simplejson is much faster comparing to
Python 2.6 json module and has the same functions set.
@@ -268,6 +263,7 @@ class Script(object):
return False
+
def execute(self):
"""
Sets up logging;
@@ -366,9 +362,17 @@ class Script(object):
if self.should_expose_component_version():
self.save_component_version_to_structured_out()
+
def get_version(self, env):
+ """
+ A dummy method which is used to ask for the version of installed
components on startup using t
+ he topology information which is sent to the agent.
+ :param env:
+ :return: nothing
+ """
pass
+
def execute_prefix_function(self, command_name, afix, env):
"""
Execute action afix (prefix or suffix) based on command_name and afix type
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatProcessor.java
b/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatProcessor.java
index 205aa7f..d9099f9 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatProcessor.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatProcessor.java
@@ -33,8 +33,6 @@ import java.util.concurrent.TimeUnit;
import javax.annotation.Nullable;
-import org.apache.ambari.annotations.Experimental;
-import org.apache.ambari.annotations.ExperimentalFeature;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.Role;
import org.apache.ambari.server.RoleCommand;
@@ -81,6 +79,7 @@ import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.base.MoreObjects;
import com.google.common.util.concurrent.AbstractService;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.google.gson.Gson;
@@ -593,9 +592,6 @@ public class HeartbeatProcessor extends AbstractService{
* Process reports of components versions
* @throws AmbariException
*/
- @Experimental(
- feature = ExperimentalFeature.VERSION_REPORTING,
- comment = "This needs to be rewritten using the new structured output")
public void processVersionReports(ComponentVersionReports versionReports,
String hostname) throws AmbariException {
Set<Cluster> clusters = clusterFsm.getClustersForHost(hostname);
for (Cluster cl : clusters) {
@@ -604,19 +600,16 @@ public class HeartbeatProcessor extends AbstractService{
if (Long.valueOf(status.getKey()).equals(cl.getClusterId())) {
for (ComponentVersionReport versionReport : status.getValue()) {
try {
- Service svc = cl.getService(versionReport.getServiceName());
+ Service svc = cl.getService(versionReport.getServiceGroupName(),
versionReport.getServiceName());
String componentName = versionReport.getComponentName();
if (svc.getServiceComponents().containsKey(componentName)) {
- ServiceComponent svcComp = svc.getServiceComponent(
- componentName);
- ServiceComponentHost scHost = svcComp.getServiceComponentHost(
- hostname);
-
- String version = versionReport.getVersion();
+ ServiceComponent svcComp =
svc.getServiceComponent(componentName);
+ ServiceComponentHost scHost =
svcComp.getServiceComponentHost(hostname);
HostComponentVersionAdvertisedEvent event = new
HostComponentVersionAdvertisedEvent(cl,
- scHost, null);
+ scHost, versionReport.getVersionStructuredOutput());
+
versionEventPublisher.publish(event);
}
} catch (ServiceNotFoundException e) {
@@ -825,6 +818,16 @@ public class HeartbeatProcessor extends AbstractService{
@SerializedName("version")
public String version;
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("mpackVersion", mpackVersion)
+ .add("version", version).toString();
+ }
}
/**
@@ -834,5 +837,14 @@ public class HeartbeatProcessor extends AbstractService{
public static class UpgradeSummaryStructuredOuut {
@SerializedName("direction")
public String direction;
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("direction", direction).toString();
+ }
}
}
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/agent/stomp/TopologyHolder.java
b/ambari-server/src/main/java/org/apache/ambari/server/agent/stomp/TopologyHolder.java
index a2710231..c1a4641 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/agent/stomp/TopologyHolder.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/agent/stomp/TopologyHolder.java
@@ -22,6 +22,7 @@ import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
+import java.util.stream.Collectors;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.ClusterNotFoundException;
@@ -44,7 +45,6 @@ import org.apache.commons.collections.CollectionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.collect.Sets;
import com.google.common.eventbus.Subscribe;
import com.google.inject.Inject;
import com.google.inject.Singleton;
@@ -90,15 +90,28 @@ public class TopologyHolder extends
AgentClusterDataHolder<TopologyUpdateEvent>
for (Service service : cl.getServices()) {
for (ServiceComponent component :
service.getServiceComponents().values()) {
Map<String, ServiceComponentHost> componentsMap =
component.getServiceComponentHosts();
- for (ServiceComponentHost sch : componentsMap.values()) {
+ if (!componentsMap.isEmpty()) {
+
+ //TODO will be a need to change to multi-instance usage
+ ServiceComponentHost sch =
componentsMap.entrySet().iterator().next().getValue();
+
+ Set<String> hostNames = cl.getHosts(sch.getServiceName(),
sch.getServiceComponentName());
+ Set<Long> hostOrderIds = clusterHosts.stream()
+ .filter(h -> hostNames.contains(h.getHostName()))
+ .map(Host::getHostId)
+ .collect(Collectors.toSet());
+ String serviceGroupName = sch.getServiceGroupName();
+
TopologyComponent topologyComponent =
TopologyComponent.newBuilder()
- .setComponentName(sch.getServiceComponentName())
- .setServiceName(sch.getServiceName())
- .setVersion(sch.getVersion())
- .setHostIds(Sets.newHashSet(sch.getHost().getHostId()))
-
.setComponentLevelParams(ambariManagementController.getTopologyComponentLevelParams(sch))
-
.setCommandParams(ambariManagementController.getTopologyCommandParams(sch))
- .build();
+ .setComponentName(sch.getServiceComponentName())
+ .setServiceName(sch.getServiceName())
+ .setServiceGroupName(serviceGroupName)
+ .setVersion(sch.getVersion())
+ .setMpackVersion(sch.getMpackVersion())
+ .setHostIds(hostOrderIds)
+
.setComponentLevelParams(ambariManagementController.getTopologyComponentLevelParams(sch))
+
.setCommandParams(ambariManagementController.getTopologyCommandParams(sch))
+ .build();
topologyComponents.add(topologyComponent);
}
}
@@ -206,19 +219,22 @@ public class TopologyHolder extends
AgentClusterDataHolder<TopologyUpdateEvent>
Map<String, ServiceComponentHost> componentsMap =
component.getServiceComponentHosts();
if (!componentsMap.isEmpty()) {
- for (ServiceComponentHost sch : componentsMap.values()) {
- TopologyComponent topologyComponent =
TopologyComponent.newBuilder()
+ //TODO will be a need to change to multi-instance usage
+ ServiceComponentHost sch =
componentsMap.entrySet().iterator().next().getValue();
+
+ TopologyComponent topologyComponent = TopologyComponent.newBuilder()
.setComponentName(sch.getServiceComponentName())
.setServiceName(sch.getServiceName())
+ .setServiceGroupName(sch.getServiceGroupName())
.setVersion(sch.getVersion())
+ .setMpackVersion(sch.getMpackVersion())
.setCommandParams(ambariManagementController.getTopologyCommandParams(sch))
.setComponentLevelParams(ambariManagementController.getTopologyComponentLevelParams(sch))
.build();
- topologyComponents.add(topologyComponent);
- }
+ topologyComponents.add(topologyComponent);
}
}
}
return topologyComponents;
}
-}
+}
\ No newline at end of file
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/agent/stomp/dto/ComponentVersionReport.java
b/ambari-server/src/main/java/org/apache/ambari/server/agent/stomp/dto/ComponentVersionReport.java
index 6619096..1202c8d 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/agent/stomp/dto/ComponentVersionReport.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/agent/stomp/dto/ComponentVersionReport.java
@@ -18,20 +18,29 @@
package org.apache.ambari.server.agent.stomp.dto;
+import
org.apache.ambari.server.agent.HeartbeatProcessor.ComponentVersionStructuredOut;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
public class ComponentVersionReport {
private String componentName;
private String serviceName;
- private String version;
+ private String serviceGroupName;
private Long clusterId;
+ @JsonProperty("version_reporting")
+ private ComponentVersionStructuredOut componentVersionStructuredOut;
+
public ComponentVersionReport() {
}
- public ComponentVersionReport(String componentName, String serviceName,
String version, Long clusterId) {
+ public ComponentVersionReport(Long clusterId, String serviceGroupName,
String serviceName, String componentName,
+ ComponentVersionStructuredOut componentVersionStructuredOut) {
this.componentName = componentName;
this.serviceName = serviceName;
- this.version = version;
+ this.serviceGroupName = serviceGroupName;
this.clusterId = clusterId;
+ this.componentVersionStructuredOut = componentVersionStructuredOut;
}
public String getComponentName() {
@@ -42,6 +51,14 @@ public class ComponentVersionReport {
this.componentName = componentName;
}
+ public String getServiceGroupName() {
+ return serviceGroupName;
+ }
+
+ public void setServiceGroupName(String serviceGroupName) {
+ this.serviceGroupName = serviceGroupName;
+ }
+
public String getServiceName() {
return serviceName;
}
@@ -50,12 +67,13 @@ public class ComponentVersionReport {
this.serviceName = serviceName;
}
- public String getVersion() {
- return version;
+ public ComponentVersionStructuredOut getVersionStructuredOutput() {
+ return componentVersionStructuredOut;
}
- public void setVersion(String version) {
- this.version = version;
+ public void setVersionStructuredOutput(
+ ComponentVersionStructuredOut componentVersionStructuredOut) {
+ this.componentVersionStructuredOut = componentVersionStructuredOut;
}
public Long getClusterId() {
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/agent/stomp/dto/TopologyComponent.java
b/ambari-server/src/main/java/org/apache/ambari/server/agent/stomp/dto/TopologyComponent.java
index b3da819..c633b63 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/agent/stomp/dto/TopologyComponent.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/agent/stomp/dto/TopologyComponent.java
@@ -20,6 +20,7 @@ package org.apache.ambari.server.agent.stomp.dto;
import java.util.HashSet;
import java.util.Map;
+import java.util.Objects;
import java.util.Set;
import java.util.TreeMap;
@@ -34,8 +35,10 @@ import com.fasterxml.jackson.annotation.JsonInclude;
public class TopologyComponent {
private String componentName;
private String serviceName;
+ private String serviceGroupName;
private String serviceType;
private String version;
+ private String mpackVersion;
private Set<Long> hostIds = new HashSet<>();
private Set<String> hostNames = new HashSet<>();
private Set<String> publicHostNames = new HashSet<>();
@@ -65,6 +68,11 @@ public class TopologyComponent {
return this;
}
+ public Builder setServiceGroupName(String serviceGroupName) {
+ TopologyComponent.this.setServiceGroupName(serviceGroupName);
+ return this;
+ }
+
public Builder setServiceType(String serviceType) {
TopologyComponent.this.setServiceType(serviceType);
return this;
@@ -75,6 +83,11 @@ public class TopologyComponent {
return this;
}
+ public Builder setMpackVersion(String mpackVersion) {
+ TopologyComponent.this.setMpackVersion(mpackVersion);
+ return this;
+ }
+
public Builder setHostIds(Set<Long> hostIds) {
TopologyComponent.this.setHostIds(hostIds);
return this;
@@ -187,6 +200,7 @@ public class TopologyComponent {
return TopologyComponent.newBuilder().setComponentName(getComponentName())
.setServiceType(getServiceType())
.setServiceName(getServiceName())
+ .setServiceGroupName(getServiceGroupName())
.setComponentLevelParams(getComponentLevelParams() == null ? null :
new TreeMap<>(getComponentLevelParams()))
.setHostIds(getHostIds() == null ? null : new HashSet<>(getHostIds()))
.setHostNames(getHostNames() == null ? null : new
HashSet<>(getHostNames()))
@@ -211,6 +225,14 @@ public class TopologyComponent {
this.serviceName = serviceName;
}
+ public String getServiceGroupName() {
+ return serviceGroupName;
+ }
+
+ public void setServiceGroupName(String serviceGroupName) {
+ this.serviceGroupName = serviceGroupName;
+ }
+
public String getVersion() {
return version;
}
@@ -219,6 +241,14 @@ public class TopologyComponent {
this.version = version;
}
+ public String getMpackVersion() {
+ return mpackVersion;
+ }
+
+ public void setMpackVersion(String mpackVersion) {
+ this.mpackVersion = mpackVersion;
+ }
+
public Set<Long> getHostIds() {
return hostIds;
}
@@ -228,11 +258,11 @@ public class TopologyComponent {
}
public void addHostId(Long hostId) {
- this.hostIds.add(hostId);
+ hostIds.add(hostId);
}
public void addHostName(String hostName) {
- this.hostNames.add(hostName);
+ hostNames.add(hostName);
}
public TreeMap<String, String> getComponentLevelParams() {
@@ -285,19 +315,21 @@ public class TopologyComponent {
@Override
public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
TopologyComponent that = (TopologyComponent) o;
-
- if (!componentName.equals(that.componentName)) return false;
- return serviceName.equals(that.serviceName);
+ return Objects.equals(serviceGroupName, that.serviceGroupName)
+ && Objects.equals(serviceName, that.serviceName)
+ && Objects.equals(componentName, that.componentName);
}
@Override
public int hashCode() {
- int result = componentName.hashCode();
- result = 31 * result + serviceName.hashCode();
- return result;
+ return Objects.hash(serviceGroupName, serviceName, componentName);
}
}
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
index ddc083e..6af49b5 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
@@ -822,6 +822,7 @@ public class AmbariManagementControllerImpl implements
AmbariManagementControlle
TopologyComponent newComponent = TopologyComponent.newBuilder()
.setComponentName(sch.getServiceComponentName())
.setServiceName(sc.getServiceName())
+ .setServiceGroupName(sc.getServiceGroupName())
.setServiceType(sc.getServiceType())
.setVersion(sch.getVersion())
.setHostIds(hostIds)
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/DeleteHostComponentStatusMetaData.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/DeleteHostComponentStatusMetaData.java
index 7237f5b..1160ad9 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/DeleteHostComponentStatusMetaData.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/DeleteHostComponentStatusMetaData.java
@@ -33,9 +33,9 @@ public class DeleteHostComponentStatusMetaData extends
DeleteStatusMetaData {
removedHostComponents = new HashSet<>();
}
- public void addDeletedHostComponent(String componentName, String
serviceName, String hostName, Long hostId,
+ public void addDeletedHostComponent(String componentName, String
serviceName, String serviceGroupName, String hostName, Long hostId,
String clusterId, String version, State
lastComponentState) {
- removedHostComponents.add(new HostComponent(componentName, serviceName,
hostId,
+ removedHostComponents.add(new HostComponent(componentName, serviceName,
serviceGroupName, hostId,
hostName, clusterId, version, lastComponentState));
addDeletedKey(componentName + "/" + hostName);
}
@@ -55,16 +55,18 @@ public class DeleteHostComponentStatusMetaData extends
DeleteStatusMetaData {
public class HostComponent {
private String componentName;
private String serviceName;
+ private String serviceGroupName;
private Long hostId;
private String hostName;
private String clusterId;
private String version;
private State lastComponentState;
- public HostComponent(String componentName, String serviceName, Long
hostId, String hostName,
+ public HostComponent(String componentName, String serviceName, String
serviceGroupName, Long hostId, String hostName,
String clusterId, String version, State
lastComponentState) {
this.componentName = componentName;
this.serviceName = serviceName;
+ this.serviceGroupName = serviceGroupName;
this.hostId = hostId;
this.hostName = hostName;
this.clusterId = clusterId;
@@ -104,6 +106,14 @@ public class DeleteHostComponentStatusMetaData extends
DeleteStatusMetaData {
this.serviceName = serviceName;
}
+ public String getServiceGroupName() {
+ return serviceGroupName;
+ }
+
+ public void setServiceGroupName(String serviceGroupName) {
+ this.serviceGroupName = serviceGroupName;
+ }
+
public String getHostName() {
return hostName;
}
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java
b/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java
index 3b29037..a8046b8 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java
@@ -1461,6 +1461,7 @@ public class ServiceComponentHostImpl implements
ServiceComponentHost {
eventPublisher.publish(event);
deleteMetaData.addDeletedHostComponent(componentName,
serviceName,
+ serviceGroupName,
hostName,
getHost().getHostId(),
Long.toString(clusterId),
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/topology/STOMPComponentsDeleteHandler.java
b/ambari-server/src/main/java/org/apache/ambari/server/topology/STOMPComponentsDeleteHandler.java
index c48d4cf..69d7bff 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/topology/STOMPComponentsDeleteHandler.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/topology/STOMPComponentsDeleteHandler.java
@@ -119,6 +119,7 @@ public class STOMPComponentsDeleteHandler {
TopologyComponent deletedComponent = TopologyComponent.newBuilder()
.setComponentName(hostComponent.getComponentName())
.setServiceName(hostComponent.getServiceName())
+ .setServiceGroupName(hostComponent.getServiceGroupName())
.setVersion(hostComponent.getVersion())
.setHostIds(new HashSet<>(Arrays.asList(hostComponent.getHostId())))
.setHostNames(new
HashSet<>(Arrays.asList(hostComponent.getHostName())))
diff --git
a/ambari-server/src/main/resources/custom_actions/scripts/stack_select_set_all.py
b/ambari-server/src/main/resources/custom_actions/scripts/stack_select_set_all.py
deleted file mode 100644
index 9ae27ba..0000000
---
a/ambari-server/src/main/resources/custom_actions/scripts/stack_select_set_all.py
+++ /dev/null
@@ -1,116 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-Ambari Agent
-
-"""
-
-import os
-import socket
-from ambari_commons.os_check import OSCheck
-from resource_management.libraries.script import Script
-from resource_management.libraries.functions import stack_tools
-from resource_management.libraries.functions import upgrade_summary
-from resource_management.libraries.functions.constants import Direction
-from resource_management.core import shell
-from resource_management.core.logger import Logger
-from resource_management.libraries.functions.decorator import experimental
-
-class UpgradeSetAll(Script):
- """
- This script is a part of stack upgrade workflow and is used to set the
- all of the component versions as a final step in the upgrade process
- """
- @experimental(feature="PATCH_UPGRADES", disable = False, comment = "The
stack-select tool will only be invoked if this is a standard upgrade which
cannot be downgraded.")
- def actionexecute(self, env):
- summary = upgrade_summary.get_upgrade_summary()
- if summary is None:
- Logger.warning("There is no upgrade in progress")
- return
-
- if summary.associated_version is None:
- Logger.warning("There is no version associated with the upgrade in
progress")
- return
-
- if summary.orchestration != "STANDARD":
- Logger.warning("The 'stack-select set all' command can only be invoked
during STANDARD upgrades")
- return
-
- if summary.direction.lower() != Direction.UPGRADE or
summary.is_downgrade_allowed or summary.is_revert:
- Logger.warning("The 'stack-select set all' command can only be invoked
during an UPGRADE which cannot be downgraded")
- return
-
- # other os?
- if OSCheck.is_redhat_family():
- cmd = ('/usr/bin/yum', 'clean', 'all')
- code, out = shell.call(cmd, sudo=True)
-
- stack_selector_path =
stack_tools.get_stack_tool_path(stack_tools.STACK_SELECTOR_NAME)
-
- # this script runs on all hosts; if this host doesn't have stack
components,
- # then don't invoke the stack tool
- # (no need to log that it's skipped - the function will do that)
- if is_host_skippable(stack_selector_path, summary.associated_version):
- return
-
- # invoke "set all"
- cmd = ('ambari-python-wrap', stack_selector_path, 'set', 'all',
summary.associated_version)
- code, out = shell.call(cmd, sudo=True)
- if code != 0:
- raise Exception("Command '{0}' exit code is nonzero".format(cmd))
-
-
-def is_host_skippable(stack_selector_path, associated_version):
- """
- Gets whether this host should not have the stack select tool called.
- :param stack_selector_path the path to the stack selector tool.
- :param associated_version: the version to use with the stack selector tool.
- :return: True if this host should be skipped, False otherwise.
- """
- if not os.path.exists(stack_selector_path):
- Logger.info("{0} does not have any stack components installed and will not
invoke {1}".format(
- socket.gethostname(), stack_selector_path))
-
- return True
-
- # invoke the tool, checking its output
- cmd = ('ambari-python-wrap', stack_selector_path, "versions")
- code, out = shell.call(cmd, sudo=True)
-
- if code != 0:
- Logger.info("{0} is unable to determine which stack versions are available
using {1}".format(
- socket.gethostname(), stack_selector_path))
-
- return True
-
- # check to see if the output is empty, indicating no versions installed
- if not out.strip():
- Logger.info("{0} has no stack versions
installed".format(socket.gethostname()))
- return True
-
- # some pre-prepped systems may have a version, so there may be a version, so
- # add the extra check if it is available
- if not associated_version in out:
- Logger.info("{0} is not found in the list of versions
{1}".format(associated_version, out))
- return True
-
- return False
-
-
-if __name__ == "__main__":
- UpgradeSetAll().execute()
diff --git a/ambari-server/src/main/resources/scripts/mpack_advisor_wrapper.py
b/ambari-server/src/main/resources/scripts/mpack_advisor_wrapper.py
old mode 100644
new mode 100755
diff --git
a/ambari-server/src/test/python/custom_actions/test_stack_select_set_all.py
b/ambari-server/src/test/python/custom_actions/test_stack_select_set_all.py
deleted file mode 100644
index 3bcd0bb..0000000
--- a/ambari-server/src/test/python/custom_actions/test_stack_select_set_all.py
+++ /dev/null
@@ -1,343 +0,0 @@
-# !/usr/bin/env python
-
-'''
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-'''
-
-# Python Imports
-import os
-import json
-
-from mock.mock import patch
-from mock.mock import MagicMock
-
-from stacks.utils.RMFTestCase import experimental_mock
-patch('resource_management.libraries.functions.decorator.experimental',
experimental_mock).start()
-
-# Module imports
-from stacks.utils.RMFTestCase import *
-from resource_management import Script, ConfigDictionary
-from resource_management.libraries.functions.default import default
-from resource_management.core.logger import Logger
-from ambari_commons.os_check import OSCheck
-from resource_management.core.environment import Environment
-import pprint
-
-
-def fake_call(command, **kwargs):
- """
- Instead of shell.call, call a command whose output equals the command.
- :param command: Command that will be echoed.
- :return: Returns a tuple of (process output code, output)
- """
- return (0, str(command))
-
-class TestStackSelectSetAll(RMFTestCase):
- def get_custom_actions_dir(self):
- return os.path.join(self.get_src_folder(),
"test/resources/custom_actions/")
-
- @patch.object(Logger, "info")
- @patch.object(Logger, "error")
- def setUp(self, error_mock, info_mock):
-
- Logger.logger = MagicMock()
-
- # Import the class under test. This is done here as opposed to the rest of
the imports because the get_os_type()
- # method needs to be patched first.
- from stack_select_set_all import UpgradeSetAll
- global UpgradeSetAll
-
- def tearDown(self):
- Logger.logger = None
-
- @patch("os.path.exists")
- @patch("resource_management.core.shell.call")
- @patch.object(Script, 'get_config')
- @patch.object(OSCheck, 'is_redhat_family')
- def test_execution(self, family_mock, get_config_mock, call_mock,
exists_mock):
- # Mock the config objects
- json_file_path = os.path.join(self.get_custom_actions_dir(),
"ru_execute_tasks_namenode_prepare.json")
- self.assertTrue(os.path.isfile(json_file_path))
-
- with open(json_file_path, "r") as json_file:
- json_payload = json.load(json_file)
-
- json_payload["configurations"]["cluster-env"]["stack_tools"] =
self.get_stack_tools()
- json_payload["configurations"]["cluster-env"]["stack_features"] =
self.get_stack_features()
- json_payload["upgradeSummary"] =
TestStackSelectSetAll._get_upgrade_summary_no_downgrade()["upgradeSummary"]
-
- config_dict = ConfigDictionary(json_payload)
-
- def hdp_select_call(command, **kwargs):
- # return no versions
- if "versions" in command:
- return (0,"2.5.9.9-9999")
-
- return (0,command)
-
- family_mock.return_value = True
- get_config_mock.return_value = config_dict
- call_mock.side_effect = hdp_select_call
- exists_mock.return_value = True
-
- # Ensure that the json file was actually read.
- stack_name = default("/clusterLevelParams/stack_name", None)
- stack_version = default("/clusterLevelParams/stack_version", None)
- service_package_folder = default('/roleParams/service_package_folder',
None)
-
- self.assertEqual(stack_name, "HDP")
- self.assertEqual(stack_version, '2.2')
- self.assertEqual(service_package_folder,
"common-services/HDFS/2.1.0.2.0/package")
-
- # Begin the test
- ru_execute = UpgradeSetAll()
- ru_execute.actionexecute(None)
-
- call_mock.assert_called_with(('ambari-python-wrap', '/usr/bin/hdp-select',
'set', 'all', u'2.5.9.9-9999'), sudo=True)
-
-
- @patch("os.path.exists")
- @patch("resource_management.core.shell.call")
- @patch.object(Script, 'get_config')
- @patch.object(OSCheck, 'is_redhat_family')
- def test_skippable_hosts(self, family_mock, get_config_mock, call_mock,
exists_mock):
- """
- Tests that hosts are skippable if they don't have stack components
installed
- :return:
- """
- # Mock the config objects
- json_file_path = os.path.join(self.get_custom_actions_dir(),
- "ru_execute_tasks_namenode_prepare.json")
- self.assertTrue(os.path.isfile(json_file_path))
-
- with open(json_file_path, "r") as json_file:
- json_payload = json.load(json_file)
-
- json_payload["configurations"]["cluster-env"]["stack_tools"] =
self.get_stack_tools()
- json_payload["configurations"]["cluster-env"]["stack_features"] =
self.get_stack_features()
- json_payload["upgradeSummary"] =
TestStackSelectSetAll._get_upgrade_summary_no_downgrade()["upgradeSummary"]
-
- config_dict = ConfigDictionary(json_payload)
-
- family_mock.return_value = False
- get_config_mock.return_value = config_dict
- exists_mock.return_value = True
-
- def hdp_select_call(command, **kwargs):
- # return no versions
- if "versions" in command:
- return (0,"")
-
- return (0,command)
-
- call_mock.side_effect = hdp_select_call
-
- # Ensure that the json file was actually read.
- stack_name = default("/clusterLevelParams/stack_name", None)
- stack_version = default("/clusterLevelParams/stack_version", None)
- service_package_folder = default('/roleParams/service_package_folder',
None)
-
- self.assertEqual(stack_name, "HDP")
- self.assertEqual(stack_version, '2.2')
- self.assertEqual(service_package_folder,
"common-services/HDFS/2.1.0.2.0/package")
-
- # Begin the test
- ru_execute = UpgradeSetAll()
- ru_execute.actionexecute(None)
-
- call_mock.assert_called_with(('ambari-python-wrap',
u'/usr/bin/hdp-select', 'versions'), sudo = True)
- self.assertEqual(call_mock.call_count, 1)
-
- @patch("os.path.exists")
- @patch("resource_management.core.shell.call")
- @patch.object(Script, 'get_config')
- @patch.object(OSCheck, 'is_redhat_family')
- def test_skippable_by_list(self, family_mock, get_config_mock, call_mock,
exists_mock):
- """
- Tests that hosts are skippable if they don't have stack components
installed
- :return:
- """
- # Mock the config objects
- json_file_path = os.path.join(self.get_custom_actions_dir(),
- "ru_execute_tasks_namenode_prepare.json")
- self.assertTrue(os.path.isfile(json_file_path))
-
- with open(json_file_path, "r") as json_file:
- json_payload = json.load(json_file)
-
- json_payload["configurations"]["cluster-env"]["stack_tools"] =
self.get_stack_tools()
- json_payload["configurations"]["cluster-env"]["stack_features"] =
self.get_stack_features()
- json_payload["upgradeSummary"] =
TestStackSelectSetAll._get_upgrade_summary_no_downgrade()["upgradeSummary"]
-
- config_dict = ConfigDictionary(json_payload)
-
- family_mock.return_value = False
- get_config_mock.return_value = config_dict
- exists_mock.return_value = True
-
- def hdp_select_call(command, **kwargs):
- # return no versions
- if "versions" in command:
- return (0,"2.6.7-123")
-
- return (0,command)
-
- call_mock.side_effect = hdp_select_call
-
- # Ensure that the json file was actually read.
- stack_name = default("/clusterLevelParams/stack_name", None)
- stack_version = default("/clusterLevelParams/stack_version", None)
- service_package_folder = default('/roleParams/service_package_folder',
None)
-
- self.assertEqual(stack_name, "HDP")
- self.assertEqual(stack_version, '2.2')
- self.assertEqual(service_package_folder,
"common-services/HDFS/2.1.0.2.0/package")
-
- # Begin the test
- ru_execute = UpgradeSetAll()
- ru_execute.actionexecute(None)
-
- call_mock.assert_called_with(('ambari-python-wrap',
u'/usr/bin/hdp-select', 'versions'), sudo = True)
- self.assertEqual(call_mock.call_count, 1)
-
- @patch("os.path.exists")
- @patch("resource_management.core.shell.call")
- @patch.object(Script, 'get_config')
- @patch.object(OSCheck, 'is_redhat_family')
- def test_execution_with_downgrade_allowed(self, family_mock,
get_config_mock, call_mock, exists_mock):
- # Mock the config objects
- json_file_path = os.path.join(self.get_custom_actions_dir(),
"ru_execute_tasks_namenode_prepare.json")
- self.assertTrue(os.path.isfile(json_file_path))
-
- with open(json_file_path, "r") as json_file:
- json_payload = json.load(json_file)
-
- json_payload["configurations"]["cluster-env"]["stack_tools"] =
self.get_stack_tools()
- json_payload["configurations"]["cluster-env"]["stack_features"] =
self.get_stack_features()
- json_payload["upgradeSummary"] =
TestStackSelectSetAll._get_upgrade_summary_downgrade_allowed()["upgradeSummary"]
-
- config_dict = ConfigDictionary(json_payload)
-
- family_mock.return_value = True
- get_config_mock.return_value = config_dict
- call_mock.side_effect = fake_call # echo the command
- exists_mock.return_value = True
-
- # Ensure that the json file was actually read.
- stack_name = default("/clusterLevelParams/stack_name", None)
- stack_version = default("/clusterLevelParams/stack_version", None)
- service_package_folder = default('/roleParams/service_package_folder',
None)
-
- self.assertEqual(stack_name, "HDP")
- self.assertEqual(stack_version, '2.2')
- self.assertEqual(service_package_folder,
"common-services/HDFS/2.1.0.2.0/package")
-
- # Begin the test
- ru_execute = UpgradeSetAll()
- ru_execute.actionexecute(None)
-
- call_mock.assert_not_called()
-
-
- @patch("os.path.exists")
- @patch("resource_management.core.shell.call")
- @patch.object(Script, 'get_config')
- @patch.object(OSCheck, 'is_redhat_family')
- def test_execution_with_patch_upgrade(self, family_mock, get_config_mock,
call_mock, exists_mock):
- # Mock the config objects
- json_file_path = os.path.join(self.get_custom_actions_dir(),
"ru_execute_tasks_namenode_prepare.json")
- self.assertTrue(os.path.isfile(json_file_path))
-
- with open(json_file_path, "r") as json_file:
- json_payload = json.load(json_file)
-
- json_payload["configurations"]["cluster-env"]["stack_tools"] =
self.get_stack_tools()
- json_payload["configurations"]["cluster-env"]["stack_features"] =
self.get_stack_features()
- json_payload["upgradeSummary"] =
TestStackSelectSetAll._get_upgrade_summary_patch_upgrade()["upgradeSummary"]
-
- config_dict = ConfigDictionary(json_payload)
-
- family_mock.return_value = True
- get_config_mock.return_value = config_dict
- call_mock.side_effect = fake_call # echo the command
- exists_mock.return_value = True
-
- # Ensure that the json file was actually read.
- stack_name = default("/clusterLevelParams/stack_name", None)
- stack_version = default("/clusterLevelParams/stack_version", None)
- service_package_folder = default('/roleParams/service_package_folder',
None)
-
- self.assertEqual(stack_name, "HDP")
- self.assertEqual(stack_version, '2.2')
- self.assertEqual(service_package_folder,
"common-services/HDFS/2.1.0.2.0/package")
-
- # Begin the test
- ru_execute = UpgradeSetAll()
- ru_execute.actionexecute(None)
-
- call_mock.assert_not_called()
-
-
- @staticmethod
- def _get_upgrade_summary_no_downgrade():
- """
- A STANDARD UPGRADE that cannot be downgraded
- :return:
- """
- return {
- "upgradeSummary": {
- "services":{
- "HDFS":{
- "sourceRepositoryId":1,
- "sourceStackId":"HDP-2.4",
- "sourceVersion":"2.4.0.0-1234",
- "targetRepositoryId":2,
- "targetStackId":"HDP-2.5",
- "targetVersion":"2.5.9.9-9999"
- }
- },
- "direction":"UPGRADE",
- "type":"rolling_upgrade",
- "isRevert":False,
- "orchestration":"STANDARD",
- "associatedStack": "HDP-2.5",
- "associatedVersion":"2.5.9.9-9999",
- "isDowngradeAllowed": False,
- "isSwitchBits": False
- }
- }
-
- @staticmethod
- def _get_upgrade_summary_downgrade_allowed():
- """
- A STANDARD UPGRADE that can be downgraded
- :return:
- """
- upgrade_summary = TestStackSelectSetAll._get_upgrade_summary_no_downgrade()
- upgrade_summary["upgradeSummary"]["isDowngradeAllowed"] = True
- return upgrade_summary
-
-
- @staticmethod
- def _get_upgrade_summary_patch_upgrade():
- """
- A STANDARD UPGRADE that can be downgraded
- :return:
- """
- upgrade_summary = TestStackSelectSetAll._get_upgrade_summary_no_downgrade()
- upgrade_summary["upgradeSummary"]["orchestration"] = "PATCH"
- return upgrade_summary
\ No newline at end of file
