This is an automated email from the ASF dual-hosted git repository. adoroszlai pushed a commit to branch branch-feature-AMBARI-14714 in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/branch-feature-AMBARI-14714 by this push: new 1d24700 AMBARI-24217. Blueprint deployment with custom service name (#1692) 1d24700 is described below commit 1d24700208c5dfe430e601b2cf47eb19a14889d7 Author: Doroszlai, Attila <6454655+adorosz...@users.noreply.github.com> AuthorDate: Wed Jul 11 18:43:44 2018 +0200 AMBARI-24217. Blueprint deployment with custom service name (#1692) --- .../execution_command/execution_command.py | 14 +- .../apache/ambari/server/actionmanager/Stage.java | 3 +- .../ambari/server/agent/ExecutionCommand.java | 25 ++-- .../ambari/server/agent/stomp/MetadataHolder.java | 13 +- .../ambari/server/agent/stomp/TopologyHolder.java | 50 ++----- .../agent/stomp/dto/MetadataServiceInfo.java | 40 ++--- .../server/agent/stomp/dto/TopologyComponent.java | 16 +- .../AmbariCustomCommandExecutionHelper.java | 4 +- .../controller/AmbariManagementControllerImpl.java | 98 +++++------- .../server/controller/ConfigurationRequest.java | 5 +- .../internal/BlueprintConfigurationProcessor.java | 16 +- .../server/metadata/ClusterMetadataGenerator.java | 23 +-- .../ambari/server/state/cluster/ClusterImpl.java | 4 + .../ambari/server/topology/AmbariContext.java | 2 +- .../topology/ClusterConfigurationRequest.java | 17 +-- .../ambari/server/topology/ClusterTopology.java | 6 +- .../server/topology/ClusterTopologyImpl.java | 24 +-- .../ambari/server/topology/ResolvedComponent.java | 7 + .../server/topology/ResolvedComponent_Builder.java | 6 +- .../validators/ClusterConfigTypeValidator.java | 2 +- .../topology/validators/HiveServiceValidator.java | 2 +- .../RequiredConfigPropertiesValidator.java | 2 +- .../server/agent/HeartbeatProcessorTest.java | 4 +- .../ambari/server/agent/HeartbeatTestHelper.java | 4 - .../agent/stomp/dto/MetadataClusterTest.java | 34 ++--- .../ambari/server/api/query/QueryImplTest.java | 1 - .../StackAdvisorBlueprintProcessorTest.java | 12 +- .../controller/AmbariManagementControllerTest.java | 165 ++++++++++----------- .../ambari/server/topology/AmbariContextTest.java | 2 +- .../topology/ClusterConfigurationRequestTest.java | 10 +- .../topology/ClusterDeploymentTestCommon.java | 2 +- .../ambari/server/topology/StackBuilder.java | 16 ++ .../server/topology/TopologyManagerTest.java | 4 +- .../validators/ClusterConfigTypeValidatorTest.java | 6 +- .../DependencyAndCardinalityValidatorTest.java | 17 ++- .../validators/HiveServiceValidatorTest.java | 4 +- .../RequiredConfigPropertiesValidatorTest.java | 2 +- .../topology/validators/TopologyValidatorTest.java | 3 +- 38 files changed, 328 insertions(+), 337 deletions(-) diff --git a/ambari-common/src/main/python/resource_management/libraries/execution_command/execution_command.py b/ambari-common/src/main/python/resource_management/libraries/execution_command/execution_command.py index 72f7c21..c147615 100644 --- a/ambari-common/src/main/python/resource_management/libraries/execution_command/execution_command.py +++ b/ambari-common/src/main/python/resource_management/libraries/execution_command/execution_command.py @@ -79,10 +79,10 @@ class ExecutionCommand(object): def get_module_name(self): """ - Retrieve service name from command.json, i.e 'zookeeper', 'hdfs' - :return: service name + Retrieve service type from command.json, eg. 'zookeeper', 'hdfs' + :return: service type """ - return self.__get_value("serviceName") + return self.__get_value("serviceType") def get_component_type(self): """ @@ -93,10 +93,12 @@ class ExecutionCommand(object): def get_component_instance_name(self): """ - At this time it returns hardcoded 'default' name - :return: 'default' string + Retrieve service name from command.json, eg. 'zk1' + :return: service name """ - return "default" + if '_CLIENTS' in self.get_module_name(): # FIXME temporary hack + return 'default' + return self.__get_value("serviceName") # multi-service, but not multi-component per service def get_servicegroup_name(self): """ diff --git a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/Stage.java b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/Stage.java index 7cdd401..5a71bfa 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/Stage.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/Stage.java @@ -419,7 +419,7 @@ public class Stage { */ public synchronized void addHostRoleExecutionCommand(Host host, Role role, RoleCommand command, ServiceComponentHostEvent event, Cluster cluster, Long mpackId, String serviceGroupName, - String serviceName, boolean retryAllowed, + String serviceType, String serviceName, boolean retryAllowed, boolean autoSkipFailure) { boolean isHostRoleCommandAutoSkippable = autoSkipFailure && supportsAutoSkipOnFailure @@ -431,6 +431,7 @@ public class Stage { commandWrapper.getExecutionCommand().setMpackId(mpackId); commandWrapper.getExecutionCommand().setServiceGroupName(serviceGroupName); commandWrapper.getExecutionCommand().setServiceName(serviceName); + commandWrapper.getExecutionCommand().setServiceType(serviceType); } /** diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java index 1009709..922b006 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java @@ -37,6 +37,7 @@ import org.slf4j.LoggerFactory; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; import com.google.gson.annotations.SerializedName; @@ -53,11 +54,11 @@ public class ExecutionCommand extends AgentCommand { super(AgentCommandType.EXECUTION_COMMAND); } - @com.fasterxml.jackson.annotation.JsonProperty("clusterId") + @JsonProperty("clusterId") private String clusterId; @SerializedName("clusterName") - @com.fasterxml.jackson.annotation.JsonProperty("clusterName") + @JsonProperty("clusterName") private String clusterName; @SerializedName("clusterSettings") @@ -67,7 +68,7 @@ public class ExecutionCommand extends AgentCommand { private Map<String, String> stackSettings; @SerializedName("requestId") - @com.fasterxml.jackson.annotation.JsonProperty("requestId") + @JsonProperty("requestId") private long requestId; @SerializedName("stageId") @@ -75,11 +76,11 @@ public class ExecutionCommand extends AgentCommand { private long stageId; @SerializedName("taskId") - @com.fasterxml.jackson.annotation.JsonProperty("taskId") + @JsonProperty("taskId") private long taskId; @SerializedName("commandId") - @com.fasterxml.jackson.annotation.JsonProperty("commandId") + @JsonProperty("commandId") private String commandId; @SerializedName("hostname") @@ -87,7 +88,7 @@ public class ExecutionCommand extends AgentCommand { private String hostname; @SerializedName("role") - @com.fasterxml.jackson.annotation.JsonProperty("role") + @JsonProperty("role") private String role; @SerializedName("hostLevelParams") @@ -95,11 +96,11 @@ public class ExecutionCommand extends AgentCommand { private Map<String, String> hostLevelParams = new HashMap<>(); @SerializedName("roleParams") - @com.fasterxml.jackson.annotation.JsonProperty("roleParams") + @JsonProperty("roleParams") private Map<String, String> roleParams = null; @SerializedName("roleCommand") - @com.fasterxml.jackson.annotation.JsonProperty("roleCommand") + @JsonProperty("roleCommand") private RoleCommand roleCommand; @SerializedName("clusterHostInfo") @@ -123,7 +124,7 @@ public class ExecutionCommand extends AgentCommand { private boolean forceRefreshConfigTagsBeforeExecution = false; @SerializedName("commandParams") - @com.fasterxml.jackson.annotation.JsonProperty("commandParams") + @JsonProperty("commandParams") private Map<String, String> commandParams = new HashMap<>(); /** @@ -136,11 +137,11 @@ public class ExecutionCommand extends AgentCommand { private String serviceGroupName; @SerializedName("serviceName") - @com.fasterxml.jackson.annotation.JsonProperty("serviceName") + @JsonProperty("serviceName") private String serviceName; @SerializedName("serviceType") - @JsonIgnore + @JsonProperty("serviceType") private String serviceType; @SerializedName("componentName") @@ -148,7 +149,7 @@ public class ExecutionCommand extends AgentCommand { private String componentName; @SerializedName("kerberosCommandParams") - @com.fasterxml.jackson.annotation.JsonProperty("kerberosCommandParams") + @JsonProperty("kerberosCommandParams") private List<Map<String, String>> kerberosCommandParams = new ArrayList<>(); @SerializedName("localComponents") diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/stomp/MetadataHolder.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/stomp/MetadataHolder.java index 47ba8e6..cc128d8 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/agent/stomp/MetadataHolder.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/stomp/MetadataHolder.java @@ -124,9 +124,9 @@ public class MetadataHolder extends AgentClusterDataHolder<MetadataUpdateEvent> } @Subscribe - public void onServiceCreate(ServiceInstalledEvent serviceInstalledEvent) throws AmbariException { - Cluster cluster = m_clusters.get().getCluster(serviceInstalledEvent.getClusterId()); - updateData(metadataGenerator.getClusterMetadataOnServiceInstall(cluster, serviceInstalledEvent.getServiceName())); + public void onServiceCreate(ServiceInstalledEvent event) throws AmbariException { + Cluster cluster = m_clusters.get().getCluster(event.getClusterId()); + updateData(metadataGenerator.getClusterMetadataOnServiceInstall(cluster, event.getServiceGroupName(), event.getServiceName())); } @Subscribe @@ -136,10 +136,9 @@ public class MetadataHolder extends AgentClusterDataHolder<MetadataUpdateEvent> } @Subscribe - public void onServiceCredentialStoreUpdate(ServiceCredentialStoreUpdateEvent serviceCredentialStoreUpdateEvent) throws AmbariException { - Cluster cluster = m_clusters.get().getCluster(serviceCredentialStoreUpdateEvent.getClusterId()); - //TODO : [AMP] Fix it - // updateData(ambariManagementController.getClusterMetadataOnServiceCredentialStoreUpdate(cluster, serviceCredentialStoreUpdateEvent.getServiceName())); + public void onServiceCredentialStoreUpdate(ServiceCredentialStoreUpdateEvent event) throws AmbariException { + Cluster cluster = m_clusters.get().getCluster(event.getClusterId()); + updateData(metadataGenerator.getClusterMetadataOnServiceCredentialStoreUpdate(cluster, event.getServiceGroupName(), event.getServiceName())); } @Subscribe diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/stomp/TopologyHolder.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/stomp/TopologyHolder.java index 1939d92..1cb3bf3 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/agent/stomp/TopologyHolder.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/stomp/TopologyHolder.java @@ -22,7 +22,6 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; import java.util.TreeMap; -import java.util.stream.Collectors; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.ClusterNotFoundException; @@ -41,11 +40,11 @@ import org.apache.ambari.server.state.Host; import org.apache.ambari.server.state.Service; import org.apache.ambari.server.state.ServiceComponent; import org.apache.ambari.server.state.ServiceComponentHost; -import org.apache.ambari.server.state.StackId; import org.apache.commons.collections.CollectionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.collect.ImmutableSet; import com.google.common.eventbus.Subscribe; import com.google.inject.Inject; import com.google.inject.Singleton; @@ -91,28 +90,15 @@ public class TopologyHolder extends AgentClusterDataHolder<TopologyUpdateEvent> for (Service service : cl.getServices()) { for (ServiceComponent component : service.getServiceComponents().values()) { Map<String, ServiceComponentHost> componentsMap = component.getServiceComponentHosts(); - if (!componentsMap.isEmpty()) { - - //TODO will be a need to change to multi-instance usage - ServiceComponentHost sch = componentsMap.entrySet().iterator().next().getValue(); - - Set<String> hostNames = cl.getHosts(sch.getServiceName(), sch.getServiceComponentName()); - Set<Long> hostOrderIds = clusterHosts.stream() - .filter(h -> hostNames.contains(h.getHostName())) - .map(Host::getHostId) - .collect(Collectors.toSet()); - String serviceName = sch.getServiceName(); - String componentName = sch.getServiceComponentName(); - StackId stackId = cl.getDesiredStackVersion(); - + for (ServiceComponentHost sch : componentsMap.values()) { TopologyComponent topologyComponent = TopologyComponent.newBuilder() - .setComponentName(sch.getServiceComponentName()) - .setServiceName(sch.getServiceName()) - .setHostIds(hostOrderIds) - .setComponentLevelParams(ambariManagementController.getTopologyComponentLevelParams(cl.getClusterId(), serviceName, - componentName, cl.getSecurityType())) - .setCommandParams(ambariManagementController.getTopologyCommandParams(cl.getClusterId(), serviceName, componentName, sch)) - .build(); + .setComponentName(sch.getServiceComponentName()) + .setServiceName(sch.getServiceName()) + .setVersion(sch.getVersion()) + .setHostIds(ImmutableSet.of(sch.getHost().getHostId())) + .setComponentLevelParams(ambariManagementController.getTopologyComponentLevelParams(sch)) + .setCommandParams(ambariManagementController.getTopologyCommandParams(sch)) + .build(); topologyComponents.add(topologyComponent); } } @@ -220,20 +206,16 @@ public class TopologyHolder extends AgentClusterDataHolder<TopologyUpdateEvent> Map<String, ServiceComponentHost> componentsMap = component.getServiceComponentHosts(); if (!componentsMap.isEmpty()) { - //TODO will be a need to change to multi-instance usage - ServiceComponentHost sch = componentsMap.entrySet().iterator().next().getValue(); - - String serviceName = sch.getServiceName(); - String componentName = sch.getServiceComponentName(); - - TopologyComponent topologyComponent = TopologyComponent.newBuilder() + for (ServiceComponentHost sch : componentsMap.values()) { + TopologyComponent topologyComponent = TopologyComponent.newBuilder() .setComponentName(sch.getServiceComponentName()) .setServiceName(sch.getServiceName()) - .setCommandParams(ambariManagementController.getTopologyCommandParams(cl.getClusterId(), serviceName, componentName, sch)) - .setComponentLevelParams(ambariManagementController.getTopologyComponentLevelParams(clusterId, - serviceName, componentName, cl.getSecurityType())) + .setVersion(sch.getVersion()) + .setCommandParams(ambariManagementController.getTopologyCommandParams(sch)) + .setComponentLevelParams(ambariManagementController.getTopologyComponentLevelParams(sch)) .build(); - topologyComponents.add(topologyComponent); + topologyComponents.add(topologyComponent); + } } } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/stomp/dto/MetadataServiceInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/stomp/dto/MetadataServiceInfo.java index d0d8374..4733908 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/agent/stomp/dto/MetadataServiceInfo.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/stomp/dto/MetadataServiceInfo.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -19,11 +19,13 @@ package org.apache.ambari.server.agent.stomp.dto; import java.util.Map; +import java.util.Objects; import com.fasterxml.jackson.annotation.JsonProperty; public class MetadataServiceInfo { + private String serviceType; private String version; private Boolean credentialStoreEnabled; @@ -36,9 +38,10 @@ public class MetadataServiceInfo { @JsonProperty("service_package_folder") private String servicePackageFolder; - public MetadataServiceInfo(String version, Boolean credentialStoreEnabled, + public MetadataServiceInfo(String serviceType, String version, Boolean credentialStoreEnabled, Map<String, Map<String, String>> credentialStoreEnabledProperties, Long statusCommandsTimeout, String servicePackageFolder) { + this.serviceType = serviceType; this.version = version; this.credentialStoreEnabled = credentialStoreEnabled; this.credentialStoreEnabledProperties = credentialStoreEnabledProperties; @@ -46,6 +49,10 @@ public class MetadataServiceInfo { this.servicePackageFolder = servicePackageFolder; } + public String getServiceType() { + return serviceType; + } + public String getVersion() { return version; } @@ -88,28 +95,25 @@ public class MetadataServiceInfo { @Override public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } MetadataServiceInfo that = (MetadataServiceInfo) o; - if (version != null ? !version.equals(that.version) : that.version != null) return false; - if (credentialStoreEnabled != null ? !credentialStoreEnabled.equals(that.credentialStoreEnabled) : that.credentialStoreEnabled != null) - return false; - if (credentialStoreEnabledProperties != null ? !credentialStoreEnabledProperties.equals(that.credentialStoreEnabledProperties) : that.credentialStoreEnabledProperties != null) - return false; - if (statusCommandsTimeout != null ? !statusCommandsTimeout.equals(that.statusCommandsTimeout) : that.statusCommandsTimeout != null) - return false; - return servicePackageFolder != null ? servicePackageFolder.equals(that.servicePackageFolder) : that.servicePackageFolder == null; + return Objects.equals(serviceType, that.serviceType) && + Objects.equals(version, that.version) && + Objects.equals(credentialStoreEnabled, that.credentialStoreEnabled) && + Objects.equals(credentialStoreEnabledProperties, that.credentialStoreEnabledProperties) && + Objects.equals(statusCommandsTimeout, that.statusCommandsTimeout) && + Objects.equals(servicePackageFolder, that.servicePackageFolder); } @Override public int hashCode() { - int result = version != null ? version.hashCode() : 0; - result = 31 * result + (credentialStoreEnabled != null ? credentialStoreEnabled.hashCode() : 0); - result = 31 * result + (credentialStoreEnabledProperties != null ? credentialStoreEnabledProperties.hashCode() : 0); - result = 31 * result + (statusCommandsTimeout != null ? statusCommandsTimeout.hashCode() : 0); - result = 31 * result + (servicePackageFolder != null ? servicePackageFolder.hashCode() : 0); - return result; + return Objects.hash(serviceType, version, credentialStoreEnabled, credentialStoreEnabledProperties, statusCommandsTimeout, servicePackageFolder); } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/stomp/dto/TopologyComponent.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/stomp/dto/TopologyComponent.java index ef0871f..b3da819 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/agent/stomp/dto/TopologyComponent.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/stomp/dto/TopologyComponent.java @@ -34,7 +34,7 @@ import com.fasterxml.jackson.annotation.JsonInclude; public class TopologyComponent { private String componentName; private String serviceName; - private String displayName; + private String serviceType; private String version; private Set<Long> hostIds = new HashSet<>(); private Set<String> hostNames = new HashSet<>(); @@ -65,8 +65,8 @@ public class TopologyComponent { return this; } - public Builder setDisplayName(String displayName) { - TopologyComponent.this.setDisplayName(displayName); + public Builder setServiceType(String serviceType) { + TopologyComponent.this.setServiceType(serviceType); return this; } @@ -185,7 +185,7 @@ public class TopologyComponent { public TopologyComponent deepCopy() { return TopologyComponent.newBuilder().setComponentName(getComponentName()) - .setDisplayName(getDisplayName()) + .setServiceType(getServiceType()) .setServiceName(getServiceName()) .setComponentLevelParams(getComponentLevelParams() == null ? null : new TreeMap<>(getComponentLevelParams())) .setHostIds(getHostIds() == null ? null : new HashSet<>(getHostIds())) @@ -251,12 +251,12 @@ public class TopologyComponent { this.hostNames = hostNames; } - public String getDisplayName() { - return displayName; + public String getServiceType() { + return serviceType; } - public void setDisplayName(String displayName) { - this.displayName = displayName; + public void setServiceType(String serviceType) { + this.serviceType = serviceType; } public Set<String> getPublicHostNames() { diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java index cb1d8ef..c882173 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java @@ -352,7 +352,7 @@ public class AmbariCustomCommandExecutionHelper { stage.addHostRoleExecutionCommand(host, Role.valueOf(componentName), RoleCommand.CUSTOM_COMMAND, new ServiceComponentHostOpInProgressEvent(componentName, hostName, nowTimestamp), - cluster, serviceGroup.getMpackId(), serviceGroupName, serviceName, retryAllowed, + cluster, serviceGroup.getMpackId(), serviceGroupName, serviceInfo.getName(), serviceName, retryAllowed, autoSkipFailure); Map<String, Map<String, String>> configurations = @@ -855,7 +855,7 @@ public class AmbariCustomCommandExecutionHelper { stage.addHostRoleExecutionCommand(host, Role.valueOf(smokeTestRole), RoleCommand.SERVICE_CHECK, new ServiceComponentHostOpInProgressEvent(componentName, hostname, nowTimestamp), - cluster, serviceGroup.getMpackId(), serviceGroupName, serviceName, retryAllowed, + cluster, serviceGroup.getMpackId(), serviceGroupName, serviceInfo.getName(), serviceName, retryAllowed, autoSkipFailure); HostRoleCommand hrc = stage.getHostRoleCommand(hostname, smokeTestRole); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java index dfdfd74..c3773cc 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java @@ -827,15 +827,14 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle TopologyComponent newComponent = TopologyComponent.newBuilder() .setComponentName(sch.getServiceComponentName()) - .setServiceName(sch.getServiceType()) - .setDisplayName(sc.getServiceName()) + .setServiceName(sc.getServiceName()) + .setServiceType(sc.getServiceType()) .setVersion(sch.getVersion()) .setHostIds(hostIds) .setHostNames(hostNames) .setPublicHostNames(publicHostNames) - .setComponentLevelParams(getTopologyComponentLevelParams(cluster.getClusterId(), serviceName, componentName, - cluster.getSecurityType())) - .setCommandParams(getTopologyCommandParams(cluster.getClusterId(), serviceName, componentName, sch)) + .setComponentLevelParams(getTopologyComponentLevelParams(sch)) + .setCommandParams(getTopologyCommandParams(sch)) .build(); String clusterId = Long.toString(cluster.getClusterId()); @@ -977,7 +976,9 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle // If the config type is for a service, then allow a user with SERVICE_MODIFY_CONFIGS to // update, else ensure the user has CLUSTER_MODIFY_CONFIGS - Service service = cluster.getServiceByConfigType(configType); + Service service = request.getServiceId() != null + ? cluster.getService(request.getServiceId()) + : cluster.getServiceByConfigType(configType); // Get the changes so that the user's intention can be determined. For example, maybe @@ -1044,10 +1045,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle } } - - - Map<String, Config> configs = cluster.getConfigsByType( - request.getType()); + Map<String, Config> configs = cluster.getConfigsByServiceIdType(request.getType(), request.getServiceId()); if (null == configs) { configs = new HashMap<>(); } @@ -1093,7 +1091,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle } Config config = createConfig(cluster, stackId, request.getType(), requestProperties, - request.getVersionTag(), propertiesAttributes, request.getServiceId() == null? null : request.getServiceId()); + request.getVersionTag(), propertiesAttributes, request.getServiceId()); LOG.info(MessageFormat.format("Creating configuration with tag ''{0}'' to cluster ''{1}'' for configuration type {2}", request.getVersionTag(), @@ -1602,39 +1600,18 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle Map<String, Config> configs = null; //Get by type if (null != request.getType()) { - //TODO : Remove after getting rid of cluster configurations - if (request.getServiceId() != null) { - configs = cluster.getConfigsByServiceIdType( - request.getType(), request.getServiceId()); - if (null != configs) { - for (Entry<String, Config> entry : configs.entrySet()) { - Config config = entry.getValue(); - response = new ConfigurationResponse( - cluster.getClusterName(), config.getStackId(), - request.getType(), - config.getTag(), entry.getValue().getVersion(), - includeProps ? config.getProperties() : new HashMap<>(), - includeProps ? config.getPropertiesAttributes() : new HashMap<>(), - config.getPropertiesTypes(), request.getServiceId(), request.getServiceGroupId()); - responses.add(response); - } - } - } - if (responses == null || responses.isEmpty()) { - configs = cluster.getConfigsByType( - request.getType()); - if (null != configs) { - for (Entry<String, Config> entry : configs.entrySet()) { - Config config = entry.getValue(); - response = new ConfigurationResponse( - cluster.getClusterName(), config.getStackId(), - request.getType(), - config.getTag(), entry.getValue().getVersion(), - includeProps ? config.getProperties() : new HashMap<>(), - includeProps ? config.getPropertiesAttributes() : new HashMap<>(), - config.getPropertiesTypes()); - responses.add(response); - } + configs = cluster.getConfigsByServiceIdType(request.getType(), request.getServiceId()); + if (null != configs) { + for (Entry<String, Config> entry : configs.entrySet()) { + Config config = entry.getValue(); + response = new ConfigurationResponse( + cluster.getClusterName(), config.getStackId(), + request.getType(), + config.getTag(), entry.getValue().getVersion(), + includeProps ? config.getProperties() : new HashMap<>(), + includeProps ? config.getPropertiesAttributes() : new HashMap<>(), + config.getPropertiesTypes(), request.getServiceId(), request.getServiceGroupId()); + responses.add(response); } } } else { @@ -1871,7 +1848,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle if (null != cr.getProperties()) { // !!! empty property sets are supported, and need to be able to use // previously-defined configs (revert) - Map<String, Config> all = cluster.getConfigsByType(configType); + Map<String, Config> all = cluster.getConfigsByServiceIdType(configType, cr.getServiceId()); if (null == all || // none set !all.containsKey(cr.getVersionTag()) || // tag not set cr.getProperties().size() > 0) { // properties to set @@ -2523,7 +2500,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle stage.addHostRoleExecutionCommand(scHost.getHost(), Role.valueOf(scHost.getServiceComponentName()), roleCommand, event, cluster, mpackId, - serviceGroupName, serviceName, false, skipFailure); + serviceGroupName, scHost.getServiceType(), serviceName, false, skipFailure); String componentName = scHost.getServiceComponentName(); String hostname = scHost.getHostName(); @@ -5510,7 +5487,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle if (null != cr.getProperties()) { // !!! empty property sets are supported, and need to be able to use // previously-defined configs (revert) - Map<String, Config> all = cluster.getConfigsByType(configType); + Map<String, Config> all = cluster.getConfigsByServiceIdType(configType, cr.getServiceId()); if (null == all || // none set !all.containsKey(cr.getVersionTag()) || // tag not set cr.getProperties().size() > 0) { // properties to set @@ -5618,16 +5595,15 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle } //TODO will be a need to change to multi-instance usage - public TreeMap<String, String> getTopologyComponentLevelParams(Long clusterId, String serviceName, String componentName, - SecurityType clusterSecurityType) throws AmbariException { + public TreeMap<String, String> getTopologyComponentLevelParams(ServiceComponentHost sch) throws AmbariException { TreeMap<String, String> statusCommandParams = new TreeMap<>(); - Cluster cluster = clusters.getCluster(clusterId); - Service service = cluster.getService(serviceName); + Cluster cluster = clusters.getCluster(sch.getClusterId()); + Service service = cluster.getService(sch.getServiceId()); StackId stackId = service.getStackId(); ComponentInfo componentInfo = ambariMetaInfo.getComponent(stackId.getStackName(), - stackId.getStackVersion(), serviceName, componentName); + stackId.getStackVersion(), sch.getServiceType(), sch.getServiceComponentName()); statusCommandParams.put(ExecutionCommand.KeyNames.CLIENTS_TO_UPDATE_CONFIGS, getClientsToUpdateConfigs(componentInfo)); @@ -5641,22 +5617,20 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle // // Set/update the unlimited_key_jce_required value as needed statusCommandParams.put(ExecutionCommand.KeyNames.UNLIMITED_KEY_JCE_REQUIRED, - Boolean.toString(getUnlimitedKeyJCERequirement(componentInfo, clusterSecurityType))); + Boolean.toString(getUnlimitedKeyJCERequirement(componentInfo, cluster.getSecurityType()))); return statusCommandParams; } //TODO will be a need to change to multi-instance usage - public TreeMap<String, String> getTopologyCommandParams(Long clusterId, String serviceName, String componentName, ServiceComponentHost sch) throws AmbariException { + public TreeMap<String, String> getTopologyCommandParams(ServiceComponentHost sch) throws AmbariException { TreeMap<String, String> commandParams = new TreeMap<>(); - Cluster cluster = clusters.getCluster(clusterId); - Service service = cluster.getService(serviceName); + Cluster cluster = clusters.getCluster(sch.getClusterId()); + Service service = cluster.getService(sch.getServiceId()); StackId stackId = service.getStackId(); - ServiceInfo serviceInfo = ambariMetaInfo.getService(stackId.getStackName(), - stackId.getStackVersion(), serviceName); - ComponentInfo componentInfo = ambariMetaInfo.getComponent(stackId.getStackName(), - stackId.getStackVersion(), serviceName, componentName); + ServiceInfo serviceInfo = ambariMetaInfo.getService(stackId.getStackName(), stackId.getStackVersion(), sch.getServiceType()); + ComponentInfo componentInfo = ambariMetaInfo.getComponent(stackId.getStackName(), stackId.getStackVersion(), sch.getServiceType(), sch.getServiceComponentName()); commandParams.put(SERVICE_PACKAGE_FOLDER, serviceInfo.getServicePackageFolder()); String scriptName = null; @@ -5669,9 +5643,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle scriptCommandTimeout = String.valueOf(script.getTimeout()); } } else { - String message = String.format( - "Component %s of service %s has not " + "command script defined", componentName, - serviceName); + String message = String.format("Component %s of service %s has no command script defined", componentInfo.getName(), serviceInfo.getName()); throw new AmbariException(message); } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/ConfigurationRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/ConfigurationRequest.java index e5b390f..81d1a02 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/ConfigurationRequest.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/ConfigurationRequest.java @@ -58,11 +58,10 @@ public class ConfigurationRequest { this.configs = configs; this.type = type; this.tag = tag; - this.configs = configs; this.configsAttributes = configsAttributes; this.includeProperties = (type != null && tag != null); - this.serviceId = serviceId == null ? null : serviceId; - this.serviceGroupId = serviceGroupId == null ? null : serviceGroupId; + this.serviceId = serviceId; + this.serviceGroupId = serviceGroupId; } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java index ba7a043..50ae6fd 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java @@ -958,7 +958,7 @@ public class BlueprintConfigurationProcessor { * @param advisedConfigurations advised configuration instance */ private void doFilterStackDefaults(Map<String, AdvisedConfiguration> advisedConfigurations) { - Configuration stackDefaults = clusterTopology.getStack().getConfiguration(clusterTopology.getServices()); + Configuration stackDefaults = clusterTopology.getStack().getConfiguration(clusterTopology.getServiceTypes()); Map<String, Map<String, String>> stackDefaultProps = stackDefaults.getProperties(); for (Map.Entry<String, AdvisedConfiguration> adConfEntry : advisedConfigurations.entrySet()) { AdvisedConfiguration advisedConfiguration = adConfEntry.getValue(); @@ -2988,7 +2988,7 @@ public class BlueprintConfigurationProcessor { } } - boolean isAtlasInCluster = topology.getServices().contains("ATLAS"); + boolean isAtlasInCluster = topology.getServiceTypes().contains("ATLAS"); boolean isAtlasHiveHookEnabled = Boolean.parseBoolean(properties.get("hive-env").get("hive.atlas.hook")); // Append atlas hook if not already present. @@ -3019,7 +3019,7 @@ public class BlueprintConfigurationProcessor { Map<String, Map<String, String>> properties, ClusterTopology topology) { - if (topology.getServices().contains("ATLAS")) { + if (topology.getServiceTypes().contains("ATLAS")) { // if original value is not set or is the default "primary" set the cluster id if (origValue == null || origValue.trim().isEmpty() || origValue.equals("primary")) { //use cluster id because cluster name may change @@ -3054,7 +3054,7 @@ public class BlueprintConfigurationProcessor { String origValue, Map<String, Map<String, String>> properties, ClusterTopology topology) { - if (topology.getServices().contains("ATLAS")) { + if (topology.getServiceTypes().contains("ATLAS")) { String host = topology.getHostAssignmentsForComponent("ATLAS_SERVER").iterator().next(); boolean tlsEnabled = Boolean.parseBoolean(properties.get("application-properties").get("atlas.enableTLS")); @@ -3111,7 +3111,7 @@ public class BlueprintConfigurationProcessor { Map<String, Map<String, String>> properties, ClusterTopology topology) { - if (topology.getServices().contains("AMBARI_METRICS")) { + if (topology.getServiceTypes().contains("AMBARI_METRICS")) { final String amsReporterClass = "org.apache.hadoop.metrics2.sink.storm.StormTimelineMetricsReporter"; if (origValue == null || origValue.isEmpty()) { return amsReporterClass; @@ -3142,7 +3142,7 @@ public class BlueprintConfigurationProcessor { Map<String, Map<String, String>> properties, ClusterTopology topology) { - if (topology.getServices().contains("AMBARI_METRICS")) { + if (topology.getServiceTypes().contains("AMBARI_METRICS")) { final String amsReportesClass = "org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter"; if (origValue == null || origValue.isEmpty()) { return amsReportesClass; @@ -3234,7 +3234,7 @@ public class BlueprintConfigurationProcessor { // AMBARI-5206 final Map<String , String> userProps = new HashMap<>(); - Collection<String> services = clusterTopology.getServices(); + Collection<String> services = clusterTopology.getServiceTypes(); if (services.contains("HDFS")) { // only add user properties to the map for // services actually included in the blueprint definition @@ -3288,7 +3288,7 @@ public class BlueprintConfigurationProcessor { * @param stack */ private void addExcludedConfigProperties(Configuration configuration, Set<String> configTypesUpdated, StackDefinition stack) { - Collection<String> blueprintServices = clusterTopology.getServices(); + Collection<String> blueprintServices = clusterTopology.getServiceTypes(); LOG.debug("Handling excluded properties for blueprint services: {}", blueprintServices); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/metadata/ClusterMetadataGenerator.java b/ambari-server/src/main/java/org/apache/ambari/server/metadata/ClusterMetadataGenerator.java index d3acb35..52043f6 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/metadata/ClusterMetadataGenerator.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/metadata/ClusterMetadataGenerator.java @@ -176,13 +176,14 @@ public class ClusterMetadataGenerator { return new MetadataUpdateEvent(metadataClusters, null, getMetadataAgentConfigs(), UpdateEventType.UPDATE); } - public MetadataUpdateEvent getClusterMetadataOnServiceInstall(Cluster cl, String serviceName) throws AmbariException { - return getClusterMetadataOnServiceCredentialStoreUpdate(cl, serviceName); + public MetadataUpdateEvent getClusterMetadataOnServiceInstall(Cluster cl, String serviceGroupName, String serviceName) throws AmbariException { + return getClusterMetadataOnServiceCredentialStoreUpdate(cl, serviceGroupName, serviceName); } - public MetadataUpdateEvent getClusterMetadataOnServiceCredentialStoreUpdate(Cluster cl, String serviceName) throws AmbariException { + public MetadataUpdateEvent getClusterMetadataOnServiceCredentialStoreUpdate(Cluster cl, String serviceGroupName, String serviceName) throws AmbariException { final SortedMap<String, MetadataCluster> metadataClusters = new TreeMap<>(); - metadataClusters.put(Long.toString(cl.getClusterId()), MetadataCluster.serviceLevelParamsMetadataCluster(null, getMetadataServiceLevelParams(cl), false)); + Service service = cl.getService(serviceGroupName, serviceName); + metadataClusters.put(Long.toString(cl.getClusterId()), MetadataCluster.serviceLevelParamsMetadataCluster(null, getMetadataServiceLevelParams(service), false)); return new MetadataUpdateEvent(metadataClusters, null, getMetadataAgentConfigs(), UpdateEventType.UPDATE); } @@ -205,18 +206,18 @@ public class ClusterMetadataGenerator { SortedMap<String, MetadataServiceInfo> serviceLevelParams = new TreeMap<>(); StackId serviceStackId = service.getStackId(); - ServiceInfo serviceInfo = ambariMetaInfo.getService(serviceStackId.getStackName(), - serviceStackId.getStackVersion(), service.getName()); + + ServiceInfo serviceInfo = ambariMetaInfo.getService(serviceStackId.getStackName(), serviceStackId.getStackVersion(), service.getServiceType()); Long statusCommandTimeout = null; if (serviceInfo.getCommandScript() != null) { statusCommandTimeout = new Long(getStatusCommandTimeout(serviceInfo)); } - - String servicePackageFolder = serviceInfo.getServicePackageFolder(); Map<String, Map<String, String>> configCredentials = configHelper.getCredentialStoreEnabledProperties(serviceStackId, service); - - serviceLevelParams.put(serviceInfo.getName(), new MetadataServiceInfo(serviceInfo.getVersion(), - service.isCredentialStoreEnabled(), configCredentials, statusCommandTimeout, servicePackageFolder)); + serviceLevelParams.put(service.getName(), new MetadataServiceInfo( + serviceInfo.getName(), serviceInfo.getVersion(), + serviceInfo.isCredentialStoreEnabled(), configCredentials, statusCommandTimeout, + serviceInfo.getServicePackageFolder() + )); return serviceLevelParams; } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java index 048ab3a..5949ae9 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java @@ -1534,6 +1534,10 @@ public class ClusterImpl implements Cluster { @Override public Map<String, Config> getConfigsByServiceIdType(String configType, Long serviceId) { + if (serviceId == null) { + return getConfigsByType(configType); + } + clusterGlobalLock.readLock().lock(); try { if (!serviceConfigs.containsKey(serviceId)) { diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java index ad1535d..5d18b6b 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java @@ -706,7 +706,7 @@ public class AmbariContext { for (Map.Entry<String, Map<String, String>> entry : userProvidedGroupProperties.entrySet()) { String type = entry.getKey(); String service = stack.getServicesForConfigType(type) - .filter(each -> topology.getServices().contains(each)) + .filter(each -> topology.getServiceTypes().contains(each)) .findFirst() // TODO check if this is required at all (might be handled by the "orphan" removal) // TODO move this validation earlier diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java index 0eb2cb7..48b55aa 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java @@ -172,9 +172,8 @@ public class ClusterConfigurationRequest { Set<String> updatedConfigTypes = new HashSet<>(); Cluster cluster = getCluster(); - Blueprint blueprint = clusterTopology.getBlueprint(); - Configuration stackDefaults = clusterTopology.getStack().getConfiguration(clusterTopology.getServices()); + Configuration stackDefaults = clusterTopology.getStack().getConfiguration(clusterTopology.getServiceTypes()); Map<String, Map<String, String>> stackDefaultProps = stackDefaults.getProperties(); // add clusterHostInfo containing components to hosts map, based on Topology, to use this one instead of @@ -186,7 +185,7 @@ public class ClusterConfigurationRequest { // generate principals & keytabs for headless identities ambariContext.getController().getKerberosHelper() .ensureHeadlessIdentities(cluster, existingConfigurations, - new HashSet<>(clusterTopology.getServices())); + new HashSet<>(clusterTopology.getServiceTypes())); // apply Kerberos specific configurations Map<String, Map<String, String>> updatedConfigs = ambariContext.getController().getKerberosHelper() @@ -344,10 +343,6 @@ public class ClusterConfigurationRequest { * @param tag config tag */ private void setConfigurationsOnCluster(ClusterTopology clusterTopology, String tag, Set<String> updatedConfigTypes) throws AmbariException { - // TODO: This version works with Ambari 3.0 where it is assumed that any service with a configuration can be identified - // by its name. Even though the cluster is multi-stack (multi-mpack), service names should not conflict across mpacks, - // except client services which have no configuration. In 3.1, mpack may have conflicting service names - //todo: also handle setting of host group scoped configuration which is updated by config processor List<Pair<String, ClusterRequest>> serviceNamesAndConfigurationRequests = new ArrayList<>(); Configuration clusterConfiguration = clusterTopology.getConfiguration(); @@ -367,10 +362,12 @@ public class ClusterConfigurationRequest { Set<String> configTypes = Sets.difference( - Sets.intersection(stack.getAllConfigurationTypes(service.getServiceName()), clusterConfigTypes), - Sets.union(stack.getExcludedConfigurationTypes(service.getServiceName()), globalConfigTypes) + Sets.intersection(stack.getAllConfigurationTypes(service.getServiceType()), clusterConfigTypes), + Sets.union(stack.getExcludedConfigurationTypes(service.getServiceType()), globalConfigTypes) ); + LOG.info("Creating config request for service {}, types {}", service.getServiceName(), configTypes); + for (String serviceConfigType: configTypes) { Map<String, String> properties = clusterProperties.get(serviceConfigType); Map<String, Map<String, String>> attributes = clusterAttributes.get(serviceConfigType); @@ -446,7 +443,7 @@ public class ClusterConfigurationRequest { } // iterate over services to deploy for (Pair<String, ClusterRequest> serviceNameAndRequest: serviceNamesAndRequests) { - LOG.info("Sending cluster config update request for service = " + serviceNameAndRequest.getLeft()); + LOG.info("Sending cluster config update request for service {}", serviceNameAndRequest.getLeft()); ambariContext.setConfigurationOnCluster(serviceNameAndRequest.getRight()); } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopology.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopology.java index 5791f40..9d622d7 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopology.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopology.java @@ -135,11 +135,11 @@ public interface ClusterTopology { Collection<String> getHostAssignmentsForComponent(String component); /** - * Get all of the services represented in the blueprint. + * Get all of the service types represented in the blueprint. * - * @return collection of all represented service names + * @return collection of all represented service types */ - Collection<String> getServices(); + Collection<String> getServiceTypes(); /** * Get all of the components represented in the blueprint. diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java index 35f359f..672ffd3 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java @@ -171,7 +171,7 @@ public class ClusterTopologyImpl implements ClusterTopology { stackIds = request.getStackIds(); stack = request.getStack(); setting = request.getSetting(); - blueprint.getConfiguration().setParentConfiguration(stack.getConfiguration(getServices())); + blueprint.getConfiguration().setParentConfiguration(stack.getConfiguration(getServiceTypes())); checkForDuplicateHosts(request.getHostGroupInfo()); registerHostGroupInfo(request.getHostGroupInfo()); @@ -265,11 +265,7 @@ public class ClusterTopologyImpl implements ClusterTopology { @Override public Set<String> getHostGroupsForComponent(ResolvedComponent component) { - return resolvedComponents.entrySet().stream() - .filter(e -> e.getValue().stream() - .anyMatch(each -> Objects.equals(each, component))) - .map(Map.Entry::getKey) - .collect(toSet()); + return getHostGroupsForComponent(resolvedComponents, component); } @Override @@ -329,9 +325,9 @@ public class ClusterTopologyImpl implements ClusterTopology { } @Override - public Collection<String> getServices() { + public Collection<String> getServiceTypes() { return getComponents() - .map(ResolvedComponent::effectiveServiceName) + .map(ResolvedComponent::serviceType) .collect(toSet()); } @@ -359,7 +355,7 @@ public class ClusterTopologyImpl implements ClusterTopology { } try { String service = getStack().getServiceForConfigType(configType); - return getServices().contains(service); + return getServiceTypes().contains(service); } catch (IllegalArgumentException e) { return false; } @@ -574,4 +570,14 @@ public class ClusterTopologyImpl implements ClusterTopology { " Be aware that host names are converted to lowercase, case differences are ignored in Ambari deployments."); } } + + // exposed for test + public static Set<String> getHostGroupsForComponent(Map<String, Set<ResolvedComponent>> hostGroups, ResolvedComponent component) { + return hostGroups.entrySet().stream() + .filter(e -> e.getValue().stream() + .map(ResolvedComponent::clearInstanceNames) + .anyMatch(each -> Objects.equals(each, component))) + .map(Map.Entry::getKey) + .collect(toSet()); + } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ResolvedComponent.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ResolvedComponent.java index f2910a8..6e1f88f 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ResolvedComponent.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ResolvedComponent.java @@ -65,6 +65,13 @@ public interface ResolvedComponent { return serviceName().orElseGet(this::serviceType); } + static ResolvedComponent clearInstanceNames(ResolvedComponent component) { + return component.toBuilder() + .clearServiceGroupName() + .clearServiceName() + .build(); + } + /** * Starts building a {@code ResolvedComponent} for the given component. */ diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ResolvedComponent_Builder.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ResolvedComponent_Builder.java index fe0b4ae..1cd8a8b 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ResolvedComponent_Builder.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ResolvedComponent_Builder.java @@ -576,6 +576,8 @@ abstract class ResolvedComponent_Builder implements ResolvedComponent { return Objects.equals(stackId, other.stackId) && Objects.equals(componentName, other.componentName) && Objects.equals(serviceInfo, other.serviceInfo) + && Objects.equals(serviceGroupName, other.serviceGroupName) + && Objects.equals(serviceName, other.serviceName) ; } @@ -584,7 +586,9 @@ abstract class ResolvedComponent_Builder implements ResolvedComponent { return Objects.hash( stackId, componentName, - serviceInfo + serviceInfo, + serviceGroupName, + serviceName ); } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/ClusterConfigTypeValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/ClusterConfigTypeValidator.java index d03fa3d..97d5fb3 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/ClusterConfigTypeValidator.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/ClusterConfigTypeValidator.java @@ -43,7 +43,7 @@ public class ClusterConfigTypeValidator implements TopologyValidator { // collecting all config types for services in the blueprint (from the related stack) Set<String> stackServiceConfigTypes = new HashSet<>(); - for (String serviceName : topology.getServices()) { + for (String serviceName : topology.getServiceTypes()) { stackServiceConfigTypes.addAll(topology.getStack().getConfigurationTypes(serviceName)); } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/HiveServiceValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/HiveServiceValidator.java index 685e354..baefd53 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/HiveServiceValidator.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/HiveServiceValidator.java @@ -36,7 +36,7 @@ public class HiveServiceValidator implements TopologyValidator { @Override public ClusterTopology validate(ClusterTopology topology) throws InvalidTopologyException { // there is no hive configured in the blueprint, nothing to do (does the validator apply?) - if (!topology.getServices().contains(HIVE_SERVICE)) { + if (!topology.getServiceTypes().contains(HIVE_SERVICE)) { LOGGER.info(" [{}] service is not listed in the blueprint, skipping hive service validation.", HIVE_SERVICE); return topology; } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredConfigPropertiesValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredConfigPropertiesValidator.java index e1f68a2..e3ef20b 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredConfigPropertiesValidator.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredConfigPropertiesValidator.java @@ -118,7 +118,7 @@ public class RequiredConfigPropertiesValidator implements TopologyValidator { Map<String, Map<String, Collection<String>>> requiredPropertiesForServiceByType = new HashMap<>(); - for (String bpService : topology.getServices()) { + for (String bpService : topology.getServiceTypes()) { LOGGER.debug("Collecting required properties for the service: {}", bpService); Collection<Stack.ConfigProperty> requiredConfigsForService = topology.getStack().getRequiredConfigurationProperties(bpService); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/agent/HeartbeatProcessorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/agent/HeartbeatProcessorTest.java index ed9623c..e682462 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/agent/HeartbeatProcessorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/agent/HeartbeatProcessorTest.java @@ -44,8 +44,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import com.google.inject.Binder; -import com.google.inject.Module; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.H2DatabaseCleaner; import org.apache.ambari.server.Role; @@ -97,9 +95,11 @@ import org.junit.Ignore; import org.junit.Test; import com.google.gson.JsonObject; +import com.google.inject.Binder; import com.google.inject.Guice; import com.google.inject.Inject; import com.google.inject.Injector; +import com.google.inject.Module; public class HeartbeatProcessorTest { diff --git a/ambari-server/src/test/java/org/apache/ambari/server/agent/HeartbeatTestHelper.java b/ambari-server/src/test/java/org/apache/ambari/server/agent/HeartbeatTestHelper.java index 9228475..4fa806f 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/agent/HeartbeatTestHelper.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/agent/HeartbeatTestHelper.java @@ -29,11 +29,9 @@ import static org.apache.ambari.server.agent.DummyHeartbeatConstants.HBASE; import java.lang.reflect.Method; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -47,7 +45,6 @@ import org.apache.ambari.server.actionmanager.Request; import org.apache.ambari.server.actionmanager.Stage; import org.apache.ambari.server.actionmanager.StageFactory; import org.apache.ambari.server.api.services.AmbariMetaInfo; -import org.apache.ambari.server.api.services.ServiceGroupKey; import org.apache.ambari.server.events.publishers.STOMPUpdatePublisher; import org.apache.ambari.server.orm.InMemoryDefaultTestModule; import org.apache.ambari.server.orm.OrmTestHelper; @@ -76,7 +73,6 @@ import org.apache.ambari.server.state.Service; import org.apache.ambari.server.state.ServiceFactory; import org.apache.ambari.server.state.ServiceGroup; import org.apache.ambari.server.state.ServiceGroupFactory; -import org.apache.ambari.server.state.ServiceGroupImpl; import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.state.cluster.ClustersImpl; import org.apache.ambari.server.state.fsm.InvalidStateTransitionException; diff --git a/ambari-server/src/test/java/org/apache/ambari/server/agent/stomp/dto/MetadataClusterTest.java b/ambari-server/src/test/java/org/apache/ambari/server/agent/stomp/dto/MetadataClusterTest.java index 1e5db3b..6427633 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/agent/stomp/dto/MetadataClusterTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/agent/stomp/dto/MetadataClusterTest.java @@ -32,9 +32,9 @@ public class MetadataClusterTest { @Test public void shouldReturnFalseWhenUpdatingServiceLevelParamsWithoutNewOrRemovedServices() throws Exception { final SortedMap<String, MetadataServiceInfo> current = new TreeMap<>(); - current.put("service1", new MetadataServiceInfo("v1", Boolean.FALSE, null, 1L, "servicePackageFolder")); - current.put("service2", new MetadataServiceInfo("v1", Boolean.FALSE, null, 1L, "servicePackageFolder")); - current.put("service3", new MetadataServiceInfo("v1", Boolean.FALSE, null, 1L, "servicePackageFolder")); + current.put("service1", new MetadataServiceInfo("anyType", "v1", Boolean.FALSE, null, 1L, "servicePackageFolder")); + current.put("service2", new MetadataServiceInfo("anyType", "v1", Boolean.FALSE, null, 1L, "servicePackageFolder")); + current.put("service3", new MetadataServiceInfo("anyType", "v1", Boolean.FALSE, null, 1L, "servicePackageFolder")); final MetadataCluster metadataCluster = MetadataCluster.serviceLevelParamsMetadataCluster(null, current, true); final SortedMap<String, MetadataServiceInfo> updated = new TreeMap<>(current); assertFalse(metadataCluster.updateServiceLevelParams(updated, true)); @@ -44,11 +44,11 @@ public class MetadataClusterTest { @Test public void shouldReturnTrueWhenUpdatingServiceLevelParamsUponServiceAddition() throws Exception { final SortedMap<String, MetadataServiceInfo> current = new TreeMap<>(); - current.put("service1", new MetadataServiceInfo("v1", Boolean.FALSE, null, 1L, "servicePackageFolder")); - current.put("service2", new MetadataServiceInfo("v1", Boolean.FALSE, null, 1L, "servicePackageFolder")); + current.put("service1", new MetadataServiceInfo("anyType", "v1", Boolean.FALSE, null, 1L, "servicePackageFolder")); + current.put("service2", new MetadataServiceInfo("anyType", "v1", Boolean.FALSE, null, 1L, "servicePackageFolder")); final MetadataCluster metadataCluster = MetadataCluster.serviceLevelParamsMetadataCluster(null, current, true); final SortedMap<String, MetadataServiceInfo> updated = new TreeMap<>(current); - updated.put("service3", new MetadataServiceInfo("v1", Boolean.FALSE, null, 1L, "servicePackageFolder")); + updated.put("service3", new MetadataServiceInfo("anyType", "v1", Boolean.FALSE, null, 1L, "servicePackageFolder")); assertTrue(metadataCluster.updateServiceLevelParams(updated, true)); assertEquals(updated, metadataCluster.getServiceLevelParams()); } @@ -56,9 +56,9 @@ public class MetadataClusterTest { @Test public void shouldReturnTrueWhenUpdatingServiceLevelParamsUponServiceRemoval() throws Exception { final SortedMap<String, MetadataServiceInfo> current = new TreeMap<>(); - current.put("service1", new MetadataServiceInfo("v1", Boolean.FALSE, null, 1L, "servicePackageFolder")); - current.put("service2", new MetadataServiceInfo("v1", Boolean.FALSE, null, 1L, "servicePackageFolder")); - current.put("service3", new MetadataServiceInfo("v1", Boolean.FALSE, null, 1L, "servicePackageFolder")); + current.put("service1", new MetadataServiceInfo("anyType", "v1", Boolean.FALSE, null, 1L, "servicePackageFolder")); + current.put("service2", new MetadataServiceInfo("anyType", "v1", Boolean.FALSE, null, 1L, "servicePackageFolder")); + current.put("service3", new MetadataServiceInfo("anyType", "v1", Boolean.FALSE, null, 1L, "servicePackageFolder")); final MetadataCluster metadataCluster = MetadataCluster.serviceLevelParamsMetadataCluster(null, current, true); final SortedMap<String, MetadataServiceInfo> updated = new TreeMap<>(current); updated.remove("service2"); @@ -69,9 +69,9 @@ public class MetadataClusterTest { @Test public void shouldReturnFalseWhenNullServiceLevelParamsArePassedBecauseOfPartialConfigurationUpdate() throws Exception { final SortedMap<String, MetadataServiceInfo> current = new TreeMap<>(); - current.put("service1", new MetadataServiceInfo("v1", Boolean.FALSE, null, 1L, "servicePackageFolder")); - current.put("service2", new MetadataServiceInfo("v1", Boolean.FALSE, null, 1L, "servicePackageFolder")); - current.put("service3", new MetadataServiceInfo("v1", Boolean.FALSE, null, 1L, "servicePackageFolder")); + current.put("service1", new MetadataServiceInfo("anyType", "v1", Boolean.FALSE, null, 1L, "servicePackageFolder")); + current.put("service2", new MetadataServiceInfo("anyType", "v1", Boolean.FALSE, null, 1L, "servicePackageFolder")); + current.put("service3", new MetadataServiceInfo("anyType", "v1", Boolean.FALSE, null, 1L, "servicePackageFolder")); final MetadataCluster metadataCluster = MetadataCluster.serviceLevelParamsMetadataCluster(null, current, true); assertFalse(metadataCluster.updateServiceLevelParams(null, true)); assertEquals(current, metadataCluster.getServiceLevelParams()); @@ -80,13 +80,13 @@ public class MetadataClusterTest { @Test public void shouldReturnTrueWhenUpdatingServiceLevelParamsWithoutFullServiceLevelMetadata() throws Exception { final SortedMap<String, MetadataServiceInfo> current = new TreeMap<>(); - current.put("service1", new MetadataServiceInfo("v1", Boolean.FALSE, null, 1L, "servicePackageFolder")); - current.put("service2", new MetadataServiceInfo("v1", Boolean.FALSE, null, 1L, "servicePackageFolder")); - current.put("service3", new MetadataServiceInfo("v1", Boolean.FALSE, null, 1L, "servicePackageFolder")); + current.put("service1", new MetadataServiceInfo("anyType", "v1", Boolean.FALSE, null, 1L, "servicePackageFolder")); + current.put("service2", new MetadataServiceInfo("anyType", "v1", Boolean.FALSE, null, 1L, "servicePackageFolder")); + current.put("service3", new MetadataServiceInfo("anyType", "v1", Boolean.FALSE, null, 1L, "servicePackageFolder")); final MetadataCluster metadataCluster = MetadataCluster.serviceLevelParamsMetadataCluster(null, current, true); final SortedMap<String, MetadataServiceInfo> updated = new TreeMap<>(); - updated.put("service3", new MetadataServiceInfo("v2", Boolean.TRUE, null, 2L, "servicePackageFolder2")); - updated.put("service4", new MetadataServiceInfo("v1", Boolean.FALSE, null, 1L, "servicePackageFolder")); + updated.put("service3", new MetadataServiceInfo("anyType", "v2", Boolean.TRUE, null, 2L, "servicePackageFolder2")); + updated.put("service4", new MetadataServiceInfo("anyType", "v1", Boolean.FALSE, null, 1L, "servicePackageFolder")); assertTrue(metadataCluster.updateServiceLevelParams(updated, false)); final SortedMap<String, MetadataServiceInfo> expected = current; expected.putAll(updated); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/query/QueryImplTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/query/QueryImplTest.java index 341f7e4..9440c8c 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/api/query/QueryImplTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/api/query/QueryImplTest.java @@ -47,7 +47,6 @@ import org.apache.ambari.server.api.query.render.Renderer; import org.apache.ambari.server.api.resources.ClusterResourceDefinition; import org.apache.ambari.server.api.resources.HostResourceDefinition; import org.apache.ambari.server.api.resources.MpackResourceDefinition; -import org.apache.ambari.server.api.resources.OperatingSystemResourceDefinition; import org.apache.ambari.server.api.resources.ResourceDefinition; import org.apache.ambari.server.api.resources.ResourceInstance; import org.apache.ambari.server.api.resources.StackResourceDefinition; diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessorTest.java index 2abc152..9a24c12 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessorTest.java @@ -94,7 +94,7 @@ public class StackAdvisorBlueprintProcessorTest { expect(stack.getName()).andReturn(STACK_ID.getStackName()).anyTimes(); expect(stack.getConfiguration(services)).andReturn(createStackDefaults()).anyTimes(); expect(stack.getServices(STACK_ID)).andReturn(services).anyTimes(); - expect(clusterTopology.getServices()).andReturn(services).anyTimes(); + expect(clusterTopology.getServiceTypes()).andReturn(services).anyTimes(); expect(blueprint.getHostGroups()).andReturn(createHostGroupMap()).anyTimes(); expect(clusterTopology.isValidConfigType("core-site")).andReturn(true).anyTimes(); expect(hostGroup.getComponentNames()).andReturn(Arrays.asList("comp1", "comp2")).anyTimes(); @@ -133,7 +133,7 @@ public class StackAdvisorBlueprintProcessorTest { expect(stack.getName()).andReturn(STACK_ID.getStackName()).anyTimes(); expect(stack.getConfiguration(services)).andReturn(createStackDefaults()).anyTimes(); expect(stack.getServices(STACK_ID)).andReturn(services).anyTimes(); - expect(clusterTopology.getServices()).andReturn(services).anyTimes(); + expect(clusterTopology.getServiceTypes()).andReturn(services).anyTimes(); expect(blueprint.getHostGroups()).andReturn(createHostGroupMap()).anyTimes(); expect(clusterTopology.isValidConfigType("core-site")).andReturn(true).anyTimes(); expect(hostGroup.getComponentNames()).andReturn(Arrays.asList("comp1", "comp2")).anyTimes(); @@ -173,7 +173,7 @@ public class StackAdvisorBlueprintProcessorTest { expect(stack.getName()).andReturn(STACK_ID.getStackName()).anyTimes(); expect(stack.getConfiguration(services)).andReturn(createStackDefaults()).anyTimes(); expect(stack.getServices(STACK_ID)).andReturn(services).anyTimes(); - expect(clusterTopology.getServices()).andReturn(services).anyTimes(); + expect(clusterTopology.getServiceTypes()).andReturn(services).anyTimes(); expect(blueprint.getHostGroups()).andReturn(createHostGroupMap()).anyTimes(); expect(clusterTopology.isValidConfigType("core-site")).andReturn(true).anyTimes(); expect(hostGroup.getComponentNames()).andReturn(Arrays.asList("comp1", "comp2")).anyTimes(); @@ -211,7 +211,7 @@ public class StackAdvisorBlueprintProcessorTest { expect(stack.getName()).andReturn(STACK_ID.getStackName()).anyTimes(); expect(stack.getConfiguration(services)).andReturn(createStackDefaults()).anyTimes(); expect(stack.getServices(STACK_ID)).andReturn(services).anyTimes(); - expect(clusterTopology.getServices()).andReturn(services).anyTimes(); + expect(clusterTopology.getServiceTypes()).andReturn(services).anyTimes(); expect(blueprint.getHostGroups()).andReturn(createHostGroupMap()).anyTimes(); expect(clusterTopology.isValidConfigType("core-site")).andReturn(true).anyTimes(); expect(hostGroup.getComponentNames()).andReturn(Arrays.asList("comp1", "comp2")).anyTimes(); @@ -248,7 +248,7 @@ public class StackAdvisorBlueprintProcessorTest { expect(blueprint.getHostGroups()).andReturn(createHostGroupMap()).anyTimes(); expect(hostGroup.getComponentNames()).andReturn(Arrays.asList("comp1", "comp2")).anyTimes(); expect(stack.getServices(STACK_ID)).andReturn(services).anyTimes(); - expect(clusterTopology.getServices()).andReturn(services).anyTimes(); + expect(clusterTopology.getServiceTypes()).andReturn(services).anyTimes(); expect(stackAdvisorHelper.recommend(anyObject(StackAdvisorRequest.class))).andThrow(new StackAdvisorException("ex")); expect(configuration.getFullProperties()).andReturn(props); @@ -280,7 +280,7 @@ public class StackAdvisorBlueprintProcessorTest { expect(stack.getName()).andReturn(STACK_ID.getStackName()).anyTimes(); expect(stack.getConfiguration(services)).andReturn(createStackDefaults()).anyTimes(); expect(stack.getServices(STACK_ID)).andReturn(services).anyTimes(); - expect(clusterTopology.getServices()).andReturn(services).anyTimes(); + expect(clusterTopology.getServiceTypes()).andReturn(services).anyTimes(); expect(blueprint.getHostGroups()).andReturn(createHostGroupMap()).anyTimes(); expect(hostGroup.getComponentNames()).andReturn(Arrays.asList("comp1", "comp2")).anyTimes(); expect(stackAdvisorHelper.recommend(anyObject(StackAdvisorRequest.class))).andReturn(new RecommendationResponse()); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java index 98d4c29..5e84ce2 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java @@ -361,16 +361,18 @@ public class AmbariManagementControllerTest { controller.createCluster(r); } - private void createService(String clusterName, String serviceGroupName, String serviceName, State desiredState) throws Exception { + private Service createService(String clusterName, String serviceGroupName, String serviceName, State desiredState) throws Exception { String dStateStr = null; if (desiredState != null) { dStateStr = desiredState.toString(); } - ServiceRequest r1 = new ServiceRequest(clusterName, serviceGroupName, serviceName, null, + ServiceRequest r1 = new ServiceRequest(clusterName, serviceGroupName, serviceName, serviceName, dStateStr, null); ServiceResourceProviderTest.createServices(controller, Collections.singleton(r1)); + + return clusters.getCluster(clusterName).getService(serviceGroupName, serviceName); } private void createServiceComponent(String clusterName, @@ -1249,7 +1251,7 @@ public class AmbariManagementControllerTest { createServiceComponentHostSimple(cluster1, host1, getUniqueName(), serviceGroupName, serviceName); Cluster cluster = clusters.getCluster(cluster1); - Service s1 = cluster.getService(serviceName); + Service s1 = cluster.getService(serviceGroupName, serviceName); // Create and attach config Map<String, String> configs = new HashMap<>(); @@ -1262,11 +1264,11 @@ public class AmbariManagementControllerTest { ConfigurationRequest cr1,cr2, cr3; cr1 = new ConfigurationRequest(cluster1, "core-site","version1", - configs, null, 1L, 1L); + configs, null, s1.getServiceId(), 1L); cr2 = new ConfigurationRequest(cluster1, "hdfs-site","version1", - configs, null, 1L, 1L); + configs, null, s1.getServiceId(), 1L); cr3 = new ConfigurationRequest(cluster1, "hadoop-env","version1", - hadoopEnvConfigs, null, 1L, 1L); + hadoopEnvConfigs, null, s1.getServiceId(), 1L); ClusterRequest crReq = new ClusterRequest(cluster.getClusterId(), cluster1, null, null); crReq.setDesiredConfig(Collections.singletonList(cr1)); @@ -1371,7 +1373,7 @@ public class AmbariManagementControllerTest { createCluster(clusterName); clusters.getCluster(clusterName).setDesiredStackVersion(new StackId(HDP_0_1)); ServiceGroupResourceProviderTest.createServiceGroup(controller, clusterName, serviceGroupName, HDP_0_1); - createService(clusterName, serviceGroupName, serviceName, null); + Service service = createService(clusterName, serviceGroupName, serviceName, null); String componentName1 = "NAMENODE"; String componentName2 = "DATANODE"; String componentName3 = "HDFS_CLIENT"; @@ -2495,7 +2497,7 @@ public class AmbariManagementControllerTest { String serviceGroupName = "CORE"; ServiceGroupResourceProviderTest.createServiceGroup(controller, cluster1, serviceGroupName, "HDP-2.0.5"); String serviceName = "HDFS"; - createService(cluster1, serviceGroupName, serviceName, null); + Service service = createService(cluster1, serviceGroupName, serviceName, null); String componentName1 = "NAMENODE"; String componentName2 = "DATANODE"; String componentName3 = "HDFS_CLIENT"; @@ -2517,9 +2519,8 @@ public class AmbariManagementControllerTest { Map<String, String> configs = new HashMap<>(); configs.put("a", "b"); - ConfigurationRequest cr1; - cr1 = new ConfigurationRequest(cluster1, "hdfs-site", "version1", - configs, null, 1L, 1L); + ConfigurationRequest cr1 = new ConfigurationRequest(cluster1, "hdfs-site", "version1", + configs, null, service.getServiceId(), 1L); ClusterRequest crReq = new ClusterRequest(clusterId, cluster1, null, null); crReq.setDesiredConfig(Collections.singletonList(cr1)); controller.updateClusters(Collections.singleton(crReq), null); @@ -4422,7 +4423,7 @@ public class AmbariManagementControllerTest { String serviceGroupName = "CORE"; ServiceGroupResourceProviderTest.createServiceGroup(controller, cluster1, serviceGroupName, HDP_0_1); String serviceName = "HDFS"; - createService(cluster1, serviceGroupName, serviceName, null); + Service service = createService(cluster1, serviceGroupName, serviceName, null); String componentName1 = "NAMENODE"; String componentName2 = "DATANODE"; String componentName3 = "HDFS_CLIENT"; @@ -4533,14 +4534,14 @@ public class AmbariManagementControllerTest { configs.put("a", "b"); ConfigurationRequest cr1, cr2, cr3, cr4, cr5, cr6, cr7, cr8; - cr1 = new ConfigurationRequest(cluster1, "typeA","v1", configs, null, 1L, 1L); - cr2 = new ConfigurationRequest(cluster1, "typeB","v1", configs, null, 1L, 1L); - cr3 = new ConfigurationRequest(cluster1, "typeC","v1", configs, null, 1L, 1L); - cr4 = new ConfigurationRequest(cluster1, "typeD","v1", configs, null, 1L, 1L); - cr5 = new ConfigurationRequest(cluster1, "typeA","v2", configs, null, 1L, 1L); - cr6 = new ConfigurationRequest(cluster1, "typeB","v2", configs, null, 1L, 1L); - cr7 = new ConfigurationRequest(cluster1, "typeC","v2", configs, null, 1L, 1L); - cr8 = new ConfigurationRequest(cluster1, "typeE","v1", configs, null, 1L, 1L); + cr1 = new ConfigurationRequest(cluster1, "typeA","v1", configs, null, service.getServiceId(), 1L); + cr2 = new ConfigurationRequest(cluster1, "typeB","v1", configs, null, service.getServiceId(), 1L); + cr3 = new ConfigurationRequest(cluster1, "typeC","v1", configs, null, service.getServiceId(), 1L); + cr4 = new ConfigurationRequest(cluster1, "typeD","v1", configs, null, service.getServiceId(), 1L); + cr5 = new ConfigurationRequest(cluster1, "typeA","v2", configs, null, service.getServiceId(), 1L); + cr6 = new ConfigurationRequest(cluster1, "typeB","v2", configs, null, service.getServiceId(), 1L); + cr7 = new ConfigurationRequest(cluster1, "typeC","v2", configs, null, service.getServiceId(), 1L); + cr8 = new ConfigurationRequest(cluster1, "typeE","v1", configs, null, service.getServiceId(), 1L); controller.createConfiguration(cr1); controller.createConfiguration(cr2); controller.createConfiguration(cr3); @@ -4620,7 +4621,7 @@ public class AmbariManagementControllerTest { String serviceGroupName = "CORE"; ServiceGroupResourceProviderTest.createServiceGroup(controller, cluster1, serviceGroupName, HDP_0_1); String serviceName = "HDFS"; - createService(cluster1, serviceGroupName, serviceName, null); + Service service = createService(cluster1, serviceGroupName, serviceName, null); String componentName1 = "NAMENODE"; String componentName2 = "DATANODE"; String componentName3 = "HDFS_CLIENT"; @@ -4674,14 +4675,14 @@ public class AmbariManagementControllerTest { configAttributes.get("final").put("a", "true"); ConfigurationRequest cr1, cr2, cr3, cr4, cr5, cr6, cr7, cr8; - cr1 = new ConfigurationRequest(cluster1, "typeA","v1", configs, configAttributes, 1L, 1L); - cr2 = new ConfigurationRequest(cluster1, "typeB","v1", configs, configAttributes, 1L, 1L); - cr3 = new ConfigurationRequest(cluster1, "typeC","v1", configs, configAttributes, 1L, 1L); - cr4 = new ConfigurationRequest(cluster1, "typeD","v1", configs, configAttributes, 1L, 1L); - cr5 = new ConfigurationRequest(cluster1, "typeA","v2", configs, configAttributes, 1L, 1L); - cr6 = new ConfigurationRequest(cluster1, "typeB","v2", configs, configAttributes, 1L, 1L); - cr7 = new ConfigurationRequest(cluster1, "typeC","v2", configs, configAttributes, 1L, 1L); - cr8 = new ConfigurationRequest(cluster1, "typeE","v1", configs, configAttributes, 1L, 1L); + cr1 = new ConfigurationRequest(cluster1, "typeA","v1", configs, configAttributes, service.getServiceId(), 1L); + cr2 = new ConfigurationRequest(cluster1, "typeB","v1", configs, configAttributes, service.getServiceId(), 1L); + cr3 = new ConfigurationRequest(cluster1, "typeC","v1", configs, configAttributes, service.getServiceId(), 1L); + cr4 = new ConfigurationRequest(cluster1, "typeD","v1", configs, configAttributes, service.getServiceId(), 1L); + cr5 = new ConfigurationRequest(cluster1, "typeA","v2", configs, configAttributes, service.getServiceId(), 1L); + cr6 = new ConfigurationRequest(cluster1, "typeB","v2", configs, configAttributes, service.getServiceId(), 1L); + cr7 = new ConfigurationRequest(cluster1, "typeC","v2", configs, configAttributes, service.getServiceId(), 1L); + cr8 = new ConfigurationRequest(cluster1, "typeE","v1", configs, configAttributes, service.getServiceId(), 1L); controller.createConfiguration(cr1); controller.createConfiguration(cr2); controller.createConfiguration(cr3); @@ -4793,14 +4794,11 @@ public class AmbariManagementControllerTest { requests.add(r); ServiceResourceProviderTest.updateServices(controller, requests, mapRequestProps, true, false); - Assert.assertEquals(State.INSTALLED, - clusters.getCluster(cluster1).getService(serviceName) - .getDesiredState()); + Service service = clusters.getCluster(cluster1).getService(serviceName); + Assert.assertEquals(State.INSTALLED, service.getDesiredState()); // manually change live state to installed as no running action manager - for (ServiceComponent sc : - clusters.getCluster(cluster1).getService(serviceName) - .getServiceComponents().values()) { + for (ServiceComponent sc : service.getServiceComponents().values()) { for (ServiceComponentHost sch : sc.getServiceComponentHosts().values()) { sch.setState(State.INSTALLED); } @@ -4812,11 +4810,11 @@ public class AmbariManagementControllerTest { ConfigurationRequest cr1,cr2,cr3; cr1 = new ConfigurationRequest(cluster1, "core-site","version1", - configs, null, 1L, 1L); + configs, null, service.getServiceId(), 1L); cr2 = new ConfigurationRequest(cluster1, "hdfs-site","version1", - configs, null, 1L, 1L); + configs, null, service.getServiceId(), 1L); cr3 = new ConfigurationRequest(cluster1, "core-site","version122", - configs, null, 1L, 1L); + configs, null, service.getServiceId(), 1L); controller.createConfiguration(cr1); controller.createConfiguration(cr2); controller.createConfiguration(cr3); @@ -4919,8 +4917,8 @@ public class AmbariManagementControllerTest { String componentName5 = "TASKTRACKER"; String componentName6 = "MAPREDUCE_CLIENT"; - createService(cluster1, serviceGroupName, serviceName1, null); - createService(cluster1, serviceGroupName, serviceName2, null); + Service service1 = createService(cluster1, serviceGroupName, serviceName1, null); + Service service2 = createService(cluster1, serviceGroupName, serviceName2, null); createServiceComponent(cluster1, serviceGroupName, serviceName1, componentName1, State.INIT); createServiceComponent(cluster1, serviceGroupName, serviceName1, componentName2, State.INIT); @@ -4956,11 +4954,11 @@ public class AmbariManagementControllerTest { ConfigurationRequest cr1,cr2,cr3,cr4; cr1 = new ConfigurationRequest(cluster1, "core-site","version1", - configs, null, 1L, 1L); + configs, null, service1.getServiceId(), 1L); cr2 = new ConfigurationRequest(cluster1, "hdfs-site","version1", - configs, null, 1L, 1L); + configs, null, service1.getServiceId(), 1L); cr4 = new ConfigurationRequest(cluster1, "kerberos-env", "version1", - configs3, null, 1L, 1L); + configs3, null, null, 1L); ConfigFactory cf = injector.getInstance(ConfigFactory.class); Config config1 = cf.createNew(cluster, "kerberos-env", "version1", @@ -4992,7 +4990,7 @@ public class AmbariManagementControllerTest { // Reconfigure cr3 = new ConfigurationRequest(cluster1, "core-site","version122", - configs2, null, 1L, 1L); + configs2, null, service1.getServiceId(), 1L); crReq = new ClusterRequest(cluster.getClusterId(), cluster1, null, null); crReq.setDesiredConfig(Collections.singletonList(cr3)); controller.updateClusters(Collections.singleton(crReq), null); @@ -5089,7 +5087,7 @@ public class AmbariManagementControllerTest { String serviceGroupName = "CORE"; ServiceGroupResourceProviderTest.createServiceGroup(controller, cluster1, serviceGroupName, HDP_0_1); String serviceName = "HDFS"; - createService(cluster1, serviceGroupName, serviceName, null); + Service service = createService(cluster1, serviceGroupName, serviceName, null); String componentName1 = "NAMENODE"; String componentName2 = "DATANODE"; String componentName3 = "HDFS_CLIENT"; @@ -5117,9 +5115,9 @@ public class AmbariManagementControllerTest { ConfigurationRequest cr1,cr2,cr3; cr1 = new ConfigurationRequest(cluster1, "core-site","version1", - configs, null, 1L, 1L); + configs, null, service.getServiceId(), 1L); cr2 = new ConfigurationRequest(cluster1, "hdfs-site","version1", - configs, null, 1L, 1L); + configs, null, service.getServiceId(), 1L); ClusterRequest crReq = new ClusterRequest(cluster.getClusterId(), cluster1, null, null); crReq.setDesiredConfig(Collections.singletonList(cr1)); @@ -5144,7 +5142,7 @@ public class AmbariManagementControllerTest { // Reconfigure cr3 = new ConfigurationRequest(cluster1, "core-site","version122", - configs2, null, 1L, 1L); + configs2, null, service.getServiceId(), 1L); crReq = new ClusterRequest(cluster.getClusterId(), cluster1, null, null); crReq.setDesiredConfig(Collections.singletonList(cr3)); controller.updateClusters(Collections.singleton(crReq), null); @@ -5677,7 +5675,7 @@ public class AmbariManagementControllerTest { String serviceGroupName = "CORE"; ServiceGroupResourceProviderTest.createServiceGroup(controller, cluster1, serviceGroupName, "HDP-2.0.7"); String serviceName = "HDFS"; - createService(cluster1, serviceGroupName, serviceName, null); + Service service = createService(cluster1, serviceGroupName, serviceName, null); String componentName1 = "NAMENODE"; String componentName2 = "DATANODE"; String componentName3 = "HDFS_CLIENT"; @@ -5711,7 +5709,7 @@ public class AmbariManagementControllerTest { ConfigurationRequest cr1; cr1 = new ConfigurationRequest(cluster1, "hdfs-site","version1", - configs, null, 1L, 1L); + configs, null, service.getServiceId(), 1L); ClusterRequest crReq = new ClusterRequest(cluster.getClusterId(), cluster1, null, null); crReq.setDesiredConfig(Collections.singletonList(cr1)); controller.updateClusters(Collections.singleton(crReq), null); @@ -6111,7 +6109,7 @@ public class AmbariManagementControllerTest { String serviceGroupName = "CORE"; ServiceGroupResourceProviderTest.createServiceGroup(controller, cluster1, serviceGroupName, HDP_0_1); String serviceName = "HDFS"; - createService(cluster1, serviceGroupName, serviceName, null); + Service service = createService(cluster1, serviceGroupName, serviceName, null); String componentName1 = "NAMENODE"; String componentName2 = "DATANODE"; String componentName3 = "HDFS_CLIENT"; @@ -6139,9 +6137,9 @@ public class AmbariManagementControllerTest { ConfigurationRequest cr1,cr2; cr1 = new ConfigurationRequest(cluster1, "core-site","version1", - configs, null, 1L, 1L); + configs, null, service.getServiceId(), 1L); cr2 = new ConfigurationRequest(cluster1, "hdfs-site","version1", - configs, null, 1L, 1L); + configs, null, service.getServiceId(), 1L); ClusterRequest crReq = new ClusterRequest(cluster.getClusterId(), cluster1, null, null); crReq.setDesiredConfig(Collections.singletonList(cr1)); @@ -6186,7 +6184,7 @@ public class AmbariManagementControllerTest { String serviceGroupName = "CORE"; ServiceGroupResourceProviderTest.createServiceGroup(controller, cluster1, serviceGroupName, HDP_0_1); String serviceName = "HDFS"; - createService(cluster1, serviceGroupName, serviceName, null); + Service service = createService(cluster1, serviceGroupName, serviceName, null); String componentName1 = "NAMENODE"; String componentName2 = "DATANODE"; String componentName3 = "HDFS_CLIENT"; @@ -6214,9 +6212,9 @@ public class AmbariManagementControllerTest { ConfigurationRequest cr1,cr2; cr1 = new ConfigurationRequest(cluster1, "core-site","version1", - configs, null, 1L, 1L); + configs, null, service.getServiceId(), 1L); cr2 = new ConfigurationRequest(cluster1, "hdfs-site","version1", - configs, null, 1L, 1L); + configs, null, service.getServiceId(), 1L); // create, but don't assign controller.createConfiguration(cr1); @@ -6316,8 +6314,8 @@ public class AmbariManagementControllerTest { ServiceGroupResourceProviderTest.createServiceGroup(controller, cluster1, serviceGroupName, "HDP-2.0.6"); String serviceName1 = "HDFS"; String serviceName2 = "MAPREDUCE2"; - createService(cluster1, serviceGroupName, serviceName1, null); - createService(cluster1, serviceGroupName, serviceName2, null); + Service service1 = createService(cluster1, serviceGroupName, serviceName1, null); + Service service2 = createService(cluster1, serviceGroupName, serviceName2, null); String componentName1 = "NAMENODE"; String componentName2 = "DATANODE"; String componentName3 = "HDFS_CLIENT"; @@ -6347,9 +6345,9 @@ public class AmbariManagementControllerTest { configs.put("a", "b"); ConfigurationRequest cr1,cr2,cr3; - cr1 = new ConfigurationRequest(cluster1, "core-site", "version1", configs, null, 1L, 1L); - cr2 = new ConfigurationRequest(cluster1, "hdfs-site", "version1", configs, null, 1L, 1L); - cr3 = new ConfigurationRequest(cluster1, "mapred-site", "version1", configs, null, 1L, 1L); + cr1 = new ConfigurationRequest(cluster1, "core-site", "version1", configs, null, service1.getServiceId(), 1L); + cr2 = new ConfigurationRequest(cluster1, "hdfs-site", "version1", configs, null, service1.getServiceId(), 1L); + cr3 = new ConfigurationRequest(cluster1, "mapred-site", "version1", configs, null, service2.getServiceId(), 1L); ClusterRequest crReq = new ClusterRequest(cluster.getClusterId(), cluster1, null, null); crReq.setDesiredConfig(Collections.singletonList(cr1)); @@ -6568,7 +6566,7 @@ public class AmbariManagementControllerTest { String serviceGroupName = "CORE"; ServiceGroupResourceProviderTest.createServiceGroup(controller, cluster1, serviceGroupName, HDP_0_1); String serviceName = "HDFS"; - createService(cluster1, serviceGroupName, serviceName, null); + Service service = createService(cluster1, serviceGroupName, serviceName, null); String componentName1 = "NAMENODE"; String componentName2 = "DATANODE"; String componentName3 = "HDFS_CLIENT"; @@ -6595,9 +6593,9 @@ public class AmbariManagementControllerTest { ConfigurationRequest cr1,cr2; cr1 = new ConfigurationRequest(cluster1, "core-site","version1", - configs, null, 1L, 1L); + configs, null, service.getServiceId(), 1L); cr2 = new ConfigurationRequest(cluster1, "hdfs-site","version1", - configs, null, 1L, 1L); + configs, null, service.getServiceId(), 1L); ClusterRequest crReq = new ClusterRequest(cluster.getClusterId(), cluster1, null, null); crReq.setDesiredConfig(Collections.singletonList(cr1)); @@ -8012,16 +8010,13 @@ public class AmbariManagementControllerTest { String serviceGroupName = "CORE"; ServiceGroupResourceProviderTest.createServiceGroup(controller, cluster1, serviceGroupName, "HDP-1.2.0"); - Set<ServiceRequest> serviceRequests = new HashSet<>(); - serviceRequests.add(new ServiceRequest(cluster1, serviceGroupName, "HDFS", null, null, null)); - - ServiceResourceProviderTest.createServices(amc, serviceRequests); + Service service = createService(cluster1, serviceGroupName, "HDFS", null); Type confType = new TypeToken<Map<String, String>>() { }.getType(); ConfigurationRequest configurationRequest = new ConfigurationRequest(cluster1, type, tag, - gson.fromJson("{ \"fs.default.name\" : \"localhost:8020\"}", confType), null, 1L, 1L); + gson.fromJson("{ \"fs.default.name\" : \"localhost:8020\"}", confType), null, service.getServiceId(), 1L); amc.createConfiguration(configurationRequest); amc.createConfiguration(configurationRequest); @@ -8064,7 +8059,7 @@ public class AmbariManagementControllerTest { long clusterId = clusters.getCluster(clusterName).getClusterId(); ConfigurationRequest configRequest = new ConfigurationRequest(clusterName, "global", "version1", - new HashMap<String, String>() {{ put("a", "b"); }}, null, 1L, 1L); + new HashMap<String, String>() {{ put("a", "b"); }}, null, null, 1L); ClusterRequest ur = new ClusterRequest(clusterId, clusterName, stackId, null); ur.setDesiredConfig(Collections.singletonList(configRequest)); amc.updateClusters(Collections.singleton(ur), new HashMap<>()); @@ -8162,23 +8157,21 @@ public class AmbariManagementControllerTest { Set<ServiceRequest> serviceRequests = new HashSet<>(); String serviceName = "HDFS"; - serviceRequests.add(new ServiceRequest(cluster1, serviceGroupName, serviceName, null, null, null)); - serviceRequests.add(new ServiceRequest(cluster1, serviceGroupName, "HIVE", null, null, null)); - - ServiceResourceProviderTest.createServices(amc, serviceRequests); + Service service = createService(cluster1, serviceGroupName, serviceName, null); + createService(cluster1, serviceGroupName, "HIVE", null); Type confType = new TypeToken<Map<String, String>>() {}.getType(); ConfigurationRequest configurationRequest = new ConfigurationRequest(cluster1, "core-site", "version1", - gson.fromJson("{ \"fs.default.name\" : \"localhost:8020\"}", confType), null, 1L, 1L); + gson.fromJson("{ \"fs.default.name\" : \"localhost:8020\"}", confType), null, service.getServiceId(), 1L); amc.createConfiguration(configurationRequest); configurationRequest = new ConfigurationRequest(cluster1, "hdfs-site", "version1", - gson.fromJson("{ \"dfs.datanode.data.dir.perm\" : \"750\"}", confType), null, 1L, 1L); + gson.fromJson("{ \"dfs.datanode.data.dir.perm\" : \"750\"}", confType), null, service.getServiceId(), 1L); amc.createConfiguration(configurationRequest); configurationRequest = new ConfigurationRequest(cluster1, "global", "version1", - gson.fromJson("{ \"hive.server2.enable.doAs\" : \"true\"}", confType), null, 1L, 1L); + gson.fromJson("{ \"hive.server2.enable.doAs\" : \"true\"}", confType), null, null, 1L); amc.createConfiguration(configurationRequest); Assert.assertTrue(clusters.getCluster(cluster1).getDesiredConfigs().containsKey("hive-site")); @@ -8396,13 +8389,13 @@ public class AmbariManagementControllerTest { org.junit.Assert.assertEquals(1, ServiceResourceProviderTest.getServices(amc, serviceRequests).size()); //Create new configs configurationRequest = new ConfigurationRequest(cluster1, "core-site", "version2", - gson.fromJson("{ \"fs.default.name\" : \"localhost:8020\"}", confType), null, 1L, 1L); + gson.fromJson("{ \"fs.default.name\" : \"localhost:8020\"}", confType), null, service.getServiceId(), 1L); amc.createConfiguration(configurationRequest); configurationRequest = new ConfigurationRequest(cluster1, "hdfs-site", "version2", - gson.fromJson("{ \"dfs.datanode.data.dir.perm\" : \"750\"}", confType), null, 1L, 1L); + gson.fromJson("{ \"dfs.datanode.data.dir.perm\" : \"750\"}", confType), null, service.getServiceId(), 1L); amc.createConfiguration(configurationRequest); configurationRequest = new ConfigurationRequest(cluster1, "global", "version2", - gson.fromJson("{ \"hbase_hdfs_root_dir\" : \"/apps/hbase/\"}", confType), null, 1L, 1L); + gson.fromJson("{ \"hbase_hdfs_root_dir\" : \"/apps/hbase/\"}", confType), null, null, 1L); amc.createConfiguration(configurationRequest); //Add configs to service serviceRequests.clear(); @@ -9487,21 +9480,21 @@ public class AmbariManagementControllerTest { // test null map with no prior cr.setDesiredConfig(Collections.singletonList( - new ConfigurationRequest(cluster1, "typeA", "v1", null, null, 1L, 1L))); + new ConfigurationRequest(cluster1, "typeA", "v1", null, null, null, 1L))); controller.updateClusters(Collections.singleton(cr), new HashMap<>()); Config config = cluster.getDesiredConfigByType("typeA"); Assert.assertNull(config); // test empty map with no prior cr.setDesiredConfig(Collections.singletonList( - new ConfigurationRequest(cluster1, "typeA", "v1", new HashMap<>(), new HashMap<>(), 1L, 1L))); + new ConfigurationRequest(cluster1, "typeA", "v1", new HashMap<>(), new HashMap<>(), null, 1L))); controller.updateClusters(Collections.singleton(cr), new HashMap<>()); config = cluster.getDesiredConfigByType("typeA"); Assert.assertNotNull(config); // test empty properties on a new version cr.setDesiredConfig(Collections.singletonList( - new ConfigurationRequest(cluster1, "typeA", "v2", new HashMap<>(), new HashMap<>(), 1L, 1L))); + new ConfigurationRequest(cluster1, "typeA", "v2", new HashMap<>(), new HashMap<>(), null, 1L))); controller.updateClusters(Collections.singleton(cr), new HashMap<>()); config = cluster.getDesiredConfigByType("typeA"); Assert.assertNotNull(config); @@ -9515,7 +9508,7 @@ public class AmbariManagementControllerTest { attributesMap.put("final", new HashMap<>()); attributesMap.get("final").put("c", "true"); cr.setDesiredConfig(Collections.singletonList( - new ConfigurationRequest(cluster1, "typeA", "v3", map, attributesMap, 1L, 1L))); + new ConfigurationRequest(cluster1, "typeA", "v3", map, attributesMap, null, 1L))); controller.updateClusters(Collections.singleton(cr), new HashMap<>()); config = cluster.getDesiredConfigByType("typeA"); Assert.assertNotNull(config); @@ -9523,7 +9516,7 @@ public class AmbariManagementControllerTest { // test reset to v2 cr.setDesiredConfig(Collections.singletonList( - new ConfigurationRequest(cluster1, "typeA", "v2", new HashMap<>(), new HashMap<>(), 1L, 1L))); + new ConfigurationRequest(cluster1, "typeA", "v2", new HashMap<>(), new HashMap<>(), null, 1L))); controller.updateClusters(Collections.singleton(cr), new HashMap<>()); config = cluster.getDesiredConfigByType("typeA"); Assert.assertEquals("v2", config.getTag()); @@ -9535,7 +9528,7 @@ public class AmbariManagementControllerTest { new ConfigurationRequest(cluster1, "typeA", "v2", new HashMap<String, String>() {{ put("a", "b"); }}, new HashMap<String, Map<String,String>>(){{put("final", new HashMap<String, String>(){{put("a", "true");}}); } - }, 1L, 1L))); + }, null, 1L))); try { controller.updateClusters(Collections.singleton(cr), new HashMap<>()); Assert.fail("Expect failure when creating a config that exists"); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java index 71e2280..31c6ec3 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java @@ -239,7 +239,7 @@ public class AmbariContextTest { expect(blueprint.getName()).andReturn(BP_NAME).anyTimes(); expect(topology.getStack()).andReturn(stack).anyTimes(); expect(topology.getStackIds()).andReturn(Collections.singleton(STACK_ID)).anyTimes(); - expect(topology.getServices()).andReturn(blueprintServices).anyTimes(); + expect(topology.getServiceTypes()).andReturn(blueprintServices).anyTimes(); expect(topology.getComponents()).andAnswer(() -> Stream.of( builderFor("service1", "s1Component1").stackId(STACK_ID).buildPartial(), builderFor("service1", "s1Component2").stackId(STACK_ID).buildPartial(), diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java index 2e67538..9793003 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java @@ -255,7 +255,7 @@ public class ClusterConfigurationRequestTest { expect(stack.getExcludedConfigurationTypes(anyString())).andReturn(Collections.emptySet()).anyTimes(); expect(stack.getConfigurationPropertiesWithMetadata(anyString(), anyString())).andReturn(Collections.emptyMap()).anyTimes(); - expect(topology.getServices()).andReturn(SERVICE_NAMES).anyTimes(); + expect(topology.getServiceTypes()).andReturn(SERVICE_NAMES).anyTimes(); expect(stack.getConfiguration(SERVICE_NAMES)).andReturn(stackDefaultConfig).once(); expect(topology.getComponents()).andAnswer(() -> Stream.of( @@ -339,7 +339,7 @@ public class ClusterConfigurationRequestTest { expect(stack.getExcludedConfigurationTypes(anyString())).andReturn(Collections.emptySet()).anyTimes(); expect(stack.getConfigurationPropertiesWithMetadata(anyString(), anyString())).andReturn(Collections.emptyMap()).anyTimes(); Set<String> serviceNames = ImmutableSet.of("HDFS", "KERBEROS", "ZOOKEEPER"); - expect(topology.getServices()).andReturn(serviceNames).anyTimes(); + expect(topology.getServiceTypes()).andReturn(serviceNames).anyTimes(); expect(topology.getAmbariContext()).andReturn(ambariContext).anyTimes(); expect(topology.getComponents()).andAnswer(Stream::empty).anyTimes(); expect(topology.getConfigRecommendationStrategy()).andReturn(ConfigRecommendationStrategy.NEVER_APPLY).anyTimes(); @@ -391,7 +391,7 @@ public class ClusterConfigurationRequestTest { expect(topology.getBlueprint()).andReturn(blueprint).anyTimes(); expect(topology.getHostGroupInfo()).andReturn(hostGroupInfoMap); expect(topology.getStack()).andReturn(stack).anyTimes(); - expect(topology.getServices()).andReturn(services).anyTimes(); + expect(topology.getServiceTypes()).andReturn(services).anyTimes(); expect(topology.isValidConfigType("hdfs-site")).andReturn(true).anyTimes(); expect(topology.isValidConfigType("admin-properties")).andReturn(true).anyTimes(); @@ -441,7 +441,7 @@ public class ClusterConfigurationRequestTest { expect(topology.getBlueprint()).andReturn(blueprint).anyTimes(); expect(topology.getHostGroupInfo()).andReturn(hostGroupInfoMap); expect(topology.getStack()).andReturn(stack).anyTimes(); - expect(topology.getServices()).andReturn(services).anyTimes(); + expect(topology.getServiceTypes()).andReturn(services).anyTimes(); expect(topology.isValidConfigType("hdfs-site")).andReturn(true).anyTimes(); expect(topology.isValidConfigType("cluster-env")).andReturn(true).anyTimes(); @@ -539,7 +539,7 @@ public class ClusterConfigurationRequestTest { expect(topology.getBlueprint()).andReturn(blueprint).anyTimes(); expect(topology.getHostGroupInfo()).andReturn(hostGroupInfoMap).anyTimes(); expect(topology.getStack()).andReturn(stack).anyTimes(); - expect(topology.getServices()).andReturn(serviceNames).anyTimes(); + expect(topology.getServiceTypes()).andReturn(serviceNames).anyTimes(); expect(topology.getConfigRecommendationStrategy()).andReturn(ConfigRecommendationStrategy.NEVER_APPLY).anyTimes(); expect(topology.isValidConfigType("zoo.cfg")).andReturn(true).anyTimes(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeploymentTestCommon.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeploymentTestCommon.java index b23d88e..960617c 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeploymentTestCommon.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeploymentTestCommon.java @@ -32,7 +32,7 @@ public class ClusterDeploymentTestCommon { * @return a {@link ServiceResponse} instance for tests */ static ServiceResponse service(String serviceName, long serviceId) { - return new ServiceResponse(CLUSTER_ID, CLUSTER_NAME, 1L, "service-group-1", serviceId, serviceName, null, null, + return new ServiceResponse(CLUSTER_ID, CLUSTER_NAME, 1L, "service-group-1", serviceId, serviceName, serviceName, null, null, null, true, true, true, true, true); } } diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/StackBuilder.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/StackBuilder.java index a4f608a..6f48bc5 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/StackBuilder.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/StackBuilder.java @@ -17,6 +17,8 @@ */ package org.apache.ambari.server.topology; +import java.util.Optional; + import org.apache.ambari.server.state.AutoDeployInfo; import org.apache.ambari.server.state.ComponentInfo; import org.apache.ambari.server.state.DependencyInfo; @@ -121,6 +123,12 @@ public class StackBuilder { return resolveComponent(currentComponent, currentService, stackId); } + public ResolvedComponent lastAddedComponentWith(String mpackInstanceName, String serviceInstanceName) { + Preconditions.checkNotNull(currentService); + Preconditions.checkNotNull(currentComponent); + return withInstanceNames(resolveComponent(currentComponent, currentService, stackId), mpackInstanceName, serviceInstanceName); + } + public ResolvedComponent componentToBeCoLocatedWith() { Preconditions.checkNotNull(currentComponent); Preconditions.checkNotNull(currentComponent.getAutoDeploy()); @@ -187,4 +195,12 @@ public class StackBuilder { return componentInfo.getAutoDeploy(); } + protected static ResolvedComponent withInstanceNames(ResolvedComponent component, String mpackInstanceName, String serviceInstanceName) { + return component.toBuilder() + .serviceGroupName(Optional.ofNullable(mpackInstanceName)) + .serviceName(Optional.ofNullable(serviceInstanceName)) + .component(new Component(component.componentName(), mpackInstanceName, serviceInstanceName, null)) + .build(); + } + } diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java index b26ffc6..7d5de53 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java @@ -225,7 +225,7 @@ public class TopologyManagerTest { private final Set<ServiceResponse> services = IntStream.range(0, SERVICE_NAMES.size()).boxed(). map( serviceId -> new ServiceResponse(CLUSTER_ID, CLUSTER_NAME, 1L, "CORE", (long)serviceId, SERVICE_NAMES.get(serviceId), - null, null, null, null, false, false, false, false, false) + SERVICE_NAMES.get(serviceId), null, null, null, false, false, false, false, false) ). collect(toSet()); @@ -313,7 +313,7 @@ public class TopologyManagerTest { expect(blueprint.getHostGroupsForComponent("component3")).andReturn(Arrays.asList(group1, group2)).anyTimes(); expect(blueprint.getHostGroupsForComponent("component4")).andReturn(Collections.singleton(group2)).anyTimes(); expect(blueprint.getName()).andReturn(BLUEPRINT_NAME).anyTimes(); - expect(clusterTopologyMock.getServices()).andReturn(SERVICE_NAMES).anyTimes(); + expect(clusterTopologyMock.getServiceTypes()).andReturn(SERVICE_NAMES).anyTimes(); expect(clusterTopologyMock.getStack()).andReturn(stack).anyTimes(); expect(clusterTopologyMock.getSecurity()).andReturn(SecurityConfiguration.NONE).anyTimes(); expect(clusterTopologyMock.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/ClusterConfigTypeValidatorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/ClusterConfigTypeValidatorTest.java index ae14426..b08d362 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/ClusterConfigTypeValidatorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/ClusterConfigTypeValidatorTest.java @@ -94,7 +94,7 @@ public class ClusterConfigTypeValidatorTest extends EasyMockSupport { clusterRequestConfigTypes = new HashSet<>(Arrays.asList("core-site", "yarn-site")); EasyMock.expect(clusterConfigurationMock.getAllConfigTypes()).andReturn(clusterRequestConfigTypes).anyTimes(); - EasyMock.expect(clusterTopologyMock.getServices()).andReturn(new HashSet<>(Arrays.asList("YARN", "HDFS"))); + EasyMock.expect(clusterTopologyMock.getServiceTypes()).andReturn(new HashSet<>(Arrays.asList("YARN", "HDFS"))); EasyMock.expect(stackMock.getConfigurationTypes("HDFS")).andReturn(Arrays.asList("core-site")); EasyMock.expect(stackMock.getConfigurationTypes("YARN")).andReturn(Arrays.asList("yarn-site")); @@ -117,7 +117,7 @@ public class ClusterConfigTypeValidatorTest extends EasyMockSupport { clusterRequestConfigTypes = new HashSet<>(Arrays.asList("oozie-site")); EasyMock.expect(clusterConfigurationMock.getAllConfigTypes()).andReturn(clusterRequestConfigTypes).anyTimes(); - EasyMock.expect(clusterTopologyMock.getServices()).andReturn(new HashSet<>(Arrays.asList("YARN", "HDFS"))); + EasyMock.expect(clusterTopologyMock.getServiceTypes()).andReturn(new HashSet<>(Arrays.asList("YARN", "HDFS"))); EasyMock.expect(stackMock.getConfigurationTypes("HDFS")).andReturn(Arrays.asList("core-site")); EasyMock.expect(stackMock.getConfigurationTypes("YARN")).andReturn(Arrays.asList("yarn-site")); @@ -138,7 +138,7 @@ public class ClusterConfigTypeValidatorTest extends EasyMockSupport { clusterRequestConfigTypes = new HashSet<>(Arrays.asList("core-site", "yarn-site", "oozie-site")); EasyMock.expect(clusterConfigurationMock.getAllConfigTypes()).andReturn(clusterRequestConfigTypes).anyTimes(); - EasyMock.expect(clusterTopologyMock.getServices()).andReturn(new HashSet<>(Arrays.asList("YARN", "HDFS"))); + EasyMock.expect(clusterTopologyMock.getServiceTypes()).andReturn(new HashSet<>(Arrays.asList("YARN", "HDFS"))); EasyMock.expect(stackMock.getConfigurationTypes("HDFS")).andReturn(Arrays.asList("core-site")); EasyMock.expect(stackMock.getConfigurationTypes("YARN")).andReturn(Arrays.asList("yarn-site")); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/DependencyAndCardinalityValidatorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/DependencyAndCardinalityValidatorTest.java index 9775dde..4d28156 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/DependencyAndCardinalityValidatorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/DependencyAndCardinalityValidatorTest.java @@ -20,7 +20,6 @@ package org.apache.ambari.server.topology.validators; import static org.junit.Assert.assertSame; -import org.apache.ambari.server.topology.Component; import org.apache.ambari.server.topology.InvalidTopologyException; import org.apache.ambari.server.topology.ResolvedComponent; import org.apache.ambari.server.topology.StackBuilder; @@ -45,10 +44,7 @@ public class DependencyAndCardinalityValidatorTest extends TopologyValidatorTest @Test public void acceptsComponentWithMpackInstanceSpecified() throws InvalidTopologyException { - ResolvedComponent component = aComponent().withCardinality("2").lastAddedComponent(); - component = component.toBuilder() - .component(new Component(component.componentName(), "mpack_instance", "service_instance", null)) - .build(); + ResolvedComponent component = aComponent().withCardinality("2").lastAddedComponentWith("mpack_instance", "service_instance"); topologyHas(2, component); replayAll(); @@ -56,6 +52,17 @@ public class DependencyAndCardinalityValidatorTest extends TopologyValidatorTest } @Test + public void acceptsMultipleInstances() throws InvalidTopologyException { + StackBuilder stackBuilder = aComponent().withCardinality("1+"); + ResolvedComponent instance1 = stackBuilder.lastAddedComponentWith(null, "instance1"); + ResolvedComponent instance2 = stackBuilder.lastAddedComponentWith(null, "instance2"); + topologyHas(instance1, instance2); + replayAll(); + + assertSame(topology, subject.validate(topology)); + } + + @Test public void acceptsInstancesOnAllHosts() throws InvalidTopologyException { topologyHas(3, aComponent().withCardinality("ALL")); replayAll(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/HiveServiceValidatorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/HiveServiceValidatorTest.java index 0f142f7..b7119fb 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/HiveServiceValidatorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/HiveServiceValidatorTest.java @@ -136,12 +136,12 @@ public class HiveServiceValidatorTest extends EasyMockSupport { expect(topology.getComponents()).andReturn(components.build().stream() .map(component -> builderFor("HIVE", component).buildPartial()) ).anyTimes(); - expect(topology.getServices()).andReturn(ImmutableSet.of("HDFS", "YARN", "HIVE")).anyTimes(); + expect(topology.getServiceTypes()).andReturn(ImmutableSet.of("HDFS", "YARN", "HIVE")).anyTimes(); replay(topology); } private void noHiveInTopology() { - expect(topology.getServices()).andReturn(ImmutableSet.of("HDFS", "YARN")).anyTimes(); + expect(topology.getServiceTypes()).andReturn(ImmutableSet.of("HDFS", "YARN")).anyTimes(); replay(topology); } diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/RequiredConfigPropertiesValidatorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/RequiredConfigPropertiesValidatorTest.java index e249feb..7f82eed 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/RequiredConfigPropertiesValidatorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/RequiredConfigPropertiesValidatorTest.java @@ -99,7 +99,7 @@ public class RequiredConfigPropertiesValidatorTest extends EasyMockSupport { EasyMock.expect(clusterTopologyMock.getBlueprint()).andReturn(blueprintMock).anyTimes(); EasyMock.expect(blueprintMock.getHostGroups()).andReturn(hostGroups); - EasyMock.expect(clusterTopologyMock.getServices()).andReturn(bpServices); + EasyMock.expect(clusterTopologyMock.getServiceTypes()).andReturn(bpServices); EasyMock.expect(clusterTopologyMock.getStack()).andReturn(stackMock).anyTimes(); EasyMock.expect(masterHostGroupMock.getName()).andReturn("master").anyTimes(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/TopologyValidatorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/TopologyValidatorTest.java index bef2a0a..3de99ec 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/TopologyValidatorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/TopologyValidatorTest.java @@ -42,6 +42,7 @@ import org.apache.ambari.server.controller.internal.Stack; import org.apache.ambari.server.controller.internal.StackDefinition; import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.topology.ClusterTopology; +import org.apache.ambari.server.topology.ClusterTopologyImpl; import org.apache.ambari.server.topology.Configuration; import org.apache.ambari.server.topology.InvalidTopologyException; import org.apache.ambari.server.topology.ResolvedComponent; @@ -164,7 +165,7 @@ public class TopologyValidatorTest extends EasyMockSupport { .anyTimes(); Capture<ResolvedComponent> componentCapture = newCapture(); expect(topology.getHostGroupsForComponent(capture(componentCapture))) - .andAnswer(() -> hostGroups.entrySet().stream().filter(each -> each.getValue().contains(componentCapture.getValue())).map(Map.Entry::getKey).collect(toSet())) + .andAnswer(() -> ClusterTopologyImpl.getHostGroupsForComponent(hostGroups, componentCapture.getValue())) .anyTimes(); expect(topology.getHostGroups()).andReturn(hostGroups.keySet()).anyTimes();