This is an automated email from the ASF dual-hosted git repository. benyoka pushed a commit to branch branch-feature-AMBARI-14714 in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/branch-feature-AMBARI-14714 by this push: new 48ddb09 [AMBARI-24162] Support for cluster-settings (remove cluster-env) (benyoka) (#1627) 48ddb09 is described below commit 48ddb0901e2b12ee881698acf8a7b3de5c02799c Author: benyoka <beny...@users.noreply.github.com> AuthorDate: Fri Jun 29 10:23:08 2018 +0200 [AMBARI-24162] Support for cluster-settings (remove cluster-env) (benyoka) (#1627) * AMBARI-24162 copy cluster settings from cluster-env + setting fixes (benyoka) * AMBARI-24162 fixint tests WIP (benyoka) * AMBARI-24162 fix unit tests #2 (benyoka) * AMBARI-24162 write new unit tests (benyoka) * AMBARI-24162 review commits and unit test fixes (benyoka) * AMBARI-24162 fix blueprint export and blueprint unit tests (benyoka) --- .../controller/AmbariManagementControllerImpl.java | 10 +- .../internal/BlueprintConfigurationProcessor.java | 48 -------- .../internal/BlueprintResourceProvider.java | 32 +++--- .../apache/ambari/server/state/ConfigHelper.java | 7 +- .../ambari/server/topology/AmbariContext.java | 44 ++++--- .../ambari/server/topology/BlueprintFactory.java | 8 +- .../ambari/server/topology/BlueprintImpl.java | 2 +- .../server/topology/ClusterTopologyImpl.java | 52 +++++++++ .../org/apache/ambari/server/topology/Setting.java | 20 ++++ .../ambari/server/topology/SettingFactory.java | 7 +- .../topology/tasks/ConfigureClusterTask.java | 1 + .../validators/StackConfigTypeValidator.java | 7 ++ .../src/main/resources/cluster-settings.xml | 21 ++++ .../BlueprintConfigurationProcessorTest.java | 101 +++------------- .../ambari/server/topology/AmbariContextTest.java | 128 ++++++++++++++------- .../topology/ClusterConfigurationRequestTest.java | 18 ++- .../topology/ClusterDeployWithStartOnlyTest.java | 55 +++++---- .../topology/ClusterDeploymentTestCommon.java | 38 ++++++ ...terInstallWithoutStartOnComponentLevelTest.java | 48 +++++--- .../topology/ClusterInstallWithoutStartTest.java | 51 +++++--- .../server/topology/ClusterTopologyImplTest.java | 50 +++++++- .../ambari/server/topology/SettingFactoryTest.java | 27 +++++ .../apache/ambari/server/topology/SettingTest.java | 35 ++++++ .../server/topology/TopologyManagerTest.java | 9 +- 24 files changed, 524 insertions(+), 295 deletions(-) diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java index aa92bac..31dea50 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java @@ -2588,15 +2588,11 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle (commandParams.get(CLUSTER_PHASE_PROPERTY).equals(CLUSTER_PHASE_INITIAL_INSTALL) || commandParams.get(CLUSTER_PHASE_PROPERTY).equals(CLUSTER_PHASE_INITIAL_START))) { String retryEnabledStr = - configHelper.getValueFromDesiredConfigurations(cluster, ConfigHelper.CLUSTER_ENV, - ConfigHelper.CLUSTER_ENV_RETRY_ENABLED); + cluster.getClusterSetting(ConfigHelper.COMMAND_RETRY_ENABLED).getClusterSettingValue(); String commandsStr = - configHelper.getValueFromDesiredConfigurations(cluster, ConfigHelper.CLUSTER_ENV, - ConfigHelper.CLUSTER_ENV_RETRY_COMMANDS); + cluster.getClusterSetting(ConfigHelper.COMMANDS_TO_RETRY).getClusterSettingValue(); String retryMaxTimeStr = - configHelper.getValueFromDesiredConfigurations(cluster, - ConfigHelper.CLUSTER_ENV, - ConfigHelper.CLUSTER_ENV_RETRY_MAX_TIME_IN_SEC); + cluster.getClusterSetting(ConfigHelper.COMMAND_RETRY_MAX_TIME_IN_SEC).getClusterSettingValue(); if (StringUtils.isNotEmpty(retryEnabledStr)) { retryEnabled = Boolean.TRUE.toString().equals(retryEnabledStr); } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java index 8275cfc..5c45712 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java @@ -74,18 +74,6 @@ public class BlueprintConfigurationProcessor { */ public static final Pattern HOST_GROUP_PLACEHOLDER_PATTERN = Pattern.compile("%HOSTGROUP::(\\S+?)%"); - private final static String COMMAND_RETRY_ENABLED_PROPERTY_NAME = "command_retry_enabled"; - - private final static String COMMANDS_TO_RETRY_PROPERTY_NAME = "commands_to_retry"; - - private final static String COMMAND_RETRY_MAX_TIME_IN_SEC_PROPERTY_NAME = "command_retry_max_time_in_sec"; - - private final static String COMMAND_RETRY_ENABLED_DEFAULT = "true"; - - private final static String COMMANDS_TO_RETRY_DEFAULT = "INSTALL,START"; - - private final static String COMMAND_RETRY_MAX_TIME_IN_SEC_DEFAULT = "600"; - private final static String CLUSTER_ENV_CONFIG_TYPE_NAME = "cluster-env"; private final static String HBASE_SITE_HBASE_COPROCESSOR_MASTER_CLASSES = "hbase.coprocessor.master.classes"; @@ -449,7 +437,6 @@ public class BlueprintConfigurationProcessor { // Explicitly set any properties that are required but not currently provided in the stack definition. setStackToolsAndFeatures(clusterConfig, configTypesUpdated); - setRetryConfiguration(clusterConfig, configTypesUpdated); setupHDFSProxyUsers(clusterConfig, configTypesUpdated); addExcludedConfigProperties(clusterConfig, configTypesUpdated, clusterTopology.getStack()); @@ -2956,41 +2943,6 @@ public class BlueprintConfigurationProcessor { } /** - * This method ensures that Ambari command retry is enabled if not explicitly overridden in - * cluster-env by the Blueprint or Cluster Creation template. The new dynamic provisioning model - * requires that retry be enabled for most multi-node clusters, to this method sets reasonable defaults - * in order to preserve backwards compatibility and to simplify Blueprint creation. - * - * If the retry-specific properties in cluster-env are not set, then the config processor - * will set these values to defaults in cluster-env. - * - * @param configuration cluster configuration - */ - private static void setRetryConfiguration(Configuration configuration, Set<String> configTypesUpdated) { - boolean wasUpdated = false; - - if (configuration.getPropertyValue(CLUSTER_ENV_CONFIG_TYPE_NAME, COMMAND_RETRY_ENABLED_PROPERTY_NAME) == null) { - configuration.setProperty(CLUSTER_ENV_CONFIG_TYPE_NAME, COMMAND_RETRY_ENABLED_PROPERTY_NAME, COMMAND_RETRY_ENABLED_DEFAULT); - wasUpdated = true; - } - - if (configuration.getPropertyValue(CLUSTER_ENV_CONFIG_TYPE_NAME, COMMANDS_TO_RETRY_PROPERTY_NAME) == null) { - configuration.setProperty(CLUSTER_ENV_CONFIG_TYPE_NAME, COMMANDS_TO_RETRY_PROPERTY_NAME, COMMANDS_TO_RETRY_DEFAULT); - wasUpdated = true; - } - - if (configuration.getPropertyValue(CLUSTER_ENV_CONFIG_TYPE_NAME, COMMAND_RETRY_MAX_TIME_IN_SEC_PROPERTY_NAME) == null) { - configuration.setProperty(CLUSTER_ENV_CONFIG_TYPE_NAME, COMMAND_RETRY_MAX_TIME_IN_SEC_PROPERTY_NAME, COMMAND_RETRY_MAX_TIME_IN_SEC_DEFAULT); - wasUpdated = true; - } - - if (wasUpdated) { - configTypesUpdated.add(CLUSTER_ENV_CONFIG_TYPE_NAME); - } - } - - - /** * Sets the read-only properties for stack features & tools, overriding * anything provided in the blueprint. * diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java index 17aed41..22f367c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java @@ -393,23 +393,29 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide if(config instanceof BlueprintConfigEntity) { Map<String, String> properties = JsonUtils.fromJson(config.getConfigData(), new TypeReference<Map<String, String>>(){}); - // TODO: use multiple mpacks - BlueprintMpackInstanceEntity mpack = - ((BlueprintConfigEntity)config).getBlueprintEntity().getMpackInstances().iterator().next(); - StackInfo metaInfoStack; - - try { - metaInfoStack = ambariMetaInfo.getStack(mpack.getMpackName(), mpack.getMpackVersion()); - } catch (AmbariException e) { - throw new NoSuchResourceException(e.getMessage()); + // TODO: use multiple mpacks, + make it work with blueprints with no mpack + Collection<BlueprintMpackInstanceEntity> mpackInstances = ((BlueprintConfigEntity) config).getBlueprintEntity().getMpackInstances(); + if (mpackInstances.isEmpty()) { + LOG.warn("This blueprint does not specify any mpacks/stacks. Configurations cannot be retrieved."); } + else { + BlueprintMpackInstanceEntity mpack = + ((BlueprintConfigEntity)config).getBlueprintEntity().getMpackInstances().iterator().next(); + StackInfo metaInfoStack; + + try { + metaInfoStack = ambariMetaInfo.getStack(mpack.getMpackName(), mpack.getMpackVersion()); + } catch (AmbariException e) { + throw new NoSuchResourceException(e.getMessage()); + } - Map<org.apache.ambari.server.state.PropertyInfo.PropertyType, Set<String>> propertiesTypes = - metaInfoStack.getConfigPropertiesTypes(type); + Map<org.apache.ambari.server.state.PropertyInfo.PropertyType, Set<String>> propertiesTypes = + metaInfoStack.getConfigPropertiesTypes(type); - SecretReference.replacePasswordsWithReferences(propertiesTypes, properties, type, -1L); + SecretReference.replacePasswordsWithReferences(propertiesTypes, properties, type, -1L); - configTypeDefinition.put(PROPERTIES_PROPERTY_ID, properties); + configTypeDefinition.put(PROPERTIES_PROPERTY_ID, properties); + } } else { Map<String, Object> properties = JsonUtils.fromJson(config.getConfigData(), new TypeReference<Map<String, Object>>(){}); configTypeDefinition.put(PROPERTIES_PROPERTY_ID, properties); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java index f918cdb..e15a87a 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java @@ -105,15 +105,14 @@ public class ConfigHelper { */ public static final String HBASE_SITE = "hbase-site"; public static final String HDFS_SITE = "hdfs-site"; - public static final String HIVE_SITE = "hive-site"; public static final String YARN_SITE = "yarn-site"; public static final String CLUSTER_ENV = "cluster-env"; public static final String CLUSTER_SETTINGS = "cluster-settings"; public static final String CLUSTER_ENV_ALERT_REPEAT_TOLERANCE = "alerts_repeat_tolerance"; - public static final String CLUSTER_ENV_RETRY_ENABLED = "command_retry_enabled"; + public static final String COMMAND_RETRY_ENABLED = "command_retry_enabled"; public static final String SERVICE_CHECK_TYPE = "service_check_type"; - public static final String CLUSTER_ENV_RETRY_COMMANDS = "commands_to_retry"; - public static final String CLUSTER_ENV_RETRY_MAX_TIME_IN_SEC = "command_retry_max_time_in_sec"; + public static final String COMMANDS_TO_RETRY = "commands_to_retry"; + public static final String COMMAND_RETRY_MAX_TIME_IN_SEC = "command_retry_max_time_in_sec"; public static final String COMMAND_RETRY_MAX_TIME_IN_SEC_DEFAULT = "600"; public static final String CLUSTER_ENV_STACK_NAME_PROPERTY = "stack_name"; public static final String CLUSTER_ENV_STACK_FEATURES_PROPERTY = "stack_features"; diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java index 9071920..021449d 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java @@ -18,13 +18,13 @@ package org.apache.ambari.server.topology; +import static java.util.stream.Collectors.toMap; import static java.util.stream.Collectors.toSet; import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.Set; import java.util.SortedSet; @@ -50,7 +50,6 @@ import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.controller.AmbariManagementController; import org.apache.ambari.server.controller.ClusterRequest; import org.apache.ambari.server.controller.ConfigGroupRequest; -import org.apache.ambari.server.controller.ConfigurationRequest; import org.apache.ambari.server.controller.RequestStatusResponse; import org.apache.ambari.server.controller.RootComponent; import org.apache.ambari.server.controller.ServiceComponentHostRequest; @@ -59,7 +58,6 @@ import org.apache.ambari.server.controller.ServiceGroupRequest; import org.apache.ambari.server.controller.ServiceGroupResponse; import org.apache.ambari.server.controller.ServiceRequest; import org.apache.ambari.server.controller.ServiceResponse; -import org.apache.ambari.server.controller.internal.AbstractResourceProvider; import org.apache.ambari.server.controller.internal.ComponentResourceProvider; import org.apache.ambari.server.controller.internal.ConfigGroupResourceProvider; import org.apache.ambari.server.controller.internal.ExportBlueprintRequest; @@ -90,6 +88,7 @@ import org.apache.ambari.server.state.Host; import org.apache.ambari.server.state.PropertyInfo; import org.apache.ambari.server.state.SecurityType; import org.apache.ambari.server.state.StackId; +import org.apache.ambari.server.state.State; import org.apache.ambari.server.state.configgroup.ConfigGroup; import org.apache.ambari.server.utils.RetryHelper; import org.slf4j.Logger; @@ -222,21 +221,21 @@ public class AmbariContext { public void createAmbariResources(ClusterTopology topology, String clusterName, SecurityType securityType) { Set<StackId> stackIds = topology.getStackIds(); - createAmbariClusterResource(clusterName, stackIds, securityType); + createAmbariClusterResource(clusterName, stackIds, securityType, topology.getSetting().getClusterSettings()); createAmbariServiceAndComponentResources(topology, clusterName); } - private void createAmbariClusterResource(String clusterName, Set<StackId> stackIds, SecurityType securityType) { + private void createAmbariClusterResource(String clusterName, Set<StackId> stackIds, SecurityType securityType, + Map<String, String> clusterSettings) { String stackInfo = stackIds.iterator().next().toString(); // temporary final ClusterRequest clusterRequest = new ClusterRequest(null, clusterName, null, securityType, stackInfo, null); - try { RetryHelper.executeWithRetry(() -> { getController().createCluster(clusterRequest); return null; }); - addDefaultClusterSettings(clusterName); + addClusterSettings(clusterName, clusterSettings); } catch (AmbariException e) { LOG.error("Failed to create Cluster resource: ", e); if (e.getCause() instanceof DuplicateResourceException) { @@ -248,10 +247,17 @@ public class AmbariContext { } // FIXME temporarily add default cluster settings -- should be provided by ClusterImpl itself - private void addDefaultClusterSettings(String clusterName) throws AmbariException { + private void addClusterSettings(String clusterName, Map<String, String> clusterSettings) throws AmbariException { Cluster cluster = getController().getClusters().getCluster(clusterName); - for (PropertyInfo p : getController().getAmbariMetaInfo().getClusterProperties()) { - cluster.addClusterSetting(p.getName(), p.getValue()); + Map<String, String> defaultClusterSettings = + getController().getAmbariMetaInfo().getClusterProperties().stream() + .collect(toMap(PropertyInfo::getName, PropertyInfo::getValue)); + + // Override default settings with those coming from blueprint / cluster template + defaultClusterSettings.putAll(clusterSettings); + + for (Map.Entry<String, String> setting : defaultClusterSettings.entrySet()) { + cluster.addClusterSetting(setting.getKey(), setting.getValue()); } } @@ -268,8 +274,16 @@ public class AmbariContext { .collect(toSet()); Set<ServiceComponentRequest> componentRequests = topology.getComponents() - .map(c -> new ServiceComponentRequest(clusterName, c.effectiveServiceGroupName(), c.effectiveServiceName(), c.componentName(), c.componentName(), - topology.getSetting().getRecoveryEnabled(c.effectiveServiceName(), c.componentName()))) // FIXME settings by service type or name? + .map( c -> + new ServiceComponentRequest( + clusterName, + c.effectiveServiceGroupName(), + c.effectiveServiceName(), + c.componentName(), + c.componentName(), + State.INIT.name(), + topology.getSetting().getRecoveryEnabled(c.effectiveServiceName(), c.componentName())) + ) // FIXME settings by service type or name? .collect(toSet()); try { @@ -399,7 +413,7 @@ public class AmbariContext { } public void downloadMissingMpacks(Set<MpackInstance> mpacks) { - ResourceProvider resourceProvider = getClusterController().ensureResourceProvider(Resource.Type.Mpack); + ResourceProvider resourceProvider = getClusterController().ensureResourceProvider(Resource.Type.Mpack); new DownloadMpacksTask(resourceProvider, ambariMetaInfo.get()).downloadMissingMpacks(mpacks); } @@ -472,10 +486,6 @@ public class AmbariContext { } } - //todo: non topology type shouldn't be returned - public List<ConfigurationRequest> createConfigurationRequests(Map<String, Object> clusterProperties) { - return AbstractResourceProvider.getConfigurationRequests("Clusters", clusterProperties); - } public void setConfigurationOnCluster(final ClusterRequest clusterRequest) { try { diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java index 0f13539..9fc08b7 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java @@ -18,6 +18,7 @@ package org.apache.ambari.server.topology; +import static java.util.stream.Collectors.toMap; import static java.util.stream.Collectors.toSet; import static org.apache.ambari.server.controller.internal.BlueprintResourceProvider.BLUEPRINT_NAME_PROPERTY_ID; import static org.apache.ambari.server.controller.internal.BlueprintResourceProvider.COMPONENT_MPACK_INSTANCE_PROPERTY; @@ -57,6 +58,7 @@ import org.slf4j.LoggerFactory; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableSet; /** * Create a Blueprint instance. @@ -115,7 +117,11 @@ public class BlueprintFactory { Collection<HostGroup> hostGroups = processHostGroups(properties); Configuration configuration = configFactory.getConfiguration((Collection<Map<String, String>>) properties.get(CONFIGURATION_PROPERTY_ID)); - Setting setting = SettingFactory.getSetting((Collection<Map<String, Object>>) properties.get(SETTING_PROPERTY_ID)); + + Map<String, Object> settingProperties = properties.entrySet().stream() + .filter(entry -> SETTING_PROPERTY_ID.equals(entry.getKey()) || entry.getKey().startsWith(SETTING_PROPERTY_ID + "/")) + .collect(toMap(Map.Entry::getKey, Map.Entry::getValue)); + Setting setting = SettingFactory.getSetting(ImmutableSet.of(settingProperties)); return new BlueprintImpl(name, hostGroups, stackIds, mpackInstances, configuration, securityConfiguration, setting); } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java index 7e60cb9..676077a 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java @@ -91,7 +91,7 @@ public class BlueprintImpl implements Blueprint { hostGroups.put(hostGroup.getName(), hostGroup); } this.configuration = configuration; - this.setting = setting != null ? setting : new Setting(ImmutableMap.of()); + this.setting = setting != null ? setting : new Setting(new HashMap<>()); repoSettings = processRepoSettings(); } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java index 428571b..8e2a044 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java @@ -43,6 +43,7 @@ import org.apache.ambari.server.controller.internal.ExportBlueprintRequest; import org.apache.ambari.server.controller.internal.ProvisionAction; import org.apache.ambari.server.controller.internal.StackDefinition; import org.apache.ambari.server.state.ConfigHelper; +import org.apache.ambari.server.state.PropertyInfo; import org.apache.ambari.server.state.StackId; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -98,6 +99,56 @@ public class ClusterTopologyImpl implements ClusterTopology { checkForDuplicateHosts(topologyRequest.getHostGroupInfo()); registerHostGroupInfo(topologyRequest.getHostGroupInfo()); + adjustTopology(); + } + + /** + * This is to collect configurations formerly (in Ambari 2.x) belonging to cluster-env and already migrated to + * cluster settings. Eventually all configurations from cluster-env should be migrated and this collection + * should be removed. + */ + private static final Set<String> SAFE_TO_REMOVE_FROM_CLUSTER_ENV = ImmutableSet.of( + ConfigHelper.COMMAND_RETRY_ENABLED, + ConfigHelper.COMMAND_RETRY_MAX_TIME_IN_SEC, + ConfigHelper.COMMANDS_TO_RETRY + ); + + /** + * This method adjusts cluster topologies coming from the Ambari 2.x blueprint structure for Ambari + * 3.x. + * Currently it extract configuration from cluster-env and transforms it into cluster settings. + */ + private void adjustTopology() { + Set<PropertyInfo> clusterProperties = ambariContext.getController().getAmbariMetaInfo().getClusterProperties(); + Set<String> clusterSettingPropertyNames = clusterProperties.stream().map(PropertyInfo::getName).collect(toSet()); + Map<String, String> clusterEnv = + configuration.getFullProperties().getOrDefault(ConfigHelper.CLUSTER_ENV, ImmutableMap.of()); + + Set<String> propertiesToConvert = Sets.intersection(clusterEnv.keySet(), clusterSettingPropertyNames); + Set<String> remainingProperties = Sets.difference(clusterEnv.keySet(), clusterSettingPropertyNames); + LOG.info("Will convert {} properties from cluster-env to cluster settings, leave {} as is. Properties to convert: {}," + + " remaining properties: {}", propertiesToConvert.size(), remainingProperties.size(), propertiesToConvert, + remainingProperties); + + // Get cluster_settings from setting or create if not exists + Map<String, String> clusterSettings = setting + .getProperties() + .computeIfAbsent( Setting.SETTING_NAME_CLUSTER_SETTINGS, __ -> { + Set<Map<String, String>> set = new HashSet<>(); + set.add(new HashMap<>()); + return set; + }) + .iterator() + .next(); + + // convert cluster-env to cluster settings + propertiesToConvert.forEach( prop -> clusterSettings.put(prop, clusterEnv.get(prop))); + + // Ideally, all converted properties should be removed from cluster-env. Since legacy Ambari code sometimes + // still uses cluster-env only remove properties that are known to have been migrated. + Sets.intersection(propertiesToConvert, SAFE_TO_REMOVE_FROM_CLUSTER_ENV).forEach( + prop -> configuration.removeProperty(ConfigHelper.CLUSTER_ENV, prop) + ); } public ClusterTopologyImpl( @@ -124,6 +175,7 @@ public class ClusterTopologyImpl implements ClusterTopology { checkForDuplicateHosts(request.getHostGroupInfo()); registerHostGroupInfo(request.getHostGroupInfo()); + adjustTopology(); } public ClusterTopologyImpl withAdditionalComponents(Map<String, Set<ResolvedComponent>> additionalComponents) throws InvalidTopologyException { diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/Setting.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/Setting.java index cf13473..1973d83 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/Setting.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/Setting.java @@ -18,7 +18,9 @@ package org.apache.ambari.server.topology; +import static java.util.Collections.emptySet; import static java.util.stream.Collectors.toList; +import static java.util.stream.Collectors.toMap; import java.util.List; import java.util.Map; @@ -35,6 +37,7 @@ public class Setting { */ private final Map<String, Set<Map<String, String>>> properties; + static final String SETTING_NAME_CLUSTER_SETTINGS = "cluster_settings"; static final String SETTING_NAME_RECOVERY_SETTINGS = "recovery_settings"; static final String SETTING_NAME_SERVICE_SETTINGS = "service_settings"; static final String SETTING_NAME_COMPONENT_SETTINGS = "component_settings"; @@ -160,6 +163,23 @@ public class Setting { .collect(toList()); } + + /** + * Extracts and returns the cluster settings. + * Currently the settings under the categories "cluster_settings" and "recovery_settings". + * Settings are flattened from sets of maps to a simple map. + * @return cluster settings + */ + Map<String, String> getClusterSettings() { + Set<String> settingsToExtract = + ImmutableSet.of(SETTING_NAME_CLUSTER_SETTINGS, SETTING_NAME_RECOVERY_SETTINGS); + return settingsToExtract.stream() + .flatMap( settingCategory -> + properties.getOrDefault(settingCategory, emptySet()).stream()) + .flatMap( map -> map.entrySet().stream() ) + .collect(toMap(e -> e.getKey(), e -> e.getValue())); + } + /** * Get whether the specified component in the service is enabled * for auto start. diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/SettingFactory.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/SettingFactory.java index ff00282..2d65ba1 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/SettingFactory.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/SettingFactory.java @@ -73,13 +73,16 @@ public class SettingFactory { for (Map.Entry<String, Object> entry : settingMap.entrySet()) { final String[] propertyNames = entry.getKey().split("/"); Set<Map<String, String>> settingValue; + String settingCategory; if (entry.getValue() instanceof Set) { + settingCategory = propertyNames[propertyNames.length - 1]; settingValue = (Set<Map<String, String>>) entry.getValue(); } else if (propertyNames.length > 1){ + settingCategory = propertyNames[0]; Map<String, String> property = new HashMap<>(); property.put(propertyNames[1], String.valueOf(entry.getValue())); - settingValue = properties.get(propertyNames[0]); + settingValue = properties.get(settingCategory); if (settingValue == null) { settingValue = new HashSet<>(); } @@ -88,7 +91,7 @@ public class SettingFactory { else { throw new IllegalArgumentException("Invalid setting schema: " + String.valueOf(entry.getValue())); } - properties.put(propertyNames[0], settingValue); + properties.put(settingCategory, settingValue); } } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/tasks/ConfigureClusterTask.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/tasks/ConfigureClusterTask.java index 0f92b3b..b00a729 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/tasks/ConfigureClusterTask.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/tasks/ConfigureClusterTask.java @@ -46,6 +46,7 @@ public class ConfigureClusterTask implements Callable<Boolean> { private static final long DEFAULT_TIMEOUT = TimeUnit.MINUTES.toMillis(30); private static final long REPEAT_DELAY = TimeUnit.SECONDS.toMillis(1); + // TODO: use cluster settings instead private static final String TIMEOUT_PROPERTY_NAME = "cluster_configure_task_timeout"; private static final Logger LOG = LoggerFactory.getLogger(ConfigureClusterTask.class); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/StackConfigTypeValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/StackConfigTypeValidator.java index d10f8cc9..4f538d2 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/StackConfigTypeValidator.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/StackConfigTypeValidator.java @@ -17,11 +17,14 @@ package org.apache.ambari.server.topology.validators; import java.util.HashSet; import java.util.Set; +import org.apache.ambari.server.state.ConfigHelper; import org.apache.ambari.server.topology.ClusterTopology; import org.apache.ambari.server.topology.InvalidTopologyException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.collect.ImmutableSet; + /** * Validates whether incoming config types (form the blueprint or the cluster creation template) are valid. * A configuration type is considered valid if the stack based on which the cluster is to be created contains such a @@ -30,6 +33,9 @@ import org.slf4j.LoggerFactory; public class StackConfigTypeValidator implements TopologyValidator { private static final Logger LOGGER = LoggerFactory.getLogger(StackConfigTypeValidator.class); + private static final Set<String> GLOBAL_CONFIG_TYPES = ImmutableSet.of( + ConfigHelper.CLUSTER_ENV); + @Override public ClusterTopology validate(ClusterTopology topology) throws InvalidTopologyException { @@ -45,6 +51,7 @@ public class StackConfigTypeValidator implements TopologyValidator { // remove all "valid" config types from the incoming set incomingConfigTypes.removeAll(stackConfigTypes); + incomingConfigTypes.removeAll(GLOBAL_CONFIG_TYPES); if (!incomingConfigTypes.isEmpty()) { // there are config types in the request that are not in the stack diff --git a/ambari-server/src/main/resources/cluster-settings.xml b/ambari-server/src/main/resources/cluster-settings.xml index 6eb81dd..2331a8d 100644 --- a/ambari-server/src/main/resources/cluster-settings.xml +++ b/ambari-server/src/main/resources/cluster-settings.xml @@ -329,4 +329,25 @@ gpgcheck=0 </description> <on-ambari-upgrade add="false"/> </property> + <property> + <name>command_retry_enabled</name> + <value>true</value> + <description>If flag is set to true, ambari will retry failed commands. + </description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>commands_to_retry</name> + <value>INSTALL,START</value> + <description>Specifies what kinds of commands should be retried on failure + </description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>command_retry_max_time_in_sec</name> + <value>600</value> + <description>Sets the timeframe in which a failed command should be retried. + </description> + <on-ambari-upgrade add="true"/> + </property> </configuration> diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java index 8e4f80c..14f1d41 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java @@ -48,6 +48,7 @@ import java.util.Map; import java.util.Set; import java.util.stream.Stream; +import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.controller.AmbariManagementController; import org.apache.ambari.server.controller.AmbariServer; import org.apache.ambari.server.controller.KerberosHelper; @@ -78,6 +79,7 @@ import org.apache.ambari.server.topology.InvalidTopologyException; import org.apache.ambari.server.topology.ResolvedComponent; import org.apache.ambari.server.topology.SecurityConfiguration; import org.apache.ambari.server.topology.SecurityConfigurationFactory; +import org.apache.ambari.server.topology.Setting; import org.apache.commons.lang.StringUtils; import org.easymock.EasyMock; import org.easymock.EasyMockRule; @@ -157,11 +159,18 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport { @Mock(type = MockType.NICE) private SecurityConfigurationFactory securityFactory; + @Mock + private AmbariMetaInfo metaInfo; + + private Setting setting; + @Before public void init() throws Exception { expect(ambariContext.composeStacks(anyObject())).andReturn(stack).anyTimes(); expect(bp.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); expect(bp.getName()).andReturn("test-bp").anyTimes(); + setting = new Setting(new HashMap<>()); + expect(bp.getSetting()).andReturn(setting).anyTimes(); expect(stack.getName()).andReturn(STACK_NAME).atLeastOnce(); expect(stack.getVersion()).andReturn(STACK_VERSION).atLeastOnce(); @@ -183,8 +192,9 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport { expect(configHelper.getDefaultStackProperties( EasyMock.eq(new StackId(STACK_NAME, STACK_VERSION)))).andReturn(stackProperties).anyTimes(); - stackProperties.put(ConfigHelper.CLUSTER_ENV, defaultClusterEnvProperties); + stackProperties.put(ConfigHelper.CLUSTER_ENV, defaultClusterEnvProperties); + expect(metaInfo.getClusterProperties()).andReturn(ImmutableSet.of()).anyTimes(); expect(ambariContext.isClusterKerberosEnabled(1)).andReturn(true).once(); expect(ambariContext.getClusterName(1L)).andReturn("clusterName").anyTimes(); @@ -195,6 +205,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport { expect(clusters.getCluster("clusterName")).andReturn(cluster).anyTimes(); expect(controller.getKerberosHelper()).andReturn(kerberosHelper).anyTimes(); expect(controller.getClusters()).andReturn(clusters).anyTimes(); + expect(controller.getAmbariMetaInfo()).andReturn(metaInfo).anyTimes(); expect(kerberosHelper.getKerberosDescriptor(cluster, false)).andReturn(kerberosDescriptor).anyTimes(); Set<String> properties = new HashSet<>(); properties.add("core-site/hadoop.security.auth_to_local"); @@ -2172,13 +2183,11 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport { assertEquals("128m", updatedVal2); assertEquals("Incorrect number of config types updated", - 3, configTypesUpdated.size()); + 2, configTypesUpdated.size()); assertTrue("Expected config type not updated", configTypesUpdated.contains("oozie-env")); assertTrue("Expected config type not updated", configTypesUpdated.contains("yarn-site")); - assertTrue("Expected config type not updated", - configTypesUpdated.contains("cluster-env")); } @Test @@ -2735,79 +2744,6 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport { } @Test - public void testDoUpdateForClusterVerifyRetrySettingsDefault() throws Exception { - Map<String, Map<String, String>> configProperties = - new HashMap<>(); - - HashMap<String, String> clusterEnvProperties = new HashMap<>(); - configProperties.put("cluster-env", clusterEnvProperties); - - Configuration clusterConfig = new Configuration(configProperties, Collections.emptyMap()); - - TestHostGroup testHostGroup = new TestHostGroup("test-host-group-one", Collections.emptySet(), Collections.emptySet()); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, Collections.singleton(testHostGroup), NEVER_APPLY); - - BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); - - Set<String> updatedConfigTypes = - updater.doUpdateForClusterCreate(); - - // after update, verify that the retry properties for commands and installs are set as expected - assertEquals("Incorrect number of properties added to cluster-env for retry", - 3, clusterEnvProperties.size()); - assertEquals("command_retry_enabled was not set to the expected default", - "true", clusterEnvProperties.get("command_retry_enabled")); - assertEquals("commands_to_retry was not set to the expected default", - "INSTALL,START", clusterEnvProperties.get("commands_to_retry")); - assertEquals("command_retry_max_time_in_sec was not set to the expected default", - "600", clusterEnvProperties.get("command_retry_max_time_in_sec")); - - assertEquals("Incorrect number of config types updated by this operation", - 1, updatedConfigTypes.size()); - - assertTrue("Expected type not included in the updated set", - updatedConfigTypes.contains("cluster-env")); - } - - @Test - public void testDoUpdateForClusterVerifyRetrySettingsCustomized() throws Exception { - Map<String, Map<String, String>> configProperties = - new HashMap<>(); - - HashMap<String, String> clusterEnvProperties = new HashMap<>(); - configProperties.put("cluster-env", clusterEnvProperties); - - clusterEnvProperties.put("command_retry_enabled", "false"); - clusterEnvProperties.put("commands_to_retry", "TEST"); - clusterEnvProperties.put("command_retry_max_time_in_sec", "1"); - - - Configuration clusterConfig = new Configuration(configProperties, Collections.emptyMap()); - - TestHostGroup testHostGroup = new TestHostGroup("test-host-group-one", Collections.emptySet(), Collections.emptySet()); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, Collections.singleton(testHostGroup), NEVER_APPLY); - - BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); - - Set<String> updatedConfigTypes = - updater.doUpdateForClusterCreate(); - - // after update, verify that the retry properties for commands and installs are set as expected - // in this case, the customer-provided overrides should be honored, rather than the retry defaults - assertEquals("Incorrect number of properties added to cluster-env for retry", - 3, clusterEnvProperties.size()); - assertEquals("command_retry_enabled was not set to the expected default", - "false", clusterEnvProperties.get("command_retry_enabled")); - assertEquals("commands_to_retry was not set to the expected default", - "TEST", clusterEnvProperties.get("commands_to_retry")); - assertEquals("command_retry_max_time_in_sec was not set to the expected default", - "1", clusterEnvProperties.get("command_retry_max_time_in_sec")); - - assertEquals("Incorrect number of config types updated", - 0, updatedConfigTypes.size()); - } - - @Test public void testDoUpdateForClusterWithNameNodeHAEnabledSpecifyingHostNamesDirectly() throws Exception { final String expectedNameService = "mynameservice"; final String expectedHostName = "c6401.apache.ambari.org"; @@ -5350,9 +5286,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport { // verify that correct configuration types were listed as updated in the returned set assertEquals("Incorrect number of updated config types returned, set = " + updatedConfigTypes, - 3, updatedConfigTypes.size()); - assertTrue("Expected config type not found in updated set", - updatedConfigTypes.contains("cluster-env")); + 2, updatedConfigTypes.size()); assertTrue("Expected config type not found in updated set", updatedConfigTypes.contains("hdfs-site")); assertTrue("Expected config type not found in updated set", @@ -5424,9 +5358,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport { // verify that correct configuration types were listed as updated in the returned set assertEquals("Incorrect number of updated config types returned, set = " + updatedConfigTypes, - 2, updatedConfigTypes.size()); - assertTrue("Expected config type 'cluster-env' not found in updated set", - updatedConfigTypes.contains("cluster-env")); + 1, updatedConfigTypes.size()); assertTrue("Expected config type 'hdfs-site' not found in updated set", updatedConfigTypes.contains("hdfs-site")); } @@ -8217,7 +8149,8 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport { throws InvalidTopologyException { - replay(stack, serviceInfo, ambariContext, configHelper, controller, kerberosHelper, kerberosDescriptor, clusters, cluster); + replay(stack, serviceInfo, ambariContext, configHelper, controller, kerberosHelper, kerberosDescriptor, clusters, + cluster, metaInfo); Map<String, HostGroupInfo> hostGroupInfo = new HashMap<>(); Map<String, HostGroup> allHostGroups = new HashMap<>(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java index 97da23b..71e2280 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java @@ -18,7 +18,9 @@ package org.apache.ambari.server.topology; +import static com.google.common.collect.Sets.newHashSet; import static java.util.Collections.emptySet; +import static java.util.stream.Collectors.toMap; import static org.apache.ambari.server.topology.StackComponentResolverTest.builderFor; import static org.easymock.EasyMock.anyObject; import static org.easymock.EasyMock.capture; @@ -34,7 +36,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import java.lang.reflect.Field; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -42,6 +43,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.stream.IntStream; import java.util.stream.Stream; import org.apache.ambari.annotations.Experimental; @@ -77,18 +79,25 @@ import org.apache.ambari.server.state.ConfigFactory; import org.apache.ambari.server.state.ConfigHelper; import org.apache.ambari.server.state.DesiredConfig; import org.apache.ambari.server.state.Host; -import org.apache.ambari.server.state.Service; +import org.apache.ambari.server.state.PropertyInfo; import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.state.configgroup.ConfigGroup; +import org.apache.commons.lang3.tuple.Pair; import org.easymock.Capture; +import org.easymock.CaptureType; import org.easymock.EasyMock; import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; import org.powermock.api.easymock.PowerMock; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.powermock.reflect.Whitebox; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; import com.google.inject.util.Providers; /** @@ -98,6 +107,8 @@ import com.google.inject.util.Providers; @Experimental( feature = ExperimentalFeature.UNIT_TEST_REQUIRED, comment = "Add test cases for mpacks and multiple/bad versions") +@RunWith(PowerMockRunner.class) +@PrepareForTest(AmbariContext.class) public class AmbariContextTest { private static final String BP_NAME = "testBP"; @@ -133,7 +144,7 @@ public class AmbariContextTest { private static final Host host1 = createNiceMock(Host.class); private static final Host host2 = createNiceMock(Host.class); private static final ConfigFactory configFactory = createNiceMock(ConfigFactory.class); - private static final Service mockService1 = createStrictMock(Service.class); +// private static final Service mockService1 = createStrictMock(Service.class); private static final Collection<String> blueprintServices = new HashSet<>(); private static final Map<Long, ConfigGroup> configGroups = new HashMap<>(); @@ -141,44 +152,27 @@ public class AmbariContextTest { private Configuration group1Configuration = null; private static final Set<String> group1Hosts = ImmutableSet.of(HOST1, HOST2); + private Capture<String> clusterSettingKeys; + private Capture<String> clusterSettingValues; + private Capture<Set<ConfigGroupRequest>> configGroupRequestCapture = EasyMock.newCapture(); - private Setting setting = createNiceMock(Setting.class); + private Setting setting; @Before public void setUp() throws Exception { reset(controller, clusterController, hostResourceProvider, serviceGroupResourceProvider, serviceResourceProvider, componentResourceProvider, metaInfo, hostComponentResourceProvider, configGroupResourceProvider, topology, blueprint, stack, clusters, - cluster, group1Info, configHelper, configGroup1, configGroup2, host1, host2, configFactory); + cluster, group1Info, configHelper, configGroup1, configGroup2, host1, host2, configFactory, metaInfo); // "inject" context state - Class<AmbariContext> clazz = AmbariContext.class; - Field f = clazz.getDeclaredField("controller"); - f.setAccessible(true); - f.set(context, Providers.of(controller)); - - f = clazz.getDeclaredField("clusterController"); - f.setAccessible(true); - f.set(null, clusterController); - - f = clazz.getDeclaredField("hostResourceProvider"); - f.setAccessible(true); - f.set(null, hostResourceProvider); - - f = clazz.getDeclaredField("serviceGroupResourceProvider"); - f.setAccessible(true); - f.set(null, serviceGroupResourceProvider); - - f = clazz.getDeclaredField("serviceResourceProvider"); - f.setAccessible(true); - f.set(null, serviceResourceProvider); - - f = clazz.getDeclaredField("componentResourceProvider"); - f.setAccessible(true); - f.set(null, componentResourceProvider); - - f = clazz.getDeclaredField("hostComponentResourceProvider"); - f.setAccessible(true); - f.set(null, hostComponentResourceProvider); + Whitebox.setInternalState(context, "controller", Providers.of(controller)); + Whitebox.setInternalState(context, "ambariMetaInfo", Providers.of(metaInfo)); + Whitebox.setInternalState(AmbariContext.class, "clusterController", clusterController); + Whitebox.setInternalState(AmbariContext.class, "hostResourceProvider", hostResourceProvider); + Whitebox.setInternalState(AmbariContext.class, "serviceGroupResourceProvider", serviceGroupResourceProvider); + Whitebox.setInternalState(AmbariContext.class, "serviceResourceProvider", serviceResourceProvider); + Whitebox.setInternalState(AmbariContext.class, "componentResourceProvider", componentResourceProvider); + Whitebox.setInternalState(AmbariContext.class, "hostComponentResourceProvider", hostComponentResourceProvider); // bp configuration Map<String, Map<String, String>> bpProperties = new HashMap<>(); @@ -252,8 +246,18 @@ public class AmbariContextTest { builderFor("service2", "s2Component1").stackId(STACK_ID).buildPartial() )).anyTimes(); expect(topology.getConfiguration()).andReturn(bpConfiguration).anyTimes(); + + setting = new Setting( + map( + Pair.of("recovery_settings", + newHashSet( + map(Pair.of("recovery_enabled", "true")))), + Pair.of("service_settings", + newHashSet( + map(Pair.of("name", "service1"), Pair.of("recovery_enabled", "true")))) + ) + ); expect(topology.getSetting()).andReturn(setting).anyTimes(); - expect(setting.getCredentialStoreEnabled("service1")).andReturn("true").anyTimes(); expect(stack.getName()).andReturn(STACK_NAME).anyTimes(); expect(stack.getVersion()).andReturn(STACK_VERSION).anyTimes(); @@ -265,7 +269,13 @@ public class AmbariContextTest { expect(controller.getClusters()).andReturn(clusters).anyTimes(); expect(controller.getConfigHelper()).andReturn(configHelper).anyTimes(); expect(controller.getAmbariMetaInfo()).andReturn(metaInfo).anyTimes(); - expect(metaInfo.getClusterProperties()).andReturn(emptySet()).anyTimes(); + expect(metaInfo.getClusterProperties()).andReturn( + Sets.newHashSet( + property("command_retry_enabled", "true"), + property("commands_to_retry", "INSTALL,START"), + property("command_retry_max_time_in_sec", "600") + ) + ).anyTimes(); expect(clusters.getCluster(CLUSTER_ID)).andReturn(cluster).anyTimes(); expect(clusters.getCluster(CLUSTER_NAME)).andReturn(cluster).anyTimes(); @@ -276,8 +286,11 @@ public class AmbariContextTest { Map<String, Host> clusterHosts = ImmutableMap.of(HOST1, host1, HOST2, host2); expect(clusters.getHostsForCluster(CLUSTER_NAME)).andReturn(clusterHosts).anyTimes(); + clusterSettingKeys = EasyMock.newCapture(CaptureType.ALL); + clusterSettingValues = EasyMock.newCapture(CaptureType.ALL); expect(cluster.getClusterId()).andReturn(CLUSTER_ID).anyTimes(); expect(cluster.getClusterName()).andReturn(CLUSTER_NAME).anyTimes(); + expect(cluster.addClusterSetting(capture(clusterSettingKeys), capture(clusterSettingValues))).andReturn(null).anyTimes(); expect(host1.getHostId()).andReturn(1L).anyTimes(); expect(host2.getHostId()).andReturn(2L).anyTimes(); @@ -287,20 +300,19 @@ public class AmbariContextTest { expect(configGroup1.getName()).andReturn(String.format("%s:%s", BP_NAME, HOST_GROUP_1)).anyTimes(); expect(configGroup2.getName()).andReturn(String.format("%s:%s", BP_NAME, HOST_GROUP_2)).anyTimes(); - } @After public void tearDown() throws Exception { verify(controller, clusterController, hostResourceProvider, serviceGroupResourceProvider, serviceResourceProvider, componentResourceProvider, metaInfo, - hostComponentResourceProvider, configGroupResourceProvider, topology, blueprint, setting, stack, clusters, - cluster, group1Info, configHelper, configGroup1, configGroup2, host1, host2, configFactory); + hostComponentResourceProvider, configGroupResourceProvider, topology, blueprint, stack, clusters, cluster, + group1Info, configHelper, configGroup1, configGroup2, host1, host2, configFactory); } private void replayAll() { replay(controller, clusterController, hostResourceProvider, serviceGroupResourceProvider, serviceResourceProvider, componentResourceProvider, metaInfo, - hostComponentResourceProvider, configGroupResourceProvider, topology, blueprint, setting, stack, clusters, - cluster, group1Info, configHelper, configGroup1, configGroup2, host1, host2, configFactory); + hostComponentResourceProvider, configGroupResourceProvider, topology, blueprint, stack, clusters, cluster, + group1Info, configHelper, configGroup1, configGroup2, host1, host2, configFactory); } @Test @@ -395,6 +407,21 @@ public class AmbariContextTest { assertEquals("STARTED", startProperties.get(ServiceResourceProvider.SERVICE_SERVICE_STATE_PROPERTY_ID)); assertEquals(new EqualsPredicate<>(ServiceResourceProvider.SERVICE_CLUSTER_NAME_PROPERTY_ID, CLUSTER_NAME), installPredicateCapture.getValue()); + + // verify that cluster settings has been configured with values coming from both defaults and the topology + Map<String, String> recordedClusterSettings = IntStream.range(0, clusterSettingKeys.getValues().size()) + .boxed() + .map(i -> Pair.of(clusterSettingKeys.getValues().get(i), clusterSettingValues.getValues().get(i))) + .collect(toMap(p -> p.getLeft(), p -> p.getRight())); + + assertEquals( + ImmutableMap.of( + "command_retry_enabled", "true", + "commands_to_retry", "INSTALL,START", + "command_retry_max_time_in_sec", "600", + "recovery_enabled", "true"), + recordedClusterSettings + ); } @Test @@ -684,14 +711,31 @@ public class AmbariContextTest { assertFalse(topologyResolved); } + private static final <K, V> Map<K, V> map(Pair<K, V>... keyValuePairs) { + Map<K, V> map = new HashMap<>(keyValuePairs.length); + for (Pair<K, V> kv: keyValuePairs) { + map.put(kv.getLeft(), kv.getRight()); + } + return map; + } + + private static PropertyInfo property(String name, String value) { + PropertyInfo info = new PropertyInfo(); + info.setName(name); + info.setValue(value); + return info; + } + @Test public void testProvisionCluster_downloadMissingMpack() throws Exception { PowerMock.mockStatic(AmbariContext.class); expect(AmbariContext.getClusterController()).andReturn(clusterController).anyTimes(); + PowerMock.replay(AmbariContext.class); // given MpackInstance mpack1 = new MpackInstance("HDPCORE", "HDPCORE", "1.0.0.0", "http://mpacks.org/hdpcore", Configuration.createEmpty()); MpackInstance mpack2 = new MpackInstance("HDF", "HDF", "3.3.0", "http://mpacks.org/hdf", Configuration.createEmpty()); + reset(metaInfo); expect(metaInfo.getStack(mpack1.getStackId())).andReturn(null); expect(metaInfo.getStack(mpack2.getStackId())).andThrow(new StackAccessException("Testing missing stack")); @@ -703,8 +747,8 @@ public class AmbariContextTest { .andReturn(new RequestStatusImpl(null, null, null)) .once(); - PowerMock.replay(AmbariContext.class); - replay(clusterController, metaInfo, mpackResourceProvider); + replay(mpackResourceProvider); + replayAll(); // when context.downloadMissingMpacks(ImmutableSet.of(mpack1, mpack2)); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java index d32bc92..2e67538 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java @@ -143,7 +143,7 @@ public class ClusterConfigurationRequestTest { Capture<? extends Set<String>> captureUpdatedConfigTypes = testProcessWithKerberos(null, "defaultTestValue", null); Set<String> updatedConfigTypes = captureUpdatedConfigTypes.getValue(); - assertEquals(2, updatedConfigTypes.size()); + assertEquals(1, updatedConfigTypes.size()); } /** @@ -158,7 +158,7 @@ public class ClusterConfigurationRequestTest { "defaultTestValue", null); Set<String> updatedConfigTypes = captureUpdatedConfigTypes.getValue(); - assertEquals(2, updatedConfigTypes.size()); + assertEquals(1, updatedConfigTypes.size()); } @@ -174,7 +174,7 @@ public class ClusterConfigurationRequestTest { "defaultTestValue", null); Set<String> updatedConfigTypes = captureUpdatedConfigTypes.getValue(); - assertEquals(1, updatedConfigTypes.size()); + assertEquals(0, updatedConfigTypes.size()); } /** @@ -188,7 +188,7 @@ public class ClusterConfigurationRequestTest { Capture<? extends Set<String>> captureUpdatedConfigTypes = testProcessWithKerberos("testPropertyValue", null, null); Set<String> updatedConfigTypes = captureUpdatedConfigTypes.getValue(); - assertEquals(1, updatedConfigTypes.size()); + assertEquals(0, updatedConfigTypes.size()); } @Test @@ -202,7 +202,7 @@ public class ClusterConfigurationRequestTest { Capture<? extends Set<String>> captureUpdatedConfigTypes = testProcessWithKerberos(null, "defaultTestValue", kerberosConfig); Set<String> updatedConfigTypes = captureUpdatedConfigTypes.getValue(); - assertEquals(1, updatedConfigTypes.size()); + assertEquals(0, updatedConfigTypes.size()); } @Test @@ -216,7 +216,7 @@ public class ClusterConfigurationRequestTest { Capture<? extends Set<String>> captureUpdatedConfigTypes = testProcessWithKerberos(null, "defaultTestValue", kerberosConfig); Set<String> updatedConfigTypes = captureUpdatedConfigTypes.getValue(); - assertEquals(1, updatedConfigTypes.size()); + assertEquals(0, updatedConfigTypes.size()); } private Capture<? extends Set<String>> testProcessWithKerberos(String blueprintPropertyValue, String @@ -277,9 +277,7 @@ public class ClusterConfigurationRequestTest { expect(ambariContext.getConfigHelper()).andReturn(configHelper).anyTimes(); expect(ambariContext.getClusterName(CLUSTER_ID)).andReturn(CLUSTER_NAME).anyTimes(); - expect(ambariContext.createConfigurationRequests(EasyMock.anyObject())).andReturn(Collections - .emptyList()).anyTimes(); - Set<ServiceResponse> services = IntStream.range(0, SERVICE_NAMES.size()).boxed().map( + Set<ServiceResponse> services = IntStream.range(0, SERVICE_NAMES.size()).boxed().map( serviceId -> new ServiceResponse(CLUSTER_ID, CLUSTER_NAME, 1L, "CORE", (long)serviceId, SERVICE_NAMES.get(serviceId), null, null, null, null, false, false, false, false, false) ).collect(toSet()); @@ -352,8 +350,6 @@ public class ClusterConfigurationRequestTest { expect(ambariContext.getConfigHelper()).andReturn(configHelper).anyTimes(); expect(ambariContext.getClusterName(1L)).andReturn(CLUSTER_NAME).anyTimes(); - expect(ambariContext.createConfigurationRequests(EasyMock.anyObject())).andReturn(Collections - .emptyList()).anyTimes(); Set<ServiceResponse> services = IntStream.range(0, serviceNames.size()).boxed(). map( serviceId -> new ServiceResponse(CLUSTER_ID, CLUSTER_NAME, 1L, "CORE", (long)serviceId, SERVICE_NAMES.get(serviceId), diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeployWithStartOnlyTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeployWithStartOnlyTest.java index 5f0e4e3..73cab9d 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeployWithStartOnlyTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeployWithStartOnlyTest.java @@ -17,7 +17,11 @@ */ package org.apache.ambari.server.topology; +import static java.util.Collections.emptySet; import static java.util.stream.Collectors.toSet; +import static org.apache.ambari.server.topology.ClusterDeploymentTestCommon.CLUSTER_ID; +import static org.apache.ambari.server.topology.ClusterDeploymentTestCommon.CLUSTER_NAME; +import static org.apache.ambari.server.topology.ClusterDeploymentTestCommon.service; import static org.apache.ambari.server.topology.StackComponentResolverTest.builderFor; import static org.easymock.EasyMock.anyBoolean; import static org.easymock.EasyMock.anyLong; @@ -31,12 +35,10 @@ import static org.easymock.EasyMock.isA; import static org.easymock.EasyMock.newCapture; import static org.junit.Assert.assertEquals; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; @@ -69,7 +71,6 @@ import org.apache.ambari.server.state.SecurityType; import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.topology.tasks.ConfigureClusterTask; import org.apache.ambari.server.topology.tasks.ConfigureClusterTaskFactory; -import org.apache.ambari.server.topology.validators.TopologyValidator; import org.easymock.Capture; import org.easymock.EasyMockRule; import org.easymock.EasyMockSupport; @@ -88,12 +89,11 @@ import org.powermock.reflect.Whitebox; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; @RunWith(PowerMockRunner.class) -@PrepareForTest({ AmbariServer.class }) +@PrepareForTest({ AmbariContext.class, AmbariServer.class }) public class ClusterDeployWithStartOnlyTest extends EasyMockSupport { - private static final String CLUSTER_NAME = "test-cluster"; - private static final long CLUSTER_ID = 1; private static final String BLUEPRINT_NAME = "test-bp"; private static final String STACK_NAME = "test-stack"; private static final String STACK_VERSION = "test-stack-version"; @@ -181,6 +181,9 @@ public class ClusterDeployWithStartOnlyTest extends EasyMockSupport { @Mock private ComponentResolver componentResolver; + @Mock(type = MockType.NICE) + private ClusterTopology clusterTopology; + private final Configuration stackConfig = new Configuration(new HashMap<>(), new HashMap<>()); private final Configuration bpConfiguration = new Configuration(new HashMap<>(), @@ -211,8 +214,6 @@ public class ClusterDeployWithStartOnlyTest extends EasyMockSupport { private String predicate = "Hosts/host_name=foo"; - private List<TopologyValidator> topologyValidators = new ArrayList<>(); - private Capture<Map<String, Object>> configRequestPropertiesCapture; private Capture<Map<String, Object>> configRequestPropertiesCapture2; private Capture<Map<String, Object>> configRequestPropertiesCapture3; @@ -270,7 +271,6 @@ public class ClusterDeployWithStartOnlyTest extends EasyMockSupport { expect(blueprint.getHostGroupsForComponent("component3")).andReturn(Arrays.asList(group1, group2)).anyTimes(); expect(blueprint.getHostGroupsForComponent("component4")).andReturn(Collections.singleton(group2)).anyTimes(); expect(blueprint.getName()).andReturn(BLUEPRINT_NAME).anyTimes(); - expect(ambariContext.composeStacks(anyObject())).andReturn(stack).anyTimes(); expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); expect(blueprint.getSecurity()).andReturn(SecurityConfiguration.NONE).anyTimes(); expect(blueprint.getMpacks()).andReturn(ImmutableSet.of()).anyTimes(); @@ -303,9 +303,9 @@ public class ClusterDeployWithStartOnlyTest extends EasyMockSupport { expect(stack.getVersion()).andReturn(STACK_VERSION).anyTimes(); expect(stack.getServiceForConfigType("service1-site")).andReturn("service1").anyTimes(); expect(stack.getServiceForConfigType("service2-site")).andReturn("service2").anyTimes(); - expect(stack.getDependenciesForComponent(anyString())).andReturn(Collections.emptySet()).anyTimes(); - expect(stack.getExcludedConfigurationTypes("service1")).andReturn(Collections.emptySet()).anyTimes(); - expect(stack.getExcludedConfigurationTypes("service2")).andReturn(Collections.emptySet()).anyTimes(); + expect(stack.getDependenciesForComponent(anyString())).andReturn(emptySet()).anyTimes(); + expect(stack.getExcludedConfigurationTypes("service1")).andReturn(emptySet()).anyTimes(); + expect(stack.getExcludedConfigurationTypes("service2")).andReturn(emptySet()).anyTimes(); expect(stack.getServiceForComponent("component1")).andReturn("service1").anyTimes(); expect(stack.getServiceForComponent("component2")).andReturn("service2").anyTimes(); expect(stack.getServiceForComponent("component3")).andReturn("service1").anyTimes(); @@ -319,7 +319,7 @@ public class ClusterDeployWithStartOnlyTest extends EasyMockSupport { expect(request.getHostGroupInfo()).andReturn(groupInfoMap).anyTimes(); expect(request.getConfigRecommendationStrategy()).andReturn(ConfigRecommendationStrategy.NEVER_APPLY).anyTimes(); expect(request.getProvisionAction()).andReturn(ProvisionAction.START_ONLY).anyTimes(); - expect(request.getSecurityConfiguration()).andReturn(null).anyTimes(); + expect(request.getSecurityConfiguration()).andReturn(SecurityConfiguration.NONE).anyTimes(); expect(request.shouldValidateTopology()).andReturn(true).anyTimes(); expect(request.getStackIds()).andReturn(ImmutableSet.of()).anyTimes(); expect(request.getMpacks()).andReturn(ImmutableSet.of()).anyTimes(); @@ -361,7 +361,9 @@ public class ClusterDeployWithStartOnlyTest extends EasyMockSupport { PowerMock.mockStatic(AmbariServer.class); expect(AmbariServer.getController()).andReturn(managementController).anyTimes(); PowerMock.replay(AmbariServer.class); - expect(ambariContext.getClusterController()).andReturn(clusterController).anyTimes(); + PowerMock.mockStatic(AmbariContext.class); + expect(AmbariContext.getClusterController()).andReturn(clusterController).anyTimes(); + PowerMock.replay(AmbariContext.class); expect(clusterController.ensureResourceProvider(Resource.Type.Mpack)).andReturn(mpackResourceProvider).anyTimes(); expect(clusterController.ensureResourceProvider(Resource.Type.Artifact)).andReturn(artifactResourceProvider).anyTimes(); RequestStatus completedStatus = createNiceMock(RequestStatus.class); @@ -372,7 +374,11 @@ public class ClusterDeployWithStartOnlyTest extends EasyMockSupport { expect(clusters.getClusterById(anyLong())).andReturn(cluster).anyTimes(); expect(cluster.getClusterName()).andReturn(CLUSTER_NAME).anyTimes(); + expect(ambariContext.composeStacks(anyObject())).andReturn(stack).anyTimes(); + expect(ambariContext.getServices(anyString())).andReturn( + Sets.newHashSet(service("service1", 1L), service("service2", 2L))).anyTimes(); expect(ambariContext.getPersistedTopologyState()).andReturn(persistedState).anyTimes(); + expect(ambariContext.getController()).andReturn(managementController).anyTimes(); //todo: don't ignore param ambariContext.createAmbariResources(isA(ClusterTopology.class), eq(CLUSTER_NAME), eq(SecurityType.NONE)); expectLastCall().once(); @@ -380,20 +386,23 @@ public class ClusterDeployWithStartOnlyTest extends EasyMockSupport { expect(ambariContext.isClusterKerberosEnabled(CLUSTER_ID)).andReturn(false).anyTimes(); expect(ambariContext.getClusterId(CLUSTER_NAME)).andReturn(CLUSTER_ID).anyTimes(); expect(ambariContext.getClusterName(CLUSTER_ID)).andReturn(CLUSTER_NAME).anyTimes(); - // so only INITIAL config - expect(ambariContext.createConfigurationRequests(capture(configRequestPropertiesCapture))). - andReturn(Collections.singletonList(configurationRequest)); - expect(ambariContext.createConfigurationRequests(capture(configRequestPropertiesCapture2))). - andReturn(Collections.singletonList(configurationRequest2)).once(); - expect(ambariContext.createConfigurationRequests(capture(configRequestPropertiesCapture3))). - andReturn(Collections.singletonList(configurationRequest3)).once(); // INSTALL task expectation - expect(ambariContext.createAmbariTask(anyLong(), anyLong(), eq("component3"), anyString(), eq(AmbariContext.TaskType.INSTALL), anyBoolean())).andReturn(hostRoleCommandInstallComponent3).times(3); expect(ambariContext.createAmbariTask(anyLong(), anyLong(), eq("component4"), anyString(), eq(AmbariContext.TaskType.INSTALL), anyBoolean())).andReturn(hostRoleCommandInstallComponent4).times(2); + expect(ambariContext.createClusterTopology(request)).andReturn(clusterTopology); + + expect(clusterTopology.getSecurity()).andReturn(SecurityConfiguration.NONE).anyTimes(); + expect(clusterTopology.getClusterId()).andReturn(CLUSTER_ID).anyTimes(); + expect(clusterTopology.getSetting()).andReturn(new Setting(ImmutableMap.of())).anyTimes(); + expect(clusterTopology.getBlueprint()).andReturn(blueprint).anyTimes(); + expect(clusterTopology.getProvisionAction()).andReturn(ProvisionAction.START_ONLY).anyTimes(); + expect(clusterTopology.getAmbariContext()).andReturn(ambariContext).anyTimes(); + expect(clusterTopology.getConfiguration()).andReturn(bpConfiguration).anyTimes(); + expect(clusterTopology.getStack()).andReturn(stack).anyTimes(); + expect(clusterTopology.getStackIds()).andReturn(ImmutableSet.of(new StackId(STACK_NAME, STACK_VERSION))).anyTimes(); expect(hostRoleCommandInstallComponent3.getTaskId()).andReturn(1L).atLeastOnce(); expect(hostRoleCommandInstallComponent3.getRole()).andReturn(Role.valueOf("component3")).atLeastOnce(); @@ -421,7 +430,6 @@ public class ClusterDeployWithStartOnlyTest extends EasyMockSupport { expect(hostRoleCommandStartComponent2.getRole()).andReturn(Role.NAMENODE).atLeastOnce(); expect(hostRoleCommandStartComponent2.getStatus()).andReturn(HostRoleStatus.COMPLETED).atLeastOnce(); - ambariContext.setConfigurationOnCluster(capture(updateClusterConfigRequestCapture)); expectLastCall().times(3); ambariContext.persistInstallStateForUI(CLUSTER_NAME, STACK_ID); @@ -456,4 +464,5 @@ public class ClusterDeployWithStartOnlyTest extends EasyMockSupport { LogicalRequest request = topologyManager.getRequest(1); assertEquals(request.getHostRequests().size(), 3); } + } diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeploymentTestCommon.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeploymentTestCommon.java new file mode 100644 index 0000000..b23d88e --- /dev/null +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeploymentTestCommon.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.topology; + +import org.apache.ambari.server.controller.ServiceResponse; + +/** + * Common functionality for cluster deployment tests + */ +public class ClusterDeploymentTestCommon { + + static final String CLUSTER_NAME = "test-cluster"; + static final long CLUSTER_ID = 1; + + /** + * @return a {@link ServiceResponse} instance for tests + */ + static ServiceResponse service(String serviceName, long serviceId) { + return new ServiceResponse(CLUSTER_ID, CLUSTER_NAME, 1L, "service-group-1", serviceId, serviceName, null, null, + null, null, true, true, true, true, true); + } +} diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartOnComponentLevelTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartOnComponentLevelTest.java index 9ea744f..65a8f6f 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartOnComponentLevelTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartOnComponentLevelTest.java @@ -20,6 +20,9 @@ package org.apache.ambari.server.topology; import static java.util.stream.Collectors.toSet; import static org.apache.ambari.server.controller.internal.ProvisionAction.INSTALL_AND_START; +import static org.apache.ambari.server.topology.ClusterDeploymentTestCommon.CLUSTER_ID; +import static org.apache.ambari.server.topology.ClusterDeploymentTestCommon.CLUSTER_NAME; +import static org.apache.ambari.server.topology.ClusterDeploymentTestCommon.service; import static org.apache.ambari.server.topology.StackComponentResolverTest.builderFor; import static org.easymock.EasyMock.anyBoolean; import static org.easymock.EasyMock.anyLong; @@ -32,12 +35,10 @@ import static org.easymock.EasyMock.expectLastCall; import static org.easymock.EasyMock.isA; import static org.easymock.EasyMock.newCapture; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; @@ -46,6 +47,7 @@ import org.apache.ambari.server.Role; import org.apache.ambari.server.RoleCommand; import org.apache.ambari.server.actionmanager.HostRoleCommand; import org.apache.ambari.server.actionmanager.HostRoleStatus; +import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.controller.AmbariManagementController; import org.apache.ambari.server.controller.AmbariServer; import org.apache.ambari.server.controller.ClusterRequest; @@ -70,7 +72,6 @@ import org.apache.ambari.server.state.SecurityType; import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.topology.tasks.ConfigureClusterTask; import org.apache.ambari.server.topology.tasks.ConfigureClusterTaskFactory; -import org.apache.ambari.server.topology.validators.TopologyValidator; import org.easymock.Capture; import org.easymock.EasyMockRule; import org.easymock.EasyMockSupport; @@ -89,12 +90,11 @@ import org.powermock.reflect.Whitebox; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; @RunWith(PowerMockRunner.class) -@PrepareForTest({ AmbariServer.class }) +@PrepareForTest({ AmbariContext.class, AmbariServer.class }) public class ClusterInstallWithoutStartOnComponentLevelTest extends EasyMockSupport { - private static final String CLUSTER_NAME = "test-cluster"; - private static final long CLUSTER_ID = 1; private static final String BLUEPRINT_NAME = "test-bp"; private static final String STACK_NAME = "test-stack"; private static final String STACK_VERSION = "test-stack-version"; @@ -165,6 +165,8 @@ public class ClusterInstallWithoutStartOnComponentLevelTest extends EasyMockSupp private ArtifactResourceProvider artifactResourceProvider; @Mock(type = MockType.NICE) private MpackResourceProvider mpackResourceProvider; + @Mock + private AmbariMetaInfo ambariMetaInfo; @Mock(type = MockType.NICE) @@ -178,6 +180,9 @@ public class ClusterInstallWithoutStartOnComponentLevelTest extends EasyMockSupp @Mock private ComponentResolver componentResolver; + @Mock(type = MockType.NICE) + private ClusterTopology clusterTopology; + private final Configuration stackConfig = new Configuration(new HashMap<>(), new HashMap<>()); private final Configuration bpConfiguration = new Configuration(new HashMap<>(), @@ -208,8 +213,6 @@ public class ClusterInstallWithoutStartOnComponentLevelTest extends EasyMockSupp private String predicate = "Hosts/host_name=foo"; - private List<TopologyValidator> topologyValidators = new ArrayList<>(); - private Capture<Map<String, Object>> configRequestPropertiesCapture; private Capture<Map<String, Object>> configRequestPropertiesCapture2; private Capture<Map<String, Object>> configRequestPropertiesCapture3; @@ -269,6 +272,8 @@ public class ClusterInstallWithoutStartOnComponentLevelTest extends EasyMockSupp expect(blueprint.getHostGroupsForComponent("component4")).andReturn(Collections.singleton(group2)).anyTimes(); expect(blueprint.getName()).andReturn(BLUEPRINT_NAME).anyTimes(); expect(ambariContext.composeStacks(anyObject())).andReturn(stack).anyTimes(); + expect(ambariContext.getServices(anyString())).andReturn( + Sets.newHashSet(service("service1", 1L), service("service2", 2L))).anyTimes(); expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); expect(blueprint.getSecurity()).andReturn(SecurityConfiguration.NONE).anyTimes(); expect(blueprint.getMpacks()).andReturn(ImmutableSet.of()).anyTimes(); @@ -365,7 +370,10 @@ public class ClusterInstallWithoutStartOnComponentLevelTest extends EasyMockSupp PowerMock.mockStatic(AmbariServer.class); expect(AmbariServer.getController()).andReturn(managementController).anyTimes(); PowerMock.replay(AmbariServer.class); - expect(ambariContext.getClusterController()).andReturn(clusterController).anyTimes(); + + PowerMock.mockStatic(AmbariContext.class); + expect(AmbariContext.getClusterController()).andReturn(clusterController).anyTimes(); + PowerMock.replay(AmbariContext.class); expect(clusterController.ensureResourceProvider(Resource.Type.Mpack)).andReturn(mpackResourceProvider).anyTimes(); expect(clusterController.ensureResourceProvider(Resource.Type.Artifact)).andReturn(artifactResourceProvider).anyTimes(); RequestStatus completedStatus = createNiceMock(RequestStatus.class); @@ -376,7 +384,11 @@ public class ClusterInstallWithoutStartOnComponentLevelTest extends EasyMockSupp expect(clusters.getClusterById(anyLong())).andReturn(cluster).anyTimes(); expect(cluster.getClusterName()).andReturn(CLUSTER_NAME).anyTimes(); + expect(managementController.getAmbariMetaInfo()).andReturn(ambariMetaInfo).anyTimes(); + expect(ambariMetaInfo.getClusterProperties()).andReturn(Sets.newHashSet()).anyTimes(); + expect(ambariContext.getPersistedTopologyState()).andReturn(persistedState).anyTimes(); + expect(ambariContext.getController()).andReturn(managementController).anyTimes(); //todo: don't ignore param ambariContext.createAmbariResources(isA(ClusterTopology.class), eq(CLUSTER_NAME), eq(SecurityType.NONE)); expectLastCall().once(); @@ -384,13 +396,6 @@ public class ClusterInstallWithoutStartOnComponentLevelTest extends EasyMockSupp expect(ambariContext.isClusterKerberosEnabled(CLUSTER_ID)).andReturn(false).anyTimes(); expect(ambariContext.getClusterId(CLUSTER_NAME)).andReturn(CLUSTER_ID).anyTimes(); expect(ambariContext.getClusterName(CLUSTER_ID)).andReturn(CLUSTER_NAME).anyTimes(); - // so only INITIAL config - expect(ambariContext.createConfigurationRequests(capture(configRequestPropertiesCapture))). - andReturn(Collections.singletonList(configurationRequest)); - expect(ambariContext.createConfigurationRequests(capture(configRequestPropertiesCapture2))). - andReturn(Collections.singletonList(configurationRequest2)).once(); - expect(ambariContext.createConfigurationRequests(capture(configRequestPropertiesCapture3))). - andReturn(Collections.singletonList(configurationRequest3)).once(); // INSTALL task expectation expect(ambariContext.createAmbariTask(anyLong(), anyLong(), anyString(), anyString(), eq(AmbariContext.TaskType.INSTALL), anyBoolean())).andReturn(hostRoleCommand).times(7); @@ -417,6 +422,17 @@ public class ClusterInstallWithoutStartOnComponentLevelTest extends EasyMockSupp persistedState.persistLogicalRequest((LogicalRequest) anyObject(), anyLong()); expectLastCall().once(); + expect(ambariContext.createClusterTopology(request)).andReturn(clusterTopology); + expect(clusterTopology.getSecurity()).andReturn(SecurityConfiguration.NONE).anyTimes(); + expect(clusterTopology.getClusterId()).andReturn(CLUSTER_ID).anyTimes(); + expect(clusterTopology.getSetting()).andReturn(new Setting(ImmutableMap.of())).anyTimes(); + expect(clusterTopology.getBlueprint()).andReturn(blueprint).anyTimes(); + expect(clusterTopology.getProvisionAction()).andReturn(ProvisionAction.INSTALL_AND_START).anyTimes(); + expect(clusterTopology.getAmbariContext()).andReturn(ambariContext).anyTimes(); + expect(clusterTopology.getConfiguration()).andReturn(bpConfiguration).anyTimes(); + expect(clusterTopology.getStack()).andReturn(stack).anyTimes(); + expect(clusterTopology.getStackIds()).andReturn(ImmutableSet.of(new StackId(STACK_NAME, STACK_VERSION))).anyTimes(); + replayAll(); Whitebox.setInternalState(topologyManager, "executor", executor); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java index 295817b..525e691 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java @@ -20,6 +20,9 @@ package org.apache.ambari.server.topology; import static java.util.stream.Collectors.toSet; import static org.apache.ambari.server.controller.internal.ProvisionAction.INSTALL_ONLY; +import static org.apache.ambari.server.topology.ClusterDeploymentTestCommon.CLUSTER_ID; +import static org.apache.ambari.server.topology.ClusterDeploymentTestCommon.CLUSTER_NAME; +import static org.apache.ambari.server.topology.ClusterDeploymentTestCommon.service; import static org.apache.ambari.server.topology.StackComponentResolverTest.builderFor; import static org.easymock.EasyMock.anyBoolean; import static org.easymock.EasyMock.anyLong; @@ -32,12 +35,10 @@ import static org.easymock.EasyMock.expectLastCall; import static org.easymock.EasyMock.isA; import static org.easymock.EasyMock.newCapture; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ExecutorService; @@ -47,6 +48,7 @@ import org.apache.ambari.server.Role; import org.apache.ambari.server.RoleCommand; import org.apache.ambari.server.actionmanager.HostRoleCommand; import org.apache.ambari.server.actionmanager.HostRoleStatus; +import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.controller.AmbariManagementController; import org.apache.ambari.server.controller.AmbariServer; import org.apache.ambari.server.controller.ClusterRequest; @@ -71,7 +73,6 @@ import org.apache.ambari.server.state.SecurityType; import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.topology.tasks.ConfigureClusterTask; import org.apache.ambari.server.topology.tasks.ConfigureClusterTaskFactory; -import org.apache.ambari.server.topology.validators.TopologyValidator; import org.easymock.Capture; import org.easymock.EasyMockRule; import org.easymock.EasyMockSupport; @@ -90,13 +91,13 @@ import org.powermock.reflect.Whitebox; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; @RunWith(PowerMockRunner.class) -@PrepareForTest({ AmbariServer.class }) +@PrepareForTest({ AmbariContext.class, AmbariServer.class }) public class ClusterInstallWithoutStartTest extends EasyMockSupport { - private static final String CLUSTER_NAME = "test-cluster"; - private static final long CLUSTER_ID = 1; + private static final String BLUEPRINT_NAME = "test-bp"; private static final String STACK_NAME = "test-stack"; private static final String STACK_VERSION = "test-stack-version"; @@ -168,7 +169,8 @@ public class ClusterInstallWithoutStartTest extends EasyMockSupport { private ArtifactResourceProvider artifactResourceProvider; @Mock(type = MockType.NICE) private MpackResourceProvider mpackResourceProvider; - + @Mock + private AmbariMetaInfo ambariMetaInfo; @Mock(type = MockType.NICE) private ComponentInfo serviceComponentInfo; @@ -181,6 +183,9 @@ public class ClusterInstallWithoutStartTest extends EasyMockSupport { @Mock private ComponentResolver componentResolver; + @Mock(type = MockType.NICE) + private ClusterTopology clusterTopology; + private final Configuration stackConfig = new Configuration(new HashMap<>(), new HashMap<>()); private final Configuration bpConfiguration = new Configuration(new HashMap<>(), @@ -211,8 +216,6 @@ public class ClusterInstallWithoutStartTest extends EasyMockSupport { private String predicate = "Hosts/host_name=foo"; - private List<TopologyValidator> topologyValidators = new ArrayList<>(); - private Capture<ClusterTopology> clusterTopologyCapture; private Capture<Map<String, Object>> configRequestPropertiesCapture; private Capture<Map<String, Object>> configRequestPropertiesCapture2; @@ -273,6 +276,8 @@ public class ClusterInstallWithoutStartTest extends EasyMockSupport { expect(blueprint.getHostGroupsForComponent("component4")).andReturn(Collections.singleton(group2)).anyTimes(); expect(blueprint.getName()).andReturn(BLUEPRINT_NAME).anyTimes(); expect(ambariContext.composeStacks(anyObject())).andReturn(stack).anyTimes(); + expect(ambariContext.getServices(anyString())).andReturn( + Sets.newHashSet(service("service1", 1L), service("service2", 2L))).anyTimes(); expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); expect(blueprint.getSecurity()).andReturn(SecurityConfiguration.NONE).anyTimes(); expect(blueprint.getMpacks()).andReturn(ImmutableSet.of()).anyTimes(); @@ -368,7 +373,11 @@ public class ClusterInstallWithoutStartTest extends EasyMockSupport { PowerMock.mockStatic(AmbariServer.class); expect(AmbariServer.getController()).andReturn(managementController).anyTimes(); PowerMock.replay(AmbariServer.class); - expect(ambariContext.getClusterController()).andReturn(clusterController).anyTimes(); + + PowerMock.mockStatic(AmbariContext.class); + expect(AmbariContext.getClusterController()).andReturn(clusterController).anyTimes(); + PowerMock.replay(AmbariContext.class); + expect(clusterController.ensureResourceProvider(Resource.Type.Mpack)).andReturn(mpackResourceProvider).anyTimes(); expect(clusterController.ensureResourceProvider(Resource.Type.Artifact)).andReturn(artifactResourceProvider).anyTimes(); RequestStatus completedStatus = createNiceMock(RequestStatus.class); @@ -379,7 +388,11 @@ public class ClusterInstallWithoutStartTest extends EasyMockSupport { expect(clusters.getClusterById(anyLong())).andReturn(cluster).anyTimes(); expect(cluster.getClusterName()).andReturn(CLUSTER_NAME).anyTimes(); + expect(managementController.getAmbariMetaInfo()).andReturn(ambariMetaInfo).anyTimes(); + expect(ambariMetaInfo.getClusterProperties()).andReturn(Sets.newHashSet()).anyTimes(); + expect(ambariContext.getPersistedTopologyState()).andReturn(persistedState).anyTimes(); + expect(ambariContext.getController()).andReturn(managementController).anyTimes(); //todo: don't ignore param ambariContext.createAmbariResources(isA(ClusterTopology.class), eq(CLUSTER_NAME), eq(SecurityType.NONE)); expectLastCall().once(); @@ -387,13 +400,6 @@ public class ClusterInstallWithoutStartTest extends EasyMockSupport { expect(ambariContext.isClusterKerberosEnabled(CLUSTER_ID)).andReturn(false).anyTimes(); expect(ambariContext.getClusterId(CLUSTER_NAME)).andReturn(CLUSTER_ID).anyTimes(); expect(ambariContext.getClusterName(CLUSTER_ID)).andReturn(CLUSTER_NAME).anyTimes(); - // so only INITIAL config - expect(ambariContext.createConfigurationRequests(capture(configRequestPropertiesCapture))). - andReturn(Collections.singletonList(configurationRequest)); - expect(ambariContext.createConfigurationRequests(capture(configRequestPropertiesCapture2))). - andReturn(Collections.singletonList(configurationRequest2)).once(); - expect(ambariContext.createConfigurationRequests(capture(configRequestPropertiesCapture3))). - andReturn(Collections.singletonList(configurationRequest3)).once(); // INSTALL task expectation expect(ambariContext.createAmbariTask(anyLong(), anyLong(), anyString(), anyString(), eq(AmbariContext.TaskType.INSTALL), anyBoolean())).andReturn(hostRoleCommand).atLeastOnce(); @@ -418,6 +424,17 @@ public class ClusterInstallWithoutStartTest extends EasyMockSupport { persistedState.persistLogicalRequest((LogicalRequest) anyObject(), anyLong()); expectLastCall().once(); + expect(ambariContext.createClusterTopology(request)).andReturn(clusterTopology); + expect(clusterTopology.getSecurity()).andReturn(SecurityConfiguration.NONE).anyTimes(); + expect(clusterTopology.getClusterId()).andReturn(CLUSTER_ID).anyTimes(); + expect(clusterTopology.getSetting()).andReturn(new Setting(ImmutableMap.of())).anyTimes(); + expect(clusterTopology.getBlueprint()).andReturn(blueprint).anyTimes(); + expect(clusterTopology.getProvisionAction()).andReturn(ProvisionAction.INSTALL_ONLY).anyTimes(); + expect(clusterTopology.getAmbariContext()).andReturn(ambariContext).anyTimes(); + expect(clusterTopology.getConfiguration()).andReturn(bpConfiguration).anyTimes(); + expect(clusterTopology.getStack()).andReturn(stack).anyTimes(); + expect(clusterTopology.getStackIds()).andReturn(ImmutableSet.of(new StackId(STACK_NAME, STACK_VERSION))).anyTimes(); + replayAll(); Whitebox.setInternalState(topologyManager, "executor", executor); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterTopologyImplTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterTopologyImplTest.java index bab1b0c..074d87f 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterTopologyImplTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterTopologyImplTest.java @@ -32,9 +32,12 @@ import java.util.Map; import java.util.Set; import java.util.stream.Stream; +import org.apache.ambari.server.api.services.AmbariMetaInfo; +import org.apache.ambari.server.controller.AmbariManagementController; import org.apache.ambari.server.controller.internal.ProvisionAction; import org.apache.ambari.server.controller.internal.StackDefinition; import org.apache.ambari.server.state.ComponentInfo; +import org.apache.ambari.server.state.PropertyInfo; import org.apache.ambari.server.state.ServiceInfo; import org.apache.ambari.server.state.StackId; import org.apache.commons.lang3.tuple.Pair; @@ -52,6 +55,8 @@ import org.junit.runner.RunWith; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; /** * Unit tests for ClusterTopologyImpl. @@ -106,14 +111,23 @@ public class ClusterTopologyImplTest extends EasyMockSupport { "group4", ImmutableSet.of( builderFor("any_service", "component5")) ); + private final AmbariManagementController controller = mock(AmbariManagementController.class); + private final AmbariMetaInfo metaInfo = mock(AmbariMetaInfo.class); private BlueprintBasedClusterProvisionRequest provisionRequest; private Configuration bpconfiguration; @Before public void setUp() throws Exception { - bpconfiguration = Configuration.createEmpty(); - + bpconfiguration = new Configuration( + Maps.newHashMap(ImmutableMap.of( + "cluster-env", + Maps.newHashMap(ImmutableMap.of( + "commands_to_retry", "INSTALL", + "command_retry_max_time_in_sec", "500", + "unknown_property_that_should_not_become_cluster_setting", "some_value")) + )), + new HashMap<>()); HostGroupInfo group1Info = new HostGroupInfo("group1"); HostGroupInfo group2Info = new HostGroupInfo("group2"); HostGroupInfo group3Info = new HostGroupInfo("group3"); @@ -158,6 +172,7 @@ public class ClusterTopologyImplTest extends EasyMockSupport { expect(stack.getServicesForComponent("ZOOKEEPER_CLIENT")).andAnswer(() -> Stream.of(Pair.of(STACK_ID, aServiceWith(aComponent("ZOOKEEPER_CLIENT"))))).anyTimes(); expect(ambariContext.composeStacks(STACK_IDS)).andReturn(stack).anyTimes(); + expect(ambariContext.getController()).andReturn(controller).anyTimes(); expect(blueprint.getMpacks()).andReturn(ImmutableSet.of()).anyTimes(); expect(blueprint.getHostGroups()).andReturn(hostGroupMap).anyTimes(); @@ -167,12 +182,20 @@ public class ClusterTopologyImplTest extends EasyMockSupport { expect(hostGroup.getName()).andReturn(name).anyTimes(); expect(blueprint.getHostGroup(name)).andReturn(hostGroup).anyTimes(); } + expect(blueprint.getSetting()).andReturn(new Setting(new HashMap<>())); expect(group1.getConfiguration()).andReturn(configuration).anyTimes(); expect(group2.getConfiguration()).andReturn(configuration).anyTimes(); expect(group3.getConfiguration()).andReturn(configuration).anyTimes(); expect(group4.getConfiguration()).andReturn(configuration).anyTimes(); + expect(controller.getAmbariMetaInfo()).andReturn(metaInfo).anyTimes(); + expect(metaInfo.getClusterProperties()).andReturn( + Sets.newHashSet( + propertyInfo("command_retry_enabled", "true"), + propertyInfo("commands_to_retry", "INSTALL,START"), + propertyInfo("command_retry_max_time_in_sec", "600"))).anyTimes(); + replayAll(); provisionRequest = new BlueprintBasedClusterProvisionRequest(ambariContext, null, blueprint, new TestTopologyRequest()); @@ -262,6 +285,29 @@ public class ClusterTopologyImplTest extends EasyMockSupport { assertFalse(topology.isComponentHadoopCompatible("ZOOKEEPER_CLIENT")); } + @Test + public void testAdjustTopology() throws Exception { + ClusterTopologyImpl topology = new ClusterTopologyImpl(ambariContext, provisionRequest, resolvedComponents); + Map<String, String> clusterSettings = topology.getSetting().getClusterSettings(); + assertEquals( + ImmutableMap.of( + "commands_to_retry", "INSTALL", + "command_retry_max_time_in_sec", "500"), + clusterSettings + ); + assertEquals( + ImmutableMap.of("unknown_property_that_should_not_become_cluster_setting", "some_value"), + topology.getConfiguration().getFullProperties().get("cluster-env") + ); + } + + private static PropertyInfo propertyInfo(String name, String value) { + PropertyInfo info = new PropertyInfo(); + info.setName(name); + info.setValue(value); + return info; + } + private ServiceInfo aHCFSWith(ComponentInfo... components) { ServiceInfo service = aServiceWith(components); service.setServiceType(ServiceInfo.HADOOP_COMPATIBLE_FS); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/SettingFactoryTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/SettingFactoryTest.java index 94c412d..e8a775e 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/SettingFactoryTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/SettingFactoryTest.java @@ -30,6 +30,9 @@ import java.util.Set; import org.junit.Test; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; + /** * Test the SettingFactory class */ @@ -57,6 +60,29 @@ public class SettingFactoryTest { } /** + * Test creating setting defined in {"settings/recovery_settings" -> { ... } } style maps (typically coming from + * REST API). + */ + @Test + public void testGetSettingFromSlashedPropertymaps() { + Set<Map<String, String>> clusterSettings = + ImmutableSet.of( + ImmutableMap.of("command_retry_enabled", "true", + "commands_to_retry", "INSTALL,START")); + + Map<String, Object> propertyMap = ImmutableMap.of( + "settings/recovery_settings", + ImmutableSet.of(ImmutableMap.of("recovery_enabled", "true")), + "settings/cluster_settings", clusterSettings + ); + Setting setting = SettingFactory.getSetting(ImmutableSet.of(propertyMap)); + + assertEquals("true", setting.getRecoveryEnabled("any service", "any component")); + assertEquals(clusterSettings, setting.getProperties().get("cluster_settings")); + + } + + /** * { * "recovery_settings":[ * { @@ -134,4 +160,5 @@ public class SettingFactoryTest { return setting; } + } diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/SettingTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/SettingTest.java index ecd5acb..a4ee1a6 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/SettingTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/SettingTest.java @@ -28,6 +28,7 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; +import org.apache.ambari.server.state.ConfigHelper; import org.junit.BeforeClass; import org.junit.Test; @@ -193,4 +194,38 @@ public class SettingTest { assertFalse(setting.shouldSkipFailure()); } + @Test + public void testGetClusterSettings() throws Exception { + Setting setting = new Setting( + ImmutableMap.of( + "cluster_settings", + ImmutableSet.of( + ImmutableMap.of( + ConfigHelper.COMMAND_RETRY_ENABLED, "true"), + ImmutableMap.of( + ConfigHelper.COMMAND_RETRY_MAX_TIME_IN_SEC, "600", + ConfigHelper.COMMANDS_TO_RETRY, "INSTALL,START")), + "recovery_settings", + ImmutableSet.of( + ImmutableMap.of( + Setting.SETTING_NAME_RECOVERY_ENABLED, "true", + "recovery_type", "AUTO_START"), + ImmutableMap.of( + "recovery_lifetime_max_count", "1024", + "recovery_max_count", "6")) + )); + + Map<String, String> expectedClusterSettings = ImmutableMap.<String, String>builder() + .put(ConfigHelper.COMMAND_RETRY_ENABLED, "true") + .put(ConfigHelper.COMMAND_RETRY_MAX_TIME_IN_SEC, "600") + .put(ConfigHelper.COMMANDS_TO_RETRY, "INSTALL,START") + .put(Setting.SETTING_NAME_RECOVERY_ENABLED, "true") + .put("recovery_type", "AUTO_START") + .put("recovery_lifetime_max_count", "1024") + .put("recovery_max_count", "6") + .build(); + + assertEquals(expectedClusterSettings, setting.getClusterSettings()); + } + } diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java index e549ce4..8dc4520 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java @@ -396,19 +396,13 @@ public class TopologyManagerTest { expect(ambariContext.getClusterId(CLUSTER_NAME)).andReturn(CLUSTER_ID).anyTimes(); expect(ambariContext.getClusterName(CLUSTER_ID)).andReturn(CLUSTER_NAME).anyTimes(); // cluster configuration task run() isn't executed by mock executor - // so only INITIAL config - expect(ambariContext.createConfigurationRequests(capture(configRequestPropertiesCapture))). - andReturn(Collections.singletonList(configurationRequest)).anyTimes(); - expect(ambariContext.createConfigurationRequests(capture(configRequestPropertiesCapture2))). - andReturn(Collections.singletonList(configurationRequest2)).anyTimes(); - expect(ambariContext.createConfigurationRequests(capture(configRequestPropertiesCapture3))). - andReturn(Collections.singletonList(configurationRequest3)).anyTimes(); ambariContext.setConfigurationOnCluster(capture(updateClusterConfigRequestCapture)); expectLastCall().anyTimes(); ambariContext.persistInstallStateForUI(CLUSTER_NAME, STACK_ID); expectLastCall().anyTimes(); expect(ambariContext.getServices(anyString())).andReturn(services).anyTimes(); + expect(ambariContext.getController()).andReturn(controller).anyTimes(); expect(resourceProvider.createResources((anyObject()))).andReturn(new RequestStatusImpl(null, null, null)).anyTimes(); // persist raw request expect(clusterController.ensureResourceProvider(anyObject(Resource.Type.class))).andReturn(resourceProvider); @@ -433,6 +427,7 @@ public class TopologyManagerTest { return null; }). anyTimes(); + expect(metaInfo.getClusterProperties()).andReturn(new HashSet<>()); expect(controller.getAmbariMetaInfo()).andReturn(metaInfo).anyTimes(); mockStatic(AmbariServer.class); expect(AmbariServer.getController()).andReturn(controller).anyTimes();