This is an automated email from the ASF dual-hosted git repository.
benyoka pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new f3aa02f AMBARI-25024 Config recommendation for add service request
(benyoka) (#2704)
f3aa02f is described below
commit f3aa02f7bb4c765885af9d740c7661dd1cdbca80
Author: benyoka <[email protected]>
AuthorDate: Fri Dec 14 00:59:39 2018 +0100
AMBARI-25024 Config recommendation for add service request (benyoka) (#2704)
* AMBARI-25024 Config recommendation for add service request (benyoka)
* AMBARI-25024 review comments (benyoka)
* AMBARI-25024 fix unit test (benyoka)
* AMBARI-25024 fix unit tests, work without layout recommendation
* AMBARI-25024 fix merge compile issue (benyoka)
* AMBARI-25024 fix NPE (benyoka)
---
ambari-server/pom.xml | 2 +-
.../services/stackadvisor/StackAdvisorRequest.java | 30 ++
.../recommendations/RecommendationResponse.java | 32 ++
.../server/controller/AddServiceRequest.java | 2 +-
.../internal/BlueprintConfigurationProcessor.java | 2 +
.../ambari/server/controller/internal/Stack.java | 18 +-
.../server/controller/internal/UnitUpdater.java | 35 +-
.../ambari/server/state/ValueAttributesInfo.java | 17 +
.../topology/ConfigRecommendationStrategy.java | 30 +-
.../ambari/server/topology/ConfigurableHelper.java | 17 +
.../ambari/server/topology/Configuration.java | 16 +-
.../server/topology/addservice/AddServiceInfo.java | 20 +-
.../addservice/AddServiceOrchestrator.java | 3 +-
.../addservice/LayoutRecommendationInfo.java | 53 +++
.../topology/addservice/RequestValidator.java | 6 +-
.../topology/addservice/StackAdvisorAdapter.java | 197 ++++++++--
.../server/controller/AddServiceRequestTest.java | 8 +-
.../controller/internal/UnitUpdaterTest.java | 61 ++-
.../server/testutils/TestCollectionUtils.java | 50 +++
.../ambari/server/topology/ConfigurableTest.java | 16 +
.../ambari/server/topology/ConfigurationTest.java | 49 +++
.../topology/addservice/RequestValidatorTest.java | 4 +-
.../addservice/StackAdvisorAdapterTest.java | 434 +++++++++++++++++++--
23 files changed, 998 insertions(+), 104 deletions(-)
diff --git a/ambari-server/pom.xml b/ambari-server/pom.xml
index 535f6b5..fd8db02 100644
--- a/ambari-server/pom.xml
+++ b/ambari-server/pom.xml
@@ -1323,7 +1323,7 @@
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
- <version>3.1</version>
+ <version>3.8.1</version>
</dependency>
<dependency>
<groupId>commons-net</groupId>
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java
b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java
index 10baa33..5fb137c 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java
@@ -18,6 +18,8 @@
package org.apache.ambari.server.api.services.stackadvisor;
+import static java.util.stream.Collectors.toMap;
+
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -30,9 +32,12 @@ import java.util.Set;
import
org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse;
import org.apache.ambari.server.state.ChangedConfigInfo;
import org.apache.ambari.server.state.StackId;
+import org.apache.ambari.server.topology.Configuration;
import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.tuple.Pair;
import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
/**
* Stack advisor request.
@@ -151,6 +156,22 @@ public class StackAdvisorRequest {
this.stackVersion = stackVersion;
}
+ public StackAdvisorRequestBuilder builder() {
+ return StackAdvisorRequestBuilder.forStack(stackName, stackVersion)
+ .ofType(requestType)
+ .forHosts(hosts)
+ .forServices(services)
+ .forHostComponents(hostComponents)
+ .forHostsGroupBindings(hostGroupBindings)
+ .withComponentHostsMap(componentHostsMap)
+ .withConfigurations(configurations)
+ .withChangedConfigurations(changedConfigurations)
+ .withConfigGroups(configGroups)
+ .withUserContext(userContext)
+ .withGPLLicenseAccepted(gplLicenseAccepted)
+ .withLdapConfig(ldapConfig);
+ }
+
public static class StackAdvisorRequestBuilder {
StackAdvisorRequest instance;
@@ -204,6 +225,15 @@ public class StackAdvisorRequest {
return this;
}
+ public StackAdvisorRequestBuilder withConfigurations(Configuration
configuration) {
+ Map<String, Map<String, String>> properties =
configuration.getFullProperties();
+ this.instance.configurations = properties.entrySet().stream()
+ .map( e -> Pair.of(e.getKey(), ImmutableMap.of("properties",
e.getValue())))
+ .collect(toMap(Pair::getKey, Pair::getValue));
+ return this;
+ }
+
+
public StackAdvisorRequestBuilder withChangedConfigurations(
List<ChangedConfigInfo> changedConfigurations) {
this.instance.changedConfigurations = changedConfigurations;
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/recommendations/RecommendationResponse.java
b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/recommendations/RecommendationResponse.java
index 99e9ab2..5d5247e 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/recommendations/RecommendationResponse.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/recommendations/RecommendationResponse.java
@@ -18,6 +18,7 @@
package org.apache.ambari.server.api.services.stackadvisor.recommendations;
+import static com.google.common.collect.Maps.transformValues;
import static java.util.stream.Collectors.groupingBy;
import static java.util.stream.Collectors.mapping;
import static java.util.stream.Collectors.toMap;
@@ -27,13 +28,16 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.Optional;
import java.util.Set;
import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorResponse;
import org.apache.ambari.server.state.ValueAttributesInfo;
+import org.apache.ambari.server.topology.ConfigurableHelper;
import org.apache.commons.lang3.tuple.Pair;
import org.codehaus.jackson.annotate.JsonIgnore;
import org.codehaus.jackson.annotate.JsonProperty;
+import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jackson.map.annotate.JsonSerialize;
import com.google.common.collect.ImmutableMap;
@@ -150,6 +154,24 @@ public class RecommendationResponse extends
StackAdvisorResponse {
@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL)
private Map<String, ValueAttributesInfo> propertyAttributes = null;
+ /**
+ *
+ * @param properties properties in <i>name -> value</i> format
+ * @param attributes attributes in <i>attribute name -> property name ->
value</i> format
+ */
+ public static BlueprintConfigurations create(Map<String, String>
properties, Map<String, Map<String, String>> attributes) {
+ BlueprintConfigurations config = new BlueprintConfigurations();
+ config.setProperties(properties);
+ if (attributes != null) {
+ // transform map to property name -> attribute name -> value format
+ Map<String, Map<String, String>> transformedAttributes =
ConfigurableHelper.transformAttributesMap(attributes);
+ ObjectMapper mapper = new ObjectMapper();
+ config.setPropertyAttributes(
+ new HashMap<>(transformValues(transformedAttributes, attr ->
ValueAttributesInfo.fromMap(attr, Optional.of(mapper)))));
+ }
+ return config;
+ }
+
public BlueprintConfigurations() {
}
@@ -176,6 +198,16 @@ public class RecommendationResponse extends
StackAdvisorResponse {
return propertyAttributes;
}
+ /**
+ * @return value attributes in <i>attribute name -> property name ->
value</i> format
+ */
+ @JsonIgnore
+ public Map<String, Map<String, String>> getPropertyAttributesAsMap() {
+ ObjectMapper mapper = new ObjectMapper();
+ return null == propertyAttributes ? null :
+ ConfigurableHelper.transformAttributesMap(
transformValues(propertyAttributes, vaInfo ->
vaInfo.toMap(Optional.of(mapper))) );
+ }
+
public void setPropertyAttributes(Map<String, ValueAttributesInfo>
propertyAttributes) {
this.propertyAttributes = propertyAttributes;
}
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/controller/AddServiceRequest.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AddServiceRequest.java
index 02577ca..e169f82 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/controller/AddServiceRequest.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AddServiceRequest.java
@@ -124,7 +124,7 @@ public class AddServiceRequest {
Configuration configuration
) {
this.operationType = null != operationType ? operationType :
OperationType.ADD_SERVICE;
- this.recommendationStrategy = null != recommendationStrategy ?
recommendationStrategy : ConfigRecommendationStrategy.NEVER_APPLY;
+ this.recommendationStrategy = null != recommendationStrategy ?
recommendationStrategy : ConfigRecommendationStrategy.defaultForAddService();
this.provisionAction = null != provisionAction ? provisionAction :
ProvisionAction.INSTALL_AND_START;
this.validationType = validationType != null ? validationType :
ValidationType.DEFAULT;
this.stackName = stackName;
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
index 46d7e9b..12ef314 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
@@ -1023,6 +1023,8 @@ public class BlueprintConfigurationProcessor {
}
}
+
+
/**
* Update configuration properties based on advised configuration properties.
* @param configuration configuration being processed
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java
index c83999b..3bfdeb3 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java
@@ -49,7 +49,6 @@ import org.apache.ambari.server.state.StackId;
import org.apache.ambari.server.state.ValueAttributesInfo;
import org.apache.ambari.server.topology.Cardinality;
import org.apache.ambari.server.topology.Configuration;
-import org.apache.ambari.server.topology.validators.UnitValidatedProperty;
/**
* Encapsulates stack information.
@@ -657,23 +656,20 @@ public class Stack {
* (eg. some properties need a unit to be appended)
*/
public Configuration getValidDefaultConfig() {
- Configuration config = getConfiguration();
-
- for (UnitValidatedProperty p : UnitValidatedProperty.ALL) {
- if (config.isPropertySet(p.getConfigType(), p.getPropertyName())) {
- String value = config.getPropertyValue(p.getConfigType(),
p.getPropertyName());
- String updatedValue = UnitUpdater.updateForClusterCreate(this,
p.getServiceName(), p.getConfigType(), p.getPropertyName(), value);
- config.setProperty(p.getConfigType(), p.getPropertyName(),
updatedValue);
- }
- }
+ Configuration config = getDefaultConfig();
+ UnitUpdater.updateUnits(config, this);
+ return config;
+ }
+ public Configuration getDefaultConfig() {
+ Configuration config = getConfiguration();
config.getProperties().values().forEach(
each -> each.values().removeIf(Objects::isNull)
);
-
return config;
}
+
/**
* Parse components for the specified service from the stack definition.
*
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UnitUpdater.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UnitUpdater.java
index 6691519..c6188b1 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UnitUpdater.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UnitUpdater.java
@@ -22,9 +22,12 @@ import static org.apache.commons.lang.StringUtils.isBlank;
import java.util.Collection;
import java.util.Collections;
import java.util.Map;
+import java.util.Objects;
import java.util.Optional;
+import java.util.function.BiFunction;
import org.apache.ambari.server.topology.ClusterTopology;
+import org.apache.ambari.server.topology.Configuration;
import org.apache.ambari.server.topology.validators.UnitValidatedProperty;
/**
@@ -65,12 +68,38 @@ public class UnitUpdater implements
BlueprintConfigurationProcessor.PropertyUpda
}
}
+ public static void updateUnits(Configuration configuration, Stack stack) {
+ updateAllUnitValidatedProperties(configuration,
+ (property, value) -> updateForClusterCreate(stack,
property.getServiceName(), property.getConfigType(),
property.getPropertyName(), value));
+ }
+
+ public static void removeUnits(Configuration configuration, Stack stack) {
+ updateAllUnitValidatedProperties(configuration,
+ (property, value) -> removeStackUnit(stack, property.getServiceName(),
property.getConfigType(), property.getPropertyName(), value));
+ }
+
+ private static void updateAllUnitValidatedProperties(Configuration
configuration, BiFunction<UnitValidatedProperty, String, String> valueUpdater) {
+ for (UnitValidatedProperty p : UnitValidatedProperty.ALL) {
+ if (configuration.isPropertySet(p.getConfigType(), p.getPropertyName()))
{
+ String value = configuration.getPropertyValue(p.getConfigType(),
p.getPropertyName());
+ String updatedValue = valueUpdater.apply(p, value);
+ if (!Objects.equals(value, updatedValue)) {
+ configuration.setProperty(p.getConfigType(), p.getPropertyName(),
updatedValue);
+ }
+ }
+ }
+ }
+
/**
* @return property value with removed unit
*/
@Override
public String updateForBlueprintExport(String propertyName, String
origValue, Map<String, Map<String, String>> properties, ClusterTopology
topology) {
- PropertyUnit stackUnit =
PropertyUnit.of(topology.getBlueprint().getStack(), serviceName, configType,
propertyName);
+ return removeStackUnit(topology.getBlueprint().getStack(), serviceName,
configType, propertyName, origValue);
+ }
+
+ static String removeStackUnit(Stack stack, String serviceName, String
configType, String propertyName, String origValue) {
+ PropertyUnit stackUnit = PropertyUnit.of(stack, serviceName, configType,
propertyName);
PropertyValue value = PropertyValue.of(propertyName, origValue);
return value.withoutUnit(stackUnit);
}
@@ -150,6 +179,10 @@ public class UnitUpdater implements
BlueprintConfigurationProcessor.PropertyUpda
}
public boolean hasAnyUnit() {
+ return hasAnyUnit(value);
+ }
+
+ static boolean hasAnyUnit(String value) {
return !Character.isDigit(value.charAt(value.length() -1));
}
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/state/ValueAttributesInfo.java
b/ambari-server/src/main/java/org/apache/ambari/server/state/ValueAttributesInfo.java
index 156157a..f8ddf0b 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/state/ValueAttributesInfo.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/state/ValueAttributesInfo.java
@@ -19,6 +19,8 @@
package org.apache.ambari.server.state;
import java.util.Collection;
+import java.util.Map;
+import java.util.Optional;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
@@ -28,7 +30,9 @@ import javax.xml.bind.annotation.XmlElements;
import org.apache.ambari.server.controller.ApiModel;
import org.codehaus.jackson.annotate.JsonProperty;
+import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jackson.map.annotate.JsonSerialize;
+import org.codehaus.jackson.type.TypeReference;
import io.swagger.annotations.ApiModelProperty;
@@ -396,6 +400,19 @@ public class ValueAttributesInfo implements ApiModel {
return result;
}
+ public Map<String, String> toMap(Optional<ObjectMapper> mapper) {
+ Map<String, String> map =
+ mapper.orElseGet(ObjectMapper::new).convertValue(this, new
TypeReference<Map<String, String>>(){});
+ if ( !Boolean.parseBoolean(map.get("keyStore")) ) { // keyStore is
declared as a primitive value instead of Boolean -> treat false as unset
+ map.remove("keyStore");
+ }
+ return map;
+ }
+
+ public static ValueAttributesInfo fromMap(Map<String, String> attributes,
Optional<ObjectMapper> mapper) {
+ return mapper.orElseGet(ObjectMapper::new).convertValue(attributes,
ValueAttributesInfo.class);
+ }
+
@Override
public String toString() {
return "ValueAttributesInfo{" +
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/topology/ConfigRecommendationStrategy.java
b/ambari-server/src/main/java/org/apache/ambari/server/topology/ConfigRecommendationStrategy.java
index bf3eacc..25d89e0 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/topology/ConfigRecommendationStrategy.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/topology/ConfigRecommendationStrategy.java
@@ -19,24 +19,46 @@
package org.apache.ambari.server.topology;
public enum ConfigRecommendationStrategy {
+
/**
* Configuration recommendations are always applied, overriding stack
defaults and
* configuration defined by the user in the Blueprint and/or Cluster
Creation Template.
*/
- ALWAYS_APPLY,
+ ALWAYS_APPLY(true, true),
/**
* Configuration recommendations are ignored with this option, both for
stack defaults
* and configuration defined by the user in the Blueprint and/or Cluster
Creation Template.
*/
- NEVER_APPLY,
+ NEVER_APPLY(false, false),
+
/**
* Configuration recommendations are always applied for properties listed
as stack defaults,
* but not for configurations defined by the user in the Blueprint and/or
Cluster Creation Template.
*/
- ONLY_STACK_DEFAULTS_APPLY,
+ ONLY_STACK_DEFAULTS_APPLY(true, false),
/**
* Configuration recommendations are always applied, overriding stack
defaults but they don't
* override configuration defined by the user in the Blueprint and/or
Cluster Creation Template.
*/
- ALWAYS_APPLY_DONT_OVERRIDE_CUSTOM_VALUES;
+ ALWAYS_APPLY_DONT_OVERRIDE_CUSTOM_VALUES(true, false);
+
+ public static ConfigRecommendationStrategy defaultForAddService() {
+ return ALWAYS_APPLY_DONT_OVERRIDE_CUSTOM_VALUES;
+ }
+
+ private final boolean useStackAdvisor;
+ private final boolean overrideCustomValues;
+
+ public boolean shouldUseStackAdvisor() {
+ return useStackAdvisor;
+ }
+
+ public boolean shouldOverrideCustomValues() {
+ return overrideCustomValues;
+ }
+
+ ConfigRecommendationStrategy(boolean useStackAdvisor, boolean
overrideCustomValues) {
+ this.useStackAdvisor = useStackAdvisor;
+ this.overrideCustomValues = overrideCustomValues;
+ }
}
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/topology/ConfigurableHelper.java
b/ambari-server/src/main/java/org/apache/ambari/server/topology/ConfigurableHelper.java
index 572f12d..9e75b0e 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/topology/ConfigurableHelper.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/topology/ConfigurableHelper.java
@@ -19,6 +19,8 @@
package org.apache.ambari.server.topology;
import static com.google.common.base.Preconditions.checkArgument;
+import static java.util.stream.Collectors.groupingBy;
+import static java.util.stream.Collectors.toMap;
import static
org.apache.ambari.server.controller.internal.BlueprintResourceProvider.PROPERTIES_ATTRIBUTES_PROPERTY_ID;
import static
org.apache.ambari.server.controller.internal.BlueprintResourceProvider.PROPERTIES_PROPERTY_ID;
import static
org.apache.ambari.server.topology.ConfigurationFactory.isKeyInLegacyFormat;
@@ -35,6 +37,8 @@ import java.util.Set;
import javax.annotation.Nullable;
+import org.apache.commons.lang3.tuple.Triple;
+
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
@@ -182,4 +186,17 @@ public class ConfigurableHelper {
return null != object ? object.getClass().getName() : null;
}
+
+ /**
+ * Transform attibutes map from <i>attributeName -> propertyName ->
value</i> to <i>propertyName -> attributeName -> value</i>
+ * or vice versa
+ * @param input the input map
+ * @return the output map
+ */
+ public static Map<String, Map<String, String>>
transformAttributesMap(Map<String, Map<String, String>> input) {
+ return input.entrySet().stream()
+ .flatMap(outer -> outer.getValue().entrySet().stream().map(inner ->
Triple.of(outer.getKey(), inner.getKey(), inner.getValue())))
+ .collect(groupingBy(Triple::getMiddle, toMap(Triple::getLeft,
Triple::getRight)));
+ }
+
}
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java
b/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java
index fc02297..e6b1d88 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java
@@ -29,6 +29,7 @@ import org.apache.commons.lang3.tuple.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
/**
* Configuration for a topology entity such as a blueprint, hostgroup or
cluster.
*/
@@ -149,8 +150,8 @@ public class Configuration {
/**
* Configuration.
*
- * @param properties properties
- * @param attributes attributes
+ * @param properties properties in configType -> propertyName -> value
format
+ * @param attributes attributes in configType -> attributeName ->
propertyName -> attributeValue format
*/
public Configuration(Map<String, Map<String, String>> properties,
Map<String, Map<String, Map<String, String>>>
attributes) {
@@ -441,6 +442,17 @@ public class Configuration {
return allTypes;
}
+ public boolean containsConfigType(String configType) {
+ return properties.containsKey(configType) ||
attributes.containsKey(configType) ||
+ (parentConfiguration != null &&
parentConfiguration.containsConfigType(configType));
+ }
+
+ public boolean containsConfig(String configType, String propertyName) {
+ return (properties.containsKey(configType) &&
properties.get(configType).containsKey(propertyName))
+ || (attributes.containsKey(configType) &&
attributes.get(configType).values().stream().filter(map ->
map.containsKey(propertyName)).findAny().isPresent())
+ || (parentConfiguration != null &&
parentConfiguration.containsConfig(configType, propertyName));
+ }
+
/**
* Get the parent configuration.
*
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/topology/addservice/AddServiceInfo.java
b/ambari-server/src/main/java/org/apache/ambari/server/topology/addservice/AddServiceInfo.java
index 46fe3f3..25b9a3b 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/topology/addservice/AddServiceInfo.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/topology/addservice/AddServiceInfo.java
@@ -41,6 +41,7 @@ public final class AddServiceInfo {
private final Map<String, Map<String, Set<String>>> newServices;
private final RequestStageContainer stages;
private final Configuration config;
+ private final LayoutRecommendationInfo recommendationInfo;
public AddServiceInfo(
AddServiceRequest request,
@@ -49,8 +50,9 @@ public final class AddServiceInfo {
Configuration config,
KerberosDescriptor kerberosDescriptor,
RequestStageContainer stages,
- Map<String, Map<String, Set<String>>> newServices
- ) {
+ Map<String, Map<String,
+ Set<String>>> newServices,
+ LayoutRecommendationInfo recommendationInfo) {
this.request = request;
this.clusterName = clusterName;
this.stack = stack;
@@ -58,10 +60,16 @@ public final class AddServiceInfo {
this.newServices = newServices;
this.stages = stages;
this.config = config;
+ this.recommendationInfo = recommendationInfo;
}
- public AddServiceInfo withNewServices(Map<String, Map<String, Set<String>>>
services) {
- return new AddServiceInfo(request, clusterName, stack, config,
kerberosDescriptor, stages, services);
+ public AddServiceInfo withLayoutRecommendation(Map<String, Map<String,
Set<String>>> services,
+ LayoutRecommendationInfo
recommendation) {
+ return new AddServiceInfo(request, clusterName, stack, config,
kerberosDescriptor, stages, services, recommendation);
+ }
+
+ public AddServiceInfo withConfig(Configuration newConfig) {
+ return new AddServiceInfo(request, clusterName, stack, newConfig,
kerberosDescriptor, stages, newServices, recommendationInfo);
}
@Override
@@ -97,6 +105,10 @@ public final class AddServiceInfo {
return config;
}
+ public Optional<LayoutRecommendationInfo> getRecommendationInfo() {
+ return Optional.ofNullable(recommendationInfo);
+ }
+
public Optional<KerberosDescriptor> getKerberosDescriptor() {
return Optional.ofNullable(kerberosDescriptor);
}
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/topology/addservice/AddServiceOrchestrator.java
b/ambari-server/src/main/java/org/apache/ambari/server/topology/addservice/AddServiceOrchestrator.java
index c4b6529..4ff1301 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/topology/addservice/AddServiceOrchestrator.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/topology/addservice/AddServiceOrchestrator.java
@@ -143,8 +143,7 @@ public class AddServiceOrchestrator {
*/
private AddServiceInfo recommendConfiguration(AddServiceInfo request) {
LOG.info("Recommending configuration for {}", request);
- // TODO implement
- return request;
+ return stackAdvisorAdapter.recommendConfigurations(request);
}
/**
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/topology/addservice/LayoutRecommendationInfo.java
b/ambari-server/src/main/java/org/apache/ambari/server/topology/addservice/LayoutRecommendationInfo.java
new file mode 100644
index 0000000..03257ee
--- /dev/null
+++
b/ambari-server/src/main/java/org/apache/ambari/server/topology/addservice/LayoutRecommendationInfo.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.topology.addservice;
+
+import static java.util.stream.Collectors.toList;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Encapsulates information from layout recommendation that can be used or
reused for config recommendation.
+ */
+public class LayoutRecommendationInfo {
+ private final Map<String, Set<String>> hostGroups;
+ private final Map<String, Map<String, Set<String>>> allServiceComponentHosts;
+
+ public LayoutRecommendationInfo(Map<String, Set<String>> hostGroups,
Map<String, Map<String, Set<String>>> allServiceComponentHosts) {
+ this.hostGroups = hostGroups;
+ this.allServiceComponentHosts = allServiceComponentHosts;
+ }
+
+ public Map<String, Set<String>> getHostGroups() {
+ return hostGroups;
+ }
+
+ public Map<String, Map<String, Set<String>>> getAllServiceLayouts() {
+ return allServiceComponentHosts;
+ }
+
+ public List<String> getHosts() {
+ return getHostsFromHostGroups(hostGroups);
+ }
+
+ public static List<String> getHostsFromHostGroups(Map<String, Set<String>>
hostGroups) {
+ return hostGroups.values().stream().flatMap(Set::stream).collect(toList());
+ }
+}
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/topology/addservice/RequestValidator.java
b/ambari-server/src/main/java/org/apache/ambari/server/topology/addservice/RequestValidator.java
index 4a8d14c..29ac8c8 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/topology/addservice/RequestValidator.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/topology/addservice/RequestValidator.java
@@ -37,6 +37,7 @@ import org.apache.ambari.server.controller.AddServiceRequest;
import org.apache.ambari.server.controller.AmbariManagementController;
import org.apache.ambari.server.controller.internal.RequestStageContainer;
import org.apache.ambari.server.controller.internal.Stack;
+import org.apache.ambari.server.controller.internal.UnitUpdater;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.ConfigHelper;
import org.apache.ambari.server.state.SecurityType;
@@ -117,7 +118,7 @@ public class RequestValidator {
RequestStageContainer stages = new
RequestStageContainer(actionManager.getNextRequestId(), null, requestFactory,
actionManager);
AddServiceInfo validatedRequest = new AddServiceInfo(request,
cluster.getClusterName(),
state.getStack(), state.getConfig(), state.getKerberosDescriptor(),
- stages, state.getNewServices());
+ stages, state.getNewServices(), null);
stages.setRequestContext(validatedRequest.describe());
return validatedRequest;
}
@@ -250,9 +251,10 @@ public class RequestValidator {
}
Configuration clusterConfig = getClusterDesiredConfigs();
-
clusterConfig.setParentConfiguration(state.getStack().getValidDefaultConfig());
+ clusterConfig.setParentConfiguration(state.getStack().getDefaultConfig());
config.setParentConfiguration(clusterConfig);
+ UnitUpdater.removeUnits(config, state.getStack()); // stack advisor
doesn't like units; they'll be added back after recommendation
state = state.with(config);
}
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/topology/addservice/StackAdvisorAdapter.java
b/ambari-server/src/main/java/org/apache/ambari/server/topology/addservice/StackAdvisorAdapter.java
index 45f0e59..7504d45 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/topology/addservice/StackAdvisorAdapter.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/topology/addservice/StackAdvisorAdapter.java
@@ -40,10 +40,11 @@ import
org.apache.ambari.server.api.services.stackadvisor.StackAdvisorHelper;
import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest;
import
org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse;
import
org.apache.ambari.server.api.services.stackadvisor.validations.ValidationResponse;
-import org.apache.ambari.server.configuration.Configuration;
import org.apache.ambari.server.controller.AmbariManagementController;
+import org.apache.ambari.server.controller.internal.UnitUpdater;
import org.apache.ambari.server.state.Cluster;
-import org.apache.ambari.server.state.StackId;
+import org.apache.ambari.server.topology.ConfigRecommendationStrategy;
+import org.apache.ambari.server.topology.Configuration;
import org.apache.commons.lang3.tuple.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -64,7 +65,7 @@ public class StackAdvisorAdapter {
private StackAdvisorHelper stackAdvisorHelper;
@Inject
- private Configuration serverConfig;
+ private org.apache.ambari.server.configuration.Configuration serverConfig;
@Inject
private Injector injector;
@@ -79,19 +80,14 @@ public class StackAdvisorAdapter {
*/
AddServiceInfo recommendLayout(AddServiceInfo info) {
try {
- Cluster cluster =
managementController.getClusters().getCluster(info.clusterName());
- Map<String, Map<String, Set<String>>> clusterServices = transformValues(
- cluster.getServices(),
- service -> transformValues(service.getServiceComponents(), component
-> component.getServiceComponentsHosts()));
-
// Requested component layout will be added to the StackAdvisor input in
addition to existing
// component layout.
- Map<String, Map<String, Set<String>>> allServices =
mergeDisjunctMaps(clusterServices, info.newServices());
+ Map<String, Map<String, Set<String>>> allServices = getAllServices(info);
Map<String, Set<String>> componentsToHosts =
getComponentHostMap(allServices);
Map<String, Set<String>> hostsToComponents =
getHostComponentMap(componentsToHosts);
- List<String> hosts = ImmutableList.copyOf(cluster.getHostNames());
+ List<String> hosts =
ImmutableList.copyOf(getCluster(info).getHostNames());
hosts.forEach( host -> hostsToComponents.putIfAbsent(host, new
HashSet<>())); // just in case there are hosts that have no components
Map<String, Set<String>> hostGroups =
getHostGroupStrategy().calculateHostGroups(hostsToComponents);
@@ -104,6 +100,7 @@ public class StackAdvisorAdapter {
.forHostComponents(hostsToComponents)
.forHostsGroupBindings(hostGroups)
.withComponentHostsMap(componentsToHosts)
+ .withConfigurations(info.getConfig())
.withGPLLicenseAccepted(serverConfig.getGplLicenseAccepted())
.build();
RecommendationResponse response = stackAdvisorHelper.recommend(request);
@@ -113,47 +110,162 @@ public class StackAdvisorAdapter {
response.getRecommendations().getBlueprint().getHostgroupComponentMap(),
info.getStack()::getServiceForComponent);
- Set<ValidationResponse.ValidationItem> validationItems =
validateRecommendedLayout(info.getStack().getStackId(),
- recommendedLayout,
-
response.getRecommendations().getBlueprintClusterBinding().getHostgroupHostMap());
- if (!validationItems.isEmpty()) {
- LOG.warn("Issues found during recommended topology validation:\n{}",
Joiner.on('\n').join(validationItems));
- }
-
- // Keep the recommendations for new services only
- keepNewServicesOnly(recommendedLayout, info.newServices());
+ // Validate layout
+ Map<String, Set<String>> recommendedComponentHosts =
getComponentHostMap(recommendedLayout);
+ StackAdvisorRequest validationRequest = request.builder()
+
.forHostsGroupBindings(response.getRecommendations().getBlueprintClusterBinding().getHostgroupHostMap())
+ .withComponentHostsMap(recommendedComponentHosts)
+
.forHostComponents(getHostComponentMap(recommendedComponentHosts)).build();
+ validate(validationRequest);
- return info.withNewServices(recommendedLayout);
+ Map<String,Map<String,Set<String>>> newServiceRecommendations =
keepNewServicesOnly(recommendedLayout, info.newServices());
+ LayoutRecommendationInfo recommendationInfo = new
LayoutRecommendationInfo(
+
response.getRecommendations().getBlueprintClusterBinding().getHostgroupHostMap(),
+ recommendedLayout);
+ return info.withLayoutRecommendation(newServiceRecommendations,
recommendationInfo);
}
catch (AmbariException|StackAdvisorException ex) {
throw new IllegalArgumentException("Layout recommendation failed.", ex);
}
}
- Set<ValidationResponse.ValidationItem> validateRecommendedLayout(StackId
stackId,
-
Map<String,Map<String,Set<String>>> recommendedLayout,
- Map<String,
Set<String>> recommendedHostgroups) throws StackAdvisorException {
- Map<String, Set<String>> componentsToHosts =
getComponentHostMap(recommendedLayout);
- Map<String, Set<String>> hostsToComponents =
getHostComponentMap(componentsToHosts);
- List<String> hosts = ImmutableList.copyOf(hostsToComponents.keySet());
-
- StackAdvisorRequest request =
StackAdvisorRequest.StackAdvisorRequestBuilder
- .forStack(stackId)
- .ofType(StackAdvisorRequest.StackAdvisorRequestType.HOST_GROUPS)
- .forHosts(hosts)
- .forServices(recommendedLayout.keySet())
- .forHostComponents(hostsToComponents)
- .forHostsGroupBindings(recommendedHostgroups)
- .withComponentHostsMap(componentsToHosts)
- .withGPLLicenseAccepted(serverConfig.getGplLicenseAccepted())
- .build();
- ValidationResponse response = stackAdvisorHelper.validate(request);
-
- return response.getItems();
+ /**
+ * Gets all services from the cluster together together with services from
AddServiceInfo.
+ * @param info
+ * @return A map of <i>service name -> component name -> hosts</i> for all
services in the cluster and the input
+ * AddServiceInfo combined.
+ */
+ Map<String, Map<String, Set<String>>> getAllServices(AddServiceInfo info)
throws AmbariException {
+ Cluster cluster =
managementController.getClusters().getCluster(info.clusterName());
+ Map<String, Map<String, Set<String>>> clusterServices = transformValues(
+ cluster.getServices(),
+ service -> transformValues(service.getServiceComponents(), component ->
component.getServiceComponentsHosts()));
+
+ return mergeDisjunctMaps(clusterServices, info.newServices());
+ }
+
+ private Cluster getCluster(AddServiceInfo info) throws AmbariException {
+ return managementController.getClusters().getCluster(info.clusterName());
+ }
+
+ AddServiceInfo recommendConfigurations(AddServiceInfo info) {
+ Configuration config = info.getConfig();
+ if (info.getRequest().getRecommendationStrategy().shouldUseStackAdvisor())
{
+ LayoutRecommendationInfo layoutInfo = getLayoutRecommendationInfo(info);
+
+ Map<String, Set<String>> componentHostMap =
getComponentHostMap(layoutInfo.getAllServiceLayouts());
+ Map<String, Set<String>> hostComponentMap =
getHostComponentMap(componentHostMap);
+ StackAdvisorRequest request =
StackAdvisorRequest.StackAdvisorRequestBuilder
+ .forStack(info.getStack().getStackId())
+ .ofType(StackAdvisorRequest.StackAdvisorRequestType.CONFIGURATIONS)
+ .forHosts(layoutInfo.getHosts())
+ .forServices(layoutInfo.getAllServiceLayouts().keySet())
+ .forHostComponents(hostComponentMap)
+ .forHostsGroupBindings(layoutInfo.getHostGroups())
+ .withComponentHostsMap(componentHostMap)
+ .withConfigurations(config)
+ .withGPLLicenseAccepted(serverConfig.getGplLicenseAccepted())
+ .build();
+ RecommendationResponse response;
+ try {
+ response = stackAdvisorHelper.recommend(request);
+ }
+ catch (StackAdvisorException|AmbariException ex) {
+ throw new IllegalArgumentException("Configuration recommendation
failed.", ex);
+ }
+ Map<String, RecommendationResponse.BlueprintConfigurations>
configRecommendations =
response.getRecommendations().getBlueprint().getConfigurations();
+
+ // remove recommendations for existing services
+ configRecommendations.keySet().removeIf(configType ->
!info.newServices().containsKey(info.getStack().getServiceForConfigType(configType)));
+
+ if (info.getRequest().getRecommendationStrategy() ==
ConfigRecommendationStrategy.ONLY_STACK_DEFAULTS_APPLY) {
+
removeNonStackConfigRecommendations(info.getConfig().getParentConfiguration().getParentConfiguration(),
configRecommendations);
+ }
+
+ Configuration recommendedConfig = toConfiguration(configRecommendations);
+
+ Configuration userConfig = config;
+ Configuration clusterAndStackConfig =
userConfig.getParentConfiguration();
+
+ if
(info.getRequest().getRecommendationStrategy().shouldOverrideCustomValues()) {
+ config = recommendedConfig;
+ config.setParentConfiguration(userConfig);
+ }
+ else {
+ config = userConfig;
+ config.setParentConfiguration(recommendedConfig);
+ recommendedConfig.setParentConfiguration(clusterAndStackConfig);
+ }
+
+ StackAdvisorRequest validationRequest =
request.builder().withConfigurations(config).build();
+ validate(validationRequest);
+ }
+
+ UnitUpdater.updateUnits(config, info.getStack());
+ return info.withConfig(config);
+ }
+
+ /**
+ * Reuse information from layout recommendation if it happened
+ */
+ LayoutRecommendationInfo getLayoutRecommendationInfo(AddServiceInfo info) {
+ if (info.getRecommendationInfo().isPresent()) {
+ return info.getRecommendationInfo().get();
+ }
+ try {
+ Map<String, Map<String, Set<String>>> allServices = getAllServices(info);
+ Map<String, Set<String>> hostGroups =
+
getHostGroupStrategy().calculateHostGroups(getHostComponentMap(getComponentHostMap(allServices)));
+ return new LayoutRecommendationInfo(hostGroups, allServices);
+ }
+ catch (AmbariException ex) {
+ throw new IllegalArgumentException("Error gathering host groups and
services", ex);
+ }
+ }
+
+ static void removeNonStackConfigRecommendations(Configuration stackConfig,
Map<String, RecommendationResponse.BlueprintConfigurations>
configRecommendations) {
+ configRecommendations.keySet().removeIf(configType ->
!stackConfig.containsConfigType(configType));
+ configRecommendations.entrySet().forEach( e -> {
+ String cfgType = e.getKey();
+ RecommendationResponse.BlueprintConfigurations cfg = e.getValue();
+ cfg.getProperties().keySet().removeIf(propName ->
!stackConfig.containsConfig(cfgType, propName));
+ if (null != cfg.getPropertyAttributes()) {
+ cfg.getPropertyAttributes().keySet().removeIf(propName ->
!stackConfig.containsConfig(cfgType, propName));
+ }
+ });
+ configRecommendations.values().removeIf(cfg ->
cfg.getProperties().isEmpty() && cfg.getPropertyAttributes().isEmpty());
}
- static void keepNewServicesOnly(Map<String,Map<String,Set<String>>>
recommendedLayout, Map<String,Map<String,Set<String>>> newServices) {
- recommendedLayout.keySet().retainAll(newServices.keySet());
+ private void validate(StackAdvisorRequest request) {
+ try {
+ Set<ValidationResponse.ValidationItem> items =
stackAdvisorHelper.validate(request).getItems();
+ if (!items.isEmpty()) {
+ LOG.warn("Issues found during recommended {} validation:\n{}",
request.getRequestType(), Joiner.on('\n').join(items));
+ }
+ }
+ catch (StackAdvisorException ex) {
+ LOG.error(request.getRequestType() + " validation failed", ex);
+ }
+ }
+
+ static Configuration toConfiguration(Map<String,
RecommendationResponse.BlueprintConfigurations> configs) {
+ Map<String, Map<String, String>> properties = configs.entrySet().stream()
+ .filter( e -> e.getValue().getProperties() != null &&
!e.getValue().getProperties().isEmpty())
+ .map(e -> Pair.of(e.getKey(), e.getValue().getProperties()))
+ .collect(toMap(Pair::getKey, Pair::getValue));
+
+ Map<String, Map<String, Map<String, String>>> propertyAttributes =
configs.entrySet().stream()
+ .filter( e -> e.getValue().getPropertyAttributes() != null &&
!e.getValue().getPropertyAttributes().isEmpty())
+ .map(e -> Pair.of(e.getKey(),
e.getValue().getPropertyAttributesAsMap()))
+ .collect(toMap(Pair::getKey, Pair::getValue));
+
+ return new Configuration(properties, propertyAttributes);
+ }
+
+ static Map<String,Map<String,Set<String>>>
keepNewServicesOnly(Map<String,Map<String,Set<String>>> recommendedLayout,
Map<String,Map<String,Set<String>>> newServices) {
+ HashMap<String, java.util.Map<String, Set<String>>>
newServiceRecommendations = new HashMap<>(recommendedLayout);
+ newServiceRecommendations.keySet().retainAll(newServices.keySet());
+ return newServiceRecommendations;
}
static Map<String, Map<String, Set<String>>>
getRecommendedLayout(Map<String, Set<String>> hostGroupHosts,
@@ -170,6 +282,7 @@ public class StackAdvisorAdapter {
toMap(Map.Entry::getKey, Map.Entry::getValue)));
}
+
/**
* Transform a map of component -> hosts to a map of hosts -> components
* @param componentHostMap the map to transform
diff --git
a/ambari-server/src/test/java/org/apache/ambari/server/controller/AddServiceRequestTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/controller/AddServiceRequestTest.java
index 2d97bd0..4b90374 100644
---
a/ambari-server/src/test/java/org/apache/ambari/server/controller/AddServiceRequestTest.java
+++
b/ambari-server/src/test/java/org/apache/ambari/server/controller/AddServiceRequestTest.java
@@ -26,7 +26,6 @@ import static
org.apache.ambari.server.controller.AddServiceRequest.ValidationTy
import static
org.apache.ambari.server.controller.internal.ProvisionAction.INSTALL_AND_START;
import static
org.apache.ambari.server.controller.internal.ProvisionAction.INSTALL_ONLY;
import static
org.apache.ambari.server.topology.ConfigRecommendationStrategy.ALWAYS_APPLY;
-import static
org.apache.ambari.server.topology.ConfigRecommendationStrategy.NEVER_APPLY;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
@@ -40,6 +39,7 @@ import java.util.Optional;
import org.apache.ambari.server.security.encryption.CredentialStoreType;
import org.apache.ambari.server.state.SecurityType;
+import org.apache.ambari.server.topology.ConfigRecommendationStrategy;
import org.apache.ambari.server.topology.Configuration;
import org.apache.ambari.server.topology.Credential;
import org.apache.ambari.server.topology.SecurityConfiguration;
@@ -135,7 +135,7 @@ public class AddServiceRequestTest {
// default / empty values
assertEquals(ADD_SERVICE, request.getOperationType());
- assertEquals(NEVER_APPLY, request.getRecommendationStrategy());
+ assertEquals(ConfigRecommendationStrategy.defaultForAddService(),
request.getRecommendationStrategy());
assertEquals(INSTALL_AND_START, request.getProvisionAction());
assertEquals(STRICT, request.getValidationType());
assertNull(request.getStackName());
@@ -159,7 +159,7 @@ public class AddServiceRequestTest {
// default / empty values
assertEquals(ADD_SERVICE, request.getOperationType());
- assertEquals(NEVER_APPLY, request.getRecommendationStrategy());
+ assertEquals(ConfigRecommendationStrategy.defaultForAddService(),
request.getRecommendationStrategy());
assertEquals(INSTALL_AND_START, request.getProvisionAction());
assertNull(request.getStackName());
assertNull(request.getStackVersion());
@@ -182,7 +182,7 @@ public class AddServiceRequestTest {
// default / empty values
assertEquals(ADD_SERVICE, request.getOperationType());
- assertEquals(NEVER_APPLY, request.getRecommendationStrategy());
+ assertEquals(ConfigRecommendationStrategy.defaultForAddService(),
request.getRecommendationStrategy());
assertEquals(INSTALL_AND_START, request.getProvisionAction());
assertNull(request.getStackName());
assertNull(request.getStackVersion());
diff --git
a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UnitUpdaterTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UnitUpdaterTest.java
index 6de6cd1..fbf448d 100644
---
a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UnitUpdaterTest.java
+++
b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UnitUpdaterTest.java
@@ -17,6 +17,7 @@
*/
package org.apache.ambari.server.controller.internal;
+import static org.apache.ambari.server.testutils.TestCollectionUtils.map;
import static org.easymock.EasyMock.expect;
import static org.junit.Assert.assertEquals;
@@ -28,7 +29,7 @@ import
org.apache.ambari.server.controller.StackConfigurationResponse;
import org.apache.ambari.server.state.ValueAttributesInfo;
import org.apache.ambari.server.topology.Blueprint;
import org.apache.ambari.server.topology.ClusterTopology;
-import org.apache.ambari.server.topology.InvalidTopologyException;
+import org.apache.ambari.server.topology.Configuration;
import org.easymock.EasyMockRule;
import org.easymock.EasyMockSupport;
import org.easymock.Mock;
@@ -86,10 +87,54 @@ public class UnitUpdaterTest extends EasyMockSupport {
updateUnit(OOZIE, OOZIE_ENV, HEAPSIZE, "");
}
+ @Test
+ public void updateUnits() {
+ stackUnitIs(HEAPSIZE, "MB");
+ setUpStack(OOZIE, OOZIE_ENV);
+
+ Map<String, Map<String, String>> properties = map(
+ OOZIE_ENV, map(HEAPSIZE, "1024"),
+ "core-site", map("fs.trash.interval", "360"));
+ Configuration configuration = new Configuration(properties, new
HashMap<>());
+
+ UnitUpdater.updateUnits(configuration, stack);
+
+ Map<String, Map<String, String>> expected = map(
+ OOZIE_ENV, map(HEAPSIZE, "1024m"),
+ "core-site", map("fs.trash.interval", "360"));
+
+ assertEquals(expected, configuration.getProperties());
+ }
+
+ @Test
+ public void removeUnits() {
+ stackUnitIs(HEAPSIZE, "MB");
+ setUpStack(OOZIE, OOZIE_ENV);
+
+ Map<String, Map<String, String>> properties = map(
+ OOZIE_ENV, map(HEAPSIZE, "1024m"),
+ "core-site", map("fs.trash.interval", "360"));
+ Configuration configuration = new Configuration(properties, new
HashMap<>());
+
+ UnitUpdater.removeUnits(configuration, stack);
+
+ Map<String, Map<String, String>> expected = map(
+ OOZIE_ENV, map(HEAPSIZE, "1024"),
+ "core-site", map("fs.trash.interval", "360"));
+
+ assertEquals(expected, configuration.getProperties());
+ }
+
private void stackUnitIs(String name, String unit) {
ValueAttributesInfo propertyValueAttributes = new ValueAttributesInfo();
propertyValueAttributes.setUnit(unit);
- stackConfigWithMetadata.put(name, new Stack.ConfigProperty(new
StackConfigurationResponse(
+ stackConfigWithMetadata.put(name, configProperty(name, unit));
+ }
+
+ public static Stack.ConfigProperty configProperty(String name, String unit) {
+ ValueAttributesInfo propertyValueAttributes = new ValueAttributesInfo();
+ propertyValueAttributes.setUnit(unit);
+ return new Stack.ConfigProperty(new StackConfigurationResponse(
name,
"any",
"any",
@@ -99,16 +144,20 @@ public class UnitUpdaterTest extends EasyMockSupport {
Collections.emptySet(),
Collections.emptyMap(),
propertyValueAttributes,
- Collections.emptySet()
- )));
+ Collections.emptySet()));
}
- private String updateUnit(String serviceName, String configType, String
propName, String propValue) throws InvalidTopologyException,
ConfigurationTopologyException {
- UnitUpdater updater = new UnitUpdater(serviceName, configType);
+ private void setUpStack(String serviceName, String configType) {
expect(clusterTopology.getBlueprint()).andReturn(blueprint).anyTimes();
expect(blueprint.getStack()).andReturn(stack).anyTimes();
expect(stack.getConfigurationPropertiesWithMetadata(serviceName,
configType)).andReturn(stackConfigWithMetadata).anyTimes();
replayAll();
+ }
+
+ private String updateUnit(String serviceName, String configType, String
propName, String propValue) {
+ UnitUpdater updater = new UnitUpdater(serviceName, configType);
+ setUpStack(serviceName, configType);
return updater.updateForClusterCreate(propName, propValue,
Collections.emptyMap(), clusterTopology);
}
+
}
\ No newline at end of file
diff --git
a/ambari-server/src/test/java/org/apache/ambari/server/testutils/TestCollectionUtils.java
b/ambari-server/src/test/java/org/apache/ambari/server/testutils/TestCollectionUtils.java
new file mode 100644
index 0000000..0a7c5ac
--- /dev/null
+++
b/ambari-server/src/test/java/org/apache/ambari/server/testutils/TestCollectionUtils.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.testutils;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+/**
+ * Utilities for collections used in unit tests.
+ */
+public class TestCollectionUtils {
+
+ /**
+ * A simple (but not production quality) way to create mutable hashmaps for
unit tests
+ * @param firstKey the first key in the map
+ * @param firstValue the first value in the map
+ * @param others further keys and values
+ * @param <K> key type
+ * @param <V> value type
+ * @return the map
+ */
+ @SuppressWarnings("unchecked")
+ public static <K, V> Map<K, V> map(K firstKey, V firstValue, Object...
others) {
+ Map<K, V> map = new HashMap<>();
+ map.put(firstKey, firstValue);
+ Iterator iterator = Arrays.asList(others).iterator();
+ while (iterator.hasNext()) {
+ map.put((K)iterator.next(), (V)iterator.next());
+ }
+ return map;
+ }
+
+}
diff --git
a/ambari-server/src/test/java/org/apache/ambari/server/topology/ConfigurableTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/topology/ConfigurableTest.java
index 752e3af..cc4c33b 100644
---
a/ambari-server/src/test/java/org/apache/ambari/server/topology/ConfigurableTest.java
+++
b/ambari-server/src/test/java/org/apache/ambari/server/topology/ConfigurableTest.java
@@ -25,6 +25,7 @@ import static org.junit.Assert.fail;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.List;
+import java.util.Map;
import java.util.regex.Pattern;
import org.apache.commons.lang3.exception.ExceptionUtils;
@@ -135,6 +136,21 @@ public class ConfigurableTest {
configurable.getConfiguration().getProperties());
}
+ @Test
+ public void testTransformAttributesMap() {
+ Map<String, Map<String, String>> attributes = ImmutableMap.of(
+ "propertyName1", ImmutableMap.of("minimum", "3000", "maximum", "4000"),
+ "propertyName2", ImmutableMap.of("minimum", "3500", "maximum", "4500",
"hidden", "true"));
+
+ Map<String, Map<String, String>> transformed = ImmutableMap.of(
+ "minimum", ImmutableMap.of("propertyName1", "3000", "propertyName2",
"3500"),
+ "maximum", ImmutableMap.of("propertyName1", "4000", "propertyName2",
"4500"),
+ "hidden", ImmutableMap.of("propertyName2", "true"));
+
+ assertEquals(transformed,
ConfigurableHelper.transformAttributesMap(attributes));
+ assertEquals(attributes,
ConfigurableHelper.transformAttributesMap(transformed));
+ }
+
static class TestConfigurable implements Configurable {
Configuration configuration;
diff --git
a/ambari-server/src/test/java/org/apache/ambari/server/topology/ConfigurationTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/topology/ConfigurationTest.java
index 23b7e81..51b7193 100644
---
a/ambari-server/src/test/java/org/apache/ambari/server/topology/ConfigurationTest.java
+++
b/ambari-server/src/test/java/org/apache/ambari/server/topology/ConfigurationTest.java
@@ -158,6 +158,55 @@ public class ConfigurationTest {
}
@Test
+ public void containsConfigType() {
+ Configuration configuration = createConfigurationWithParents_PropsOnly();
+ assertTrue(configuration.containsConfigType("type1"));
+ assertTrue(configuration.containsConfigType("type2"));
+ assertTrue(configuration.containsConfigType("type3"));
+ assertTrue(configuration.containsConfigType("type4"));
+ assertFalse(configuration.containsConfigType("type5"));
+
+ configuration = createConfigurationWithParents_AttributesOnly();
+ assertTrue(configuration.containsConfigType("type1"));
+ assertTrue(configuration.containsConfigType("type2"));
+ assertFalse(configuration.containsConfigType("type3"));
+ }
+
+ @Test
+ public void containsConfig() {
+ Configuration configuration = createConfigurationWithParents_PropsOnly();
+ assertTrue(configuration.containsConfig("type1", "prop1"));
+ assertTrue(configuration.containsConfig("type1", "prop2"));
+ assertTrue(configuration.containsConfig("type1", "prop3"));
+ assertTrue(configuration.containsConfig("type2", "prop4"));
+ assertTrue(configuration.containsConfig("type2", "prop5"));
+ assertTrue(configuration.containsConfig("type1", "prop6"));
+ assertTrue(configuration.containsConfig("type1", "prop9"));
+ assertTrue(configuration.containsConfig("type3", "prop7"));
+ assertTrue(configuration.containsConfig("type3", "prop8"));
+ assertTrue(configuration.containsConfig("type4", "prop10"));
+ assertTrue(configuration.containsConfig("type4", "prop11"));
+ assertFalse(configuration.containsConfig("type1", "prop99"));
+ assertFalse(configuration.containsConfig("core-site",
"io.file.buffer.size"));
+
+ configuration = createConfigurationWithParents_AttributesOnly();
+ assertTrue(configuration.containsConfig("type1", "prop1"));
+ assertTrue(configuration.containsConfig("type1", "prop2"));
+ assertTrue(configuration.containsConfig("type1", "prop3"));
+ assertTrue(configuration.containsConfig("type1", "prop6"));
+ assertTrue(configuration.containsConfig("type1", "prop7"));
+ assertTrue(configuration.containsConfig("type1", "prop8"));
+ assertTrue(configuration.containsConfig("type1", "prop9"));
+ assertTrue(configuration.containsConfig("type1", "prop10"));
+ assertTrue(configuration.containsConfig("type1", "prop11"));
+ assertTrue(configuration.containsConfig("type2", "prop100"));
+ assertTrue(configuration.containsConfig("type2", "prop101"));
+ assertTrue(configuration.containsConfig("type2", "prop102"));
+ assertFalse(configuration.containsConfig("type1", "prop99"));
+ assertFalse(configuration.containsConfig("core-site",
"io.file.buffer.size"));
+ }
+
+ @Test
public void testGetFullProperties_withParent_specifyDepth() {
Configuration configuration = createConfigurationWithParents_PropsOnly();
// specify a depth of 1 which means to include only 1 level up the parent
chain
diff --git
a/ambari-server/src/test/java/org/apache/ambari/server/topology/addservice/RequestValidatorTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/topology/addservice/RequestValidatorTest.java
index e9c3ff7..13fc281 100644
---
a/ambari-server/src/test/java/org/apache/ambari/server/topology/addservice/RequestValidatorTest.java
+++
b/ambari-server/src/test/java/org/apache/ambari/server/topology/addservice/RequestValidatorTest.java
@@ -48,6 +48,7 @@ import org.apache.ambari.server.state.StackId;
import org.apache.ambari.server.state.kerberos.KerberosDescriptor;
import org.apache.ambari.server.state.kerberos.KerberosDescriptorFactory;
import org.apache.ambari.server.state.kerberos.KerberosServiceDescriptor;
+import org.apache.ambari.server.topology.ConfigRecommendationStrategy;
import org.apache.ambari.server.topology.Configuration;
import org.apache.ambari.server.topology.SecurityConfiguration;
import org.apache.ambari.server.topology.SecurityConfigurationFactory;
@@ -467,6 +468,7 @@ public class RequestValidatorTest extends EasyMockSupport {
requestConfig.setProperty("kafka-broker", "zookeeper.connect",
"zookeeper.connect:request");
requestConfig.setProperty("kafka-env", "custom_property",
"custom_property:request");
expect(request.getConfiguration()).andReturn(requestConfig.copy()).anyTimes();
+
expect(request.getRecommendationStrategy()).andReturn(ConfigRecommendationStrategy.ALWAYS_APPLY).anyTimes();
Configuration clusterConfig = Configuration.newEmpty();
clusterConfig.setProperty("zookeeper-env", "zk_user",
"zk_user:cluster_level");
@@ -477,7 +479,7 @@ public class RequestValidatorTest extends EasyMockSupport {
stackConfig.setProperty("zookeeper-env", "zk_user",
"zk_user:stack_default");
stackConfig.setProperty("zookeeper-env", "zk_log_dir",
"zk_log_dir:stack_default");
stackConfig.setProperty("kafka-broker", "zookeeper.connect",
"zookeeper.connect:stack_default");
- expect(stack.getValidDefaultConfig()).andReturn(stackConfig).anyTimes();
+ expect(stack.getDefaultConfig()).andReturn(stackConfig).anyTimes();
replayAll();
diff --git
a/ambari-server/src/test/java/org/apache/ambari/server/topology/addservice/StackAdvisorAdapterTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/topology/addservice/StackAdvisorAdapterTest.java
index abd570a..78f481e 100644
---
a/ambari-server/src/test/java/org/apache/ambari/server/topology/addservice/StackAdvisorAdapterTest.java
+++
b/ambari-server/src/test/java/org/apache/ambari/server/topology/addservice/StackAdvisorAdapterTest.java
@@ -21,6 +21,9 @@ package org.apache.ambari.server.topology.addservice;
import static java.util.Collections.emptyMap;
import static java.util.Collections.emptySet;
import static java.util.stream.Collectors.toMap;
+import static
org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest.StackAdvisorRequestType.HOST_GROUPS;
+import static
org.apache.ambari.server.controller.internal.UnitUpdaterTest.configProperty;
+import static org.apache.ambari.server.testutils.TestCollectionUtils.map;
import static org.easymock.EasyMock.anyObject;
import static org.easymock.EasyMock.anyString;
import static org.easymock.EasyMock.expect;
@@ -28,15 +31,18 @@ import static org.easymock.EasyMock.getCurrentArguments;
import static org.easymock.EasyMock.mock;
import static org.easymock.EasyMock.replay;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertSame;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorHelper;
+import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest;
import
org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse;
import
org.apache.ambari.server.api.services.stackadvisor.validations.ValidationResponse;
-import org.apache.ambari.server.configuration.Configuration;
+import org.apache.ambari.server.controller.AddServiceRequest;
import org.apache.ambari.server.controller.AmbariManagementController;
import org.apache.ambari.server.controller.internal.Stack;
import org.apache.ambari.server.state.Cluster;
@@ -44,8 +50,11 @@ import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.state.Service;
import org.apache.ambari.server.state.ServiceComponent;
import org.apache.ambari.server.state.StackId;
+import org.apache.ambari.server.topology.ConfigRecommendationStrategy;
+import org.apache.ambari.server.topology.Configuration;
import org.apache.commons.lang3.tuple.Pair;
import org.easymock.EasyMockRunner;
+import org.easymock.IExpectationSetters;
import org.easymock.Mock;
import org.easymock.TestSubject;
import org.junit.Before;
@@ -66,7 +75,7 @@ public class StackAdvisorAdapterTest {
private StackAdvisorHelper stackAdvisorHelper;
@Mock
- private Configuration serverConfig;
+ private org.apache.ambari.server.configuration.Configuration serverConfig;
@Mock
private Injector injector;
@@ -158,20 +167,67 @@ public class StackAdvisorAdapterTest {
}
@Test
+ public void getLayoutRecommendationInfo() {
+ Map<String, Map<String, Set<String>>> newServices = ImmutableMap.of(
+ "KAFKA", ImmutableMap.of(
+ "KAFKA_BROKER", ImmutableSet.of("c7401")),
+ "SPARK2", ImmutableMap.of(
+ "SPARK2_JOBHISTORYSERVER", ImmutableSet.of("c7402"),
+ "SPARK2_CLIENT", ImmutableSet.of("c7403", "c7404")),
+ "OOZIE", ImmutableMap.of(
+ "OOZIE_SERVER", ImmutableSet.of("c7401"),
+ "OOZIE_CLIENT", ImmutableSet.of("c7403", "c7404")));
+
+ AddServiceInfo info = new
AddServiceInfo(request(ConfigRecommendationStrategy.ALWAYS_APPLY), "c1", stack
, Configuration.newEmpty(),
+ null, null, newServices, null); // No LayoutReommendationInfo -> needs
to be calculated
+
+ LayoutRecommendationInfo layoutRecommendationInfo =
adapter.getLayoutRecommendationInfo(info);
+ layoutRecommendationInfo.getAllServiceLayouts();
+
+ assertEquals(
+ ImmutableMap.of(
+ "host_group_1", ImmutableSet.of("c7401"),
+ "host_group_2", ImmutableSet.of("c7402"),
+ "host_group_3", ImmutableSet.of("c7403", "c7404")),
+ layoutRecommendationInfo.getHostGroups());
+
+ assertEquals(
+ ImmutableMap.<String, Map<String, Set<String>>>builder()
+ .put("KAFKA", ImmutableMap.of(
+ "KAFKA_BROKER", ImmutableSet.of("c7401")))
+ .put("SPARK2", ImmutableMap.of(
+ "SPARK2_JOBHISTORYSERVER", ImmutableSet.of("c7402"),
+ "SPARK2_CLIENT", ImmutableSet.of("c7403", "c7404")))
+ .put("OOZIE", ImmutableMap.of(
+ "OOZIE_SERVER", ImmutableSet.of("c7401"),
+ "OOZIE_CLIENT", ImmutableSet.of("c7403", "c7404")))
+ .put("HDFS", ImmutableMap.of(
+ "NAMENODE", ImmutableSet.of("c7401"),
+ "HDFS_CLIENT", ImmutableSet.of("c7401", "c7402")))
+ .put("ZOOKEEPER", ImmutableMap.of(
+ "ZOOKEEPER_SERVER", ImmutableSet.of("c7401"),
+ "ZOOKEEPER_CLIENT", ImmutableSet.of("c7401", "c7402")))
+ .put("MAPREDUCE2", ImmutableMap.of(
+ "HISTORYSERVER", ImmutableSet.of("c7401")))
+ .build(),
+ layoutRecommendationInfo.getAllServiceLayouts());
+ }
+
+ @Test
public void keepNewServicesOnly() {
Map<String, Map<String, Set<String>>> newServices = ImmutableMap.of(
"KAFKA", emptyMap(),
"PIG", emptyMap());
- Map<String, Map<String, Set<String>>> recommendationForNewServices =
ImmutableMap.of(
+ Map<String, Map<String, Set<String>>> expectedNewServiceRecommendations =
ImmutableMap.of(
"KAFKA", ImmutableMap.of("KAFKA_BROKER", ImmutableSet.of("c7405")),
"PIG", ImmutableMap.of("PIG_CLIENT", ImmutableSet.of("c7405", "c7406")));
Map<String, Map<String, Set<String>>> recommendations = new
HashMap<>(SERVICE_COMPONENT_HOST_MAP_1);
- recommendations.putAll(recommendationForNewServices);
+ recommendations.putAll(expectedNewServiceRecommendations);
- StackAdvisorAdapter.keepNewServicesOnly(recommendations, newServices);
- assertEquals(recommendationForNewServices, recommendations);
+ Map<String, Map<String, Set<String>>> newServiceRecommendations =
StackAdvisorAdapter.keepNewServicesOnly(recommendations, newServices);
+ assertEquals(expectedNewServiceRecommendations, newServiceRecommendations);
}
@Before
@@ -182,39 +238,71 @@ public class StackAdvisorAdapterTest {
"HDFS",
service("HDFS", ImmutableMap.of("NAMENODE", ImmutableSet.of("c7401"),
"HDFS_CLIENT", ImmutableSet.of("c7401", "c7402"))),
"ZOOKEEPER",
- service("ZOOKEEPER", ImmutableMap.of("ZOOKEEPER_SERVER",
ImmutableSet.of("c7401"), "ZOOKEEPER_CLIENT", ImmutableSet.of("c7401",
"c7402")))));
+ service("ZOOKEEPER", ImmutableMap.of("ZOOKEEPER_SERVER",
ImmutableSet.of("c7401"), "ZOOKEEPER_CLIENT", ImmutableSet.of("c7401",
"c7402"))),
+ "MAPREDUCE2",
+ service("MAPREDUCE2", ImmutableMap.of("HISTORYSERVER",
ImmutableSet.of("c7401")))));
Clusters clusters = mock(Clusters.class);
expect(clusters.getCluster(anyString())).andReturn(cluster).anyTimes();
expect(managementController.getClusters()).andReturn(clusters).anyTimes();
replay(clusters, cluster, managementController);
expect(serverConfig.getGplLicenseAccepted()).andReturn(Boolean.FALSE).anyTimes();
-
expect(serverConfig.getAddServiceHostGroupStrategyClass()).andReturn((Class)GroupByComponentsStrategy.class).anyTimes();
+ @SuppressWarnings("unchecked")
+ IExpectationSetters iExpectationSetters =
expect(serverConfig.getAddServiceHostGroupStrategyClass()).andReturn((Class)
GroupByComponentsStrategy.class).anyTimes();
replay(serverConfig);
expect(injector.getInstance(GroupByComponentsStrategy.class)).andReturn(new
GroupByComponentsStrategy()).anyTimes();
replay(injector);
- RecommendationResponse response = new RecommendationResponse();
- RecommendationResponse.Recommendation recommendation = new
RecommendationResponse.Recommendation();
- response.setRecommendations(recommendation);
RecommendationResponse.BlueprintClusterBinding binding =
RecommendationResponse.BlueprintClusterBinding.fromHostGroupHostMap(
ImmutableMap.of(
"hostgroup-1", ImmutableSet.of("c7401"),
"hostgroup-2", ImmutableSet.of("c7402")));
- recommendation.setBlueprintClusterBinding(binding);
RecommendationResponse.Blueprint blueprint = new
RecommendationResponse.Blueprint();
blueprint.setHostGroups(RecommendationResponse.HostGroup.fromHostGroupComponents(
ImmutableMap.of(
- "hostgroup-1", ImmutableSet.of("NAMENODE", "HDFS_CLIENT",
"ZOOKEEPER_SERVER", "ZOOKEEPER_CLIENT"),
+ "hostgroup-1", ImmutableSet.of("NAMENODE", "HDFS_CLIENT",
"ZOOKEEPER_SERVER", "ZOOKEEPER_CLIENT", "HISTORYSERVER"),
"hostgroup-2", ImmutableSet.of("HDFS_CLIENT", "ZOOKEEPER_CLIENT",
"KAFKA_BROKER"))
));
- recommendation.setBlueprint(blueprint);
- expect(stackAdvisorHelper.recommend(anyObject())).andReturn(response);
+ RecommendationResponse layoutResponse = createRecommendation(blueprint,
binding);
+
+ RecommendationResponse.Blueprint configBlueprint = new
RecommendationResponse.Blueprint();
+ RecommendationResponse.BlueprintConfigurations kafkaBroker =
RecommendationResponse.BlueprintConfigurations.create(
+ ImmutableMap.of("log.dirs", "/kafka-logs",
"offsets.topic.replication.factor", "1"),
+ ImmutableMap.of("maximum",
ImmutableMap.of("offsets.topic.replication.factor", "10")));
+ RecommendationResponse.BlueprintConfigurations spark2Defaults =
RecommendationResponse.BlueprintConfigurations.create(
+ ImmutableMap.of("spark.yarn.queue", "default"), null);
+ RecommendationResponse.BlueprintConfigurations mapredSite =
RecommendationResponse.BlueprintConfigurations.create(
+ ImmutableMap.of("mapreduce.map.memory.mb", "682",
"mapreduce.reduce.memory.mb", "1364"),
+ ImmutableMap.of(
+ "minimum", ImmutableMap.of("mapreduce.map.memory.mb", "682",
"mapreduce.reduce.memory.mb", "682"),
+ "maximum" , ImmutableMap.of("mapreduce.map.memory.mb", "2046",
"mapreduce.reduce.memory.mb", "2046")));
+ configBlueprint.setConfigurations(map(
+ "kafka-broker", kafkaBroker,
+ "spark2-defaults", spark2Defaults,
+ "mapred-site", mapredSite
+ ));
+ RecommendationResponse configResponse =
createRecommendation(configBlueprint, binding);
+
+ expect(stackAdvisorHelper.recommend(anyObject())).andAnswer(() -> {
+ StackAdvisorRequest request = (StackAdvisorRequest)
getCurrentArguments()[0];
+ assertNotNull(request.getHosts());
+ assertNotNull(request.getServices());
+ assertNotNull(request.getStackName());
+ assertNotNull(request.getStackVersion());
+ assertNotNull(request.getConfigurations());
+ assertNotNull(request.getHostComponents());
+ assertNotNull(request.getComponentHostsMap());
+ assertNotNull(request.getHostGroupBindings());
+ assertNotNull(request.getLdapConfig());
+ assertNotNull(request.getRequestType());
+ return request.getRequestType() == HOST_GROUPS ? layoutResponse :
configResponse;
+ });
ValidationResponse validationResponse = new ValidationResponse();
validationResponse.setItems(emptySet());
expect(stackAdvisorHelper.validate(anyObject())).andReturn(validationResponse);
+
replay(stackAdvisorHelper);
expect(stack.getStackId()).andReturn(new StackId("HDP", "3.0")).anyTimes();
@@ -224,11 +312,32 @@ public class StackAdvisorAdapterTest {
.put("HDFS_CLIENT", "HDFS")
.put("ZOOKEEPER_SERVER", "ZOOKEEPER")
.put("ZOOKEEPER_CLIENT", "ZOOKEEPER")
+ .put("HISTORYSERVER", "MAPREDUCE2")
.build();
expect(stack.getServiceForComponent(anyString())).andAnswer(() ->
serviceComponentMap.get(getCurrentArguments()[0])).anyTimes();
+ ImmutableMap<String, String> configTypeServiceMap = ImmutableMap.<String,
String>builder()
+ .put("kafka-broker", "KAFKA")
+ .put("spark2-defaults", "SPARK2")
+ .put("mapred-site", "MAPREDUCE2")
+ .build();
+ expect(stack.getServiceForConfigType(anyString())).andAnswer(() ->
configTypeServiceMap.get(getCurrentArguments()[0])).anyTimes();
+ expect(stack.getConfigurationPropertiesWithMetadata("OOZIE",
"oozie-env")).andReturn(
+ ImmutableMap.of(
+ "mapreduce.map.memory.mb", configProperty("mapreduce.map.memory.mb",
"MB"),
+ "mapreduce.reduce.memory.mb",
configProperty("mapreduce.reduce.memory.mb", "MB"))).anyTimes();
replay(stack);
}
+ private static RecommendationResponse
createRecommendation(RecommendationResponse.Blueprint blueprint,
+
RecommendationResponse.BlueprintClusterBinding binding) {
+ RecommendationResponse response = new RecommendationResponse();
+ RecommendationResponse.Recommendation recommendation = new
RecommendationResponse.Recommendation();
+ response.setRecommendations(recommendation);
+ recommendation.setBlueprint(blueprint);
+ recommendation.setBlueprintClusterBinding(binding);
+ return response;
+ }
+
private static Service service(String name,
ImmutableMap<String,ImmutableSet<String>> componentHostMap) {
Service service = mock(Service.class);
expect(service.getName()).andReturn(name).anyTimes();
@@ -252,7 +361,7 @@ public class StackAdvisorAdapterTest {
"KAFKA",
ImmutableMap.of("KAFKA_BROKER", emptySet()));
- AddServiceInfo info = new AddServiceInfo(null, "c1", stack,
org.apache.ambari.server.topology.Configuration.newEmpty(), null, null,
newServices);
+ AddServiceInfo info = new AddServiceInfo(null, "c1", stack,
org.apache.ambari.server.topology.Configuration.newEmpty(), null, null,
newServices, null);
AddServiceInfo infoWithRecommendations = adapter.recommendLayout(info);
Map<String, Map<String, Set<String>>> expectedNewLayout = ImmutableMap.of(
@@ -263,13 +372,292 @@ public class StackAdvisorAdapterTest {
assertEquals(expectedNewLayout, infoWithRecommendations.newServices());
}
+ @Test
+ public void recommendConfigurations_noLayoutInfo() {
+ Map<String, Map<String, Set<String>>> newServices = ImmutableMap.of(
+ "KAFKA", ImmutableMap.of(
+ "KAFKA_BROKER", ImmutableSet.of("c7401")),
+ "SPARK2", ImmutableMap.of(
+ "SPARK2_JOBHISTORYSERVER", ImmutableSet.of("c7402"),
+ "SPARK2_CLIENT", ImmutableSet.of("c7403", "c7404")),
+ "OOZIE", ImmutableMap.of(
+ "OOZIE_SERVER", ImmutableSet.of("c7401"),
+ "OOZIE_CLIENT", ImmutableSet.of("c7403", "c7404")));
+
+ Configuration stackConfig = Configuration.newEmpty();
+ Configuration clusterConfig = new Configuration(
+ map("oozie-env", map("oozie_heapsize", "1024", "oozie_permsize", "256")),
+ emptyMap());
+ Configuration userConfig = Configuration.newEmpty();
+ userConfig.setParentConfiguration(clusterConfig);
+ clusterConfig.setParentConfiguration(stackConfig);
+
+ AddServiceInfo info = new
AddServiceInfo(request(ConfigRecommendationStrategy.ALWAYS_APPLY), "c1", stack
, userConfig,
+ null, null, newServices, null); // No LayoutRecommendationInfo
+ AddServiceInfo infoWithConfig = adapter.recommendConfigurations(info);
+
+ Configuration recommendedConfig = infoWithConfig.getConfig();
+ assertSame(userConfig, recommendedConfig.getParentConfiguration());
+ assertSame(clusterConfig, userConfig.getParentConfiguration());
+ assertSame(stackConfig, clusterConfig.getParentConfiguration());
+
+ // Yarn/Mapred config is excpected to be removed as it does not belong to
newly added services
+ assertEquals(
+ ImmutableMap.of(
+ "kafka-broker", ImmutableMap.of(
+ "log.dirs", "/kafka-logs",
+ "offsets.topic.replication.factor", "1"),
+ "spark2-defaults", ImmutableMap.of(
+ "spark.yarn.queue", "default"),
+ "oozie-env", ImmutableMap.of(
+ "oozie_heapsize", "1024m", // unit updates should happen
+ "oozie_permsize", "256m")),
+ recommendedConfig.getProperties());
+
+ assertEquals(
+ ImmutableMap.of(
+ "kafka-broker", ImmutableMap.of(
+ "maximum", ImmutableMap.of("offsets.topic.replication.factor",
"10"))),
+ recommendedConfig.getAttributes());
+ }
- private static Map<String, Map<String, Set<String>>> mutableCopy(Map<String,
Map<String, Set<String>>> map) {
- Map<String, Map<String, Set<String>>> copy = new HashMap<>();
- map.entrySet().forEach( outer -> {
- Map<String, Set<String>> innerCopy = new HashMap<>(outer.getValue());
- copy.put(outer.getKey(), innerCopy);
- });
- return copy;
+ @Test
+ public void recommendConfigurations_alwaysApply() {
+ Map<String, Map<String, Set<String>>> newServices = ImmutableMap.of(
+ "KAFKA", ImmutableMap.of(
+ "KAFKA_BROKER", ImmutableSet.of("c7401")),
+ "SPARK2", ImmutableMap.of(
+ "SPARK2_JOBHISTORYSERVER", ImmutableSet.of("c7402"),
+ "SPARK2_CLIENT", ImmutableSet.of("c7403", "c7404")),
+ "OOZIE", ImmutableMap.of(
+ "OOZIE_SERVER", ImmutableSet.of("c7401"),
+ "OOZIE_CLIENT", ImmutableSet.of("c7403", "c7404")));
+
+ Configuration stackConfig = Configuration.newEmpty();
+ Configuration clusterConfig = new Configuration(
+ map("oozie-env", map("oozie_heapsize", "1024", "oozie_permsize", "256")),
+ emptyMap());
+ Configuration userConfig = Configuration.newEmpty();
+ userConfig.setParentConfiguration(clusterConfig);
+ clusterConfig.setParentConfiguration(stackConfig);
+
+ LayoutRecommendationInfo layoutRecommendationInfo = new
LayoutRecommendationInfo(new HashMap<>(), new HashMap<>()); // contents doesn't
matter for the test
+ AddServiceInfo info = new
AddServiceInfo(request(ConfigRecommendationStrategy.ALWAYS_APPLY), "c1", stack
, userConfig,
+ null, null, newServices, layoutRecommendationInfo);
+ AddServiceInfo infoWithConfig = adapter.recommendConfigurations(info);
+
+ Configuration recommendedConfig = infoWithConfig.getConfig();
+ assertSame(userConfig, recommendedConfig.getParentConfiguration());
+ assertSame(clusterConfig, userConfig.getParentConfiguration());
+ assertSame(stackConfig, clusterConfig.getParentConfiguration());
+
+ // Yarn/Mapred config is excpected to be removed as it does not belong to
newly added services
+ assertEquals(
+ ImmutableMap.of(
+ "kafka-broker", ImmutableMap.of(
+ "log.dirs", "/kafka-logs",
+ "offsets.topic.replication.factor", "1"),
+ "spark2-defaults", ImmutableMap.of(
+ "spark.yarn.queue", "default"),
+ "oozie-env", ImmutableMap.of(
+ "oozie_heapsize", "1024m", // unit updates should happen
+ "oozie_permsize", "256m")),
+ recommendedConfig.getProperties());
+
+ assertEquals(
+ ImmutableMap.of(
+ "kafka-broker", ImmutableMap.of(
+ "maximum", ImmutableMap.of("offsets.topic.replication.factor",
"10"))),
+ recommendedConfig.getAttributes());
+ }
+
+ @Test
+ public void recommendConfigurations_alwaysDoNotOverrideCustomValues() {
+ Map<String, Map<String, Set<String>>> newServices = ImmutableMap.of(
+ "KAFKA", ImmutableMap.of(
+ "KAFKA_BROKER", ImmutableSet.of("c7401")),
+ "SPARK2", ImmutableMap.of(
+ "SPARK2_JOBHISTORYSERVER", ImmutableSet.of("c7402"),
+ "SPARK2_CLIENT", ImmutableSet.of("c7403", "c7404")),
+ "OOZIE", ImmutableMap.of(
+ "OOZIE_SERVER", ImmutableSet.of("c7401"),
+ "OOZIE_CLIENT", ImmutableSet.of("c7403", "c7404")));
+
+ Configuration stackConfig = Configuration.newEmpty();
+ Configuration clusterConfig = new Configuration(
+ map("oozie-env", map("oozie_heapsize", "1024", "oozie_permsize", "256")),
+ emptyMap());
+ Configuration userConfig = Configuration.newEmpty();
+ userConfig.setParentConfiguration(clusterConfig);
+ clusterConfig.setParentConfiguration(stackConfig);
+
+ LayoutRecommendationInfo layoutRecommendationInfo = new
LayoutRecommendationInfo(new HashMap<>(), new HashMap<>()); // contents doesn't
matter for the test
+ AddServiceInfo info = new
AddServiceInfo(request(ConfigRecommendationStrategy.ALWAYS_APPLY_DONT_OVERRIDE_CUSTOM_VALUES),
"c1", stack , userConfig,
+ null, null, newServices, layoutRecommendationInfo);
+ AddServiceInfo infoWithConfig = adapter.recommendConfigurations(info);
+
+ assertSame(userConfig, infoWithConfig.getConfig()); // user config stays
top priority
+ Configuration recommendedConfig = userConfig.getParentConfiguration();
+ assertSame(clusterConfig, recommendedConfig.getParentConfiguration());
+ assertSame(stackConfig, clusterConfig.getParentConfiguration());
+
+ // Yarn/Mapred config is excpected to be removed as it does not belong to
newly added services
+ assertEquals(
+ ImmutableMap.of(
+ "kafka-broker", ImmutableMap.of(
+ "log.dirs", "/kafka-logs",
+ "offsets.topic.replication.factor", "1"),
+ "spark2-defaults", ImmutableMap.of(
+ "spark.yarn.queue", "default")),
+ recommendedConfig.getProperties());
+
+ // the result of unit updates always happen in the top level config
+ assertEquals(
+ ImmutableMap.of(
+ "oozie-env", ImmutableMap.of(
+ "oozie_heapsize", "1024m",
+ "oozie_permsize", "256m")),
+ userConfig.getProperties());
+
+ assertEquals(
+ ImmutableMap.of(
+ "kafka-broker", ImmutableMap.of(
+ "maximum", ImmutableMap.of("offsets.topic.replication.factor",
"10"))),
+ recommendedConfig.getAttributes());
+ }
+
+ @Test
+ public void recommendConfigurations_neverApply() {
+ Map<String, Map<String, Set<String>>> newServices = ImmutableMap.of(
+ "KAFKA", ImmutableMap.of(
+ "KAFKA_BROKER", ImmutableSet.of("c7401")),
+ "SPARK2", ImmutableMap.of(
+ "SPARK2_JOBHISTORYSERVER", ImmutableSet.of("c7402"),
+ "SPARK2_CLIENT", ImmutableSet.of("c7403", "c7404")),
+ "OOZIE", ImmutableMap.of(
+ "OOZIE_SERVER", ImmutableSet.of("c7401"),
+ "OOZIE_CLIENT", ImmutableSet.of("c7403", "c7404")));
+
+ Configuration stackConfig = Configuration.newEmpty();
+ Configuration clusterConfig = new Configuration(
+ map("oozie-env", map("oozie_heapsize", "1024", "oozie_permsize", "256")),
+ emptyMap());
+ Configuration userConfig = Configuration.newEmpty();
+ userConfig.setParentConfiguration(clusterConfig);
+ clusterConfig.setParentConfiguration(stackConfig);
+
+ LayoutRecommendationInfo layoutRecommendationInfo = new
LayoutRecommendationInfo(new HashMap<>(), new HashMap<>()); // contents doesn't
matter for the test
+ AddServiceInfo info = new
AddServiceInfo(request(ConfigRecommendationStrategy.NEVER_APPLY), "c1", stack ,
userConfig,
+ null, null, newServices, layoutRecommendationInfo);
+ AddServiceInfo infoWithConfig = adapter.recommendConfigurations(info);
+
+ // No recommended config, no stack config
+ assertSame(userConfig, infoWithConfig.getConfig());
+ assertSame(clusterConfig, userConfig.getParentConfiguration());
+ assertNotNull(clusterConfig.getParentConfiguration());
+
+ // the result of unit updates always happen in the top level config
+ assertEquals(
+ ImmutableMap.of(
+ "oozie-env", ImmutableMap.of(
+ "oozie_heapsize", "1024m",
+ "oozie_permsize", "256m")),
+ userConfig.getProperties());
}
+
+ @Test
+ public void recommendConfigurations_onlyStackDefaultsApply() {
+ Map<String, Map<String, Set<String>>> newServices = ImmutableMap.of(
+ "KAFKA", ImmutableMap.of(
+ "KAFKA_BROKER", ImmutableSet.of("c7401")),
+ "SPARK2", ImmutableMap.of(
+ "SPARK2_JOBHISTORYSERVER", ImmutableSet.of("c7402"),
+ "SPARK2_CLIENT", ImmutableSet.of("c7403", "c7404")),
+ "OOZIE", ImmutableMap.of(
+ "OOZIE_SERVER", ImmutableSet.of("c7401"),
+ "OOZIE_CLIENT", ImmutableSet.of("c7403", "c7404")));
+
+ Configuration stackConfig = new Configuration(
+ ImmutableMap.of("kafka-broker", ImmutableMap.of("log.dirs",
"/kafka-logs-stackdefault")),
+ ImmutableMap.of());
+ Configuration clusterConfig = new Configuration(
+ ImmutableMap.of("oozie-env", ImmutableMap.of("oozie_heapsize", "1024",
"oozie_permsize", "256")),
+ emptyMap());
+ Configuration userConfig = Configuration.newEmpty();
+ userConfig.setParentConfiguration(clusterConfig);
+ clusterConfig.setParentConfiguration(stackConfig);
+
+ LayoutRecommendationInfo layoutRecommendationInfo = new
LayoutRecommendationInfo(new HashMap<>(), new HashMap<>()); // contents doesn't
matter for the test
+ AddServiceInfo info = new
AddServiceInfo(request(ConfigRecommendationStrategy.ONLY_STACK_DEFAULTS_APPLY),
"c1", stack , userConfig,
+ null, null, newServices, layoutRecommendationInfo);
+ AddServiceInfo infoWithConfig = adapter.recommendConfigurations(info);
+ Configuration recommendedConfig =
infoWithConfig.getConfig().getParentConfiguration();
+
+ // No recommended config
+ assertSame(userConfig, infoWithConfig.getConfig()); // user config is top
level in this case
+ assertSame(clusterConfig, recommendedConfig.getParentConfiguration());
+ assertSame(stackConfig, clusterConfig.getParentConfiguration());
+
+ assertEquals(
+ ImmutableMap.of(
+ "kafka-broker", ImmutableMap.of(
+ "log.dirs", "/kafka-logs")),
+ recommendedConfig.getProperties());
+
+ // the result of unit updates always happen in the top level config
+ assertEquals(
+ ImmutableMap.of(
+ "oozie-env", ImmutableMap.of(
+ "oozie_heapsize", "1024m",
+ "oozie_permsize", "256m")),
+ userConfig.getProperties());
+ }
+
+ @Test
+ public void removeNonStackConfigRecommendations() {
+ Map<String, Map<String, String>> stackProperties = ImmutableMap.of(
+ "kafka-broker", ImmutableMap.of(
+ "log.dirs", "/kafka-logs",
+ "offsets.topic.replication.factor", "1"),
+ "spark2-defaults", ImmutableMap.of(
+ "spark.yarn.queue", "default"));
+
+ Map<String, Map<String, Map<String, String>>> stackAttributes =
ImmutableMap.of(
+ "oozie-env",
+ ImmutableMap.of(
+ "miniumum",
+ ImmutableMap.of("oozie_heapsize", "1024", "oozie_permsize", "256")));
+
+ Configuration stackConfig = new Configuration(stackProperties,
stackAttributes);
+
+ Map<String, RecommendationResponse.BlueprintConfigurations>
recommendedConfigs =
+ map(
+ "hdfs-site", RecommendationResponse.BlueprintConfigurations.create(
+ map("dfs.namenode.name.dir", "/hadoop/hdfs/namenode"),
+ map("visible", ImmutableMap.of("dfs.namenode.name.dir", "false"))),
+ "oozie-env", RecommendationResponse.BlueprintConfigurations.create(
+ map("oozie_heapsize", "2048"),
+ new HashMap<>()),
+ "spark2-defaults",
RecommendationResponse.BlueprintConfigurations.create(
+ map("spark.yarn.queue", "spark2"),
+ new HashMap<>()));
+
+ Map<String, RecommendationResponse.BlueprintConfigurations>
recommendedConfigsForStackDefaults =
+ ImmutableMap.of(
+ "oozie-env", RecommendationResponse.BlueprintConfigurations.create(
+ ImmutableMap.of("oozie_heapsize", "2048"),
+ ImmutableMap.of()),
+ "spark2-defaults",
RecommendationResponse.BlueprintConfigurations.create(
+ ImmutableMap.of("spark.yarn.queue", "spark2"),
+ ImmutableMap.of()));
+
+ StackAdvisorAdapter.removeNonStackConfigRecommendations(stackConfig,
recommendedConfigs);
+
+ assertEquals(recommendedConfigsForStackDefaults, recommendedConfigs);
+ }
+
+ private AddServiceRequest request(ConfigRecommendationStrategy strategy) {
+ return new AddServiceRequest(null, strategy, null, null, null, null, null,
null, null, null, null);
+ }
+
}
\ No newline at end of file
