Repository: ambari Updated Branches: refs/heads/branch-2.5 0b8f8c890 -> 9ef624fe8
AMBARI-19546. Add support for call context to the stack advisor (smohanty) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9ef624fe Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9ef624fe Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9ef624fe Branch: refs/heads/branch-2.5 Commit: 9ef624fe871f8dfdeb89c2b0e3f1e9b1f51396ea Parents: 0b8f8c8 Author: Sumit Mohanty <[email protected]> Authored: Wed Jan 18 22:36:30 2017 -0800 Committer: Sumit Mohanty <[email protected]> Committed: Wed Jan 18 22:36:35 2017 -0800 ---------------------------------------------------------------------- .../StackAdvisorBlueprintProcessor.java | 8 +++++ .../stackadvisor/StackAdvisorRequest.java | 15 +++++++++ .../commands/StackAdvisorCommand.java | 4 +++ .../internal/StackAdvisorResourceProvider.java | 31 +++++++++++++++++-- .../src/main/resources/properties.json | 3 ++ .../src/main/resources/scripts/stack_advisor.py | 14 ++++++--- .../stacks/HDP/2.0.6/services/stack_advisor.py | 25 ++++++++++++--- .../src/main/resources/stacks/stack_advisor.py | 11 +++++++ .../StackAdvisorResourceProviderTest.java | 32 ++++++++++++++++++++ .../stacks/2.0.6/common/test_stack_advisor.py | 8 ++--- 10 files changed, 135 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/9ef624fe/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java index 67dfded..eb3d7e2 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java @@ -66,6 +66,13 @@ public class StackAdvisorBlueprintProcessor { stackAdvisorHelper = instance; } + private static final Map<String, String> userContext; + static + { + userContext = new HashMap<String, String>(); + userContext.put("operation", "ClusterCreate"); + } + /** * Recommend configurations by the stack advisor, then store the results in cluster topology. * @param clusterTopology cluster topology instance @@ -97,6 +104,7 @@ public class StackAdvisorBlueprintProcessor { .forHostComponents(gatherHostGroupComponents(clusterTopology)) .withComponentHostsMap(componentHostsMap) .withConfigurations(calculateConfigs(clusterTopology)) + .withUserContext(userContext) .ofType(requestType) .build(); } http://git-wip-us.apache.org/repos/asf/ambari/blob/9ef624fe/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java index b804c7e..c655816 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java @@ -47,6 +47,7 @@ public class StackAdvisorRequest { private Map<String, Map<String, Map<String, String>>> configurations = new HashMap<String, Map<String, Map<String, String>>>(); private List<ChangedConfigInfo> changedConfigurations = new LinkedList<ChangedConfigInfo>(); private Set<RecommendationResponse.ConfigGroup> configGroups; + private Map<String, String> userContext = new HashMap<String, String>(); public String getStackName() { return stackName; @@ -100,6 +101,14 @@ public class StackAdvisorRequest { this.changedConfigurations = changedConfigurations; } + public Map<String, String> getUserContext() { + return this.userContext; + } + + public void setUserContext(Map<String, String> userContext) { + this.userContext = userContext; + } + public Set<RecommendationResponse.ConfigGroup> getConfigGroups() { return configGroups; } @@ -168,6 +177,12 @@ public class StackAdvisorRequest { return this; } + public StackAdvisorRequestBuilder withUserContext( + Map<String, String> userContext) { + this.instance.userContext = userContext; + return this; + } + public StackAdvisorRequestBuilder withConfigGroups( Set<RecommendationResponse.ConfigGroup> configGroups) { this.instance.configGroups = configGroups; http://git-wip-us.apache.org/repos/asf/ambari/blob/9ef624fe/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java index 6f612af..dfd0d15 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java @@ -97,6 +97,7 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse> extend private static final String COMPONENT_HOSTNAMES_PROPERTY = "hostnames"; private static final String CONFIGURATIONS_PROPERTY = "configurations"; private static final String CHANGED_CONFIGURATIONS_PROPERTY = "changed-configurations"; + private static final String USER_CONTEXT_PROPERTY = "user-context"; private static final String AMBARI_SERVER_CONFIGURATIONS_PROPERTY = "ambari-server-properties"; private File recommendationsDir; @@ -205,6 +206,9 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse> extend JsonNode changedConfigs = mapper.valueToTree(request.getChangedConfigurations()); root.put(CHANGED_CONFIGURATIONS_PROPERTY, changedConfigs); + + JsonNode userContext = mapper.valueToTree(request.getUserContext()); + root.put(USER_CONTEXT_PROPERTY, userContext); } private void populateConfigGroups(ObjectNode root, http://git-wip-us.apache.org/repos/asf/ambari/blob/9ef624fe/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java index 0ad9126..5bb52c2 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java @@ -58,6 +58,9 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi private static final String SERVICES_PROPERTY = "services"; private static final String CHANGED_CONFIGURATIONS_PROPERTY = "changed_configurations"; + private static final String OPERATION_PROPERTY = "operation"; + private static final String OPERATION_DETAILS_PROPERTY = "operation_details"; + private static final String BLUEPRINT_HOST_GROUPS_PROPERTY = "recommendations/blueprint/host_groups"; private static final String BINDING_HOST_GROUPS_PROPERTY = "recommendations/blueprint_cluster_binding/host_groups"; @@ -75,6 +78,8 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi private static final String CONFIG_GROUPS_HOSTS_PROPERTY = "hosts"; protected static StackAdvisorHelper saHelper; + protected static final String USER_CONTEXT_OPERATION_PROPERTY = "user_context/operation"; + protected static final String USER_CONTEXT_OPERATION_DETAILS_PROPERTY = "user_context/operation_details"; @Inject public static void init(StackAdvisorHelper instance) { @@ -109,6 +114,7 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi Map<String, Set<String>> componentHostsMap = calculateComponentHostsMap(hgComponentsMap, hgHostsMap); Map<String, Map<String, Map<String, String>>> configurations = calculateConfigurations(request); + Map<String, String> userContext = readUserContext(request); List<ChangedConfigInfo> changedConfigurations = requestType == StackAdvisorRequestType.CONFIGURATION_DEPENDENCIES ? @@ -122,9 +128,10 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi withComponentHostsMap(componentHostsMap). withConfigurations(configurations). withConfigGroups(configGroups). - withChangedConfigurations(changedConfigurations).build(); + withChangedConfigurations(changedConfigurations). + withUserContext(userContext).build(); } catch (Exception e) { - LOG.warn("Error occured during preparation of stack advisor request", e); + LOG.warn("Error occurred during preparation of stack advisor request", e); Response response = Response.status(Status.BAD_REQUEST) .entity(String.format("Request body is not correct, error: %s", e.getMessage())).build(); // TODO: Hosts and services must not be empty @@ -240,6 +247,26 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi return configGroups; } + /** + * Parse the user contex for the call. Typical structure + * { "operation" : "createCluster" } + * { "operation" : "addService", "services" : "Atlas,Slider" } + * @param request + * @return + */ + protected Map<String, String> readUserContext(Request request) { + HashMap<String, String> userContext = new HashMap<>(); + if (null != getRequestProperty(request, USER_CONTEXT_OPERATION_PROPERTY)) { + userContext.put(OPERATION_PROPERTY, + (String) getRequestProperty(request, USER_CONTEXT_OPERATION_PROPERTY)); + } + if (null != getRequestProperty(request, USER_CONTEXT_OPERATION_DETAILS_PROPERTY)) { + userContext.put(OPERATION_DETAILS_PROPERTY, + (String) getRequestProperty(request, USER_CONTEXT_OPERATION_DETAILS_PROPERTY)); + } + return userContext; + } + protected static final String CONFIGURATIONS_PROPERTY_ID = "recommendations/blueprint/configurations/"; protected Map<String, Map<String, Map<String, String>>> calculateConfigurations(Request request) { http://git-wip-us.apache.org/repos/asf/ambari/blob/9ef624fe/ambari-server/src/main/resources/properties.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/properties.json b/ambari-server/src/main/resources/properties.json index b7e0988..698b6c5 100644 --- a/ambari-server/src/main/resources/properties.json +++ b/ambari-server/src/main/resources/properties.json @@ -405,6 +405,9 @@ "hosts", "services", "changed_configurations", + "user_context", + "user_context/operation", + "user_context/operation_details", "recommendations", "recommendations/blueprint", "recommendations/blueprint/configurations", http://git-wip-us.apache.org/repos/asf/ambari/blob/9ef624fe/ambari-server/src/main/resources/scripts/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/scripts/stack_advisor.py b/ambari-server/src/main/resources/scripts/stack_advisor.py index 56bd34a..66ab248 100755 --- a/ambari-server/src/main/resources/scripts/stack_advisor.py +++ b/ambari-server/src/main/resources/scripts/stack_advisor.py @@ -42,6 +42,10 @@ STACK_ADVISOR_DEFAULT_IMPL_CLASS = 'DefaultStackAdvisor' STACK_ADVISOR_IMPL_PATH_TEMPLATE = os.path.join(SCRIPT_DIRECTORY, './../stacks/{0}/{1}/services/stack_advisor.py') STACK_ADVISOR_IMPL_CLASS_TEMPLATE = '{0}{1}StackAdvisor' +ADVISOR_CONTEXT = "advisor_context" +CALL_TYPE = "call_type" + + class StackAdvisorException(Exception): pass @@ -100,23 +104,23 @@ def main(argv=None): hosts = stackAdvisor.filterHostMounts(hosts, services) if action == RECOMMEND_COMPONENT_LAYOUT_ACTION: - services['context'] = {'call_type': 'recommendComponentLayout'} + services[ADVISOR_CONTEXT] = {CALL_TYPE : 'recommendComponentLayout'} result = stackAdvisor.recommendComponentLayout(services, hosts) result_file = os.path.join(actionDir, "component-layout.json") elif action == VALIDATE_COMPONENT_LAYOUT_ACTION: - services['context'] = {'call_type': 'validateComponentLayout'} + services[ADVISOR_CONTEXT] = {CALL_TYPE : 'validateComponentLayout'} result = stackAdvisor.validateComponentLayout(services, hosts) result_file = os.path.join(actionDir, "component-layout-validation.json") elif action == RECOMMEND_CONFIGURATIONS: - services['context'] = {'call_type': 'recommendConfigurations'} + services[ADVISOR_CONTEXT] = {CALL_TYPE : 'recommendConfigurations'} result = stackAdvisor.recommendConfigurations(services, hosts) result_file = os.path.join(actionDir, "configurations.json") elif action == RECOMMEND_CONFIGURATION_DEPENDENCIES: - services['context'] = {'call_type': 'recommendConfigurationDependencies'} + services[ADVISOR_CONTEXT] = {CALL_TYPE : 'recommendConfigurationDependencies'} result = stackAdvisor.recommendConfigurationDependencies(services, hosts) result_file = os.path.join(actionDir, "configurations.json") else: # action == VALIDATE_CONFIGURATIONS - services['context'] = {'call_type': 'validateConfigurations'} + services[ADVISOR_CONTEXT] = {CALL_TYPE: 'validateConfigurations'} result = stackAdvisor.validateConfigurations(services, hosts) result_file = os.path.join(actionDir, "configurations-validation.json") http://git-wip-us.apache.org/repos/asf/ambari/blob/9ef624fe/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py index 968a94f..628f183 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py @@ -1044,9 +1044,16 @@ class HDP206StackAdvisor(DefaultStackAdvisor): suggestedMinContainerRam = 1024 # new smaller value for YARN min container callContext = getCallContext(services) + operation = getUserOperationContext(services, DefaultStackAdvisor.OPERATION) + if operation: + Logger.info("user operation context : " + str(operation)) + if services: # its never None but some unit tests pass it as None - if None != getOldValue(self, services, "yarn-site", "yarn.scheduler.minimum-allocation-mb") or \ - 'recommendConfigurations' != callContext: + # If min container value is changed (user is changing it) + # if its a validation call - just used what ever value is set + # If its not a cluster create or add yarn service (TBD) + if (getOldValue(self, services, "yarn-site", "yarn.scheduler.minimum-allocation-mb") or \ + 'recommendConfigurations' != callContext) and operation != DefaultStackAdvisor.CLUSTER_CREATE_OPERATION: '''yarn.scheduler.minimum-allocation-mb has changed - then pick this value up''' if "yarn-site" in services["configurations"] and \ "yarn.scheduler.minimum-allocation-mb" in services["configurations"]["yarn-site"]["properties"] and \ @@ -1964,9 +1971,9 @@ class HDP206StackAdvisor(DefaultStackAdvisor): def getCallContext(services): if services: - if 'context' in services: - Logger.info("context : " + str (services['context'])) - return services['context']['call_type'] + if DefaultStackAdvisor.ADVISOR_CONTEXT in services: + Logger.info("call type context : " + str(services[DefaultStackAdvisor.ADVISOR_CONTEXT])) + return services[DefaultStackAdvisor.ADVISOR_CONTEXT][DefaultStackAdvisor.CALL_TYPE] return "" @@ -1979,6 +1986,14 @@ def getOldValue(self, services, configType, propertyName): return changedConfig["old_value"] return None +def getUserOperationContext(services, contextName): + if services: + if 'user-context' in services.keys(): + userContext = services["user-context"] + if contextName in userContext: + return userContext[contextName] + return None + # Validation helper methods def getSiteProperties(configurations, siteName): siteConfig = configurations.get(siteName) http://git-wip-us.apache.org/repos/asf/ambari/blob/9ef624fe/ambari-server/src/main/resources/stacks/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/stack_advisor.py b/ambari-server/src/main/resources/stacks/stack_advisor.py index 6b29bc3..2dce53a 100644 --- a/ambari-server/src/main/resources/stacks/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/stack_advisor.py @@ -300,6 +300,17 @@ class StackAdvisor(object): class DefaultStackAdvisor(StackAdvisor): + + CLUSTER_CREATE_OPERATION = "ClusterCreate" + ADD_SERVICE_OPERATION = "AddService" + EDIT_CONFIG_OPERATION = "EditConfig" + RECOMMEND_ATTRIBUTE_OPERATION = "RecommendAttribute" + OPERATION = "operation" + OPERATION_DETAILS = "operation_details" + + ADVISOR_CONTEXT = "advisor_context" + CALL_TYPE = "call_type" + """ Default stack advisor implementation. http://git-wip-us.apache.org/repos/asf/ambari/blob/9ef624fe/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProviderTest.java index e3b89b8..5a3c8af 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProviderTest.java @@ -32,6 +32,8 @@ import java.util.Map; import java.util.Set; import static org.apache.ambari.server.controller.internal.StackAdvisorResourceProvider.CONFIGURATIONS_PROPERTY_ID; +import static org.apache.ambari.server.controller.internal.StackAdvisorResourceProvider.USER_CONTEXT_OPERATION_PROPERTY; +import static org.apache.ambari.server.controller.internal.StackAdvisorResourceProvider.USER_CONTEXT_OPERATION_DETAILS_PROPERTY; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -76,6 +78,36 @@ public class StackAdvisorResourceProviderTest { } @Test + public void testReadUserContext() throws Exception { + + Map<Resource.Type, String> keyPropertyIds = Collections.emptyMap(); + Set<String> propertyIds = Collections.emptySet(); + AmbariManagementController ambariManagementController = mock(AmbariManagementController.class); + RecommendationResourceProvider provider = new RecommendationResourceProvider(propertyIds, + keyPropertyIds, ambariManagementController); + + Request request = mock(Request.class); + Set<Map<String, Object>> propertiesSet = new HashSet<Map<String, Object>>(); + Map<String, Object> propertiesMap = new HashMap<String, Object>(); + propertiesMap.put(CONFIGURATIONS_PROPERTY_ID + "site/properties/string_prop", "string"); + List<Object> array = new ArrayList<Object>(); + array.add("array1"); + array.add("array2"); + propertiesMap.put(USER_CONTEXT_OPERATION_PROPERTY, "op1"); + propertiesMap.put(USER_CONTEXT_OPERATION_DETAILS_PROPERTY, "op_det"); + propertiesSet.add(propertiesMap); + + doReturn(propertiesSet).when(request).getProperties(); + + Map<String, String> userContext = provider.readUserContext(request); + + assertNotNull(userContext); + assertEquals(2, userContext.size()); + assertEquals("op1", userContext.get("operation")); + assertEquals("op_det", userContext.get("operation_details")); + } + + @Test public void testCalculateConfigurationsWithNullPropertyValues() throws Exception { Map<Resource.Type, String> keyPropertyIds = Collections.emptyMap(); http://git-wip-us.apache.org/repos/asf/ambari/blob/9ef624fe/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py index f39703c..c2c2c2d 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py @@ -526,7 +526,7 @@ class TestHDP206StackAdvisor(TestCase): } }, "changed-configurations": [], - "context": {'call_type': 'validateConfigurations'} + "advisor_context": {'call_type': 'validateConfigurations'} } result = self.stackAdvisor.getConfigurationClusterSummary(servicesList, hosts, components, services) self.assertEquals(result, expected) @@ -571,7 +571,7 @@ class TestHDP206StackAdvisor(TestCase): } }, "changed-configurations": [], - "context": {'call_type': 'validateConfigurations'} + "advisor_context": {'call_type': 'validateConfigurations'} } expected_2048 = { @@ -636,7 +636,7 @@ class TestHDP206StackAdvisor(TestCase): } }, "changed-configurations": [], - "context": {'call_type': 'recommendConfigurationDependencies'} + "advisor_context": {'call_type': 'recommendConfigurationDependencies'} } expected_2048 = { @@ -701,7 +701,7 @@ class TestHDP206StackAdvisor(TestCase): } }, "changed-configurations": [], - "context": {'call_type': 'recommendConfigurationDependencies'} + "advisor_context": {'call_type': 'recommendConfigurationDependencies'} } expected_4096 = {
