Repository: ambari Updated Branches: refs/heads/trunk 988775704 -> b5b9493ff
AMBARI-17835 Maximum validation failure for 'yarn.scheduler.maximum-allocation-mb' after dependency change (dsen) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b5b9493f Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b5b9493f Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b5b9493f Branch: refs/heads/trunk Commit: b5b9493ff7da82f342a7b39c2c8d273d763c8eae Parents: 9887757 Author: Dmytro Sen <[email protected]> Authored: Fri Jul 22 15:14:13 2016 +0300 Committer: Dmytro Sen <[email protected]> Committed: Fri Jul 22 15:14:13 2016 +0300 ---------------------------------------------------------------------- .../commands/StackAdvisorCommand.java | 6 ++++ .../stacks/HDP/2.2/services/stack_advisor.py | 34 ++++++++++++++++---- .../src/main/resources/stacks/stack_advisor.py | 16 ++++++--- .../commands/StackAdvisorCommandTest.java | 18 +++++++++++ .../stacks/2.2/common/test_stack_advisor.py | 21 +++++------- 5 files changed, 71 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/b5b9493f/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java index 6f612af..f93e180 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java @@ -98,6 +98,7 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse> extend private static final String CONFIGURATIONS_PROPERTY = "configurations"; private static final String CHANGED_CONFIGURATIONS_PROPERTY = "changed-configurations"; private static final String AMBARI_SERVER_CONFIGURATIONS_PROPERTY = "ambari-server-properties"; + private static final String STACK_ADVISOR_COMMAND_TYPE_PROPERTY = "stack-advisor-command-type"; private File recommendationsDir; private String recommendationsArtifactsLifetime; @@ -163,6 +164,7 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse> extend populateConfigurations(root, request); populateConfigGroups(root, request); populateAmbariServerInfo(root); + populateCommandType(root); data.servicesJSON = mapper.writeValueAsString(root); } catch (Exception e) { // should not happen @@ -183,6 +185,10 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse> extend } } + protected void populateCommandType(ObjectNode root) throws StackAdvisorException { + root.put(STACK_ADVISOR_COMMAND_TYPE_PROPERTY, getCommandType().toString()); + } + private void populateConfigurations(ObjectNode root, StackAdvisorRequest request) { Map<String, Map<String, Map<String, String>>> configurations = http://git-wip-us.apache.org/repos/asf/ambari/blob/b5b9493f/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py index b77980a..416447e 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py @@ -26,6 +26,8 @@ import socket import re import xml.etree.ElementTree as ET +from stack_advisor import ActionType + class HDP22StackAdvisor(HDP21StackAdvisor): def getServiceConfigurationRecommenderDict(self): @@ -70,14 +72,11 @@ class HDP22StackAdvisor(HDP21StackAdvisor): putYarnPropertyAttribute('yarn.scheduler.maximum-allocation-mb', 'maximum', configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"]) # Above is the default calculated 'maximum' values derived purely from hosts. - # However, there are 'maximum' and other attributes that actually change based on the values - # of other configs. We need to update those values. + # However, there are 'maximum' and other attributes that actually should come from services["configurations"] during validations. # On a first stackadvisor invocation from the UI we have stack defaults in services["configurations"]. # So services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"] is always 5120 during first recommendation. - # Get a value from recommendations instead of services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"] - # in this case (if 'changed-configurations' are empty). - if ('changed-configurations' in services.keys() and services['changed-configurations'] and "yarn-site" in services["configurations"]): + if (self.getActionType(services) == ActionType.VALIDATE_CONFIGURATIONS and "yarn-site" in services["configurations"]): if ("yarn.nodemanager.resource.memory-mb" in services["configurations"]["yarn-site"]["properties"]): putYarnPropertyAttribute('yarn.scheduler.maximum-allocation-mb', 'maximum', services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"]) putYarnPropertyAttribute('yarn.scheduler.minimum-allocation-mb', 'maximum', services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"]) @@ -355,8 +354,19 @@ class HDP22StackAdvisor(HDP21StackAdvisor): container_size = min(clusterData['containers'] * clusterData['ramPerContainer'], container_size, yarnMaxAllocationSize) putHiveSiteProperty("hive.tez.container.size", min(int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]), container_size)) - putHiveSitePropertyAttribute("hive.tez.container.size", "minimum", int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])) - putHiveSitePropertyAttribute("hive.tez.container.size", "maximum", int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"])) + + # "maximum" and "minimum" attributes should come from services["configurations"] during validations. + # If one of dependencies are in changed-confogurations another can be absent. In this case we need to take both values from 'services'. + if self.getActionType(services) == ActionType.VALIDATE_CONFIGURATIONS or ('changed-configurations' in services and services['changed-configurations'] \ + and (self.isPropertyInChangedConfigs('yarn-site', 'yarn.scheduler.minimum-allocation-mb', services['changed-configurations']) or self.isPropertyInChangedConfigs('yarn-site', 'yarn.scheduler.maximum-allocation-mb', services['changed-configurations']))): + if "yarn-site" in services["configurations"]: + if "yarn.scheduler.minimum-allocation-mb" in services["configurations"]["yarn-site"]["properties"]: + putHiveSitePropertyAttribute("hive.tez.container.size", "minimum", int(services["configurations"]["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])) + if "yarn.scheduler.maximum-allocation-mb" in services["configurations"]["yarn-site"]["properties"]: + putHiveSitePropertyAttribute("hive.tez.container.size", "maximum", int(services["configurations"]["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"])) + else: + putHiveSitePropertyAttribute("hive.tez.container.size", "minimum", int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])) + putHiveSitePropertyAttribute("hive.tez.container.size", "maximum", int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"])) putHiveSiteProperty("hive.prewarm.enabled", "false") putHiveSiteProperty("hive.prewarm.numcontainers", "3") @@ -1037,6 +1047,16 @@ class HDP22StackAdvisor(HDP21StackAdvisor): putMapredPropertyAttribute = self.putPropertyAttribute(configurations, "mapred-site") yarnMinAllocationSize = int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]) yarnMaxAllocationSize = min(30 * int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]), int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"])) + + # "maximum" and "minimum" attributes should come from services["configurations"] during validations. + # If one of dependencies are in changed-confogurations another can be absent. In this case we need to take both values from 'services'. + if self.getActionType(services) == ActionType.VALIDATE_CONFIGURATIONS or ('changed-configurations' in services and services['changed-configurations'] \ + and (self.isPropertyInChangedConfigs('yarn-site', 'yarn.scheduler.minimum-allocation-mb', services['changed-configurations']) or self.isPropertyInChangedConfigs('yarn-site', 'yarn.scheduler.maximum-allocation-mb', services['changed-configurations']))): + if "yarn-site" in services["configurations"] and "yarn.scheduler.minimum-allocation-mb" in services["configurations"]["yarn-site"]["properties"] \ + and "yarn.scheduler.maximum-allocation-mb" in services["configurations"]["yarn-site"]["properties"]: + yarnMinAllocationSize = int(services["configurations"]["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]) + yarnMaxAllocationSize = min(30 * int(services["configurations"]["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]), int(services["configurations"]["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"])) + putMapredPropertyAttribute("mapreduce.map.memory.mb", "maximum", yarnMaxAllocationSize) putMapredPropertyAttribute("mapreduce.map.memory.mb", "minimum", yarnMinAllocationSize) putMapredPropertyAttribute("mapreduce.reduce.memory.mb", "maximum", yarnMaxAllocationSize) http://git-wip-us.apache.org/repos/asf/ambari/blob/b5b9493f/ambari-server/src/main/resources/stacks/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/stack_advisor.py b/ambari-server/src/main/resources/stacks/stack_advisor.py index 437fe4f..380c8f4 100644 --- a/ambari-server/src/main/resources/stacks/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/stack_advisor.py @@ -298,6 +298,12 @@ class StackAdvisor(object): """ pass +class ActionType: + RECOMMEND_COMPONENT_LAYOUT_ACTION = 'recommend-component-layout' + VALIDATE_COMPONENT_LAYOUT_ACTION = 'validate-component-layout' + RECOMMEND_CONFIGURATIONS = 'recommend-configurations' + RECOMMEND_CONFIGURATION_DEPENDENCIES = 'recommend-configuration-dependencies' + VALIDATE_CONFIGURATIONS = 'validate-configurations' class DefaultStackAdvisor(StackAdvisor): """ @@ -313,7 +319,6 @@ class DefaultStackAdvisor(StackAdvisor): # Dictionary that maps serviceName or componentName to serviceAdvisor self.serviceAdvisorsDict = {} - def getActiveHosts(self, hosts): """ Filters the list of specified hosts object and returns a list of hosts which are not in maintenance mode. """ @@ -987,13 +992,13 @@ class DefaultStackAdvisor(StackAdvisor): config[configType]["properties"] = {} def appendProperty(key, value): # If property exists in changedConfigs, do not override, use user defined property - if self.__isPropertyInChangedConfigs(configType, key, changedConfigs): + if self.isPropertyInChangedConfigs(configType, key, changedConfigs): config[configType]["properties"][key] = userConfigs[configType]['properties'][key] else: config[configType]["properties"][key] = str(value) return appendProperty - def __isPropertyInChangedConfigs(self, configType, propertyName, changedConfigs): + def isPropertyInChangedConfigs(self, configType, propertyName, changedConfigs): for changedConfig in changedConfigs: if changedConfig['type']==configType and changedConfig['name']==propertyName: return True @@ -1016,7 +1021,7 @@ class DefaultStackAdvisor(StackAdvisor): def updatePropertyWithCallback(key, value, callback): # If property exists in changedConfigs, do not override, use user defined property - if self.__isPropertyInChangedConfigs(configType, key, changedConfigs): + if self.isPropertyInChangedConfigs(configType, key, changedConfigs): config[configType]["properties"][key] = userConfigs[configType]['properties'][key] else: # Give the callback an empty string if the mapping doesn't exist @@ -1083,3 +1088,6 @@ class DefaultStackAdvisor(StackAdvisor): def getServiceNames(self, services): return [service["StackServices"]["service_name"] for service in services["services"]] + + def getActionType(self, services): + return services.get("stack-advisor-command-type", ActionType.RECOMMEND_CONFIGURATIONS) http://git-wip-us.apache.org/repos/asf/ambari/blob/b5b9493f/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommandTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommandTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommandTest.java index 25d634b..a956789 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommandTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommandTest.java @@ -241,6 +241,24 @@ public class StackAdvisorCommandTest { } @Test + public void testPopulateCommandType() throws Exception { + File file = mock(File.class); + String recommendationsArtifactsLifetime = "1w"; + StackAdvisorRunner stackAdvisorRunner = mock(StackAdvisorRunner.class); + AmbariMetaInfo ambariMetaInfo = mock(AmbariMetaInfo.class); + StackAdvisorCommand<TestResource> cmd = new TestStackAdvisorCommand(file, recommendationsArtifactsLifetime, "test", 1, + stackAdvisorRunner, ambariMetaInfo); + ObjectNode objectNode = (ObjectNode) cmd.mapper.readTree("{\"Versions\": " + + "{\"stack_name\": \"stack\", \"stack_version\":\"1.0.0\"}}"); + + cmd.populateCommandType(objectNode); + + JsonNode commandType = objectNode.get("stack-advisor-command-type"); + assertNotNull(commandType); + assertEquals(StackAdvisorCommandType.RECOMMEND_COMPONENT_LAYOUT.toString(), commandType.getTextValue()); + } + + @Test public void testPopulateStackHierarchy_noParents() throws Exception { File file = mock(File.class); String recommendationsArtifactsLifetime = "1w"; http://git-wip-us.apache.org/repos/asf/ambari/blob/b5b9493f/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py index 5fe534a..41d8409 100644 --- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py @@ -1136,24 +1136,19 @@ class TestHDP22StackAdvisor(TestCase): self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, hosts) self.assertEquals(configurations, expected) - # With no 'changed-configurations', we should get updated 'maximum's from recommendations - # Else update from services. + # During a recommendation request, we should get updated 'maximum's from recommendations + # During a validation request, update from services. # # On a first stackadvisor invocation from the UI we have stack defaults in services["configurations"]. # So services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"] is always 5120 during first recommendation. - # Get a value from recommendations instead of services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"] - # in this case (if 'changed-configurations' are empty). - # - # Test not empty 'changed-configurations': - services["changed-configurations"] = [ - { - "type": "yarn-env", - "name": "min_user_id" - } - ] - services.pop("configurations", None) + + # Test validate-configurations: + services.pop("changed-configurations", None) + services["stack-advisor-command-type"] = 'validate-configurations' services["configurations"] = {"yarn-site": {"properties": {"yarn.nodemanager.resource.memory-mb": '4321', "yarn.nodemanager.resource.cpu-vcores": '9'}}, "yarn-env": {"properties": {"min_user_id": "500"}}} + + # Expected recommendations from 'servicess' expected["yarn-site"]["property_attributes"]["yarn.scheduler.minimum-allocation-vcores"]["maximum"] = '9' expected["yarn-site"]["property_attributes"]["yarn.scheduler.maximum-allocation-vcores"]["maximum"] = '9' expected["yarn-site"]["property_attributes"]["yarn.scheduler.maximum-allocation-mb"]["maximum"] = '4321'
