Repository: ambari Updated Branches: refs/heads/branch-2.4 57e13815e -> 9e6fbf90a
Revert "AMBARI-17835 Maximum validation failure for 'yarn.scheduler.maximum-allocation-mb' after dependency change (dsen)" Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9e6fbf90 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9e6fbf90 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9e6fbf90 Branch: refs/heads/branch-2.4 Commit: 9e6fbf90af46ec08ca7546d05cb251fb2384dd7b Parents: 57e1381 Author: Dmytro Sen <[email protected]> Authored: Fri Jul 22 19:10:16 2016 +0300 Committer: Dmytro Sen <[email protected]> Committed: Fri Jul 22 19:11:11 2016 +0300 ---------------------------------------------------------------------- .../commands/StackAdvisorCommand.java | 6 ---- .../stacks/HDP/2.2/services/stack_advisor.py | 34 ++++---------------- .../src/main/resources/stacks/stack_advisor.py | 16 +++------ .../commands/StackAdvisorCommandTest.java | 18 ----------- .../stacks/2.2/common/test_stack_advisor.py | 21 +++++++----- 5 files changed, 24 insertions(+), 71 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/9e6fbf90/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java index f93e180..6f612af 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java @@ -98,7 +98,6 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse> extend private static final String CONFIGURATIONS_PROPERTY = "configurations"; private static final String CHANGED_CONFIGURATIONS_PROPERTY = "changed-configurations"; private static final String AMBARI_SERVER_CONFIGURATIONS_PROPERTY = "ambari-server-properties"; - private static final String STACK_ADVISOR_COMMAND_TYPE_PROPERTY = "stack-advisor-command-type"; private File recommendationsDir; private String recommendationsArtifactsLifetime; @@ -164,7 +163,6 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse> extend populateConfigurations(root, request); populateConfigGroups(root, request); populateAmbariServerInfo(root); - populateCommandType(root); data.servicesJSON = mapper.writeValueAsString(root); } catch (Exception e) { // should not happen @@ -185,10 +183,6 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse> extend } } - protected void populateCommandType(ObjectNode root) throws StackAdvisorException { - root.put(STACK_ADVISOR_COMMAND_TYPE_PROPERTY, getCommandType().toString()); - } - private void populateConfigurations(ObjectNode root, StackAdvisorRequest request) { Map<String, Map<String, Map<String, String>>> configurations = http://git-wip-us.apache.org/repos/asf/ambari/blob/9e6fbf90/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py index cfbcbbf..fc38ced 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py @@ -26,8 +26,6 @@ import socket import re import xml.etree.ElementTree as ET -from stack_advisor import ActionType - class HDP22StackAdvisor(HDP21StackAdvisor): def getServiceConfigurationRecommenderDict(self): @@ -72,11 +70,14 @@ class HDP22StackAdvisor(HDP21StackAdvisor): putYarnPropertyAttribute('yarn.scheduler.maximum-allocation-mb', 'maximum', configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"]) # Above is the default calculated 'maximum' values derived purely from hosts. - # However, there are 'maximum' and other attributes that actually should come from services["configurations"] during validations. + # However, there are 'maximum' and other attributes that actually change based on the values + # of other configs. We need to update those values. # On a first stackadvisor invocation from the UI we have stack defaults in services["configurations"]. # So services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"] is always 5120 during first recommendation. - if (self.getActionType(services) == ActionType.VALIDATE_CONFIGURATIONS and "yarn-site" in services["configurations"]): + # Get a value from recommendations instead of services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"] + # in this case (if 'changed-configurations' are empty). + if ('changed-configurations' in services.keys() and services['changed-configurations'] and "yarn-site" in services["configurations"]): if ("yarn.nodemanager.resource.memory-mb" in services["configurations"]["yarn-site"]["properties"]): putYarnPropertyAttribute('yarn.scheduler.maximum-allocation-mb', 'maximum', services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"]) putYarnPropertyAttribute('yarn.scheduler.minimum-allocation-mb', 'maximum', services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"]) @@ -354,19 +355,8 @@ class HDP22StackAdvisor(HDP21StackAdvisor): container_size = min(clusterData['containers'] * clusterData['ramPerContainer'], container_size, yarnMaxAllocationSize) putHiveSiteProperty("hive.tez.container.size", min(int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]), container_size)) - - # "maximum" and "minimum" attributes should come from services["configurations"] during validations. - # If one of dependencies are in changed-confogurations another can be absent. In this case we need to take both values from 'services'. - if self.getActionType(services) == ActionType.VALIDATE_CONFIGURATIONS or ('changed-configurations' in services and services['changed-configurations'] \ - and (self.isPropertyInChangedConfigs('yarn-site', 'yarn.scheduler.minimum-allocation-mb', services['changed-configurations']) or self.isPropertyInChangedConfigs('yarn-site', 'yarn.scheduler.maximum-allocation-mb', services['changed-configurations']))): - if "yarn-site" in services["configurations"]: - if "yarn.scheduler.minimum-allocation-mb" in services["configurations"]["yarn-site"]["properties"]: - putHiveSitePropertyAttribute("hive.tez.container.size", "minimum", int(services["configurations"]["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])) - if "yarn.scheduler.maximum-allocation-mb" in services["configurations"]["yarn-site"]["properties"]: - putHiveSitePropertyAttribute("hive.tez.container.size", "maximum", int(services["configurations"]["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"])) - else: - putHiveSitePropertyAttribute("hive.tez.container.size", "minimum", int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])) - putHiveSitePropertyAttribute("hive.tez.container.size", "maximum", int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"])) + putHiveSitePropertyAttribute("hive.tez.container.size", "minimum", int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])) + putHiveSitePropertyAttribute("hive.tez.container.size", "maximum", int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"])) putHiveSiteProperty("hive.prewarm.enabled", "false") putHiveSiteProperty("hive.prewarm.numcontainers", "3") @@ -1047,16 +1037,6 @@ class HDP22StackAdvisor(HDP21StackAdvisor): putMapredPropertyAttribute = self.putPropertyAttribute(configurations, "mapred-site") yarnMinAllocationSize = int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]) yarnMaxAllocationSize = min(30 * int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]), int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"])) - - # "maximum" and "minimum" attributes should come from services["configurations"] during validations. - # If one of dependencies are in changed-confogurations another can be absent. In this case we need to take both values from 'services'. - if self.getActionType(services) == ActionType.VALIDATE_CONFIGURATIONS or ('changed-configurations' in services and services['changed-configurations'] \ - and (self.isPropertyInChangedConfigs('yarn-site', 'yarn.scheduler.minimum-allocation-mb', services['changed-configurations']) or self.isPropertyInChangedConfigs('yarn-site', 'yarn.scheduler.maximum-allocation-mb', services['changed-configurations']))): - if "yarn-site" in services["configurations"] and "yarn.scheduler.minimum-allocation-mb" in services["configurations"]["yarn-site"]["properties"] \ - and "yarn.scheduler.maximum-allocation-mb" in services["configurations"]["yarn-site"]["properties"]: - yarnMinAllocationSize = int(services["configurations"]["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]) - yarnMaxAllocationSize = min(30 * int(services["configurations"]["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]), int(services["configurations"]["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"])) - putMapredPropertyAttribute("mapreduce.map.memory.mb", "maximum", yarnMaxAllocationSize) putMapredPropertyAttribute("mapreduce.map.memory.mb", "minimum", yarnMinAllocationSize) putMapredPropertyAttribute("mapreduce.reduce.memory.mb", "maximum", yarnMaxAllocationSize) http://git-wip-us.apache.org/repos/asf/ambari/blob/9e6fbf90/ambari-server/src/main/resources/stacks/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/stack_advisor.py b/ambari-server/src/main/resources/stacks/stack_advisor.py index 380c8f4..437fe4f 100644 --- a/ambari-server/src/main/resources/stacks/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/stack_advisor.py @@ -298,12 +298,6 @@ class StackAdvisor(object): """ pass -class ActionType: - RECOMMEND_COMPONENT_LAYOUT_ACTION = 'recommend-component-layout' - VALIDATE_COMPONENT_LAYOUT_ACTION = 'validate-component-layout' - RECOMMEND_CONFIGURATIONS = 'recommend-configurations' - RECOMMEND_CONFIGURATION_DEPENDENCIES = 'recommend-configuration-dependencies' - VALIDATE_CONFIGURATIONS = 'validate-configurations' class DefaultStackAdvisor(StackAdvisor): """ @@ -319,6 +313,7 @@ class DefaultStackAdvisor(StackAdvisor): # Dictionary that maps serviceName or componentName to serviceAdvisor self.serviceAdvisorsDict = {} + def getActiveHosts(self, hosts): """ Filters the list of specified hosts object and returns a list of hosts which are not in maintenance mode. """ @@ -992,13 +987,13 @@ class DefaultStackAdvisor(StackAdvisor): config[configType]["properties"] = {} def appendProperty(key, value): # If property exists in changedConfigs, do not override, use user defined property - if self.isPropertyInChangedConfigs(configType, key, changedConfigs): + if self.__isPropertyInChangedConfigs(configType, key, changedConfigs): config[configType]["properties"][key] = userConfigs[configType]['properties'][key] else: config[configType]["properties"][key] = str(value) return appendProperty - def isPropertyInChangedConfigs(self, configType, propertyName, changedConfigs): + def __isPropertyInChangedConfigs(self, configType, propertyName, changedConfigs): for changedConfig in changedConfigs: if changedConfig['type']==configType and changedConfig['name']==propertyName: return True @@ -1021,7 +1016,7 @@ class DefaultStackAdvisor(StackAdvisor): def updatePropertyWithCallback(key, value, callback): # If property exists in changedConfigs, do not override, use user defined property - if self.isPropertyInChangedConfigs(configType, key, changedConfigs): + if self.__isPropertyInChangedConfigs(configType, key, changedConfigs): config[configType]["properties"][key] = userConfigs[configType]['properties'][key] else: # Give the callback an empty string if the mapping doesn't exist @@ -1088,6 +1083,3 @@ class DefaultStackAdvisor(StackAdvisor): def getServiceNames(self, services): return [service["StackServices"]["service_name"] for service in services["services"]] - - def getActionType(self, services): - return services.get("stack-advisor-command-type", ActionType.RECOMMEND_CONFIGURATIONS) http://git-wip-us.apache.org/repos/asf/ambari/blob/9e6fbf90/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommandTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommandTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommandTest.java index a956789..25d634b 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommandTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommandTest.java @@ -241,24 +241,6 @@ public class StackAdvisorCommandTest { } @Test - public void testPopulateCommandType() throws Exception { - File file = mock(File.class); - String recommendationsArtifactsLifetime = "1w"; - StackAdvisorRunner stackAdvisorRunner = mock(StackAdvisorRunner.class); - AmbariMetaInfo ambariMetaInfo = mock(AmbariMetaInfo.class); - StackAdvisorCommand<TestResource> cmd = new TestStackAdvisorCommand(file, recommendationsArtifactsLifetime, "test", 1, - stackAdvisorRunner, ambariMetaInfo); - ObjectNode objectNode = (ObjectNode) cmd.mapper.readTree("{\"Versions\": " + - "{\"stack_name\": \"stack\", \"stack_version\":\"1.0.0\"}}"); - - cmd.populateCommandType(objectNode); - - JsonNode commandType = objectNode.get("stack-advisor-command-type"); - assertNotNull(commandType); - assertEquals(StackAdvisorCommandType.RECOMMEND_COMPONENT_LAYOUT.toString(), commandType.getTextValue()); - } - - @Test public void testPopulateStackHierarchy_noParents() throws Exception { File file = mock(File.class); String recommendationsArtifactsLifetime = "1w"; http://git-wip-us.apache.org/repos/asf/ambari/blob/9e6fbf90/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py index df897d5..818390d 100644 --- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py @@ -1136,19 +1136,24 @@ class TestHDP22StackAdvisor(TestCase): self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, hosts) self.assertEquals(configurations, expected) - # During a recommendation request, we should get updated 'maximum's from recommendations - # During a validation request, update from services. + # With no 'changed-configurations', we should get updated 'maximum's from recommendations + # Else update from services. # # On a first stackadvisor invocation from the UI we have stack defaults in services["configurations"]. # So services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"] is always 5120 during first recommendation. - - # Test validate-configurations: - services.pop("changed-configurations", None) - services["stack-advisor-command-type"] = 'validate-configurations' + # Get a value from recommendations instead of services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"] + # in this case (if 'changed-configurations' are empty). + # + # Test not empty 'changed-configurations': + services["changed-configurations"] = [ + { + "type": "yarn-env", + "name": "min_user_id" + } + ] + services.pop("configurations", None) services["configurations"] = {"yarn-site": {"properties": {"yarn.nodemanager.resource.memory-mb": '4321', "yarn.nodemanager.resource.cpu-vcores": '9'}}, "yarn-env": {"properties": {"min_user_id": "500"}}} - - # Expected recommendations from 'servicess' expected["yarn-site"]["property_attributes"]["yarn.scheduler.minimum-allocation-vcores"]["maximum"] = '9' expected["yarn-site"]["property_attributes"]["yarn.scheduler.maximum-allocation-vcores"]["maximum"] = '9' expected["yarn-site"]["property_attributes"]["yarn.scheduler.maximum-allocation-mb"]["maximum"] = '4321'
