Repository: ambari Updated Branches: refs/heads/trunk 9cb96ad08 -> a08bdc061
AMBARI-19236. Stackadvisor: incorrect recommendations when only one of two dependencies was changed.(vbrodetskyi) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a08bdc06 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a08bdc06 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a08bdc06 Branch: refs/heads/trunk Commit: a08bdc0618ca99d98ae6b93a0a15c278eeec5f75 Parents: 9cb96ad Author: Vitaly Brodetskyi <[email protected]> Authored: Mon Dec 19 17:42:57 2016 +0200 Committer: Vitaly Brodetskyi <[email protected]> Committed: Mon Dec 19 17:42:57 2016 +0200 ---------------------------------------------------------------------- .../src/main/resources/stacks/stack_advisor.py | 19 ++++++- .../stacks/2.2/common/test_stack_advisor.py | 53 ++++++++++++++++++++ 2 files changed, 71 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/a08bdc06/ambari-server/src/main/resources/stacks/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/stack_advisor.py b/ambari-server/src/main/resources/stacks/stack_advisor.py index 8148379..8865b70 100644 --- a/ambari-server/src/main/resources/stacks/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/stack_advisor.py @@ -313,6 +313,10 @@ class DefaultStackAdvisor(StackAdvisor): # Dictionary that maps serviceName or componentName to serviceAdvisor self.serviceAdvisorsDict = {} + # Contains requested properties during 'recommend-configuration-dependencies' request. + # It's empty during other requests. + self.allRequestedProperties = None + def getActiveHosts(self, hosts): """ Filters the list of specified hosts object and returns @@ -880,6 +884,7 @@ class DefaultStackAdvisor(StackAdvisor): return component["StackServiceComponents"]["hostnames"] def recommendConfigurationDependencies(self, services, hosts): + self.allRequestedProperties = self.getAllRequestedProperties(services) result = self.recommendConfigurations(services, hosts) return self.filterResult(result, services) @@ -1000,7 +1005,8 @@ class DefaultStackAdvisor(StackAdvisor): config[configType]["properties"] = {} def appendProperty(key, value): # If property exists in changedConfigs, do not override, use user defined property - if self.__isPropertyInChangedConfigs(configType, key, changedConfigs): + if not self.isPropertyRequested(configType, key, changedConfigs) \ + and configType in userConfigs and key in userConfigs[configType]['properties']: config[configType]["properties"][key] = userConfigs[configType]['properties'][key] else: config[configType]["properties"][key] = str(value) @@ -1012,6 +1018,17 @@ class DefaultStackAdvisor(StackAdvisor): return True return False + def isPropertyRequested(self, configType, propertyName, changedConfigs): + # When the property depends on more than one property, we need to recalculate it based on the actual values + # of all related properties. But "changed-configurations" usually contains only one on the dependent on properties. + # So allRequestedProperties is used to avoid recommendations of other properties that are not requested. + # Calculations should use user provided values for all properties that we depend on, not only the one that + # came in the "changed-configurations". + if self.allRequestedProperties: + return configType in self.allRequestedProperties and propertyName in self.allRequestedProperties[configType] + else: + return not self.__isPropertyInChangedConfigs(configType, propertyName, changedConfigs) + def updateProperty(self, config, configType, services=None): userConfigs = {} changedConfigs = [] http://git-wip-us.apache.org/repos/asf/ambari/blob/a08bdc06/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py index 70ce79e..54349a2 100644 --- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py @@ -1173,6 +1173,59 @@ class TestHDP22StackAdvisor(TestCase): self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, hosts) self.assertEquals(configurations, expected) + def test_multipleDependsOn(self): + configurations = { + "yarn-env": { + "properties": { + "min_user_id": "500" + } + }, + "yarn-site": { + "properties": { + "yarn.nodemanager.resource.memory-mb": "1280", + "yarn.scheduler.minimum-allocation-mb": "350", + "yarn.scheduler.maximum-allocation-mb": "1000", + }, + }, + "mapred-site": { + "properties": { + "mapreduce.map.memory.mb": "0", + "mapreduce.reduce.memory.mb": "111" + } + } + } + clusterData = { + "cpu": 4, + "containers" : 5, + "ramPerContainer": 256 + } + + services = { + "configurations": configurations, + "services": [], + "changed-configurations": [ + { + "type": "yarn-site", + "name": "yarn.scheduler.maximum-allocation-mb", + "old_value": "512" + }, + ] + + } + hosts = {} + + # immitate recommend-configuration-dependencies request with only "yarn.scheduler.maximum-allocation-mb" in "changed-configurations" + self.stackAdvisor.allRequestedProperties = {'yarn-site': ['yarn.scheduler.maximum-allocation-mb'], 'mapred-site': ['mapreduce.map.memory.mb']} + + self.stackAdvisor.recommendMapReduce2Configurations(configurations, clusterData, services, hosts) + + # changed-configurations contain only "yarn.scheduler.maximum-allocation-mb". + # Ensure that user provided value (350) for "yarn.scheduler.minimum-allocation-mb" is used. + # The recommended default for "yarn.scheduler.minimum-allocation-mb" is 256. + self.assertEquals(configurations['mapred-site']['properties']['mapreduce.map.memory.mb'], '350') # should not be 256 + + # assert that not requested property was not changed + self.assertEquals(configurations['mapred-site']['properties']['mapreduce.reduce.memory.mb'], '111') def test_recommendHiveConfigurationAttributes(self): self.maxDiff = None
