Repository: ambari Updated Branches: refs/heads/branch-1.7.0 7f82db52b -> 6d70bc044 refs/heads/trunk aa15c03d0 -> f93670872
AMBARI-7953. Modify HDP-2.2 stack advisor for the tez config properties (aonishuk) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f9367087 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f9367087 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f9367087 Branch: refs/heads/trunk Commit: f93670872d48a9600d8cc07569171880c44a805c Parents: aa15c03 Author: Andrew Onishuk <[email protected]> Authored: Fri Oct 24 21:15:18 2014 +0300 Committer: Andrew Onishuk <[email protected]> Committed: Fri Oct 24 21:15:18 2014 +0300 ---------------------------------------------------------------------- .../stacks/HDP/2.2/services/stack_advisor.py | 18 ++++++ .../stacks/2.2/common/test_stack_advisor.py | 66 ++++++++++++++++++++ 2 files changed, 84 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/f9367087/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py index 19b1065..b291530 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py @@ -23,6 +23,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor): parentRecommendConfDict = super(HDP22StackAdvisor, self).getServiceConfigurationRecommenderDict() childRecommendConfDict = { "HDFS": self.recommendHDFSConfigurations, + "TEZ": self.recommendTezConfigurations } parentRecommendConfDict.update(childRecommendConfDict) return parentRecommendConfDict @@ -31,14 +32,31 @@ class HDP22StackAdvisor(HDP21StackAdvisor): def recommendHDFSConfigurations(self, configurations, clusterData): self.putProperty(configurations, "hdfs-site") + def recommendTezConfigurations(self, configurations, clusterData): + putTezProperty = self.putProperty(configurations, "tez-site") + putTezProperty("tez.am.resource.memory.mb", int(clusterData['amMemory']) * 2 if int(clusterData['amMemory']) < 3072 else int(clusterData['amMemory'])) + + taskResourceMemory = clusterData['mapMemory'] if clusterData['mapMemory'] > 2048 else int(clusterData['reduceMemory']) + taskResourceMemory = min(clusterData['containers'] * clusterData['ramPerContainer'], taskResourceMemory) + putTezProperty("tez.task.resource.memory.mb", taskResourceMemory) + putTezProperty("tez.runtime.io.sort.mb", int(taskResourceMemory * 0.4) if int(taskResourceMemory * 0.4) <= 2147483644 else 2147483644) + putTezProperty("tez.runtime.unordered.output.buffer.size-mb", int(taskResourceMemory * 0.075)) + def getServiceConfigurationValidators(self): parentValidators = super(HDP22StackAdvisor, self).getServiceConfigurationValidators() childValidators = { "HDFS": ["hdfs-site", self.validateHDFSConfigurations], + "TEZ": ["tez-site", self.validateTezConfigurations] } parentValidators.update(childValidators) return parentValidators + def validateTezConfigurations(self, properties, recommendedDefaults, configurations): + validationItems = [ {"config-name": 'tez.am.resource.memory.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.am.resource.memory.mb')}, + {"config-name": 'tez.task.resource.memory.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.task.resource.memory.mb')}, + {"config-name": 'tez.runtime.io.sort.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.runtime.io.sort.mb')}, + {"config-name": 'tez.runtime.unordered.output.buffer.size-mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.runtime.unordered.output.buffer.size-mb')},] + return self.toConfigurationValidationProblems(validationItems, "tez-site") def validateHDFSConfigurations(self, properties, recommendedDefaults, configurations): # We can not access property hadoop.security.authentication from the http://git-wip-us.apache.org/repos/asf/ambari/blob/f9367087/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py index 3d6b2e6..06c476e 100644 --- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py @@ -42,6 +42,72 @@ class TestHDP22StackAdvisor(TestCase): clazz = getattr(stack_advisor_impl, hdp22StackAdvisorClassName) self.stackAdvisor = clazz() + def test_recommendTezConfigurations(self): + configurations = {} + clusterData = { + "mapMemory": 3000, + "amMemory": 2000, + "reduceMemory": 2056, + "containers": 3, + "ramPerContainer": 256 + } + expected = { + "tez-site": { + "properties": { + "tez.am.resource.memory.mb": "4000", + "tez.task.resource.memory.mb": "768", + "tez.runtime.io.sort.mb": "307", + "tez.runtime.unordered.output.buffer.size-mb": "57" + } + } + } + self.stackAdvisor.recommendTezConfigurations(configurations, clusterData) + self.assertEquals(configurations, expected) + + def test_recommendTezConfigurations_amMemoryMoreThan3072(self): + configurations = {} + clusterData = { + "mapMemory": 4000, + "amMemory": 3100, + "reduceMemory": 2056, + "containers": 3, + "ramPerContainer": 256 + } + expected = { + "tez-site": { + "properties": { + "tez.am.resource.memory.mb": "3100", + "tez.task.resource.memory.mb": "768", + "tez.runtime.io.sort.mb": "307", + "tez.runtime.unordered.output.buffer.size-mb": "57" + } + } + } + self.stackAdvisor.recommendTezConfigurations(configurations, clusterData) + self.assertEquals(configurations, expected) + + def test_recommendTezConfigurations_mapMemoryLessThan768(self): + configurations = {} + clusterData = { + "mapMemory": 760, + "amMemory": 2000, + "reduceMemory": 760, + "containers": 3, + "ramPerContainer": 256 + } + expected = { + "tez-site": { + "properties": { + "tez.am.resource.memory.mb": "4000", + "tez.task.resource.memory.mb": "760", + "tez.runtime.io.sort.mb": "304", + "tez.runtime.unordered.output.buffer.size-mb": "57" + } + } + } + self.stackAdvisor.recommendTezConfigurations(configurations, clusterData) + self.assertEquals(configurations, expected) + def test_validateHDFSConfigurations(self): self.maxDiff = None
