Repository: ambari Updated Branches: refs/heads/trunk b40d808d3 -> 8dcade74e
AMBARI-11627. Default Map memory should be less than default reduce memory (srimanth) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8dcade74 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8dcade74 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8dcade74 Branch: refs/heads/trunk Commit: 8dcade74e2c669f0ec15e5562232fe12d51ca57e Parents: b40d808 Author: Srimanth Gunturi <[email protected]> Authored: Tue Jun 2 09:50:16 2015 -0700 Committer: Srimanth Gunturi <[email protected]> Committed: Tue Jun 2 11:12:56 2015 -0700 ---------------------------------------------------------------------- .../stacks/HDP/2.2/services/stack_advisor.py | 6 ++++-- .../stacks/HDPWIN/2.2/services/stack_advisor.py | 17 +++++++++++++---- .../python/stacks/2.2/common/test_stack_advisor.py | 10 +++++----- 3 files changed, 22 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/8dcade74/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py index 1e31662..3ea3167 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py @@ -634,14 +634,16 @@ class HDP22StackAdvisor(HDP21StackAdvisor): nodemanagerMinRam = min(nodemanager["Hosts"]["total_mem"]/1024, nodemanagerMinRam) putMapredProperty('yarn.app.mapreduce.am.resource.mb', configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]) putMapredProperty('yarn.app.mapreduce.am.command-opts', "-Xmx" + str(int(0.8 * int(configurations["mapred-site"]["properties"]["yarn.app.mapreduce.am.resource.mb"]))) + "m" + " -Dhdp.version=${hdp.version}") - putMapredProperty('mapreduce.reduce.memory.mb', min(2*int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]), int(nodemanagerMinRam))) servicesList = [service["StackServices"]["service_name"] for service in services["services"]] min_mapreduce_map_memory_mb = 0 + min_mapreduce_reduce_memory_mb = 0 min_mapreduce_map_java_opts = 0 if ("PIG" in servicesList): - min_mapreduce_map_memory_mb = 1500 + min_mapreduce_map_memory_mb = 1536 + min_mapreduce_reduce_memory_mb = 1536 min_mapreduce_map_java_opts = 1024 putMapredProperty('mapreduce.map.memory.mb', max(min_mapreduce_map_memory_mb, int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]))) + putMapredProperty('mapreduce.reduce.memory.mb', max(min_mapreduce_reduce_memory_mb, min(2*int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]), int(nodemanagerMinRam)))) mapredMapXmx = int(0.8*int(configurations["mapred-site"]["properties"]["mapreduce.map.memory.mb"])); putMapredProperty('mapreduce.map.java.opts', "-Xmx" + str(max(min_mapreduce_map_java_opts, mapredMapXmx)) + "m") putMapredProperty('mapreduce.reduce.java.opts', "-Xmx" + str(int(0.8*int(configurations["mapred-site"]["properties"]["mapreduce.reduce.memory.mb"]))) + "m") http://git-wip-us.apache.org/repos/asf/ambari/blob/8dcade74/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py index 3b7c948..058b618 100644 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py @@ -230,11 +230,20 @@ class HDPWIN22StackAdvisor(HDPWIN21StackAdvisor): putMapredProperty = self.putProperty(configurations, "mapred-site", services) putMapredProperty('yarn.app.mapreduce.am.resource.mb', configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]) putMapredProperty('yarn.app.mapreduce.am.command-opts', "-Xmx" + str(int(0.8 * int(configurations["mapred-site"]["properties"]["yarn.app.mapreduce.am.resource.mb"]))) + "m" + " -Dhdp.version=${hdp.version}") - putMapredProperty('mapreduce.map.memory.mb', int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])) - putMapredProperty('mapreduce.reduce.memory.mb', int(2*int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]))) - putMapredProperty('mapreduce.map.java.opts', "-Xmx" + str(int(0.8*int(configurations["mapred-site"]["properties"]["mapreduce.map.memory.mb"]))) + "m") + servicesList = [service["StackServices"]["service_name"] for service in services["services"]] + min_mapreduce_map_memory_mb = 0 + min_mapreduce_reduce_memory_mb = 0 + min_mapreduce_map_java_opts = 0 + if ("PIG" in servicesList): + min_mapreduce_map_memory_mb = 1536 + min_mapreduce_reduce_memory_mb = 1536 + min_mapreduce_map_java_opts = 1024 + putMapredProperty('mapreduce.map.memory.mb', max(min_mapreduce_map_memory_mb, int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]))) + putMapredProperty('mapreduce.reduce.memory.mb', max(min_mapreduce_reduce_memory_mb, min(2*int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]), int(nodemanagerMinRam)))) + mapredMapXmx = int(0.8*int(configurations["mapred-site"]["properties"]["mapreduce.map.memory.mb"])); + putMapredProperty('mapreduce.map.java.opts', "-Xmx" + str(max(min_mapreduce_map_java_opts, mapredMapXmx)) + "m") putMapredProperty('mapreduce.reduce.java.opts', "-Xmx" + str(int(0.8*int(configurations["mapred-site"]["properties"]["mapreduce.reduce.memory.mb"]))) + "m") - putMapredProperty('mapreduce.task.io.sort.mb', str(int(0.7*int(configurations["mapred-site"]["properties"]["mapreduce.map.memory.mb"])))) + putMapredProperty('mapreduce.task.io.sort.mb', str(min(int(0.7*mapredMapXmx), 2047))) # Property Attributes putMapredPropertyAttribute = self.putPropertyAttribute(configurations, "mapred-site") yarnMinAllocationSize = int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]) http://git-wip-us.apache.org/repos/asf/ambari/blob/8dcade74/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py index 3a641f1..8d462d4 100644 --- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py @@ -1209,13 +1209,13 @@ class TestHDP22StackAdvisor(TestCase): }, "mapred-site": { "properties": { - "mapreduce.map.memory.mb": "1500", - "mapreduce.reduce.memory.mb": "200", + "mapreduce.map.memory.mb": "1536", + "mapreduce.reduce.memory.mb": "1536", "yarn.app.mapreduce.am.command-opts": "-Xmx80m -Dhdp.version=${hdp.version}", - "mapreduce.reduce.java.opts": "-Xmx160m", + "mapreduce.reduce.java.opts": "-Xmx1228m", "yarn.app.mapreduce.am.resource.mb": "100", - "mapreduce.map.java.opts": "-Xmx1200m", - "mapreduce.task.io.sort.mb": "840" + "mapreduce.map.java.opts": "-Xmx1228m", + "mapreduce.task.io.sort.mb": "859" }, "property_attributes": { 'mapreduce.task.io.sort.mb': {'maximum': '2047'},
