Repository: ambari Updated Branches: refs/heads/trunk a30d15738 -> 2516a059a
AMBARI-17818. Tez & Hive: Memory management should use scaled down Xmx for sizing buffers. (Gopal V via Swapan Shridhar) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2516a059 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2516a059 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2516a059 Branch: refs/heads/trunk Commit: 2516a059a7ca116d218bb6411fccd4ce11ad59ff Parents: a30d157 Author: Swapan Shridhar <[email protected]> Authored: Thu Jul 21 14:35:38 2016 -0700 Committer: Swapan Shridhar <[email protected]> Committed: Thu Jul 21 14:37:36 2016 -0700 ---------------------------------------------------------------------- .../resources/stacks/HDP/2.2/services/stack_advisor.py | 2 +- .../resources/stacks/HDP/2.3/services/stack_advisor.py | 3 ++- .../test/python/stacks/2.2/common/test_stack_advisor.py | 4 ++-- .../test/python/stacks/2.3/common/test_stack_advisor.py | 10 +++++----- 4 files changed, 10 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/2516a059/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py index 1dd7a07..b77980a 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py @@ -364,7 +364,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor): putHiveSiteProperty("hive.tez.dynamic.partition.pruning", "true") container_size = configurations["hive-site"]["properties"]["hive.tez.container.size"] - container_size_bytes = int(container_size)*1024*1024 + container_size_bytes = int(int(container_size)*0.8*1024*1024) # Xmx == 80% of container # Memory putHiveSiteProperty("hive.auto.convert.join.noconditionaltask.size", int(round(container_size_bytes/3))) putHiveSitePropertyAttribute("hive.auto.convert.join.noconditionaltask.size", "maximum", container_size_bytes) http://git-wip-us.apache.org/repos/asf/ambari/blob/2516a059/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py index 24047ce..8d5cdc9 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py @@ -92,7 +92,8 @@ class HDP23StackAdvisor(HDP22StackAdvisor): # remove 2gb limit for tez.runtime.io.sort.mb # in HDP 2.3 "tez.runtime.sorter.class" is set by default to PIPELINED, in other case comment calculation code below taskResourceMemory = int(configurations["tez-site"]["properties"]["tez.task.resource.memory.mb"]) - putTezProperty("tez.runtime.io.sort.mb", int(taskResourceMemory * 0.4)) + # fit io.sort.mb into tenured regions + putTezProperty("tez.runtime.io.sort.mb", int(taskResourceMemory * 0.8 * 0.33)) if "tez-site" in services["configurations"] and "tez.runtime.sorter.class" in services["configurations"]["tez-site"]["properties"]: if services["configurations"]["tez-site"]["properties"]["tez.runtime.sorter.class"] == "LEGACY": http://git-wip-us.apache.org/repos/asf/ambari/blob/2516a059/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py index 9d3ce4c..5fe534a 100644 --- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py @@ -1203,7 +1203,7 @@ class TestHDP22StackAdvisor(TestCase): 'hive.server2.tez.default.queues': "queue1,queue2", 'hive.server2.tez.initialize.default.sessions': 'false', 'hive.server2.tez.sessions.per.default.queue': '1', - 'hive.auto.convert.join.noconditionaltask.size': '268435456', + 'hive.auto.convert.join.noconditionaltask.size': '214748364', 'hive.compactor.initiator.on': 'false', 'hive.compactor.worker.threads': '0', 'hive.compute.query.using.stats': 'true', @@ -1238,7 +1238,7 @@ class TestHDP22StackAdvisor(TestCase): 'hive.security.authorization.manager': 'org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdConfOnlyAuthorizerFactory' }, 'property_attributes': { - 'hive.auto.convert.join.noconditionaltask.size': {'maximum': '805306368'}, + 'hive.auto.convert.join.noconditionaltask.size': {'maximum': '644245094'}, 'hive.tez.container.size': {'maximum': '8192', 'minimum': '256'}, 'hive.server2.authentication.pam.services': {'delete': 'true'}, 'hive.server2.custom.authentication.class': {'delete': 'true'}, http://git-wip-us.apache.org/repos/asf/ambari/blob/2516a059/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py index faad01c..34bebe8 100644 --- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py @@ -617,7 +617,7 @@ class TestHDP23StackAdvisor(TestCase): 'hive.server2.tez.default.queues': "queue1,queue2", 'hive.server2.tez.initialize.default.sessions': 'false', 'hive.server2.tez.sessions.per.default.queue': '1', - 'hive.auto.convert.join.noconditionaltask.size': '268435456', + 'hive.auto.convert.join.noconditionaltask.size': '214748364', 'hive.compactor.initiator.on': 'false', 'hive.compactor.worker.threads': '0', 'hive.compute.query.using.stats': 'true', @@ -652,7 +652,7 @@ class TestHDP23StackAdvisor(TestCase): 'hive.security.authorization.manager': 'org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdConfOnlyAuthorizerFactory' }, 'property_attributes': { - 'hive.auto.convert.join.noconditionaltask.size': {'maximum': '805306368'}, + 'hive.auto.convert.join.noconditionaltask.size': {'maximum': '644245094'}, 'hive.server2.authentication.pam.services': {'delete': 'true'}, 'hive.server2.custom.authentication.class': {'delete': 'true'}, 'hive.server2.authentication.kerberos.principal': {'delete': 'true'}, @@ -884,7 +884,7 @@ class TestHDP23StackAdvisor(TestCase): 'hive.server2.tez.default.queues': "queue1,queue2", 'hive.server2.tez.initialize.default.sessions': 'false', 'hive.server2.tez.sessions.per.default.queue': '1', - 'hive.auto.convert.join.noconditionaltask.size': '268435456', + 'hive.auto.convert.join.noconditionaltask.size': '214748364', 'hive.compactor.initiator.on': 'false', 'hive.compactor.worker.threads': '0', 'hive.compute.query.using.stats': 'true', @@ -919,7 +919,7 @@ class TestHDP23StackAdvisor(TestCase): 'hive.security.authorization.manager': 'org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdConfOnlyAuthorizerFactory' }, 'property_attributes': { - 'hive.auto.convert.join.noconditionaltask.size': {'maximum': '805306368'}, + 'hive.auto.convert.join.noconditionaltask.size': {'maximum': '644245094'}, 'hive.tez.container.size': {'maximum': '8192', 'minimum': '256'}, 'hive.server2.authentication.pam.services': {'delete': 'true'}, 'hive.server2.custom.authentication.class': {'delete': 'true'}, @@ -1161,7 +1161,7 @@ class TestHDP23StackAdvisor(TestCase): "tez.task.resource.memory.mb": "768", "tez.am.launch.cmd-opts": "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC", "tez.task.launch.cmd-opts": "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC", - "tez.runtime.io.sort.mb": "307", + "tez.runtime.io.sort.mb": "202", "tez.session.am.dag.submit.timeout.secs": "600", "tez.runtime.unordered.output.buffer.size-mb": "57", "tez.am.resource.memory.mb": "4000",
