AMBARI-19338 AM sizing for LLAP - min container size changes (dsen)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d256e82e Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d256e82e Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d256e82e Branch: refs/heads/branch-dev-patch-upgrade Commit: d256e82e628a6ba86acd17a21c2b925a7b3ae9c5 Parents: 20a5076 Author: Dmytro Sen <[email protected]> Authored: Tue Jan 3 20:24:16 2017 +0200 Committer: Dmytro Sen <[email protected]> Committed: Tue Jan 3 20:24:35 2017 +0200 ---------------------------------------------------------------------- .../stacks/HDP/2.0.6/services/stack_advisor.py | 15 +++-- .../stacks/HDP/2.2/services/stack_advisor.py | 2 +- .../stacks/2.0.6/common/test_stack_advisor.py | 43 ++++++++------ .../stacks/2.2/common/test_stack_advisor.py | 34 +++++++---- .../stacks/2.3/common/test_stack_advisor.py | 2 +- .../stacks/2.5/common/test_stack_advisor.py | 60 +++++++++++++------- 6 files changed, 98 insertions(+), 58 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/d256e82e/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py index 77ed527..7736301 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py @@ -126,7 +126,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor): if "referenceNodeManagerHost" in clusterData: nodemanagerMinRam = min(clusterData["referenceNodeManagerHost"]["total_mem"]/1024, nodemanagerMinRam) putYarnProperty('yarn.nodemanager.resource.memory-mb', int(round(min(clusterData['containers'] * clusterData['ramPerContainer'], nodemanagerMinRam)))) - putYarnProperty('yarn.scheduler.minimum-allocation-mb', int(clusterData['ramPerContainer'])) + putYarnProperty('yarn.scheduler.minimum-allocation-mb', int(clusterData['minContainerRam'])) putYarnProperty('yarn.scheduler.maximum-allocation-mb', int(configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"])) putYarnEnvProperty('min_user_id', self.get_system_min_uid()) @@ -1042,7 +1042,8 @@ class HDP206StackAdvisor(DefaultStackAdvisor): cluster["minContainerSize"] = { - cluster["ram"] <= 4: 256, + cluster["ram"] <= 3: 128, + 3 < cluster["ram"] <= 4: 256, 4 < cluster["ram"] <= 8: 512, 8 < cluster["ram"] <= 24: 1024, 24 < cluster["ram"]: 2048 @@ -1053,20 +1054,22 @@ class HDP206StackAdvisor(DefaultStackAdvisor): totalAvailableRam -= cluster["hbaseRam"] cluster["totalAvailableRam"] = max(512, totalAvailableRam * 1024) '''containers = max(3, min (2*cores,min (1.8*DISKS,(Total available RAM) / MIN_CONTAINER_SIZE))))''' - cluster["containers"] = round(max(3, + cluster["containers"] = int(round(max(3, min(2 * cluster["cpu"], min(ceil(1.8 * cluster["disk"]), - cluster["totalAvailableRam"] / cluster["minContainerSize"])))) + cluster["totalAvailableRam"] / cluster["minContainerSize"]))))) '''ramPerContainers = max(2GB, RAM - reservedRam - hBaseRam) / containers''' - cluster["ramPerContainer"] = abs(cluster["totalAvailableRam"] / cluster["containers"]) + cluster["ramPerContainer"] = int(abs(cluster["totalAvailableRam"] / cluster["containers"])) '''If greater than 1GB, value will be in multiples of 512.''' if cluster["ramPerContainer"] > 1024: cluster["ramPerContainer"] = int(cluster["ramPerContainer"] / 512) * 512 + cluster["minContainerRam"] = min(1024, cluster["ramPerContainer"]) + cluster["mapMemory"] = int(cluster["ramPerContainer"]) cluster["reduceMemory"] = cluster["ramPerContainer"] - cluster["amMemory"] = max(cluster["mapMemory"], cluster["reduceMemory"]) + cluster["amMemory"] = min(cluster["mapMemory"], cluster["minContainerRam"]) return cluster http://git-wip-us.apache.org/repos/asf/ambari/blob/d256e82e/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py index 6f91ca8..b61b0ab 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py @@ -829,7 +829,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor): yarnMaxAllocationSize = min(30 * int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]), int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"])) putTezProperty = self.putProperty(configurations, "tez-site", services) - putTezProperty("tez.am.resource.memory.mb", min(int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]), int(clusterData['amMemory']) * 2 if int(clusterData['amMemory']) < 3072 else int(clusterData['amMemory']))) + putTezProperty("tez.am.resource.memory.mb", min(int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]), int(clusterData['amMemory']))) taskResourceMemory = clusterData['mapMemory'] if clusterData['mapMemory'] > 2048 else int(clusterData['reduceMemory']) taskResourceMemory = min(clusterData['containers'] * clusterData['ramPerContainer'], taskResourceMemory, yarnMaxAllocationSize) http://git-wip-us.apache.org/repos/asf/ambari/blob/d256e82e/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py index 4b3397f..ac3028e 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py @@ -258,7 +258,7 @@ class TestHDP206StackAdvisor(TestCase): result = self.stackAdvisor.validateConfigurations(services, hosts) expectedItems = [ - {"message": "Value is less than the recommended default of 512", "level": "WARN"}, + {"message": "Value is less than the recommended default of 510", "level": "WARN"}, {'message': 'Value should be set for yarn.nodemanager.linux-container-executor.group', 'level': 'ERROR'}, {"message": "Value should be integer", "level": "ERROR"}, {"message": "Value should be set", "level": "ERROR"} @@ -419,6 +419,7 @@ class TestHDP206StackAdvisor(TestCase): "reservedRam": 2, "hbaseRam": 1, "minContainerSize": 512, + "minContainerRam" : 512, "totalAvailableRam": 3072, "containers": 6, "ramPerContainer": 512, @@ -487,15 +488,16 @@ class TestHDP206StackAdvisor(TestCase): }) expected["referenceHost"] = hosts["items"][1]["Hosts"] expected["referenceNodeManagerHost"] = hosts["items"][1]["Hosts"] - expected["amMemory"] = 170.66666666666666 - expected["containers"] = 3.0 + expected["amMemory"] = 128 + expected["containers"] = 4 expected["cpu"] = 4 expected["totalAvailableRam"] = 512 - expected["mapMemory"] = 170 - expected["minContainerSize"] = 256 - expected["reduceMemory"] = 170.66666666666666 + expected["mapMemory"] = 128 + expected["minContainerSize"] = 128 + expected["reduceMemory"] = 128 + expected["minContainerRam"] = 128 expected["ram"] = 0 - expected["ramPerContainer"] = 170.66666666666666 + expected["ramPerContainer"] = 128 expected["reservedRam"] = 1 result = self.stackAdvisor.getConfigurationClusterSummary(servicesList, hosts, components, services) self.assertEquals(result, expected) @@ -536,7 +538,8 @@ class TestHDP206StackAdvisor(TestCase): "ramPerContainer": 3072, "mapMemory": 3072, "reduceMemory": 3072, - "amMemory": 3072, + "amMemory": 1024, + "minContainerRam": 1024, "referenceHost": hosts["items"][0]["Hosts"] } @@ -592,7 +595,8 @@ class TestHDP206StackAdvisor(TestCase): services = {"configurations": configurations, "services": []} clusterData = { "containers" : 5, - "ramPerContainer": 256 + "ramPerContainer": 256, + "minContainerRam": 256 } expected = { "yarn-env": { @@ -619,7 +623,8 @@ class TestHDP206StackAdvisor(TestCase): clusterData = { "mapMemory": 567, "reduceMemory": 345.6666666666666, - "amMemory": 123.54 + "amMemory": 123.54, + "minContainerRam": 123.54 } expected = { "mapred-site": { @@ -655,13 +660,14 @@ class TestHDP206StackAdvisor(TestCase): "ram": 0, "reservedRam": 1, "hbaseRam": 1, - "minContainerSize": 256, + "minContainerSize": 128, "totalAvailableRam": 512, "containers": 3, - "ramPerContainer": 170.66666666666666, + "ramPerContainer": 170, "mapMemory": 170, - "reduceMemory": 170.66666666666666, - "amMemory": 170.66666666666666 + "reduceMemory": 170, + "amMemory": 170, + "minContainerRam" : 170 } self.assertEquals(result, expected) @@ -1171,7 +1177,8 @@ class TestHDP206StackAdvisor(TestCase): } clusterData = { - "totalAvailableRam": 2048 + "totalAvailableRam": 2048, + "totalAvailableRam": 256 } ambariHostName = socket.getfqdn() expected = {'oozie-env': @@ -1210,7 +1217,7 @@ class TestHDP206StackAdvisor(TestCase): 'namenode_heapsize': '1024', 'proxyuser_group': 'users', 'namenode_opt_maxnewsize': '256', - 'namenode_opt_newsize': '256'}}} + 'namenode_opt_newsize': '128'}}} # Apart from testing other HDFS recommendations, also tests 'hadoop.proxyuser.hive.hosts' config value which includes both HiveServer # and Hive Server Interactive Host (installed on different host compared to HiveServer). @@ -1345,7 +1352,7 @@ class TestHDP206StackAdvisor(TestCase): 'namenode_heapsize': '1024', 'proxyuser_group': 'users', 'namenode_opt_maxnewsize': '256', - 'namenode_opt_newsize': '256'}}} + 'namenode_opt_newsize': '128'}}} # Apart from testing other HDFS recommendations, also tests 'hadoop.proxyuser.hive.hosts' config value which includes both HiveServer # and Hive Server Interactive Host (installed on same host compared to HiveServer). @@ -1478,7 +1485,7 @@ class TestHDP206StackAdvisor(TestCase): 'namenode_heapsize': '1024', 'proxyuser_group': 'users', 'namenode_opt_maxnewsize': '256', - 'namenode_opt_newsize': '256'}}} + 'namenode_opt_newsize': '128'}}} self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, services3, hosts) self.assertEquals(configurations, expected) http://git-wip-us.apache.org/repos/asf/ambari/blob/d256e82e/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py index beebe28..1c9238e 100644 --- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py @@ -86,10 +86,11 @@ class TestHDP22StackAdvisor(TestCase): } clusterData = { "mapMemory": 3000, - "amMemory": 2000, + "amMemory": 2048, "reduceMemory": 2056, "containers": 3, - "ramPerContainer": 256 + "ramPerContainer": 256, + "minContainerRam": 256 } expected = { "tez-site": { @@ -183,6 +184,7 @@ class TestHDP22StackAdvisor(TestCase): "amMemory": 3100, "reduceMemory": 2056, "containers": 3, + "minContainerRam": 256, "ramPerContainer": 256 } expected = { @@ -265,10 +267,11 @@ class TestHDP22StackAdvisor(TestCase): } clusterData = { "mapMemory": 760, - "amMemory": 2000, + "amMemory": 2048, "reduceMemory": 760, "containers": 3, - "ramPerContainer": 256 + "ramPerContainer": 256, + "minContainerRam": 256 } expected = { "tez-site": { @@ -884,7 +887,8 @@ class TestHDP22StackAdvisor(TestCase): clusterData = { "cpu": 4, "containers" : 5, - "ramPerContainer": 256 + "ramPerContainer": 256, + "minContainerRam": 256 } expected = { "yarn-env": { @@ -923,7 +927,8 @@ class TestHDP22StackAdvisor(TestCase): clusterData = { "cpu": 4, "containers": 5, - "ramPerContainer": 256 + "ramPerContainer": 256, + "minContainerRam": 256 } expected = { "spark-defaults": { @@ -955,7 +960,8 @@ class TestHDP22StackAdvisor(TestCase): clusterData = { "cpu": 4, "containers" : 5, - "ramPerContainer": 256 + "ramPerContainer": 256, + "minContainerRam": 256 } expected = { "yarn-env": { @@ -1197,7 +1203,8 @@ class TestHDP22StackAdvisor(TestCase): clusterData = { "cpu": 4, "containers" : 5, - "ramPerContainer": 256 + "ramPerContainer": 256, + "minContainerRam": 256 } services = { @@ -1244,7 +1251,8 @@ class TestHDP22StackAdvisor(TestCase): "amMemory": 2000, "reduceMemory": 2056, "containers": 3, - "ramPerContainer": 256 + "ramPerContainer": 256, + "minContainerRam": 256 } expected = { @@ -1735,6 +1743,7 @@ class TestHDP22StackAdvisor(TestCase): "containers" : 7, "ramPerContainer": 256, "totalAvailableRam": 4096, + "minContainerRam": 256 } expected = { "cluster-env": { @@ -2002,7 +2011,8 @@ class TestHDP22StackAdvisor(TestCase): clusterData = { "cpu": 4, "containers" : 5, - "ramPerContainer": 256 + "ramPerContainer": 256, + "minContainerRam": 256 } expected = { "yarn-env": { @@ -3790,7 +3800,7 @@ class TestHDP22StackAdvisor(TestCase): "yarn.nodemanager.linux-container-executor.cgroups.mount-path": "/cgroup", "yarn.nodemanager.linux-container-executor.cgroups.mount": "true", "yarn.nodemanager.resource.memory-mb": "39424", - "yarn.scheduler.minimum-allocation-mb": "3584", + "yarn.scheduler.minimum-allocation-mb": "1024", "yarn.scheduler.maximum-allocation-vcores": "4", "yarn.scheduler.minimum-allocation-vcores": "1", "yarn.nodemanager.resource.cpu-vcores": "4", @@ -3849,7 +3859,7 @@ class TestHDP22StackAdvisor(TestCase): "yarn.nodemanager.linux-container-executor.cgroups.mount-path": "/cgroup", "yarn.nodemanager.linux-container-executor.cgroups.mount": "true", "yarn.nodemanager.resource.memory-mb": "39424", - "yarn.scheduler.minimum-allocation-mb": "3584", + "yarn.scheduler.minimum-allocation-mb": "1024", "yarn.scheduler.maximum-allocation-vcores": "4", "yarn.scheduler.minimum-allocation-vcores": "1", "yarn.nodemanager.resource.cpu-vcores": "4", http://git-wip-us.apache.org/repos/asf/ambari/blob/d256e82e/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py index 378522e..88fa6d0 100644 --- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py @@ -1165,7 +1165,7 @@ class TestHDP23StackAdvisor(TestCase): "tez.runtime.io.sort.mb": "202", "tez.session.am.dag.submit.timeout.secs": "600", "tez.runtime.unordered.output.buffer.size-mb": "57", - "tez.am.resource.memory.mb": "4000", + "tez.am.resource.memory.mb": "2000", "tez.queue.name": "queue2", } }, http://git-wip-us.apache.org/repos/asf/ambari/blob/d256e82e/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py index a7ccdfe..d4a301c 100644 --- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py @@ -374,7 +374,8 @@ class TestHDP25StackAdvisor(TestCase): clusterData = { "cpu": 4, "containers": 5, - "ramPerContainer": 256 + "ramPerContainer": 256, + "minContainerRam": 256 } expected = { "spark2-defaults": { @@ -801,7 +802,8 @@ class TestHDP25StackAdvisor(TestCase): "ramPerContainer": 512, "referenceNodeManagerHost" : { "total_mem" : 10240 * 1024 - } + }, + "minContainerRam": 512 } @@ -969,7 +971,8 @@ class TestHDP25StackAdvisor(TestCase): "ramPerContainer": 512, "referenceNodeManagerHost" : { "total_mem" : 10240 * 1024 - } + }, + "minContainerRam": 512 } configurations = { @@ -1149,7 +1152,8 @@ class TestHDP25StackAdvisor(TestCase): "ramPerContainer": 512, "referenceNodeManagerHost" : { "total_mem" : 10240 * 1024 - } + }, + "minContainerRam": 512 } @@ -1348,7 +1352,8 @@ class TestHDP25StackAdvisor(TestCase): "ramPerContainer": 512, "referenceNodeManagerHost" : { "total_mem" : 10240 * 2048 - } + }, + "minContainerRam": 512 } @@ -1542,7 +1547,8 @@ class TestHDP25StackAdvisor(TestCase): "ramPerContainer": 1024, "referenceNodeManagerHost" : { "total_mem" : 51200 * 1024 - } + }, + "minContainerRam": 1024 } configurations = { @@ -1744,7 +1750,8 @@ class TestHDP25StackAdvisor(TestCase): "ramPerContainer": 2048, "referenceNodeManagerHost" : { "total_mem" : 40960 * 1024 - } + }, + "minContainerRam": 1024 } configurations = { @@ -1940,7 +1947,8 @@ class TestHDP25StackAdvisor(TestCase): "ramPerContainer": 341, "referenceNodeManagerHost" : { "total_mem" : 12288 * 1024 - } + }, + "minContainerRam": 341 } @@ -2138,7 +2146,8 @@ class TestHDP25StackAdvisor(TestCase): "ramPerContainer": 2048, "referenceNodeManagerHost" : { "total_mem" : 204800 * 1024 - } + }, + "minContainerRam": 1024 } configurations = { @@ -2340,7 +2349,8 @@ class TestHDP25StackAdvisor(TestCase): "ramPerContainer": 3072, "referenceNodeManagerHost" : { "total_mem" : 40960 * 1024 - } + }, + "minContainerRam": 1024 } @@ -2538,7 +2548,8 @@ class TestHDP25StackAdvisor(TestCase): "ramPerContainer": 341, "referenceNodeManagerHost" : { "total_mem" : 204800 * 1024 - } + }, + "minContainerRam": 341 } @@ -2735,7 +2746,8 @@ class TestHDP25StackAdvisor(TestCase): "ramPerContainer": 82240, "referenceNodeManagerHost" : { "total_mem" : 204800 * 1024 - } + }, + "minContainerRam": 1024 } configurations = { @@ -2927,7 +2939,8 @@ class TestHDP25StackAdvisor(TestCase): "ramPerContainer": 82240, "referenceNodeManagerHost" : { "total_mem" : 328960 * 1024 - } + }, + "minContainerRam": 1024 } @@ -3148,7 +3161,8 @@ class TestHDP25StackAdvisor(TestCase): "ramPerContainer": 512, "referenceNodeManagerHost" : { "total_mem" : 10240 * 1024 - } + }, + "minContainerRam": 512 } configurations = { @@ -3374,7 +3388,8 @@ class TestHDP25StackAdvisor(TestCase): "ramPerContainer": 512, "referenceNodeManagerHost" : { "total_mem" : 10240 * 1024 - } + }, + "minContainerRam": 512 } configurations = { @@ -3566,7 +3581,8 @@ class TestHDP25StackAdvisor(TestCase): "ramPerContainer": 82240, "referenceNodeManagerHost" : { "total_mem" : 328960 * 1024 - } + }, + "minContainerRam": 1024 } configurations = { @@ -3748,7 +3764,8 @@ class TestHDP25StackAdvisor(TestCase): "ramPerContainer": 82240, "referenceNodeManagerHost" : { "total_mem" : 328960 * 1024 - } + }, + "minContainerRam": 1024 } configurations = { @@ -3880,7 +3897,8 @@ class TestHDP25StackAdvisor(TestCase): "ramPerContainer": 512, "referenceNodeManagerHost" : { "total_mem" : 10240 * 1024 - } + }, + "minContainerRam": 512 } configurations = { @@ -3925,7 +3943,8 @@ class TestHDP25StackAdvisor(TestCase): "amMemory": 2000, "reduceMemory": 2056, "containers": 3, - "ramPerContainer": 256 + "ramPerContainer": 256, + "minContainerRam": 256 } expected = { 'application-properties': { @@ -4660,7 +4679,8 @@ class TestHDP25StackAdvisor(TestCase): "cpu": 4, "referenceNodeManagerHost" : { "total_mem" : 328960 * 1024 - } + }, + "minContainerRam": 256 } hosts = { "items" : [
