Repository: ambari Updated Branches: refs/heads/branch-2.5 8a9d5fd52 -> 61ba63f21
Revert "AMBARI-19338 AM sizing for LLAP - min container size changes (dsen)" This reverts commit 2a85ec05e2c14e624bff5222a1291b0dcf12e83a. Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/61ba63f2 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/61ba63f2 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/61ba63f2 Branch: refs/heads/branch-2.5 Commit: 61ba63f21236026b3857c0c01c960d73eee70649 Parents: 8a9d5fd Author: Sumit Mohanty <[email protected]> Authored: Mon Jan 9 10:57:44 2017 -0800 Committer: Sumit Mohanty <[email protected]> Committed: Mon Jan 9 10:57:44 2017 -0800 ---------------------------------------------------------------------- .../stacks/HDP/2.0.6/services/stack_advisor.py | 15 ++--- .../stacks/HDP/2.2/services/stack_advisor.py | 2 +- .../stacks/2.0.6/common/test_stack_advisor.py | 43 ++++++-------- .../stacks/2.2/common/test_stack_advisor.py | 34 ++++------- .../stacks/2.3/common/test_stack_advisor.py | 2 +- .../stacks/2.5/common/test_stack_advisor.py | 60 +++++++------------- 6 files changed, 58 insertions(+), 98 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/61ba63f2/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py index f0c792c..105ffb6 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py @@ -125,7 +125,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor): if "referenceNodeManagerHost" in clusterData: nodemanagerMinRam = min(clusterData["referenceNodeManagerHost"]["total_mem"]/1024, nodemanagerMinRam) putYarnProperty('yarn.nodemanager.resource.memory-mb', int(round(min(clusterData['containers'] * clusterData['ramPerContainer'], nodemanagerMinRam)))) - putYarnProperty('yarn.scheduler.minimum-allocation-mb', int(clusterData['minContainerRam'])) + putYarnProperty('yarn.scheduler.minimum-allocation-mb', int(clusterData['ramPerContainer'])) putYarnProperty('yarn.scheduler.maximum-allocation-mb', int(configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"])) putYarnEnvProperty('min_user_id', self.get_system_min_uid()) @@ -1005,8 +1005,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor): cluster["minContainerSize"] = { - cluster["ram"] <= 3: 128, - 3 < cluster["ram"] <= 4: 256, + cluster["ram"] <= 4: 256, 4 < cluster["ram"] <= 8: 512, 8 < cluster["ram"] <= 24: 1024, 24 < cluster["ram"]: 2048 @@ -1017,22 +1016,20 @@ class HDP206StackAdvisor(DefaultStackAdvisor): totalAvailableRam -= cluster["hbaseRam"] cluster["totalAvailableRam"] = max(512, totalAvailableRam * 1024) '''containers = max(3, min (2*cores,min (1.8*DISKS,(Total available RAM) / MIN_CONTAINER_SIZE))))''' - cluster["containers"] = int(round(max(3, + cluster["containers"] = round(max(3, min(2 * cluster["cpu"], min(ceil(1.8 * cluster["disk"]), - cluster["totalAvailableRam"] / cluster["minContainerSize"]))))) + cluster["totalAvailableRam"] / cluster["minContainerSize"])))) '''ramPerContainers = max(2GB, RAM - reservedRam - hBaseRam) / containers''' - cluster["ramPerContainer"] = int(abs(cluster["totalAvailableRam"] / cluster["containers"])) + cluster["ramPerContainer"] = abs(cluster["totalAvailableRam"] / cluster["containers"]) '''If greater than 1GB, value will be in multiples of 512.''' if cluster["ramPerContainer"] > 1024: cluster["ramPerContainer"] = int(cluster["ramPerContainer"] / 512) * 512 - cluster["minContainerRam"] = min(1024, cluster["ramPerContainer"]) - cluster["mapMemory"] = int(cluster["ramPerContainer"]) cluster["reduceMemory"] = cluster["ramPerContainer"] - cluster["amMemory"] = min(cluster["mapMemory"], cluster["minContainerRam"]) + cluster["amMemory"] = max(cluster["mapMemory"], cluster["reduceMemory"]) return cluster http://git-wip-us.apache.org/repos/asf/ambari/blob/61ba63f2/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py index 63e4b20..cf7134e 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py @@ -818,7 +818,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor): yarnMaxAllocationSize = min(30 * int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]), int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"])) putTezProperty = self.putProperty(configurations, "tez-site", services) - putTezProperty("tez.am.resource.memory.mb", min(int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]), int(clusterData['amMemory']))) + putTezProperty("tez.am.resource.memory.mb", min(int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]), int(clusterData['amMemory']) * 2 if int(clusterData['amMemory']) < 3072 else int(clusterData['amMemory']))) taskResourceMemory = clusterData['mapMemory'] if clusterData['mapMemory'] > 2048 else int(clusterData['reduceMemory']) taskResourceMemory = min(clusterData['containers'] * clusterData['ramPerContainer'], taskResourceMemory, yarnMaxAllocationSize) http://git-wip-us.apache.org/repos/asf/ambari/blob/61ba63f2/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py index 22bf401..d781147 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py @@ -258,7 +258,7 @@ class TestHDP206StackAdvisor(TestCase): result = self.stackAdvisor.validateConfigurations(services, hosts) expectedItems = [ - {"message": "Value is less than the recommended default of 510", "level": "WARN"}, + {"message": "Value is less than the recommended default of 512", "level": "WARN"}, {'message': 'Value should be set for yarn.nodemanager.linux-container-executor.group', 'level': 'ERROR'}, {"message": "Value should be integer", "level": "ERROR"}, {"message": "Value should be set", "level": "ERROR"} @@ -419,7 +419,6 @@ class TestHDP206StackAdvisor(TestCase): "reservedRam": 2, "hbaseRam": 1, "minContainerSize": 512, - "minContainerRam" : 512, "totalAvailableRam": 3072, "containers": 6, "ramPerContainer": 512, @@ -488,16 +487,15 @@ class TestHDP206StackAdvisor(TestCase): }) expected["referenceHost"] = hosts["items"][1]["Hosts"] expected["referenceNodeManagerHost"] = hosts["items"][1]["Hosts"] - expected["amMemory"] = 128 - expected["containers"] = 4 + expected["amMemory"] = 170.66666666666666 + expected["containers"] = 3.0 expected["cpu"] = 4 expected["totalAvailableRam"] = 512 - expected["mapMemory"] = 128 - expected["minContainerSize"] = 128 - expected["reduceMemory"] = 128 - expected["minContainerRam"] = 128 + expected["mapMemory"] = 170 + expected["minContainerSize"] = 256 + expected["reduceMemory"] = 170.66666666666666 expected["ram"] = 0 - expected["ramPerContainer"] = 128 + expected["ramPerContainer"] = 170.66666666666666 expected["reservedRam"] = 1 result = self.stackAdvisor.getConfigurationClusterSummary(servicesList, hosts, components, services) self.assertEquals(result, expected) @@ -538,8 +536,7 @@ class TestHDP206StackAdvisor(TestCase): "ramPerContainer": 3072, "mapMemory": 3072, "reduceMemory": 3072, - "amMemory": 1024, - "minContainerRam": 1024, + "amMemory": 3072, "referenceHost": hosts["items"][0]["Hosts"] } @@ -595,8 +592,7 @@ class TestHDP206StackAdvisor(TestCase): services = {"configurations": configurations, "services": []} clusterData = { "containers" : 5, - "ramPerContainer": 256, - "minContainerRam": 256 + "ramPerContainer": 256 } expected = { "yarn-env": { @@ -623,8 +619,7 @@ class TestHDP206StackAdvisor(TestCase): clusterData = { "mapMemory": 567, "reduceMemory": 345.6666666666666, - "amMemory": 123.54, - "minContainerRam": 123.54 + "amMemory": 123.54 } expected = { "mapred-site": { @@ -660,14 +655,13 @@ class TestHDP206StackAdvisor(TestCase): "ram": 0, "reservedRam": 1, "hbaseRam": 1, - "minContainerSize": 128, + "minContainerSize": 256, "totalAvailableRam": 512, "containers": 3, - "ramPerContainer": 170, + "ramPerContainer": 170.66666666666666, "mapMemory": 170, - "reduceMemory": 170, - "amMemory": 170, - "minContainerRam" : 170 + "reduceMemory": 170.66666666666666, + "amMemory": 170.66666666666666 } self.assertEquals(result, expected) @@ -1177,8 +1171,7 @@ class TestHDP206StackAdvisor(TestCase): } clusterData = { - "totalAvailableRam": 2048, - "totalAvailableRam": 256 + "totalAvailableRam": 2048 } ambariHostName = socket.getfqdn() expected = {'oozie-env': @@ -1217,7 +1210,7 @@ class TestHDP206StackAdvisor(TestCase): 'namenode_heapsize': '1024', 'proxyuser_group': 'users', 'namenode_opt_maxnewsize': '256', - 'namenode_opt_newsize': '128'}}} + 'namenode_opt_newsize': '256'}}} # Apart from testing other HDFS recommendations, also tests 'hadoop.proxyuser.hive.hosts' config value which includes both HiveServer # and Hive Server Interactive Host (installed on different host compared to HiveServer). @@ -1352,7 +1345,7 @@ class TestHDP206StackAdvisor(TestCase): 'namenode_heapsize': '1024', 'proxyuser_group': 'users', 'namenode_opt_maxnewsize': '256', - 'namenode_opt_newsize': '128'}}} + 'namenode_opt_newsize': '256'}}} # Apart from testing other HDFS recommendations, also tests 'hadoop.proxyuser.hive.hosts' config value which includes both HiveServer # and Hive Server Interactive Host (installed on same host compared to HiveServer). @@ -1479,7 +1472,7 @@ class TestHDP206StackAdvisor(TestCase): 'namenode_heapsize': '1024', 'proxyuser_group': 'users', 'namenode_opt_maxnewsize': '256', - 'namenode_opt_newsize': '128'}}} + 'namenode_opt_newsize': '256'}}} self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, services3, hosts) self.assertEquals(configurations, expected) http://git-wip-us.apache.org/repos/asf/ambari/blob/61ba63f2/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py index 9d364ed..02be812 100644 --- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py @@ -86,11 +86,10 @@ class TestHDP22StackAdvisor(TestCase): } clusterData = { "mapMemory": 3000, - "amMemory": 2048, + "amMemory": 2000, "reduceMemory": 2056, "containers": 3, - "ramPerContainer": 256, - "minContainerRam": 256 + "ramPerContainer": 256 } expected = { "tez-site": { @@ -184,7 +183,6 @@ class TestHDP22StackAdvisor(TestCase): "amMemory": 3100, "reduceMemory": 2056, "containers": 3, - "minContainerRam": 256, "ramPerContainer": 256 } expected = { @@ -267,11 +265,10 @@ class TestHDP22StackAdvisor(TestCase): } clusterData = { "mapMemory": 760, - "amMemory": 2048, + "amMemory": 2000, "reduceMemory": 760, "containers": 3, - "ramPerContainer": 256, - "minContainerRam": 256 + "ramPerContainer": 256 } expected = { "tez-site": { @@ -887,8 +884,7 @@ class TestHDP22StackAdvisor(TestCase): clusterData = { "cpu": 4, "containers" : 5, - "ramPerContainer": 256, - "minContainerRam": 256 + "ramPerContainer": 256 } expected = { "yarn-env": { @@ -927,8 +923,7 @@ class TestHDP22StackAdvisor(TestCase): clusterData = { "cpu": 4, "containers": 5, - "ramPerContainer": 256, - "minContainerRam": 256 + "ramPerContainer": 256 } expected = { "spark-defaults": { @@ -960,8 +955,7 @@ class TestHDP22StackAdvisor(TestCase): clusterData = { "cpu": 4, "containers" : 5, - "ramPerContainer": 256, - "minContainerRam": 256 + "ramPerContainer": 256 } expected = { "yarn-env": { @@ -1222,8 +1216,7 @@ class TestHDP22StackAdvisor(TestCase): clusterData = { "cpu": 4, "containers" : 5, - "ramPerContainer": 256, - "minContainerRam": 256 + "ramPerContainer": 256 } services = { @@ -1270,8 +1263,7 @@ class TestHDP22StackAdvisor(TestCase): "amMemory": 2000, "reduceMemory": 2056, "containers": 3, - "ramPerContainer": 256, - "minContainerRam": 256 + "ramPerContainer": 256 } expected = { @@ -1762,7 +1754,6 @@ class TestHDP22StackAdvisor(TestCase): "containers" : 7, "ramPerContainer": 256, "totalAvailableRam": 4096, - "minContainerRam": 256 } expected = { "cluster-env": { @@ -2030,8 +2021,7 @@ class TestHDP22StackAdvisor(TestCase): clusterData = { "cpu": 4, "containers" : 5, - "ramPerContainer": 256, - "minContainerRam": 256 + "ramPerContainer": 256 } expected = { "yarn-env": { @@ -3813,7 +3803,7 @@ class TestHDP22StackAdvisor(TestCase): "properties": { "yarn.nodemanager.linux-container-executor.group": "hadoop", "yarn.nodemanager.resource.memory-mb": "43008", - "yarn.scheduler.minimum-allocation-mb": "1024", + "yarn.scheduler.minimum-allocation-mb": "14336", "yarn.scheduler.maximum-allocation-vcores": "4", "yarn.scheduler.minimum-allocation-vcores": "1", "yarn.nodemanager.resource.cpu-vcores": "4", @@ -3873,7 +3863,7 @@ class TestHDP22StackAdvisor(TestCase): "properties": { "yarn.nodemanager.linux-container-executor.group": "hadoop", "yarn.nodemanager.resource.memory-mb": "43008", - "yarn.scheduler.minimum-allocation-mb": "1024", + "yarn.scheduler.minimum-allocation-mb": "14336", "yarn.scheduler.maximum-allocation-vcores": "4", "yarn.scheduler.minimum-allocation-vcores": "1", "yarn.nodemanager.resource.cpu-vcores": "4", http://git-wip-us.apache.org/repos/asf/ambari/blob/61ba63f2/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py index 88fa6d0..378522e 100644 --- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py @@ -1165,7 +1165,7 @@ class TestHDP23StackAdvisor(TestCase): "tez.runtime.io.sort.mb": "202", "tez.session.am.dag.submit.timeout.secs": "600", "tez.runtime.unordered.output.buffer.size-mb": "57", - "tez.am.resource.memory.mb": "2000", + "tez.am.resource.memory.mb": "4000", "tez.queue.name": "queue2", } }, http://git-wip-us.apache.org/repos/asf/ambari/blob/61ba63f2/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py index d4a301c..a7ccdfe 100644 --- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py @@ -374,8 +374,7 @@ class TestHDP25StackAdvisor(TestCase): clusterData = { "cpu": 4, "containers": 5, - "ramPerContainer": 256, - "minContainerRam": 256 + "ramPerContainer": 256 } expected = { "spark2-defaults": { @@ -802,8 +801,7 @@ class TestHDP25StackAdvisor(TestCase): "ramPerContainer": 512, "referenceNodeManagerHost" : { "total_mem" : 10240 * 1024 - }, - "minContainerRam": 512 + } } @@ -971,8 +969,7 @@ class TestHDP25StackAdvisor(TestCase): "ramPerContainer": 512, "referenceNodeManagerHost" : { "total_mem" : 10240 * 1024 - }, - "minContainerRam": 512 + } } configurations = { @@ -1152,8 +1149,7 @@ class TestHDP25StackAdvisor(TestCase): "ramPerContainer": 512, "referenceNodeManagerHost" : { "total_mem" : 10240 * 1024 - }, - "minContainerRam": 512 + } } @@ -1352,8 +1348,7 @@ class TestHDP25StackAdvisor(TestCase): "ramPerContainer": 512, "referenceNodeManagerHost" : { "total_mem" : 10240 * 2048 - }, - "minContainerRam": 512 + } } @@ -1547,8 +1542,7 @@ class TestHDP25StackAdvisor(TestCase): "ramPerContainer": 1024, "referenceNodeManagerHost" : { "total_mem" : 51200 * 1024 - }, - "minContainerRam": 1024 + } } configurations = { @@ -1750,8 +1744,7 @@ class TestHDP25StackAdvisor(TestCase): "ramPerContainer": 2048, "referenceNodeManagerHost" : { "total_mem" : 40960 * 1024 - }, - "minContainerRam": 1024 + } } configurations = { @@ -1947,8 +1940,7 @@ class TestHDP25StackAdvisor(TestCase): "ramPerContainer": 341, "referenceNodeManagerHost" : { "total_mem" : 12288 * 1024 - }, - "minContainerRam": 341 + } } @@ -2146,8 +2138,7 @@ class TestHDP25StackAdvisor(TestCase): "ramPerContainer": 2048, "referenceNodeManagerHost" : { "total_mem" : 204800 * 1024 - }, - "minContainerRam": 1024 + } } configurations = { @@ -2349,8 +2340,7 @@ class TestHDP25StackAdvisor(TestCase): "ramPerContainer": 3072, "referenceNodeManagerHost" : { "total_mem" : 40960 * 1024 - }, - "minContainerRam": 1024 + } } @@ -2548,8 +2538,7 @@ class TestHDP25StackAdvisor(TestCase): "ramPerContainer": 341, "referenceNodeManagerHost" : { "total_mem" : 204800 * 1024 - }, - "minContainerRam": 341 + } } @@ -2746,8 +2735,7 @@ class TestHDP25StackAdvisor(TestCase): "ramPerContainer": 82240, "referenceNodeManagerHost" : { "total_mem" : 204800 * 1024 - }, - "minContainerRam": 1024 + } } configurations = { @@ -2939,8 +2927,7 @@ class TestHDP25StackAdvisor(TestCase): "ramPerContainer": 82240, "referenceNodeManagerHost" : { "total_mem" : 328960 * 1024 - }, - "minContainerRam": 1024 + } } @@ -3161,8 +3148,7 @@ class TestHDP25StackAdvisor(TestCase): "ramPerContainer": 512, "referenceNodeManagerHost" : { "total_mem" : 10240 * 1024 - }, - "minContainerRam": 512 + } } configurations = { @@ -3388,8 +3374,7 @@ class TestHDP25StackAdvisor(TestCase): "ramPerContainer": 512, "referenceNodeManagerHost" : { "total_mem" : 10240 * 1024 - }, - "minContainerRam": 512 + } } configurations = { @@ -3581,8 +3566,7 @@ class TestHDP25StackAdvisor(TestCase): "ramPerContainer": 82240, "referenceNodeManagerHost" : { "total_mem" : 328960 * 1024 - }, - "minContainerRam": 1024 + } } configurations = { @@ -3764,8 +3748,7 @@ class TestHDP25StackAdvisor(TestCase): "ramPerContainer": 82240, "referenceNodeManagerHost" : { "total_mem" : 328960 * 1024 - }, - "minContainerRam": 1024 + } } configurations = { @@ -3897,8 +3880,7 @@ class TestHDP25StackAdvisor(TestCase): "ramPerContainer": 512, "referenceNodeManagerHost" : { "total_mem" : 10240 * 1024 - }, - "minContainerRam": 512 + } } configurations = { @@ -3943,8 +3925,7 @@ class TestHDP25StackAdvisor(TestCase): "amMemory": 2000, "reduceMemory": 2056, "containers": 3, - "ramPerContainer": 256, - "minContainerRam": 256 + "ramPerContainer": 256 } expected = { 'application-properties': { @@ -4679,8 +4660,7 @@ class TestHDP25StackAdvisor(TestCase): "cpu": 4, "referenceNodeManagerHost" : { "total_mem" : 328960 * 1024 - }, - "minContainerRam": 256 + } } hosts = { "items" : [
