AMBARI-19578. Fix issues around modifying yarn min container size (smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e0d78edb Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e0d78edb Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e0d78edb Branch: refs/heads/branch-dev-patch-upgrade Commit: e0d78edb94cbbfe56b075487d241143f337a0a87 Parents: 6ccff93 Author: Sumit Mohanty <[email protected]> Authored: Tue Jan 17 20:59:33 2017 -0800 Committer: Sumit Mohanty <[email protected]> Committed: Tue Jan 17 21:01:21 2017 -0800 ---------------------------------------------------------------------- .../stacks/HDP/2.0.6/services/stack_advisor.py | 46 ++++++++++++++------ .../stacks/2.0.6/common/test_stack_advisor.py | 21 +++++---- .../stacks/2.2/common/test_stack_advisor.py | 20 ++++----- .../stacks/2.5/common/test_stack_advisor.py | 44 +++++++++---------- 4 files changed, 74 insertions(+), 57 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/e0d78edb/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py index 3596798..fc989fe 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py @@ -196,8 +196,26 @@ class HDP206StackAdvisor(DefaultStackAdvisor): nodemanagerMinRam = 1048576 # 1TB in mb if "referenceNodeManagerHost" in clusterData: nodemanagerMinRam = min(clusterData["referenceNodeManagerHost"]["total_mem"]/1024, nodemanagerMinRam) + + callContext = getCallContext(services) putYarnProperty('yarn.nodemanager.resource.memory-mb', int(round(min(clusterData['containers'] * clusterData['ramPerContainer'], nodemanagerMinRam)))) - putYarnProperty('yarn.scheduler.minimum-allocation-mb', int(clusterData['minContainerRam'])) + # read from the supplied config + #if 'recommendConfigurations' != callContext and \ + # "yarn-site" in services["configurations"] and \ + # "yarn.nodemanager.resource.memory-mb" in services["configurations"]["yarn-site"]["properties"]: + # putYarnProperty('yarn.nodemanager.resource.memory-mb', int(services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"])) + if 'recommendConfigurations' == callContext: + putYarnProperty('yarn.nodemanager.resource.memory-mb', int(round(min(clusterData['containers'] * clusterData['ramPerContainer'], nodemanagerMinRam)))) + else: + # read from the supplied config + if "yarn-site" in services["configurations"] and "yarn.nodemanager.resource.memory-mb" in services["configurations"]["yarn-site"]["properties"]: + putYarnProperty('yarn.nodemanager.resource.memory-mb', int(services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"])) + else: + putYarnProperty('yarn.nodemanager.resource.memory-mb', int(round(min(clusterData['containers'] * clusterData['ramPerContainer'], nodemanagerMinRam)))) + pass + pass + + putYarnProperty('yarn.scheduler.minimum-allocation-mb', int(clusterData['yarnMinContainerSize'])) putYarnProperty('yarn.scheduler.maximum-allocation-mb', int(configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"])) putYarnEnvProperty('min_user_id', self.get_system_min_uid()) @@ -887,7 +905,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor): cluster["totalAvailableRam"] = max(512, totalAvailableRam * 1024) Logger.info("Memory for YARN apps - cluster[totalAvailableRam]: " + str(cluster["totalAvailableRam"])) - suggestedMinContainerRam = 1024 + suggestedMinContainerRam = 1024 # new smaller value for YARN min container callContext = getCallContext(services) if services: # its never None but some unit tests pass it as None @@ -899,14 +917,14 @@ class HDP206StackAdvisor(DefaultStackAdvisor): str(services["configurations"]["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]).isdigit(): Logger.info("Using user provided yarn.scheduler.minimum-allocation-mb = " + str(services["configurations"]["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])) - cluster["minContainerRam"] = int(services["configurations"]["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]) - Logger.info("Minimum ram per container due to user input - cluster[minContainerRam]: " + str(cluster["minContainerRam"])) - if cluster["minContainerRam"] > cluster["totalAvailableRam"]: - cluster["minContainerRam"] = cluster["totalAvailableRam"] - Logger.info("Minimum ram per container after checking against limit - cluster[minContainerRam]: " + str(cluster["minContainerRam"])) + cluster["yarnMinContainerSize"] = int(services["configurations"]["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]) + Logger.info("Minimum ram per container due to user input - cluster[yarnMinContainerSize]: " + str(cluster["yarnMinContainerSize"])) + if cluster["yarnMinContainerSize"] > cluster["totalAvailableRam"]: + cluster["yarnMinContainerSize"] = cluster["totalAvailableRam"] + Logger.info("Minimum ram per container after checking against limit - cluster[yarnMinContainerSize]: " + str(cluster["yarnMinContainerSize"])) pass - cluster["minContainerSize"] = cluster["minContainerRam"] - suggestedMinContainerRam = cluster["minContainerRam"] + cluster["minContainerSize"] = cluster["yarnMinContainerSize"] # set to what user has suggested as YARN min container size + suggestedMinContainerRam = cluster["yarnMinContainerSize"] pass pass pass @@ -925,19 +943,19 @@ class HDP206StackAdvisor(DefaultStackAdvisor): pass cluster["ramPerContainer"] = int(abs(cluster["totalAvailableRam"] / cluster["containers"])) - cluster["minContainerRam"] = min(suggestedMinContainerRam, cluster["ramPerContainer"]) + cluster["yarnMinContainerSize"] = min(suggestedMinContainerRam, cluster["ramPerContainer"]) Logger.info("Ram per containers before normalization - cluster[ramPerContainer]: " + str(cluster["ramPerContainer"])) - '''If greater than cluster["minContainerRam"], value will be in multiples of cluster["minContainerRam"]''' - if cluster["ramPerContainer"] > cluster["minContainerRam"]: - cluster["ramPerContainer"] = int(cluster["ramPerContainer"] / cluster["minContainerRam"]) * cluster["minContainerRam"] + '''If greater than cluster["yarnMinContainerSize"], value will be in multiples of cluster["yarnMinContainerSize"]''' + if cluster["ramPerContainer"] > cluster["yarnMinContainerSize"]: + cluster["ramPerContainer"] = int(cluster["ramPerContainer"] / cluster["yarnMinContainerSize"]) * cluster["yarnMinContainerSize"] cluster["mapMemory"] = int(cluster["ramPerContainer"]) cluster["reduceMemory"] = cluster["ramPerContainer"] cluster["amMemory"] = max(cluster["mapMemory"], cluster["reduceMemory"]) - Logger.info("Min container size - cluster[minContainerRam]: " + str(cluster["minContainerRam"])) + Logger.info("Min container size - cluster[yarnMinContainerSize]: " + str(cluster["yarnMinContainerSize"])) Logger.info("Available memory for map - cluster[mapMemory]: " + str(cluster["mapMemory"])) Logger.info("Available memory for reduce - cluster[reduceMemory]: " + str(cluster["reduceMemory"])) Logger.info("Available memory for am - cluster[amMemory]: " + str(cluster["amMemory"])) http://git-wip-us.apache.org/repos/asf/ambari/blob/e0d78edb/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py index a486fb3..2a97b75 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py @@ -258,7 +258,6 @@ class TestHDP206StackAdvisor(TestCase): result = self.stackAdvisor.validateConfigurations(services, hosts) expectedItems = [ - {"message": "Value is less than the recommended default of 510", "level": "WARN"}, {'message': 'Value should be set for yarn.nodemanager.linux-container-executor.group', 'level': 'ERROR'}, {"message": "Value should be integer", "level": "ERROR"}, {"message": "Value should be set", "level": "ERROR"} @@ -472,7 +471,7 @@ class TestHDP206StackAdvisor(TestCase): "reservedRam": 4, "hbaseRam": 4, "minContainerSize": 1024, - "minContainerRam" : 1024, + "yarnMinContainerSize" : 1024, "totalAvailableRam": 15360, "containers": 4, "ramPerContainer": 3072, @@ -584,7 +583,7 @@ class TestHDP206StackAdvisor(TestCase): "reservedRam": 4, "hbaseRam": 4, "minContainerSize": 2048, - "minContainerRam" : 2048, + "yarnMinContainerSize" : 2048, "totalAvailableRam": 15360, "containers": 4, "ramPerContainer": 2048, @@ -649,7 +648,7 @@ class TestHDP206StackAdvisor(TestCase): "reservedRam": 4, "hbaseRam": 4, "minContainerSize": 2048, - "minContainerRam" : 2048, + "yarnMinContainerSize" : 2048, "totalAvailableRam": 15360, "containers": 4, "ramPerContainer": 2048, @@ -714,7 +713,7 @@ class TestHDP206StackAdvisor(TestCase): "reservedRam": 4, "hbaseRam": 4, "minContainerSize": 4096, - "minContainerRam" : 4096, + "yarnMinContainerSize" : 4096, "totalAvailableRam": 15360, "containers": 3, "ramPerContainer": 4096, @@ -761,7 +760,7 @@ class TestHDP206StackAdvisor(TestCase): "reservedRam": 2, "hbaseRam": 1, "minContainerSize": 512, - "minContainerRam" : 512, + "yarnMinContainerSize" : 512, "totalAvailableRam": 3072, "containers": 6, "ramPerContainer": 512, @@ -837,7 +836,7 @@ class TestHDP206StackAdvisor(TestCase): expected["mapMemory"] = 128 expected["minContainerSize"] = 128 expected["reduceMemory"] = 128 - expected["minContainerRam"] = 128 + expected["yarnMinContainerSize"] = 128 expected["ram"] = 0 expected["ramPerContainer"] = 128 expected["reservedRam"] = 1 @@ -881,7 +880,7 @@ class TestHDP206StackAdvisor(TestCase): "mapMemory": 3072, "reduceMemory": 3072, "amMemory": 3072, - "minContainerRam": 1024, + "yarnMinContainerSize": 1024, "referenceHost": hosts["items"][0]["Hosts"] } @@ -938,7 +937,7 @@ class TestHDP206StackAdvisor(TestCase): clusterData = { "containers" : 5, "ramPerContainer": 256, - "minContainerRam": 256 + "yarnMinContainerSize": 256 } expected = { "yarn-env": { @@ -966,7 +965,7 @@ class TestHDP206StackAdvisor(TestCase): "mapMemory": 567, "reduceMemory": 345.6666666666666, "amMemory": 123.54, - "minContainerRam": 123.54 + "yarnMinContainerSize": 123.54 } expected = { "mapred-site": { @@ -1009,7 +1008,7 @@ class TestHDP206StackAdvisor(TestCase): "mapMemory": 170, "reduceMemory": 170, "amMemory": 170, - "minContainerRam" : 170 + "yarnMinContainerSize" : 170 } self.assertEquals(result, expected) http://git-wip-us.apache.org/repos/asf/ambari/blob/e0d78edb/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py index 8a82c99..a26b661 100644 --- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py @@ -90,7 +90,7 @@ class TestHDP22StackAdvisor(TestCase): "reduceMemory": 2056, "containers": 3, "ramPerContainer": 256, - "minContainerRam": 256 + "yarnMinContainerSize": 256 } expected = { "tez-site": { @@ -184,7 +184,7 @@ class TestHDP22StackAdvisor(TestCase): "amMemory": 3100, "reduceMemory": 2056, "containers": 3, - "minContainerRam": 256, + "yarnMinContainerSize": 256, "ramPerContainer": 256 } expected = { @@ -271,7 +271,7 @@ class TestHDP22StackAdvisor(TestCase): "reduceMemory": 760, "containers": 3, "ramPerContainer": 256, - "minContainerRam": 256 + "yarnMinContainerSize": 256 } expected = { "tez-site": { @@ -888,7 +888,7 @@ class TestHDP22StackAdvisor(TestCase): "cpu": 4, "containers" : 5, "ramPerContainer": 256, - "minContainerRam": 256 + "yarnMinContainerSize": 256 } expected = { "yarn-env": { @@ -928,7 +928,7 @@ class TestHDP22StackAdvisor(TestCase): "cpu": 4, "containers": 5, "ramPerContainer": 256, - "minContainerRam": 256 + "yarnMinContainerSize": 256 } expected = { "spark-defaults": { @@ -961,7 +961,7 @@ class TestHDP22StackAdvisor(TestCase): "cpu": 4, "containers" : 5, "ramPerContainer": 256, - "minContainerRam": 256 + "yarnMinContainerSize": 256 } expected = { "yarn-env": { @@ -1223,7 +1223,7 @@ class TestHDP22StackAdvisor(TestCase): "cpu": 4, "containers" : 5, "ramPerContainer": 256, - "minContainerRam": 256 + "yarnMinContainerSize": 256 } services = { @@ -1271,7 +1271,7 @@ class TestHDP22StackAdvisor(TestCase): "reduceMemory": 2056, "containers": 3, "ramPerContainer": 256, - "minContainerRam": 256 + "yarnMinContainerSize": 256 } expected = { @@ -1762,7 +1762,7 @@ class TestHDP22StackAdvisor(TestCase): "containers" : 7, "ramPerContainer": 256, "totalAvailableRam": 4096, - "minContainerRam": 256 + "yarnMinContainerSize": 256 } expected = { "cluster-env": { @@ -2031,7 +2031,7 @@ class TestHDP22StackAdvisor(TestCase): "cpu": 4, "containers" : 5, "ramPerContainer": 256, - "minContainerRam": 256 + "yarnMinContainerSize": 256 } expected = { "yarn-env": { http://git-wip-us.apache.org/repos/asf/ambari/blob/e0d78edb/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py index 790c6a7..ad962fd 100644 --- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py @@ -375,7 +375,7 @@ class TestHDP25StackAdvisor(TestCase): "cpu": 4, "containers": 5, "ramPerContainer": 256, - "minContainerRam": 256 + "yarnMinContainerSize": 256 } expected = { "spark2-defaults": { @@ -803,7 +803,7 @@ class TestHDP25StackAdvisor(TestCase): "referenceNodeManagerHost" : { "total_mem" : 10240 * 1024 }, - "minContainerRam": 512 + "yarnMinContainerSize": 512 } @@ -972,7 +972,7 @@ class TestHDP25StackAdvisor(TestCase): "referenceNodeManagerHost" : { "total_mem" : 10240 * 1024 }, - "minContainerRam": 512 + "yarnMinContainerSize": 512 } configurations = { @@ -1153,7 +1153,7 @@ class TestHDP25StackAdvisor(TestCase): "referenceNodeManagerHost" : { "total_mem" : 10240 * 1024 }, - "minContainerRam": 512 + "yarnMinContainerSize": 512 } @@ -1353,7 +1353,7 @@ class TestHDP25StackAdvisor(TestCase): "referenceNodeManagerHost" : { "total_mem" : 10240 * 2048 }, - "minContainerRam": 512 + "yarnMinContainerSize": 512 } @@ -1546,7 +1546,7 @@ class TestHDP25StackAdvisor(TestCase): "referenceNodeManagerHost" : { "total_mem" : 51200 * 1024 }, - "minContainerRam": 1024 + "yarnMinContainerSize": 1024 } configurations = { @@ -1747,7 +1747,7 @@ class TestHDP25StackAdvisor(TestCase): "referenceNodeManagerHost" : { "total_mem" : 40960 * 1024 }, - "minContainerRam": 1024 + "yarnMinContainerSize": 1024 } configurations = { @@ -1942,7 +1942,7 @@ class TestHDP25StackAdvisor(TestCase): "referenceNodeManagerHost" : { "total_mem" : 12288 * 1024 }, - "minContainerRam": 341 + "yarnMinContainerSize": 341 } @@ -2140,7 +2140,7 @@ class TestHDP25StackAdvisor(TestCase): "referenceNodeManagerHost" : { "total_mem" : 204800 * 1024 }, - "minContainerRam": 1024 + "yarnMinContainerSize": 1024 } configurations = { @@ -2341,7 +2341,7 @@ class TestHDP25StackAdvisor(TestCase): "referenceNodeManagerHost" : { "total_mem" : 40960 * 1024 }, - "minContainerRam": 1024 + "yarnMinContainerSize": 1024 } @@ -2538,7 +2538,7 @@ class TestHDP25StackAdvisor(TestCase): "referenceNodeManagerHost" : { "total_mem" : 204800 * 1024 }, - "minContainerRam": 341 + "yarnMinContainerSize": 341 } @@ -2734,7 +2734,7 @@ class TestHDP25StackAdvisor(TestCase): "referenceNodeManagerHost" : { "total_mem" : 204800 * 1024 }, - "minContainerRam": 1024 + "yarnMinContainerSize": 1024 } configurations = { @@ -2925,7 +2925,7 @@ class TestHDP25StackAdvisor(TestCase): "referenceNodeManagerHost" : { "total_mem" : 328960 * 1024 }, - "minContainerRam": 1024 + "yarnMinContainerSize": 1024 } @@ -3145,7 +3145,7 @@ class TestHDP25StackAdvisor(TestCase): "referenceNodeManagerHost" : { "total_mem" : 10240 * 1024 }, - "minContainerRam": 512 + "yarnMinContainerSize": 512 } configurations = { @@ -3159,12 +3159,12 @@ class TestHDP25StackAdvisor(TestCase): self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.server2.tez.sessions.per.default.queue'], {'maximum': '1.0', 'minimum': '1'}) self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes'], 3) - self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '9728') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '204288') self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors'], '3') self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.threadpool.size'], '3') - self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size'], '3584') + self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size'], '198144') self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.enabled'], 'true') self.assertEqual(configurations['hive-interactive-env']['properties']['llap_heap_size'], '4915') @@ -3370,7 +3370,7 @@ class TestHDP25StackAdvisor(TestCase): "referenceNodeManagerHost" : { "total_mem" : 10240 * 1024 }, - "minContainerRam": 512 + "yarnMinContainerSize": 512 } configurations = { @@ -3563,7 +3563,7 @@ class TestHDP25StackAdvisor(TestCase): "referenceNodeManagerHost" : { "total_mem" : 328960 * 1024 }, - "minContainerRam": 1024 + "yarnMinContainerSize": 1024 } configurations = { @@ -3746,7 +3746,7 @@ class TestHDP25StackAdvisor(TestCase): "referenceNodeManagerHost" : { "total_mem" : 328960 * 1024 }, - "minContainerRam": 1024 + "yarnMinContainerSize": 1024 } configurations = { @@ -3877,7 +3877,7 @@ class TestHDP25StackAdvisor(TestCase): "referenceNodeManagerHost" : { "total_mem" : 10240 * 1024 }, - "minContainerRam": 512 + "yarnMinContainerSize": 512 } configurations = { @@ -3923,7 +3923,7 @@ class TestHDP25StackAdvisor(TestCase): "reduceMemory": 2056, "containers": 3, "ramPerContainer": 256, - "minContainerRam": 256 + "yarnMinContainerSize": 256 } expected = { 'application-properties': { @@ -4659,7 +4659,7 @@ class TestHDP25StackAdvisor(TestCase): "referenceNodeManagerHost" : { "total_mem" : 328960 * 1024 }, - "minContainerRam": 256 + "yarnMinContainerSize": 256 } hosts = { "items" : [
