Repository: ambari Updated Branches: refs/heads/branch-2.1 158f2e13d -> 73b7d7273 refs/heads/trunk 4fd35351b -> 47f925eff
AMBARI-11993. Tez, YARN, MR2 checks failed on 2.2 cluster (aonishuk) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/47f925ef Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/47f925ef Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/47f925ef Branch: refs/heads/trunk Commit: 47f925efff2ddfa1d5483aeaf8b25d042ea43879 Parents: 4fd3535 Author: Andrew Onishuk <[email protected]> Authored: Thu Jun 18 18:01:56 2015 +0300 Committer: Andrew Onishuk <[email protected]> Committed: Thu Jun 18 18:01:56 2015 +0300 ---------------------------------------------------------------------- .../stacks/HDP/2.0.6/services/stack_advisor.py | 2 +- .../stacks/HDP/2.2/services/stack_advisor.py | 2 +- .../stacks/HDPWIN/2.1/services/stack_advisor.py | 2 +- .../stacks/HDPWIN/2.2/services/stack_advisor.py | 4 +-- .../stacks/2.0.6/common/test_stack_advisor.py | 26 ++++++++++---------- .../stacks/2.2/common/test_stack_advisor.py | 3 ++- 6 files changed, 20 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/47f925ef/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py index d9da407..f12524a 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py @@ -306,7 +306,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor): totalAvailableRam = cluster["ram"] - cluster["reservedRam"] if cluster["hBaseInstalled"]: totalAvailableRam -= cluster["hbaseRam"] - cluster["totalAvailableRam"] = max(2048, totalAvailableRam * 1024) + cluster["totalAvailableRam"] = max(512, totalAvailableRam * 1024) '''containers = max(3, min (2*cores,min (1.8*DISKS,(Total available RAM) / MIN_CONTAINER_SIZE))))''' cluster["containers"] = round(max(3, min(2 * cluster["cpu"], http://git-wip-us.apache.org/repos/asf/ambari/blob/47f925ef/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py index bdbe9b1..9de8421 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py @@ -638,7 +638,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor): min_mapreduce_map_memory_mb = 0 min_mapreduce_reduce_memory_mb = 0 min_mapreduce_map_java_opts = 0 - if ("PIG" in servicesList): + if ("PIG" in servicesList) and clusterData["totalAvailableRam"] >= 4096: min_mapreduce_map_memory_mb = 1536 min_mapreduce_reduce_memory_mb = 1536 min_mapreduce_map_java_opts = 1024 http://git-wip-us.apache.org/repos/asf/ambari/blob/47f925ef/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py index b8334e3..75b427e 100644 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py @@ -292,7 +292,7 @@ class HDPWIN21StackAdvisor(DefaultStackAdvisor): totalAvailableRam = cluster["ram"] - cluster["reservedRam"] if cluster["hBaseInstalled"]: totalAvailableRam -= cluster["hbaseRam"] - cluster["totalAvailableRam"] = max(2048, totalAvailableRam * 1024) + cluster["totalAvailableRam"] = max(512, totalAvailableRam * 1024) '''containers = max(3, min (2*cores,min (1.8*DISKS,(Total available RAM) / MIN_CONTAINER_SIZE))))''' cluster["containers"] = round(max(3, min(2 * cluster["cpu"], http://git-wip-us.apache.org/repos/asf/ambari/blob/47f925ef/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py index 6d527ea..c6941f0 100644 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py @@ -236,7 +236,7 @@ class HDPWIN22StackAdvisor(HDPWIN21StackAdvisor): min_mapreduce_map_memory_mb = 0 min_mapreduce_reduce_memory_mb = 0 min_mapreduce_map_java_opts = 0 - if ("PIG" in servicesList): + if ("PIG" in servicesList) and clusterData["totalAvailableRam"] >= 8192: min_mapreduce_map_memory_mb = 1536 min_mapreduce_reduce_memory_mb = 1536 min_mapreduce_map_java_opts = 1024 @@ -641,7 +641,7 @@ class HDPWIN22StackAdvisor(HDPWIN21StackAdvisor): servicesList = [service["StackServices"]["service_name"] for service in services["services"]] min_mapreduce_map_memory_mb = 0 min_mapreduce_map_java_opts = 0 - if ("PIG" in servicesList): + if ("PIG" in servicesList) and clusterData["totalAvailableRam"] >= 8192: min_mapreduce_map_memory_mb = 1500 min_mapreduce_map_java_opts = 1024 putMapredProperty('mapreduce.map.memory.mb', max(min_mapreduce_map_memory_mb, int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]))) http://git-wip-us.apache.org/repos/asf/ambari/blob/47f925ef/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py index 00622a7..65fa78a 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py @@ -36,7 +36,7 @@ class TestHDP206StackAdvisor(TestCase): stack_advisor_impl = imp.load_module('stack_advisor_impl', fp, hdp206StackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE)) clazz = getattr(stack_advisor_impl, hdp206StackAdvisorClassName) self.stackAdvisor = clazz() - + self.maxDiff = None # substitute method in the instance self.get_system_min_uid_real = self.stackAdvisor.get_system_min_uid self.stackAdvisor.get_system_min_uid = self.get_system_min_uid_magic @@ -201,7 +201,7 @@ class TestHDP206StackAdvisor(TestCase): result = self.stackAdvisor.validateConfigurations(services, hosts) expectedItems = [ - {"message": "Value is less than the recommended default of 2048", "level": "WARN"}, + {"message": "Value is less than the recommended default of 512", "level": "WARN"}, {"message": "Value should be integer", "level": "ERROR"}, {"message": "Value should be set", "level": "ERROR"} ] @@ -407,15 +407,15 @@ class TestHDP206StackAdvisor(TestCase): }) expected["referenceHost"] = hosts["items"][1]["Hosts"] expected["referenceNodeManagerHost"] = hosts["items"][1]["Hosts"] - expected["amMemory"] = 256 - expected["containers"] = 8 + expected["amMemory"] = 170.66666666666666 + expected["containers"] = 3.0 expected["cpu"] = 4 - expected["totalAvailableRam"] = 2048 - expected["mapMemory"] = 256 + expected["totalAvailableRam"] = 512 + expected["mapMemory"] = 170 expected["minContainerSize"] = 256 - expected["reduceMemory"] = 256 + expected["reduceMemory"] = 170.66666666666666 expected["ram"] = 0 - expected["ramPerContainer"] = 256 + expected["ramPerContainer"] = 170.66666666666666 expected["reservedRam"] = 1 result = self.stackAdvisor.getConfigurationClusterSummary(servicesList, hosts, components, services) self.assertEquals(result, expected) @@ -529,12 +529,12 @@ class TestHDP206StackAdvisor(TestCase): "reservedRam": 1, "hbaseRam": 1, "minContainerSize": 256, - "totalAvailableRam": 2048, + "totalAvailableRam": 512, "containers": 3, - "ramPerContainer": 682.6666666666666, - "mapMemory": 682, - "reduceMemory": 682.6666666666666, - "amMemory": 682.6666666666666 + "ramPerContainer": 170.66666666666666, + "mapMemory": 170, + "reduceMemory": 170.66666666666666, + "amMemory": 170.66666666666666 } self.assertEquals(result, expected) http://git-wip-us.apache.org/repos/asf/ambari/blob/47f925ef/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py index a984a7a..78acce3 100644 --- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py @@ -1265,7 +1265,8 @@ class TestHDP22StackAdvisor(TestCase): clusterData = { "cpu": 4, "containers" : 5, - "ramPerContainer": 256 + "ramPerContainer": 256, + "totalAvailableRam": 4096, } expected = { "yarn-env": {
