Repository: ambari Updated Branches: refs/heads/branch-2.4 dd8dfdd22 -> b0c77fb1f
AMBARI-17764 Oozie service check failed after deployment (dsen) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b0c77fb1 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b0c77fb1 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b0c77fb1 Branch: refs/heads/branch-2.4 Commit: b0c77fb1ff2dd747006be670d4d1985ff91cff58 Parents: dd8dfdd Author: Dmytro Sen <[email protected]> Authored: Mon Jul 18 13:00:25 2016 +0300 Committer: Dmytro Sen <[email protected]> Committed: Mon Jul 18 13:01:35 2016 +0300 ---------------------------------------------------------------------- .../stacks/HDP/2.0.6/services/stack_advisor.py | 34 +++++++++++++------- .../stacks/2.0.6/common/test_stack_advisor.py | 10 +++--- .../stacks/2.1/common/test_stack_advisor.py | 4 +-- .../stacks/2.2/common/test_stack_advisor.py | 13 ++++---- 4 files changed, 36 insertions(+), 25 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/b0c77fb1/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py index ee37fe0..5806e52 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py @@ -307,22 +307,32 @@ class HDP206StackAdvisor(DefaultStackAdvisor): putHDFSSiteProperty('dfs.datanode.data.dir', dataDirs) else: dataDirs = hdfsSiteProperties['dfs.datanode.data.dir'].split(",") - #dfs.datanode.du.reserved should be set to 10-15% of volume size - mountPoints = [] - mountPointDiskAvailableSpace = [] #kBytes + + # dfs.datanode.du.reserved should be set to 10-15% of volume size + # For each host selects maximum size of the volume. Then gets minimum for all hosts. + # This ensures that each host will have at least one data dir with available space. + reservedSizeRecommendation = 0l #kBytes for host in hosts["items"]: + mountPoints = [] + mountPointDiskAvailableSpace = [] #kBytes for diskInfo in host["Hosts"]["disk_info"]: mountPoints.append(diskInfo["mountpoint"]) mountPointDiskAvailableSpace.append(long(diskInfo["size"])) - maxFreeVolumeSize = 0l #kBytes - for dataDir in dataDirs: - mp = getMountPointForDir(dataDir, mountPoints) - for i in range(len(mountPoints)): - if mp == mountPoints[i]: - if mountPointDiskAvailableSpace[i] > maxFreeVolumeSize: - maxFreeVolumeSize = mountPointDiskAvailableSpace[i] - - putHDFSSiteProperty('dfs.datanode.du.reserved', maxFreeVolumeSize * 1024 / 8) #Bytes + + maxFreeVolumeSizeForHost = 0l #kBytes + for dataDir in dataDirs: + mp = getMountPointForDir(dataDir, mountPoints) + for i in range(len(mountPoints)): + if mp == mountPoints[i]: + if mountPointDiskAvailableSpace[i] > maxFreeVolumeSizeForHost: + maxFreeVolumeSizeForHost = mountPointDiskAvailableSpace[i] + + if not reservedSizeRecommendation or maxFreeVolumeSizeForHost and maxFreeVolumeSizeForHost < reservedSizeRecommendation: + reservedSizeRecommendation = maxFreeVolumeSizeForHost + + if reservedSizeRecommendation: + reservedSizeRecommendation = max(reservedSizeRecommendation * 1024 / 8, 1073741824) # At least 1Gb is reserved + putHDFSSiteProperty('dfs.datanode.du.reserved', reservedSizeRecommendation) #Bytes # recommendations for "hadoop.proxyuser.*.hosts", "hadoop.proxyuser.*.groups" properties in core-site self.recommendHadoopProxyUsers(configurations, services, hosts) http://git-wip-us.apache.org/repos/asf/ambari/blob/b0c77fb1/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py index 1feeed2..380861f 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py @@ -983,7 +983,7 @@ class TestHDP206StackAdvisor(TestCase): "rack_info": "/default-rack", "total_mem": 2097152, "disk_info": [{ - "size": '8', + "size": '80000000', "mountpoint": "/" }] } @@ -1000,7 +1000,7 @@ class TestHDP206StackAdvisor(TestCase): "rack_info": "/default-rack", "total_mem": 1048576, "disk_info": [{ - "size": '8', + "size": '800000000', "mountpoint": "/" }] } @@ -1106,7 +1106,7 @@ class TestHDP206StackAdvisor(TestCase): 'hdfs-site': {'properties': {'dfs.datanode.data.dir': '/hadoop/hdfs/data', - 'dfs.datanode.du.reserved': '1024'}}, + 'dfs.datanode.du.reserved': '10240000000'}}, 'hive-env': {'properties': {'hive_user': 'hive', @@ -1162,7 +1162,7 @@ class TestHDP206StackAdvisor(TestCase): 'hdfs-site': {'properties': {'dfs.datanode.data.dir': '/hadoop/hdfs/data', - 'dfs.datanode.du.reserved': '1024'}}, + 'dfs.datanode.du.reserved': '10240000000'}}, 'hadoop-env': {'properties': {'hdfs_user': 'hdfs1', @@ -1182,7 +1182,7 @@ class TestHDP206StackAdvisor(TestCase): expected["hdfs-site"] = { 'properties': { 'dfs.datanode.data.dir': '/hadoop/hdfs/data', - 'dfs.datanode.du.reserved': '1024', + 'dfs.datanode.du.reserved': '10240000000', 'dfs.internal.nameservices': 'mycluster', 'dfs.ha.namenodes.mycluster': 'nn1,nn2' }, http://git-wip-us.apache.org/repos/asf/ambari/blob/b0c77fb1/ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py index 1232d97..e174461 100644 --- a/ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py @@ -364,7 +364,7 @@ class TestHDP21StackAdvisor(TestCase): { "Hosts": { "disk_info": [{ - "size": '8', + "size": '80000000', "mountpoint": "/" }] } @@ -404,7 +404,7 @@ class TestHDP21StackAdvisor(TestCase): "hdfs-site": { "properties": { 'dfs.datanode.data.dir': '/hadoop/hdfs/data', - 'dfs.datanode.du.reserved': '1024' + 'dfs.datanode.du.reserved': '10240000000' } } } http://git-wip-us.apache.org/repos/asf/ambari/blob/b0c77fb1/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py index c6c27c2..5f272b5 100644 --- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py @@ -2968,7 +2968,7 @@ class TestHDP22StackAdvisor(TestCase): }, 'hdfs-site': { 'properties': { - 'dfs.datanode.du.reserved': '1024', + 'dfs.datanode.du.reserved': '10240000000', 'dfs.datanode.max.transfer.threads': '16384', 'dfs.namenode.safemode.threshold-pct': '1.000', 'dfs.datanode.failed.volumes.tolerated': '1', @@ -3086,6 +3086,7 @@ class TestHDP22StackAdvisor(TestCase): "configurations": configurations, "ambari-server-properties": {"ambari-server.user":"ambari_user"} } + # One host has bigger volume size. Minimum should be used for the calculations of dfs.datanode.du.reserved hosts = { "items" : [ { @@ -3100,7 +3101,7 @@ class TestHDP22StackAdvisor(TestCase): "rack_info" : "/default-rack", "total_mem" : 2097152, "disk_info": [{ - "size": '8', + "size": '80000000', "mountpoint": "/" }] } @@ -3117,7 +3118,7 @@ class TestHDP22StackAdvisor(TestCase): "rack_info" : "/default-rack", "total_mem" : 10485760, "disk_info": [{ - "size": '8', + "size": '80000000000', "mountpoint": "/" }] } @@ -3145,7 +3146,7 @@ class TestHDP22StackAdvisor(TestCase): "rack_info" : "/default-rack", "total_mem" : 2097152, "disk_info": [{ - "size": '8', + "size": '80000000', "mountpoint": "/" }] } @@ -3172,7 +3173,7 @@ class TestHDP22StackAdvisor(TestCase): "rack_info" : "/default-rack", "total_mem" : 2097152, "disk_info": [{ - "size": '8', + "size": '80000000', "mountpoint": "/" }] } @@ -3201,7 +3202,7 @@ class TestHDP22StackAdvisor(TestCase): "rack_info" : "/default-rack", "total_mem" : 2097152, "disk_info": [{ - "size": '8', + "size": '80000000', "mountpoint": "/" }] }
