HDFS-9902. Support different values of dfs.datanode.du.reserved per storage type. (Contributed by Brahma Reddy Battula)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6d77d6ea Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6d77d6ea Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6d77d6ea Branch: refs/heads/HDFS-7240 Commit: 6d77d6eab7790ed7ae2cad5b327ba5d1deb485db Parents: ed54f5f Author: Arpit Agarwal <a...@apache.org> Authored: Tue May 3 16:52:43 2016 -0700 Committer: Arpit Agarwal <a...@apache.org> Committed: Tue May 3 16:52:43 2016 -0700 ---------------------------------------------------------------------- .../datanode/fsdataset/impl/FsVolumeImpl.java | 6 ++-- .../src/main/resources/hdfs-default.xml | 5 +++ .../fsdataset/impl/TestFsVolumeList.java | 36 +++++++++++++++++++- 3 files changed, 44 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d77d6ea/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java index 73514b6..68e2537 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java @@ -58,6 +58,7 @@ import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.util.CloseableReferenceCount; import org.apache.hadoop.util.DiskChecker.DiskErrorException; +import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Time; import org.apache.hadoop.util.Timer; import org.codehaus.jackson.annotate.JsonProperty; @@ -118,9 +119,10 @@ public class FsVolumeImpl implements FsVolumeSpi { Configuration conf, StorageType storageType) throws IOException { this.dataset = dataset; this.storageID = storageID; - this.reserved = conf.getLong( + this.reserved = conf.getLong(DFSConfigKeys.DFS_DATANODE_DU_RESERVED_KEY + + "." + StringUtils.toLowerCase(storageType.toString()), conf.getLong( DFSConfigKeys.DFS_DATANODE_DU_RESERVED_KEY, - DFSConfigKeys.DFS_DATANODE_DU_RESERVED_DEFAULT); + DFSConfigKeys.DFS_DATANODE_DU_RESERVED_DEFAULT)); this.reservedForReplicas = new AtomicLong(0L); this.currentDir = currentDir; File parent = currentDir.getParentFile(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d77d6ea/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 842ccbf..79f7911 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -321,6 +321,11 @@ <name>dfs.datanode.du.reserved</name> <value>0</value> <description>Reserved space in bytes per volume. Always leave this much space free for non dfs use. + Specific storage type based reservation is also supported. The property can be followed with + corresponding storage types ([ssd]/[disk]/[archive]/[ram_disk]) for cluster with heterogeneous storage. + For example, reserved space for RAM_DISK storage can be configured using property + 'dfs.datanode.du.reserved.ram_disk'. If specific storage type reservation is not configured + then dfs.datanode.du.reserved will be used. </description> </property> http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d77d6ea/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java index e24c725..796d249 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeReference; import org.apache.hadoop.hdfs.server.datanode.fsdataset.RoundRobinVolumeChoosingPolicy; import org.apache.hadoop.hdfs.server.datanode.fsdataset.VolumeChoosingPolicy; import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.util.StringUtils; import org.junit.Before; import org.junit.Test; @@ -36,7 +37,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.concurrent.TimeoutException; - +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.fail; @@ -143,4 +144,37 @@ public class TestFsVolumeList { volumeList.addVolume(ref); assertNull(ref.getVolume()); } + + @Test + public void testDfsReservedForDifferentStorageTypes() throws IOException { + Configuration conf = new Configuration(); + conf.setLong(DFSConfigKeys.DFS_DATANODE_DU_RESERVED_KEY, 100L); + + File volDir = new File(baseDir, "volume-0"); + volDir.mkdirs(); + // when storage type reserved is not configured,should consider + // dfs.datanode.du.reserved. + FsVolumeImpl volume = new FsVolumeImpl(dataset, "storage-id", volDir, conf, + StorageType.RAM_DISK); + assertEquals("", 100L, volume.getReserved()); + // when storage type reserved is configured. + conf.setLong( + DFSConfigKeys.DFS_DATANODE_DU_RESERVED_KEY + "." + + StringUtils.toLowerCase(StorageType.RAM_DISK.toString()), 1L); + conf.setLong( + DFSConfigKeys.DFS_DATANODE_DU_RESERVED_KEY + "." + + StringUtils.toLowerCase(StorageType.SSD.toString()), 2L); + FsVolumeImpl volume1 = new FsVolumeImpl(dataset, "storage-id", volDir, + conf, StorageType.RAM_DISK); + assertEquals("", 1L, volume1.getReserved()); + FsVolumeImpl volume2 = new FsVolumeImpl(dataset, "storage-id", volDir, + conf, StorageType.SSD); + assertEquals("", 2L, volume2.getReserved()); + FsVolumeImpl volume3 = new FsVolumeImpl(dataset, "storage-id", volDir, + conf, StorageType.DISK); + assertEquals("", 100L, volume3.getReserved()); + FsVolumeImpl volume4 = new FsVolumeImpl(dataset, "storage-id", volDir, + conf, StorageType.DEFAULT); + assertEquals("", 100L, volume4.getReserved()); + } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
