HDFS-8822. Add SSD storagepolicy tests in TestBlockStoragePolicy#testDefaultPolicies (Contributed by Vinayakumar B)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/975e138d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/975e138d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/975e138d Branch: refs/heads/HADOOP-12111 Commit: 975e138df316f59e8bb0642e138d4b1170fb8184 Parents: 50887e5 Author: Vinayakumar B <[email protected]> Authored: Wed Jul 29 11:50:48 2015 +0530 Committer: Vinayakumar B <[email protected]> Committed: Wed Jul 29 11:50:48 2015 +0530 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../hadoop/hdfs/TestBlockStoragePolicy.java | 88 ++++++++++++++++---- 2 files changed, 75 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/975e138d/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 654c40f..d30c258 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -760,6 +760,9 @@ Release 2.8.0 - UNRELEASED HDFS-8811. Move BlockStoragePolicy name's constants from HdfsServerConstants.java to HdfsConstants.java (vinayakumarb) + HDFS-8822. Add SSD storagepolicy tests in TestBlockStoragePolicy# + testDefaultPolicies (vinayakumarb) + OPTIMIZATIONS HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than http://git-wip-us.apache.org/repos/asf/hadoop/blob/975e138d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java index 78e1e58..631d9f7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java @@ -67,7 +67,11 @@ public class TestBlockStoragePolicy { static final EnumSet<StorageType> none = EnumSet.noneOf(StorageType.class); static final EnumSet<StorageType> archive = EnumSet.of(StorageType.ARCHIVE); static final EnumSet<StorageType> disk = EnumSet.of(StorageType.DISK); - static final EnumSet<StorageType> both = EnumSet.of(StorageType.DISK, StorageType.ARCHIVE); + static final EnumSet<StorageType> ssd = EnumSet.of(StorageType.SSD); + static final EnumSet<StorageType> disk_archive = EnumSet.of(StorageType.DISK, + StorageType.ARCHIVE); + static final EnumSet<StorageType> all = EnumSet.of(StorageType.SSD, + StorageType.DISK, StorageType.ARCHIVE); static final long FILE_LEN = 1024; static final short REPLICATION = 3; @@ -153,8 +157,8 @@ public class TestBlockStoragePolicy { final List<StorageType> computed = cold.chooseStorageTypes(replication); assertStorageType(computed, replication, StorageType.ARCHIVE); } - assertCreationFallback(cold, null, null, null); - assertReplicationFallback(cold, null, null, null); + assertCreationFallback(cold, null, null, null, null, null); + assertReplicationFallback(cold, null, null, null, null); } { // check Warm policy @@ -163,8 +167,10 @@ public class TestBlockStoragePolicy { final List<StorageType> computed = warm.chooseStorageTypes(replication); assertStorageType(computed, replication, StorageType.DISK, StorageType.ARCHIVE); } - assertCreationFallback(warm, StorageType.DISK, StorageType.DISK, StorageType.ARCHIVE); - assertReplicationFallback(warm, StorageType.DISK, StorageType.DISK, StorageType.ARCHIVE); + assertCreationFallback(warm, StorageType.DISK, StorageType.DISK, + StorageType.ARCHIVE, StorageType.DISK, null); + assertReplicationFallback(warm, StorageType.DISK, StorageType.DISK, + StorageType.ARCHIVE, StorageType.DISK); } { // check Hot policy @@ -173,8 +179,50 @@ public class TestBlockStoragePolicy { final List<StorageType> computed = hot.chooseStorageTypes(replication); assertStorageType(computed, replication, StorageType.DISK); } - assertCreationFallback(hot, null, null, null); - assertReplicationFallback(hot, StorageType.ARCHIVE, null, StorageType.ARCHIVE); + assertCreationFallback(hot, null, null, null, null, null); + assertReplicationFallback(hot, StorageType.ARCHIVE, null, + StorageType.ARCHIVE, StorageType.ARCHIVE); + } + + { // check ONE_SSD policy + final BlockStoragePolicy onessd = POLICY_SUITE.getPolicy(ONESSD); + for (short replication = 1; replication < 6; replication++) { + final List<StorageType> computed = onessd + .chooseStorageTypes(replication); + assertStorageType(computed, replication, StorageType.SSD, + StorageType.DISK); + } + assertCreationFallback(onessd, StorageType.SSD, StorageType.SSD, + StorageType.SSD, StorageType.DISK, StorageType.SSD); + assertReplicationFallback(onessd, StorageType.SSD, StorageType.SSD, + StorageType.SSD, StorageType.DISK); + } + + { // check ALL_SSD policy + final BlockStoragePolicy allssd = POLICY_SUITE.getPolicy(ALLSSD); + for (short replication = 1; replication < 6; replication++) { + final List<StorageType> computed = allssd + .chooseStorageTypes(replication); + assertStorageType(computed, replication, StorageType.SSD); + } + assertCreationFallback(allssd, StorageType.DISK, StorageType.DISK, null, + StorageType.DISK, null); + assertReplicationFallback(allssd, StorageType.DISK, StorageType.DISK, + null, StorageType.DISK); + } + + { // check LAZY_PERSIST policy + final BlockStoragePolicy lazyPersist = POLICY_SUITE + .getPolicy(LAZY_PERSIST); + for (short replication = 1; replication < 6; replication++) { + final List<StorageType> computed = lazyPersist + .chooseStorageTypes(replication); + assertStorageType(computed, replication, StorageType.DISK); + } + assertCreationFallback(lazyPersist, StorageType.DISK, StorageType.DISK, + null, StorageType.DISK, null); + assertReplicationFallback(lazyPersist, StorageType.DISK, + StorageType.DISK, null, StorageType.DISK); } } @@ -199,20 +247,28 @@ public class TestBlockStoragePolicy { } } - static void assertCreationFallback(BlockStoragePolicy policy, StorageType noneExpected, - StorageType archiveExpected, StorageType diskExpected) { + static void assertCreationFallback(BlockStoragePolicy policy, + StorageType noneExpected, StorageType archiveExpected, + StorageType diskExpected, StorageType ssdExpected, + StorageType disk_archiveExpected) { Assert.assertEquals(noneExpected, policy.getCreationFallback(none)); Assert.assertEquals(archiveExpected, policy.getCreationFallback(archive)); Assert.assertEquals(diskExpected, policy.getCreationFallback(disk)); - Assert.assertEquals(null, policy.getCreationFallback(both)); + Assert.assertEquals(ssdExpected, policy.getCreationFallback(ssd)); + Assert.assertEquals(disk_archiveExpected, + policy.getCreationFallback(disk_archive)); + Assert.assertEquals(null, policy.getCreationFallback(all)); } - static void assertReplicationFallback(BlockStoragePolicy policy, StorageType noneExpected, - StorageType archiveExpected, StorageType diskExpected) { + static void assertReplicationFallback(BlockStoragePolicy policy, + StorageType noneExpected, StorageType archiveExpected, + StorageType diskExpected, StorageType ssdExpected) { Assert.assertEquals(noneExpected, policy.getReplicationFallback(none)); - Assert.assertEquals(archiveExpected, policy.getReplicationFallback(archive)); + Assert + .assertEquals(archiveExpected, policy.getReplicationFallback(archive)); Assert.assertEquals(diskExpected, policy.getReplicationFallback(disk)); - Assert.assertEquals(null, policy.getReplicationFallback(both)); + Assert.assertEquals(ssdExpected, policy.getReplicationFallback(ssd)); + Assert.assertEquals(null, policy.getReplicationFallback(all)); } private static interface CheckChooseStorageTypes { @@ -260,7 +316,7 @@ public class TestBlockStoragePolicy { public void checkChooseStorageTypes(BlockStoragePolicy p, short replication, List<StorageType> chosen, StorageType... expected) { final List<StorageType> types = p.chooseStorageTypes(replication, - chosen, both, true); + chosen, disk_archive, true); assertStorageTypes(types, expected); } }; @@ -272,7 +328,7 @@ public class TestBlockStoragePolicy { public void checkChooseStorageTypes(BlockStoragePolicy p, short replication, List<StorageType> chosen, StorageType... expected) { final List<StorageType> types = p.chooseStorageTypes(replication, - chosen, both, false); + chosen, disk_archive, false); assertStorageTypes(types, expected); } };
