HDFS-7072. Fix TestBlockManager and TestStorageMover. Contributed by Jing Zhao
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bd2e409d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bd2e409d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bd2e409d Branch: refs/heads/trunk Commit: bd2e409de128dcf4143f69c1d572e03a51511517 Parents: 432b863 Author: Tsz-Wo Nicholas Sze <szets...@hortonworks.com> Authored: Wed Sep 17 13:48:32 2014 +0800 Committer: Tsz-Wo Nicholas Sze <szets...@hortonworks.com> Committed: Wed Sep 17 13:48:32 2014 +0800 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../blockmanagement/DatanodeStorageInfo.java | 2 +- .../server/blockmanagement/TestBlockManager.java | 1 + .../hdfs/server/mover/TestStorageMover.java | 18 ++++++++++++++++++ 4 files changed, 23 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/bd2e409d/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 7584582..59a130e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -23,6 +23,9 @@ HDFS-6584: Archival Storage HDFS-6847. Support storage policy on directories and include storage policy in HdfsFileStatus. (Jing Zhao via szetszwo) + HDFS-7072. Fix TestBlockManager and TestStorageMover. (Jing Zhao + via szetszwo) + Trunk (Unreleased) INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/bd2e409d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java index c363f4b..58ca2ac 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java @@ -109,7 +109,7 @@ public class DatanodeStorageInfo { private long capacity; private long dfsUsed; - private long remaining; + private volatile long remaining; private long blockPoolUsed; private volatile BlockInfo blockList = null; http://git-wip-us.apache.org/repos/asf/hadoop/blob/bd2e409d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java index 9b980dc..7c0623c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java @@ -666,6 +666,7 @@ public class TestBlockManager { excessTypes.add(StorageType.DEFAULT); Assert.assertTrue(BlockManager.useDelHint(true, delHint, null, moreThan1Racks, excessTypes)); + excessTypes.remove(0); excessTypes.add(StorageType.SSD); Assert.assertFalse(BlockManager.useDelHint(true, delHint, null, moreThan1Racks, excessTypes)); http://git-wip-us.apache.org/repos/asf/hadoop/blob/bd2e409d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java index ceedfc2..eeea62b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java @@ -588,6 +588,20 @@ public class TestStorageMover { } } + private void waitForAllReplicas(int expectedReplicaNum, Path file, + DistributedFileSystem dfs) throws Exception { + for (int i = 0; i < 5; i++) { + LocatedBlocks lbs = dfs.getClient().getLocatedBlocks(file.toString(), 0, + BLOCK_SIZE); + LocatedBlock lb = lbs.get(0); + if (lb.getLocations().length >= expectedReplicaNum) { + return; + } else { + Thread.sleep(1000); + } + } + } + /** * Test DISK is running out of spaces. */ @@ -618,6 +632,7 @@ public class TestStorageMover { for (; ; hotFileCount++) { final Path p = new Path(pathPolicyMap.hot, "file" + hotFileCount); DFSTestUtil.createFile(test.dfs, p, BLOCK_SIZE, replication, 0L); + waitForAllReplicas(replication, p, test.dfs); } } catch (IOException e) { LOG.info("Expected: hotFileCount=" + hotFileCount, e); @@ -632,6 +647,7 @@ public class TestStorageMover { for (; ; hotFileCount_r1++) { final Path p = new Path(pathPolicyMap.hot, "file_r1_" + hotFileCount_r1); DFSTestUtil.createFile(test.dfs, p, BLOCK_SIZE, (short) 1, 0L); + waitForAllReplicas(1, p, test.dfs); } } catch (IOException e) { LOG.info("Expected: hotFileCount_r1=" + hotFileCount_r1, e); @@ -699,6 +715,7 @@ public class TestStorageMover { for (; ; coldFileCount++) { final Path p = new Path(pathPolicyMap.cold, "file" + coldFileCount); DFSTestUtil.createFile(test.dfs, p, BLOCK_SIZE, replication, 0L); + waitForAllReplicas(replication, p, test.dfs); } } catch (IOException e) { LOG.info("Expected: coldFileCount=" + coldFileCount, e); @@ -713,6 +730,7 @@ public class TestStorageMover { for (; ; coldFileCount_r1++) { final Path p = new Path(pathPolicyMap.cold, "file_r1_" + coldFileCount_r1); DFSTestUtil.createFile(test.dfs, p, BLOCK_SIZE, (short) 1, 0L); + waitForAllReplicas(1, p, test.dfs); } } catch (IOException e) { LOG.info("Expected: coldFileCount_r1=" + coldFileCount_r1, e);
