HDFS-8879. Quota by storage type usage incorrectly initialized upon namenode restart. Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3e715a4f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3e715a4f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3e715a4f Branch: refs/heads/HDFS-7285 Commit: 3e715a4f4c46bcd8b3054cb0566e526c46bd5d66 Parents: e5003be Author: Xiaoyu Yao <x...@apache.org> Authored: Tue Aug 11 21:42:53 2015 -0700 Committer: Xiaoyu Yao <x...@apache.org> Committed: Wed Aug 12 15:20:54 2015 -0700 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../apache/hadoop/hdfs/server/namenode/FSImage.java | 6 ++---- .../hdfs/server/namenode/TestQuotaByStorageType.java | 14 ++++++++++---- 3 files changed, 15 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e715a4f/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 6c2e0f9..df9b742 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1128,6 +1128,9 @@ Release 2.8.0 - UNRELEASED HDFS-8866. Typo in docs: Rumtime -> Runtime. (Gabor Liptak via jghoman) + HDFS-8879. Quota by storage type usage incorrectly initialized upon namenode + restart. (xyao) + Release 2.7.2 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e715a4f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java index 0dd7855..1f8cea3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java @@ -907,11 +907,9 @@ public class FSImage implements Closeable { + " quota = " + ssQuota + " < consumed = " + ssConsumed); } - final EnumCounters<StorageType> typeSpaces = - new EnumCounters<StorageType>(StorageType.class); + final EnumCounters<StorageType> typeSpaces = counts.getTypeSpaces(); for (StorageType t : StorageType.getTypesSupportingQuota()) { - final long typeSpace = counts.getTypeSpaces().get(t) - - parentTypeSpaces.get(t); + final long typeSpace = typeSpaces.get(t) - parentTypeSpaces.get(t); final long typeQuota = q.getTypeSpaces().get(t); if (Quota.isViolated(typeQuota, typeSpace)) { LOG.warn("Storage type quota violation in image for " http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e715a4f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaByStorageType.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaByStorageType.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaByStorageType.java index 6703066..f56c5a2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaByStorageType.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaByStorageType.java @@ -67,10 +67,7 @@ public class TestQuotaByStorageType { .storageTypes(new StorageType[]{StorageType.SSD, StorageType.DEFAULT}) .build(); cluster.waitActive(); - - fsdir = cluster.getNamesystem().getFSDirectory(); - dfs = cluster.getFileSystem(); - fsn = cluster.getNamesystem(); + refreshClusterState(); } @After @@ -80,6 +77,13 @@ public class TestQuotaByStorageType { } } + // Cluster state must be refreshed after each start/restart in the test + private void refreshClusterState() throws IOException{ + fsdir = cluster.getNamesystem().getFSDirectory(); + dfs = cluster.getFileSystem(); + fsn = cluster.getNamesystem(); + } + @Test(timeout = 60000) public void testQuotaByStorageTypeWithFileCreateOneSSD() throws Exception { testQuotaByStorageTypeWithFileCreateCase( @@ -662,6 +666,7 @@ public class TestQuotaByStorageType { // Restart namenode to make sure the editlog is correct cluster.restartNameNode(true); + refreshClusterState(); INode testDirNodeAfterNNRestart = fsdir.getINode4Write(testDir.toString()); // Verify quota is still set @@ -714,6 +719,7 @@ public class TestQuotaByStorageType { dfs.saveNamespace(); dfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE); cluster.restartNameNode(true); + refreshClusterState(); INode testDirNodeAfterNNRestart = fsdir.getINode4Write(testDir.toString()); assertTrue(testDirNode.isDirectory());
