HDFS-8809. HDFS fsck reports under construction blocks as CORRUPT. Contributed by Jing Zhao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c8bca627 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c8bca627 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c8bca627 Branch: refs/heads/HDFS-7240 Commit: c8bca62718203a1dad9b70d164bdf10cc71b40cd Parents: 7642f64 Author: Jing Zhao <ji...@apache.org> Authored: Thu Aug 20 16:31:24 2015 -0700 Committer: Jing Zhao <ji...@apache.org> Committed: Thu Aug 20 16:31:24 2015 -0700 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ .../apache/hadoop/hdfs/server/namenode/NamenodeFsck.java | 10 ++++++++++ .../org/apache/hadoop/hdfs/server/namenode/TestFsck.java | 2 ++ 3 files changed, 14 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8bca627/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index dcc5d58..d9d176b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1194,6 +1194,8 @@ Release 2.8.0 - UNRELEASED HDFS-8922. Link the native_mini_dfs test library with libdl, since IBM Java requires it (Ayappan via Colin P. McCabe) + HDFS-8809. HDFS fsck reports under construction blocks as "CORRUPT". (jing9) + Release 2.7.2 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8bca627/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java index 2f989d1..c7892b5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java @@ -528,6 +528,9 @@ public class NamenodeFsck implements DataEncryptionKeyFactory { LocatedBlocks blocks) throws IOException { String path = file.getFullName(parent); boolean isOpen = blocks.isUnderConstruction(); + if (isOpen && !showOpenFiles) { + return; + } int missing = 0; int corrupt = 0; long missize = 0; @@ -536,8 +539,15 @@ public class NamenodeFsck implements DataEncryptionKeyFactory { int misReplicatedPerFile = 0; StringBuilder report = new StringBuilder(); int blockNumber = 0; + final LocatedBlock lastBlock = blocks.getLastLocatedBlock(); for (LocatedBlock lBlk : blocks.getLocatedBlocks()) { ExtendedBlock block = lBlk.getBlock(); + if (!blocks.isLastBlockComplete() && lastBlock != null && + lastBlock.getBlock().equals(block)) { + // this is the last block and this is not complete. ignore it since + // it is under construction + continue; + } BlockManager bm = namenode.getNamesystem().getBlockManager(); final BlockInfo storedBlock = bm.getStoredBlock( http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8bca627/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java index 8818f17..2226947 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java @@ -65,6 +65,7 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSClient; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSInputStream; +import org.apache.hadoop.hdfs.DFSOutputStream; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; @@ -612,6 +613,7 @@ public class TestFsck { out.write(randomString.getBytes()); writeCount++; } + ((DFSOutputStream) out.getWrappedStream()).hflush(); // We expect the filesystem to be HEALTHY and show one open file outStr = runFsck(conf, 0, true, topDir); System.out.println(outStr);