Repository: hadoop Updated Branches: refs/heads/branch-2 a0450b96a -> cd0c6b578
HDFS-8809. HDFS fsck reports under construction blocks as CORRUPT. Contributed by Jing Zhao. (cherry picked from commit c8bca62718203a1dad9b70d164bdf10cc71b40cd) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cd0c6b57 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cd0c6b57 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cd0c6b57 Branch: refs/heads/branch-2 Commit: cd0c6b57895d3fd6f5f43c166a8a1cd363173701 Parents: a0450b9 Author: Jing Zhao <[email protected]> Authored: Thu Aug 20 16:31:24 2015 -0700 Committer: Jing Zhao <[email protected]> Committed: Thu Aug 20 16:32:04 2015 -0700 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ .../apache/hadoop/hdfs/server/namenode/NamenodeFsck.java | 10 ++++++++++ .../org/apache/hadoop/hdfs/server/namenode/TestFsck.java | 2 ++ 3 files changed, 14 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/cd0c6b57/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 842fb2e..81ee9d0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -850,6 +850,8 @@ Release 2.8.0 - UNRELEASED HDFS-8922. Link the native_mini_dfs test library with libdl, since IBM Java requires it (Ayappan via Colin P. McCabe) + HDFS-8809. HDFS fsck reports under construction blocks as "CORRUPT". (jing9) + Release 2.7.2 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/cd0c6b57/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java index e96e674..6b951f3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java @@ -523,6 +523,9 @@ public class NamenodeFsck implements DataEncryptionKeyFactory { LocatedBlocks blocks) throws IOException { String path = file.getFullName(parent); boolean isOpen = blocks.isUnderConstruction(); + if (isOpen && !showOpenFiles) { + return; + } int missing = 0; int corrupt = 0; long missize = 0; @@ -530,8 +533,15 @@ public class NamenodeFsck implements DataEncryptionKeyFactory { int misReplicatedPerFile = 0; StringBuilder report = new StringBuilder(); int blockNumber = 0; + final LocatedBlock lastBlock = blocks.getLastLocatedBlock(); for (LocatedBlock lBlk : blocks.getLocatedBlocks()) { ExtendedBlock block = lBlk.getBlock(); + if (!blocks.isLastBlockComplete() && lastBlock != null && + lastBlock.getBlock().equals(block)) { + // this is the last block and this is not complete. ignore it since + // it is under construction + continue; + } BlockManager bm = namenode.getNamesystem().getBlockManager(); final BlockInfo storedBlock = bm.getStoredBlock( http://git-wip-us.apache.org/repos/asf/hadoop/blob/cd0c6b57/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java index b1e7a35..3495898 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java @@ -65,6 +65,7 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSClient; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSInputStream; +import org.apache.hadoop.hdfs.DFSOutputStream; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; @@ -601,6 +602,7 @@ public class TestFsck { out.write(randomString.getBytes()); writeCount++; } + ((DFSOutputStream) out.getWrappedStream()).hflush(); // We expect the filesystem to be HEALTHY and show one open file outStr = runFsck(conf, 0, true, topDir); System.out.println(outStr);
