Repository: hadoop Updated Branches: refs/heads/branch-3.1 b3f29e350 -> b38649c59
HDFS-13813. Exit NameNode if dangling child inode is detected when saving FsImage. Contributed by Siyao Meng. (cherry picked from commit 23854443efa62aa70a1c30c32c3816750e5d7a5b) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b38649c5 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b38649c5 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b38649c5 Branch: refs/heads/branch-3.1 Commit: b38649c59a70b3112811443464b3b3180f4b9873 Parents: b3f29e3 Author: Wei-Chiu Chuang <weic...@apache.org> Authored: Mon Aug 13 16:12:37 2018 -0700 Committer: Wei-Chiu Chuang <weic...@apache.org> Committed: Mon Aug 13 16:13:36 2018 -0700 ---------------------------------------------------------------------- .../server/namenode/FSImageFormatPBINode.java | 28 ++++++++++++++++++-- .../server/namenode/FSImageFormatProtobuf.java | 9 ++++--- 2 files changed, 32 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b38649c5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java index 5e60038..a233d26 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java @@ -439,6 +439,8 @@ public final class FSImageFormatPBINode { } public final static class Saver { + private long numImageErrors; + private static long buildPermissionStatus(INodeAttributes n, final SaverContext.DeduplicationMap<String> stringMap) { long userId = stringMap.getId(n.getUserName()); @@ -563,11 +565,13 @@ public final class FSImageFormatPBINode { this.summary = summary; this.context = parent.getContext(); this.fsn = context.getSourceNamesystem(); + this.numImageErrors = 0; } void serializeINodeDirectorySection(OutputStream out) throws IOException { - Iterator<INodeWithAdditionalFields> iter = fsn.getFSDirectory() - .getINodeMap().getMapIterator(); + FSDirectory dir = fsn.getFSDirectory(); + Iterator<INodeWithAdditionalFields> iter = dir.getINodeMap() + .getMapIterator(); final ArrayList<INodeReference> refList = parent.getSaverContext() .getRefList(); int i = 0; @@ -583,6 +587,17 @@ public final class FSImageFormatPBINode { INodeDirectorySection.DirEntry.Builder b = INodeDirectorySection. DirEntry.newBuilder().setParent(n.getId()); for (INode inode : children) { + // Error if the child inode doesn't exist in inodeMap + if (dir.getInode(inode.getId()) == null) { + FSImage.LOG.error( + "FSImageFormatPBINode#serializeINodeDirectorySection: " + + "Dangling child pointer found. Missing INode in " + + "inodeMap: id=" + inode.getId() + + "; path=" + inode.getFullPathName() + + "; parent=" + (inode.getParent() == null ? "null" : + inode.getParent().getFullPathName())); + ++numImageErrors; + } if (!inode.isReference()) { b.addChildren(inode.getId()); } else { @@ -711,6 +726,15 @@ public final class FSImageFormatPBINode { .setId(n.getId()) .setName(ByteString.copyFrom(n.getLocalNameBytes())); } + + /** + * Number of non-fatal errors detected while writing the + * INodeSection and INodeDirectorySection sections. + * @return the number of non-fatal errors detected. + */ + public long getNumImageErrors() { + return numImageErrors; + } } private FSImageFormatPBINode() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/b38649c5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java index 4ac20ad..9752733 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java @@ -474,13 +474,15 @@ public final class FSImageFormatProtobuf { out.write(lengthBytes); } - private void saveInodes(FileSummary.Builder summary) throws IOException { + private long saveInodes(FileSummary.Builder summary) throws IOException { FSImageFormatPBINode.Saver saver = new FSImageFormatPBINode.Saver(this, summary); saver.serializeINodeSection(sectionOutputStream); saver.serializeINodeDirectorySection(sectionOutputStream); saver.serializeFilesUCSection(sectionOutputStream); + + return saver.getNumImageErrors(); } /** @@ -543,8 +545,9 @@ public final class FSImageFormatProtobuf { step = new Step(StepType.INODES, filePath); prog.beginStep(Phase.SAVING_CHECKPOINT, step); - saveInodes(b); - long numErrors = saveSnapshots(b); + // Count number of non-fatal errors when saving inodes and snapshots. + long numErrors = saveInodes(b); + numErrors += saveSnapshots(b); prog.endStep(Phase.SAVING_CHECKPOINT, step); step = new Step(StepType.DELEGATION_TOKENS, filePath); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
