HDFS-10983. OIV tool should make an EC file explicit. Contributed by Manoj Govindassamy.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5ca6ef0c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5ca6ef0c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5ca6ef0c Branch: refs/heads/HADOOP-13345 Commit: 5ca6ef0c268b1acb3abf12505b9ead6fe7e38a23 Parents: d7762a5 Author: Andrew Wang <w...@apache.org> Authored: Wed Mar 8 15:36:19 2017 -0800 Committer: Andrew Wang <w...@apache.org> Committed: Wed Mar 8 15:36:19 2017 -0800 ---------------------------------------------------------------------- .../server/namenode/FSImageFormatPBINode.java | 1 + .../OfflineImageReconstructor.java | 4 + .../offlineImageViewer/PBImageXmlWriter.java | 15 ++- .../hdfs/server/namenode/TestFSImage.java | 1 + .../TestOfflineImageViewer.java | 99 +++++++++++++++++++- 5 files changed, 112 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ca6ef0c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java index 17b1da7..ef334f7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java @@ -332,6 +332,7 @@ public final class FSImageFormatPBINode { BlockType blockType = PBHelperClient.convert(f.getBlockType()); LoaderContext state = parent.getLoaderContext(); boolean isStriped = f.hasErasureCodingPolicyID(); + assert ((!isStriped) || (isStriped && !f.hasReplication())); Short replication = (!isStriped ? (short) f.getReplication() : null); ErasureCodingPolicy ecPolicy = isStriped ? ErasureCodingPolicyManager.getPolicyByPolicyID( http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ca6ef0c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java index ed348d3..e80f4d4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java @@ -647,6 +647,10 @@ class OfflineImageReconstructor { break; case "STRIPED": bld.setBlockType(HdfsProtos.BlockTypeProto.STRIPED); + ival = node.removeChildInt(INODE_SECTION_EC_POLICY_ID); + if (ival != null) { + bld.setErasureCodingPolicyID(ival); + } break; default: throw new IOException("INode XML found with unknown <blocktype> " + http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ca6ef0c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java index f8734cb..5a42a6b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java @@ -40,7 +40,6 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheD import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveInfoProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CachePoolInfoProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto; -import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockTypeProto; import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos; import org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode; import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf.SectionName; @@ -59,6 +58,7 @@ import org.apache.hadoop.hdfs.server.namenode.FsImageProto.SecretManagerSection; import org.apache.hadoop.hdfs.server.namenode.FsImageProto.SnapshotDiffSection; import org.apache.hadoop.hdfs.server.namenode.FsImageProto.SnapshotSection; import org.apache.hadoop.hdfs.server.namenode.FsImageProto.StringTableSection; +import org.apache.hadoop.hdfs.server.namenode.INodeFile; import org.apache.hadoop.hdfs.util.XMLUtils; import org.apache.hadoop.util.LimitInputStream; import com.google.common.collect.ImmutableList; @@ -132,6 +132,8 @@ public final class PBImageXmlWriter { public static final String INODE_SECTION_STORAGE_POLICY_ID = "storagePolicyId"; public static final String INODE_SECTION_BLOCK_TYPE = "blockType"; + public static final String INODE_SECTION_EC_POLICY_ID = + "erasureCodingPolicyId"; public static final String INODE_SECTION_NS_QUOTA = "nsquota"; public static final String INODE_SECTION_DS_QUOTA = "dsquota"; public static final String INODE_SECTION_TYPE_QUOTA = "typeQuota"; @@ -472,8 +474,12 @@ public final class PBImageXmlWriter { } private void dumpINodeFile(INodeSection.INodeFile f) { - o(SECTION_REPLICATION, f.getReplication()) - .o(INODE_SECTION_MTIME, f.getModificationTime()) + if (f.hasErasureCodingPolicyID()) { + o(SECTION_REPLICATION, INodeFile.DEFAULT_REPL_FOR_STRIPED_BLOCKS); + } else { + o(SECTION_REPLICATION, f.getReplication()); + } + o(INODE_SECTION_MTIME, f.getModificationTime()) .o(INODE_SECTION_ATIME, f.getAccessTime()) .o(INODE_SECTION_PREFERRED_BLOCK_SIZE, f.getPreferredBlockSize()) .o(INODE_SECTION_PERMISSION, dumpPermission(f.getPermission())); @@ -495,8 +501,9 @@ public final class PBImageXmlWriter { if (f.hasStoragePolicyID()) { o(INODE_SECTION_STORAGE_POLICY_ID, f.getStoragePolicyID()); } - if (f.getBlockType() != BlockTypeProto.CONTIGUOUS) { + if (f.hasErasureCodingPolicyID()) { o(INODE_SECTION_BLOCK_TYPE, f.getBlockType().name()); + o(INODE_SECTION_EC_POLICY_ID, f.getErasureCodingPolicyID()); } if (f.hasFileUC()) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ca6ef0c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java index ae15491..57416ed 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java @@ -223,6 +223,7 @@ public class TestFSImage { // blocks to/from legacy fsimage assertEquals(3, fileByLoaded.getBlocks().length); assertEquals(preferredBlockSize, fileByLoaded.getPreferredBlockSize()); + assertEquals(file.getFileReplication(), fileByLoaded.getFileReplication()); if (isUC) { assertEquals(client, http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ca6ef0c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java index b587017..e52d536 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java @@ -76,11 +76,13 @@ import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.protocol.BlockType; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil; +import org.apache.hadoop.hdfs.server.namenode.INodeFile; import org.apache.hadoop.hdfs.server.namenode.NameNodeLayoutVersion; import org.apache.hadoop.hdfs.web.WebHdfsFileSystem; import org.apache.hadoop.io.IOUtils; @@ -91,9 +93,8 @@ import org.apache.log4j.Level; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.TemporaryFolder; +import org.xml.sax.Attributes; import org.xml.sax.InputSource; import org.xml.sax.SAXException; import org.xml.sax.helpers.DefaultHandler; @@ -112,7 +113,6 @@ public class TestOfflineImageViewer { // namespace as written to dfs, to be compared with viewer's output final static HashMap<String, FileStatus> writtenFiles = Maps.newHashMap(); static int dirCount = 0; - private static File tempDir; // Create a populated namespace for later testing. Save its contents to a @@ -358,6 +358,96 @@ public class TestOfflineImageViewer { assertEquals(0, status); } + /** + * SAX handler to verify EC Files and their policies. + */ + class ECXMLHandler extends DefaultHandler { + + private boolean isInode = false; + private boolean isAttrRepl = false; + private boolean isAttrName = false; + private boolean isXAttrs = false; + private boolean isAttrECPolicy = false; + private boolean isAttrBlockType = false; + private String currentInodeName; + private String currentECPolicy; + private String currentBlockType; + private String currentRepl; + + @Override + public void startElement(String uri, String localName, String qName, + Attributes attributes) throws SAXException { + super.startElement(uri, localName, qName, attributes); + if (qName.equalsIgnoreCase(PBImageXmlWriter.INODE_SECTION_INODE)) { + isInode = true; + } else if (isInode && !isXAttrs && qName.equalsIgnoreCase( + PBImageXmlWriter.SECTION_NAME)) { + isAttrName = true; + } else if (isInode && qName.equalsIgnoreCase( + PBImageXmlWriter.SECTION_REPLICATION)) { + isAttrRepl = true; + } else if (isInode && + qName.equalsIgnoreCase(PBImageXmlWriter.INODE_SECTION_EC_POLICY_ID)) { + isAttrECPolicy = true; + } else if (isInode && qName.equalsIgnoreCase( + PBImageXmlWriter.INODE_SECTION_BLOCK_TYPE)) { + isAttrBlockType = true; + } else if (isInode && qName.equalsIgnoreCase( + PBImageXmlWriter.INODE_SECTION_XATTRS)) { + isXAttrs = true; + } + } + + @Override + public void endElement(String uri, String localName, String qName) + throws SAXException { + super.endElement(uri, localName, qName); + if (qName.equalsIgnoreCase(PBImageXmlWriter.INODE_SECTION_INODE)) { + if (currentInodeName != null && currentInodeName.length() > 0) { + if (currentBlockType != null && currentBlockType.equalsIgnoreCase( + BlockType.STRIPED.name())) { + Assert.assertEquals("INode '" + + currentInodeName + "' has unexpected EC Policy!", + Byte.parseByte(currentECPolicy), + ErasureCodingPolicyManager.getPolicyByPolicyID( + HdfsConstants.XOR_2_1_POLICY_ID).getId()); + Assert.assertEquals("INode '" + + currentInodeName + "' has unexpected replication!", + currentRepl, + Short.toString(INodeFile.DEFAULT_REPL_FOR_STRIPED_BLOCKS)); + } + } + isInode = false; + currentInodeName = ""; + currentECPolicy = ""; + currentRepl = ""; + } else if (qName.equalsIgnoreCase( + PBImageXmlWriter.INODE_SECTION_XATTRS)) { + isXAttrs = false; + } + } + + @Override + public void characters(char[] ch, int start, int length) + throws SAXException { + super.characters(ch, start, length); + String value = new String(ch, start, length); + if (isAttrName) { + currentInodeName = value; + isAttrName = false; + } else if (isAttrRepl) { + currentRepl = value; + isAttrRepl = false; + } else if (isAttrECPolicy) { + currentECPolicy = value; + isAttrECPolicy = false; + } else if (isAttrBlockType) { + currentBlockType = value; + isAttrBlockType = false; + } + } + } + @Test public void testPBImageXmlWriter() throws IOException, SAXException, ParserConfigurationException { @@ -368,7 +458,8 @@ public class TestOfflineImageViewer { SAXParserFactory spf = SAXParserFactory.newInstance(); SAXParser parser = spf.newSAXParser(); final String xml = output.toString(); - parser.parse(new InputSource(new StringReader(xml)), new DefaultHandler()); + ECXMLHandler ecxmlHandler = new ECXMLHandler(); + parser.parse(new InputSource(new StringReader(xml)), ecxmlHandler); } @Test --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
