Added: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java?rev=1437256&view=auto ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java (added) +++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java Wed Jan 23 02:48:01 2013 @@ -0,0 +1,351 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode.snapshot; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.apache.hadoop.hdfs.DFSUtil; +import org.apache.hadoop.hdfs.server.namenode.FSDirectory; +import org.apache.hadoop.hdfs.server.namenode.FSImageFormat; +import org.apache.hadoop.hdfs.server.namenode.FSImageFormat.Loader; +import org.apache.hadoop.hdfs.server.namenode.FSImageSerialization; +import org.apache.hadoop.hdfs.server.namenode.INode; +import org.apache.hadoop.hdfs.server.namenode.INodeDirectory; +import org.apache.hadoop.hdfs.server.namenode.INodeFile; +import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectoryWithSnapshot.Diff; +import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectoryWithSnapshot.SnapshotDiff; +import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot.Root; +import org.apache.hadoop.hdfs.util.ReadOnlyList; + +/** + * A helper class defining static methods for reading/writing snapshot related + * information from/to FSImage. + */ +public class SnapshotFSImageFormat { + + /** + * Save snapshots and snapshot quota for a snapshottable directory. + * @param current The directory that the snapshots belongs to. + * @param out The {@link DataOutputStream} to write. + * @throws IOException + */ + public static void saveSnapshots(INodeDirectorySnapshottable current, + DataOutputStream out) throws IOException { + // list of snapshots in snapshotsByNames + ReadOnlyList<Snapshot> snapshots = current.getSnapshotsByNames(); + out.writeInt(snapshots.size()); + for (Snapshot ss : snapshots) { + // write the snapshot + ss.write(out); + } + // snapshot quota + out.writeInt(current.getSnapshotQuota()); + } + + /** + * Save SnapshotDiff list for an INodeDirectoryWithSnapshot. + * @param sNode The directory that the SnapshotDiff list belongs to. + * @param out The {@link DataOutputStream} to write. + */ + public static void saveSnapshotDiffs(INodeDirectoryWithSnapshot sNode, + DataOutputStream out) throws IOException { + // # of SnapshotDiff + List<SnapshotDiff> diffs = sNode.getSnapshotDiffs(); + // Record the SnapshotDiff in reversed order, so that we can find the + // correct reference for INodes in the created list when loading the + // FSImage + out.writeInt(diffs.size()); + for (int i = diffs.size() - 1; i >= 0; i--) { + SnapshotDiff sdiff = diffs.get(i); + sdiff.write(out); + } + } + + /** + * Load a node stored in the created list from fsimage. + * @param createdNodeName The name of the created node. + * @param parent The directory that the created list belongs to. + * @return The created node. + */ + private static INode loadCreated(byte[] createdNodeName, + INodeDirectoryWithSnapshot parent) throws IOException { + // the INode in the created list should be a reference to another INode + // in posterior SnapshotDiffs or one of the current children + for (SnapshotDiff postDiff : parent.getSnapshotDiffs()) { + INode created = findCreated(createdNodeName, postDiff.getDiff()); + if (created != null) { + return created; + } // else go to the next SnapshotDiff + } + // use the current child + INode currentChild = parent.getChild(createdNodeName, null); + if (currentChild == null) { + throw new IOException("Cannot find an INode associated with the INode " + + DFSUtil.bytes2String(createdNodeName) + + " in created list while loading FSImage."); + } + return currentChild; + } + + /** + * Search the given {@link Diff} to find an inode matching the specific name. + * @param createdNodeName The name of the node for searching. + * @param diff The given {@link Diff} where to search the node. + * @return The matched inode. Return null if no matched inode can be found. + */ + private static INode findCreated(byte[] createdNodeName, Diff diff) { + INode c = diff.searchCreated(createdNodeName); + INode d = diff.searchDeleted(createdNodeName); + if (c == null && d != null) { + // if an INode with the same name is only contained in the deleted + // list, then the node should be the snapshot copy of a deleted + // node, and the node in the created list should be its reference + return d; + } else if (c != null && d != null) { + // in a posterior SnapshotDiff, if the created/deleted lists both + // contains nodes with the same name (c & d), there are two + // possibilities: + // + // 1) c and d are used to represent a modification, and + // 2) d indicates the deletion of the node, while c was originally + // contained in the created list of a later snapshot, but c was + // moved here because of the snapshot deletion. + // + // For case 1), c and d should be both INodeFile and should share + // the same blockInfo list. + if (c.isFile() + && ((INodeFile) c).getBlocks() == ((INodeFile) d).getBlocks()) { + return c; + } else { + return d; + } + } + return null; + } + + /** + * Load the created list from fsimage. + * @param parent The directory that the created list belongs to. + * @param in The {@link DataInputStream} to read. + * @return The created list. + */ + private static List<INode> loadCreatedList(INodeDirectoryWithSnapshot parent, + DataInputStream in) throws IOException { + // read the size of the created list + int createdSize = in.readInt(); + List<INode> createdList = new ArrayList<INode>(createdSize); + for (int i = 0; i < createdSize; i++) { + byte[] createdNodeName = new byte[in.readShort()]; + in.readFully(createdNodeName); + INode created = loadCreated(createdNodeName, parent); + createdList.add(created); + } + return createdList; + } + + /** + * Load the deleted list from the fsimage. + * + * @param parent The directory that the deleted list belongs to. + * @param createdList The created list associated with the deleted list in + * the same Diff. + * @param in The {@link DataInputStream} to read. + * @param loader The {@link Loader} instance. Used to call the + * {@link Loader#loadINode(DataInputStream)} method. + * @return The deleted list. + */ + private static List<INode> loadDeletedList(INodeDirectoryWithSnapshot parent, + List<INode> createdList, DataInputStream in, FSImageFormat.Loader loader) + throws IOException { + int deletedSize = in.readInt(); + List<INode> deletedList = new ArrayList<INode>(deletedSize); + for (int i = 0; i < deletedSize; i++) { + byte[] deletedNodeName = new byte[in.readShort()]; + in.readFully(deletedNodeName); + INode deleted = loader.loadINode(in); + deleted.setLocalName(deletedNodeName); + deletedList.add(deleted); + // set parent: the parent field of an INode in the deleted list is not + // useful, but set the parent here to be consistent with the original + // fsdir tree. + deleted.setParent(parent); + if (deleted instanceof INodeFile + && ((INodeFile) deleted).getBlocks() == null) { + // if deleted is an INodeFile, and its blocks is null, then deleted + // must be an INodeFileWithLink, and we need to rebuild its next link + int c = Collections.binarySearch(createdList, deletedNodeName); + if (c < 0) { + throw new IOException( + "Cannot find the INode linked with the INode " + + DFSUtil.bytes2String(deletedNodeName) + + " in deleted list while loading FSImage."); + } + // deleted must be an INodeFileSnapshot + INodeFileSnapshot deletedWithLink = (INodeFileSnapshot) deleted; + INodeFile cNode = (INodeFile) createdList.get(c); + INodeFileWithSnapshot cNodeWithLink = (INodeFileWithSnapshot) cNode; + deletedWithLink.setBlocks(cNode.getBlocks()); + // insert deleted into the circular list + cNodeWithLink.insertBefore(deletedWithLink); + } + } + return deletedList; + } + + /** + * Load snapshots and snapshotQuota for a Snapshottable directory. + * @param snapshottableParent The snapshottable directory for loading. + * @param numSnapshots The number of snapshots that the directory has. + * @param in The {@link DataInputStream} instance to read. + * @param loader The {@link Loader} instance that this loading procedure is + * using. + */ + public static void loadSnapshotList( + INodeDirectorySnapshottable snapshottableParent, int numSnapshots, + DataInputStream in, FSImageFormat.Loader loader) throws IOException { + for (int i = 0; i < numSnapshots; i++) { + // read snapshots + Snapshot ss = loadSnapshot(snapshottableParent, in, loader); + snapshottableParent.addSnapshot(ss); + } + int snapshotQuota = in.readInt(); + snapshottableParent.setSnapshotQuota(snapshotQuota); + } + + /** + * Load a {@link Snapshot} from fsimage. + * @param parent The directory that the snapshot belongs to. + * @param in The {@link DataInputStream} instance to read. + * @param loader The {@link Loader} instance that this loading procedure is + * using. + * @return The snapshot. + */ + private static Snapshot loadSnapshot(INodeDirectorySnapshottable parent, + DataInputStream in, FSImageFormat.Loader loader) throws IOException { + int snapshotId = in.readInt(); + byte[] snapshotName = new byte[in.readShort()]; + in.readFully(snapshotName); + INode rootNode = loader.loadINode(in); + rootNode.setLocalName(snapshotName); + rootNode.setParent(parent); + return new Snapshot(snapshotId, (INodeDirectory) rootNode); + } + + /** + * Load the {@link SnapshotDiff} list for the INodeDirectoryWithSnapshot + * directory. + * @param snapshottableParent The snapshottable directory for loading. + * @param numSnapshotDiffs The number of {@link SnapshotDiff} that the + * directory has. + * @param in The {@link DataInputStream} instance to read. + * @param loader The {@link Loader} instance that this loading procedure is + * using. + */ + public static void loadSnapshotDiffList( + INodeDirectoryWithSnapshot parentWithSnapshot, int numSnapshotDiffs, + DataInputStream in, FSImageFormat.Loader loader) + throws IOException { + for (int i = 0; i < numSnapshotDiffs; i++) { + SnapshotDiff diff = loadSnapshotDiff(parentWithSnapshot, in, loader); + parentWithSnapshot.insertDiff(diff); + } + } + + /** + * Use the given full path to a {@link Root} directory to find the + * associated snapshot. + */ + private static Snapshot findSnapshot(String sRootFullPath, FSDirectory fsdir) + throws IOException { + // find the root + INode root = fsdir.getINode(sRootFullPath); + INodeDirectorySnapshottable snapshotRoot = INodeDirectorySnapshottable + .valueOf(root.getParent(), root.getParent().getFullPathName()); + // find the snapshot + return snapshotRoot.getSnapshot(root.getLocalNameBytes()); + } + + /** + * Load the snapshotINode field of {@link SnapshotDiff}. + * @param snapshot The Snapshot associated with the {@link SnapshotDiff}. + * @param in The {@link DataInputStream} to read. + * @param loader The {@link Loader} instance that this loading procedure is + * using. + * @return The snapshotINode. + */ + private static INodeDirectory loadSnapshotINodeInSnapshotDiff( + Snapshot snapshot, DataInputStream in, FSImageFormat.Loader loader) + throws IOException { + // read the boolean indicating whether snapshotINode == Snapshot.Root + boolean useRoot = in.readBoolean(); + if (useRoot) { + return snapshot.getRoot(); + } else { + // another boolean is used to indicate whether snapshotINode is non-null + if (in.readBoolean()) { + byte[] localName = new byte[in.readShort()]; + in.readFully(localName); + INodeDirectory snapshotINode = (INodeDirectory) loader.loadINode(in); + snapshotINode.setLocalName(localName); + return snapshotINode; + } + } + return null; + } + + /** + * Load {@link SnapshotDiff} from fsimage. + * @param parent The directory that the SnapshotDiff belongs to. + * @param in The {@link DataInputStream} instance to read. + * @param loader The {@link Loader} instance that this loading procedure is + * using. + * @return A {@link SnapshotDiff}. + */ + private static SnapshotDiff loadSnapshotDiff( + INodeDirectoryWithSnapshot parent, DataInputStream in, + FSImageFormat.Loader loader) throws IOException { + // 1. Load SnapshotDiff#childrenSize + int childrenSize = in.readInt(); + // 2. Read the full path of the Snapshot's Root, identify + // SnapshotDiff#Snapshot + Snapshot snapshot = findSnapshot(FSImageSerialization.readString(in), + loader.getFSDirectoryInLoading()); + + // 3. Load SnapshotDiff#snapshotINode + INodeDirectory snapshotINode = loadSnapshotINodeInSnapshotDiff(snapshot, + in, loader); + + // 4. Load the created list in SnapshotDiff#Diff + List<INode> createdList = loadCreatedList(parent, in); + + // 5. Load the deleted list in SnapshotDiff#Diff + List<INode> deletedList = loadDeletedList(parent, createdList, in, loader); + + // 6. Compose the SnapshotDiff + SnapshotDiff sdiff = parent.new SnapshotDiff(snapshot, childrenSize, + snapshotINode, parent.getSnapshotDiffs().isEmpty() ? null : parent + .getSnapshotDiffs().get(0), createdList, deletedList); + return sdiff; + } + +}
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java?rev=1437256&r1=1437255&r2=1437256&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java Wed Jan 23 02:48:01 2013 @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hdfs.server.namenode.snapshot; +import java.io.DataInput; +import java.io.DataOutput; import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -48,7 +50,7 @@ public class SnapshotManager implements private final AtomicInteger numSnapshottableDirs = new AtomicInteger(); private final AtomicInteger numSnapshots = new AtomicInteger(); - private int snapshotID = 0; + private int snapshotCounter = 0; /** All snapshottable directories in the namesystem. */ private final List<INodeDirectorySnapshottable> snapshottables @@ -117,10 +119,10 @@ public class SnapshotManager implements final INodesInPath i = fsdir.getMutableINodesInPath(path); final INodeDirectorySnapshottable srcRoot = INodeDirectorySnapshottable.valueOf(i.getLastINode(), path); - srcRoot.addSnapshot(snapshotID, snapshotName); + srcRoot.addSnapshot(snapshotCounter, snapshotName); //create success, update id - snapshotID++; + snapshotCounter++; numSnapshots.getAndIncrement(); } @@ -181,6 +183,26 @@ public class SnapshotManager implements } /** + * Write {@link #snapshotCounter}, {@link #numSnapshots}, and + * {@link #numSnapshottableDirs} to the DataOutput. + */ + public void write(DataOutput out) throws IOException { + out.writeInt(snapshotCounter); + out.writeInt(numSnapshots.get()); + out.writeInt(numSnapshottableDirs.get()); + } + + /** + * Read values of {@link #snapshotCounter}, {@link #numSnapshots}, and + * {@link #numSnapshottableDirs} from the DataInput + */ + public void read(DataInput in) throws IOException { + snapshotCounter = in.readInt(); + numSnapshots.set(in.readInt()); + numSnapshottableDirs.set(in.readInt()); + } + + /** * @return All the current snapshottable directories */ public SnapshottableDirectoryStatus[] getSnapshottableDirListing() { Added: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java?rev=1437256&view=auto ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java (added) +++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java Wed Jan 23 02:48:01 2013 @@ -0,0 +1,162 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode; + +import static org.junit.Assert.assertEquals; + +import java.io.File; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DFSTestUtil; +import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeFile; +import org.apache.hadoop.hdfs.util.Canceler; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +/** + * Test FSImage save/load when Snapshot is supported + */ +public class TestFSImageWithSnapshot { + static final long seed = 0; + static final short REPLICATION = 3; + static final long BLOCKSIZE = 1024; + static final long txid = 1; + + private final Path dir = new Path("/TestSnapshot"); + private static String testDir = + System.getProperty("test.build.data", "build/test/data"); + + Configuration conf; + MiniDFSCluster cluster; + FSNamesystem fsn; + DistributedFileSystem hdfs; + + @Before + public void setUp() throws Exception { + conf = new Configuration(); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(REPLICATION) + .build(); + cluster.waitActive(); + fsn = cluster.getNamesystem(); + hdfs = cluster.getFileSystem(); + } + + @After + public void tearDown() throws Exception { + if (cluster != null) { + cluster.shutdown(); + } + } + + /** + * Testing steps: + * <pre> + * 1. Creating/modifying directories/files while snapshots are being taken. + * 2. Dump the FSDirectory tree of the namesystem. + * 3. Save the namesystem to a temp file (FSImage saving). + * 4. Restart the cluster and format the namesystem. + * 5. Load the namesystem from the temp file (FSImage loading). + * 6. Dump the FSDirectory again and compare the two dumped string. + * </pre> + */ + @Test + public void testSaveLoadImage() throws Exception { + // make changes to the namesystem + hdfs.mkdirs(dir); + hdfs.allowSnapshot(dir.toString()); + hdfs.createSnapshot(dir, "s0"); + + Path sub1 = new Path(dir, "sub1"); + Path sub1file1 = new Path(sub1, "sub1file1"); + Path sub1file2 = new Path(sub1, "sub1file2"); + DFSTestUtil.createFile(hdfs, sub1file1, BLOCKSIZE, REPLICATION, seed); + DFSTestUtil.createFile(hdfs, sub1file2, BLOCKSIZE, REPLICATION, seed); + + hdfs.createSnapshot(dir, "s1"); + + Path sub2 = new Path(dir, "sub2"); + Path sub2file1 = new Path(sub2, "sub2file1"); + Path sub2file2 = new Path(sub2, "sub2file2"); + DFSTestUtil.createFile(hdfs, sub2file1, BLOCKSIZE, REPLICATION, seed); + DFSTestUtil.createFile(hdfs, sub2file2, BLOCKSIZE, REPLICATION, seed); + hdfs.setReplication(sub1file1, (short) (REPLICATION - 1)); + hdfs.delete(sub1file2, true); + + hdfs.createSnapshot(dir, "s2"); + hdfs.setOwner(sub2, "dr.who", "unknown"); + hdfs.delete(sub2file2, true); + + // dump the fsdir tree + StringBuffer fsnStrBefore = fsn.getFSDirectory().rootDir + .dumpTreeRecursively(); + + // save the namesystem to a temp file + SaveNamespaceContext context = new SaveNamespaceContext(fsn, txid, + new Canceler()); + FSImageFormat.Saver saver = new FSImageFormat.Saver(context); + FSImageCompression compression = FSImageCompression.createCompression(conf); + File dstFile = getStorageFile(testDir, txid); + fsn.readLock(); + try { + saver.save(dstFile, compression); + } finally { + fsn.readUnlock(); + } + + // restart the cluster, and format the cluster + cluster.shutdown(); + cluster = new MiniDFSCluster.Builder(conf).format(true) + .numDataNodes(REPLICATION).build(); + cluster.waitActive(); + fsn = cluster.getNamesystem(); + hdfs = cluster.getFileSystem(); + + // load the namesystem from the temp file + FSImageFormat.Loader loader = new FSImageFormat.Loader(conf, fsn); + fsn.writeLock(); + try { + loader.load(dstFile); + } finally { + fsn.writeUnlock(); + } + + // dump the fsdir tree again + StringBuffer fsnStrAfter = fsn.getFSDirectory().rootDir + .dumpTreeRecursively(); + + // compare two dumped tree + System.out.println(fsnStrBefore.toString()); + System.out.println("\n" + fsnStrAfter.toString()); + assertEquals(fsnStrBefore.toString(), fsnStrAfter.toString()); + } + + /** + * Create a temp fsimage file for testing. + * @param dir The directory where the fsimage file resides + * @param imageTxId The transaction id of the fsimage + * @return The file of the image file + */ + private File getStorageFile(String dir, long imageTxId) { + return new File(dir, String.format("%s_%019d", NameNodeFile.IMAGE, + imageTxId)); + } +} \ No newline at end of file Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotTestHelper.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotTestHelper.java?rev=1437256&r1=1437255&r2=1437256&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotTestHelper.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotTestHelper.java Wed Jan 23 02:48:01 2013 @@ -127,7 +127,6 @@ public class SnapshotTestHelper { static class TestDirectoryTree { /** Height of the directory tree */ final int height; - final FileSystem fs; /** Top node of the directory tree */ final Node topNode; /** A map recording nodes for each tree level */ @@ -138,12 +137,11 @@ public class SnapshotTestHelper { */ TestDirectoryTree(int height, FileSystem fs) throws Exception { this.height = height; - this.fs = fs; this.topNode = new Node(new Path("/TestSnapshot"), 0, null, fs); this.levelMap = new HashMap<Integer, ArrayList<Node>>(); addDirNode(topNode, 0); - genChildren(topNode, height - 1); + genChildren(topNode, height - 1, fs); } /** @@ -164,9 +162,11 @@ public class SnapshotTestHelper { * * @param parent The parent node * @param level The remaining levels to generate + * @param fs The FileSystem where to generate the files/dirs * @throws Exception */ - void genChildren(Node parent, int level) throws Exception { + private void genChildren(Node parent, int level, FileSystem fs) + throws Exception { if (level == 0) { return; } @@ -176,8 +176,8 @@ public class SnapshotTestHelper { "right" + ++id), height - level, parent, fs); addDirNode(parent.leftChild, parent.leftChild.level); addDirNode(parent.rightChild, parent.rightChild.level); - genChildren(parent.leftChild, level - 1); - genChildren(parent.rightChild, level - 1); + genChildren(parent.leftChild, level - 1, fs); + genChildren(parent.rightChild, level - 1, fs); } /** @@ -246,7 +246,6 @@ public class SnapshotTestHelper { * directory creation/deletion */ final ArrayList<Node> nonSnapshotChildren; - final FileSystem fs; Node(Path path, int level, Node parent, FileSystem fs) throws Exception { @@ -254,7 +253,6 @@ public class SnapshotTestHelper { this.level = level; this.parent = parent; this.nonSnapshotChildren = new ArrayList<Node>(); - this.fs = fs; fs.mkdirs(nodePath); } @@ -262,8 +260,8 @@ public class SnapshotTestHelper { * Create files and add them in the fileList. Initially the last element * in the fileList is set to null (where we start file creation). */ - void initFileList(String namePrefix, long fileLen, short replication, - long seed, int numFiles) throws Exception { + void initFileList(FileSystem fs, String namePrefix, long fileLen, + short replication, long seed, int numFiles) throws Exception { fileList = new ArrayList<Path>(numFiles); for (int i = 0; i < numFiles; i++) { Path file = new Path(nodePath, namePrefix + "-f" + i); Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestINodeDirectoryWithSnapshot.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestINodeDirectoryWithSnapshot.java?rev=1437256&r1=1437255&r2=1437256&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestINodeDirectoryWithSnapshot.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestINodeDirectoryWithSnapshot.java Wed Jan 23 02:48:01 2013 @@ -148,7 +148,7 @@ public class TestINodeDirectoryWithSnaps // combine all diffs final Diff combined = diffs[0]; for(int i = 1; i < diffs.length; i++) { - combined.combinePostDiff(diffs[i], null); + combined.combinePostDiff(diffs[i], null, false); } { @@ -284,7 +284,7 @@ public class TestINodeDirectoryWithSnaps before = toString(diff); } - final Triple<Integer, INode, Integer> undoInfo = diff.delete(inode); + final Triple<Integer, INode, Integer> undoInfo = diff.delete(inode, true); if (testUndo) { final String after = toString(diff); @@ -292,7 +292,7 @@ public class TestINodeDirectoryWithSnaps diff.undoDelete(inode, undoInfo); assertDiff(before, diff); //re-do - diff.delete(inode); + diff.delete(inode, true); assertDiff(after, diff); } } @@ -314,7 +314,7 @@ public class TestINodeDirectoryWithSnaps before = toString(diff); } - final Triple<Integer, INode, Integer> undoInfo = diff.modify(oldinode, newinode); + final Triple<Integer, INode, Integer> undoInfo = diff.modify(oldinode, newinode, true); if (testUndo) { final String after = toString(diff); @@ -322,7 +322,7 @@ public class TestINodeDirectoryWithSnaps diff.undoModify(oldinode, newinode, undoInfo); assertDiff(before, diff); //re-do - diff.modify(oldinode, newinode); + diff.modify(oldinode, newinode, true); assertDiff(after, diff); } } Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshot.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshot.java?rev=1437256&r1=1437255&r2=1437256&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshot.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshot.java Wed Jan 23 02:48:01 2013 @@ -36,6 +36,7 @@ import org.apache.hadoop.fs.permission.F import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotTestHelper.TestDirectoryTree; import org.apache.hadoop.ipc.RemoteException; @@ -55,7 +56,7 @@ public class TestSnapshot { private static final long seed = Time.now(); protected static final short REPLICATION = 3; protected static final long BLOCKSIZE = 1024; - /** The number of times snapshots are created for a snapshottable directory */ + /** The number of times snapshots are created for a snapshottable directory */ public static final int SNAPSHOT_ITERATION_NUMBER = 20; /** Height of directory tree used for testing */ public static final int DIRECTORY_TREE_LEVEL = 5; @@ -143,6 +144,49 @@ public class TestSnapshot { return nodes; } + /** + * Restart the cluster to check edit log applying and fsimage saving/loading + */ + private void checkFSImage() throws Exception { + String rootDir = "/"; + StringBuffer fsnStrBefore = fsn.getFSDirectory().getINode(rootDir) + .dumpTreeRecursively(); + + cluster.shutdown(); + cluster = new MiniDFSCluster.Builder(conf).format(false) + .numDataNodes(REPLICATION).build(); + cluster.waitActive(); + fsn = cluster.getNamesystem(); + hdfs = cluster.getFileSystem(); + // later check fsnStrMiddle to see if the edit log is recorded and applied + // correctly + StringBuffer fsnStrMiddle = fsn.getFSDirectory().getINode(rootDir) + .dumpTreeRecursively(); + + // save namespace and restart cluster + hdfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); + hdfs.saveNamespace(); + hdfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); + cluster.shutdown(); + cluster = new MiniDFSCluster.Builder(conf).format(false) + .numDataNodes(REPLICATION).build(); + cluster.waitActive(); + fsn = cluster.getNamesystem(); + hdfs = cluster.getFileSystem(); + // dump the namespace loaded from fsimage + StringBuffer fsnStrAfter = fsn.getFSDirectory().getINode(rootDir) + .dumpTreeRecursively(); + + System.out.println("================== Original FSDir =================="); + System.out.println(fsnStrBefore.toString()); + System.out.println("================== FSDir After Applying Edit Logs =================="); + System.out.println(fsnStrMiddle.toString()); + System.out.println("================ FSDir After FSImage Saving/Loading ================"); + System.out.println(fsnStrAfter.toString()); + System.out.println("===================================================="); + assertEquals(fsnStrBefore.toString(), fsnStrMiddle.toString()); + assertEquals(fsnStrBefore.toString(), fsnStrAfter.toString()); + } /** * Main test, where we will go in the following loop: @@ -184,6 +228,9 @@ public class TestSnapshot { Modification chown = new FileChown(chownDir.nodePath, hdfs, userGroup[0], userGroup[1]); modifyCurrentDirAndCheckSnapshots(new Modification[]{chmod, chown}); + + // check fsimage saving/loading + checkFSImage(); } System.out.println("XXX done:"); SnapshotTestHelper.dumpTreeRecursively(fsn.getFSDirectory().getINode("/")); @@ -244,7 +291,8 @@ public class TestSnapshot { for (TestDirectoryTree.Node node : nodes) { // If the node does not have files in it, create files if (node.fileList == null) { - node.initFileList(node.nodePath.getName(), BLOCKSIZE, REPLICATION, seed, 6); + node.initFileList(hdfs, node.nodePath.getName(), BLOCKSIZE, + REPLICATION, seed, 5); } // @@ -270,18 +318,21 @@ public class TestSnapshot { Modification delete = new FileDeletion( node.fileList.get((node.nullFileIndex + 1) % node.fileList.size()), hdfs); - Modification append = new FileAppend( - node.fileList.get((node.nullFileIndex + 2) % node.fileList.size()), - hdfs, (int) BLOCKSIZE); + + // TODO: temporarily disable file append testing before supporting + // INodeFileUnderConstructionWithSnapshot in FSImage saving/loading +// Modification append = new FileAppend( +// node.fileList.get((node.nullFileIndex + 2) % node.fileList.size()), +// hdfs, (int) BLOCKSIZE); Modification chmod = new FileChangePermission( - node.fileList.get((node.nullFileIndex + 3) % node.fileList.size()), + node.fileList.get((node.nullFileIndex + 2) % node.fileList.size()), hdfs, genRandomPermission()); String[] userGroup = genRandomOwner(); Modification chown = new FileChown( - node.fileList.get((node.nullFileIndex + 4) % node.fileList.size()), + node.fileList.get((node.nullFileIndex + 3) % node.fileList.size()), hdfs, userGroup[0], userGroup[1]); Modification replication = new FileChangeReplication( - node.fileList.get((node.nullFileIndex + 5) % node.fileList.size()), + node.fileList.get((node.nullFileIndex + 4) % node.fileList.size()), hdfs, (short) (random.nextInt(REPLICATION) + 1)); node.nullFileIndex = (node.nullFileIndex + 1) % node.fileList.size(); Modification dirChange = new DirCreationOrDeletion(node.nodePath, hdfs, @@ -289,7 +340,8 @@ public class TestSnapshot { mList.add(create); mList.add(delete); - mList.add(append); + //TODO + //mList.add(append); mList.add(chmod); mList.add(chown); mList.add(replication); @@ -606,7 +658,7 @@ public class TestSnapshot { /** * Directory creation or deletion. */ - static class DirCreationOrDeletion extends Modification { + class DirCreationOrDeletion extends Modification { private final TestDirectoryTree.Node node; private final boolean isCreation; private final Path changedPath; @@ -656,15 +708,16 @@ public class TestSnapshot { if (isCreation) { // creation TestDirectoryTree.Node newChild = new TestDirectoryTree.Node( - changedPath, node.level + 1, node, node.fs); + changedPath, node.level + 1, node, hdfs); // create file under the new non-snapshottable directory - newChild.initFileList(node.nodePath.getName(), BLOCKSIZE, REPLICATION, seed, 2); + newChild.initFileList(hdfs, node.nodePath.getName(), BLOCKSIZE, + REPLICATION, seed, 2); node.nonSnapshotChildren.add(newChild); } else { // deletion TestDirectoryTree.Node childToDelete = node.nonSnapshotChildren .remove(node.nonSnapshotChildren.size() - 1); - node.fs.delete(childToDelete.nodePath, true); + hdfs.delete(childToDelete.nodePath, true); } } Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotRename.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotRename.java?rev=1437256&r1=1437255&r2=1437256&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotRename.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotRename.java Wed Jan 23 02:48:01 2013 @@ -32,6 +32,7 @@ import org.apache.hadoop.hdfs.MiniDFSClu import org.apache.hadoop.hdfs.server.namenode.FSDirectory; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectoryWithSnapshot.SnapshotDiff; +import org.apache.hadoop.hdfs.util.ReadOnlyList; import org.apache.hadoop.ipc.RemoteException; import org.junit.After; import org.junit.Before; @@ -85,7 +86,7 @@ public class TestSnapshotRename { */ private void checkSnapshotList(INodeDirectorySnapshottable srcRoot, String[] sortedNames, String[] names) { - List<Snapshot> listByName = srcRoot.getSnapshotsByNames(); + ReadOnlyList<Snapshot> listByName = srcRoot.getSnapshotsByNames(); assertEquals(sortedNames.length, listByName.size()); for (int i = 0; i < listByName.size(); i++) { assertEquals(sortedNames[i], listByName.get(i).getRoot().getLocalName());
