Author: szetszwo
Date: Thu Nov 15 23:08:25 2012
New Revision: 1410097
URL: http://svn.apache.org/viewvc?rev=1410097&view=rev
Log:
HDFS-4187. Add tests for replication handling in snapshots. Contributed by Jing
Zhao
Added:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotReplication.java
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-2802.txt
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-2802.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-2802.txt?rev=1410097&r1=1410096&r2=1410097&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-2802.txt
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-2802.txt
Thu Nov 15 23:08:25 2012
@@ -73,3 +73,6 @@ Branch-2802 Snapshot (Unreleased)
HDFS-4188. Add Snapshot.ID_COMPARATOR for comparing IDs and fix a bug in
ReadOnlyList.Util.binarySearch(..). (szetszwo)
+
+ HDFS-4187. Add tests for replication handling in snapshots. (Jing Zhao via
+ szetszwo)
Added:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotReplication.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotReplication.java?rev=1410097&view=auto
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotReplication.java
(added)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotReplication.java
Thu Nov 15 23:08:25 2012
@@ -0,0 +1,229 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode.snapshot;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
+import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
+import org.apache.hadoop.hdfs.server.namenode.INode;
+import org.apache.hadoop.hdfs.server.namenode.INodeFile;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * This class tests the replication handling/calculation of snapshots. In
+ * particular, {@link INodeFile#getFileReplication()} and
+ * {@link INodeFileWithLink#getBlockReplication()} are tested to make sure
+ * the number of replication is calculated correctly with/without snapshots.
+ */
+public class TestSnapshotReplication {
+
+ private static final long seed = 0;
+ private static final short REPLICATION = 3;
+ private static final int NUMDATANODE = 5;
+ private static final long BLOCKSIZE = 1024;
+
+ private final Path dir = new Path("/TestSnapshot");
+ private final Path sub1 = new Path(dir, "sub1");
+ private final Path file1 = new Path(sub1, "file1");
+
+ Configuration conf;
+ MiniDFSCluster cluster;
+ FSNamesystem fsn;
+ DistributedFileSystem hdfs;
+ FSDirectory fsdir;
+
+ @Before
+ public void setUp() throws Exception {
+ conf = new Configuration();
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUMDATANODE)
+ .build();
+ cluster.waitActive();
+ fsn = cluster.getNamesystem();
+ hdfs = cluster.getFileSystem();
+ fsdir = fsn.getFSDirectory();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ /**
+ * Check the replication of a given file. We test both
+ * {@link INodeFile#getFileReplication()} and
+ * {@link INodeFile#getBlockReplication()}.
+ *
+ * @param file The given file
+ * @param replication The expected replication number
+ * @param blockReplication The expected replication number for the block
+ * @throws Exception
+ */
+ private void checkFileReplication(Path file, short replication,
+ short blockReplication) throws Exception {
+ // Get FileStatus of file1, and identify the replication number of file1.
+ // Note that the replication number in FileStatus was derived from
+ // INodeFile#getFileReplication().
+ short fileReplication = hdfs.getFileStatus(file1).getReplication();
+ assertEquals(replication, fileReplication);
+ // Check the correctness of getBlockReplication()
+ INode inode = fsdir.getINode(file1.toString());
+ assertTrue(inode instanceof INodeFile);
+ assertEquals(blockReplication, ((INodeFile) inode).getBlockReplication());
+ }
+
+ /**
+ * Test replication number calculation for a normal file without snapshots.
+ */
+ @Test
+ public void testReplicationWithoutSnapshot() throws Exception {
+ // Create file1, set its replication to REPLICATION
+ DFSTestUtil.createFile(hdfs, file1, BLOCKSIZE, REPLICATION, seed);
+ // Check the replication of file1
+ checkFileReplication(file1, REPLICATION, REPLICATION);
+ // Change the replication factor of file1 from 3 to 2
+ hdfs.setReplication(file1, (short) (REPLICATION - 1));
+ // Check the replication again
+ checkFileReplication(file1, (short) (REPLICATION - 1),
+ (short) (REPLICATION - 1));
+ }
+
+ /**
+ * Check the replication for both the current file and all its prior
snapshots
+ *
+ * @param currentFile
+ * the Path of the current file
+ * @param snapshotRepMap
+ * A map maintaining all the snapshots of the current file, as well
+ * as their expected replication number stored in their
corresponding
+ * INodes
+ * @param expectedBlockRep
+ * The expected replication number that should be returned by
+ * {@link INodeFileWithLink#getBlockReplication()} of all the INodes
+ * @throws Exception
+ */
+ private void checkSnapshotFileReplication(Path currentFile,
+ Map<Path, Short> snapshotRepMap, short expectedBlockRep) throws
Exception {
+ // First check the getBlockReplication for the INode of the currentFile
+ INodeFileWithLink inodeOfCurrentFile = (INodeFileWithLink) fsdir
+ .getINode(currentFile.toString());
+ assertEquals(expectedBlockRep, inodeOfCurrentFile.getBlockReplication());
+ // Then check replication for every snapshot
+ for (Path ss : snapshotRepMap.keySet()) {
+ INodeFileWithLink ssInode = (INodeFileWithLink) fsdir.getINode(ss
+ .toString());
+ // The replication number derived from the
+ // INodeFileWithLink#getBlockReplication should always ==
expectedBlockRep
+ assertEquals(expectedBlockRep, ssInode.getBlockReplication());
+ // Also check the number derived from INodeFile#getFileReplication
+ assertEquals(snapshotRepMap.get(ss).shortValue(),
+ ssInode.getFileReplication());
+ }
+ }
+
+ /**
+ * Test replication number calculation for a file with snapshots.
+ */
+ @Test
+ public void testReplicationWithSnapshot() throws Exception {
+ short fileRep = 1;
+ // Create file1, set its replication to 1
+ DFSTestUtil.createFile(hdfs, file1, BLOCKSIZE, fileRep, seed);
+ Map<Path, Short> snapshotRepMap = new HashMap<Path, Short>();
+ // Change replication factor from 1 to 5. In the meanwhile, keep taking
+ // snapshots for sub1
+ for (; fileRep < NUMDATANODE; ) {
+ // Create snapshot for sub1
+ Path snapshotRoot = SnapshotTestHelper.createSnapshot(hdfs, sub1, "s"
+ + fileRep);
+ Path snapshot = new Path(snapshotRoot, file1.getName());
+
+ // Check the replication stored in the INode of the snapshot of file1
+ INode inode = fsdir.getINode(snapshot.toString());
+ assertTrue(inode instanceof INodeFileWithLink);
+ assertEquals(fileRep, ((INodeFileWithLink) inode).getFileReplication());
+ snapshotRepMap.put(snapshot, fileRep);
+
+ // Increase the replication factor by 1
+ hdfs.setReplication(file1, ++fileRep);
+ // Check the replication for file1
+ checkFileReplication(file1, fileRep, fileRep);
+ // Also check the replication for all the prior snapshots of file1
+ checkSnapshotFileReplication(file1, snapshotRepMap, fileRep);
+ }
+
+ // Change replication factor back to 3.
+ hdfs.setReplication(file1, REPLICATION);
+ // Check the replication for file1
+ // Currently the max replication among snapshots should be 4
+ checkFileReplication(file1, REPLICATION, (short) (NUMDATANODE - 1));
+ // Also check the replication for all the prior snapshots of file1.
+ // Currently the max replication among snapshots should be 4
+ checkSnapshotFileReplication(file1, snapshotRepMap,
+ (short) (NUMDATANODE - 1));
+ }
+
+ /**
+ * Test replication for a file with snapshots, also including the scenario
+ * where the original file is deleted
+ */
+ @Test
+ public void testReplicationAfterDeletion() throws Exception {
+ // Create file1, set its replication to 3
+ DFSTestUtil.createFile(hdfs, file1, BLOCKSIZE, REPLICATION, seed);
+ Map<Path, Short> snapshotRepMap = new HashMap<Path, Short>();
+ // Take 3 snapshots of sub1
+ for (int i = 1; i <= 3; i++) {
+ Path root = SnapshotTestHelper.createSnapshot(hdfs, sub1, "s" + i);
+ Path ssFile = new Path(root, file1.getName());
+ snapshotRepMap.put(ssFile, REPLICATION);
+ }
+ // Check replication
+ checkFileReplication(file1, REPLICATION, REPLICATION);
+ checkSnapshotFileReplication(file1, snapshotRepMap, REPLICATION);
+
+ // TODO: check replication after deleting snapshot(s)
+ // Delete file1
+ hdfs.delete(file1, true);
+ // Check replication of snapshots
+ for (Path ss : snapshotRepMap.keySet()) {
+ INodeFileWithLink ssInode = (INodeFileWithLink) fsdir.getINode(ss
+ .toString());
+ // The replication number derived from the
+ // INodeFileWithLink#getBlockReplication should always ==
expectedBlockRep
+ assertEquals(REPLICATION, ssInode.getBlockReplication());
+ // Also check the number derived from INodeFile#getFileReplication
+ assertEquals(snapshotRepMap.get(ss).shortValue(),
+ ssInode.getFileReplication());
+ }
+ }
+
+}
