Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java?rev=1242635&r1=1242634&r2=1242635&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java Fri Feb 10 01:49:08 2012 @@ -17,14 +17,14 @@ */ package org.apache.hadoop.hdfs.server.namenode; +import static org.junit.Assert.*; + import java.io.File; import java.io.IOException; import java.net.InetSocketAddress; import java.util.Collections; import java.util.List; -import junit.framework.TestCase; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.impl.Log4JLogger; @@ -42,13 +42,15 @@ import org.apache.hadoop.hdfs.server.nam import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; import org.apache.hadoop.test.GenericTestUtils; import org.apache.log4j.Level; +import org.junit.Before; +import org.junit.Test; import com.google.common.base.Supplier; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; -public class TestBackupNode extends TestCase { +public class TestBackupNode { public static final Log LOG = LogFactory.getLog(TestBackupNode.class); @@ -59,8 +61,8 @@ public class TestBackupNode extends Test static final String BASE_DIR = MiniDFSCluster.getBaseDirectory(); - protected void setUp() throws Exception { - super.setUp(); + @Before + public void setUp() throws Exception { File baseDir = new File(BASE_DIR); if(baseDir.exists()) if(!(FileUtil.fullyDelete(baseDir))) @@ -91,8 +93,7 @@ public class TestBackupNode extends Test return (BackupNode)NameNode.createNameNode(new String[]{startupOpt.getName()}, c); } - void waitCheckpointDone( - MiniDFSCluster cluster, BackupNode backup, long txid) { + void waitCheckpointDone(MiniDFSCluster cluster, long txid) { long thisCheckpointTxId; do { try { @@ -100,16 +101,16 @@ public class TestBackupNode extends Test "checkpoint txid should increase above " + txid); Thread.sleep(1000); } catch (Exception e) {} - thisCheckpointTxId = backup.getFSImage().getStorage() + // The checkpoint is not done until the nn has received it from the bn + thisCheckpointTxId = cluster.getNameNode().getFSImage().getStorage() .getMostRecentCheckpointTxId(); - } while (thisCheckpointTxId < txid); - // Check that the checkpoint got uploaded to NN successfully FSImageTestUtil.assertNNHasCheckpoints(cluster, Collections.singletonList((int)thisCheckpointTxId)); } + @Test public void testCheckpointNode() throws Exception { testCheckpoint(StartupOption.CHECKPOINT); } @@ -119,6 +120,7 @@ public class TestBackupNode extends Test * and keep in sync, even while the NN rolls, checkpoints * occur, etc. */ + @Test public void testBackupNodeTailsEdits() throws Exception { Configuration conf = new HdfsConfiguration(); HAUtil.setAllowStandbyReads(conf, true); @@ -237,6 +239,7 @@ public class TestBackupNode extends Test FSImageTestUtil.assertParallelFilesAreIdentical(dirs, ImmutableSet.of("VERSION")); } + @Test public void testBackupNode() throws Exception { testCheckpoint(StartupOption.BACKUP); } @@ -277,7 +280,7 @@ public class TestBackupNode extends Test // long txid = cluster.getNameNodeRpc().getTransactionID(); backup = startBackupNode(conf, op, 1); - waitCheckpointDone(cluster, backup, txid); + waitCheckpointDone(cluster, txid); } catch(IOException e) { LOG.error("Error in TestBackupNode:", e); assertTrue(e.getLocalizedMessage(), false); @@ -312,7 +315,7 @@ public class TestBackupNode extends Test // backup = startBackupNode(conf, op, 1); long txid = cluster.getNameNodeRpc().getTransactionID(); - waitCheckpointDone(cluster, backup, txid); + waitCheckpointDone(cluster, txid); for (int i = 0; i < 10; i++) { fileSys.mkdirs(new Path("file_" + i)); @@ -320,11 +323,11 @@ public class TestBackupNode extends Test txid = cluster.getNameNodeRpc().getTransactionID(); backup.doCheckpoint(); - waitCheckpointDone(cluster, backup, txid); + waitCheckpointDone(cluster, txid); txid = cluster.getNameNodeRpc().getTransactionID(); backup.doCheckpoint(); - waitCheckpointDone(cluster, backup, txid); + waitCheckpointDone(cluster, txid); // Try BackupNode operations InetSocketAddress add = backup.getNameNodeAddress();
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java?rev=1242635&r1=1242634&r2=1242635&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java Fri Feb 10 01:49:08 2012 @@ -38,6 +38,9 @@ import org.apache.hadoop.hdfs.server.pro import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo; import org.apache.hadoop.hdfs.server.protocol.RegisterCommand; +import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport; +import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks; +import org.apache.hadoop.hdfs.server.protocol.StorageReport; import org.junit.After; import org.junit.Test; @@ -110,19 +113,22 @@ public class TestDeadDatanode { new Block(0), ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, null) }; + StorageReceivedDeletedBlocks[] storageBlocks = { + new StorageReceivedDeletedBlocks(reg.getStorageID(), blocks) }; // Ensure blockReceived call from dead datanode is rejected with IOException try { - dnp.blockReceivedAndDeleted(reg, poolId, blocks); + dnp.blockReceivedAndDeleted(reg, poolId, storageBlocks); Assert.fail("Expected IOException is not thrown"); } catch (IOException ex) { // Expected } // Ensure blockReport from dead datanode is rejected with IOException - long[] blockReport = new long[] { 0L, 0L, 0L }; + StorageBlockReport[] report = { new StorageBlockReport(reg.getStorageID(), + new long[] { 0L, 0L, 0L }) }; try { - dnp.blockReport(reg, poolId, blockReport); + dnp.blockReport(reg, poolId, report); Assert.fail("Expected IOException is not thrown"); } catch (IOException ex) { // Expected @@ -130,8 +136,9 @@ public class TestDeadDatanode { // Ensure heartbeat from dead datanode is rejected with a command // that asks datanode to register again - DatanodeCommand[] cmd = dnp.sendHeartbeat(reg, 0, 0, 0, 0, 0, 0, 0) - .getCommands(); + StorageReport[] rep = { new StorageReport(reg.getStorageID(), false, 0, 0, + 0, 0) }; + DatanodeCommand[] cmd = dnp.sendHeartbeat(reg, rep, 0, 0, 0).getCommands(); Assert.assertEquals(1, cmd.length); Assert.assertEquals(cmd[0].getAction(), RegisterCommand.REGISTER .getAction()); Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java?rev=1242635&r1=1242634&r2=1242635&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java Fri Feb 10 01:49:08 2012 @@ -116,12 +116,12 @@ public class TestEditLog extends TestCas int numTransactions; short replication = 3; long blockSize = 64; - final int id; + int startIndex; - Transactions(FSNamesystem ns, int num, int id) { + Transactions(FSNamesystem ns, int numTx, int startIdx) { namesystem = ns; - numTransactions = num; - this.id = id; + numTransactions = numTx; + startIndex = startIdx; } // add a bunch of transactions. @@ -133,9 +133,8 @@ public class TestEditLog extends TestCas for (int i = 0; i < numTransactions; i++) { INodeFileUnderConstruction inode = new INodeFileUnderConstruction( p, replication, blockSize, 0, "", "", null); - String fileName = "/filename-" + id + "-" + i; - editLog.logOpenFile(fileName, inode); - editLog.logCloseFile(fileName, inode); + editLog.logOpenFile("/filename" + (startIndex + i), inode); + editLog.logCloseFile("/filename" + (startIndex + i), inode); editLog.logSync(); } } @@ -283,7 +282,8 @@ public class TestEditLog extends TestCas // Create threads and make them run transactions concurrently. Thread threadId[] = new Thread[NUM_THREADS]; for (int i = 0; i < NUM_THREADS; i++) { - Transactions trans = new Transactions(namesystem, NUM_TRANSACTIONS, i); + Transactions trans = + new Transactions(namesystem, NUM_TRANSACTIONS, i*NUM_TRANSACTIONS); threadId[i] = new Thread(trans, "TransactionThread-" + i); threadId[i].start(); } @@ -296,11 +296,16 @@ public class TestEditLog extends TestCas i--; // retry } } - + + // Reopen some files as for append + Transactions trans = + new Transactions(namesystem, NUM_TRANSACTIONS, NUM_TRANSACTIONS / 2); + trans.run(); + // Roll another time to finalize edits_inprogress_3 fsimage.rollEditLog(); - long expectedTxns = (NUM_THREADS * 2 * NUM_TRANSACTIONS) + 2; // +2 for start/end txns + long expectedTxns = ((NUM_THREADS+1) * 2 * NUM_TRANSACTIONS) + 2; // +2 for start/end txns // Verify that we can read in all the transactions that we have written. // If there were any corruptions, it is likely that the reading in Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java?rev=1242635&r1=1242634&r2=1242635&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java Fri Feb 10 01:49:08 2012 @@ -25,6 +25,7 @@ import javax.management.ObjectName; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; +import org.apache.hadoop.util.VersionInfo; import org.junit.Test; import junit.framework.Assert; @@ -57,6 +58,8 @@ public class TestNameNodeMXBean { // get attribute "Version" String version = (String) mbs.getAttribute(mxbeanName, "Version"); Assert.assertEquals(fsn.getVersion(), version); + Assert.assertTrue(version.equals(VersionInfo.getVersion() + + ", r" + VersionInfo.getRevision())); // get attribute "Used" Long used = (Long) mbs.getAttribute(mxbeanName, "Used"); Assert.assertEquals(fsn.getUsed(), used.longValue()); Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java?rev=1242635&r1=1242634&r2=1242635&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java Fri Feb 10 01:49:08 2012 @@ -18,9 +18,10 @@ package org.apache.hadoop.hdfs.server.namenode; import static org.apache.hadoop.hdfs.server.common.Util.fileAsURI; - -import static org.junit.Assert.*; - +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import static org.mockito.Matchers.anyObject; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doThrow; @@ -28,6 +29,7 @@ import static org.mockito.Mockito.spy; import java.io.File; import java.io.IOException; +import java.io.OutputStream; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -38,16 +40,19 @@ import org.apache.commons.logging.LogFac import org.apache.commons.logging.impl.Log4JLogger; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.permission.PermissionStatus; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; +import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; import org.apache.hadoop.hdfs.util.MD5FileUtils; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.test.GenericTestUtils.DelayAnswer; import org.apache.log4j.Level; @@ -565,6 +570,34 @@ public class TestSaveNamespace { } } } + + /** + * Test for save namespace should succeed when parent directory renamed with + * open lease and destination directory exist. + * This test is a regression for HDFS-2827 + */ + @Test + public void testSaveNamespaceWithRenamedLease() throws Exception { + MiniDFSCluster cluster = new MiniDFSCluster.Builder(new Configuration()) + .numDataNodes(1).build(); + cluster.waitActive(); + DistributedFileSystem fs = (DistributedFileSystem) cluster.getFileSystem(); + OutputStream out = null; + try { + fs.mkdirs(new Path("/test-target")); + out = fs.create(new Path("/test-source/foo")); // don't close + fs.rename(new Path("/test-source/"), new Path("/test-target/")); + + fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); + cluster.getNameNodeRpc().saveNamespace(); + fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); + } finally { + IOUtils.cleanup(LOG, out, fs); + if (cluster != null) { + cluster.shutdown(); + } + } + } private void doAnEdit(FSNamesystem fsn, int id) throws IOException { // Make an edit Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java?rev=1242635&r1=1242634&r2=1242635&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java Fri Feb 10 01:49:08 2012 @@ -53,6 +53,7 @@ import org.apache.hadoop.hdfs.server.nam import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; +import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.test.GenericTestUtils.DelayAnswer; @@ -501,7 +502,7 @@ public class TestDNFencing { .when(spy).blockReport( Mockito.<DatanodeRegistration>anyObject(), Mockito.anyString(), - Mockito.<long[]>anyObject()); + Mockito.<StorageBlockReport[]>anyObject()); dn.scheduleAllBlockReport(0); delayer.waitForCall();
