Modified: hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java?rev=1228651&r1=1228650&r2=1228651&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java (original) +++ hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java Sat Jan 7 16:28:27 2012 @@ -23,6 +23,9 @@ import java.net.URI; import java.util.Collection; import java.util.Iterator; import java.util.List; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Arrays; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -37,6 +40,7 @@ import org.apache.hadoop.fs.ChecksumExce import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.*; +import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.HdfsConfiguration; @@ -54,6 +58,7 @@ import org.aspectj.util.FileUtil; import org.mockito.Mockito; import org.junit.Test; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import static org.apache.hadoop.test.MetricsAsserts.*; @@ -76,7 +81,7 @@ public class TestEditLog extends TestCas static final int NUM_TRANSACTIONS = 100; static final int NUM_THREADS = 100; - private static final File TEST_DIR = new File( + static final File TEST_DIR = new File( System.getProperty("test.build.data","build/test/data")); /** An edits log with 3 edits from 0.20 - the result of @@ -623,13 +628,23 @@ public class TestEditLog extends TestCas } public void testCrashRecoveryEmptyLogOneDir() throws Exception { - doTestCrashRecoveryEmptyLog(false); + doTestCrashRecoveryEmptyLog(false, true); } public void testCrashRecoveryEmptyLogBothDirs() throws Exception { - doTestCrashRecoveryEmptyLog(true); + doTestCrashRecoveryEmptyLog(true, true); + } + + public void testCrashRecoveryEmptyLogOneDirNoUpdateSeenTxId() + throws Exception { + doTestCrashRecoveryEmptyLog(false, false); } + public void testCrashRecoveryEmptyLogBothDirsNoUpdateSeenTxId() + throws Exception { + doTestCrashRecoveryEmptyLog(true, false); + } + /** * Test that the NN handles the corruption properly * after it crashes just after creating an edit log @@ -642,8 +657,14 @@ public class TestEditLog extends TestCas * will only be in one of the directories. In both cases, the * NN should fail to start up, because it's aware that txid 3 * was reached, but unable to find a non-corrupt log starting there. + * @param updateTransactionIdFile if true update the seen_txid file. + * If false, the it will not be updated. This will simulate a case + * where the NN crashed between creating the new segment and updating + * seen_txid. */ - private void doTestCrashRecoveryEmptyLog(boolean inBothDirs) throws Exception { + private void doTestCrashRecoveryEmptyLog(boolean inBothDirs, + boolean updateTransactionIdFile) + throws Exception { // start a cluster Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = null; @@ -661,6 +682,14 @@ public class TestEditLog extends TestCas // Make a truncated edits_3_inprogress File log = new File(currentDir, NNStorage.getInProgressEditsFileName(3)); + NNStorage storage = new NNStorage(conf, + Collections.<URI>emptyList(), + Lists.newArrayList(uri)); + if (updateTransactionIdFile) { + storage.writeTransactionIdFileToStorage(3); + } + storage.close(); + new EditLogFileOutputStream(log, 1024).create(); if (!inBothDirs) { break; @@ -671,9 +700,9 @@ public class TestEditLog extends TestCas cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(NUM_DATA_NODES).format(false).build(); fail("Did not fail to start with all-corrupt logs"); - } catch (IllegalStateException ise) { + } catch (IOException ioe) { GenericTestUtils.assertExceptionContains( - "No non-corrupt logs for txid 3", ise); + "No non-corrupt logs for txid 3", ioe); } cluster.shutdown(); } @@ -698,8 +727,18 @@ public class TestEditLog extends TestCas reader = new FSEditLogOp.Reader(in, version); } + + @Override + public long getFirstTxId() throws IOException { + return HdfsConstants.INVALID_TXID; + } @Override + public long getLastTxId() throws IOException { + return HdfsConstants.INVALID_TXID; + } + + @Override public long length() throws IOException { return len; } @@ -848,6 +887,168 @@ public class TestEditLog extends TestCas Mockito.doReturn(sds).when(storage).dirIterable(NameNodeDirType.EDITS); return storage; } - - + + /** + * Specification for a failure during #setupEdits + */ + static class AbortSpec { + final int roll; + final int logindex; + + /** + * Construct the failure specification. + * @param roll number to fail after. e.g. 1 to fail after the first roll + * @param loginfo index of journal to fail. + */ + AbortSpec(int roll, int logindex) { + this.roll = roll; + this.logindex = logindex; + } + } + + final static int TXNS_PER_ROLL = 10; + final static int TXNS_PER_FAIL = 2; + + /** + * Set up directories for tests. + * + * Each rolled file is 10 txns long. + * A failed file is 2 txns long. + * + * @param editUris directories to create edit logs in + * @param numrolls number of times to roll the edit log during setup + * @param abortAtRolls Specifications for when to fail, see AbortSpec + */ + public static NNStorage setupEdits(List<URI> editUris, int numrolls, + AbortSpec... abortAtRolls) + throws IOException { + List<AbortSpec> aborts = new ArrayList<AbortSpec>(Arrays.asList(abortAtRolls)); + NNStorage storage = new NNStorage(new Configuration(), + Collections.<URI>emptyList(), + editUris); + storage.format("test-cluster-id"); + FSEditLog editlog = new FSEditLog(storage); + // open the edit log and add two transactions + // logGenerationStamp is used, simply because it doesn't + // require complex arguments. + editlog.open(); + for (int i = 2; i < TXNS_PER_ROLL; i++) { + editlog.logGenerationStamp((long)0); + } + editlog.logSync(); + + // Go into edit log rolling loop. + // On each roll, the abortAtRolls abort specs are + // checked to see if an abort is required. If so the + // the specified journal is aborted. It will be brought + // back into rotation automatically by rollEditLog + for (int i = 0; i < numrolls; i++) { + editlog.rollEditLog(); + + editlog.logGenerationStamp((long)i); + editlog.logSync(); + + while (aborts.size() > 0 + && aborts.get(0).roll == (i+1)) { + AbortSpec spec = aborts.remove(0); + editlog.getJournals().get(spec.logindex).abort(); + } + + for (int j = 3; j < TXNS_PER_ROLL; j++) { + editlog.logGenerationStamp((long)i); + } + editlog.logSync(); + } + editlog.close(); + + FSImageTestUtil.logStorageContents(LOG, storage); + return storage; + } + + /** + * Test loading an editlog which has had both its storage fail + * on alternating rolls. Two edit log directories are created. + * The first on fails on odd rolls, the second on even. Test + * that we are able to load the entire editlog regardless. + */ + @Test + public void testAlternatingJournalFailure() throws IOException { + File f1 = new File(TEST_DIR + "/alternatingjournaltest0"); + File f2 = new File(TEST_DIR + "/alternatingjournaltest1"); + + List<URI> editUris = ImmutableList.of(f1.toURI(), f2.toURI()); + + NNStorage storage = setupEdits(editUris, 10, + new AbortSpec(1, 0), + new AbortSpec(2, 1), + new AbortSpec(3, 0), + new AbortSpec(4, 1), + new AbortSpec(5, 0), + new AbortSpec(6, 1), + new AbortSpec(7, 0), + new AbortSpec(8, 1), + new AbortSpec(9, 0), + new AbortSpec(10, 1)); + long totaltxnread = 0; + FSEditLog editlog = new FSEditLog(storage); + long startTxId = 1; + Iterable<EditLogInputStream> editStreams = editlog.selectInputStreams(startTxId, + TXNS_PER_ROLL*11); + + for (EditLogInputStream edits : editStreams) { + FSEditLogLoader.EditLogValidation val = FSEditLogLoader.validateEditLog(edits); + long read = val.getNumTransactions(); + LOG.info("Loading edits " + edits + " read " + read); + assertEquals(startTxId, val.getStartTxId()); + startTxId += read; + totaltxnread += read; + } + + editlog.close(); + storage.close(); + assertEquals(TXNS_PER_ROLL*11, totaltxnread); + } + + /** + * Test loading an editlog with gaps. A single editlog directory + * is set up. On of the edit log files is deleted. This should + * fail when selecting the input streams as it will not be able + * to select enough streams to load up to 4*TXNS_PER_ROLL. + * There should be 4*TXNS_PER_ROLL transactions as we rolled 3 + * times. + */ + @Test + public void testLoadingWithGaps() throws IOException { + File f1 = new File(TEST_DIR + "/gaptest0"); + List<URI> editUris = ImmutableList.of(f1.toURI()); + + NNStorage storage = setupEdits(editUris, 3); + + final long startGapTxId = 1*TXNS_PER_ROLL + 1; + final long endGapTxId = 2*TXNS_PER_ROLL; + + File[] files = new File(f1, "current").listFiles(new FilenameFilter() { + public boolean accept(File dir, String name) { + if (name.startsWith(NNStorage.getFinalizedEditsFileName(startGapTxId, + endGapTxId))) { + return true; + } + return false; + } + }); + assertEquals(1, files.length); + assertTrue(files[0].delete()); + + FSEditLog editlog = new FSEditLog(storage); + long startTxId = 1; + try { + Iterable<EditLogInputStream> editStreams + = editlog.selectInputStreams(startTxId, 4*TXNS_PER_ROLL); + + fail("Should have thrown exception"); + } catch (IOException ioe) { + GenericTestUtils.assertExceptionContains( + "No non-corrupt logs for txid " + startGapTxId, ioe); + } + } }
Modified: hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java?rev=1228651&r1=1228650&r2=1228651&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java (original) +++ hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java Sat Jan 7 16:28:27 2012 @@ -63,8 +63,8 @@ public class TestEditLogFileOutputStream EditLogValidation validation = EditLogFileInputStream.validateEditLog(editLog); assertEquals("Edit log should contain a header as valid length", - HEADER_LEN, validation.validLength); - assertEquals(1, validation.numTransactions); + HEADER_LEN, validation.getValidLength()); + assertEquals(1, validation.getNumTransactions()); assertEquals("Edit log should have 1MB of bytes allocated", 1024*1024, editLog.length()); @@ -72,12 +72,12 @@ public class TestEditLogFileOutputStream cluster.getFileSystem().mkdirs(new Path("/tmp"), new FsPermission((short)777)); - long oldLength = validation.validLength; + long oldLength = validation.getValidLength(); validation = EditLogFileInputStream.validateEditLog(editLog); assertTrue("Edit log should have more valid data after writing a txn " + - "(was: " + oldLength + " now: " + validation.validLength + ")", - validation.validLength > oldLength); - assertEquals(2, validation.numTransactions); + "(was: " + oldLength + " now: " + validation.getValidLength() + ")", + validation.getValidLength() > oldLength); + assertEquals(2, validation.getNumTransactions()); assertEquals("Edit log should be 1MB long", 1024 * 1024, editLog.length()); Modified: hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java?rev=1228651&r1=1228650&r2=1228651&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java (original) +++ hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java Sat Jan 7 16:28:27 2012 @@ -186,8 +186,8 @@ public class TestFSEditLogLoader { // Make sure that uncorrupted log has the expected length and number // of transactions. EditLogValidation validation = EditLogFileInputStream.validateEditLog(logFile); - assertEquals(NUM_TXNS + 2, validation.numTransactions); - assertEquals(validLength, validation.validLength); + assertEquals(NUM_TXNS + 2, validation.getNumTransactions()); + assertEquals(validLength, validation.getValidLength()); // Back up the uncorrupted log File logFileBak = new File(testDir, logFile.getName() + ".bak"); @@ -203,8 +203,8 @@ public class TestFSEditLogLoader { truncateFile(logFile, txOffset); validation = EditLogFileInputStream.validateEditLog(logFile); assertEquals("Failed when truncating to length " + txOffset, - txid - 1, validation.numTransactions); - assertEquals(txOffset, validation.validLength); + txid - 1, validation.getNumTransactions()); + assertEquals(txOffset, validation.getValidLength()); // Restore backup, truncate the file with one byte in the txn, // also isn't valid @@ -212,24 +212,24 @@ public class TestFSEditLogLoader { truncateFile(logFile, txOffset + 1); validation = EditLogFileInputStream.validateEditLog(logFile); assertEquals("Failed when truncating to length " + (txOffset + 1), - txid - 1, validation.numTransactions); - assertEquals(txOffset, validation.validLength); + txid - 1, validation.getNumTransactions()); + assertEquals(txOffset, validation.getValidLength()); // Restore backup, corrupt the txn opcode Files.copy(logFileBak, logFile); corruptByteInFile(logFile, txOffset); validation = EditLogFileInputStream.validateEditLog(logFile); assertEquals("Failed when corrupting txn opcode at " + txOffset, - txid - 1, validation.numTransactions); - assertEquals(txOffset, validation.validLength); + txid - 1, validation.getNumTransactions()); + assertEquals(txOffset, validation.getValidLength()); // Restore backup, corrupt a byte a few bytes into the txn Files.copy(logFileBak, logFile); corruptByteInFile(logFile, txOffset+5); validation = EditLogFileInputStream.validateEditLog(logFile); assertEquals("Failed when corrupting txn data at " + (txOffset+5), - txid - 1, validation.numTransactions); - assertEquals(txOffset, validation.validLength); + txid - 1, validation.getNumTransactions()); + assertEquals(txOffset, validation.getValidLength()); } // Corrupt the log at every offset to make sure that validation itself @@ -240,8 +240,8 @@ public class TestFSEditLogLoader { Files.copy(logFileBak, logFile); corruptByteInFile(logFile, offset); EditLogValidation val = EditLogFileInputStream.validateEditLog(logFile); - assertTrue(val.numTransactions >= prevNumValid); - prevNumValid = val.numTransactions; + assertTrue(val.getNumTransactions() >= prevNumValid); + prevNumValid = val.getNumTransactions(); } } Modified: hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageStorageInspector.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageStorageInspector.java?rev=1228651&r1=1228650&r2=1228651&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageStorageInspector.java (original) +++ hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageStorageInspector.java Sat Jan 7 16:28:27 2012 @@ -36,9 +36,6 @@ import static org.apache.hadoop.hdfs.ser import org.apache.hadoop.hdfs.server.namenode.FileJournalManager.EditLogFile; import org.apache.hadoop.hdfs.server.namenode.FSImageStorageInspector.FSImageFile; -import org.apache.hadoop.hdfs.server.namenode.FSImageTransactionalStorageInspector.TransactionalLoadPlan; -import org.apache.hadoop.hdfs.server.namenode.FSImageTransactionalStorageInspector.LogGroup; -import org.apache.hadoop.hdfs.server.namenode.FSImageStorageInspector.LoadPlan; import org.junit.Test; import org.mockito.Mockito; @@ -63,335 +60,14 @@ public class TestFSImageStorageInspector "/foo/current/" + getInProgressEditsFileName(457)); inspector.inspectDirectory(mockDir); - mockLogValidation(inspector, - "/foo/current/" + getInProgressEditsFileName(457), 10); - - assertEquals(2, inspector.foundEditLogs.size()); assertEquals(2, inspector.foundImages.size()); - assertTrue(inspector.foundEditLogs.get(1).isInProgress()); - + FSImageFile latestImage = inspector.getLatestImage(); assertEquals(456, latestImage.txId); assertSame(mockDir, latestImage.sd); assertTrue(inspector.isUpgradeFinalized()); - LoadPlan plan = inspector.createLoadPlan(); - LOG.info("Plan: " + plan); - assertEquals(new File("/foo/current/"+getImageFileName(456)), - plan.getImageFile()); - assertArrayEquals(new File[] { - new File("/foo/current/" + getInProgressEditsFileName(457)) }, - plan.getEditsFiles().toArray(new File[0])); - } - - /** - * Test that we check for gaps in txids when devising a load plan. - */ - @Test - public void testPlanWithGaps() throws IOException { - FSImageTransactionalStorageInspector inspector = - new FSImageTransactionalStorageInspector(); - - StorageDirectory mockDir = FSImageTestUtil.mockStorageDirectory( - NameNodeDirType.IMAGE_AND_EDITS, - false, - "/foo/current/" + getImageFileName(123), - "/foo/current/" + getImageFileName(456), - "/foo/current/" + getFinalizedEditsFileName(457,900), - "/foo/current/" + getFinalizedEditsFileName(901,950), - "/foo/current/" + getFinalizedEditsFileName(952,1000)); // <-- missing edit 951! - - inspector.inspectDirectory(mockDir); - try { - inspector.createLoadPlan(); - fail("Didn't throw IOE trying to load with gaps in edits"); - } catch (IOException ioe) { - assertTrue(ioe.getMessage().contains( - "would start at txid 951 but starts at txid 952")); - } - } - - /** - * Test the case where an in-progress log comes in the middle of a sequence - * of logs - */ - @Test - public void testPlanWithInProgressInMiddle() throws IOException { - FSImageTransactionalStorageInspector inspector = - new FSImageTransactionalStorageInspector(); - - StorageDirectory mockDir = FSImageTestUtil.mockStorageDirectory( - NameNodeDirType.IMAGE_AND_EDITS, - false, - "/foo/current/" + getImageFileName(123), - "/foo/current/" + getImageFileName(456), - "/foo/current/" + getFinalizedEditsFileName(457,900), - "/foo/current/" + getInProgressEditsFileName(901), // <-- inprogress in middle - "/foo/current/" + getFinalizedEditsFileName(952,1000)); - - inspector.inspectDirectory(mockDir); - mockLogValidation(inspector, - "/foo/current/" + getInProgressEditsFileName(901), 51); - - LoadPlan plan = inspector.createLoadPlan(); - LOG.info("Plan: " + plan); - - assertEquals(new File("/foo/current/" + getImageFileName(456)), - plan.getImageFile()); - assertArrayEquals(new File[] { - new File("/foo/current/" + getFinalizedEditsFileName(457,900)), - new File("/foo/current/" + getInProgressEditsFileName(901)), - new File("/foo/current/" + getFinalizedEditsFileName(952,1000)) }, - plan.getEditsFiles().toArray(new File[0])); - - } - - - /** - * Test case for the usual case where no recovery of a log group is necessary - * (i.e all logs have the same start and end txids and finalized) - */ - @Test - public void testLogGroupRecoveryNoop() throws IOException { - FSImageTransactionalStorageInspector inspector = - new FSImageTransactionalStorageInspector(); - - inspector.inspectDirectory( - mockDirectoryWithEditLogs("/foo1/current/" - + getFinalizedEditsFileName(123,456))); - inspector.inspectDirectory( - mockDirectoryWithEditLogs("/foo2/current/" - + getFinalizedEditsFileName(123,456))); - inspector.inspectDirectory( - mockDirectoryWithEditLogs("/foo3/current/" - + getFinalizedEditsFileName(123,456))); - LogGroup lg = inspector.logGroups.get(123L); - assertEquals(3, lg.logs.size()); - - lg.planRecovery(); - - assertFalse(lg.logs.get(0).isCorrupt()); - assertFalse(lg.logs.get(1).isCorrupt()); - assertFalse(lg.logs.get(2).isCorrupt()); - } - - /** - * Test case where we have some in-progress and some finalized logs - * for a given txid. - */ - @Test - public void testLogGroupRecoveryMixed() throws IOException { - FSImageTransactionalStorageInspector inspector = - new FSImageTransactionalStorageInspector(); - - inspector.inspectDirectory( - mockDirectoryWithEditLogs("/foo1/current/" - + getFinalizedEditsFileName(123,456))); - inspector.inspectDirectory( - mockDirectoryWithEditLogs("/foo2/current/" - + getFinalizedEditsFileName(123,456))); - inspector.inspectDirectory( - mockDirectoryWithEditLogs("/foo3/current/" - + getInProgressEditsFileName(123))); - inspector.inspectDirectory(FSImageTestUtil.mockStorageDirectory( - NameNodeDirType.IMAGE, - false, - "/foo4/current/" + getImageFileName(122))); - - LogGroup lg = inspector.logGroups.get(123L); - assertEquals(3, lg.logs.size()); - EditLogFile inProgressLog = lg.logs.get(2); - assertTrue(inProgressLog.isInProgress()); - - LoadPlan plan = inspector.createLoadPlan(); - - // Check that it was marked corrupt. - assertFalse(lg.logs.get(0).isCorrupt()); - assertFalse(lg.logs.get(1).isCorrupt()); - assertTrue(lg.logs.get(2).isCorrupt()); - - - // Calling recover should move it aside - inProgressLog = spy(inProgressLog); - Mockito.doNothing().when(inProgressLog).moveAsideCorruptFile(); - lg.logs.set(2, inProgressLog); - - plan.doRecovery(); - - Mockito.verify(inProgressLog).moveAsideCorruptFile(); - } - - /** - * Test case where we have finalized logs with different end txids - */ - @Test - public void testLogGroupRecoveryInconsistentEndTxIds() throws IOException { - FSImageTransactionalStorageInspector inspector = - new FSImageTransactionalStorageInspector(); - inspector.inspectDirectory( - mockDirectoryWithEditLogs("/foo1/current/" - + getFinalizedEditsFileName(123,456))); - inspector.inspectDirectory( - mockDirectoryWithEditLogs("/foo2/current/" - + getFinalizedEditsFileName(123,678))); - - LogGroup lg = inspector.logGroups.get(123L); - assertEquals(2, lg.logs.size()); - - try { - lg.planRecovery(); - fail("Didn't throw IOE on inconsistent end txids"); - } catch (IOException ioe) { - assertTrue(ioe.getMessage().contains("More than one ending txid")); - } - } - - /** - * Test case where we have only in-progress logs and need to synchronize - * based on valid length. - */ - @Test - public void testLogGroupRecoveryInProgress() throws IOException { - String paths[] = new String[] { - "/foo1/current/" + getInProgressEditsFileName(123), - "/foo2/current/" + getInProgressEditsFileName(123), - "/foo3/current/" + getInProgressEditsFileName(123) - }; - FSImageTransactionalStorageInspector inspector = - new FSImageTransactionalStorageInspector(); - inspector.inspectDirectory(mockDirectoryWithEditLogs(paths[0])); - inspector.inspectDirectory(mockDirectoryWithEditLogs(paths[1])); - inspector.inspectDirectory(mockDirectoryWithEditLogs(paths[2])); - - // Inject spies to return the valid counts we would like to see - mockLogValidation(inspector, paths[0], 2000); - mockLogValidation(inspector, paths[1], 2000); - mockLogValidation(inspector, paths[2], 1000); - - LogGroup lg = inspector.logGroups.get(123L); - assertEquals(3, lg.logs.size()); - - lg.planRecovery(); - - // Check that the short one was marked corrupt - assertFalse(lg.logs.get(0).isCorrupt()); - assertFalse(lg.logs.get(1).isCorrupt()); - assertTrue(lg.logs.get(2).isCorrupt()); - - // Calling recover should move it aside - EditLogFile badLog = lg.logs.get(2); - Mockito.doNothing().when(badLog).moveAsideCorruptFile(); - Mockito.doNothing().when(lg.logs.get(0)).finalizeLog(); - Mockito.doNothing().when(lg.logs.get(1)).finalizeLog(); - - lg.recover(); - - Mockito.verify(badLog).moveAsideCorruptFile(); - Mockito.verify(lg.logs.get(0)).finalizeLog(); - Mockito.verify(lg.logs.get(1)).finalizeLog(); - } - - /** - * Mock out the log at the given path to return a specified number - * of transactions upon validation. - */ - private void mockLogValidation( - FSImageTransactionalStorageInspector inspector, - String path, int numValidTransactions) throws IOException { - - for (LogGroup lg : inspector.logGroups.values()) { - List<EditLogFile> logs = lg.logs; - for (int i = 0; i < logs.size(); i++) { - EditLogFile log = logs.get(i); - if (log.getFile().getPath().equals(path)) { - // mock out its validation - EditLogFile spyLog = spy(log); - doReturn(new FSEditLogLoader.EditLogValidation(-1, numValidTransactions)) - .when(spyLog).validateLog(); - logs.set(i, spyLog); - return; - } - } - } - fail("No log found to mock out at " + path); - } - - /** - * Test when edits and image are in separate directories. - */ - @Test - public void testCurrentSplitEditsAndImage() throws IOException { - FSImageTransactionalStorageInspector inspector = - new FSImageTransactionalStorageInspector(); - - StorageDirectory mockImageDir = FSImageTestUtil.mockStorageDirectory( - NameNodeDirType.IMAGE, - false, - "/foo/current/" + getImageFileName(123)); - StorageDirectory mockImageDir2 = FSImageTestUtil.mockStorageDirectory( - NameNodeDirType.IMAGE, - false, - "/foo2/current/" + getImageFileName(456)); - StorageDirectory mockEditsDir = FSImageTestUtil.mockStorageDirectory( - NameNodeDirType.EDITS, - false, - "/foo3/current/" + getFinalizedEditsFileName(123, 456), - "/foo3/current/" + getInProgressEditsFileName(457)); - - inspector.inspectDirectory(mockImageDir); - inspector.inspectDirectory(mockEditsDir); - inspector.inspectDirectory(mockImageDir2); - - mockLogValidation(inspector, - "/foo3/current/" + getInProgressEditsFileName(457), 2); - - assertEquals(2, inspector.foundEditLogs.size()); - assertEquals(2, inspector.foundImages.size()); - assertTrue(inspector.foundEditLogs.get(1).isInProgress()); - assertTrue(inspector.isUpgradeFinalized()); - - // Check plan - TransactionalLoadPlan plan = - (TransactionalLoadPlan)inspector.createLoadPlan(); - FSImageFile pickedImage = plan.image; - assertEquals(456, pickedImage.txId); - assertSame(mockImageDir2, pickedImage.sd); - assertEquals(new File("/foo2/current/" + getImageFileName(456)), - plan.getImageFile()); - assertArrayEquals(new File[] { - new File("/foo3/current/" + getInProgressEditsFileName(457)) - }, plan.getEditsFiles().toArray(new File[0])); - } - - /** - * Test case where an in-progress log is in an earlier name directory - * than a finalized log. Previously, getEditLogManifest wouldn't - * see this log. - */ - @Test - public void testLogManifestInProgressComesFirst() throws IOException { - FSImageTransactionalStorageInspector inspector = - new FSImageTransactionalStorageInspector(); - inspector.inspectDirectory( - mockDirectoryWithEditLogs("/foo1/current/" - + getFinalizedEditsFileName(2622,2623), - "/foo1/current/" - + getFinalizedEditsFileName(2624,2625), - "/foo1/current/" - + getInProgressEditsFileName(2626))); - inspector.inspectDirectory( - mockDirectoryWithEditLogs("/foo2/current/" - + getFinalizedEditsFileName(2622,2623), - "/foo2/current/" - + getFinalizedEditsFileName(2624,2625), - "/foo2/current/" - + getFinalizedEditsFileName(2626,2627), - "/foo2/current/" - + getFinalizedEditsFileName(2628,2629))); - } - - static StorageDirectory mockDirectoryWithEditLogs(String... fileNames) { - return FSImageTestUtil.mockStorageDirectory(NameNodeDirType.EDITS, false, fileNames); + latestImage.getFile()); } } Modified: hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java?rev=1228651&r1=1228650&r2=1228651&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java (original) +++ hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java Sat Jan 7 16:28:27 2012 @@ -19,17 +19,277 @@ package org.apache.hadoop.hdfs.server.na import static org.junit.Assert.*; -import java.io.IOException; +import java.net.URI; +import java.util.Collections; +import java.util.Arrays; +import java.util.List; +import java.util.ArrayList; +import java.util.Iterator; +import java.io.RandomAccessFile; +import java.io.File; +import java.io.FilenameFilter; +import java.io.BufferedInputStream; +import java.io.DataInputStream; +import java.io.IOException; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.security.SecurityUtil; +import org.junit.Test; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType; import org.apache.hadoop.test.GenericTestUtils; -import org.junit.Test; +import static org.apache.hadoop.hdfs.server.namenode.TestEditLog.setupEdits; +import static org.apache.hadoop.hdfs.server.namenode.TestEditLog.AbortSpec; +import static org.apache.hadoop.hdfs.server.namenode.TestEditLog.TXNS_PER_ROLL; +import static org.apache.hadoop.hdfs.server.namenode.TestEditLog.TXNS_PER_FAIL; +import com.google.common.collect.ImmutableList; import com.google.common.base.Joiner; +import java.util.zip.CheckedInputStream; +import java.util.zip.Checksum; + public class TestFileJournalManager { + /** + * Test the normal operation of loading transactions from + * file journal manager. 3 edits directories are setup without any + * failures. Test that we read in the expected number of transactions. + */ + @Test + public void testNormalOperation() throws IOException { + File f1 = new File(TestEditLog.TEST_DIR + "/normtest0"); + File f2 = new File(TestEditLog.TEST_DIR + "/normtest1"); + File f3 = new File(TestEditLog.TEST_DIR + "/normtest2"); + + List<URI> editUris = ImmutableList.of(f1.toURI(), f2.toURI(), f3.toURI()); + NNStorage storage = setupEdits(editUris, 5); + + long numJournals = 0; + for (StorageDirectory sd : storage.dirIterable(NameNodeDirType.EDITS)) { + FileJournalManager jm = new FileJournalManager(sd); + assertEquals(6*TXNS_PER_ROLL, jm.getNumberOfTransactions(1)); + numJournals++; + } + assertEquals(3, numJournals); + } + + /** + * Test that inprogress files are handled correct. Set up a single + * edits directory. Fail on after the last roll. Then verify that the + * logs have the expected number of transactions. + */ + @Test + public void testInprogressRecovery() throws IOException { + File f = new File(TestEditLog.TEST_DIR + "/filejournaltest0"); + // abort after the 5th roll + NNStorage storage = setupEdits(Collections.<URI>singletonList(f.toURI()), + 5, new AbortSpec(5, 0)); + StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next(); + + FileJournalManager jm = new FileJournalManager(sd); + assertEquals(5*TXNS_PER_ROLL + TXNS_PER_FAIL, + jm.getNumberOfTransactions(1)); + } + + /** + * Test a mixture of inprogress files and finalised. Set up 3 edits + * directories and fail the second on the last roll. Verify that reading + * the transactions, reads from the finalised directories. + */ + @Test + public void testInprogressRecoveryMixed() throws IOException { + File f1 = new File(TestEditLog.TEST_DIR + "/mixtest0"); + File f2 = new File(TestEditLog.TEST_DIR + "/mixtest1"); + File f3 = new File(TestEditLog.TEST_DIR + "/mixtest2"); + + List<URI> editUris = ImmutableList.of(f1.toURI(), f2.toURI(), f3.toURI()); + + // abort after the 5th roll + NNStorage storage = setupEdits(editUris, + 5, new AbortSpec(5, 1)); + Iterator<StorageDirectory> dirs = storage.dirIterator(NameNodeDirType.EDITS); + StorageDirectory sd = dirs.next(); + FileJournalManager jm = new FileJournalManager(sd); + assertEquals(6*TXNS_PER_ROLL, jm.getNumberOfTransactions(1)); + + sd = dirs.next(); + jm = new FileJournalManager(sd); + assertEquals(5*TXNS_PER_ROLL + TXNS_PER_FAIL, jm.getNumberOfTransactions(1)); + + sd = dirs.next(); + jm = new FileJournalManager(sd); + assertEquals(6*TXNS_PER_ROLL, jm.getNumberOfTransactions(1)); + } + + /** + * Test that FileJournalManager behaves correctly despite inprogress + * files in all its edit log directories. Set up 3 directories and fail + * all on the last roll. Verify that the correct number of transaction + * are then loaded. + */ + @Test + public void testInprogressRecoveryAll() throws IOException { + File f1 = new File(TestEditLog.TEST_DIR + "/failalltest0"); + File f2 = new File(TestEditLog.TEST_DIR + "/failalltest1"); + File f3 = new File(TestEditLog.TEST_DIR + "/failalltest2"); + + List<URI> editUris = ImmutableList.of(f1.toURI(), f2.toURI(), f3.toURI()); + // abort after the 5th roll + NNStorage storage = setupEdits(editUris, 5, + new AbortSpec(5, 0), + new AbortSpec(5, 1), + new AbortSpec(5, 2)); + Iterator<StorageDirectory> dirs = storage.dirIterator(NameNodeDirType.EDITS); + StorageDirectory sd = dirs.next(); + FileJournalManager jm = new FileJournalManager(sd); + assertEquals(5*TXNS_PER_ROLL + TXNS_PER_FAIL, jm.getNumberOfTransactions(1)); + + sd = dirs.next(); + jm = new FileJournalManager(sd); + assertEquals(5*TXNS_PER_ROLL + TXNS_PER_FAIL, jm.getNumberOfTransactions(1)); + + sd = dirs.next(); + jm = new FileJournalManager(sd); + assertEquals(5*TXNS_PER_ROLL + TXNS_PER_FAIL, jm.getNumberOfTransactions(1)); + } + + /** + * Corrupt an edit log file after the start segment transaction + */ + private void corruptAfterStartSegment(File f) throws IOException { + RandomAccessFile raf = new RandomAccessFile(f, "rw"); + raf.seek(0x16); // skip version and first tranaction and a bit of next transaction + for (int i = 0; i < 1000; i++) { + raf.writeInt(0xdeadbeef); + } + raf.close(); + } + + /** + * Test that we can read from a stream created by FileJournalManager. + * Create a single edits directory, failing it on the final roll. + * Then try loading from the point of the 3rd roll. Verify that we read + * the correct number of transactions from this point. + */ + @Test + public void testReadFromStream() throws IOException { + File f = new File(TestEditLog.TEST_DIR + "/filejournaltest1"); + // abort after 10th roll + NNStorage storage = setupEdits(Collections.<URI>singletonList(f.toURI()), + 10, new AbortSpec(10, 0)); + StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next(); + + FileJournalManager jm = new FileJournalManager(sd); + long expectedTotalTxnCount = TXNS_PER_ROLL*10 + TXNS_PER_FAIL; + assertEquals(expectedTotalTxnCount, jm.getNumberOfTransactions(1)); + + long skippedTxns = (3*TXNS_PER_ROLL); // skip first 3 files + long startingTxId = skippedTxns + 1; + + long numTransactionsToLoad = jm.getNumberOfTransactions(startingTxId); + long numLoaded = 0; + while (numLoaded < numTransactionsToLoad) { + EditLogInputStream editIn = jm.getInputStream(startingTxId); + FSEditLogLoader.EditLogValidation val = FSEditLogLoader.validateEditLog(editIn); + long count = val.getNumTransactions(); + + editIn.close(); + startingTxId += count; + numLoaded += count; + } + + assertEquals(expectedTotalTxnCount - skippedTxns, numLoaded); + } + + /** + * Try to make a request with a start transaction id which doesn't + * match the start ID of some log segment. + * This should fail as edit logs must currently be treated as indevisable + * units. + */ + @Test(expected=IOException.class) + public void testAskForTransactionsMidfile() throws IOException { + File f = new File(TestEditLog.TEST_DIR + "/filejournaltest2"); + NNStorage storage = setupEdits(Collections.<URI>singletonList(f.toURI()), + 10); + StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next(); + + FileJournalManager jm = new FileJournalManager(sd); + jm.getNumberOfTransactions(2); + } + + /** + * Test that we receive the correct number of transactions when we count + * the number of transactions around gaps. + * Set up a single edits directory, with no failures. Delete the 4th logfile. + * Test that getNumberOfTransactions returns the correct number of + * transactions before this gap and after this gap. Also verify that if you + * try to count on the gap that an exception is thrown. + */ + @Test + public void testManyLogsWithGaps() throws IOException { + File f = new File(TestEditLog.TEST_DIR + "/filejournaltest3"); + NNStorage storage = setupEdits(Collections.<URI>singletonList(f.toURI()), 10); + StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next(); + + final long startGapTxId = 3*TXNS_PER_ROLL + 1; + final long endGapTxId = 4*TXNS_PER_ROLL; + File[] files = new File(f, "current").listFiles(new FilenameFilter() { + public boolean accept(File dir, String name) { + if (name.startsWith(NNStorage.getFinalizedEditsFileName(startGapTxId, endGapTxId))) { + return true; + } + return false; + } + }); + assertEquals(1, files.length); + assertTrue(files[0].delete()); + + FileJournalManager jm = new FileJournalManager(sd); + assertEquals(startGapTxId-1, jm.getNumberOfTransactions(1)); + + try { + jm.getNumberOfTransactions(startGapTxId); + fail("Should have thrown an exception by now"); + } catch (IOException ioe) { + assertTrue(true); + } + + // rolled 10 times so there should be 11 files. + assertEquals(11*TXNS_PER_ROLL - endGapTxId, + jm.getNumberOfTransactions(endGapTxId+1)); + } + + /** + * Test that we can load an edits directory with a corrupt inprogress file. + * The corrupt inprogress file should be moved to the side. + */ + @Test + public void testManyLogsWithCorruptInprogress() throws IOException { + File f = new File(TestEditLog.TEST_DIR + "/filejournaltest5"); + NNStorage storage = setupEdits(Collections.<URI>singletonList(f.toURI()), 10, new AbortSpec(10, 0)); + StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next(); + + File[] files = new File(f, "current").listFiles(new FilenameFilter() { + public boolean accept(File dir, String name) { + if (name.startsWith("edits_inprogress")) { + return true; + } + return false; + } + }); + assertEquals(files.length, 1); + + corruptAfterStartSegment(files[0]); + + FileJournalManager jm = new FileJournalManager(sd); + assertEquals(10*TXNS_PER_ROLL+1, + jm.getNumberOfTransactions(1)); + } + @Test public void testGetRemoteEditLog() throws IOException { StorageDirectory sd = FSImageTestUtil.mockStorageDirectory( @@ -58,5 +318,4 @@ public class TestFileJournalManager { FileJournalManager fjm, long firstTxId) throws IOException { return Joiner.on(",").join(fjm.getRemoteEditLogs(firstTxId)); } - } Modified: hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameEditsConfigs.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameEditsConfigs.java?rev=1228651&r1=1228650&r2=1228651&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameEditsConfigs.java (original) +++ hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameEditsConfigs.java Sat Jan 7 16:28:27 2012 @@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.server.na import junit.framework.TestCase; import java.io.*; import java.util.Random; +import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; @@ -80,10 +81,12 @@ public class TestNameEditsConfigs extend assertTrue("Expect no images in " + dir, ins.foundImages.isEmpty()); } + List<FileJournalManager.EditLogFile> editlogs + = FileJournalManager.matchEditLogs(new File(dir, "current").listFiles()); if (shouldHaveEdits) { - assertTrue("Expect edits in " + dir, ins.foundEditLogs.size() > 0); + assertTrue("Expect edits in " + dir, editlogs.size() > 0); } else { - assertTrue("Expect no edits in " + dir, ins.foundEditLogs.isEmpty()); + assertTrue("Expect no edits in " + dir, editlogs.isEmpty()); } }
