Author: atm
Date: Wed Jan 11 08:32:10 2012
New Revision: 1229931
URL: http://svn.apache.org/viewvc?rev=1229931&view=rev
Log:
HDFS-2738. FSEditLog.selectinputStreams is reading through in-progress streams
even when non-in-progress are requested. Contributed by Aaron T. Myers
Modified:
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperJournalManager.java
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupJournalManager.java
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java
Modified:
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt?rev=1229931&r1=1229930&r2=1229931&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt
(original)
+++
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt
Wed Jan 11 08:32:10 2012
@@ -101,3 +101,5 @@ HDFS-2773. Reading edit logs from an ear
HDFS-2775. Fix TestStandbyCheckpoints.testBothNodesInStandbyState failing
intermittently. (todd)
HDFS-2766. Test for case where standby partially reads log and then performs
checkpoint. (atm)
+
+HDFS-2738. FSEditLog.selectinputStreams is reading through in-progress streams
even when non-in-progress are requested. (atm)
Modified:
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java?rev=1229931&r1=1229930&r2=1229931&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java
(original)
+++
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java
Wed Jan 11 08:32:10 2012
@@ -312,8 +312,10 @@ public class BookKeeperJournalManager im
}
}
+ // TODO(HA): Handle inProgressOk
@Override
- public EditLogInputStream getInputStream(long fromTxnId) throws IOException {
+ public EditLogInputStream getInputStream(long fromTxnId, boolean
inProgressOk)
+ throws IOException {
for (EditLogLedgerMetadata l : getLedgerList()) {
if (l.getFirstTxId() == fromTxnId) {
try {
@@ -329,8 +331,10 @@ public class BookKeeperJournalManager im
throw new IOException("No ledger for fromTxnId " + fromTxnId + " found.");
}
+ // TODO(HA): Handle inProgressOk
@Override
- public long getNumberOfTransactions(long fromTxnId) throws IOException {
+ public long getNumberOfTransactions(long fromTxnId, boolean inProgressOk)
+ throws IOException {
long count = 0;
long expectedStart = 0;
for (EditLogLedgerMetadata l : getLedgerList()) {
Modified:
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperJournalManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperJournalManager.java?rev=1229931&r1=1229930&r2=1229931&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperJournalManager.java
(original)
+++
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperJournalManager.java
Wed Jan 11 08:32:10 2012
@@ -195,7 +195,7 @@ public class TestBookKeeperJournalManage
out.close();
bkjm.finalizeLogSegment(1, 100);
- long numTrans = bkjm.getNumberOfTransactions(1);
+ long numTrans = bkjm.getNumberOfTransactions(1, true);
assertEquals(100, numTrans);
}
@@ -218,17 +218,17 @@ public class TestBookKeeperJournalManage
}
zkc.delete(bkjm.finalizedLedgerZNode(DEFAULT_SEGMENT_SIZE+1,
DEFAULT_SEGMENT_SIZE*2), -1);
- long numTrans = bkjm.getNumberOfTransactions(1);
+ long numTrans = bkjm.getNumberOfTransactions(1, true);
assertEquals(DEFAULT_SEGMENT_SIZE, numTrans);
try {
- numTrans = bkjm.getNumberOfTransactions(DEFAULT_SEGMENT_SIZE+1);
+ numTrans = bkjm.getNumberOfTransactions(DEFAULT_SEGMENT_SIZE+1, true);
fail("Should have thrown corruption exception by this point");
} catch (JournalManager.CorruptionException ce) {
// if we get here, everything is going good
}
- numTrans = bkjm.getNumberOfTransactions((DEFAULT_SEGMENT_SIZE*2)+1);
+ numTrans = bkjm.getNumberOfTransactions((DEFAULT_SEGMENT_SIZE*2)+1, true);
assertEquals(DEFAULT_SEGMENT_SIZE, numTrans);
}
@@ -262,7 +262,7 @@ public class TestBookKeeperJournalManage
out.abort();
out.close();
- long numTrans = bkjm.getNumberOfTransactions(1);
+ long numTrans = bkjm.getNumberOfTransactions(1, true);
assertEquals((txid-1), numTrans);
}
@@ -357,7 +357,7 @@ public class TestBookKeeperJournalManage
bkjm.finalizeLogSegment(1, numTransactions);
- EditLogInputStream in = bkjm.getInputStream(1);
+ EditLogInputStream in = bkjm.getInputStream(1, true);
try {
assertEquals(numTransactions,
FSEditLogTestUtil.countTransactionsInStream(in));
@@ -392,4 +392,4 @@ public class TestBookKeeperJournalManage
assertNotNull(zkc.exists(bkjm.finalizedLedgerZNode(1, 100), false));
assertNull(zkc.exists(bkjm.inprogressZNode(), false));
}
-}
\ No newline at end of file
+}
Modified:
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupJournalManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupJournalManager.java?rev=1229931&r1=1229930&r2=1229931&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupJournalManager.java
(original)
+++
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupJournalManager.java
Wed Jan 11 08:32:10 2012
@@ -58,7 +58,7 @@ class BackupJournalManager implements Jo
}
@Override
- public long getNumberOfTransactions(long fromTxnId)
+ public long getNumberOfTransactions(long fromTxnId, boolean inProgressOk)
throws IOException, CorruptionException {
// This JournalManager is never used for input. Therefore it cannot
// return any transactions
@@ -66,7 +66,8 @@ class BackupJournalManager implements Jo
}
@Override
- public EditLogInputStream getInputStream(long fromTxnId) throws IOException {
+ public EditLogInputStream getInputStream(long fromTxnId, boolean
inProgressOk)
+ throws IOException {
// This JournalManager is never used for input. Therefore it cannot
// return any transactions
throw new IOException("Unsupported operation");
Modified:
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java?rev=1229931&r1=1229930&r2=1229931&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
(original)
+++
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
Wed Jan 11 08:32:10 2012
@@ -252,7 +252,7 @@ public class FSEditLog {
long segmentTxId = getLastWrittenTxId() + 1;
// Safety check: we should never start a segment if there are
// newer txids readable.
- EditLogInputStream s = journalSet.getInputStream(segmentTxId);
+ EditLogInputStream s = journalSet.getInputStream(segmentTxId, true);
try {
Preconditions.checkState(s == null,
"Cannot start writing at txid %s when there is a stream " +
@@ -1071,19 +1071,19 @@ public class FSEditLog {
public Collection<EditLogInputStream> selectInputStreams(long fromTxId,
long toAtLeastTxId, boolean inProgressOk) throws IOException {
List<EditLogInputStream> streams = new ArrayList<EditLogInputStream>();
- EditLogInputStream stream = journalSet.getInputStream(fromTxId);
+ EditLogInputStream stream = journalSet.getInputStream(fromTxId,
inProgressOk);
while (stream != null) {
- if (inProgressOk || !stream.isInProgress()) {
- streams.add(stream);
- }
+ streams.add(stream);
// We're now looking for a higher range, so reset the fromTxId
fromTxId = stream.getLastTxId() + 1;
- stream = journalSet.getInputStream(fromTxId);
+ stream = journalSet.getInputStream(fromTxId, inProgressOk);
}
+
if (fromTxId <= toAtLeastTxId) {
closeAllStreams(streams);
- throw new IOException("No non-corrupt logs for txid "
- + fromTxId);
+ throw new IOException(String.format("Gap in transactions. Expected to "
+ + "be able to read up until at least txid %d but unable to find any "
+ + "edit logs containing txid %d", toAtLeastTxId, fromTxId));
}
return streams;
}
Modified:
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java?rev=1229931&r1=1229930&r2=1229931&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
(original)
+++
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
Wed Jan 11 08:32:10 2012
@@ -585,9 +585,12 @@ public class FSImage implements Closeabl
if (LayoutVersion.supports(Feature.TXID_BASED_LAYOUT,
getLayoutVersion())) {
+ // If we're open for write, we're either non-HA or we're the active NN,
so
+ // we better be able to load all the edits. If we're the standby NN, it's
+ // OK to not be able to read all of edits right now.
+ long toAtLeastTxId = editLog.isOpenForWrite() ?
inspector.getMaxSeenTxId() : 0;
editStreams = editLog.selectInputStreams(imageFile.getCheckpointTxId() +
1,
- inspector.getMaxSeenTxId(),
- false);
+ toAtLeastTxId, false);
} else {
editStreams = FSImagePreTransactionalStorageInspector
.getEditLogStreams(storage);
Modified:
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java?rev=1229931&r1=1229930&r2=1229931&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
(original)
+++
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
Wed Jan 11 08:32:10 2012
@@ -31,6 +31,7 @@ import java.util.regex.Pattern;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
+import
org.apache.hadoop.hdfs.server.namenode.JournalManager.CorruptionException;
import
org.apache.hadoop.hdfs.server.namenode.NNStorageRetentionManager.StoragePurger;
import
org.apache.hadoop.hdfs.server.namenode.FSEditLogLoader.EditLogValidation;
import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeFile;
@@ -192,10 +193,13 @@ class FileJournalManager implements Jour
}
@Override
- synchronized public EditLogInputStream getInputStream(long fromTxId)
- throws IOException {
+ synchronized public EditLogInputStream getInputStream(long fromTxId,
+ boolean inProgressOk) throws IOException {
for (EditLogFile elf : getLogFiles(fromTxId)) {
if (elf.containsTxId(fromTxId)) {
+ if (!inProgressOk && elf.isInProgress()) {
+ continue;
+ }
if (elf.isInProgress()) {
elf.validateLog();
}
@@ -219,7 +223,7 @@ class FileJournalManager implements Jour
}
@Override
- public long getNumberOfTransactions(long fromTxId)
+ public long getNumberOfTransactions(long fromTxId, boolean inProgressOk)
throws IOException, CorruptionException {
long numTxns = 0L;
@@ -232,6 +236,10 @@ class FileJournalManager implements Jour
+ fromTxId + " - " + (elf.getFirstTxId() - 1));
break;
} else if (elf.containsTxId(fromTxId)) {
+ if (!inProgressOk && elf.isInProgress()) {
+ break;
+ }
+
if (elf.isInProgress()) {
elf.validateLog();
}
@@ -253,7 +261,7 @@ class FileJournalManager implements Jour
+ " txns from " + fromTxId);
}
- long max = findMaxTransaction();
+ long max = findMaxTransaction(inProgressOk);
// fromTxId should be greater than max, as it points to the next
// transaction we should expect to find. If it is less than or equal
@@ -276,7 +284,7 @@ class FileJournalManager implements Jour
// make sure journal is aware of max seen transaction before moving
corrupt
// files aside
- findMaxTransaction();
+ findMaxTransaction(true);
for (EditLogFile elf : allLogFiles) {
if (elf.getFile().equals(currentInProgress)) {
@@ -318,9 +326,13 @@ class FileJournalManager implements Jour
* tranaction id in the case that it was the maximum transaction in
* the journal.
*/
- private long findMaxTransaction()
+ private long findMaxTransaction(boolean inProgressOk)
throws IOException {
for (EditLogFile elf : getLogFiles(0)) {
+ if (elf.isInProgress() && !inProgressOk) {
+ continue;
+ }
+
if (elf.isInProgress()) {
maxSeenTransaction = Math.max(elf.getFirstTxId(), maxSeenTransaction);
elf.validateLog();
Modified:
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java?rev=1229931&r1=1229930&r2=1229931&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java
(original)
+++
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java
Wed Jan 11 08:32:10 2012
@@ -48,20 +48,23 @@ public interface JournalManager extends
/**
* Get the input stream starting with fromTxnId from this journal manager
* @param fromTxnId the first transaction id we want to read
+ * @param inProgressOk whether or not in-progress streams should be returned
* @return the stream starting with transaction fromTxnId
* @throws IOException if a stream cannot be found.
*/
- EditLogInputStream getInputStream(long fromTxnId) throws IOException;
+ EditLogInputStream getInputStream(long fromTxnId, boolean inProgressOk)
+ throws IOException;
/**
* Get the number of transaction contiguously available from fromTxnId.
*
* @param fromTxnId Transaction id to count from
+ * @param inProgressOk whether or not in-progress streams should be counted
* @return The number of transactions available from fromTxnId
* @throws IOException if the journal cannot be read.
* @throws CorruptionException if there is a gap in the journal at fromTxnId.
*/
- long getNumberOfTransactions(long fromTxnId)
+ long getNumberOfTransactions(long fromTxnId, boolean inProgressOk)
throws IOException, CorruptionException;
/**
Modified:
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java?rev=1229931&r1=1229930&r2=1229931&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java
(original)
+++
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java
Wed Jan 11 08:32:10 2012
@@ -198,7 +198,8 @@ public class JournalSet implements Journ
* or null if no more exist
*/
@Override
- public EditLogInputStream getInputStream(long fromTxnId) throws IOException {
+ public EditLogInputStream getInputStream(long fromTxnId, boolean
inProgressOk)
+ throws IOException {
JournalManager bestjm = null;
long bestjmNumTxns = 0;
CorruptionException corruption = null;
@@ -209,7 +210,8 @@ public class JournalSet implements Journ
JournalManager candidate = jas.getManager();
long candidateNumTxns = 0;
try {
- candidateNumTxns = candidate.getNumberOfTransactions(fromTxnId);
+ candidateNumTxns = candidate.getNumberOfTransactions(fromTxnId,
+ inProgressOk);
} catch (CorruptionException ce) {
corruption = ce;
} catch (IOException ioe) {
@@ -232,18 +234,20 @@ public class JournalSet implements Journ
return null;
}
}
- return bestjm.getInputStream(fromTxnId);
+ return bestjm.getInputStream(fromTxnId, inProgressOk);
}
@Override
- public long getNumberOfTransactions(long fromTxnId) throws IOException {
+ public long getNumberOfTransactions(long fromTxnId, boolean inProgressOk)
+ throws IOException {
long num = 0;
for (JournalAndStream jas: journals) {
if (jas.isDisabled()) {
LOG.info("Skipping jas " + jas + " since it's disabled");
continue;
} else {
- long newNum = jas.getManager().getNumberOfTransactions(fromTxnId);
+ long newNum = jas.getManager().getNumberOfTransactions(fromTxnId,
+ inProgressOk);
if (newNum > num) {
num = newNum;
}
Modified:
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java?rev=1229931&r1=1229930&r2=1229931&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
(original)
+++
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
Wed Jan 11 08:32:10 2012
@@ -936,11 +936,11 @@ public class TestEditLog extends TestCas
*
* @param editUris directories to create edit logs in
* @param numrolls number of times to roll the edit log during setup
+ * @param closeOnFinish whether to close the edit log after setup
* @param abortAtRolls Specifications for when to fail, see AbortSpec
*/
- public static NNStorage setupEdits(List<URI> editUris, int numrolls,
- AbortSpec... abortAtRolls)
- throws IOException {
+ public static NNStorage setupEdits(List<URI> editUris, int numrolls,
+ boolean closeOnFinish, AbortSpec... abortAtRolls) throws IOException {
List<AbortSpec> aborts = new
ArrayList<AbortSpec>(Arrays.asList(abortAtRolls));
NNStorage storage = new NNStorage(new Configuration(),
Collections.<URI>emptyList(),
@@ -979,16 +979,34 @@ public class TestEditLog extends TestCas
}
editlog.logSync();
}
- editlog.close();
+
+ if (closeOnFinish) {
+ editlog.close();
+ }
FSImageTestUtil.logStorageContents(LOG, storage);
return storage;
}
+
+ /**
+ * Set up directories for tests.
+ *
+ * Each rolled file is 10 txns long.
+ * A failed file is 2 txns long.
+ *
+ * @param editUris directories to create edit logs in
+ * @param numrolls number of times to roll the edit log during setup
+ * @param abortAtRolls Specifications for when to fail, see AbortSpec
+ */
+ public static NNStorage setupEdits(List<URI> editUris, int numrolls,
+ AbortSpec... abortAtRolls) throws IOException {
+ return setupEdits(editUris, numrolls, true, abortAtRolls);
+ }
/**
* Test loading an editlog which has had both its storage fail
* on alternating rolls. Two edit log directories are created.
- * The first on fails on odd rolls, the second on even. Test
+ * The first one fails on odd rolls, the second on even. Test
* that we are able to load the entire editlog regardless.
*/
@Test
Modified:
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java?rev=1229931&r1=1229930&r2=1229931&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java
(original)
+++
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java
Wed Jan 11 08:32:10 2012
@@ -60,7 +60,7 @@ public class TestFileJournalManager {
long numJournals = 0;
for (StorageDirectory sd : storage.dirIterable(NameNodeDirType.EDITS)) {
FileJournalManager jm = new FileJournalManager(sd);
- assertEquals(6*TXNS_PER_ROLL, jm.getNumberOfTransactions(1));
+ assertEquals(6*TXNS_PER_ROLL, jm.getNumberOfTransactions(1, true));
numJournals++;
}
assertEquals(3, numJournals);
@@ -81,7 +81,7 @@ public class TestFileJournalManager {
FileJournalManager jm = new FileJournalManager(sd);
assertEquals(5*TXNS_PER_ROLL + TXNS_PER_FAIL,
- jm.getNumberOfTransactions(1));
+ jm.getNumberOfTransactions(1, true));
}
/**
@@ -103,15 +103,16 @@ public class TestFileJournalManager {
Iterator<StorageDirectory> dirs =
storage.dirIterator(NameNodeDirType.EDITS);
StorageDirectory sd = dirs.next();
FileJournalManager jm = new FileJournalManager(sd);
- assertEquals(6*TXNS_PER_ROLL, jm.getNumberOfTransactions(1));
+ assertEquals(6*TXNS_PER_ROLL, jm.getNumberOfTransactions(1, true));
sd = dirs.next();
jm = new FileJournalManager(sd);
- assertEquals(5*TXNS_PER_ROLL + TXNS_PER_FAIL,
jm.getNumberOfTransactions(1));
+ assertEquals(5*TXNS_PER_ROLL + TXNS_PER_FAIL, jm.getNumberOfTransactions(1,
+ true));
sd = dirs.next();
jm = new FileJournalManager(sd);
- assertEquals(6*TXNS_PER_ROLL, jm.getNumberOfTransactions(1));
+ assertEquals(6*TXNS_PER_ROLL, jm.getNumberOfTransactions(1, true));
}
/**
@@ -135,15 +136,18 @@ public class TestFileJournalManager {
Iterator<StorageDirectory> dirs =
storage.dirIterator(NameNodeDirType.EDITS);
StorageDirectory sd = dirs.next();
FileJournalManager jm = new FileJournalManager(sd);
- assertEquals(5*TXNS_PER_ROLL + TXNS_PER_FAIL,
jm.getNumberOfTransactions(1));
+ assertEquals(5*TXNS_PER_ROLL + TXNS_PER_FAIL, jm.getNumberOfTransactions(1,
+ true));
sd = dirs.next();
jm = new FileJournalManager(sd);
- assertEquals(5*TXNS_PER_ROLL + TXNS_PER_FAIL,
jm.getNumberOfTransactions(1));
+ assertEquals(5*TXNS_PER_ROLL + TXNS_PER_FAIL, jm.getNumberOfTransactions(1,
+ true));
sd = dirs.next();
jm = new FileJournalManager(sd);
- assertEquals(5*TXNS_PER_ROLL + TXNS_PER_FAIL,
jm.getNumberOfTransactions(1));
+ assertEquals(5*TXNS_PER_ROLL + TXNS_PER_FAIL, jm.getNumberOfTransactions(1,
+ true));
}
/**
@@ -174,15 +178,15 @@ public class TestFileJournalManager {
FileJournalManager jm = new FileJournalManager(sd);
long expectedTotalTxnCount = TXNS_PER_ROLL*10 + TXNS_PER_FAIL;
- assertEquals(expectedTotalTxnCount, jm.getNumberOfTransactions(1));
+ assertEquals(expectedTotalTxnCount, jm.getNumberOfTransactions(1, true));
long skippedTxns = (3*TXNS_PER_ROLL); // skip first 3 files
long startingTxId = skippedTxns + 1;
- long numTransactionsToLoad = jm.getNumberOfTransactions(startingTxId);
+ long numTransactionsToLoad = jm.getNumberOfTransactions(startingTxId,
true);
long numLoaded = 0;
while (numLoaded < numTransactionsToLoad) {
- EditLogInputStream editIn = jm.getInputStream(startingTxId);
+ EditLogInputStream editIn = jm.getInputStream(startingTxId, true);
FSEditLogLoader.EditLogValidation val =
FSEditLogLoader.validateEditLog(editIn);
long count = val.getNumTransactions();
@@ -212,7 +216,8 @@ public class TestFileJournalManager {
// 10 rolls, so 11 rolled files, 110 txids total.
final int TOTAL_TXIDS = 10 * 11;
for (int txid = 1; txid <= TOTAL_TXIDS; txid++) {
- assertEquals((TOTAL_TXIDS - txid) + 1, jm.getNumberOfTransactions(txid));
+ assertEquals((TOTAL_TXIDS - txid) + 1, jm.getNumberOfTransactions(txid,
+ true));
}
}
@@ -244,10 +249,10 @@ public class TestFileJournalManager {
assertTrue(files[0].delete());
FileJournalManager jm = new FileJournalManager(sd);
- assertEquals(startGapTxId-1, jm.getNumberOfTransactions(1));
+ assertEquals(startGapTxId-1, jm.getNumberOfTransactions(1, true));
try {
- jm.getNumberOfTransactions(startGapTxId);
+ jm.getNumberOfTransactions(startGapTxId, true);
fail("Should have thrown an exception by now");
} catch (IOException ioe) {
assertTrue(true);
@@ -255,7 +260,7 @@ public class TestFileJournalManager {
// rolled 10 times so there should be 11 files.
assertEquals(11*TXNS_PER_ROLL - endGapTxId,
- jm.getNumberOfTransactions(endGapTxId+1));
+ jm.getNumberOfTransactions(endGapTxId + 1, true));
}
/**
@@ -282,7 +287,7 @@ public class TestFileJournalManager {
FileJournalManager jm = new FileJournalManager(sd);
assertEquals(10*TXNS_PER_ROLL+1,
- jm.getNumberOfTransactions(1));
+ jm.getNumberOfTransactions(1, true));
}
@Test
@@ -323,11 +328,37 @@ public class TestFileJournalManager {
FileJournalManager jm = new FileJournalManager(sd);
- EditLogInputStream elis = jm.getInputStream(5);
+ EditLogInputStream elis = jm.getInputStream(5, true);
FSEditLogOp op = elis.readOp();
assertEquals("read unexpected op", op.getTransactionId(), 5);
}
+ /**
+ * Make sure that in-progress streams aren't counted if we don't ask for
+ * them.
+ */
+ @Test
+ public void testExcludeInProgressStreams() throws CorruptionException,
+ IOException {
+ File f = new File(TestEditLog.TEST_DIR + "/filejournaltest2");
+
+ // Don't close the edit log once the files have been set up.
+ NNStorage storage = setupEdits(Collections.<URI>singletonList(f.toURI()),
+ 10, false);
+ StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next();
+
+ FileJournalManager jm = new FileJournalManager(sd);
+
+ // If we exclude the in-progess stream, we should only have 100 tx.
+ assertEquals(100, jm.getNumberOfTransactions(1, false));
+
+ EditLogInputStream elis = jm.getInputStream(90, false);
+ FSEditLogOp lastReadOp = null;
+ while ((lastReadOp = elis.readOp()) != null) {
+ assertTrue(lastReadOp.getTransactionId() <= 100);
+ }
+ }
+
private static String getLogsAsString(
FileJournalManager fjm, long firstTxId) throws IOException {
return Joiner.on(",").join(fjm.getRemoteEditLogs(firstTxId));
Modified:
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java?rev=1229931&r1=1229930&r2=1229931&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java
(original)
+++
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java
Wed Jan 11 08:32:10 2012
@@ -144,13 +144,13 @@ public class TestGenericJournalConf {
}
@Override
- public EditLogInputStream getInputStream(long fromTxnId)
+ public EditLogInputStream getInputStream(long fromTxnId, boolean
inProgressOk)
throws IOException {
return null;
}
@Override
- public long getNumberOfTransactions(long fromTxnId)
+ public long getNumberOfTransactions(long fromTxnId, boolean inProgressOk)
throws IOException {
return 0;
}
