Repository: hadoop Updated Branches: refs/heads/branch-3.0 1fb87df87 -> 26395aef6 refs/heads/branch-3.1 9b37d1869 -> c58cec250 refs/heads/trunk b4f1ba141 -> ae290a4bb
HDFS-13145. SBN crash when transition to ANN with in-progress edit tailing enabled. Contributed by Chao Sun. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ae290a4b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ae290a4b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ae290a4b Branch: refs/heads/trunk Commit: ae290a4bb4e514e2fe9b40d28426a7589afe2a3f Parents: b4f1ba1 Author: Chao Sun <sunc...@apache.org> Authored: Mon Feb 26 15:37:27 2018 -0800 Committer: Konstantin V Shvachko <s...@apache.org> Committed: Mon Feb 26 16:15:00 2018 -0800 ---------------------------------------------------------------------- .../qjournal/client/QuorumJournalManager.java | 4 ++- .../client/TestQuorumJournalManager.java | 26 +++++++++++++++++++- 2 files changed, 28 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae290a4b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java index 7dff9b4..7a70a3d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java @@ -496,7 +496,9 @@ public class QuorumJournalManager implements JournalManager { // If it's bounded by durable Txns, endTxId could not be larger // than committedTxnId. This ensures the consistency. - if (onlyDurableTxns && inProgressOk) { + // We don't do the following for finalized log segments, since all + // edits in those are guaranteed to be committed. + if (onlyDurableTxns && inProgressOk && remoteLog.isInProgress()) { endTxId = Math.min(endTxId, committedTxnId); if (endTxId < remoteLog.getStartTxId()) { LOG.warn("Found endTxId (" + endTxId + ") that is less than " + http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae290a4b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java index ce1d404..34a0348 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java @@ -933,7 +933,31 @@ public class TestQuorumJournalManager { verifyEdits(streams, 25, 50); } - + + @Test + public void testInProgressRecovery() throws Exception { + // Test the case when in-progress edit log tailing is on, and + // new active performs recovery when the old active crashes + // without closing the last log segment. + // See HDFS-13145 for more details. + + // Write two batches of edits. After these, the commitId on the + // journals should be 5, and endTxnId should be 8. + EditLogOutputStream stm = qjm.startLogSegment(1, + NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION); + writeTxns(stm, 1, 5); + writeTxns(stm, 6, 3); + + // Do recovery from a separate QJM, just like in failover. + QuorumJournalManager qjm2 = createSpyingQJM(); + qjm2.recoverUnfinalizedSegments(); + checkRecovery(cluster, 1, 8); + + // When selecting input stream, we should see all txns up to 8. + List<EditLogInputStream> streams = new ArrayList<>(); + qjm2.selectInputStreams(streams, 1, true, true); + verifyEdits(streams, 1, 8); + } private QuorumJournalManager createSpyingQJM() throws IOException, URISyntaxException { --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
