Repository: hadoop Updated Branches: refs/heads/trunk 03bab8dea -> 5cb1e0118
HDFS-9533. seen_txid in the shared edits directory is modified during bootstrapping. Contributed by Kihwal Lee. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5cb1e011 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5cb1e011 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5cb1e011 Branch: refs/heads/trunk Commit: 5cb1e0118b173a95c1f7bdfae1e58d7833d61c26 Parents: 03bab8d Author: Kihwal Lee <kih...@apache.org> Authored: Thu Dec 17 17:11:16 2015 -0600 Committer: Kihwal Lee <kih...@apache.org> Committed: Thu Dec 17 17:11:16 2015 -0600 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hadoop/hdfs/server/namenode/NNStorage.java | 18 +++++++++++++++++- .../hdfs/server/namenode/ha/BootstrapStandby.java | 6 ++++-- .../hdfs/server/namenode/FSImageTestUtil.java | 7 +++++++ .../server/namenode/ha/TestBootstrapStandby.java | 11 +++++++++++ 5 files changed, 42 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cb1e011/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index fc9dfe9..f896544 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -2540,6 +2540,9 @@ Release 2.7.3 - UNRELEASED HDFS-9516. Truncate file fails with data dirs on multiple disks. (Plamen Jeliazkov via shv) + HDFS-9533. seen_txid in the shared edits directory is modified during + bootstrapping (kihwal) + Release 2.7.2 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cb1e011/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java index 9b63e72..ed36b27 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java @@ -482,8 +482,24 @@ public class NNStorage extends Storage implements Closeable, * @param txid the txid that has been reached */ public void writeTransactionIdFileToStorage(long txid) { + writeTransactionIdFileToStorage(txid, null); + } + + /** + * Write a small file in all available storage directories that + * indicates that the namespace has reached some given transaction ID. + * + * This is used when the image is loaded to avoid accidental rollbacks + * in the case where an edit log is fully deleted but there is no + * checkpoint. See TestNameEditsConfigs.testNameEditsConfigsFailure() + * @param txid the txid that has been reached + * @param type the type of directory + */ + public void writeTransactionIdFileToStorage(long txid, + NameNodeDirType type) { // Write txid marker in all storage directories - for (StorageDirectory sd : storageDirs) { + for (Iterator<StorageDirectory> it = dirIterator(type); it.hasNext();) { + StorageDirectory sd = it.next(); try { writeTransactionIdFile(sd, txid); } catch(IOException e) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cb1e011/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java index f694a7f..d84b0fa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java @@ -51,6 +51,7 @@ import org.apache.hadoop.hdfs.server.namenode.EditLogInputStream; import org.apache.hadoop.hdfs.server.namenode.FSImage; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.NNStorage; +import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType; import org.apache.hadoop.hdfs.server.namenode.NNUpgradeUtil; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.TransferFsImage; @@ -329,13 +330,14 @@ public class BootstrapStandby implements Tool, Configurable { return ERR_CODE_LOGS_UNAVAILABLE; } - image.getStorage().writeTransactionIdFileToStorage(curTxId); - // Download that checkpoint into our storage directories. MD5Hash hash = TransferFsImage.downloadImageToStorage( proxyInfo.getHttpAddress(), imageTxId, storage, true, true); image.saveDigestAndRenameCheckpointImage(NameNodeFile.IMAGE, imageTxId, hash); + + // Write seen_txid to the formatted image directories. + storage.writeTransactionIdFileToStorage(imageTxId, NameNodeDirType.IMAGE); } catch (IOException ioe) { throw ioe; } finally { http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cb1e011/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java index 68eff19..3884f87 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java @@ -568,4 +568,11 @@ public abstract class FSImageTestUtil { FSImageTestUtil.assertParallelFilesAreIdentical(curDirs, ignoredFiles); } + + public static long getStorageTxId(NameNode node, URI storageUri) + throws IOException { + StorageDirectory sDir = getFSImage(node).getStorage(). + getStorageDirectory(storageUri); + return NNStorage.readTransactionIdFile(sDir); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cb1e011/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java index 8c39548..48d505c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java @@ -146,6 +146,13 @@ public class TestBootstrapStandby { .getFSImage().getMostRecentCheckpointTxId(); assertEquals(6, expectedCheckpointTxId); + // advance the current txid + cluster.getFileSystem(0).create(new Path("/test_txid"), (short)1).close(); + + // obtain the content of seen_txid + URI editsUri = cluster.getSharedEditsDir(0, maxNNCount - 1); + long seen_txid_shared = FSImageTestUtil.getStorageTxId(nn0, editsUri); + for (int i = 1; i < maxNNCount; i++) { assertEquals(0, forceBootstrap(i)); @@ -156,6 +163,10 @@ public class TestBootstrapStandby { } FSImageTestUtil.assertNNFilesMatch(cluster); + // Make sure the seen_txid was not modified by the standby + assertEquals(seen_txid_shared, + FSImageTestUtil.getStorageTxId(nn0, editsUri)); + // We should now be able to start the standby successfully. restartNameNodesFromIndex(1); }