Author: todd
Date: Tue Feb 7 01:39:14 2012
New Revision: 1241317
URL: http://svn.apache.org/viewvc?rev=1241317&view=rev
Log:
HDFS-2794. Active NN may purge edit log files before standby NN has a chance to
read them. Contributed by Todd Lipcon.
Modified:
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionFunctional.java
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java
Modified:
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt?rev=1241317&r1=1241316&r2=1241317&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt
(original)
+++
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt
Tue Feb 7 01:39:14 2012
@@ -176,3 +176,5 @@ HDFS-2752. HA: exit if multiple shared d
HDFS-2894. HA: automatically determine the nameservice Id if only one
nameservice is configured. (eli)
HDFS-2733. Document HA configuration and CLI. (atm)
+
+HDFS-2794. Active NN may purge edit log files before standby NN has a chance
to read them (todd)
Modified:
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1241317&r1=1241316&r2=1241317&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
(original)
+++
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
Tue Feb 7 01:39:14 2012
@@ -144,6 +144,9 @@ public class DFSConfigKeys extends Commo
public static final boolean DFS_NAMENODE_SUPPORT_ALLOW_FORMAT_DEFAULT = true;
public static final String DFS_NAMENODE_NUM_CHECKPOINTS_RETAINED_KEY =
"dfs.namenode.num.checkpoints.retained";
public static final int DFS_NAMENODE_NUM_CHECKPOINTS_RETAINED_DEFAULT =
2;
+ public static final String DFS_NAMENODE_NUM_EXTRA_EDITS_RETAINED_KEY =
"dfs.namenode.num.extra.edits.retained";
+ public static final int DFS_NAMENODE_NUM_EXTRA_EDITS_RETAINED_DEFAULT =
1000000; //1M
+
public static final String DFS_NAMENODE_EDITS_DIR_MINIMUM_KEY =
"dfs.namenode.edits.dir.minimum";
public static final int DFS_NAMENODE_EDITS_DIR_MINIMUM_DEFAULT = 1;
Modified:
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java?rev=1241317&r1=1241316&r2=1241317&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java
(original)
+++
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java
Tue Feb 7 01:39:14 2012
@@ -31,6 +31,7 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.FileJournalManager.EditLogFile;
import org.apache.hadoop.hdfs.util.MD5FileUtils;
+import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
@@ -46,6 +47,7 @@ import com.google.common.collect.Sets;
public class NNStorageRetentionManager {
private final int numCheckpointsToRetain;
+ private final long numExtraEditsToRetain;
private static final Log LOG = LogFactory.getLog(
NNStorageRetentionManager.class);
private final NNStorage storage;
@@ -60,6 +62,15 @@ public class NNStorageRetentionManager {
this.numCheckpointsToRetain = conf.getInt(
DFSConfigKeys.DFS_NAMENODE_NUM_CHECKPOINTS_RETAINED_KEY,
DFSConfigKeys.DFS_NAMENODE_NUM_CHECKPOINTS_RETAINED_DEFAULT);
+ this.numExtraEditsToRetain = conf.getLong(
+ DFSConfigKeys.DFS_NAMENODE_NUM_EXTRA_EDITS_RETAINED_KEY,
+ DFSConfigKeys.DFS_NAMENODE_NUM_EXTRA_EDITS_RETAINED_DEFAULT);
+ Preconditions.checkArgument(numCheckpointsToRetain > 0,
+ "Must retain at least one checkpoint");
+ Preconditions.checkArgument(numExtraEditsToRetain >= 0,
+ DFSConfigKeys.DFS_NAMENODE_NUM_EXTRA_EDITS_RETAINED_KEY +
+ " must not be negative");
+
this.storage = storage;
this.editLog = editLog;
this.purger = purger;
@@ -79,8 +90,12 @@ public class NNStorageRetentionManager {
purgeCheckpointsOlderThan(inspector, minImageTxId);
// If fsimage_N is the image we want to keep, then we need to keep
// all txns > N. We can remove anything < N+1, since fsimage_N
- // reflects the state up to and including N.
- editLog.purgeLogsOlderThan(minImageTxId + 1);
+ // reflects the state up to and including N. However, we also
+ // provide a "cushion" of older txns that we keep, which is
+ // handy for HA, where a remote node may not have as many
+ // new images.
+ long purgeLogsFrom = Math.max(0, minImageTxId + 1 - numExtraEditsToRetain);
+ editLog.purgeLogsOlderThan(purgeLogsFrom);
}
private void purgeCheckpointsOlderThan(
Modified:
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml?rev=1241317&r1=1241316&r2=1241317&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
(original)
+++
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
Tue Feb 7 01:39:14 2012
@@ -638,6 +638,19 @@
</property>
<property>
+ <name>dfs.namenode.num.extra.edits.retained</name>
+ <value>1000000</value>
+ <description>The number of extra transactions which should be retained
+ beyond what is minimally necessary for a NN restart. This can be useful for
+ audit purposes or for an HA setup where a remote Standby Node may have
+ been offline for some time and need to have a longer backlog of retained
+ edits in order to start again.
+ Typically each edit is on the order of a few hundred bytes, so the default
+ of 1 million edits should be on the order of hundreds of MBs or low GBs.
+ </description>
+</property>
+
+<property>
<name>dfs.namenode.delegation.key.update-interval</name>
<value>86400000</value>
<description>The update interval for master key for delegation tokens
Modified:
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionFunctional.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionFunctional.java?rev=1241317&r1=1241316&r2=1241317&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionFunctional.java
(original)
+++
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionFunctional.java
Tue Feb 7 01:39:14 2012
@@ -61,6 +61,7 @@ public class TestNNStorageRetentionFunct
throws IOException {
MiniDFSCluster cluster = null;
Configuration conf = new HdfsConfiguration();
+ conf.setLong(DFSConfigKeys.DFS_NAMENODE_NUM_EXTRA_EDITS_RETAINED_KEY, 0);
File sd0 = new File(TEST_ROOT_DIR, "nn0");
File sd1 = new File(TEST_ROOT_DIR, "nn1");
Modified:
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java?rev=1241317&r1=1241316&r2=1241317&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java
(original)
+++
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java
Tue Feb 7 01:39:14 2012
@@ -23,6 +23,7 @@ import java.util.Map;
import java.util.Set;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
import org.apache.hadoop.hdfs.server.namenode.FileJournalManager.EditLogFile;
import
org.apache.hadoop.hdfs.server.namenode.FSImageStorageInspector.FSImageFile;
@@ -33,6 +34,7 @@ import static org.apache.hadoop.hdfs.ser
import
org.apache.hadoop.hdfs.server.namenode.NNStorageRetentionManager.StoragePurger;
import org.junit.Assert;
+import org.junit.Before;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.Mockito;
@@ -46,6 +48,17 @@ import com.google.common.collect.Sets;
public class TestNNStorageRetentionManager {
+ Configuration conf = new Configuration();
+
+ /**
+ * For the purpose of this test, purge as many edits as we can
+ * with no extra "safety cushion"
+ */
+ @Before
+ public void setNoExtraEditRetention() {
+ conf.setLong(DFSConfigKeys.DFS_NAMENODE_NUM_EXTRA_EDITS_RETAINED_KEY, 0);
+ }
+
/**
* Test the "easy case" where we have more images in the
* directory than we need to keep. Should purge the
@@ -163,9 +176,27 @@ public class TestNNStorageRetentionManag
runTest(tc);
}
- private void runTest(TestCaseDescription tc) throws IOException {
- Configuration conf = new Configuration();
+ @Test
+ public void testRetainExtraLogs() throws IOException {
+ conf.setLong(DFSConfigKeys.DFS_NAMENODE_NUM_EXTRA_EDITS_RETAINED_KEY,
+ 50);
+ TestCaseDescription tc = new TestCaseDescription();
+ tc.addRoot("/foo1", NameNodeDirType.IMAGE);
+ tc.addRoot("/foo2", NameNodeDirType.EDITS);
+ tc.addImage("/foo1/current/" + getImageFileName(100), true);
+ tc.addImage("/foo1/current/" + getImageFileName(200), true);
+ tc.addImage("/foo1/current/" + getImageFileName(300), false);
+ tc.addImage("/foo1/current/" + getImageFileName(400), false);
+ tc.addLog("/foo2/current/" + getFinalizedEditsFileName(101, 200), true);
+ // Since we need 50 extra edits, *do* retain the 201-300 segment
+ tc.addLog("/foo2/current/" + getFinalizedEditsFileName(201, 300), false);
+ tc.addLog("/foo2/current/" + getFinalizedEditsFileName(301, 400), false);
+ tc.addLog("/foo2/current/" + getInProgressEditsFileName(401), false);
+ runTest(tc);
+ }
+
+ private void runTest(TestCaseDescription tc) throws IOException {
StoragePurger mockPurger =
Mockito.mock(NNStorageRetentionManager.StoragePurger.class);
ArgumentCaptor<FSImageFile> imagesPurgedCaptor =
