Author: atm
Date: Tue Nov 13 01:26:20 2012
New Revision: 1408531
URL: http://svn.apache.org/viewvc?rev=1408531&view=rev
Log:
HDFS-3921. NN will prematurely consider blocks missing when entering active
state while still in safe mode. Contributed by Aaron T. Myers.
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1408531&r1=1408530&r2=1408531&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Tue Nov 13
01:26:20 2012
@@ -580,6 +580,9 @@ Release 2.0.3-alpha - Unreleased
HDFS-4164. fuse_dfs: add -lrt to the compiler command line on Linux.
(Colin Patrick McCabe via eli)
+ HDFS-3921. NN will prematurely consider blocks missing when entering active
+ state while still in safe mode. (atm)
+
Release 2.0.2-alpha - 2012-09-07
INCOMPATIBLE CHANGES
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1408531&r1=1408530&r2=1408531&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
(original)
+++
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
Tue Nov 13 01:26:20 2012
@@ -660,13 +660,17 @@ public class FSNamesystem implements Nam
LOG.info("Catching up to latest edits from old active before " +
"taking over writer role in edits logs");
editLogTailer.catchupDuringFailover();
- blockManager.setPostponeBlocksFromFuture(false);
- LOG.info("Reprocessing replication and invalidation queues");
+ blockManager.setPostponeBlocksFromFuture(false);
blockManager.getDatanodeManager().markAllDatanodesStale();
blockManager.clearQueues();
blockManager.processAllPendingDNMessages();
- blockManager.processMisReplicatedBlocks();
+
+ if (!isInSafeMode() ||
+ (isInSafeMode() && safeMode.isPopulatingReplQueues())) {
+ LOG.info("Reprocessing replication and invalidation queues");
+ blockManager.processMisReplicatedBlocks();
+ }
if (LOG.isDebugEnabled()) {
LOG.debug("NameNode metadata after re-processing " +
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java?rev=1408531&r1=1408530&r2=1408531&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java
(original)
+++
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java
Tue Nov 13 01:26:20 2012
@@ -631,6 +631,32 @@ public class TestHASafeMode {
}
/**
+ * Make sure that when we transition to active in safe mode that we don't
+ * prematurely consider blocks missing just because not all DNs have reported
+ * yet.
+ *
+ * This is a regression test for HDFS-3921.
+ */
+ @Test
+ public void testNoPopulatingReplQueuesWhenStartingActiveInSafeMode()
+ throws IOException {
+ DFSTestUtil.createFile(fs, new Path("/test"), 15*BLOCK_SIZE, (short)3, 1L);
+
+ // Stop the DN so that when the NN restarts not all blocks wil be reported
+ // and the NN won't leave safe mode.
+ cluster.stopDataNode(1);
+ // Restart the namenode but don't wait for it to hear from all DNs (since
+ // one DN is deliberately shut down.)
+ cluster.restartNameNode(0, false);
+ cluster.transitionToActive(0);
+
+ assertTrue(cluster.getNameNode(0).isInSafeMode());
+ // We shouldn't yet consider any blocks "missing" since we're in startup
+ // safemode, i.e. not all DNs may have reported.
+ assertEquals(0, cluster.getNamesystem(0).getMissingBlocksCount());
+ }
+
+ /**
* Print a big banner in the test log to make debug easier.
*/
static void banner(String string) {
