Author: rangadi
Date: Mon May 19 10:54:58 2008
New Revision: 657888
URL: http://svn.apache.org/viewvc?rev=657888&view=rev
Log:
HADOOP-3396. TestDatanodeBlockScanner occationally fails. (Lohit Vijayarenu via
rangadi)
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java
hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestDatanodeBlockScanner.java
Modified: hadoop/core/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=657888&r1=657887&r2=657888&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Mon May 19 10:54:58 2008
@@ -275,6 +275,9 @@
HADOOP-3399. A debug message was logged at info level. (rangadi)
+ HADOOP-3396. TestDatanodeBlockScanner occationally fails.
+ (Lohit Vijayarenu via rangadi)
+
Release 0.17.0 - 2008-05-18
INCOMPATIBLE CHANGES
Modified: hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java?rev=657888&r1=657887&r2=657888&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java
(original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java Mon May
19 10:54:58 2008
@@ -1329,7 +1329,17 @@
" as corrupt because datanode " + dn.getName() +
" does not exist. ");
}
- corruptReplicas.addToCorruptReplicasMap(blk, node);
+ // Check if the block is associated with an inode, if not
+ // ignore the request for now. This could happen when BlockScanner
+ // thread of Datanode reports bad block before Block reports are sent
+ // by the Datanode on startup
+ if (blocksMap.getINode(blk) == null)
+ NameNode.stateChangeLog.info("BLOCK NameSystem.markBlockAsCorrupt: " +
+ "block " + blk + " could not be marked " +
+ "as corrupt as it does not exists in " +
+ "blocksMap");
+ else
+ corruptReplicas.addToCorruptReplicasMap(blk, node);
}
/**
Modified:
hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestDatanodeBlockScanner.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestDatanodeBlockScanner.java?rev=657888&r1=657887&r2=657888&view=diff
==============================================================================
---
hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestDatanodeBlockScanner.java
(original)
+++
hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestDatanodeBlockScanner.java
Mon May 19 10:54:58 2008
@@ -163,11 +163,13 @@
public void testBlockCorruptionPolicy() throws IOException {
Configuration conf = new Configuration();
+ conf.setLong("dfs.blockreport.intervalMsec", 1000L);
Random random = new Random();
FileSystem fs = null;
DFSClient dfsClient = null;
LocatedBlocks blocks = null;
int blockCount = 0;
+ int rand = random.nextInt(3);
MiniDFSCluster cluster = new MiniDFSCluster(conf, 3, true, null);
cluster.waitActive();
@@ -178,27 +180,35 @@
dfsClient = new DFSClient(new InetSocketAddress("localhost",
cluster.getNameNodePort()), conf);
- blocks = dfsClient.namenode.
+ do {
+ blocks = dfsClient.namenode.
getBlockLocations(file1.toString(), 0, Long.MAX_VALUE);
- blockCount = blocks.get(0).getLocations().length;
- assertTrue(blockCount == 3);
+ blockCount = blocks.get(0).getLocations().length;
+ try {
+ LOG.info("Looping until expected blockCount of 3 is received");
+ Thread.sleep(1000);
+ } catch (InterruptedException ignore) {
+ }
+ } while (blockCount != 3);
assertTrue(blocks.get(0).isCorrupt() == false);
// Corrupt random replica of block
- corruptReplica(block, random.nextInt(3));
- cluster.shutdown();
+ corruptReplica(block, rand);
+
+ // Restart the datanode hoping the corrupt block to be reported
+ cluster.restartDataNode(rand);
- // Restart the cluster hoping the corrupt block to be reported
// We have 2 good replicas and block is not corrupt
- cluster = new MiniDFSCluster(conf, 3, false, null);
- cluster.waitActive();
- fs = cluster.getFileSystem();
- dfsClient = new DFSClient(new InetSocketAddress("localhost",
- cluster.getNameNodePort()), conf);
- blocks = dfsClient.namenode.
+ do {
+ blocks = dfsClient.namenode.
getBlockLocations(file1.toString(), 0, Long.MAX_VALUE);
- blockCount = blocks.get(0).getLocations().length;
- assertTrue (blockCount == 2);
+ blockCount = blocks.get(0).getLocations().length;
+ try {
+ LOG.info("Looping until expected blockCount of 2 is received");
+ Thread.sleep(1000);
+ } catch (InterruptedException ignore) {
+ }
+ } while (blockCount != 2);
assertTrue(blocks.get(0).isCorrupt() == false);
// Corrupt all replicas. Now, block should be marked as corrupt
@@ -215,12 +225,18 @@
// Ignore exception
}
- // We now have he blocks to be marked as corrup and we get back all
+ // We now have the blocks to be marked as corrupt and we get back all
// its replicas
- blocks = dfsClient.namenode.
+ do {
+ blocks = dfsClient.namenode.
getBlockLocations(file1.toString(), 0, Long.MAX_VALUE);
- blockCount = blocks.get(0).getLocations().length;
- assertTrue (blockCount == 3);
+ blockCount = blocks.get(0).getLocations().length;
+ try {
+ LOG.info("Looping until expected blockCount of 3 is received");
+ Thread.sleep(1000);
+ } catch (InterruptedException ignore) {
+ }
+ } while (blockCount != 3);
assertTrue(blocks.get(0).isCorrupt() == true);
cluster.shutdown();
