Author: shv
Date: Tue Jul 8 16:32:18 2008
New Revision: 675055
URL: http://svn.apache.org/viewvc?rev=675055&view=rev
Log:
HADOOP-3002. Merge -r 675011:675012 from trunk to branch 0.17.
Modified:
hadoop/core/branches/branch-0.17/CHANGES.txt
hadoop/core/branches/branch-0.17/src/java/org/apache/hadoop/dfs/DatanodeDescriptor.java
hadoop/core/branches/branch-0.17/src/java/org/apache/hadoop/dfs/FSNamesystem.java
hadoop/core/branches/branch-0.17/src/java/org/apache/hadoop/dfs/NameNode.java
Modified: hadoop/core/branches/branch-0.17/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.17/CHANGES.txt?rev=675055&r1=675054&r2=675055&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.17/CHANGES.txt (original)
+++ hadoop/core/branches/branch-0.17/CHANGES.txt Tue Jul 8 16:32:18 2008
@@ -7,6 +7,8 @@
HADOOP-3681. DFSClient can get into an infinite loop while closing
a file if there are some errors. (Lohit Vijayarenu via rangadi)
+ HADOOP-3002. Hold off block removal while in safe mode. (shv)
+
Release 0.17.1 - 2008-06-23
INCOMPATIBLE CHANGES
Modified:
hadoop/core/branches/branch-0.17/src/java/org/apache/hadoop/dfs/DatanodeDescriptor.java
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.17/src/java/org/apache/hadoop/dfs/DatanodeDescriptor.java?rev=675055&r1=675054&r2=675055&view=diff
==============================================================================
---
hadoop/core/branches/branch-0.17/src/java/org/apache/hadoop/dfs/DatanodeDescriptor.java
(original)
+++
hadoop/core/branches/branch-0.17/src/java/org/apache/hadoop/dfs/DatanodeDescriptor.java
Tue Jul 8 16:32:18 2008
@@ -332,7 +332,8 @@
void reportDiff(BlocksMap blocksMap,
BlockListAsLongs newReport,
Collection<Block> toAdd,
- Collection<Block> toRemove) {
+ Collection<Block> toRemove,
+ Collection<Block> toInvalidate) {
// place a deilimiter in the list which separates blocks
// that have been reported from those that have not
BlockInfo delimiter = new BlockInfo(new Block(), 1);
@@ -347,8 +348,9 @@
for (int i = 0; i < newReport.getNumberOfBlocks(); ++i) {
iblk.set(newReport.getBlockId(i), newReport.getBlockLen(i));
BlockInfo storedBlock = blocksMap.getStoredBlock(iblk);
- if(storedBlock == null) { // Brand new block
- toAdd.add(new Block(iblk));
+ if(storedBlock == null) {
+ // If block is not in blocksMap it does not belong to any file
+ toInvalidate.add(new Block(iblk));
continue;
}
if(storedBlock.findDatanode(this) < 0) {// Known block, but not on the DN
Modified:
hadoop/core/branches/branch-0.17/src/java/org/apache/hadoop/dfs/FSNamesystem.java
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.17/src/java/org/apache/hadoop/dfs/FSNamesystem.java?rev=675055&r1=675054&r2=675055&view=diff
==============================================================================
---
hadoop/core/branches/branch-0.17/src/java/org/apache/hadoop/dfs/FSNamesystem.java
(original)
+++
hadoop/core/branches/branch-0.17/src/java/org/apache/hadoop/dfs/FSNamesystem.java
Tue Jul 8 16:32:18 2008
@@ -2289,6 +2289,9 @@
int workFound = 0;
int blocksToProcess = 0;
int nodesToProcess = 0;
+ // blocks should not be replicated or removed if safe mode is on
+ if (isInSafeMode())
+ return workFound;
synchronized(heartbeats) {
blocksToProcess = (int)(heartbeats.size()
* ReplicationMonitor.REPLICATION_WORK_MULTIPLIER_PER_ITERATION);
@@ -2325,9 +2328,6 @@
private synchronized int computeReplicationWork(
int blocksToProcess) throws IOException {
int scheduledReplicationCount = 0;
- // blocks should not be replicated or removed if safe mode is on
- if (isInSafeMode())
- return scheduledReplicationCount;
synchronized(neededReplications) {
// # of blocks to process equals either twice the number of live
@@ -2695,9 +2695,9 @@
* The given node is reporting all its blocks. Use this info to
* update the (machine-->blocklist) and (block-->machinelist) tables.
*/
- public synchronized Block[] processReport(DatanodeID nodeID,
- BlockListAsLongs newReport
- ) throws IOException {
+ public synchronized void processReport(DatanodeID nodeID,
+ BlockListAsLongs newReport
+ ) throws IOException {
long startTime = now();
if (NameNode.stateChangeLog.isDebugEnabled()) {
NameNode.stateChangeLog.debug("BLOCK* NameSystem.processReport: "
@@ -2719,7 +2719,7 @@
if (node.getNetworkLocation().equals(NetworkTopology.UNRESOLVED)) {
LOG.info("Ignoring block report from " + nodeID.getName() +
" because rack location for this datanode is still to be
resolved.");
- return null; //drop the block report if the dn hasn't been resolved
+ return; //drop the block report if the dn hasn't been resolved
}
node.setBlockReportProcessed(true);
@@ -2729,7 +2729,8 @@
//
Collection<Block> toAdd = new LinkedList<Block>();
Collection<Block> toRemove = new LinkedList<Block>();
- node.reportDiff(blocksMap, newReport, toAdd, toRemove);
+ Collection<Block> toInvalidate = new LinkedList<Block>();
+ node.reportDiff(blocksMap, newReport, toAdd, toRemove, toInvalidate);
for (Block b : toRemove) {
removeStoredBlock(b, node);
@@ -2737,41 +2738,13 @@
for (Block b : toAdd) {
addStoredBlock(b, node, null);
}
-
- //
- // We've now completely updated the node's block report profile.
- // We now go through all its blocks and find which ones are invalid,
- // no longer pending, or over-replicated.
- //
- // (Note it's not enough to just invalidate blocks at lease expiry
- // time; datanodes can go down before the client's lease on
- // the failed file expires and miss the "expire" event.)
- //
- // This function considers every block on a datanode, and thus
- // should only be invoked infrequently.
- //
- Collection<Block> obsolete = new ArrayList<Block>();
- for (Iterator<Block> it = node.getBlockIterator(); it.hasNext();) {
- Block b = it.next();
-
- //
- // A block report can only send BLOCK_INVALIDATE_CHUNK number of
- // blocks to be deleted. If there are more blocks to be deleted,
- // they are added to recentInvalidateSets and will be sent out
- // thorugh succeeding heartbeat responses.
- //
- if (!isValidBlock(b)) {
- if (obsolete.size() > blockInvalidateLimit) {
- addToInvalidates(b, node);
- } else {
- obsolete.add(b);
- }
- NameNode.stateChangeLog.debug("BLOCK* NameSystem.processReport: "
- +"ask "+nodeID.getName()+" to delete
"+b.getBlockName());
- }
+ for (Block b : toInvalidate) {
+ NameNode.stateChangeLog.info("BLOCK* NameSystem.processReport: block "
+ + b + " on " + node.getName() + " size " + b.getNumBytes()
+ + " does not belong to any file.");
+ addToInvalidates(b, node);
}
NameNode.getNameNodeMetrics().blockReport.inc((int) (now() - startTime));
- return obsolete.toArray(new Block[obsolete.size()]);
}
/**
Modified:
hadoop/core/branches/branch-0.17/src/java/org/apache/hadoop/dfs/NameNode.java
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.17/src/java/org/apache/hadoop/dfs/NameNode.java?rev=675055&r1=675054&r2=675055&view=diff
==============================================================================
---
hadoop/core/branches/branch-0.17/src/java/org/apache/hadoop/dfs/NameNode.java
(original)
+++
hadoop/core/branches/branch-0.17/src/java/org/apache/hadoop/dfs/NameNode.java
Tue Jul 8 16:32:18 2008
@@ -607,9 +607,7 @@
stateChangeLog.debug("*BLOCK* NameNode.blockReport: "
+"from "+nodeReg.getName()+" "+blist.getNumberOfBlocks() +"
blocks");
- Block blocksToDelete[] = namesystem.processReport(nodeReg, blist);
- if (blocksToDelete != null && blocksToDelete.length > 0)
- return new BlockCommand(blocksToDelete);
+ namesystem.processReport(nodeReg, blist);
if (getFSImage().isUpgradeFinalized())
return new DatanodeCommand(DatanodeProtocol.DNA_FINALIZE);
return null;
