Author: cutting
Date: Fri Feb 16 14:23:31 2007
New Revision: 508622
URL: http://svn.apache.org/viewvc?view=rev&rev=508622
Log:
HADOOP-947. Improve performance of datanode decomissioning. Contributed by
Dhruba.
Modified:
lucene/hadoop/trunk/CHANGES.txt
lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java
Modified: lucene/hadoop/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?view=diff&rev=508622&r1=508621&r2=508622
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Fri Feb 16 14:23:31 2007
@@ -71,6 +71,9 @@
21. HADOOP-333. Add validator for sort benchmark output.
(Arun C Murthy via cutting)
+22. HADOOP-947. Improve performance of datanode decomissioning.
+ (Dhruba Borthakur via cutting)
+
Branch 0.11 (unreleased)
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java?view=diff&rev=508622&r1=508621&r2=508622
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java Fri
Feb 16 14:23:31 2007
@@ -2515,18 +2515,26 @@
return nonCommissionedNodeList;
}
/*
- * Return true if there are any blocks in neededReplication that
- * reside on the specified node. Otherwise returns false.
+ * Return true if there are any blocks on this node that have not
+ * yet reached their replication factor. Otherwise returns false.
*/
private boolean isReplicationInProgress(DatanodeDescriptor srcNode) {
- for (Iterator<Block> it = neededReplications.iterator();
it.hasNext();){
- Block block = it.next();
+ Block decommissionBlocks[] = srcNode.getBlocks();
+ for (int i = 0; i < decommissionBlocks.length; i++) {
+ Block block = decommissionBlocks[i];
+ FSDirectory.INode fileINode = dir.getFileByBlock(block);
+ if (fileINode == null) {
+ continue;
+ }
Collection<DatanodeDescriptor> containingNodes =
blocksMap.get(block);
- if (containingNodes.contains(srcNode)) {
- return true;
+ List<DatanodeDescriptor> nodes =
+ filterDecommissionedNodes(containingNodes);
+ int numCurrentReplica = nodes.size();
+ if (fileINode.getReplication() > numCurrentReplica) {
+ return true;
}
}
- return false;
+ return false;
}
/**
@@ -2535,8 +2543,8 @@
*/
private boolean checkDecommissionStateInternal(DatanodeDescriptor node) {
//
- // Check to see if there are any blocks in the neededReplication
- // data structure that has a replica on the node being decommissioned.
+ // Check to see if there are all blocks in this decommisioned
+ // node has reached their target replication factor.
//
if (node.isDecommissionInProgress()) {
if (!isReplicationInProgress(node)) {
