Author: suresh
Date: Wed Mar 7 06:31:47 2012
New Revision: 1297861
URL: http://svn.apache.org/viewvc?rev=1297861&view=rev
Log:
HDFS-2477. Merging change r1196676 from trunk to 0.23
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1297861&r1=1297860&r2=1297861&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Wed Mar 7 06:31:47 2012
@@ -102,8 +102,14 @@ Release 0.23.3 - UNRELEASED
HDFS-2334. Add Closeable to JournalManager. (Ivan Kelly via jitendra)
+<<<<<<< .working
OPTIMIZATIONS
+=======
+ HDFS-2477. Optimize computing the diff between a block report and the
+ namenode state. (Tomasz Nykiel via hairong)
+
+>>>>>>> .merge-right.r1196676
BUG FIXES
HDFS-2481. Unknown protocol:
org.apache.hadoop.hdfs.protocol.ClientProtocol.
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java?rev=1297861&r1=1297860&r2=1297861&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
Wed Mar 7 06:31:47 2012
@@ -122,6 +122,38 @@ public class BlockInfo extends Block imp
triplets[index*3+2] = to;
}
+ /**
+ * Return the previous block on the block list for the datanode at
+ * position index. Set the previous block on the list to "to".
+ *
+ * @param index - the datanode index
+ * @param to - block to be set to previous on the list of blocks
+ * @return current previous block on the list of blocks
+ */
+ BlockInfo getSetPrevious(int index, BlockInfo to) {
+ assert this.triplets != null : "BlockInfo is not initialized";
+ assert index >= 0 && index*3+1 < triplets.length : "Index is out of
bound";
+ BlockInfo info = (BlockInfo)triplets[index*3+1];
+ triplets[index*3+1] = to;
+ return info;
+ }
+
+ /**
+ * Return the next block on the block list for the datanode at
+ * position index. Set the next block on the list to "to".
+ *
+ * @param index - the datanode index
+ * @param to - block to be set to next on the list of blocks
+ * * @return current next block on the list of blocks
+ */
+ BlockInfo getSetNext(int index, BlockInfo to) {
+ assert this.triplets != null : "BlockInfo is not initialized";
+ assert index >= 0 && index*3+2 < triplets.length : "Index is out of
bound";
+ BlockInfo info = (BlockInfo)triplets[index*3+2];
+ triplets[index*3+2] = to;
+ return info;
+ }
+
int getCapacity() {
assert this.triplets != null : "BlockInfo is not initialized";
assert triplets.length % 3 == 0 : "Malformed BlockInfo";
@@ -258,6 +290,27 @@ public class BlockInfo extends Block imp
}
/**
+ * Remove this block from the list of blocks related to the specified
+ * DatanodeDescriptor. Insert it into the head of the list of blocks.
+ *
+ * @return the new head of the list.
+ */
+ public BlockInfo moveBlockToHead(BlockInfo head, DatanodeDescriptor dn,
+ int curIndex, int headIndex) {
+ if (head == this) {
+ return this;
+ }
+ BlockInfo next = this.getSetNext(curIndex, head);
+ BlockInfo prev = this.getSetPrevious(curIndex, null);
+
+ head.setPrevious(headIndex, this);
+ prev.setNext(prev.findDatanode(dn), next);
+ if (next != null)
+ next.setPrevious(next.findDatanode(dn), prev);
+ return this;
+ }
+
+ /**
* BlockInfo represents a block that is not being constructed.
* In order to start modifying the block, the BlockInfo should be converted
* to {@link BlockInfoUnderConstruction}.
@@ -315,4 +368,4 @@ public class BlockInfo extends Block imp
public void setNext(LightWeightGSet.LinkedElement next) {
this.nextLinkedElement = next;
}
-}
\ No newline at end of file
+}
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java?rev=1297861&r1=1297860&r2=1297861&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
Wed Mar 7 06:31:47 2012
@@ -75,6 +75,7 @@ import com.google.common.annotations.Vis
*/
@InterfaceAudience.Private
public class BlockManager {
+
static final Log LOG = LogFactory.getLog(BlockManager.class);
/** Default load factor of map */
@@ -1471,7 +1472,10 @@ public class BlockManager {
BlockInfo delimiter = new BlockInfo(new Block(), 1);
boolean added = dn.addBlock(delimiter);
assert added : "Delimiting block cannot be present in the node";
- if(newReport == null)
+ int headIndex = 0; //currently the delimiter is in the head of the list
+ int curIndex;
+
+ if (newReport == null)
newReport = new BlockListAsLongs();
// scan the report and process newly reported blocks
BlockReportIterator itBR = newReport.getBlockReportIterator();
@@ -1481,8 +1485,9 @@ public class BlockManager {
BlockInfo storedBlock = processReportedBlock(dn, iblk, iState,
toAdd, toInvalidate, toCorrupt, toUC);
// move block to the head of the list
- if(storedBlock != null && storedBlock.findDatanode(dn) >= 0)
- dn.moveBlockToHead(storedBlock);
+ if (storedBlock != null && (curIndex = storedBlock.findDatanode(dn)) >=
0) {
+ headIndex = dn.moveBlockToHead(storedBlock, curIndex, headIndex);
+ }
}
// collect blocks that have not been reported
// all of them are next to the delimiter
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java?rev=1297861&r1=1297860&r2=1297861&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
Wed Mar 7 06:31:47 2012
@@ -244,15 +244,24 @@ public class DatanodeDescriptor extends
/**
* Move block to the head of the list of blocks belonging to the data-node.
+ * @return the index of the head of the blockList
*/
- void moveBlockToHead(BlockInfo b) {
- blockList = b.listRemove(blockList, this);
- blockList = b.listInsert(blockList, this);
+ int moveBlockToHead(BlockInfo b, int curIndex, int headIndex) {
+ blockList = b.moveBlockToHead(blockList, this, curIndex, headIndex);
+ return curIndex;
+ }
+
+ /**
+ * Used for testing only
+ * @return the head of the blockList
+ */
+ protected BlockInfo getHead(){
+ return blockList;
}
/**
* Replace specified old block with a new one in the DataNodeDescriptor.
- *
+ *
* @param oldBlock - block to be replaced
* @param newBlock - a replacement block
* @return the new block
