Author: rangadi
Date: Mon Jun 22 21:48:07 2009
New Revision: 787417
URL: http://svn.apache.org/viewvc?rev=787417&view=rev
Log:
HDFS-381. Remove blocks from DataNode maps when corresponding file
is deleted. (Suresh Srinivas via rangadi)
Modified:
hadoop/hdfs/trunk/CHANGES.txt
hadoop/hdfs/trunk/src/java/hdfs-default.xml
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BlocksMap.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
Modified: hadoop/hdfs/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/hdfs/trunk/CHANGES.txt?rev=787417&r1=787416&r2=787417&view=diff
==============================================================================
--- hadoop/hdfs/trunk/CHANGES.txt (original)
+++ hadoop/hdfs/trunk/CHANGES.txt Mon Jun 22 21:48:07 2009
@@ -2,6 +2,12 @@
Trunk (unreleased changes)
+ IMPROVEMENTS
+
+ HDFS-381. Remove blocks from DataNode maps when corresponding file
+ is deleted. (Suresh Srinivas via rangadi)
+
+ BUG FIXES
HDFS-76. Better error message to users when commands fail because of
lack of quota. Allow quota to be set even if the limit is lower than
current consumption. (Boris Shkolnik via rangadi)
Modified: hadoop/hdfs/trunk/src/java/hdfs-default.xml
URL:
http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/hdfs-default.xml?rev=787417&r1=787416&r2=787417&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/hdfs-default.xml (original)
+++ hadoop/hdfs/trunk/src/java/hdfs-default.xml Mon Jun 22 21:48:07 2009
@@ -284,7 +284,7 @@
<property>
<name>dfs.blockreport.intervalMsec</name>
- <value>3600000</value>
+ <value>21600000</value>
<description>Determines block reporting interval in
milliseconds.</description>
</property>
Modified:
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java
URL:
http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java?rev=787417&r1=787416&r2=787417&view=diff
==============================================================================
---
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java
(original)
+++
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java
Mon Jun 22 21:48:07 2009
@@ -547,11 +547,10 @@
// initialize data structure for the return value
List<List<Block>> blocksToReplicate = new ArrayList<List<Block>>(
UnderReplicatedBlocks.LEVEL);
+ for (int i = 0; i < UnderReplicatedBlocks.LEVEL; i++) {
+ blocksToReplicate.add(new ArrayList<Block>());
+ }
synchronized (namesystem) {
- for (int i = 0; i < UnderReplicatedBlocks.LEVEL; i++) {
- blocksToReplicate.add(new ArrayList<Block>());
- }
-
synchronized (neededReplications) {
if (neededReplications.size() == 0) {
missingBlocksInCurIter = 0;
@@ -1284,9 +1283,9 @@
}
void removeBlock(Block block) {
- blocksMap.removeINode(block);
- corruptReplicas.removeFromCorruptReplicasMap(block);
addToInvalidates(block);
+ corruptReplicas.removeFromCorruptReplicasMap(block);
+ blocksMap.removeBlock(block);
}
BlockInfo getStoredBlock(Block block) {
@@ -1400,10 +1399,6 @@
return blocksMap.addINode(block, iNode);
}
- void removeINode(Block block) {
- blocksMap.removeINode(block);
- }
-
INodeFile getINode(Block b) {
return blocksMap.getINode(b);
}
@@ -1416,8 +1411,8 @@
return corruptReplicas.numCorruptReplicas(block);
}
- void removeBlockFromMap(BlockInfo blockInfo) {
- blocksMap.removeBlock(blockInfo);
+ void removeBlockFromMap(Block block) {
+ blocksMap.removeBlock(block);
}
public int getCapacity() {
Modified:
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BlocksMap.java
URL:
http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BlocksMap.java?rev=787417&r1=787416&r2=787417&view=diff
==============================================================================
---
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BlocksMap.java
(original)
+++
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BlocksMap.java
Mon Jun 22 21:48:07 2009
@@ -347,36 +347,22 @@
}
/**
- * Remove INode reference from block b.
- * If it does not belong to any file and data-nodes,
- * then remove the block from the block map.
- */
- void removeINode(Block b) {
- BlockInfo info = map.get(b);
- if (info != null) {
- info.inode = null;
- if (info.getDatanode(0) == null) { // no datanodes left
- map.remove(b); // remove block from the map
- }
- }
- }
-
- /**
* Remove the block from the block map;
* remove it from all data-node lists it belongs to;
* and remove all data-node locations associated with the block.
*/
- void removeBlock(BlockInfo blockInfo) {
+ void removeBlock(Block block) {
+ BlockInfo blockInfo = map.remove(block);
if (blockInfo == null)
return;
+
blockInfo.inode = null;
for(int idx = blockInfo.numNodes()-1; idx >= 0; idx--) {
DatanodeDescriptor dn = blockInfo.getDatanode(idx);
dn.removeBlock(blockInfo); // remove from the list and wipe the location
}
- map.remove(blockInfo); // remove block from the map
}
-
+
/** Returns the block object it it exists in the map. */
BlockInfo getStoredBlock(Block b) {
return map.get(b);
Modified:
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
URL:
http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java?rev=787417&r1=787416&r2=787417&view=diff
==============================================================================
---
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
(original)
+++
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
Mon Jun 22 21:48:07 2009
@@ -328,7 +328,7 @@
synchronized (rootDir) {
// modify file-> block and blocksMap
fileNode.removeBlock(block);
- getBlockManager().removeINode(block);
+ getBlockManager().removeBlockFromMap(block);
// If block is removed from blocksMap remove it from corruptReplicasMap
getBlockManager().removeFromCorruptReplicasMap(block);
