Author: eli
Date: Thu Nov 18 21:26:01 2010
New Revision: 1036642
URL: http://svn.apache.org/viewvc?rev=1036642&view=rev
Log:
HDFS-1487. svn merge -c 1036310 from trunk
Modified:
hadoop/hdfs/branches/branch-0.22/ (props changed)
hadoop/hdfs/branches/branch-0.22/CHANGES.txt
hadoop/hdfs/branches/branch-0.22/build.xml (props changed)
hadoop/hdfs/branches/branch-0.22/src/c++/libhdfs/ (props changed)
hadoop/hdfs/branches/branch-0.22/src/contrib/hdfsproxy/ (props changed)
hadoop/hdfs/branches/branch-0.22/src/java/ (props changed)
hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java
(props changed)
hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
hadoop/hdfs/branches/branch-0.22/src/test/hdfs/ (props changed)
hadoop/hdfs/branches/branch-0.22/src/test/hdfs/org/apache/hadoop/hdfs/TestAbandonBlock.java
hadoop/hdfs/branches/branch-0.22/src/webapps/datanode/ (props changed)
hadoop/hdfs/branches/branch-0.22/src/webapps/hdfs/ (props changed)
hadoop/hdfs/branches/branch-0.22/src/webapps/secondary/ (props changed)
Propchange: hadoop/hdfs/branches/branch-0.22/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Nov 18 21:26:01 2010
@@ -1,4 +1,4 @@
/hadoop/core/branches/branch-0.19/hdfs:713112
/hadoop/hdfs/branches/HDFS-265:796829-820463
/hadoop/hdfs/branches/branch-0.21:820487
-/hadoop/hdfs/trunk:1036213,1036303
+/hadoop/hdfs/trunk:1036213,1036303,1036310
Modified: hadoop/hdfs/branches/branch-0.22/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.22/CHANGES.txt?rev=1036642&r1=1036641&r2=1036642&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.22/CHANGES.txt (original)
+++ hadoop/hdfs/branches/branch-0.22/CHANGES.txt Thu Nov 18 21:26:01 2010
@@ -375,6 +375,9 @@ Release 0.22.0 - Unreleased
HDFS-1507. TestAbandonBlock should abandon a block. (eli)
+ HDFS-1487. FSDirectory.removeBlock() should update diskspace count
+ of the block owner node (Zhong Wang via eli).
+
Release 0.21.1 - Unreleased
IMPROVEMENTS
Propchange: hadoop/hdfs/branches/branch-0.22/build.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Nov 18 21:26:01 2010
@@ -2,4 +2,4 @@
/hadoop/core/trunk/build.xml:779102
/hadoop/hdfs/branches/HDFS-265/build.xml:796829-820463
/hadoop/hdfs/branches/branch-0.21/build.xml:820487
-/hadoop/hdfs/trunk/build.xml:1036213,1036303
+/hadoop/hdfs/trunk/build.xml:1036213,1036303,1036310
Propchange: hadoop/hdfs/branches/branch-0.22/src/c++/libhdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Nov 18 21:26:01 2010
@@ -1,3 +1,3 @@
/hadoop/core/branches/branch-0.19/mapred/src/c++/libhdfs:713112
/hadoop/core/trunk/src/c++/libhdfs:776175-784663
-/hadoop/hdfs/trunk/src/c++/libhdfs:1036213,1036303
+/hadoop/hdfs/trunk/src/c++/libhdfs:1036213,1036303,1036310
Propchange: hadoop/hdfs/branches/branch-0.22/src/contrib/hdfsproxy/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Nov 18 21:26:01 2010
@@ -2,4 +2,4 @@
/hadoop/core/trunk/src/contrib/hdfsproxy:776175-784663
/hadoop/hdfs/branches/HDFS-265/src/contrib/hdfsproxy:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/contrib/hdfsproxy:820487
-/hadoop/hdfs/trunk/src/contrib/hdfsproxy:1036213,1036303
+/hadoop/hdfs/trunk/src/contrib/hdfsproxy:1036213,1036303,1036310
Propchange: hadoop/hdfs/branches/branch-0.22/src/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Nov 18 21:26:01 2010
@@ -2,4 +2,4 @@
/hadoop/core/trunk/src/hdfs:776175-785643,785929-786278
/hadoop/hdfs/branches/HDFS-265/src/java:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/java:820487
-/hadoop/hdfs/trunk/src/java:1036213,1036303
+/hadoop/hdfs/trunk/src/java:1036213,1036303,1036310
Propchange:
hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Nov 18 21:26:01 2010
@@ -4,4 +4,4 @@
/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:776175-785643,785929-786278
/hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:820487
-/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:1036213,1036303
+/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:1036213,1036303,1036310
Modified:
hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
URL:
http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java?rev=1036642&r1=1036641&r2=1036642&view=diff
==============================================================================
---
hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
(original)
+++
hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
Thu Nov 18 21:26:01 2010
@@ -453,6 +453,11 @@ class FSDirectory implements Closeable {
+path+" with "+block
+" block is added to the file system");
}
+
+ // update space consumed
+ INode[] pathINodes = getExistingPathINodes(path);
+ updateCount(pathINodes, pathINodes.length-1, 0,
+ -fileNode.getPreferredBlockSize()*fileNode.getReplication(), true);
} finally {
writeUnlock();
}
Propchange: hadoop/hdfs/branches/branch-0.22/src/test/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Nov 18 21:26:01 2010
@@ -2,4 +2,4 @@
/hadoop/core/trunk/src/test/hdfs:776175-785643
/hadoop/hdfs/branches/HDFS-265/src/test/hdfs:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/test/hdfs:820487
-/hadoop/hdfs/trunk/src/test/hdfs:1036213,1036303
+/hadoop/hdfs/trunk/src/test/hdfs:1036213,1036303,1036310
Modified:
hadoop/hdfs/branches/branch-0.22/src/test/hdfs/org/apache/hadoop/hdfs/TestAbandonBlock.java
URL:
http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.22/src/test/hdfs/org/apache/hadoop/hdfs/TestAbandonBlock.java?rev=1036642&r1=1036641&r2=1036642&view=diff
==============================================================================
---
hadoop/hdfs/branches/branch-0.22/src/test/hdfs/org/apache/hadoop/hdfs/TestAbandonBlock.java
(original)
+++
hadoop/hdfs/branches/branch-0.22/src/test/hdfs/org/apache/hadoop/hdfs/TestAbandonBlock.java
Thu Nov 18 21:26:01 2010
@@ -23,9 +23,12 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
+import org.apache.hadoop.hdfs.protocol.FSConstants;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
+import static org.junit.Assert.*;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -76,4 +79,30 @@ public class TestAbandonBlock {
// And close the file
fout.close();
}
+
+ @Test
+ /** Make sure that the quota is decremented correctly when a block is
abandoned */
+ public void testQuotaUpdatedWhenBlockAbandoned() throws IOException {
+ DistributedFileSystem dfs = (DistributedFileSystem)fs;
+ // Setting diskspace quota to 3MB
+ dfs.setQuota(new Path("/"), FSConstants.QUOTA_DONT_SET, 3 * 1024 * 1024);
+
+ // Start writing a file with 2 replicas to ensure each datanode has one.
+ // Block Size is 1MB.
+ String src = FILE_NAME_PREFIX + "test_quota1";
+ FSDataOutputStream fout = fs.create(new Path(src), true, 4096, (short)2,
1024 * 1024);
+ for (int i = 0; i < 1024; i++) {
+ fout.writeByte(123);
+ }
+
+ // Shutdown one datanode, causing the block abandonment.
+ cluster.getDataNodes().get(0).shutdown();
+
+ // Close the file, new block will be allocated with 2MB pending size.
+ try {
+ fout.close();
+ } catch (QuotaExceededException e) {
+ fail("Unexpected quota exception when closing fout");
+ }
+ }
}
Propchange: hadoop/hdfs/branches/branch-0.22/src/webapps/datanode/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Nov 18 21:26:01 2010
@@ -2,4 +2,4 @@
/hadoop/core/trunk/src/webapps/datanode:776175-784663
/hadoop/hdfs/branches/HDFS-265/src/webapps/datanode:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/webapps/datanode:820487
-/hadoop/hdfs/trunk/src/webapps/datanode:1036213,1036303
+/hadoop/hdfs/trunk/src/webapps/datanode:1036213,1036303,1036310
Propchange: hadoop/hdfs/branches/branch-0.22/src/webapps/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Nov 18 21:26:01 2010
@@ -2,4 +2,4 @@
/hadoop/core/trunk/src/webapps/hdfs:776175-784663
/hadoop/hdfs/branches/HDFS-265/src/webapps/hdfs:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/webapps/hdfs:820487
-/hadoop/hdfs/trunk/src/webapps/hdfs:1036213,1036303
+/hadoop/hdfs/trunk/src/webapps/hdfs:1036213,1036303,1036310
Propchange: hadoop/hdfs/branches/branch-0.22/src/webapps/secondary/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Nov 18 21:26:01 2010
@@ -2,4 +2,4 @@
/hadoop/core/trunk/src/webapps/secondary:776175-784663
/hadoop/hdfs/branches/HDFS-265/src/webapps/secondary:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/webapps/secondary:820487
-/hadoop/hdfs/trunk/src/webapps/secondary:1036213,1036303
+/hadoop/hdfs/trunk/src/webapps/secondary:1036213,1036303,1036310
