Author: hairong
Date: Fri Apr 30 22:06:04 2010
New Revision: 939841
URL: http://svn.apache.org/viewvc?rev=939841&view=rev
Log:
HDFS-1104. Fsck triggers full GC on NameNode. Contributed by Hairong Kuang.
Modified:
hadoop/hdfs/trunk/CHANGES.txt
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
Modified: hadoop/hdfs/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/hdfs/trunk/CHANGES.txt?rev=939841&r1=939840&r2=939841&view=diff
==============================================================================
--- hadoop/hdfs/trunk/CHANGES.txt (original)
+++ hadoop/hdfs/trunk/CHANGES.txt Fri Apr 30 22:06:04 2010
@@ -312,6 +312,8 @@ Trunk (unreleased changes)
HDFS-1078. Create static and dynamic versions of libhdfs.
(Sam Rash via dhruba)
+ HDFS-1104. Fsck triggers full GC on NameNode. (hairong)
+
Release 0.21.0 - Unreleased
INCOMPATIBLE CHANGES
Modified:
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL:
http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=939841&r1=939840&r2=939841&view=diff
==============================================================================
---
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
(original)
+++
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
Fri Apr 30 22:06:04 2010
@@ -706,10 +706,6 @@ public class FSNamesystem implements FSC
*/
LocatedBlocks getBlockLocations(String clientMachine, String src,
long offset, long length) throws IOException, UnresolvedLinkException {
- if (isPermissionEnabled) {
- checkPathAccess(src, FsAction.READ);
- }
-
LocatedBlocks blocks = getBlockLocations(src, offset, length, true);
if (blocks != null) {
//sort the blocks
@@ -729,6 +725,10 @@ public class FSNamesystem implements FSC
*/
LocatedBlocks getBlockLocations(String src, long offset, long length,
boolean doAccessTime) throws IOException, UnresolvedLinkException {
+ if (isPermissionEnabled) {
+ checkPathAccess(src, FsAction.READ);
+ }
+
if (offset < 0) {
throw new IOException("Negative offset is not supported. File: " + src );
}
Modified:
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
URL:
http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=939841&r1=939840&r2=939841&view=diff
==============================================================================
---
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
(original)
+++
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
Fri Apr 30 22:06:04 2010
@@ -31,6 +31,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FsServerDefaults;
@@ -587,6 +588,19 @@ public class NameNode implements Namenod
src, offset, length);
}
+ /**
+ * The specification of this method matches that of
+ * {...@link getBlockLocations(Path)}
+ * except that it does not update the file's access time.
+ */
+ LocatedBlocks getBlockLocationsNoATime(String src,
+ long offset,
+ long length)
+ throws IOException {
+ myMetrics.numGetBlockLocations.inc();
+ return namesystem.getBlockLocations(src, offset, length, false);
+ }
+
private static String getClientMachine() {
String clientMachine = Server.getRemoteAddress();
if (clientMachine == null) {
Modified:
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
URL:
http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java?rev=939841&r1=939840&r2=939841&view=diff
==============================================================================
---
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
(original)
+++
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
Fri Apr 30 22:06:04 2010
@@ -37,7 +37,6 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.BlockReader;
import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.protocol.Block;
-import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
@@ -87,7 +86,7 @@ public class NamenodeFsck {
/** Delete corrupted files. */
public static final int FIXING_DELETE = 2;
- private final ClientProtocol namenode;
+ private final NameNode namenode;
private final NetworkTopology networktopology;
private final int totalDatanodes;
private final short minReplication;
@@ -115,7 +114,7 @@ public class NamenodeFsck {
* @param response the object into which this servelet writes the url
contents
* @throws IOException
*/
- NamenodeFsck(Configuration conf, ClientProtocol namenode,
+ NamenodeFsck(Configuration conf, NameNode namenode,
NetworkTopology networktopology,
Map<String,String[]> pmap, PrintWriter out,
int totalDatanodes, short minReplication) {
@@ -260,7 +259,7 @@ public class NamenodeFsck {
return;
}
long fileLen = file.getLen();
- LocatedBlocks blocks = namenode.getBlockLocations(path, 0, fileLen);
+ LocatedBlocks blocks = namenode.getBlockLocationsNoATime(path, 0, fileLen);
if (blocks == null) { // the file is deleted
return;
}
Modified:
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
URL:
http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestFsck.java?rev=939841&r1=939840&r2=939841&view=diff
==============================================================================
---
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
(original)
+++
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
Fri Apr 30 22:06:04 2010
@@ -40,6 +40,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSClient;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
@@ -79,12 +80,19 @@ public class TestFsck extends TestCase {
FileSystem fs = null;
try {
Configuration conf = new HdfsConfiguration();
+ final long precision = 1L;
+ conf.setLong(DFSConfigKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_KEY,
precision);
conf.setLong("dfs.blockreport.intervalMsec", 10000L);
cluster = new MiniDFSCluster(conf, 4, true, null);
fs = cluster.getFileSystem();
- util.createFiles(fs, "/srcdat");
- util.waitReplication(fs, "/srcdat", (short)3);
+ final String fileName = "/srcdat";
+ util.createFiles(fs, fileName);
+ util.waitReplication(fs, fileName, (short)3);
+ final Path file = new Path(fileName);
+ long aTime = fs.getFileStatus(file).getAccessTime();
+ Thread.sleep(precision);
String outStr = runFsck(conf, 0, true, "/");
+ assertEquals(aTime, fs.getFileStatus(file).getAccessTime());
assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
System.out.println(outStr);
if (fs != null) {try{fs.close();} catch(Exception e){}}
