Author: kihwal
Date: Fri Aug 9 19:20:05 2013
New Revision: 1512456
URL: http://svn.apache.org/r1512456
Log:
HDFS-4993. fsck can fail if a file is renamed or deleted. Contributed by Robert
Parker.
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1512456&r1=1512455&r2=1512456&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Fri Aug 9 19:20:05 2013
@@ -34,6 +34,9 @@ Release 0.23.10 - UNRELEASED
HDFS-3020. Fix editlog to automatically sync when buffer is full. (todd)
+ HDFS-4993. Fsck can fail if a file is renamed or deleted. (Robert Parker
+ via kihwal)
+
Release 0.23.9 - 2013-07-08
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java?rev=1512456&r1=1512455&r2=1512456&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
Fri Aug 9 19:20:05 2013
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintWriter;
@@ -123,7 +124,7 @@ public class NamenodeFsck {
/**
* Filesystem checker.
* @param conf configuration (namenode config)
- * @param nn namenode that this fsck is going to use
+ * @param namenode namenode that this fsck is going to use
* @param pmap key=value[] map passed to the http servlet as url parameters
* @param out output stream to write the fsck output
* @param totalDatanodes number of live datanodes
@@ -276,8 +277,13 @@ public class NamenodeFsck {
long fileLen = file.getLen();
// Get block locations without updating the file access time
// and without block access tokens
- LocatedBlocks blocks = namenode.getNamesystem().getBlockLocations(path, 0,
- fileLen, false, false);
+ LocatedBlocks blocks;
+ try {
+ blocks = namenode.getNamesystem().getBlockLocations(path, 0,
+ fileLen, false, false);
+ } catch (FileNotFoundException fnfe) {
+ blocks = null;
+ }
if (blocks == null) { // the file is deleted
return;
}
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java?rev=1512456&r1=1512455&r2=1512456&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
Fri Aug 9 19:20:05 2013
@@ -23,6 +23,7 @@ import static org.junit.Assert.*;
import java.io.BufferedReader;
import java.io.ByteArrayOutputStream;
import java.io.File;
+import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.PrintStream;
@@ -49,6 +50,7 @@ import org.apache.hadoop.fs.permission.F
import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
@@ -69,6 +71,8 @@ import org.apache.log4j.PatternLayout;
import org.apache.log4j.RollingFileAppender;
import org.junit.Test;
+import static org.mockito.Mockito.*;
+
/**
* A JUnit test for doing fsck
*/
@@ -635,6 +639,56 @@ public class TestFsck {
}
}
+ /** Test fsck with FileNotFound */
+ @Test
+ public void testFsckFileNotFound() throws Exception {
+
+ // Number of replicas to actually start
+ final short NUM_REPLICAS = 1;
+
+ Configuration conf = new Configuration();
+ NameNode namenode = mock(NameNode.class);
+ NetworkTopology nettop = mock(NetworkTopology.class);
+ Map<String,String[]> pmap = new HashMap<String, String[]>();
+ Writer result = new StringWriter();
+ PrintWriter out = new PrintWriter(result, true);
+ InetAddress remoteAddress = InetAddress.getLocalHost();
+ FSNamesystem fsName = mock(FSNamesystem.class);
+ when(namenode.getNamesystem()).thenReturn(fsName);
+ when(fsName.getBlockLocations(anyString(), anyLong(), anyLong(),
+ anyBoolean(), anyBoolean())).
+ thenThrow(new FileNotFoundException()) ;
+
+ NamenodeFsck fsck = new NamenodeFsck(conf, namenode, nettop, pmap, out,
+ NUM_REPLICAS, (short)1, remoteAddress);
+
+ String pathString = "/tmp/testFile";
+
+ long length = 123L;
+ boolean isDir = false;
+ int blockReplication = 1;
+ long blockSize = 128 *1024L;
+ long modTime = 123123123L;
+ long accessTime = 123123120L;
+ FsPermission perms = FsPermission.getDefault();
+ String owner = "foo";
+ String group = "bar";
+ byte [] symlink = null;
+ byte [] path = new byte[128];
+ path = DFSUtil.string2Bytes(pathString);
+
+ HdfsFileStatus file = new HdfsFileStatus(length, isDir, blockReplication,
+ blockSize, modTime, accessTime, perms, owner, group, symlink, path);
+ Result res = new Result(conf);
+
+ try {
+ fsck.check(pathString, file, res);
+ } catch (Exception e) {
+ fail("Unexpected exception "+ e.getMessage());
+ }
+ assertTrue(res.toString().contains("HEALTHY"));
+ }
+
/** Test fsck with symlinks in the filesystem */
@Test
public void testFsckSymlink() throws Exception {
