Author: eli
Date: Thu Mar 22 21:13:18 2012
New Revision: 1304067
URL: http://svn.apache.org/viewvc?rev=1304067&view=rev
Log:
HDFS-3044. svn merge -c 1304063 from trunk
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/
(props changed)
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
(props changed)
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
(props changed)
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
(props changed)
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
(props changed)
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
(props changed)
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
(props changed)
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
Propchange: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1304063
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1304067&r1=1304066&r2=1304067&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Thu Mar 22 21:13:18 2012
@@ -156,6 +156,9 @@ Release 0.23.3 - UNRELEASED
HDFS-309. FSEditLog should log progress during replay. (Sho Shimauchi
via todd)
+ HDFS-3044. fsck move should be non-destructive by default.
+ (Colin Patrick McCabe via eli)
+
OPTIMIZATIONS
HDFS-2477. Optimize computing the diff between a block report and the
Propchange:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1304063
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java?rev=1304067&r1=1304066&r2=1304067&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
Thu Mar 22 21:13:18 2012
@@ -85,13 +85,6 @@ public class NamenodeFsck {
public static final String NONEXISTENT_STATUS = "does not exist";
public static final String FAILURE_STATUS = "FAILED";
- /** Don't attempt any fixing . */
- public static final int FIXING_NONE = 0;
- /** Move corrupted files to /lost+found . */
- public static final int FIXING_MOVE = 1;
- /** Delete corrupted files. */
- public static final int FIXING_DELETE = 2;
-
private final NameNode namenode;
private final NetworkTopology networktopology;
private final int totalDatanodes;
@@ -107,7 +100,21 @@ public class NamenodeFsck {
private boolean showLocations = false;
private boolean showRacks = false;
private boolean showCorruptFileBlocks = false;
- private int fixing = FIXING_NONE;
+
+ /**
+ * True if the user specified the -move option.
+ *
+ * Whe this option is in effect, we will copy salvaged blocks into the lost
+ * and found. */
+ private boolean doMove = false;
+
+ /**
+ * True if the user specified the -delete option.
+ *
+ * Whe this option is in effect, we will delete corrupted files.
+ */
+ private boolean doDelete = false;
+
private String path = "/";
// We return back N files that are corrupt; the list of files returned is
@@ -144,8 +151,8 @@ public class NamenodeFsck {
for (Iterator<String> it = pmap.keySet().iterator(); it.hasNext();) {
String key = it.next();
if (key.equals("path")) { this.path = pmap.get("path")[0]; }
- else if (key.equals("move")) { this.fixing = FIXING_MOVE; }
- else if (key.equals("delete")) { this.fixing = FIXING_DELETE; }
+ else if (key.equals("move")) { this.doMove = true; }
+ else if (key.equals("delete")) { this.doDelete = true; }
else if (key.equals("files")) { this.showFiles = true; }
else if (key.equals("blocks")) { this.showBlocks = true; }
else if (key.equals("locations")) { this.showLocations = true; }
@@ -377,16 +384,20 @@ public class NamenodeFsck {
+ " blocks of total size " + missize + " B.");
}
res.corruptFiles++;
- switch(fixing) {
- case FIXING_NONE:
- break;
- case FIXING_MOVE:
- if (!isOpen)
- lostFoundMove(parent, file, blocks);
- break;
- case FIXING_DELETE:
- if (!isOpen)
- namenode.getRpcServer().delete(path, true);
+ try {
+ if (doMove) {
+ if (!isOpen) {
+ copyBlocksToLostFound(parent, file, blocks);
+ }
+ }
+ if (doDelete) {
+ if (!isOpen) {
+ LOG.warn("\n - deleting corrupted file " + path);
+ namenode.getRpcServer().delete(path, true);
+ }
+ }
+ } catch (IOException e) {
+ LOG.error("error processing " + path + ": " + e.toString());
}
}
if (showFiles) {
@@ -401,8 +412,8 @@ public class NamenodeFsck {
}
}
- private void lostFoundMove(String parent, HdfsFileStatus file, LocatedBlocks
blocks)
- throws IOException {
+ private void copyBlocksToLostFound(String parent, HdfsFileStatus file,
+ LocatedBlocks blocks) throws IOException {
final DFSClient dfs = new DFSClient(NameNode.getAddress(conf), conf);
try {
if (!lfInited) {
@@ -436,12 +447,10 @@ public class NamenodeFsck {
}
if (fos == null) {
fos = dfs.create(target + "/" + chain, true);
- if (fos != null) chain++;
+ if (fos != null)
+ chain++;
else {
- LOG.warn(errmsg + ": could not store chain " + chain);
- // perhaps we should bail out here...
- // return;
- continue;
+ throw new IOException(errmsg + ": could not store chain " + chain);
}
}
@@ -458,8 +467,7 @@ public class NamenodeFsck {
}
}
if (fos != null) fos.close();
- LOG.warn("\n - moved corrupted file " + fullName + " to /lost+found");
- dfs.delete(fullName, true);
+ LOG.warn("\n - copied corrupted file " + fullName + " to /lost+found");
} catch (Exception e) {
e.printStackTrace();
LOG.warn(errmsg + ": " + e.getMessage());
Propchange:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native:r1304063
Propchange:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:r1304063
Propchange:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:r1304063
Propchange:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:r1304063
Propchange:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:r1304063
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java?rev=1304067&r1=1304066&r2=1304067&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
Thu Mar 22 21:13:18 2012
@@ -227,7 +227,7 @@ public class TestFsck extends TestCase {
}
}
- public void testFsckMove() throws Exception {
+ public void testFsckMoveAndDelete() throws Exception {
DFSTestUtil util = new DFSTestUtil("TestFsck", 5, 3, 8*1024);
MiniDFSCluster cluster = null;
FileSystem fs = null;
@@ -248,8 +248,9 @@ public class TestFsck extends TestCase {
String[] fileNames = util.getFileNames(topDir);
DFSClient dfsClient = new DFSClient(new InetSocketAddress("localhost",
cluster.getNameNodePort()), conf);
+ String corruptFileName = fileNames[0];
ExtendedBlock block = dfsClient.getNamenode().getBlockLocations(
- fileNames[0], 0, Long.MAX_VALUE).get(0).getBlock();
+ corruptFileName, 0, Long.MAX_VALUE).get(0).getBlock();
for (int i=0; i<4; i++) {
File blockFile = MiniDFSCluster.getBlockFile(i, block);
if(blockFile != null && blockFile.exists()) {
@@ -267,8 +268,21 @@ public class TestFsck extends TestCase {
outStr = runFsck(conf, 1, false, "/");
}
+ // After a fsck -move, the corrupted file should still exist.
+ outStr = runFsck(conf, 1, true, "/", "-move" );
+ assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS));
+ String[] newFileNames = util.getFileNames(topDir);
+ boolean found = false;
+ for (String f : newFileNames) {
+ if (f.equals(corruptFileName)) {
+ found = true;
+ break;
+ }
+ }
+ assertTrue(found);
+
// Fix the filesystem by moving corrupted files to lost+found
- outStr = runFsck(conf, 1, true, "/", "-move");
+ outStr = runFsck(conf, 1, true, "/", "-move", "-delete");
assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS));
// Check to make sure we have healthy filesystem
