Author: cmccabe
Date: Thu Jul 10 04:07:14 2014
New Revision: 1609386
URL: http://svn.apache.org/r1609386
Log:
Rename and AddBlock may race and produce invalid edits (kihwal via cmccabe)
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/ (props changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/ (props
changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
(props changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java
Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project:r1609384
Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1609384
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1609386&r1=1609385&r2=1609386&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Thu Jul 10 04:07:14 2014
@@ -618,6 +618,9 @@ Release 2.5.0 - UNRELEASED
HDFS-6618. FSNamesystem#delete drops the FSN lock between removing INodes
from the tree and deleting them from the inode map (kihwal via cmccabe)
+ HDFS-6622. Rename and AddBlock may race and produce invalid edits (kihwal
+ via cmccabe)
+
Release 2.4.1 - 2014-06-23
INCOMPATIBLE CHANGES
Propchange:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1609384
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1609386&r1=1609385&r2=1609386&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
Thu Jul 10 04:07:14 2014
@@ -2764,9 +2764,10 @@ public class FSNamesystem implements Nam
checkOperation(OperationCategory.READ);
src = FSDirectory.resolvePath(src, pathComponents, dir);
LocatedBlock[] onRetryBlock = new LocatedBlock[1];
- final INodeFile pendingFile = analyzeFileState(
+ FileState fileState = analyzeFileState(
src, fileId, clientName, previous, onRetryBlock);
- src = pendingFile.getFullPathName();
+ final INodeFile pendingFile = fileState.inode;
+ src = fileState.path;
if (onRetryBlock[0] != null && onRetryBlock[0].getLocations().length >
0) {
// This is a retry. Just return the last block if having locations.
@@ -2802,8 +2803,10 @@ public class FSNamesystem implements Nam
// Run the full analysis again, since things could have changed
// while chooseTarget() was executing.
LocatedBlock[] onRetryBlock = new LocatedBlock[1];
- final INodeFile pendingFile =
+ FileState fileState =
analyzeFileState(src, fileId, clientName, previous, onRetryBlock);
+ final INodeFile pendingFile = fileState.inode;
+ src = fileState.path;
if (onRetryBlock[0] != null) {
if (onRetryBlock[0].getLocations().length > 0) {
@@ -2839,7 +2842,17 @@ public class FSNamesystem implements Nam
return makeLocatedBlock(newBlock, targets, offset);
}
- INodeFile analyzeFileState(String src,
+ static class FileState {
+ public final INodeFile inode;
+ public final String path;
+
+ public FileState(INodeFile inode, String fullPath) {
+ this.inode = inode;
+ this.path = fullPath;
+ }
+ }
+
+ FileState analyzeFileState(String src,
long fileId,
String clientName,
ExtendedBlock previous,
@@ -2927,7 +2940,7 @@ public class FSNamesystem implements Nam
onRetryBlock[0] = makeLocatedBlock(lastBlockInFile,
((BlockInfoUnderConstruction)lastBlockInFile).getExpectedStorageLocations(),
offset);
- return pendingFile;
+ return new FileState(pendingFile, src);
} else {
// Case 3
throw new IOException("Cannot allocate block in " + src + ": " +
@@ -2940,7 +2953,7 @@ public class FSNamesystem implements Nam
if (!checkFileProgress(pendingFile, false)) {
throw new NotReplicatedYetException("Not replicated yet: " + src);
}
- return pendingFile;
+ return new FileState(pendingFile, src);
}
LocatedBlock makeLocatedBlock(Block blk, DatanodeStorageInfo[] locs,
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java?rev=1609386&r1=1609385&r2=1609386&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java
Thu Jul 10 04:07:14 2014
@@ -146,4 +146,62 @@ public class TestDeleteRace {
}
}
}
+
+ private class RenameThread extends Thread {
+ private FileSystem fs;
+ private Path from;
+ private Path to;
+
+ RenameThread(FileSystem fs, Path from, Path to) {
+ this.fs = fs;
+ this.from = from;
+ this.to = to;
+ }
+
+ @Override
+ public void run() {
+ try {
+ Thread.sleep(1000);
+ LOG.info("Renaming " + from + " to " + to);
+
+ fs.rename(from, to);
+ LOG.info("Renamed " + from + " to " + to);
+ } catch (Exception e) {
+ LOG.info(e);
+ }
+ }
+ }
+
+ @Test
+ public void testRenameRace() throws Exception {
+ try {
+ conf.setClass(DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_KEY,
+ SlowBlockPlacementPolicy.class, BlockPlacementPolicy.class);
+ cluster = new MiniDFSCluster.Builder(conf).build();
+ FileSystem fs = cluster.getFileSystem();
+ Path dirPath1 = new Path("/testRenameRace1");
+ Path dirPath2 = new Path("/testRenameRace2");
+ Path filePath = new Path("/testRenameRace1/file1");
+
+
+ fs.mkdirs(dirPath1);
+ FSDataOutputStream out = fs.create(filePath);
+ Thread renameThread = new RenameThread(fs, dirPath1, dirPath2);
+ renameThread.start();
+
+ // write data and close to make sure a block is allocated.
+ out.write(new byte[32], 0, 32);
+ out.close();
+
+ // Restart name node so that it replays edit. If old path was
+ // logged in edit, it will fail to come up.
+ cluster.restartNameNode(0);
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+
+ }
}
