Author: szetszwo
Date: Thu Oct 25 19:09:34 2012
New Revision: 1402278
URL: http://svn.apache.org/viewvc?rev=1402278&view=rev
Log:
Merge r1401869 through r1402273 from trunk.
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/ (props
changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
(props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JNStorage.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileUnderConstruction.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
(props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
(props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
(props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
(props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
(props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestUnderReplicatedBlocks.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockUnderConstruction.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
Propchange: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1401869-1402273
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1402278&r1=1402277&r2=1402278&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Thu Oct 25 19:09:34 2012
@@ -150,6 +150,8 @@ Trunk (Unreleased)
HDFS-4052. BlockManager#invalidateWork should print log outside the lock.
(Jing Zhao via suresh)
+ HDFS-4110. Refine a log printed in JNStorage. (Liang Xie via suresh)
+
OPTIMIZATIONS
BUG FIXES
@@ -233,6 +235,9 @@ Trunk (Unreleased)
HDFS-2434. TestNameNodeMetrics.testCorruptBlock fails intermittently.
(Jing Zhao via suresh)
+ HDFS-4067. TestUnderReplicatedBlocks intermittently fails due to
+ ReplicaAlreadyExistsException. (Jing Zhao via suresh)
+
BREAKDOWN OF HDFS-3077 SUBTASKS
HDFS-3077. Quorum-based protocol for reading and writing edit logs.
@@ -413,6 +418,9 @@ Release 2.0.3-alpha - Unreleased
HDFS-4099. Clean up replication code and add more javadoc. (szetszwo)
+ HDFS-4107. Add utility methods for casting INode to INodeFile and
+ INodeFileUnderConstruction. (szetszwo)
+
OPTIMIZATIONS
BUG FIXES
@@ -488,6 +496,10 @@ Release 2.0.3-alpha - Unreleased
HDFS-4022. Replication not happening for appended block.
(Vinay via umamahesh)
+ HDFS-3948. Do not use hflush in TestWebHDFS.testNamenodeRestart() since the
+ out stream returned by WebHdfsFileSystem does not support it. (Jing Zhao
+ via szetszwo)
+
Release 2.0.2-alpha - 2012-09-07
INCOMPATIBLE CHANGES
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1401869-1402273
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JNStorage.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JNStorage.java?rev=1402278&r1=1402277&r2=1402278&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JNStorage.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JNStorage.java
Thu Oct 25 19:09:34 2012
@@ -171,8 +171,7 @@ class JNStorage extends Storage {
void format(NamespaceInfo nsInfo) throws IOException {
setStorageInfo(nsInfo);
- LOG.info("Formatting journal storage directory " +
- sd + " with nsid: " + getNamespaceID());
+ LOG.info("Formatting journal " + sd + " with nsid: " + getNamespaceID());
// Unlock the directory before formatting, because we will
// re-analyze it after format(). The analyzeStorage() call
// below is reponsible for re-locking it. This is a no-op
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java?rev=1402278&r1=1402277&r2=1402278&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
Thu Oct 25 19:09:34 2012
@@ -1012,7 +1012,7 @@ public class FSDirectory implements Clos
int i = 0;
int totalBlocks = 0;
for(String src : srcs) {
- INodeFile srcInode = getFileINode(src);
+ INodeFile srcInode = (INodeFile)getINode(src);
allSrcInodes[i++] = srcInode;
totalBlocks += srcInode.blocks.length;
}
@@ -1301,24 +1301,12 @@ public class FSDirectory implements Clos
}
/**
- * Get {@link INode} associated with the file.
- */
- INodeFile getFileINode(String src) throws UnresolvedLinkException {
- INode inode = getINode(src);
- if (inode == null || inode.isDirectory())
- return null;
- assert !inode.isLink();
- return (INodeFile) inode;
- }
-
- /**
* Get {@link INode} associated with the file / directory.
*/
public INode getINode(String src) throws UnresolvedLinkException {
readLock();
try {
- INode iNode = rootDir.getNode(src, true);
- return iNode;
+ return rootDir.getNode(src, true);
} finally {
readUnlock();
}
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java?rev=1402278&r1=1402277&r2=1402278&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
Thu Oct 25 19:09:34 2012
@@ -477,8 +477,8 @@ public class FSEditLogLoader {
Lease lease = fsNamesys.leaseManager.getLease(
reassignLeaseOp.leaseHolder);
INodeFileUnderConstruction pendingFile =
- (INodeFileUnderConstruction) fsDir.getFileINode(
- reassignLeaseOp.path);
+ INodeFileUnderConstruction.valueOf(
+ fsDir.getINode(reassignLeaseOp.path), reassignLeaseOp.path);
fsNamesys.reassignLeaseInternal(lease,
reassignLeaseOp.path, reassignLeaseOp.newHolder, pendingFile);
break;
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java?rev=1402278&r1=1402277&r2=1402278&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
Thu Oct 25 19:09:34 2012
@@ -365,14 +365,7 @@ class FSImageFormat {
// verify that file exists in namespace
String path = cons.getLocalName();
- INode old = fsDir.getFileINode(path);
- if (old == null) {
- throw new IOException("Found lease for non-existent file " + path);
- }
- if (old.isDirectory()) {
- throw new IOException("Found lease for directory " + path);
- }
- INodeFile oldnode = (INodeFile) old;
+ INodeFile oldnode = INodeFile.valueOf(fsDir.getINode(path), path);
fsDir.replaceNode(path, oldnode, cons);
namesystem.leaseManager.addLease(cons.getClientName(), path);
}
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1402278&r1=1402277&r2=1402278&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
Thu Oct 25 19:09:34 2012
@@ -1270,11 +1270,7 @@ public class FSNamesystem implements Nam
}
long now = now();
- INodeFile inode = dir.getFileINode(src);
- if (inode == null) {
- throw new FileNotFoundException("File does not exist: " + src);
- }
- assert !inode.isLink();
+ final INodeFile inode = INodeFile.valueOf(dir.getINode(src), src);
if (doAccessTime && isAccessTimeSupported()) {
if (now <= inode.getAccessTime() + getAccessTimePrecision()) {
// if we have to set access time but we only have the readlock,
then
@@ -1390,28 +1386,27 @@ public class FSNamesystem implements Nam
// we put the following prerequisite for the operation
// replication and blocks sizes should be the same for ALL the blocks
- // check the target
- INode inode = dir.getFileINode(target);
- if(inode == null) {
- throw new IllegalArgumentException("concat: trg file doesn't exist");
- }
- if(inode.isUnderConstruction()) {
- throw new IllegalArgumentException("concat: trg file is uner
construction");
+ // check the target
+ final INodeFile trgInode = INodeFile.valueOf(dir.getINode(target), target);
+ if(trgInode.isUnderConstruction()) {
+ throw new HadoopIllegalArgumentException("concat: target file "
+ + target + " is under construction");
}
-
- INodeFile trgInode = (INodeFile) inode;
-
- // per design trg shouldn't be empty and all the blocks same size
+ // per design target shouldn't be empty and all the blocks same size
if(trgInode.blocks.length == 0) {
- throw new IllegalArgumentException("concat: "+ target + " file is
empty");
+ throw new HadoopIllegalArgumentException("concat: target file "
+ + target + " is empty");
}
long blockSize = trgInode.getPreferredBlockSize();
// check the end block to be full
if(blockSize != trgInode.blocks[trgInode.blocks.length-1].getNumBytes()) {
- throw new IllegalArgumentException(target + " blocks size should be the
same");
+ throw new HadoopIllegalArgumentException("The last block in " + target
+ + " is not full; last block size = "
+ + trgInode.blocks[trgInode.blocks.length-1].getNumBytes()
+ + " but file block size = " + blockSize);
}
si.add(trgInode);
@@ -1424,21 +1419,21 @@ public class FSNamesystem implements Nam
if(i==srcs.length-1)
endSrc=true;
- INodeFile srcInode = dir.getFileINode(src);
-
+ final INodeFile srcInode = INodeFile.valueOf(dir.getINode(src), src);
if(src.isEmpty()
- || srcInode == null
|| srcInode.isUnderConstruction()
|| srcInode.blocks.length == 0) {
- throw new IllegalArgumentException("concat: file " + src +
- " is invalid or empty or underConstruction");
+ throw new HadoopIllegalArgumentException("concat: source file " + src
+ + " is invalid or empty or underConstruction");
}
// check replication and blocks size
if(repl != srcInode.getFileReplication()) {
- throw new IllegalArgumentException(src + " and " + target + " " +
- "should have same replication: "
- + repl + " vs. " + srcInode.getFileReplication());
+ throw new HadoopIllegalArgumentException("concat: the soruce file "
+ + src + " and the target file " + target
+ + " should have the same replication: source replication is "
+ + srcInode.getBlockReplication()
+ + " but target replication is " + repl);
}
//boolean endBlock=false;
@@ -1448,8 +1443,11 @@ public class FSNamesystem implements Nam
if(endSrc)
idx = srcInode.blocks.length-2; // end block of endSrc is OK not to be
full
if(idx >= 0 && srcInode.blocks[idx].getNumBytes() != blockSize) {
- throw new IllegalArgumentException("concat: blocks sizes of " +
- src + " and " + target + " should all be the same");
+ throw new HadoopIllegalArgumentException("concat: the soruce file "
+ + src + " and the target file " + target
+ + " should have the same blocks sizes: target block size is "
+ + blockSize + " but the size of source block " + idx + " is "
+ + srcInode.blocks[idx].getNumBytes());
}
si.add(srcInode);
@@ -1458,7 +1456,8 @@ public class FSNamesystem implements Nam
// make sure no two files are the same
if(si.size() < srcs.length+1) { // trg + srcs
// it means at least two files are the same
- throw new IllegalArgumentException("at least two files are the same");
+ throw new HadoopIllegalArgumentException(
+ "concat: at least two of the source files are the same");
}
if(NameNode.stateChangeLog.isDebugEnabled()) {
@@ -1797,13 +1796,9 @@ public class FSNamesystem implements Nam
}
try {
- INodeFile myFile = dir.getFileINode(src);
- try {
- blockManager.verifyReplication(src, replication, clientMachine);
- } catch(IOException e) {
- throw new IOException("failed to create "+e.getMessage());
- }
+ blockManager.verifyReplication(src, replication, clientMachine);
boolean create = flag.contains(CreateFlag.CREATE);
+ final INode myFile = dir.getINode(src);
if (myFile == null) {
if (!create) {
throw new FileNotFoundException("failed to overwrite or append to
non-existent file "
@@ -1829,8 +1824,9 @@ public class FSNamesystem implements Nam
blockManager.getDatanodeManager().getDatanodeByHost(clientMachine);
if (append && myFile != null) {
+ final INodeFile f = INodeFile.valueOf(myFile, src);
return prepareFileForWrite(
- src, myFile, holder, clientMachine, clientNode, true);
+ src, f, holder, clientMachine, clientNode, true);
} else {
// Now we can add the name to the filesystem. This file has no
// blocks associated with it.
@@ -1925,11 +1921,7 @@ public class FSNamesystem implements Nam
throw new IOException("Invalid file name: " + src);
}
- INode inode = dir.getFileINode(src);
- if (inode == null) {
- throw new FileNotFoundException("File not found " + src);
- }
-
+ final INodeFile inode = INodeFile.valueOf(dir.getINode(src), src);
if (!inode.isUnderConstruction()) {
return true;
}
@@ -2330,35 +2322,32 @@ public class FSNamesystem implements Nam
private INodeFileUnderConstruction checkLease(String src, String holder)
throws LeaseExpiredException, UnresolvedLinkException {
assert hasReadOrWriteLock();
- INodeFile file = dir.getFileINode(src);
- checkLease(src, holder, file);
- return (INodeFileUnderConstruction)file;
+ return checkLease(src, holder, dir.getINode(src));
}
- private void checkLease(String src, String holder, INode file)
- throws LeaseExpiredException {
+ private INodeFileUnderConstruction checkLease(String src, String holder,
+ INode file) throws LeaseExpiredException {
assert hasReadOrWriteLock();
- if (file == null || file.isDirectory()) {
+ if (file == null || !(file instanceof INodeFile)) {
Lease lease = leaseManager.getLease(holder);
- throw new LeaseExpiredException("No lease on " + src +
- " File does not exist. " +
- (lease != null ? lease.toString() :
- "Holder " + holder +
- " does not have any open files."));
+ throw new LeaseExpiredException(
+ "No lease on " + src + ": File does not exist. "
+ + (lease != null ? lease.toString()
+ : "Holder " + holder + " does not have any open files."));
}
if (!file.isUnderConstruction()) {
Lease lease = leaseManager.getLease(holder);
- throw new LeaseExpiredException("No lease on " + src +
- " File is not open for writing. " +
- (lease != null ? lease.toString() :
- "Holder " + holder +
- " does not have any open files."));
+ throw new LeaseExpiredException(
+ "No lease on " + src + ": File is not open for writing. "
+ + (lease != null ? lease.toString()
+ : "Holder " + holder + " does not have any open files."));
}
INodeFileUnderConstruction pendingFile = (INodeFileUnderConstruction)file;
if (holder != null && !pendingFile.getClientName().equals(holder)) {
throw new LeaseExpiredException("Lease mismatch on " + src + " owned by "
+ pendingFile.getClientName() + " but is accessed by " + holder);
}
+ return pendingFile;
}
/**
@@ -2400,15 +2389,15 @@ public class FSNamesystem implements Nam
try {
pendingFile = checkLease(src, holder);
} catch (LeaseExpiredException lee) {
- INodeFile file = dir.getFileINode(src);
- if (file != null && !file.isUnderConstruction()) {
+ final INode inode = dir.getINode(src);
+ if (inode != null && inode instanceof INodeFile &&
!inode.isUnderConstruction()) {
// This could be a retry RPC - i.e the client tried to close
// the file, but missed the RPC response. Thus, it is trying
// again to close the file. If the file still exists and
// the client's view of the last block matches the actual
// last block, then we'll treat it as a successful close.
// See HDFS-3031.
- Block realLastBlock = file.getLastBlock();
+ final Block realLastBlock = ((INodeFile)inode).getLastBlock();
if (Block.matchingIdAndGenStamp(last, realLastBlock)) {
NameNode.stateChangeLog.info("DIR* NameSystem.completeFile: " +
"received request from " + holder + " to complete file " + src +
@@ -2994,23 +2983,9 @@ public class FSNamesystem implements Nam
LOG.info("Recovering lease=" + lease + ", src=" + src);
assert !isInSafeMode();
assert hasWriteLock();
- INodeFile iFile = dir.getFileINode(src);
- if (iFile == null) {
- final String message = "DIR* NameSystem.internalReleaseLease: "
- + "attempt to release a create lock on "
- + src + " file does not exist.";
- NameNode.stateChangeLog.warn(message);
- throw new IOException(message);
- }
- if (!iFile.isUnderConstruction()) {
- final String message = "DIR* NameSystem.internalReleaseLease: "
- + "attempt to release a create lock on "
- + src + " but file is already closed.";
- NameNode.stateChangeLog.warn(message);
- throw new IOException(message);
- }
- INodeFileUnderConstruction pendingFile = (INodeFileUnderConstruction)
iFile;
+ final INodeFileUnderConstruction pendingFile
+ = INodeFileUnderConstruction.valueOf(dir.getINode(src), src);
int nrBlocks = pendingFile.numBlocks();
BlockInfo[] blocks = pendingFile.getBlocks();
@@ -4318,17 +4293,14 @@ public class FSNamesystem implements Nam
try {
for (Lease lease : leaseManager.getSortedLeases()) {
for (String path : lease.getPaths()) {
- INode node;
+ final INodeFileUnderConstruction cons;
try {
- node = dir.getFileINode(path);
+ cons = INodeFileUnderConstruction.valueOf(dir.getINode(path),
path);
} catch (UnresolvedLinkException e) {
throw new AssertionError("Lease files should reside on this FS");
+ } catch (IOException e) {
+ throw new RuntimeException(e);
}
- assert node != null : "Found a lease for nonexisting file.";
- assert node.isUnderConstruction() :
- "Found a lease for file " + path + " that is not under
construction." +
- " lease=" + lease;
- INodeFileUnderConstruction cons = (INodeFileUnderConstruction) node;
BlockInfo[] blocks = cons.getBlocks();
if(blocks == null)
continue;
@@ -4911,21 +4883,12 @@ public class FSNamesystem implements Nam
for (Lease lease : leaseManager.getSortedLeases()) {
for(String path : lease.getPaths()) {
// verify that path exists in namespace
- INode node;
+ final INodeFileUnderConstruction cons;
try {
- node = dir.getFileINode(path);
+ cons = INodeFileUnderConstruction.valueOf(dir.getINode(path),
path);
} catch (UnresolvedLinkException e) {
throw new AssertionError("Lease files should reside on this FS");
}
- if (node == null) {
- throw new IOException("saveLeases found path " + path +
- " but no matching entry in namespace.");
- }
- if (!node.isUnderConstruction()) {
- throw new IOException("saveLeases found path " + path +
- " but is not under construction.");
- }
- INodeFileUnderConstruction cons = (INodeFileUnderConstruction) node;
FSImageSerialization.writeINodeUnderConstruction(out, cons, path);
}
}
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java?rev=1402278&r1=1402277&r2=1402278&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
Thu Oct 25 19:09:34 2012
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.List;
@@ -32,6 +33,17 @@ import org.apache.hadoop.hdfs.server.blo
/** I-node for closed file. */
@InterfaceAudience.Private
public class INodeFile extends INode implements BlockCollection {
+ /** Cast INode to INodeFile. */
+ public static INodeFile valueOf(INode inode, String path) throws IOException
{
+ if (inode == null) {
+ throw new FileNotFoundException("File does not exist: " + path);
+ }
+ if (!(inode instanceof INodeFile)) {
+ throw new FileNotFoundException("Path is not a file: " + path);
+ }
+ return (INodeFile)inode;
+ }
+
static final FsPermission UMASK = FsPermission.createImmutable((short)0111);
//Number of bits for Block size
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileUnderConstruction.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileUnderConstruction.java?rev=1402278&r1=1402277&r2=1402278&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileUnderConstruction.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileUnderConstruction.java
Thu Oct 25 19:09:34 2012
@@ -25,8 +25,8 @@ import org.apache.hadoop.hdfs.protocol.B
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import
org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
-import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
import org.apache.hadoop.hdfs.server.blockmanagement.MutableBlockCollection;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
import com.google.common.base.Joiner;
@@ -35,6 +35,16 @@ import com.google.common.base.Joiner;
*/
@InterfaceAudience.Private
class INodeFileUnderConstruction extends INodeFile implements
MutableBlockCollection {
+ /** Cast INode to INodeFileUnderConstruction. */
+ public static INodeFileUnderConstruction valueOf(INode inode, String path
+ ) throws IOException {
+ final INodeFile file = INodeFile.valueOf(inode, path);
+ if (!file.isUnderConstruction()) {
+ throw new IOException("File is not under construction: " + path);
+ }
+ return (INodeFileUnderConstruction)file;
+ }
+
private String clientName; // lease holder
private final String clientMachine;
private final DatanodeDescriptor clientNode; // if client is a cluster node
too.
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java?rev=1402278&r1=1402277&r2=1402278&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
Thu Oct 25 19:09:34 2012
@@ -253,7 +253,7 @@ public class LeaseManager {
private String findPath(INodeFileUnderConstruction pendingFile) {
try {
for (String src : paths) {
- if (fsnamesystem.dir.getFileINode(src) == pendingFile) {
+ if (fsnamesystem.dir.getINode(src) == pendingFile) {
return src;
}
}
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native:r1401869-1402273
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:r1401869-1402273
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:r1401869-1402273
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:r1401869-1402273
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:r1401869-1402273
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java?rev=1402278&r1=1402277&r2=1402278&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
Thu Oct 25 19:09:34 2012
@@ -876,7 +876,14 @@ public class TestDFSClientRetries {
new Random().nextBytes(bytes);
out4.write(bytes);
out4.write(bytes);
- out4.hflush();
+ if (isWebHDFS) {
+ // WebHDFS does not support hflush. To avoid DataNode communicating
with
+ // NN while we're shutting down NN, we call out4.close() to finish
+ // writing the data
+ out4.close();
+ } else {
+ out4.hflush();
+ }
//shutdown namenode
assertTrue(HdfsUtils.isHealthy(uri));
@@ -889,10 +896,12 @@ public class TestDFSClientRetries {
public void run() {
try {
//write some more data and then close the file
- out4.write(bytes);
- out4.write(bytes);
- out4.write(bytes);
- out4.close();
+ if (!isWebHDFS) {
+ out4.write(bytes);
+ out4.write(bytes);
+ out4.write(bytes);
+ out4.close();
+ }
} catch (Exception e) {
exceptions.add(e);
}
@@ -975,7 +984,11 @@ public class TestDFSClientRetries {
Assert.assertEquals(String.format("count=%d", count),
bytes[count % bytes.length], (byte)r);
}
- Assert.assertEquals(5 * bytes.length, count);
+ if (!isWebHDFS) {
+ Assert.assertEquals(5 * bytes.length, count);
+ } else {
+ Assert.assertEquals(2 * bytes.length, count);
+ }
in.close();
}
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java?rev=1402278&r1=1402277&r2=1402278&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
Thu Oct 25 19:09:34 2012
@@ -119,8 +119,6 @@ public class TestDistributedFileSystem {
DFSTestUtil.createFile(fileSys, p, 1L, (short)1, 0L);
DFSTestUtil.readFile(fileSys, p);
- DFSClient client = ((DistributedFileSystem)fileSys).dfs;
-
fileSys.close();
} finally {
@@ -476,7 +474,7 @@ public class TestDistributedFileSystem {
fail("Expecting FileNotFoundException");
} catch (FileNotFoundException e) {
assertTrue("Not throwing the intended exception message", e.getMessage()
- .contains("File does not exist: /test/TestExistingDir"));
+ .contains("Path is not a file: /test/TestExistingDir"));
}
//hftp
@@ -712,7 +710,6 @@ public class TestDistributedFileSystem {
@Test
public void testCreateWithCustomChecksum() throws Exception {
Configuration conf = getTestConfiguration();
- final long grace = 1000L;
MiniDFSCluster cluster = null;
Path testBasePath = new Path("/test/csum");
// create args
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestUnderReplicatedBlocks.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestUnderReplicatedBlocks.java?rev=1402278&r1=1402277&r2=1402278&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestUnderReplicatedBlocks.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestUnderReplicatedBlocks.java
Thu Oct 25 19:09:34 2012
@@ -27,6 +27,7 @@ import org.apache.hadoop.hdfs.DFSTestUti
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
import org.junit.Test;
public class TestUnderReplicatedBlocks {
@@ -49,6 +50,12 @@ public class TestUnderReplicatedBlocks {
ExtendedBlock b = DFSTestUtil.getFirstBlock(fs, FILE_PATH);
DatanodeDescriptor dn =
bm.blocksMap.nodeIterator(b.getLocalBlock()).next();
bm.addToInvalidates(b.getLocalBlock(), dn);
+ // Compute the invalidate work in NN, and trigger the heartbeat from DN
+ BlockManagerTestUtil.computeAllPendingWork(bm);
+ DataNodeTestUtils.triggerHeartbeat(cluster.getDataNode(dn.getIpcPort()));
+ // Wait to make sure the DataNode receives the deletion request
+ Thread.sleep(1000);
+ // Remove the record from blocksMap
bm.blocksMap.removeNode(b.getLocalBlock(), dn);
// increment this file's replication factor
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockUnderConstruction.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockUnderConstruction.java?rev=1402278&r1=1402277&r2=1402278&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockUnderConstruction.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockUnderConstruction.java
Thu Oct 25 19:09:34 2012
@@ -83,8 +83,7 @@ public class TestBlockUnderConstruction
private void verifyFileBlocks(String file,
boolean isFileOpen) throws IOException {
FSNamesystem ns = cluster.getNamesystem();
- INodeFile inode = ns.dir.getFileINode(file);
- assertTrue("File does not exist: " + inode.toString(), inode != null);
+ final INodeFile inode = INodeFile.valueOf(ns.dir.getINode(file), file);
assertTrue("File " + inode.toString() +
" isUnderConstruction = " + inode.isUnderConstruction() +
" expected to be " + isFileOpen,
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java?rev=1402278&r1=1402277&r2=1402278&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
Thu Oct 25 19:09:34 2012
@@ -18,13 +18,17 @@
package org.apache.hadoop.hdfs.server.namenode;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
-
import org.junit.Test;
public class TestINodeFile {
@@ -199,4 +203,88 @@ public class TestINodeFile {
return iNodes;
}
+
+ /**
+ * Test for the static {@link INodeFile#valueOf(INode, String)}
+ * and {@link INodeFileUnderConstruction#valueOf(INode, String)} methods.
+ * @throws IOException
+ */
+ @Test
+ public void testValueOf () throws IOException {
+ final String path = "/testValueOf";
+ final PermissionStatus perm = new PermissionStatus(
+ userName, null, FsPermission.getDefault());
+ final short replication = 3;
+
+ {//cast from null
+ final INode from = null;
+
+ //cast to INodeFile, should fail
+ try {
+ INodeFile.valueOf(from, path);
+ fail();
+ } catch(FileNotFoundException fnfe) {
+ assertTrue(fnfe.getMessage().contains("File does not exist"));
+ }
+
+ //cast to INodeFileUnderConstruction, should fail
+ try {
+ INodeFileUnderConstruction.valueOf(from, path);
+ fail();
+ } catch(FileNotFoundException fnfe) {
+ assertTrue(fnfe.getMessage().contains("File does not exist"));
+ }
+ }
+
+ {//cast from INodeFile
+ final INode from = new INodeFile(
+ perm, null, replication, 0L, 0L, preferredBlockSize);
+
+ //cast to INodeFile, should success
+ final INodeFile f = INodeFile.valueOf(from, path);
+ assertTrue(f == from);
+
+ //cast to INodeFileUnderConstruction, should fail
+ try {
+ INodeFileUnderConstruction.valueOf(from, path);
+ fail();
+ } catch(IOException ioe) {
+ assertTrue(ioe.getMessage().contains("File is not under
construction"));
+ }
+ }
+
+ {//cast from INodeFileUnderConstruction
+ final INode from = new INodeFileUnderConstruction(
+ perm, replication, 0L, 0L, "client", "machine", null);
+
+ //cast to INodeFile, should success
+ final INodeFile f = INodeFile.valueOf(from, path);
+ assertTrue(f == from);
+
+ //cast to INodeFileUnderConstruction, should success
+ final INodeFileUnderConstruction u = INodeFileUnderConstruction.valueOf(
+ from, path);
+ assertTrue(u == from);
+ }
+
+ {//cast from INodeDirectory
+ final INode from = new INodeDirectory(perm, 0L);
+
+ //cast to INodeFile, should fail
+ try {
+ INodeFile.valueOf(from, path);
+ fail();
+ } catch(FileNotFoundException fnfe) {
+ assertTrue(fnfe.getMessage().contains("Path is not a file"));
+ }
+
+ //cast to INodeFileUnderConstruction, should fail
+ try {
+ INodeFileUnderConstruction.valueOf(from, path);
+ fail();
+ } catch(FileNotFoundException fnfe) {
+ assertTrue(fnfe.getMessage().contains("Path is not a file"));
+ }
+ }
+ }
}
