Author: szetszwo
Date: Tue Nov 6 21:11:42 2012
New Revision: 1406337
URL: http://svn.apache.org/viewvc?rev=1406337&view=rev
Log:
Merge r1406007 through r1406326 from trunk.
Added:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/TestHdfsProtoUtil.java
- copied unchanged from r1406326,
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/TestHdfsProtoUtil.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestStartSecureDataNode.java
- copied unchanged from r1406326,
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestStartSecureDataNode.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecureNameNodeWithExternalKdc.java
- copied unchanged from r1406326,
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecureNameNodeWithExternalKdc.java
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/ (props
changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
(props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsProtoUtil.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/DataTransferProtoUtil.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/SecureDataNodeStarter.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
(props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/datatransfer.proto
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
(props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
(props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
(props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
(props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
Propchange: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1405252-1406005,1406007-1406326
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1406337&r1=1406336&r2=1406337&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Tue Nov 6 21:11:42 2012
@@ -154,6 +154,9 @@ Trunk (Unreleased)
HDFS-4151. Change the methods in FSDirectory to pass INodesInPath instead
of INode[] as a parameter. (szetszwo)
+ HDFS-4152. Add a new class BlocksMapUpdateInfo for the parameter in
+ INode.collectSubtreeBlocksAndClear(..). (Jing Zhao via szetszwo)
+
OPTIMIZATIONS
BUG FIXES
@@ -451,6 +454,9 @@ Release 2.0.3-alpha - Unreleased
HDFS-4143. Change blocks to private in INodeFile and renames isLink() to
isSymlink() in INode. (szetszwo)
+ HDFS-4046. Rename ChecksumTypeProto enum NULL since it is illegal in
+ C/C++. (Binglin Chang via suresh)
+
OPTIMIZATIONS
BUG FIXES
@@ -548,6 +554,8 @@ Release 2.0.3-alpha - Unreleased
HDFS-4132. When libwebhdfs is not enabled, nativeMiniDfsClient frees
uninitialized memory (Colin Patrick McCabe via todd)
+ HDFS-1331. dfs -test should work like /bin/test (Andy Isaacson via daryn)
+
Release 2.0.2-alpha - 2012-09-07
INCOMPATIBLE CHANGES
@@ -1935,6 +1943,8 @@ Release 0.23.5 - UNRELEASED
OPTIMIZATIONS
+ HDFS-4075. Reduce recommissioning overhead (Kihwal Lee via daryn)
+
BUG FIXES
HDFS-3829. TestHftpURLTimeouts fails intermittently with JDK7 (Trevor
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1405252-1406005,1406007-1406326
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsProtoUtil.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsProtoUtil.java?rev=1406337&r1=1406336&r2=1406337&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsProtoUtil.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsProtoUtil.java
Tue Nov 6 21:11:42 2012
@@ -157,11 +157,11 @@ public abstract class HdfsProtoUtil {
}
public static DataChecksum.Type fromProto(HdfsProtos.ChecksumTypeProto type)
{
- return DataChecksum.Type.valueOf(type.name());
+ return DataChecksum.Type.valueOf(type.getNumber());
}
public static HdfsProtos.ChecksumTypeProto toProto(DataChecksum.Type type) {
- return HdfsProtos.ChecksumTypeProto.valueOf(type.name());
+ return HdfsProtos.ChecksumTypeProto.valueOf(type.id);
}
public static InputStream vintPrefixed(final InputStream input)
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/DataTransferProtoUtil.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/DataTransferProtoUtil.java?rev=1406337&r1=1406336&r2=1406337&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/DataTransferProtoUtil.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/DataTransferProtoUtil.java
Tue Nov 6 21:11:42 2012
@@ -52,7 +52,7 @@ public abstract class DataTransferProtoU
}
public static ChecksumProto toProto(DataChecksum checksum) {
- ChecksumTypeProto type =
ChecksumTypeProto.valueOf(checksum.getChecksumType().name());
+ ChecksumTypeProto type = HdfsProtoUtil.toProto(checksum.getChecksumType());
if (type == null) {
throw new IllegalArgumentException(
"Can't convert checksum to protobuf: " + checksum);
@@ -68,7 +68,7 @@ public abstract class DataTransferProtoU
if (proto == null) return null;
int bytesPerChecksum = proto.getBytesPerChecksum();
- DataChecksum.Type type = DataChecksum.Type.valueOf(proto.getType().name());
+ DataChecksum.Type type = HdfsProtoUtil.fromProto(proto.getType());
return DataChecksum.newDataChecksum(type, bytesPerChecksum);
}
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java?rev=1406337&r1=1406336&r2=1406337&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
Tue Nov 6 21:11:42 2012
@@ -39,6 +39,7 @@ import org.apache.hadoop.hdfs.protocol.H
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
+import org.apache.hadoop.hdfs.protocol.HdfsProtoUtil;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos;
@@ -67,7 +68,6 @@ import org.apache.hadoop.hdfs.protocol.p
import
org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlocksWithLocationsProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.CheckpointCommandProto;
import
org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.CheckpointSignatureProto;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ChecksumTypeProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ContentSummaryProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.CorruptFileBlocksProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeIDProto;
@@ -129,7 +129,6 @@ import org.apache.hadoop.hdfs.server.pro
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest;
import org.apache.hadoop.io.EnumSetWritable;
import org.apache.hadoop.io.Text;
-import org.apache.hadoop.util.DataChecksum;
import org.apache.hadoop.security.token.Token;
import com.google.protobuf.ByteString;
@@ -961,7 +960,7 @@ public class PBHelper {
fs.getFileBufferSize(),
fs.getEncryptDataTransfer(),
fs.getTrashInterval(),
- DataChecksum.Type.valueOf(fs.getChecksumType().name()));
+ HdfsProtoUtil.fromProto(fs.getChecksumType()));
}
public static FsServerDefaultsProto convert(FsServerDefaults fs) {
@@ -974,7 +973,7 @@ public class PBHelper {
.setFileBufferSize(fs.getFileBufferSize())
.setEncryptDataTransfer(fs.getEncryptDataTransfer())
.setTrashInterval(fs.getTrashInterval())
- .setChecksumType(ChecksumTypeProto.valueOf(fs.getChecksumType().name()))
+ .setChecksumType(HdfsProtoUtil.toProto(fs.getChecksumType()))
.build();
}
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java?rev=1406337&r1=1406336&r2=1406337&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
Tue Nov 6 21:11:42 2012
@@ -2696,6 +2696,7 @@ assert storedBlock.findDatanode(dn) < 0
void processOverReplicatedBlocksOnReCommission(
final DatanodeDescriptor srcNode) {
final Iterator<? extends Block> it = srcNode.getBlockIterator();
+ int numOverReplicated = 0;
while(it.hasNext()) {
final Block block = it.next();
BlockCollection bc = blocksMap.getBlockCollection(block);
@@ -2705,8 +2706,11 @@ assert storedBlock.findDatanode(dn) < 0
if (numCurrentReplica > expectedReplication) {
// over-replicated block
processOverReplicatedBlock(block, expectedReplication, null, null);
+ numOverReplicated++;
}
}
+ LOG.info("Invalidated " + numOverReplicated + " over-replicated blocks on
" +
+ srcNode + " during recommissioning");
}
/**
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java?rev=1406337&r1=1406336&r2=1406337&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
Tue Nov 6 21:11:42 2012
@@ -608,7 +608,11 @@ public class DatanodeManager {
if (node.isDecommissionInProgress() || node.isDecommissioned()) {
LOG.info("Stop Decommissioning " + node);
heartbeatManager.stopDecommission(node);
- blockManager.processOverReplicatedBlocksOnReCommission(node);
+ // Over-replicated blocks will be detected and processed when
+ // the dead node comes back and send in its full block report.
+ if (node.isAlive) {
+ blockManager.processOverReplicatedBlocksOnReCommission(node);
+ }
}
}
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/SecureDataNodeStarter.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/SecureDataNodeStarter.java?rev=1406337&r1=1406336&r2=1406337&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/SecureDataNodeStarter.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/SecureDataNodeStarter.java
Tue Nov 6 21:11:42 2012
@@ -38,6 +38,8 @@ import org.mortbay.jetty.security.SslSoc
import javax.net.ssl.SSLServerSocketFactory;
+import com.google.common.annotations.VisibleForTesting;
+
/**
* Utility class to start a datanode in a secure cluster, first obtaining
* privileged resources before main startup and handing them to the datanode.
@@ -73,6 +75,25 @@ public class SecureDataNodeStarter imple
// Stash command-line arguments for regular datanode
args = context.getArguments();
+ sslFactory = new SSLFactory(SSLFactory.Mode.SERVER, conf);
+ resources = getSecureResources(sslFactory, conf);
+ }
+
+ @Override
+ public void start() throws Exception {
+ System.err.println("Starting regular datanode initialization");
+ DataNode.secureMain(args, resources);
+ }
+
+ @Override public void destroy() {
+ sslFactory.destroy();
+ }
+
+ @Override public void stop() throws Exception { /* Nothing to do */ }
+
+ @VisibleForTesting
+ public static SecureResources getSecureResources(final SSLFactory sslFactory,
+ Configuration conf) throws Exception {
// Obtain secure port for data streaming to datanode
InetSocketAddress streamingAddr = DataNode.getStreamingAddr(conf);
int socketWriteTimeout =
conf.getInt(DFSConfigKeys.DFS_DATANODE_SOCKET_WRITE_TIMEOUT_KEY,
@@ -85,13 +106,12 @@ public class SecureDataNodeStarter imple
// Check that we got the port we need
if (ss.getLocalPort() != streamingAddr.getPort()) {
throw new RuntimeException("Unable to bind on specified streaming port
in secure " +
- "context. Needed " + streamingAddr.getPort() + ", got " +
ss.getLocalPort());
+ "context. Needed " + streamingAddr.getPort() + ", got " +
ss.getLocalPort());
}
// Obtain secure listener for web server
Connector listener;
if (HttpConfig.isSecure()) {
- sslFactory = new SSLFactory(SSLFactory.Mode.SERVER, conf);
try {
sslFactory.init();
} catch (GeneralSecurityException ex) {
@@ -126,18 +146,7 @@ public class SecureDataNodeStarter imple
}
System.err.println("Opened streaming server at " + streamingAddr);
System.err.println("Opened info server at " + infoSocAddr);
- resources = new SecureResources(ss, listener);
+ return new SecureResources(ss, listener);
}
- @Override
- public void start() throws Exception {
- System.err.println("Starting regular datanode initialization");
- DataNode.secureMain(args, resources);
- }
-
- @Override public void destroy() {
- sslFactory.destroy();
- }
-
- @Override public void stop() throws Exception { /* Nothing to do */ }
}
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java?rev=1406337&r1=1406336&r2=1406337&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
Tue Nov 6 21:11:42 2012
@@ -57,6 +57,7 @@ import org.apache.hadoop.hdfs.server.blo
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
+import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
import org.apache.hadoop.hdfs.server.namenode.INodeDirectory.INodesInPath;
import
org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable;
import org.apache.hadoop.hdfs.util.ByteArray;
@@ -758,7 +759,7 @@ public class FSDirectory implements Clos
if (removedDst != null) {
INode rmdst = removedDst;
removedDst = null;
- List<Block> collectedBlocks = new ArrayList<Block>();
+ BlocksMapUpdateInfo collectedBlocks = new BlocksMapUpdateInfo();
filesDeleted = rmdst.collectSubtreeBlocksAndClear(collectedBlocks);
getFSNamesystem().removePathAndBlocks(src, collectedBlocks);
}
@@ -997,7 +998,7 @@ public class FSDirectory implements Clos
* @param collectedBlocks Blocks under the deleted directory
* @return true on successful deletion; else false
*/
- boolean delete(String src, List<Block>collectedBlocks)
+ boolean delete(String src, BlocksMapUpdateInfo collectedBlocks)
throws IOException {
if (NameNode.stateChangeLog.isDebugEnabled()) {
NameNode.stateChangeLog.debug("DIR* FSDirectory.delete: " + src);
@@ -1084,7 +1085,7 @@ public class FSDirectory implements Clos
void unprotectedDelete(String src, long mtime)
throws UnresolvedLinkException {
assert hasWriteLock();
- List<Block> collectedBlocks = new ArrayList<Block>();
+ BlocksMapUpdateInfo collectedBlocks = new BlocksMapUpdateInfo();
int filesRemoved = 0;
final INodesInPath inodesInPath = rootDir.getExistingPathINodes(
@@ -1108,8 +1109,8 @@ public class FSDirectory implements Clos
* @param mtime the time the inode is removed
* @return the number of inodes deleted; 0 if no inodes are deleted.
*/
- int unprotectedDelete(INodesInPath inodesInPath, List<Block>
collectedBlocks,
- long mtime) {
+ int unprotectedDelete(INodesInPath inodesInPath,
+ BlocksMapUpdateInfo collectedBlocks, long mtime) {
assert hasWriteLock();
final INode[] inodes = inodesInPath.getINodes();
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1406337&r1=1406336&r2=1406337&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
Tue Nov 6 21:11:42 2012
@@ -159,6 +159,7 @@ import org.apache.hadoop.hdfs.server.com
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirType;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
import org.apache.hadoop.hdfs.server.common.Util;
+import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
import org.apache.hadoop.hdfs.server.namenode.INodeDirectory.INodesInPath;
import org.apache.hadoop.hdfs.server.namenode.LeaseManager.Lease;
import org.apache.hadoop.hdfs.server.namenode.NameNode.OperationCategory;
@@ -2676,7 +2677,7 @@ public class FSNamesystem implements Nam
boolean enforcePermission)
throws AccessControlException, SafeModeException,
UnresolvedLinkException,
IOException {
- ArrayList<Block> collectedBlocks = new ArrayList<Block>();
+ BlocksMapUpdateInfo collectedBlocks = new BlocksMapUpdateInfo();
writeLock();
try {
@@ -2707,21 +2708,26 @@ public class FSNamesystem implements Nam
return true;
}
- /**
+ /**
* From the given list, incrementally remove the blocks from blockManager
* Writelock is dropped and reacquired every BLOCK_DELETION_INCREMENT to
* ensure that other waiters on the lock can get in. See HDFS-2938
+ *
+ * @param blocks
+ * An instance of {@link BlocksMapUpdateInfo} which contains a list
+ * of blocks that need to be removed from blocksMap
*/
- private void removeBlocks(List<Block> blocks) {
+ private void removeBlocks(BlocksMapUpdateInfo blocks) {
int start = 0;
int end = 0;
- while (start < blocks.size()) {
+ List<Block> toDeleteList = blocks.getToDeleteList();
+ while (start < toDeleteList.size()) {
end = BLOCK_DELETION_INCREMENT + start;
- end = end > blocks.size() ? blocks.size() : end;
+ end = end > toDeleteList.size() ? toDeleteList.size() : end;
writeLock();
try {
for (int i = start; i < end; i++) {
- blockManager.removeBlock(blocks.get(i));
+ blockManager.removeBlock(toDeleteList.get(i));
}
} finally {
writeUnlock();
@@ -2730,7 +2736,12 @@ public class FSNamesystem implements Nam
}
}
- void removePathAndBlocks(String src, List<Block> blocks) {
+ /**
+ * Remove leases and blocks related to a given path
+ * @param src The given path
+ * @param blocks Containing the list of blocks to be deleted from blocksMap
+ */
+ void removePathAndBlocks(String src, BlocksMapUpdateInfo blocks) {
assert hasWriteLock();
leaseManager.removeLeaseWithPrefixPath(src);
if (blocks == null) {
@@ -2743,7 +2754,7 @@ public class FSNamesystem implements Nam
boolean trackBlockCounts = isSafeModeTrackingBlocks();
int numRemovedComplete = 0, numRemovedSafe = 0;
- for (Block b : blocks) {
+ for (Block b : blocks.getToDeleteList()) {
if (trackBlockCounts) {
BlockInfo bi = blockManager.getStoredBlock(b);
if (bi.isComplete()) {
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java?rev=1406337&r1=1406336&r2=1406337&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
Tue Nov 6 21:11:42 2012
@@ -188,11 +188,15 @@ public abstract class INode implements C
}
/**
- * Collect all the blocks in all children of this INode.
- * Count and return the number of files in the sub tree.
- * Also clears references since this INode is deleted.
+ * Collect all the blocks in all children of this INode. Count and return the
+ * number of files in the sub tree. Also clears references since this INode
is
+ * deleted.
+ *
+ * @param info
+ * Containing all the blocks collected from the children of this
+ * INode. These blocks later should be removed from the blocksMap.
*/
- abstract int collectSubtreeBlocksAndClear(List<Block> v);
+ abstract int collectSubtreeBlocksAndClear(BlocksMapUpdateInfo info);
/** Compute {@link ContentSummary}. */
public final ContentSummary computeContentSummary() {
@@ -493,4 +497,48 @@ public abstract class INode implements C
out.print(s.substring(s.lastIndexOf(getClass().getSimpleName())));
out.println(")");
}
+
+ /**
+ * Information used for updating the blocksMap when deleting files.
+ */
+ public static class BlocksMapUpdateInfo {
+ /**
+ * The list of blocks that need to be removed from blocksMap
+ */
+ private List<Block> toDeleteList;
+
+ public BlocksMapUpdateInfo(List<Block> toDeleteList) {
+ this.toDeleteList = toDeleteList == null ? new ArrayList<Block>()
+ : toDeleteList;
+ }
+
+ public BlocksMapUpdateInfo() {
+ toDeleteList = new ArrayList<Block>();
+ }
+
+ /**
+ * @return The list of blocks that need to be removed from blocksMap
+ */
+ public List<Block> getToDeleteList() {
+ return toDeleteList;
+ }
+
+ /**
+ * Add a to-be-deleted block into the
+ * {@link BlocksMapUpdateInfo#toDeleteList}
+ * @param toDelete the to-be-deleted block
+ */
+ public void addDeleteBlock(Block toDelete) {
+ if (toDelete != null) {
+ toDeleteList.add(toDelete);
+ }
+ }
+
+ /**
+ * Clear {@link BlocksMapUpdateInfo#toDeleteList}
+ */
+ public void clear() {
+ toDeleteList.clear();
+ }
+ }
}
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java?rev=1406337&r1=1406336&r2=1406337&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
Tue Nov 6 21:11:42 2012
@@ -28,7 +28,6 @@ import java.util.List;
import org.apache.hadoop.fs.UnresolvedLinkException;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.UnresolvedPathException;
import
org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable;
@@ -473,13 +472,13 @@ public class INodeDirectory extends INod
}
@Override
- int collectSubtreeBlocksAndClear(List<Block> v) {
+ int collectSubtreeBlocksAndClear(BlocksMapUpdateInfo info) {
int total = 1;
if (children == null) {
return total;
}
for (INode child : children) {
- total += child.collectSubtreeBlocksAndClear(v);
+ total += child.collectSubtreeBlocksAndClear(info);
}
parent = null;
children = null;
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java?rev=1406337&r1=1406336&r2=1406337&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
Tue Nov 6 21:11:42 2012
@@ -19,7 +19,6 @@ package org.apache.hadoop.hdfs.server.na
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.permission.FsAction;
@@ -162,11 +161,11 @@ public class INodeFile extends INode imp
}
@Override
- protected int collectSubtreeBlocksAndClear(List<Block> v) {
+ protected int collectSubtreeBlocksAndClear(BlocksMapUpdateInfo info) {
parent = null;
- if(blocks != null && v != null) {
+ if(blocks != null && info != null) {
for (BlockInfo blk : blocks) {
- v.add(blk);
+ info.addDeleteBlock(blk);
blk.setBlockCollection(null);
}
}
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java?rev=1406337&r1=1406336&r2=1406337&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java
Tue Nov 6 21:11:42 2012
@@ -17,12 +17,9 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
-import java.util.List;
-
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.hdfs.protocol.Block;
/**
* An INode representing a symbolic link.
@@ -72,7 +69,7 @@ public class INodeSymlink extends INode
}
@Override
- int collectSubtreeBlocksAndClear(List<Block> v) {
+ int collectSubtreeBlocksAndClear(BlocksMapUpdateInfo info) {
return 1;
}
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native:r1405252-1406326
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/datatransfer.proto
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/datatransfer.proto?rev=1406337&r1=1406336&r2=1406337&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/datatransfer.proto
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/datatransfer.proto
Tue Nov 6 21:11:42 2012
@@ -181,5 +181,5 @@ message OpBlockChecksumResponseProto {
required uint32 bytesPerCrc = 1;
required uint64 crcPerBlock = 2;
required bytes md5 = 3;
- optional ChecksumTypeProto crcType = 4 [default = CRC32];
+ optional ChecksumTypeProto crcType = 4 [default = CHECKSUM_CRC32];
}
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto?rev=1406337&r1=1406336&r2=1406337&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
Tue Nov 6 21:11:42 2012
@@ -181,11 +181,13 @@ message HdfsFileStatusProto {
/**
* Checksum algorithms/types used in HDFS
+ * Make sure this enum's integer values match enum values' id properties
defined
+ * in org.apache.hadoop.util.DataChecksum.Type
*/
enum ChecksumTypeProto {
- NULL = 0;
- CRC32 = 1;
- CRC32C = 2;
+ CHECKSUM_NULL = 0;
+ CHECKSUM_CRC32 = 1;
+ CHECKSUM_CRC32C = 2;
}
/**
@@ -199,7 +201,7 @@ message FsServerDefaultsProto {
required uint32 fileBufferSize = 5;
optional bool encryptDataTransfer = 6 [default = false];
optional uint64 trashInterval = 7 [default = 0];
- optional ChecksumTypeProto checksumType = 8 [default = CRC32];
+ optional ChecksumTypeProto checksumType = 8 [default = CHECKSUM_CRC32];
}
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:r1405252-1406326
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:r1405252-1406326
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:r1405252-1406326
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:r1405252-1406326
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java?rev=1406337&r1=1406336&r2=1406337&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
Tue Nov 6 21:11:42 2012
@@ -81,6 +81,8 @@ import org.apache.hadoop.hdfs.server.com
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
import org.apache.hadoop.hdfs.server.datanode.DataStorage;
+import org.apache.hadoop.hdfs.server.datanode.SecureDataNodeStarter;
+import
org.apache.hadoop.hdfs.server.datanode.SecureDataNodeStarter.SecureResources;
import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
@@ -95,6 +97,7 @@ import org.apache.hadoop.net.StaticMappi
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.ProxyUsers;
+import org.apache.hadoop.security.ssl.SSLFactory;
import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.ToolRunner;
@@ -145,6 +148,7 @@ public class MiniDFSCluster {
private boolean setupHostsFile = false;
private MiniDFSNNTopology nnTopology = null;
private boolean checkExitOnShutdown = true;
+ private boolean checkDataNodeAddrConfig = false;
private boolean checkDataNodeHostConfig = false;
public Builder(Configuration conf) {
@@ -266,6 +270,14 @@ public class MiniDFSCluster {
/**
* Default: false
*/
+ public Builder checkDataNodeAddrConfig(boolean val) {
+ this.checkDataNodeAddrConfig = val;
+ return this;
+ }
+
+ /**
+ * Default: false
+ */
public Builder checkDataNodeHostConfig(boolean val) {
this.checkDataNodeHostConfig = val;
return this;
@@ -336,6 +348,7 @@ public class MiniDFSCluster {
builder.setupHostsFile,
builder.nnTopology,
builder.checkExitOnShutdown,
+ builder.checkDataNodeAddrConfig,
builder.checkDataNodeHostConfig);
}
@@ -343,11 +356,14 @@ public class MiniDFSCluster {
DataNode datanode;
Configuration conf;
String[] dnArgs;
+ SecureResources secureResources;
- DataNodeProperties(DataNode node, Configuration conf, String[] args) {
+ DataNodeProperties(DataNode node, Configuration conf, String[] args,
+ SecureResources secureResources) {
this.datanode = node;
this.conf = conf;
this.dnArgs = args;
+ this.secureResources = secureResources;
}
}
@@ -573,7 +589,7 @@ public class MiniDFSCluster {
manageNameDfsDirs, true, manageDataDfsDirs, manageDataDfsDirs,
operation, racks, hosts,
simulatedCapacities, null, true, false,
- MiniDFSNNTopology.simpleSingleNN(nameNodePort, 0), true, false);
+ MiniDFSNNTopology.simpleSingleNN(nameNodePort, 0), true, false, false);
}
private void initMiniDFSCluster(
@@ -584,6 +600,7 @@ public class MiniDFSCluster {
String[] hosts, long[] simulatedCapacities, String clusterId,
boolean waitSafeMode, boolean setupHostsFile,
MiniDFSNNTopology nnTopology, boolean checkExitOnShutdown,
+ boolean checkDataNodeAddrConfig,
boolean checkDataNodeHostConfig)
throws IOException {
ExitUtil.disableSystemExit();
@@ -647,7 +664,7 @@ public class MiniDFSCluster {
// Start the DataNodes
startDataNodes(conf, numDataNodes, manageDataDfsDirs, operation, racks,
- hosts, simulatedCapacities, setupHostsFile, false,
checkDataNodeHostConfig);
+ hosts, simulatedCapacities, setupHostsFile, checkDataNodeAddrConfig,
checkDataNodeHostConfig);
waitClusterUp();
//make sure ProxyUsers uses the latest conf
ProxyUsers.refreshSuperUserGroupsConfiguration(conf);
@@ -1161,7 +1178,18 @@ public class MiniDFSCluster {
if (hosts != null) {
NetUtils.addStaticResolution(hosts[i - curDatanodesNum], "localhost");
}
- DataNode dn = DataNode.instantiateDataNode(dnArgs, dnConf);
+
+ SecureResources secureResources = null;
+ if (UserGroupInformation.isSecurityEnabled()) {
+ SSLFactory sslFactory = new SSLFactory(SSLFactory.Mode.SERVER, dnConf);
+ try {
+ secureResources =
SecureDataNodeStarter.getSecureResources(sslFactory, dnConf);
+ } catch (Exception ex) {
+ ex.printStackTrace();
+ }
+ }
+ DataNode dn = DataNode.instantiateDataNode(dnArgs, dnConf,
+ secureResources);
if(dn == null)
throw new IOException("Cannot start DataNode in "
+ dnConf.get(DFS_DATANODE_DATA_DIR_KEY));
@@ -1176,7 +1204,7 @@ public class MiniDFSCluster {
racks[i-curDatanodesNum]);
}
dn.runDatanodeDaemon();
- dataNodes.add(new DataNodeProperties(dn, newconf, dnArgs));
+ dataNodes.add(new DataNodeProperties(dn, newconf, dnArgs,
secureResources));
}
curDatanodesNum += numDataNodes;
this.numDataNodes += numDataNodes;
@@ -1607,14 +1635,16 @@ public class MiniDFSCluster {
boolean keepPort) throws IOException {
Configuration conf = dnprop.conf;
String[] args = dnprop.dnArgs;
+ SecureResources secureResources = dnprop.secureResources;
Configuration newconf = new HdfsConfiguration(conf); // save cloned config
if (keepPort) {
InetSocketAddress addr = dnprop.datanode.getXferAddress();
conf.set(DFS_DATANODE_ADDRESS_KEY,
addr.getAddress().getHostAddress() + ":" + addr.getPort());
}
- dataNodes.add(new DataNodeProperties(DataNode.createDataNode(args, conf),
- newconf, args));
+ dataNodes.add(new DataNodeProperties(
+ DataNode.createDataNode(args, conf, secureResources),
+ newconf, args, secureResources));
numDataNodes++;
return true;
}
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java?rev=1406337&r1=1406336&r2=1406337&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
Tue Nov 6 21:11:42 2012
@@ -1243,7 +1243,106 @@ public class TestDFSShell {
}
assertEquals(0, val);
}
-
+
+ // Verify -test -f negative case (missing file)
+ {
+ String[] args = new String[3];
+ args[0] = "-test";
+ args[1] = "-f";
+ args[2] = "/test/mkdirs/noFileHere";
+ int val = -1;
+ try {
+ val = shell.run(args);
+ } catch (Exception e) {
+ System.err.println("Exception raised from DFSShell.run " +
+ e.getLocalizedMessage());
+ }
+ assertEquals(1, val);
+ }
+
+ // Verify -test -f negative case (directory rather than file)
+ {
+ String[] args = new String[3];
+ args[0] = "-test";
+ args[1] = "-f";
+ args[2] = "/test/mkdirs";
+ int val = -1;
+ try {
+ val = shell.run(args);
+ } catch (Exception e) {
+ System.err.println("Exception raised from DFSShell.run " +
+ e.getLocalizedMessage());
+ }
+ assertEquals(1, val);
+ }
+
+ // Verify -test -f positive case
+ {
+ writeFile(fileSys, myFile);
+ assertTrue(fileSys.exists(myFile));
+
+ String[] args = new String[3];
+ args[0] = "-test";
+ args[1] = "-f";
+ args[2] = myFile.toString();
+ int val = -1;
+ try {
+ val = shell.run(args);
+ } catch (Exception e) {
+ System.err.println("Exception raised from DFSShell.run " +
+ e.getLocalizedMessage());
+ }
+ assertEquals(0, val);
+ }
+
+ // Verify -test -s negative case (missing file)
+ {
+ String[] args = new String[3];
+ args[0] = "-test";
+ args[1] = "-s";
+ args[2] = "/test/mkdirs/noFileHere";
+ int val = -1;
+ try {
+ val = shell.run(args);
+ } catch (Exception e) {
+ System.err.println("Exception raised from DFSShell.run " +
+ e.getLocalizedMessage());
+ }
+ assertEquals(1, val);
+ }
+
+ // Verify -test -s negative case (zero length file)
+ {
+ String[] args = new String[3];
+ args[0] = "-test";
+ args[1] = "-s";
+ args[2] = "/test/mkdirs/isFileHere";
+ int val = -1;
+ try {
+ val = shell.run(args);
+ } catch (Exception e) {
+ System.err.println("Exception raised from DFSShell.run " +
+ e.getLocalizedMessage());
+ }
+ assertEquals(1, val);
+ }
+
+ // Verify -test -s positive case (nonzero length file)
+ {
+ String[] args = new String[3];
+ args[0] = "-test";
+ args[1] = "-s";
+ args[2] = myFile.toString();
+ int val = -1;
+ try {
+ val = shell.run(args);
+ } catch (Exception e) {
+ System.err.println("Exception raised from DFSShell.run " +
+ e.getLocalizedMessage());
+ }
+ assertEquals(0, val);
+ }
+
} finally {
try {
fileSys.close();
