Author: szetszwo
Date: Thu Feb 20 04:38:30 2014
New Revision: 1570084
URL: http://svn.apache.org/r1570084
Log:
Merge r1569890 through r1570083 from trunk.
Added:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockHasMultipleReplicasOnSameDN.java
- copied unchanged from r1570083,
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockHasMultipleReplicasOnSameDN.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestReadOnlySharedStorage.java
- copied unchanged from r1570083,
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestReadOnlySharedStorage.java
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/ (props
changed)
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
(props changed)
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/ReplicaOutputStreams.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeStorage.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DataNodeCluster.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSClusterWithNodeGroup.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestInjectionForSimulatedStorage.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
Propchange: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1569890-1570083
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1570084&r1=1570083&r2=1570084&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Thu Feb 20 04:38:30 2014
@@ -416,6 +416,14 @@ Release 2.4.0 - UNRELEASED
HDFS-5726. Fix compilation error in AbstractINodeDiff for JDK7. (jing9)
+ HDFS-5973. add DomainSocket#shutdown method (cmccabe)
+
+ HDFS-5318. Support read-only and read-write paths to shared replicas.
+ (Eric Sirianni via Arpit Agarwal)
+
+ HDFS-5868. Make hsync implementation pluggable on the DataNode.
+ (Buddy Taylor via Arpit Agarwal)
+
OPTIMIZATIONS
HDFS-5790. LeaseManager.findPath is very slow when many leases need
recovery
@@ -521,6 +529,14 @@ Release 2.4.0 - UNRELEASED
HDFS-5961. OIV cannot load fsimages containing a symbolic link. (kihwal)
+ HDFS-5483. NN should gracefully handle multiple block replicas on same DN.
+ (Arpit Agarwal)
+
+ HDFS-5742. DatanodeCluster (mini cluster of DNs) fails to start.
+ (Arpit Agarwal)
+
+ HDFS-5979. Typo and logger fix for fsimage PB code. (wang)
+
BREAKDOWN OF HDFS-5698 SUBTASKS AND RELATED JIRAS
HDFS-5717. Save FSImage header in protobuf. (Haohui Mai via jing9)
Propchange:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1569890-1570083
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java?rev=1570084&r1=1570083&r2=1570084&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
Thu Feb 20 04:38:30 2014
@@ -1581,8 +1581,8 @@ public class PBHelper {
private static StorageState convertState(State state) {
switch(state) {
- case READ_ONLY:
- return StorageState.READ_ONLY;
+ case READ_ONLY_SHARED:
+ return StorageState.READ_ONLY_SHARED;
case NORMAL:
default:
return StorageState.NORMAL;
@@ -1610,8 +1610,8 @@ public class PBHelper {
private static State convertState(StorageState state) {
switch(state) {
- case READ_ONLY:
- return DatanodeStorage.State.READ_ONLY;
+ case READ_ONLY_SHARED:
+ return DatanodeStorage.State.READ_ONLY_SHARED;
case NORMAL:
default:
return DatanodeStorage.State.NORMAL;
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java?rev=1570084&r1=1570083&r2=1570084&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
Thu Feb 20 04:38:30 2014
@@ -73,6 +73,7 @@ import org.apache.hadoop.hdfs.server.pro
import
org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations.BlockWithLocations;
import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage.State;
import org.apache.hadoop.hdfs.server.protocol.KeyUpdateCommand;
import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo;
import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
@@ -501,7 +502,10 @@ public class BlockManager {
chooseSourceDatanode(block, containingNodes,
containingLiveReplicasNodes, numReplicas,
UnderReplicatedBlocks.LEVEL);
- assert containingLiveReplicasNodes.size() == numReplicas.liveReplicas();
+
+ // containingLiveReplicasNodes can include READ_ONLY_SHARED replicas which
are
+ // not included in the numReplicas.liveReplicas() count
+ assert containingLiveReplicasNodes.size() >= numReplicas.liveReplicas();
int usableReplicas = numReplicas.liveReplicas() +
numReplicas.decommissionedReplicas();
@@ -1040,7 +1044,7 @@ public class BlockManager {
*/
private void addToInvalidates(Block b) {
StringBuilder datanodes = new StringBuilder();
- for(DatanodeStorageInfo storage : blocksMap.getStorages(b)) {
+ for(DatanodeStorageInfo storage : blocksMap.getStorages(b, State.NORMAL)) {
final DatanodeDescriptor node = storage.getDatanodeDescriptor();
invalidateBlocks.add(b, node, false);
datanodes.append(node).append(" ");
@@ -1254,7 +1258,10 @@ public class BlockManager {
continue;
}
- assert liveReplicaNodes.size() == numReplicas.liveReplicas();
+ // liveReplicaNodes can include READ_ONLY_SHARED replicas which
are
+ // not included in the numReplicas.liveReplicas() count
+ assert liveReplicaNodes.size() >= numReplicas.liveReplicas();
+
// do not schedule more if enough replicas is already pending
numEffectiveReplicas = numReplicas.liveReplicas() +
pendingReplications.getNumReplicas(block);
@@ -1494,15 +1501,16 @@ public class BlockManager {
final DatanodeDescriptor node = storage.getDatanodeDescriptor();
LightWeightLinkedSet<Block> excessBlocks =
excessReplicateMap.get(node.getDatanodeUuid());
+ int countableReplica = storage.getState() == State.NORMAL ? 1 : 0;
if ((nodesCorrupt != null) && (nodesCorrupt.contains(node)))
- corrupt++;
+ corrupt += countableReplica;
else if (node.isDecommissionInProgress() || node.isDecommissioned())
- decommissioned++;
+ decommissioned += countableReplica;
else if (excessBlocks != null && excessBlocks.contains(block)) {
- excess++;
+ excess += countableReplica;
} else {
nodesContainingLiveReplicas.add(storage);
- live++;
+ live += countableReplica;
}
containingNodes.add(node);
// Check if this replica is corrupt
@@ -1880,7 +1888,8 @@ public class BlockManager {
iblk, iState, toAdd, toInvalidate, toCorrupt, toUC);
// move block to the head of the list
- if (storedBlock != null && (curIndex = storedBlock.findDatanode(dn)) >=
0) {
+ if (storedBlock != null &&
+ (curIndex = storedBlock.findStorageInfo(storageInfo)) >= 0) {
headIndex = storageInfo.moveBlockToHead(storedBlock, curIndex,
headIndex);
}
}
@@ -2581,7 +2590,7 @@ assert storedBlock.findDatanode(dn) < 0
Collection<DatanodeDescriptor> nonExcess = new
ArrayList<DatanodeDescriptor>();
Collection<DatanodeDescriptor> corruptNodes = corruptReplicas
.getNodes(block);
- for(DatanodeStorageInfo storage : blocksMap.getStorages(block)) {
+ for(DatanodeStorageInfo storage : blocksMap.getStorages(block,
State.NORMAL)) {
final DatanodeDescriptor cur = storage.getDatanodeDescriptor();
if (storage.areBlockContentsStale()) {
LOG.info("BLOCK* processOverReplicatedBlock: " +
@@ -2910,7 +2919,7 @@ assert storedBlock.findDatanode(dn) < 0
int excess = 0;
int stale = 0;
Collection<DatanodeDescriptor> nodesCorrupt = corruptReplicas.getNodes(b);
- for(DatanodeStorageInfo storage : blocksMap.getStorages(b)) {
+ for(DatanodeStorageInfo storage : blocksMap.getStorages(b, State.NORMAL)) {
final DatanodeDescriptor node = storage.getDatanodeDescriptor();
if ((nodesCorrupt != null) && (nodesCorrupt.contains(node))) {
corrupt++;
@@ -2949,7 +2958,7 @@ assert storedBlock.findDatanode(dn) < 0
// else proceed with fast case
int live = 0;
Collection<DatanodeDescriptor> nodesCorrupt = corruptReplicas.getNodes(b);
- for(DatanodeStorageInfo storage : blocksMap.getStorages(b)) {
+ for(DatanodeStorageInfo storage : blocksMap.getStorages(b, State.NORMAL)) {
final DatanodeDescriptor node = storage.getDatanodeDescriptor();
if ((nodesCorrupt == null) || (!nodesCorrupt.contains(node)))
live++;
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java?rev=1570084&r1=1570083&r2=1570084&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
Thu Feb 20 04:38:30 2014
@@ -605,7 +605,7 @@ public class BlockPlacementPolicyDefault
+ storageType);
return false;
}
- if (storage.getState() == State.READ_ONLY) {
+ if (storage.getState() == State.READ_ONLY_SHARED) {
logNodeIsNotChosen(storage, "storage is read-only");
return false;
}
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java?rev=1570084&r1=1570083&r2=1570084&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
Thu Feb 20 04:38:30 2014
@@ -20,10 +20,14 @@ package org.apache.hadoop.hdfs.server.bl
import java.util.Iterator;
import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
import org.apache.hadoop.util.GSet;
import org.apache.hadoop.util.LightWeightGSet;
import org.apache.hadoop.util.LightWeightGSet.SetIterator;
+import com.google.common.base.Predicate;
+import com.google.common.collect.Iterables;
+
/**
* This class maintains the map from a block to its metadata.
* block's metadata currently includes blockCollection it belongs to and
@@ -136,6 +140,22 @@ class BlocksMap {
}
/**
+ * Searches for the block in the BlocksMap and
+ * returns {@link Iterable} of the storages the block belongs to
+ * <i>that are of the given {@link DatanodeStorage.State state}</i>.
+ *
+ * @param state DatanodeStorage state by which to filter the returned
Iterable
+ */
+ Iterable<DatanodeStorageInfo> getStorages(Block b, final
DatanodeStorage.State state) {
+ return Iterables.filter(getStorages(blocks.get(b)), new
Predicate<DatanodeStorageInfo>() {
+ @Override
+ public boolean apply(DatanodeStorageInfo storage) {
+ return storage.getState() == state;
+ }
+ });
+ }
+
+ /**
* For a block that has already been retrieved from the BlocksMap
* returns {@link Iterable} of the storages the block belongs to.
*/
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java?rev=1570084&r1=1570083&r2=1570084&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
Thu Feb 20 04:38:30 2014
@@ -78,7 +78,6 @@ class BlockReceiver implements Closeable
private boolean needsChecksumTranslation;
private OutputStream out = null; // to block file at local disk
private FileDescriptor outFd;
- private OutputStream cout = null; // output stream for cehcksum file
private DataOutputStream checksumOut = null; // to crc file at local disk
private int bytesPerChecksum;
private int checksumSize;
@@ -223,9 +222,8 @@ class BlockReceiver implements Closeable
LOG.warn("Could not get file descriptor for outputstream of class " +
out.getClass());
}
- this.cout = streams.getChecksumOut();
this.checksumOut = new DataOutputStream(new BufferedOutputStream(
- cout, HdfsConstants.SMALL_BUFFER_SIZE));
+ streams.getChecksumOut(), HdfsConstants.SMALL_BUFFER_SIZE));
// write data chunk header if creating a new replica
if (isCreate) {
BlockMetadataHeader.writeHeader(checksumOut, diskChecksum);
@@ -280,9 +278,9 @@ class BlockReceiver implements Closeable
long flushStartNanos = System.nanoTime();
checksumOut.flush();
long flushEndNanos = System.nanoTime();
- if (syncOnClose && (cout instanceof FileOutputStream)) {
+ if (syncOnClose) {
long fsyncStartNanos = flushEndNanos;
- ((FileOutputStream)cout).getChannel().force(true);
+ streams.syncChecksumOut();
datanode.metrics.addFsyncNanos(System.nanoTime() - fsyncStartNanos);
}
flushTotalNanos += flushEndNanos - flushStartNanos;
@@ -302,9 +300,9 @@ class BlockReceiver implements Closeable
long flushStartNanos = System.nanoTime();
out.flush();
long flushEndNanos = System.nanoTime();
- if (syncOnClose && (out instanceof FileOutputStream)) {
+ if (syncOnClose) {
long fsyncStartNanos = flushEndNanos;
- ((FileOutputStream)out).getChannel().force(true);
+ streams.syncDataOut();
datanode.metrics.addFsyncNanos(System.nanoTime() - fsyncStartNanos);
}
flushTotalNanos += flushEndNanos - flushStartNanos;
@@ -338,9 +336,9 @@ class BlockReceiver implements Closeable
long flushStartNanos = System.nanoTime();
checksumOut.flush();
long flushEndNanos = System.nanoTime();
- if (isSync && (cout instanceof FileOutputStream)) {
+ if (isSync) {
long fsyncStartNanos = flushEndNanos;
- ((FileOutputStream)cout).getChannel().force(true);
+ streams.syncChecksumOut();
datanode.metrics.addFsyncNanos(System.nanoTime() - fsyncStartNanos);
}
flushTotalNanos += flushEndNanos - flushStartNanos;
@@ -349,9 +347,9 @@ class BlockReceiver implements Closeable
long flushStartNanos = System.nanoTime();
out.flush();
long flushEndNanos = System.nanoTime();
- if (isSync && (out instanceof FileOutputStream)) {
+ if (isSync) {
long fsyncStartNanos = flushEndNanos;
- ((FileOutputStream)out).getChannel().force(true);
+ streams.syncDataOut();
datanode.metrics.addFsyncNanos(System.nanoTime() - fsyncStartNanos);
}
flushTotalNanos += flushEndNanos - flushStartNanos;
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/ReplicaOutputStreams.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/ReplicaOutputStreams.java?rev=1570084&r1=1570083&r2=1570084&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/ReplicaOutputStreams.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/ReplicaOutputStreams.java
Thu Feb 20 04:38:30 2014
@@ -18,7 +18,9 @@
package org.apache.hadoop.hdfs.server.datanode.fsdataset;
import java.io.Closeable;
+import java.io.FileOutputStream;
import java.io.OutputStream;
+import java.io.IOException;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.DataChecksum;
@@ -62,4 +64,23 @@ public class ReplicaOutputStreams implem
IOUtils.closeStream(dataOut);
IOUtils.closeStream(checksumOut);
}
-}
\ No newline at end of file
+
+ /**
+ * Sync the data stream if it supports it.
+ */
+ public void syncDataOut() throws IOException {
+ if (dataOut instanceof FileOutputStream) {
+ ((FileOutputStream)dataOut).getChannel().force(true);
+ }
+ }
+
+ /**
+ * Sync the checksum stream if it supports it.
+ */
+ public void syncChecksumOut() throws IOException {
+ if (checksumOut instanceof FileOutputStream) {
+ ((FileOutputStream)checksumOut).getChannel().force(true);
+ }
+ }
+
+}
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java?rev=1570084&r1=1570083&r2=1570084&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
Thu Feb 20 04:38:30 2014
@@ -139,6 +139,7 @@ public class FSImage implements Closeabl
"FSImage.format should be called with an uninitialized namesystem, has
" +
fileCount + " files");
NamespaceInfo ns = NNStorage.newNamespaceInfo();
+ LOG.info("Allocated new BlockPoolId: " + ns.getBlockPoolID());
ns.clusterID = clusterId;
storage.format(ns);
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java?rev=1570084&r1=1570083&r2=1570084&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
Thu Feb 20 04:38:30 2014
@@ -75,7 +75,7 @@ public final class FSImageFormatPBINode
private static final AclEntryType[] ACL_ENTRY_TYPE_VALUES = AclEntryType
.values();
- private static final Log LOG =
LogFactory.getLog(FSImageFormatProtobuf.class);
+ private static final Log LOG = LogFactory.getLog(FSImageFormatPBINode.class);
public final static class Loader {
public static PermissionStatus loadPermission(long id,
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java?rev=1570084&r1=1570083&r2=1570084&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
Thu Feb 20 04:38:30 2014
@@ -267,7 +267,7 @@ public final class FSImageFormatProtobuf
}
break;
default:
- LOG.warn("Unregconized section " + n);
+ LOG.warn("Unrecognized section " + n);
break;
}
}
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java?rev=1570084&r1=1570083&r2=1570084&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
Thu Feb 20 04:38:30 2014
@@ -58,6 +58,8 @@ import org.apache.hadoop.hdfs.protocol.L
import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementStatus;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.NumberReplicas;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.datanode.CachingStrategy;
import org.apache.hadoop.net.NetUtils;
@@ -378,11 +380,13 @@ public class NamenodeFsck {
boolean isCorrupt = lBlk.isCorrupt();
String blkName = block.toString();
DatanodeInfo[] locs = lBlk.getLocations();
- res.totalReplicas += locs.length;
+ NumberReplicas numberReplicas =
namenode.getNamesystem().getBlockManager().countNodes(block.getLocalBlock());
+ int liveReplicas = numberReplicas.liveReplicas();
+ res.totalReplicas += liveReplicas;
short targetFileReplication = file.getReplication();
res.numExpectedReplicas += targetFileReplication;
- if (locs.length > targetFileReplication) {
- res.excessiveReplicas += (locs.length - targetFileReplication);
+ if (liveReplicas > targetFileReplication) {
+ res.excessiveReplicas += (liveReplicas - targetFileReplication);
res.numOverReplicatedBlocks += 1;
}
// Check if block is Corrupt
@@ -392,10 +396,10 @@ public class NamenodeFsck {
out.print("\n" + path + ": CORRUPT blockpool " +
block.getBlockPoolId() +
" block " + block.getBlockName()+"\n");
}
- if (locs.length >= minReplication)
+ if (liveReplicas >= minReplication)
res.numMinReplicatedBlocks++;
- if (locs.length < targetFileReplication && locs.length > 0) {
- res.missingReplicas += (targetFileReplication - locs.length);
+ if (liveReplicas < targetFileReplication && liveReplicas > 0) {
+ res.missingReplicas += (targetFileReplication - liveReplicas);
res.numUnderReplicatedBlocks += 1;
underReplicatedPerFile++;
if (!showFiles) {
@@ -404,7 +408,7 @@ public class NamenodeFsck {
out.println(" Under replicated " + block +
". Target Replicas is " +
targetFileReplication + " but found " +
- locs.length + " replica(s).");
+ liveReplicas + " replica(s).");
}
// verify block placement policy
BlockPlacementStatus blockPlacementStatus = bpPolicy
@@ -421,13 +425,13 @@ public class NamenodeFsck {
block + ". " + blockPlacementStatus.getErrorDescription());
}
report.append(i + ". " + blkName + " len=" + block.getNumBytes());
- if (locs.length == 0) {
+ if (liveReplicas == 0) {
report.append(" MISSING!");
res.addMissing(block.toString(), block.getNumBytes());
missing++;
missize += block.getNumBytes();
} else {
- report.append(" repl=" + locs.length);
+ report.append(" repl=" + liveReplicas);
if (showLocations || showRacks) {
StringBuilder sb = new StringBuilder("[");
for (int j = 0; j < locs.length; j++) {
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeStorage.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeStorage.java?rev=1570084&r1=1570083&r2=1570084&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeStorage.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeStorage.java
Thu Feb 20 04:38:30 2014
@@ -28,7 +28,18 @@ public class DatanodeStorage {
/** The state of the storage. */
public enum State {
NORMAL,
- READ_ONLY
+
+ /**
+ * A storage that represents a read-only path to replicas stored on a
shared storage device.
+ * Replicas on {@link #READ_ONLY_SHARED} storage are not counted towards
live replicas.
+ *
+ * <p>
+ * In certain implementations, a {@link #READ_ONLY_SHARED} storage may be
correlated to
+ * its {@link #NORMAL} counterpart using the {@link
DatanodeStorage#storageID}. This
+ * property should be used for debugging purposes only.
+ * </p>
+ */
+ READ_ONLY_SHARED;
}
private final String storageID;
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto?rev=1570084&r1=1570083&r2=1570084&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto
Thu Feb 20 04:38:30 2014
@@ -50,7 +50,7 @@ message DatanodeRegistrationProto {
message DatanodeStorageProto {
enum StorageState {
NORMAL = 0;
- READ_ONLY = 1;
+ READ_ONLY_SHARED = 1;
}
required string storageUuid = 1;
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DataNodeCluster.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DataNodeCluster.java?rev=1570084&r1=1570083&r2=1570084&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DataNodeCluster.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DataNodeCluster.java
Thu Feb 20 04:38:30 2014
@@ -68,9 +68,10 @@ public class DataNodeCluster {
static String dataNodeDirs = DATANODE_DIRS;
static final String USAGE =
"Usage: datanodecluster " +
- " -n <numDataNodes> " +
+ " -n <numDataNodes> " +
+ " -bpid <bpid>" +
" [-racks <numRacks>] " +
- " [-simulated] " +
+ " [-simulated [<simulatedCapacityPerDn>]] " +
" [-inject startingBlockId numBlocksPerDN]" +
" [-r replicationFactorForInjectedBlocks]" +
" [-d dataNodeDirs]\n" +
@@ -91,7 +92,7 @@ public class DataNodeCluster {
printUsageExit();
}
- public static void main(String[] args) {
+ public static void main(String[] args) throws InterruptedException {
int numDataNodes = 0;
int numRacks = 0;
boolean inject = false;
@@ -99,6 +100,8 @@ public class DataNodeCluster {
int numBlocksPerDNtoInject = 0;
int replication = 1;
boolean checkDataNodeAddrConfig = false;
+ long simulatedCapacityPerDn = SimulatedFSDataset.DEFAULT_CAPACITY;
+ String bpid = null;
Configuration conf = new HdfsConfiguration();
@@ -115,7 +118,7 @@ public class DataNodeCluster {
numRacks = Integer.parseInt(args[i]);
} else if (args[i].equals("-r")) {
if (++i >= args.length || args[i].startsWith("-")) {
- printUsageExit("Missing replicaiton factor");
+ printUsageExit("Missing replication factor");
}
replication = Integer.parseInt(args[i]);
} else if (args[i].equals("-d")) {
@@ -125,6 +128,14 @@ public class DataNodeCluster {
dataNodeDirs = args[i];
} else if (args[i].equals("-simulated")) {
SimulatedFSDataset.setFactory(conf);
+ if ((i+1) < args.length && !args[i+1].startsWith("-")) {
+ simulatedCapacityPerDn = Long.parseLong(args[++i]);
+ }
+ } else if (args[i].equals("-bpid")) {
+ if (++i >= args.length || args[i].startsWith("-")) {
+ printUsageExit("Missing blockpoolid parameter");
+ }
+ bpid = args[i];
} else if (args[i].equals("-inject")) {
if (!FsDatasetSpi.Factory.getFactory(conf).isSimulated()) {
System.out.print("-inject is valid only for simulated");
@@ -153,6 +164,9 @@ public class DataNodeCluster {
printUsageExit("Replication must be less than or equal to numDataNodes");
}
+ if (bpid == null) {
+ printUsageExit("BlockPoolId must be provided");
+ }
String nameNodeAdr = FileSystem.getDefaultUri(conf).getAuthority();
if (nameNodeAdr == null) {
System.out.println("No name node address and port in config");
@@ -162,9 +176,14 @@ public class DataNodeCluster {
System.out.println("Starting " + numDataNodes +
(simulated ? " Simulated " : " ") +
" Data Nodes that will connect to Name Node at " + nameNodeAdr);
-
+
System.setProperty("test.build.data", dataNodeDirs);
+ long simulatedCapacities[] = new long[numDataNodes];
+ for (int i = 0; i < numDataNodes; ++i) {
+ simulatedCapacities[i] = simulatedCapacityPerDn;
+ }
+
MiniDFSCluster mc = new MiniDFSCluster();
try {
mc.formatDataNodeDirs();
@@ -182,13 +201,12 @@ public class DataNodeCluster {
//rack4DataNode[i] = racks[i%numRacks];
rack4DataNode[i] = rackPrefix + "-" + i%numRacks;
System.out.println("Data Node " + i + " using " + rack4DataNode[i]);
-
-
}
}
try {
mc.startDataNodes(conf, numDataNodes, true, StartupOption.REGULAR,
- rack4DataNode, null, null, false, checkDataNodeAddrConfig);
+ rack4DataNode, null, simulatedCapacities, false,
checkDataNodeAddrConfig);
+ Thread.sleep(10*1000); // Give the DN some time to connect to NN and
init storage directories.
if (inject) {
long blockSize = 10;
System.out.println("Injecting " + numBlocksPerDNtoInject +
@@ -203,7 +221,7 @@ public class DataNodeCluster {
}
for (int i = 1; i <= replication; ++i) {
// inject blocks for dn_i into dn_i and replica in dn_i's
neighbors
- mc.injectBlocks((i_dn + i- 1)% numDataNodes,
Arrays.asList(blocks));
+ mc.injectBlocks((i_dn + i- 1)% numDataNodes,
Arrays.asList(blocks), bpid);
System.out.println("Injecting blocks of dn " + i_dn + " into dn"
+
((i_dn + i- 1)% numDataNodes));
}
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java?rev=1570084&r1=1570083&r2=1570084&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
Thu Feb 20 04:38:30 2014
@@ -157,6 +157,7 @@ public class MiniDFSCluster {
private boolean checkExitOnShutdown = true;
private boolean checkDataNodeAddrConfig = false;
private boolean checkDataNodeHostConfig = false;
+ private Configuration[] dnConfOverlays;
public Builder(Configuration conf) {
this.conf = conf;
@@ -334,6 +335,19 @@ public class MiniDFSCluster {
}
/**
+ * Default: null
+ *
+ * An array of {@link Configuration} objects that will overlay the
+ * global MiniDFSCluster Configuration for the corresponding DataNode.
+ *
+ * Useful for setting specific per-DataNode configuration parameters.
+ */
+ public Builder dataNodeConfOverlays(Configuration[] dnConfOverlays) {
+ this.dnConfOverlays = dnConfOverlays;
+ return this;
+ }
+
+ /**
* Construct the actual MiniDFSCluster
*/
public MiniDFSCluster build() throws IOException {
@@ -375,7 +389,8 @@ public class MiniDFSCluster {
builder.nnTopology,
builder.checkExitOnShutdown,
builder.checkDataNodeAddrConfig,
- builder.checkDataNodeHostConfig);
+ builder.checkDataNodeHostConfig,
+ builder.dnConfOverlays);
}
public class DataNodeProperties {
@@ -625,7 +640,7 @@ public class MiniDFSCluster {
manageNameDfsDirs, true, manageDataDfsDirs, manageDataDfsDirs,
operation, null, racks, hosts,
simulatedCapacities, null, true, false,
- MiniDFSNNTopology.simpleSingleNN(nameNodePort, 0), true, false, false);
+ MiniDFSNNTopology.simpleSingleNN(nameNodePort, 0), true, false, false,
null);
}
private void initMiniDFSCluster(
@@ -638,7 +653,8 @@ public class MiniDFSCluster {
boolean waitSafeMode, boolean setupHostsFile,
MiniDFSNNTopology nnTopology, boolean checkExitOnShutdown,
boolean checkDataNodeAddrConfig,
- boolean checkDataNodeHostConfig)
+ boolean checkDataNodeHostConfig,
+ Configuration[] dnConfOverlays)
throws IOException {
ExitUtil.disableSystemExit();
@@ -703,7 +719,7 @@ public class MiniDFSCluster {
startDataNodes(conf, numDataNodes, storageType, manageDataDfsDirs,
dnStartOpt != null ? dnStartOpt : startOpt,
racks, hosts, simulatedCapacities, setupHostsFile,
- checkDataNodeAddrConfig, checkDataNodeHostConfig);
+ checkDataNodeAddrConfig, checkDataNodeHostConfig, dnConfOverlays);
waitClusterUp();
//make sure ProxyUsers uses the latest conf
ProxyUsers.refreshSuperUserGroupsConfiguration(conf);
@@ -1110,7 +1126,7 @@ public class MiniDFSCluster {
long[] simulatedCapacities,
boolean setupHostsFile) throws IOException {
startDataNodes(conf, numDataNodes, StorageType.DEFAULT, manageDfsDirs,
operation, racks, hosts,
- simulatedCapacities, setupHostsFile, false, false);
+ simulatedCapacities, setupHostsFile, false, false, null);
}
/**
@@ -1124,7 +1140,7 @@ public class MiniDFSCluster {
boolean setupHostsFile,
boolean checkDataNodeAddrConfig) throws IOException {
startDataNodes(conf, numDataNodes, StorageType.DEFAULT, manageDfsDirs,
operation, racks, hosts,
- simulatedCapacities, setupHostsFile, checkDataNodeAddrConfig, false);
+ simulatedCapacities, setupHostsFile, checkDataNodeAddrConfig, false,
null);
}
/**
@@ -1151,7 +1167,8 @@ public class MiniDFSCluster {
* @param setupHostsFile add new nodes to dfs hosts files
* @param checkDataNodeAddrConfig if true, only set DataNode port addresses
if not already set in config
* @param checkDataNodeHostConfig if true, only set DataNode hostname key if
not already set in config
- *
+ * @param dnConfOverlays An array of {@link Configuration} objects that will
overlay the
+ * global MiniDFSCluster Configuration for the corresponding
DataNode.
* @throws IllegalStateException if NameNode has been shutdown
*/
public synchronized void startDataNodes(Configuration conf, int numDataNodes,
@@ -1160,7 +1177,8 @@ public class MiniDFSCluster {
long[] simulatedCapacities,
boolean setupHostsFile,
boolean checkDataNodeAddrConfig,
- boolean checkDataNodeHostConfig) throws IOException {
+ boolean checkDataNodeHostConfig,
+ Configuration[] dnConfOverlays) throws IOException {
if (operation == StartupOption.RECOVER) {
return;
}
@@ -1200,6 +1218,13 @@ public class MiniDFSCluster {
+ simulatedCapacities.length
+ "] is less than the number of datanodes [" + numDataNodes + "].");
}
+
+ if (dnConfOverlays != null
+ && numDataNodes > dnConfOverlays.length) {
+ throw new IllegalArgumentException( "The length of dnConfOverlays ["
+ + dnConfOverlays.length
+ + "] is less than the number of datanodes [" + numDataNodes + "].");
+ }
String [] dnArgs = (operation == null ||
operation != StartupOption.ROLLBACK) ?
@@ -1208,6 +1233,9 @@ public class MiniDFSCluster {
for (int i = curDatanodesNum; i < curDatanodesNum+numDataNodes; i++) {
Configuration dnConf = new HdfsConfiguration(conf);
+ if (dnConfOverlays != null) {
+ dnConf.addResource(dnConfOverlays[i]);
+ }
// Set up datanode address
setupDatanodeAddress(dnConf, setupHostsFile, checkDataNodeAddrConfig);
if (manageDfsDirs) {
@@ -2057,17 +2085,19 @@ public class MiniDFSCluster {
return result;
}
-
/**
* This method is valid only if the data nodes have simulated data
* @param dataNodeIndex - data node i which to inject - the index is same as
for getDataNodes()
* @param blocksToInject - the blocks
+ * @param bpid - (optional) the block pool id to use for injecting blocks.
+ * If not supplied then it is queried from the in-process
NameNode.
* @throws IOException
* if not simulatedFSDataset
* if any of blocks already exist in the data node
*
*/
- public void injectBlocks(int dataNodeIndex, Iterable<Block> blocksToInject)
throws IOException {
+ public void injectBlocks(int dataNodeIndex,
+ Iterable<Block> blocksToInject, String bpid) throws IOException {
if (dataNodeIndex < 0 || dataNodeIndex > dataNodes.size()) {
throw new IndexOutOfBoundsException();
}
@@ -2076,7 +2106,9 @@ public class MiniDFSCluster {
if (!(dataSet instanceof SimulatedFSDataset)) {
throw new IOException("injectBlocks is valid only for
SimilatedFSDataset");
}
- String bpid = getNamesystem().getBlockPoolId();
+ if (bpid == null) {
+ bpid = getNamesystem().getBlockPoolId();
+ }
SimulatedFSDataset sdataset = (SimulatedFSDataset) dataSet;
sdataset.injectBlocks(bpid, blocksToInject);
dataNodes.get(dataNodeIndex).datanode.scheduleAllBlockReport(0);
@@ -2102,25 +2134,6 @@ public class MiniDFSCluster {
}
/**
- * This method is valid only if the data nodes have simulated data
- * @param blocksToInject - blocksToInject[] is indexed in the same order as
the list
- * of datanodes returned by getDataNodes()
- * @throws IOException
- * if not simulatedFSDataset
- * if any of blocks already exist in the data nodes
- * Note the rest of the blocks are not injected.
- */
- public void injectBlocks(Iterable<Block>[] blocksToInject)
- throws IOException {
- if (blocksToInject.length > dataNodes.size()) {
- throw new IndexOutOfBoundsException();
- }
- for (int i = 0; i < blocksToInject.length; ++i) {
- injectBlocks(i, blocksToInject[i]);
- }
- }
-
- /**
* Set the softLimit and hardLimit of client lease periods
*/
public void setLeasePeriod(long soft, long hard) {
@@ -2166,11 +2179,13 @@ public class MiniDFSCluster {
* @return the base directory for this instance.
*/
protected String determineDfsBaseDir() {
- String dfsdir = conf.get(HDFS_MINIDFS_BASEDIR, null);
- if (dfsdir == null) {
- dfsdir = getBaseDirectory();
+ if (conf != null) {
+ final String dfsdir = conf.get(HDFS_MINIDFS_BASEDIR, null);
+ if (dfsdir != null) {
+ return dfsdir;
+ }
}
- return dfsdir;
+ return getBaseDirectory();
}
/**
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSClusterWithNodeGroup.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSClusterWithNodeGroup.java?rev=1570084&r1=1570083&r2=1570084&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSClusterWithNodeGroup.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSClusterWithNodeGroup.java
Thu Feb 20 04:38:30 2014
@@ -210,7 +210,8 @@ public class MiniDFSClusterWithNodeGroup
long[] simulatedCapacities,
boolean setupHostsFile,
boolean checkDataNodeAddrConfig,
- boolean checkDataNodeHostConfig) throws IOException {
+ boolean checkDataNodeHostConfig,
+ Configuration[] dnConfOverlays) throws IOException {
startDataNodes(conf, numDataNodes, storageType, manageDfsDirs, operation,
racks,
NODE_GROUPS, hosts, simulatedCapacities, setupHostsFile,
checkDataNodeAddrConfig, checkDataNodeHostConfig);
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestInjectionForSimulatedStorage.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestInjectionForSimulatedStorage.java?rev=1570084&r1=1570083&r2=1570084&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestInjectionForSimulatedStorage.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestInjectionForSimulatedStorage.java
Thu Feb 20 04:38:30 2014
@@ -168,7 +168,7 @@ public class TestInjectionForSimulatedSt
// Insert all the blocks in the first data node
LOG.info("Inserting " + uniqueBlocks.size() + " blocks");
- cluster.injectBlocks(0, uniqueBlocks);
+ cluster.injectBlocks(0, uniqueBlocks, null);
dfsClient = new DFSClient(new InetSocketAddress("localhost",
cluster.getNameNodePort()),
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java?rev=1570084&r1=1570083&r2=1570084&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
Thu Feb 20 04:38:30 2014
@@ -209,7 +209,7 @@ public class TestBalancer {
ClientProtocol.class).getProxy();
for(int i = 0; i < blocksDN.length; i++)
- cluster.injectBlocks(i, Arrays.asList(blocksDN[i]));
+ cluster.injectBlocks(i, Arrays.asList(blocksDN[i]), null);
final long totalCapacity = sum(capacities);
runBalancer(conf, totalUsedSpace, totalCapacity);
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java?rev=1570084&r1=1570083&r2=1570084&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
Thu Feb 20 04:38:30 2014
@@ -34,6 +34,7 @@ import javax.management.StandardMBean;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.StorageType;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
import org.apache.hadoop.hdfs.protocol.BlockLocalPathInfo;
@@ -96,6 +97,11 @@ public class SimulatedFSDataset implemen
public static final long DEFAULT_CAPACITY = 2L<<40; // 1 terabyte
public static final byte DEFAULT_DATABYTE = 9;
+ public static final String CONFIG_PROPERTY_STATE =
+ "dfs.datanode.simulateddatastorage.state";
+ private static final DatanodeStorage.State DEFAULT_STATE =
+ DatanodeStorage.State.NORMAL;
+
static final byte[] nullCrcFileData;
static {
DataChecksum checksum = DataChecksum.newDataChecksum(
@@ -325,9 +331,9 @@ public class SimulatedFSDataset implemen
private static class SimulatedStorage {
private Map<String, SimulatedBPStorage> map =
new HashMap<String, SimulatedBPStorage>();
- private final String storageUuid = "SimulatedStroage-" +
DatanodeStorage.generateUuid();
private final long capacity; // in bytes
+ private final DatanodeStorage dnStorage;
synchronized long getFree() {
return capacity - getUsed();
@@ -365,8 +371,11 @@ public class SimulatedFSDataset implemen
getBPStorage(bpid).free(amount);
}
- SimulatedStorage(long cap) {
+ SimulatedStorage(long cap, DatanodeStorage.State state) {
capacity = cap;
+ dnStorage = new DatanodeStorage(
+ "SimulatedStorage-" + DatanodeStorage.generateUuid(),
+ state, StorageType.DEFAULT);
}
synchronized void addBlockPool(String bpid) {
@@ -390,11 +399,15 @@ public class SimulatedFSDataset implemen
}
String getStorageUuid() {
- return storageUuid;
+ return dnStorage.getStorageID();
+ }
+
+ DatanodeStorage getDnStorage() {
+ return dnStorage;
}
synchronized StorageReport getStorageReport(String bpid) {
- return new StorageReport(new DatanodeStorage(getStorageUuid()),
+ return new StorageReport(dnStorage,
false, getCapacity(), getUsed(), getFree(),
map.get(bpid).getUsed());
}
@@ -417,7 +430,8 @@ public class SimulatedFSDataset implemen
registerMBean(datanodeUuid);
this.storage = new SimulatedStorage(
- conf.getLong(CONFIG_PROPERTY_CAPACITY, DEFAULT_CAPACITY));
+ conf.getLong(CONFIG_PROPERTY_CAPACITY, DEFAULT_CAPACITY),
+ conf.getEnum(CONFIG_PROPERTY_STATE, DEFAULT_STATE));
}
public synchronized void injectBlocks(String bpid,
@@ -488,7 +502,7 @@ public class SimulatedFSDataset implemen
@Override
public synchronized Map<DatanodeStorage, BlockListAsLongs> getBlockReports(
String bpid) {
- return Collections.singletonMap(new DatanodeStorage(storage.storageUuid),
getBlockReport(bpid));
+ return Collections.singletonMap(storage.getDnStorage(),
getBlockReport(bpid));
}
@Override // FsDatasetSpi
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java?rev=1570084&r1=1570083&r2=1570084&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
Thu Feb 20 04:38:30 2014
@@ -57,7 +57,7 @@ public class CreateEditsLog {
GenerationStamp.LAST_RESERVED_STAMP;
static void addFiles(FSEditLog editLog, int numFiles, short replication,
- int blocksPerFile, long startingBlockId,
+ int blocksPerFile, long startingBlockId, long
blockSize,
FileNameGenerator nameGenerator) {
PermissionStatus p = new PermissionStatus("joeDoe", "people",
@@ -66,7 +66,6 @@ public class CreateEditsLog {
INodeDirectory dirInode = new INodeDirectory(inodeId.nextValue(), null, p,
0L);
editLog.logMkDir(BASE_PATH, dirInode);
- long blockSize = 10;
BlockInfo[] blocks = new BlockInfo[blocksPerFile];
for (int iB = 0; iB < blocksPerFile; ++iB) {
blocks[iB] =
@@ -144,6 +143,7 @@ public class CreateEditsLog {
int numFiles = 0;
short replication = 1;
int numBlocksPerFile = 0;
+ long blockSize = 10;
if (args.length == 0) {
printUsageExit();
@@ -164,10 +164,16 @@ public class CreateEditsLog {
if (numFiles <=0 || numBlocksPerFile <= 0) {
printUsageExit("numFiles and numBlocksPerFile most be greater than
0");
}
+ } else if (args[i].equals("-l")) {
+ if (i + 1 >= args.length) {
+ printUsageExit(
+ "Missing block length");
+ }
+ blockSize = Long.parseLong(args[++i]);
} else if (args[i].equals("-r") || args[i+1].startsWith("-")) {
if (i + 1 >= args.length) {
printUsageExit(
- "Missing num files, starting block and/or number of blocks");
+ "Missing replication factor");
}
replication = Short.parseShort(args[++i]);
} else if (args[i].equals("-d")) {
@@ -202,7 +208,7 @@ public class CreateEditsLog {
FSEditLog editLog = FSImageTestUtil.createStandaloneEditLog(editsLogDir);
editLog.openForWrite();
addFiles(editLog, numFiles, replication, numBlocksPerFile, startingBlockId,
- nameGenerator);
+ blockSize, nameGenerator);
editLog.logSync();
editLog.close();
}
