Author: arp
Date: Wed Feb 19 23:12:34 2014
New Revision: 1569964
URL: http://svn.apache.org/r1569964
Log:
HDFS-5318. Merging r1569959 from branch-2 to branch-2.4.
Added:
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestReadOnlySharedStorage.java
- copied unchanged from r1569959,
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestReadOnlySharedStorage.java
Modified:
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeStorage.java
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSClusterWithNodeGroup.java
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
Modified:
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1569964&r1=1569963&r2=1569964&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Wed Feb 19 23:12:34 2014
@@ -77,6 +77,9 @@ Release 2.4.0 - UNRELEASED
HDFS-5973. add DomainSocket#shutdown method (cmccabe)
+ HDFS-5318. Support read-only and read-write paths to shared replicas.
+ (Eric Sirianni via Arpit Agarwal)
+
OPTIMIZATIONS
HDFS-5790. LeaseManager.findPath is very slow when many leases need
recovery
Modified:
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java?rev=1569964&r1=1569963&r2=1569964&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
(original)
+++
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
Wed Feb 19 23:12:34 2014
@@ -1497,8 +1497,8 @@ public class PBHelper {
private static StorageState convertState(State state) {
switch(state) {
- case READ_ONLY:
- return StorageState.READ_ONLY;
+ case READ_ONLY_SHARED:
+ return StorageState.READ_ONLY_SHARED;
case NORMAL:
default:
return StorageState.NORMAL;
@@ -1526,8 +1526,8 @@ public class PBHelper {
private static State convertState(StorageState state) {
switch(state) {
- case READ_ONLY:
- return DatanodeStorage.State.READ_ONLY;
+ case READ_ONLY_SHARED:
+ return DatanodeStorage.State.READ_ONLY_SHARED;
case NORMAL:
default:
return DatanodeStorage.State.NORMAL;
Modified:
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java?rev=1569964&r1=1569963&r2=1569964&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
(original)
+++
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
Wed Feb 19 23:12:34 2014
@@ -73,6 +73,7 @@ import org.apache.hadoop.hdfs.server.pro
import
org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations.BlockWithLocations;
import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage.State;
import org.apache.hadoop.hdfs.server.protocol.KeyUpdateCommand;
import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo;
import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
@@ -482,7 +483,10 @@ public class BlockManager {
chooseSourceDatanode(block, containingNodes,
containingLiveReplicasNodes, numReplicas,
UnderReplicatedBlocks.LEVEL);
- assert containingLiveReplicasNodes.size() == numReplicas.liveReplicas();
+
+ // containingLiveReplicasNodes can include READ_ONLY_SHARED replicas which
are
+ // not included in the numReplicas.liveReplicas() count
+ assert containingLiveReplicasNodes.size() >= numReplicas.liveReplicas();
int usableReplicas = numReplicas.liveReplicas() +
numReplicas.decommissionedReplicas();
@@ -1021,7 +1025,7 @@ public class BlockManager {
*/
private void addToInvalidates(Block b) {
StringBuilder datanodes = new StringBuilder();
- for(DatanodeStorageInfo storage : blocksMap.getStorages(b)) {
+ for(DatanodeStorageInfo storage : blocksMap.getStorages(b, State.NORMAL)) {
final DatanodeDescriptor node = storage.getDatanodeDescriptor();
invalidateBlocks.add(b, node, false);
datanodes.append(node).append(" ");
@@ -1234,7 +1238,10 @@ public class BlockManager {
continue;
}
- assert liveReplicaNodes.size() == numReplicas.liveReplicas();
+ // liveReplicaNodes can include READ_ONLY_SHARED replicas which
are
+ // not included in the numReplicas.liveReplicas() count
+ assert liveReplicaNodes.size() >= numReplicas.liveReplicas();
+
// do not schedule more if enough replicas is already pending
numEffectiveReplicas = numReplicas.liveReplicas() +
pendingReplications.getNumReplicas(block);
@@ -1478,15 +1485,16 @@ public class BlockManager {
final DatanodeDescriptor node = storage.getDatanodeDescriptor();
LightWeightLinkedSet<Block> excessBlocks =
excessReplicateMap.get(node.getDatanodeUuid());
+ int countableReplica = storage.getState() == State.NORMAL ? 1 : 0;
if ((nodesCorrupt != null) && (nodesCorrupt.contains(node)))
- corrupt++;
+ corrupt += countableReplica;
else if (node.isDecommissionInProgress() || node.isDecommissioned())
- decommissioned++;
+ decommissioned += countableReplica;
else if (excessBlocks != null && excessBlocks.contains(block)) {
- excess++;
+ excess += countableReplica;
} else {
nodesContainingLiveReplicas.add(storage);
- live++;
+ live += countableReplica;
}
containingNodes.add(node);
// Check if this replica is corrupt
@@ -2483,7 +2491,7 @@ assert storedBlock.findDatanode(dn) < 0
Collection<DatanodeDescriptor> nonExcess = new
ArrayList<DatanodeDescriptor>();
Collection<DatanodeDescriptor> corruptNodes = corruptReplicas
.getNodes(block);
- for(DatanodeStorageInfo storage : blocksMap.getStorages(block)) {
+ for(DatanodeStorageInfo storage : blocksMap.getStorages(block,
State.NORMAL)) {
final DatanodeDescriptor cur = storage.getDatanodeDescriptor();
if (storage.areBlockContentsStale()) {
LOG.info("BLOCK* processOverReplicatedBlock: " +
@@ -2812,7 +2820,7 @@ assert storedBlock.findDatanode(dn) < 0
int excess = 0;
int stale = 0;
Collection<DatanodeDescriptor> nodesCorrupt = corruptReplicas.getNodes(b);
- for(DatanodeStorageInfo storage : blocksMap.getStorages(b)) {
+ for(DatanodeStorageInfo storage : blocksMap.getStorages(b, State.NORMAL)) {
final DatanodeDescriptor node = storage.getDatanodeDescriptor();
if ((nodesCorrupt != null) && (nodesCorrupt.contains(node))) {
corrupt++;
@@ -2851,7 +2859,7 @@ assert storedBlock.findDatanode(dn) < 0
// else proceed with fast case
int live = 0;
Collection<DatanodeDescriptor> nodesCorrupt = corruptReplicas.getNodes(b);
- for(DatanodeStorageInfo storage : blocksMap.getStorages(b)) {
+ for(DatanodeStorageInfo storage : blocksMap.getStorages(b, State.NORMAL)) {
final DatanodeDescriptor node = storage.getDatanodeDescriptor();
if ((nodesCorrupt == null) || (!nodesCorrupt.contains(node)))
live++;
Modified:
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java?rev=1569964&r1=1569963&r2=1569964&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
(original)
+++
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
Wed Feb 19 23:12:34 2014
@@ -605,7 +605,7 @@ public class BlockPlacementPolicyDefault
+ storageType);
return false;
}
- if (storage.getState() == State.READ_ONLY) {
+ if (storage.getState() == State.READ_ONLY_SHARED) {
logNodeIsNotChosen(storage, "storage is read-only");
return false;
}
Modified:
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java?rev=1569964&r1=1569963&r2=1569964&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
(original)
+++
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
Wed Feb 19 23:12:34 2014
@@ -20,9 +20,13 @@ package org.apache.hadoop.hdfs.server.bl
import java.util.Iterator;
import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
import org.apache.hadoop.util.GSet;
import org.apache.hadoop.util.LightWeightGSet;
+import com.google.common.base.Predicate;
+import com.google.common.collect.Iterables;
+
/**
* This class maintains the map from a block to its metadata.
* block's metadata currently includes blockCollection it belongs to and
@@ -122,6 +126,22 @@ class BlocksMap {
}
/**
+ * Searches for the block in the BlocksMap and
+ * returns {@link Iterable} of the storages the block belongs to
+ * <i>that are of the given {@link DatanodeStorage.State state}</i>.
+ *
+ * @param state DatanodeStorage state by which to filter the returned
Iterable
+ */
+ Iterable<DatanodeStorageInfo> getStorages(Block b, final
DatanodeStorage.State state) {
+ return Iterables.filter(getStorages(blocks.get(b)), new
Predicate<DatanodeStorageInfo>() {
+ @Override
+ public boolean apply(DatanodeStorageInfo storage) {
+ return storage.getState() == state;
+ }
+ });
+ }
+
+ /**
* For a block that has already been retrieved from the BlocksMap
* returns {@link Iterable} of the storages the block belongs to.
*/
Modified:
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java?rev=1569964&r1=1569963&r2=1569964&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
(original)
+++
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
Wed Feb 19 23:12:34 2014
@@ -58,6 +58,8 @@ import org.apache.hadoop.hdfs.protocol.L
import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementStatus;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.NumberReplicas;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.datanode.CachingStrategy;
import org.apache.hadoop.net.NetUtils;
@@ -378,11 +380,13 @@ public class NamenodeFsck {
boolean isCorrupt = lBlk.isCorrupt();
String blkName = block.toString();
DatanodeInfo[] locs = lBlk.getLocations();
- res.totalReplicas += locs.length;
+ NumberReplicas numberReplicas =
namenode.getNamesystem().getBlockManager().countNodes(block.getLocalBlock());
+ int liveReplicas = numberReplicas.liveReplicas();
+ res.totalReplicas += liveReplicas;
short targetFileReplication = file.getReplication();
res.numExpectedReplicas += targetFileReplication;
- if (locs.length > targetFileReplication) {
- res.excessiveReplicas += (locs.length - targetFileReplication);
+ if (liveReplicas > targetFileReplication) {
+ res.excessiveReplicas += (liveReplicas - targetFileReplication);
res.numOverReplicatedBlocks += 1;
}
// Check if block is Corrupt
@@ -392,10 +396,10 @@ public class NamenodeFsck {
out.print("\n" + path + ": CORRUPT blockpool " +
block.getBlockPoolId() +
" block " + block.getBlockName()+"\n");
}
- if (locs.length >= minReplication)
+ if (liveReplicas >= minReplication)
res.numMinReplicatedBlocks++;
- if (locs.length < targetFileReplication && locs.length > 0) {
- res.missingReplicas += (targetFileReplication - locs.length);
+ if (liveReplicas < targetFileReplication && liveReplicas > 0) {
+ res.missingReplicas += (targetFileReplication - liveReplicas);
res.numUnderReplicatedBlocks += 1;
underReplicatedPerFile++;
if (!showFiles) {
@@ -404,7 +408,7 @@ public class NamenodeFsck {
out.println(" Under replicated " + block +
". Target Replicas is " +
targetFileReplication + " but found " +
- locs.length + " replica(s).");
+ liveReplicas + " replica(s).");
}
// verify block placement policy
BlockPlacementStatus blockPlacementStatus = bpPolicy
@@ -421,13 +425,13 @@ public class NamenodeFsck {
block + ". " + blockPlacementStatus.getErrorDescription());
}
report.append(i + ". " + blkName + " len=" + block.getNumBytes());
- if (locs.length == 0) {
+ if (liveReplicas == 0) {
report.append(" MISSING!");
res.addMissing(block.toString(), block.getNumBytes());
missing++;
missize += block.getNumBytes();
} else {
- report.append(" repl=" + locs.length);
+ report.append(" repl=" + liveReplicas);
if (showLocations || showRacks) {
StringBuilder sb = new StringBuilder("[");
for (int j = 0; j < locs.length; j++) {
Modified:
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeStorage.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeStorage.java?rev=1569964&r1=1569963&r2=1569964&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeStorage.java
(original)
+++
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeStorage.java
Wed Feb 19 23:12:34 2014
@@ -28,7 +28,18 @@ public class DatanodeStorage {
/** The state of the storage. */
public enum State {
NORMAL,
- READ_ONLY
+
+ /**
+ * A storage that represents a read-only path to replicas stored on a
shared storage device.
+ * Replicas on {@link #READ_ONLY_SHARED} storage are not counted towards
live replicas.
+ *
+ * <p>
+ * In certain implementations, a {@link #READ_ONLY_SHARED} storage may be
correlated to
+ * its {@link #NORMAL} counterpart using the {@link
DatanodeStorage#storageID}. This
+ * property should be used for debugging purposes only.
+ * </p>
+ */
+ READ_ONLY_SHARED;
}
private final String storageID;
Modified:
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto?rev=1569964&r1=1569963&r2=1569964&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto
(original)
+++
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto
Wed Feb 19 23:12:34 2014
@@ -49,7 +49,7 @@ message DatanodeRegistrationProto {
message DatanodeStorageProto {
enum StorageState {
NORMAL = 0;
- READ_ONLY = 1;
+ READ_ONLY_SHARED = 1;
}
required string storageUuid = 1;
Modified:
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java?rev=1569964&r1=1569963&r2=1569964&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
(original)
+++
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
Wed Feb 19 23:12:34 2014
@@ -157,6 +157,7 @@ public class MiniDFSCluster {
private boolean checkExitOnShutdown = true;
private boolean checkDataNodeAddrConfig = false;
private boolean checkDataNodeHostConfig = false;
+ private Configuration[] dnConfOverlays;
public Builder(Configuration conf) {
this.conf = conf;
@@ -326,6 +327,19 @@ public class MiniDFSCluster {
}
/**
+ * Default: null
+ *
+ * An array of {@link Configuration} objects that will overlay the
+ * global MiniDFSCluster Configuration for the corresponding DataNode.
+ *
+ * Useful for setting specific per-DataNode configuration parameters.
+ */
+ public Builder dataNodeConfOverlays(Configuration[] dnConfOverlays) {
+ this.dnConfOverlays = dnConfOverlays;
+ return this;
+ }
+
+ /**
* Construct the actual MiniDFSCluster
*/
public MiniDFSCluster build() throws IOException {
@@ -366,7 +380,8 @@ public class MiniDFSCluster {
builder.nnTopology,
builder.checkExitOnShutdown,
builder.checkDataNodeAddrConfig,
- builder.checkDataNodeHostConfig);
+ builder.checkDataNodeHostConfig,
+ builder.dnConfOverlays);
}
public class DataNodeProperties {
@@ -604,7 +619,7 @@ public class MiniDFSCluster {
manageNameDfsDirs, true, true, manageDataDfsDirs,
operation, racks, hosts,
simulatedCapacities, null, true, false,
- MiniDFSNNTopology.simpleSingleNN(nameNodePort, 0), true, false, false);
+ MiniDFSNNTopology.simpleSingleNN(nameNodePort, 0), true, false, false,
null);
}
private void initMiniDFSCluster(
@@ -616,7 +631,8 @@ public class MiniDFSCluster {
boolean waitSafeMode, boolean setupHostsFile,
MiniDFSNNTopology nnTopology, boolean checkExitOnShutdown,
boolean checkDataNodeAddrConfig,
- boolean checkDataNodeHostConfig)
+ boolean checkDataNodeHostConfig,
+ Configuration[] dnConfOverlays)
throws IOException {
ExitUtil.disableSystemExit();
@@ -681,8 +697,9 @@ public class MiniDFSCluster {
}
// Start the DataNodes
- startDataNodes(conf, numDataNodes, storageType, manageDataDfsDirs,
operation, racks,
- hosts, simulatedCapacities, setupHostsFile, checkDataNodeAddrConfig,
checkDataNodeHostConfig);
+ startDataNodes(conf, numDataNodes, storageType, manageDataDfsDirs,
+ operation, racks, hosts, simulatedCapacities, setupHostsFile,
+ checkDataNodeAddrConfig, checkDataNodeHostConfig, dnConfOverlays);
waitClusterUp();
//make sure ProxyUsers uses the latest conf
ProxyUsers.refreshSuperUserGroupsConfiguration(conf);
@@ -1077,7 +1094,7 @@ public class MiniDFSCluster {
long[] simulatedCapacities,
boolean setupHostsFile) throws IOException {
startDataNodes(conf, numDataNodes, StorageType.DEFAULT, manageDfsDirs,
operation, racks, hosts,
- simulatedCapacities, setupHostsFile, false, false);
+ simulatedCapacities, setupHostsFile, false, false, null);
}
/**
@@ -1091,7 +1108,7 @@ public class MiniDFSCluster {
boolean setupHostsFile,
boolean checkDataNodeAddrConfig) throws IOException {
startDataNodes(conf, numDataNodes, StorageType.DEFAULT, manageDfsDirs,
operation, racks, hosts,
- simulatedCapacities, setupHostsFile, checkDataNodeAddrConfig, false);
+ simulatedCapacities, setupHostsFile, checkDataNodeAddrConfig, false,
null);
}
/**
@@ -1118,7 +1135,8 @@ public class MiniDFSCluster {
* @param setupHostsFile add new nodes to dfs hosts files
* @param checkDataNodeAddrConfig if true, only set DataNode port addresses
if not already set in config
* @param checkDataNodeHostConfig if true, only set DataNode hostname key if
not already set in config
- *
+ * @param dnConfOverlays An array of {@link Configuration} objects that will
overlay the
+ * global MiniDFSCluster Configuration for the corresponding
DataNode.
* @throws IllegalStateException if NameNode has been shutdown
*/
public synchronized void startDataNodes(Configuration conf, int numDataNodes,
@@ -1127,7 +1145,8 @@ public class MiniDFSCluster {
long[] simulatedCapacities,
boolean setupHostsFile,
boolean checkDataNodeAddrConfig,
- boolean checkDataNodeHostConfig) throws IOException {
+ boolean checkDataNodeHostConfig,
+ Configuration[] dnConfOverlays) throws IOException {
if (operation == StartupOption.RECOVER) {
return;
}
@@ -1167,6 +1186,13 @@ public class MiniDFSCluster {
+ simulatedCapacities.length
+ "] is less than the number of datanodes [" + numDataNodes + "].");
}
+
+ if (dnConfOverlays != null
+ && numDataNodes > dnConfOverlays.length) {
+ throw new IllegalArgumentException( "The length of dnConfOverlays ["
+ + dnConfOverlays.length
+ + "] is less than the number of datanodes [" + numDataNodes + "].");
+ }
String [] dnArgs = (operation == null ||
operation != StartupOption.ROLLBACK) ?
@@ -1175,6 +1201,9 @@ public class MiniDFSCluster {
for (int i = curDatanodesNum; i < curDatanodesNum+numDataNodes; i++) {
Configuration dnConf = new HdfsConfiguration(conf);
+ if (dnConfOverlays != null) {
+ dnConf.addResource(dnConfOverlays[i]);
+ }
// Set up datanode address
setupDatanodeAddress(dnConf, setupHostsFile, checkDataNodeAddrConfig);
if (manageDfsDirs) {
Modified:
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSClusterWithNodeGroup.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSClusterWithNodeGroup.java?rev=1569964&r1=1569963&r2=1569964&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSClusterWithNodeGroup.java
(original)
+++
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSClusterWithNodeGroup.java
Wed Feb 19 23:12:34 2014
@@ -210,7 +210,8 @@ public class MiniDFSClusterWithNodeGroup
long[] simulatedCapacities,
boolean setupHostsFile,
boolean checkDataNodeAddrConfig,
- boolean checkDataNodeHostConfig) throws IOException {
+ boolean checkDataNodeHostConfig,
+ Configuration[] dnConfOverlays) throws IOException {
startDataNodes(conf, numDataNodes, storageType, manageDfsDirs, operation,
racks,
NODE_GROUPS, hosts, simulatedCapacities, setupHostsFile,
checkDataNodeAddrConfig, checkDataNodeHostConfig);
Modified:
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java?rev=1569964&r1=1569963&r2=1569964&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
(original)
+++
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
Wed Feb 19 23:12:34 2014
@@ -34,6 +34,7 @@ import javax.management.StandardMBean;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.StorageType;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
import org.apache.hadoop.hdfs.protocol.BlockLocalPathInfo;
@@ -96,6 +97,11 @@ public class SimulatedFSDataset implemen
public static final long DEFAULT_CAPACITY = 2L<<40; // 1 terabyte
public static final byte DEFAULT_DATABYTE = 9;
+ public static final String CONFIG_PROPERTY_STATE =
+ "dfs.datanode.simulateddatastorage.state";
+ private static final DatanodeStorage.State DEFAULT_STATE =
+ DatanodeStorage.State.NORMAL;
+
static final byte[] nullCrcFileData;
static {
DataChecksum checksum = DataChecksum.newDataChecksum(
@@ -325,9 +331,9 @@ public class SimulatedFSDataset implemen
private static class SimulatedStorage {
private Map<String, SimulatedBPStorage> map =
new HashMap<String, SimulatedBPStorage>();
- private final String storageUuid = "SimulatedStroage-" +
DatanodeStorage.generateUuid();
private final long capacity; // in bytes
+ private final DatanodeStorage dnStorage;
synchronized long getFree() {
return capacity - getUsed();
@@ -365,8 +371,11 @@ public class SimulatedFSDataset implemen
getBPStorage(bpid).free(amount);
}
- SimulatedStorage(long cap) {
+ SimulatedStorage(long cap, DatanodeStorage.State state) {
capacity = cap;
+ dnStorage = new DatanodeStorage(
+ "SimulatedStorage-" + DatanodeStorage.generateUuid(),
+ state, StorageType.DEFAULT);
}
synchronized void addBlockPool(String bpid) {
@@ -390,11 +399,15 @@ public class SimulatedFSDataset implemen
}
String getStorageUuid() {
- return storageUuid;
+ return dnStorage.getStorageID();
+ }
+
+ DatanodeStorage getDnStorage() {
+ return dnStorage;
}
synchronized StorageReport getStorageReport(String bpid) {
- return new StorageReport(new DatanodeStorage(getStorageUuid()),
+ return new StorageReport(dnStorage,
false, getCapacity(), getUsed(), getFree(),
map.get(bpid).getUsed());
}
@@ -417,7 +430,8 @@ public class SimulatedFSDataset implemen
registerMBean(datanodeUuid);
this.storage = new SimulatedStorage(
- conf.getLong(CONFIG_PROPERTY_CAPACITY, DEFAULT_CAPACITY));
+ conf.getLong(CONFIG_PROPERTY_CAPACITY, DEFAULT_CAPACITY),
+ conf.getEnum(CONFIG_PROPERTY_STATE, DEFAULT_STATE));
}
public synchronized void injectBlocks(String bpid,
@@ -488,7 +502,7 @@ public class SimulatedFSDataset implemen
@Override
public synchronized Map<DatanodeStorage, BlockListAsLongs> getBlockReports(
String bpid) {
- return Collections.singletonMap(new DatanodeStorage(storage.storageUuid),
getBlockReport(bpid));
+ return Collections.singletonMap(storage.getDnStorage(),
getBlockReport(bpid));
}
@Override // FsDatasetSpi
