http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec80de3c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java index dff8482..db5042b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java @@ -68,18 +68,18 @@ import org.apache.hadoop.util.DataChecksum; /** * This class implements a simulated FSDataset. - * + * * Blocks that are created are recorded but their data (plus their CRCs) are * discarded. * Fixed data is returned when blocks are read; a null CRC meta file is * created for such data. - * + * * This FSDataset does not remember any block information across its * restarts; it does however offer an operation to inject blocks * (See the TestInectionForSImulatedStorage() * for a usage example of injection. - * - * Note the synchronization is coarse grained - it is at each method. + * + * Note the synchronization is coarse grained - it is at each method. */ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> { public final static int BYTE_MASK = 0xff; @@ -95,7 +95,7 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> { return true; } } - + public static void setFactory(Configuration conf) { conf.set(DFSConfigKeys.DFS_DATANODE_FSDATASET_FACTORY_KEY, Factory.class.getName()); @@ -105,12 +105,12 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> { byte firstByte = (byte) (b.getBlockId() & BYTE_MASK); return (byte) ((firstByte + offsetInBlk) & BYTE_MASK); } - + public static final String CONFIG_PROPERTY_CAPACITY = "dfs.datanode.simulateddatastorage.capacity"; - + public static final long DEFAULT_CAPACITY = 2L<<40; // 1 terabyte - + public static final String CONFIG_PROPERTY_STATE = "dfs.datanode.simulateddatastorage.state"; private static final DatanodeStorage.State DEFAULT_STATE = @@ -145,7 +145,7 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> { if (theBlock.getNumBytes() < 0) { theBlock.setNumBytes(0); } - if (!storage.alloc(bpid, theBlock.getNumBytes())) { + if (!storage.alloc(bpid, theBlock.getNumBytes())) { // expected length - actual length may // be more - we find out at finalize DataNode.LOG.warn("Lack of free storage on a block alloc"); @@ -160,7 +160,7 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> { oStream = null; } } - + @Override public String getStorageUuid() { return storage.getStorageUuid(); @@ -188,7 +188,7 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> { theBlock.setNumBytes(length); } } - + synchronized SimulatedInputStream getIStream() { if (!finalized) { // throw new IOException("Trying to read an unfinalized block"); @@ -197,12 +197,12 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> { return new SimulatedInputStream(theBlock.getNumBytes(), theBlock); } } - + synchronized void finalizeBlock(String bpid, long finalSize) throws IOException { if (finalized) { throw new IOException( - "Finalizing a block that has already been finalized" + + "Finalizing a block that has already been finalized" + theBlock.getBlockId()); } if (oStream == null) { @@ -216,7 +216,7 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> { throw new IOException( "Size passed to finalize does not match the amount of data written"); } - // We had allocated the expected length when block was created; + // We had allocated the expected length when block was created; // adjust if necessary long extraLen = finalSize - theBlock.getNumBytes(); if (extraLen > 0) { @@ -227,7 +227,7 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> { } else { storage.free(bpid, -extraLen); } - theBlock.setNumBytes(finalSize); + theBlock.setNumBytes(finalSize); finalized = true; oStream = null; @@ -248,7 +248,7 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> { } SimulatedInputStream getMetaIStream() { - return new SimulatedInputStream(nullCrcFileData); + return new SimulatedInputStream(nullCrcFileData); } synchronized boolean isFinalized() { @@ -257,8 +257,7 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> { @Override synchronized public ReplicaOutputStreams createStreams(boolean isCreate, - DataChecksum requestedChecksum) - throws IOException { + DataChecksum requestedChecksum) throws IOException { if (finalized) { throw new IOException("Trying to write to a finalized replica " + theBlock); @@ -333,31 +332,31 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> { return false; } } - + /** * Class is used for tracking block pool storage utilization similar * to {@link BlockPoolSlice} */ private static class SimulatedBPStorage { private long used; // in bytes - + long getUsed() { return used; } - + void alloc(long amount) { used += amount; } - + void free(long amount) { used -= amount; } - + SimulatedBPStorage() { - used = 0; + used = 0; } } - + /** * Class used for tracking datanode level storage utilization similar * to {@link FSVolumeSet} @@ -368,15 +367,15 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> { private final long capacity; // in bytes private final DatanodeStorage dnStorage; - + synchronized long getFree() { return capacity - getUsed(); } - + long getCapacity() { return capacity; } - + synchronized long getUsed() { long used = 0; for (SimulatedBPStorage bpStorage : map.values()) { @@ -384,11 +383,11 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> { } return used; } - + synchronized long getBlockPoolUsed(String bpid) throws IOException { return getBPStorage(bpid).getUsed(); } - + int getNumFailedVolumes() { return 0; } @@ -398,20 +397,20 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> { getBPStorage(bpid).alloc(amount); return true; } - return false; + return false; } - + synchronized void free(String bpid, long amount) throws IOException { getBPStorage(bpid).free(amount); } - + SimulatedStorage(long cap, DatanodeStorage.State state) { capacity = cap; dnStorage = new DatanodeStorage( "SimulatedStorage-" + DatanodeStorage.generateUuid(), state, StorageType.DEFAULT); } - + synchronized void addBlockPool(String bpid) { SimulatedBPStorage bpStorage = map.get(bpid); if (bpStorage != null) { @@ -419,11 +418,11 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> { } map.put(bpid, new SimulatedBPStorage()); } - + synchronized void removeBlockPool(String bpid) { map.remove(bpid); } - + private SimulatedBPStorage getBPStorage(String bpid) throws IOException { SimulatedBPStorage bpStorage = map.get(bpid); if (bpStorage == null) { @@ -435,7 +434,7 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> { String getStorageUuid() { return dnStorage.getStorageID(); } - + DatanodeStorage getDnStorage() { return dnStorage; } @@ -462,7 +461,17 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> { @Override public FsVolumeReference obtainReference() throws ClosedChannelException { - return null; + return new FsVolumeReference() { + @Override + public void close() throws IOException { + // no-op. + } + + @Override + public FsVolumeSpi getVolume() { + return SimulatedVolume.this; + } + }; } @Override @@ -562,7 +571,7 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> { private final SimulatedVolume volume; private final String datanodeUuid; private final DataNode datanode; - + public SimulatedFSDataset(DataStorage storage, Configuration conf) { this(null, storage, conf); @@ -611,14 +620,14 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> { map = new HashMap<Block, BInfo>(); blockMap.put(bpid, map); } - + for (Block b: injectBlocks) { BInfo binfo = new BInfo(bpid, b, false); map.put(binfo.theBlock, binfo); } } } - + /** Get a map for a given block pool Id */ private Map<Block, BInfo> getMap(String bpid) throws IOException { final Map<Block, BInfo> map = blockMap.get(bpid); @@ -684,7 +693,7 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> { public long getBlockPoolUsed(String bpid) throws IOException { return storage.getBlockPoolUsed(bpid); } - + @Override // FSDatasetMBean public long getRemaining() { return storage.getFree(); @@ -775,7 +784,7 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> { return null; } - @Override + @Override public synchronized String getReplicaString(String bpid, long blockId) { Replica r = null; final Map<Block, BInfo> map = blockMap.get(bpid); @@ -865,14 +874,14 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> { * * @throws ReplicaNotFoundException If the replica is not found * - * @throws UnexpectedReplicaStateException If the replica is not in the + * @throws UnexpectedReplicaStateException If the replica is not in the * expected state. */ @Override // {@link FsDatasetSpi} public void checkBlock(ExtendedBlock b, long minLength, ReplicaState state) throws ReplicaNotFoundException, UnexpectedReplicaStateException { final BInfo binfo = getBInfo(b); - + if (binfo == null) { throw new ReplicaNotFoundException(b); } @@ -956,7 +965,7 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> { map.put(binfo.theBlock, binfo); return binfo; } - + @Override // FsDatasetSpi public synchronized ReplicaHandler recoverRbw( ExtendedBlock b, long newGS, long minBytesRcvd, long maxBytesRcvd) @@ -988,11 +997,11 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> { public synchronized ReplicaHandler createTemporary( StorageType storageType, ExtendedBlock b) throws IOException { if (isValidBlock(b)) { - throw new ReplicaAlreadyExistsException("Block " + b + + throw new ReplicaAlreadyExistsException("Block " + b + " is valid, and cannot be written to."); } if (isValidRbw(b)) { - throw new ReplicaAlreadyExistsException("Block " + b + + throw new ReplicaAlreadyExistsException("Block " + b + " is being written, and cannot be written to."); } final Map<Block, BInfo> map = getMap(b.getBlockPoolId()); @@ -1006,12 +1015,12 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> { final Map<Block, BInfo> map = getMap(b.getBlockPoolId()); BInfo binfo = map.get(b.getLocalBlock()); if (binfo == null) { - throw new IOException("No such Block " + b ); + throw new IOException("No such Block " + b ); } - + return binfo.getIStream(); } - + @Override // FsDatasetSpi public synchronized InputStream getBlockInputStream(ExtendedBlock b, long seekOffset) throws IOException { @@ -1033,10 +1042,10 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> { final Map<Block, BInfo> map = getMap(b.getBlockPoolId()); BInfo binfo = map.get(b.getLocalBlock()); if (binfo == null) { - throw new IOException("No such Block " + b ); + throw new IOException("No such Block " + b ); } if (!binfo.finalized) { - throw new IOException("Block " + b + + throw new IOException("Block " + b + " is being written, its meta cannot be read"); } final SimulatedInputStream sin = binfo.getMetaIStream(); @@ -1044,19 +1053,17 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> { } @Override - public Set<File> checkDataDir() { - // nothing to check for simulated data set - return null; + public void handleVolumeFailures(Set<FsVolumeSpi> failedVolumes) { } @Override // FsDatasetSpi public synchronized void adjustCrcChannelPosition(ExtendedBlock b, - ReplicaOutputStreams stream, + ReplicaOutputStreams stream, int checksumSize) throws IOException { } - /** + /** * Simulated input and output streams * */ @@ -1065,7 +1072,7 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> { int currentPos = 0; byte[] data = null; Block theBlock = null; - + /** * An input stream of size l with repeated bytes * @param l size of the stream @@ -1075,7 +1082,7 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> { length = l; theBlock = b; } - + /** * An input stream of of the supplied data * @param iData data to construct the stream @@ -1084,7 +1091,7 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> { data = iData; length = data.length; } - + /** * @return the lenght of the input stream */ @@ -1103,9 +1110,9 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> { return simulatedByte(theBlock, currentPos++) & BYTE_MASK; } } - + @Override - public int read(byte[] b) throws IOException { + public int read(byte[] b) throws IOException { if (b == null) { throw new NullPointerException(); @@ -1128,7 +1135,7 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> { return bytesRead; } } - + /** * This class implements an output stream that merely throws its data away, but records its * length. @@ -1136,15 +1143,15 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> { */ static private class SimulatedOutputStream extends OutputStream { long length = 0; - + /** * constructor for Simulated Output Steram */ SimulatedOutputStream() { } - + /** - * + * * @return the length of the data created so far. */ long getLength() { @@ -1156,17 +1163,17 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> { void setLength(long length) { this.length = length; } - + @Override public void write(int arg0) throws IOException { length++; } - + @Override public void write(byte[] b) throws IOException { length += b.length; } - + @Override public void write(byte[] b, int off, @@ -1174,11 +1181,11 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> { length += len; } } - + private ObjectName mbeanName; - + /** * Register the FSDataset MBean using the name * "hadoop:service=DataNode,name=FSDatasetState-<storageid>" @@ -1187,7 +1194,7 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> { */ void registerMBean(final String storageId) { // We wrap to bypass standard mbean naming convetion. - // This wraping can be removed in java 6 as it is more flexible in + // This wraping can be removed in java 6 as it is more flexible in // package naming for mbeans and their impl. StandardMBean bean; @@ -1198,7 +1205,7 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> { } catch (NotCompliantMBeanException e) { DataNode.LOG.warn("Error registering FSDatasetState MBean", e); } - + DataNode.LOG.info("Registered FSDatasetState MBean"); } @@ -1211,7 +1218,7 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> { public String getStorageInfo() { return "Simulated FSDataset-" + datanodeUuid; } - + @Override public boolean hasEnoughResource() { return true; @@ -1224,11 +1231,11 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> { final Map<Block, BInfo> map = getMap(b.getBlockPoolId()); BInfo binfo = map.get(b.getLocalBlock()); if (binfo == null) { - throw new IOException("No such Block " + b ); + throw new IOException("No such Block " + b ); } - return new ReplicaRecoveryInfo(binfo.getBlockId(), binfo.getBytesOnDisk(), - binfo.getGenerationStamp(), + return new ReplicaRecoveryInfo(binfo.getBlockId(), binfo.getBytesOnDisk(), + binfo.getGenerationStamp(), binfo.isFinalized()?ReplicaState.FINALIZED : ReplicaState.RBW); } @@ -1251,13 +1258,13 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> { blockMap.put(bpid, map); storage.addBlockPool(bpid); } - + @Override // FsDatasetSpi public void shutdownBlockPool(String bpid) { blockMap.remove(bpid); storage.removeBlockPool(bpid); } - + @Override // FsDatasetSpi public void deleteBlockPool(String bpid, boolean force) { return; @@ -1321,7 +1328,7 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> { @Override public FsVolumeReferences getFsVolumeReferences() { - throw new UnsupportedOperationException(); + return new FsVolumeReferences(Collections.singletonList(volume)); } @Override @@ -1386,17 +1393,17 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> { // TODO Auto-generated method stub return null; } - + @Override public void setPinning(ExtendedBlock b) throws IOException { blockMap.get(b.getBlockPoolId()).get(b.getLocalBlock()).pinned = true; } - + @Override public boolean getPinning(ExtendedBlock b) throws IOException { return blockMap.get(b.getBlockPoolId()).get(b.getLocalBlock()).pinned; } - + @Override public boolean isDeletingBlock(String bpid, long blockId) { throw new UnsupportedOperationException();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec80de3c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java index 5d4543d..25f4d5a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java @@ -66,6 +66,7 @@ import java.util.Random; import java.util.concurrent.BrokenBarrierException; import java.util.concurrent.CountDownLatch; import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; @@ -124,6 +125,8 @@ public class TestDataNodeHotSwapVolumes { 1000); /* Allow 1 volume failure */ conf.setInt(DFSConfigKeys.DFS_DATANODE_FAILED_VOLUMES_TOLERATED_KEY, 1); + conf.setTimeDuration(DFSConfigKeys.DFS_DATANODE_DISK_CHECK_MIN_GAP_KEY, + 0, TimeUnit.MILLISECONDS); MiniDFSNNTopology nnTopology = MiniDFSNNTopology.simpleFederatedTopology(numNameNodes); http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec80de3c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java index ed60560..e58f993 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java @@ -34,6 +34,7 @@ import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import org.apache.hadoop.conf.Configuration; @@ -119,6 +120,8 @@ public class TestDataNodeVolumeFailure { // Allow a single volume failure (there are two volumes) conf.setInt(DFSConfigKeys.DFS_DATANODE_FAILED_VOLUMES_TOLERATED_KEY, 1); conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 30); + conf.setTimeDuration(DFSConfigKeys.DFS_DATANODE_DISK_CHECK_MIN_GAP_KEY, + 0, TimeUnit.MILLISECONDS); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(dn_num).build(); cluster.waitActive(); fs = cluster.getFileSystem(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec80de3c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureReporting.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureReporting.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureReporting.java index e6f1366..23ec8a7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureReporting.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureReporting.java @@ -29,6 +29,7 @@ import static org.junit.Assume.assumeTrue; import java.io.File; import java.util.ArrayList; +import java.util.concurrent.TimeUnit; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -621,6 +622,8 @@ public class TestDataNodeVolumeFailureReporting { conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 1000); conf.setInt(DFSConfigKeys.DFS_DATANODE_FAILED_VOLUMES_TOLERATED_KEY, failedVolumesTolerated); + conf.setTimeDuration(DFSConfigKeys.DFS_DATANODE_DISK_CHECK_MIN_GAP_KEY, + 0, TimeUnit.MILLISECONDS); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDataNodes) .storagesPerDatanode(storagesPerDatanode).build(); cluster.waitActive(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec80de3c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureToleration.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureToleration.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureToleration.java index 2f8239e..462fdae 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureToleration.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureToleration.java @@ -24,6 +24,7 @@ import static org.junit.Assume.assumeTrue; import java.io.File; import java.io.IOException; +import java.util.concurrent.TimeUnit; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -144,6 +145,8 @@ public class TestDataNodeVolumeFailureToleration { // Bring up two additional datanodes that need both of their volumes // functioning in order to stay up. conf.setInt(DFSConfigKeys.DFS_DATANODE_FAILED_VOLUMES_TOLERATED_KEY, 0); + conf.setTimeDuration(DFSConfigKeys.DFS_DATANODE_DISK_CHECK_MIN_GAP_KEY, + 0, TimeUnit.MILLISECONDS); cluster.startDataNodes(conf, 2, true, null, null); cluster.waitActive(); final DatanodeManager dm = cluster.getNamesystem().getBlockManager( http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec80de3c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java index 06b6fcd..0553a91 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java @@ -26,7 +26,9 @@ import java.io.IOException; import java.io.RandomAccessFile; import java.net.InetSocketAddress; import java.net.Socket; +import java.util.concurrent.TimeUnit; +import com.google.common.base.Supplier; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; @@ -49,8 +51,8 @@ import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.DataChecksum; -import org.apache.hadoop.util.Time; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -69,6 +71,9 @@ public class TestDiskError { public void setUp() throws Exception { conf = new HdfsConfiguration(); conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 512L); + conf.setTimeDuration( + DFSConfigKeys.DFS_DATANODE_DISK_CHECK_MIN_GAP_KEY, + 0, TimeUnit.MILLISECONDS); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); cluster.waitActive(); fs = cluster.getFileSystem(); @@ -214,19 +219,22 @@ public class TestDiskError { * Before refactoring the code the above function was not getting called * @throws IOException, InterruptedException */ - @Test - public void testcheckDiskError() throws IOException, InterruptedException { + @Test(timeout=60000) + public void testcheckDiskError() throws Exception { if(cluster.getDataNodes().size() <= 0) { cluster.startDataNodes(conf, 1, true, null, null); cluster.waitActive(); } - DataNode dataNode = cluster.getDataNodes().get(0); - long slackTime = dataNode.checkDiskErrorInterval/2; + final DataNode dataNode = cluster.getDataNodes().get(0); //checking for disk error - dataNode.checkDiskErrorAsync(); - Thread.sleep(dataNode.checkDiskErrorInterval); - long lastDiskErrorCheck = dataNode.getLastDiskErrorCheck(); - assertTrue("Disk Error check is not performed within " + dataNode.checkDiskErrorInterval + " ms", ((Time.monotonicNow()-lastDiskErrorCheck) < (dataNode.checkDiskErrorInterval + slackTime))); + final long lastCheckTimestamp = dataNode.getLastDiskErrorCheck(); + dataNode.checkDiskError(); + GenericTestUtils.waitFor(new Supplier<Boolean>() { + @Override + public Boolean get() { + return dataNode.getLastDiskErrorCheck() > lastCheckTimestamp; + } + }, 100, 60000); } @Test http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec80de3c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestDatasetVolumeChecker.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestDatasetVolumeChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestDatasetVolumeChecker.java index a9041ab..940e73b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestDatasetVolumeChecker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestDatasetVolumeChecker.java @@ -21,7 +21,6 @@ package org.apache.hadoop.hdfs.server.datanode.checker; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import org.apache.hadoop.hdfs.HdfsConfiguration; -import org.apache.hadoop.hdfs.server.datanode.StorageLocation; import org.apache.hadoop.hdfs.server.datanode.fsdataset.*; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi.VolumeCheckContext; import org.apache.hadoop.util.DiskChecker.DiskErrorException; @@ -35,7 +34,10 @@ import org.junit.runners.Parameterized.Parameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; @@ -229,6 +231,7 @@ public class TestDatasetVolumeChecker { static FsDatasetSpi<FsVolumeSpi> makeDataset(List<FsVolumeSpi> volumes) throws Exception { // Create dataset and init volume health. + @SuppressWarnings("unchecked") // The cast is safe. final FsDatasetSpi<FsVolumeSpi> dataset = mock(FsDatasetSpi.class); final FsDatasetSpi.FsVolumeReferences references = new FsDatasetSpi.FsVolumeReferences(volumes); @@ -236,7 +239,7 @@ public class TestDatasetVolumeChecker { return dataset; } - private static List<FsVolumeSpi> makeVolumes( + static List<FsVolumeSpi> makeVolumes( int numVolumes, VolumeCheckResult health) throws Exception { final List<FsVolumeSpi> volumes = new ArrayList<>(numVolumes); for (int i = 0; i < numVolumes; ++i) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec80de3c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestDatasetVolumeCheckerFailures.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestDatasetVolumeCheckerFailures.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestDatasetVolumeCheckerFailures.java index 56446de..6f3d748 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestDatasetVolumeCheckerFailures.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestDatasetVolumeCheckerFailures.java @@ -21,10 +21,10 @@ package org.apache.hadoop.hdfs.server.datanode.checker; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.HdfsConfiguration; -import org.apache.hadoop.hdfs.server.datanode.StorageLocation; import org.apache.hadoop.hdfs.server.datanode.fsdataset.*; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi.VolumeCheckContext; import org.apache.hadoop.util.FakeTimer; +import org.junit.Before; import org.junit.Test; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; @@ -37,7 +37,6 @@ import java.util.*; import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertThat; -import static org.mockito.Matchers.anyObject; import static org.mockito.Mockito.*; @@ -48,6 +47,19 @@ public class TestDatasetVolumeCheckerFailures { public static final Logger LOG =LoggerFactory.getLogger( TestDatasetVolumeCheckerFailures.class); + private FakeTimer timer; + private Configuration conf; + + private static final long MIN_DISK_CHECK_GAP_MS = 1000; // 1 second. + + @Before + public void commonInit() { + timer = new FakeTimer(); + conf = new HdfsConfiguration(); + conf.setTimeDuration(DFSConfigKeys.DFS_DATANODE_DISK_CHECK_MIN_GAP_KEY, + MIN_DISK_CHECK_GAP_MS, TimeUnit.MILLISECONDS); + } + /** * Test timeout in {@link DatasetVolumeChecker#checkAllVolumes}. * @throws Exception @@ -62,7 +74,6 @@ public class TestDatasetVolumeCheckerFailures { TestDatasetVolumeChecker.makeDataset(volumes); // Create a disk checker with a very low timeout. - final HdfsConfiguration conf = new HdfsConfiguration(); conf.setTimeDuration(DFSConfigKeys.DFS_DATANODE_DISK_CHECK_TIMEOUT_KEY, 1, TimeUnit.SECONDS); final DatasetVolumeChecker checker = @@ -87,10 +98,10 @@ public class TestDatasetVolumeCheckerFailures { final FsDatasetSpi<FsVolumeSpi> dataset = TestDatasetVolumeChecker.makeDataset(volumes); - DatasetVolumeChecker checker = - new DatasetVolumeChecker(new HdfsConfiguration(), new FakeTimer()); + DatasetVolumeChecker checker = new DatasetVolumeChecker(conf, timer); Set<FsVolumeSpi> failedVolumes = checker.checkAllVolumes(dataset); assertThat(failedVolumes.size(), is(0)); + assertThat(checker.getNumSyncDatasetChecks(), is(0L)); // The closed volume should not have been checked as it cannot // be referenced. @@ -99,13 +110,10 @@ public class TestDatasetVolumeCheckerFailures { @Test(timeout=60000) public void testMinGapIsEnforcedForSyncChecks() throws Exception { + final List<FsVolumeSpi> volumes = + TestDatasetVolumeChecker.makeVolumes(1, VolumeCheckResult.HEALTHY); final FsDatasetSpi<FsVolumeSpi> dataset = - TestDatasetVolumeChecker.makeDataset(new ArrayList<FsVolumeSpi>()); - final FakeTimer timer = new FakeTimer(); - final Configuration conf = new HdfsConfiguration(); - final long minGapMs = 100; - conf.setTimeDuration(DFSConfigKeys.DFS_DATANODE_DISK_CHECK_MIN_GAP_KEY, - minGapMs, TimeUnit.MILLISECONDS); + TestDatasetVolumeChecker.makeDataset(volumes); final DatasetVolumeChecker checker = new DatasetVolumeChecker(conf, timer); checker.checkAllVolumes(dataset); @@ -117,7 +125,7 @@ public class TestDatasetVolumeCheckerFailures { assertThat(checker.getNumSkippedChecks(), is(1L)); // Re-check after advancing the timer. Ensure the check is performed. - timer.advance(minGapMs); + timer.advance(MIN_DISK_CHECK_GAP_MS); checker.checkAllVolumes(dataset); assertThat(checker.getNumSyncDatasetChecks(), is(2L)); assertThat(checker.getNumSkippedChecks(), is(1L)); @@ -125,13 +133,10 @@ public class TestDatasetVolumeCheckerFailures { @Test(timeout=60000) public void testMinGapIsEnforcedForASyncChecks() throws Exception { + final List<FsVolumeSpi> volumes = + TestDatasetVolumeChecker.makeVolumes(1, VolumeCheckResult.HEALTHY); final FsDatasetSpi<FsVolumeSpi> dataset = - TestDatasetVolumeChecker.makeDataset(new ArrayList<FsVolumeSpi>()); - final FakeTimer timer = new FakeTimer(); - final Configuration conf = new HdfsConfiguration(); - final long minGapMs = 100; - conf.setTimeDuration(DFSConfigKeys.DFS_DATANODE_DISK_CHECK_MIN_GAP_KEY, - minGapMs, TimeUnit.MILLISECONDS); + TestDatasetVolumeChecker.makeDataset(volumes); final DatasetVolumeChecker checker = new DatasetVolumeChecker(conf, timer); checker.checkAllVolumesAsync(dataset, null); @@ -143,7 +148,7 @@ public class TestDatasetVolumeCheckerFailures { assertThat(checker.getNumSkippedChecks(), is(1L)); // Re-check after advancing the timer. Ensure the check is performed. - timer.advance(minGapMs); + timer.advance(MIN_DISK_CHECK_GAP_MS); checker.checkAllVolumesAsync(dataset, null); assertThat(checker.getNumAsyncDatasetChecks(), is(2L)); assertThat(checker.getNumSkippedChecks(), is(1L)); http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec80de3c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java index 7548e83..731bb0b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java @@ -238,8 +238,7 @@ public class ExternalDatasetImpl implements FsDatasetSpi<ExternalVolumeImpl> { } @Override - public Set<File> checkDataDir() { - return null; + public void handleVolumeFailures(Set<FsVolumeSpi> failedVolumes) { } @Override @@ -432,14 +431,14 @@ public class ExternalDatasetImpl implements FsDatasetSpi<ExternalVolumeImpl> { } @Override - public void setPinning(ExtendedBlock block) throws IOException { + public void setPinning(ExtendedBlock block) throws IOException { } @Override public boolean getPinning(ExtendedBlock block) throws IOException { return false; } - + @Override public boolean isDeletingBlock(String bpid, long blockId) { return false; http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec80de3c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java index 7484a40..10b2f56 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java @@ -51,12 +51,9 @@ import org.apache.hadoop.hdfs.server.datanode.ReplicaInfo; import org.apache.hadoop.hdfs.server.datanode.ShortCircuitRegistry; import org.apache.hadoop.hdfs.server.datanode.StorageLocation; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; -import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeReference; -import org.apache.hadoop.hdfs.server.datanode.fsdataset.RoundRobinVolumeChoosingPolicy; import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; import org.apache.hadoop.io.MultipleIOException; import org.apache.hadoop.test.GenericTestUtils; -import org.apache.hadoop.util.DiskChecker; import org.apache.hadoop.util.FakeTimer; import org.apache.hadoop.util.StringUtils; import org.junit.Assert; @@ -64,8 +61,6 @@ import org.junit.Before; import org.junit.Test; import org.mockito.Matchers; import org.mockito.Mockito; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; import java.io.File; import java.io.FileOutputStream; @@ -74,16 +69,18 @@ import java.io.OutputStreamWriter; import java.io.Writer; import java.nio.charset.StandardCharsets; import java.util.ArrayList; -import java.util.Collections; import java.util.concurrent.CountDownLatch; import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.concurrent.TimeUnit; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DN_CACHED_DFSUSED_CHECK_INTERVAL_MS; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOURS_KEY; +import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertSame; @@ -92,13 +89,10 @@ import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyListOf; import static org.mockito.Matchers.anyString; import static org.mockito.Matchers.eq; -import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import org.slf4j.Logger; @@ -120,7 +114,7 @@ public class TestFsDatasetImpl { private DataNode datanode; private DataStorage storage; private FsDatasetImpl dataset; - + private final static String BLOCKPOOL = "BP-TEST"; private static Storage.StorageDirectory createStorageDirectory(File root) { @@ -324,64 +318,6 @@ public class TestFsDatasetImpl { assertEquals(numExistingVolumes, getNumVolumes()); } - @Test(timeout = 5000) - public void testChangeVolumeWithRunningCheckDirs() throws IOException { - RoundRobinVolumeChoosingPolicy<FsVolumeImpl> blockChooser = - new RoundRobinVolumeChoosingPolicy<>(); - conf.setLong(DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOURS_KEY, -1); - final BlockScanner blockScanner = new BlockScanner(datanode); - final FsVolumeList volumeList = new FsVolumeList( - Collections.<VolumeFailureInfo>emptyList(), blockScanner, blockChooser); - final List<FsVolumeImpl> oldVolumes = new ArrayList<>(); - - // Initialize FsVolumeList with 5 mock volumes. - final int NUM_VOLUMES = 5; - for (int i = 0; i < NUM_VOLUMES; i++) { - FsVolumeImpl volume = mock(FsVolumeImpl.class); - oldVolumes.add(volume); - when(volume.getBasePath()).thenReturn("data" + i); - when(volume.checkClosed()).thenReturn(true); - FsVolumeReference ref = mock(FsVolumeReference.class); - when(ref.getVolume()).thenReturn(volume); - volumeList.addVolume(ref); - } - - // When call checkDirs() on the 2nd volume, anther "thread" removes the 5th - // volume and add another volume. It does not affect checkDirs() running. - final FsVolumeImpl newVolume = mock(FsVolumeImpl.class); - final FsVolumeReference newRef = mock(FsVolumeReference.class); - when(newRef.getVolume()).thenReturn(newVolume); - when(newVolume.getBasePath()).thenReturn("data4"); - FsVolumeImpl blockedVolume = volumeList.getVolumes().get(1); - doAnswer(new Answer() { - @Override - public Object answer(InvocationOnMock invocationOnMock) - throws Throwable { - volumeList.removeVolume(new File("data4"), false); - volumeList.addVolume(newRef); - return null; - } - }).when(blockedVolume).checkDirs(); - - FsVolumeImpl brokenVolume = volumeList.getVolumes().get(2); - doThrow(new DiskChecker.DiskErrorException("broken")) - .when(brokenVolume).checkDirs(); - - volumeList.checkDirs(); - - // Since FsVolumeImpl#checkDirs() get a snapshot of the list of volumes - // before running removeVolume(), it is supposed to run checkDirs() on all - // the old volumes. - for (FsVolumeImpl volume : oldVolumes) { - verify(volume).checkDirs(); - } - // New volume is not visible to checkDirs() process. - verify(newVolume, never()).checkDirs(); - assertTrue(volumeList.getVolumes().contains(newVolume)); - assertFalse(volumeList.getVolumes().contains(brokenVolume)); - assertEquals(NUM_VOLUMES - 1, volumeList.getVolumes().size()); - } - @Test public void testAddVolumeFailureReleasesInUseLock() throws IOException { FsDatasetImpl spyDataset = spy(dataset); @@ -417,7 +353,7 @@ public class TestFsDatasetImpl { FsDatasetTestUtil.assertFileLockReleased(badDir.toString()); } - + @Test public void testDeletingBlocks() throws IOException { HdfsConfiguration conf = new HdfsConfiguration(); @@ -425,7 +361,7 @@ public class TestFsDatasetImpl { try { cluster.waitActive(); DataNode dn = cluster.getDataNodes().get(0); - + FsDatasetSpi<?> ds = DataNodeTestUtils.getFSDataset(dn); ds.addBlockPool(BLOCKPOOL, conf); FsVolumeImpl vol; @@ -697,6 +633,9 @@ public class TestFsDatasetImpl { Configuration config = new HdfsConfiguration(); config.setLong( DFSConfigKeys.DFS_DATANODE_XCEIVER_STOP_TIMEOUT_MILLIS_KEY, 1000); + config.setTimeDuration( + DFSConfigKeys.DFS_DATANODE_DISK_CHECK_MIN_GAP_KEY, 0, + TimeUnit.MILLISECONDS); config.setInt(DFSConfigKeys.DFS_DATANODE_FAILED_VOLUMES_TOLERATED_KEY, 1); cluster = new MiniDFSCluster.Builder(config).numDataNodes(1).build(); @@ -714,6 +653,8 @@ public class TestFsDatasetImpl { getVolume(block); File finalizedDir = volume.getFinalizedDir(cluster.getNamesystem() .getBlockPoolId()); + LocatedBlock lb = DFSTestUtil.getAllBlocks(fs, filePath).get(0); + DatanodeInfo info = lb.getLocations()[0]; if (finalizedDir.exists()) { // Remove write and execute access so that checkDiskErrorThread detects @@ -724,15 +665,14 @@ public class TestFsDatasetImpl { Assert.assertTrue("Reference count for the volume should be greater " + "than 0", volume.getReferenceCount() > 0); // Invoke the synchronous checkDiskError method - dataNode.getFSDataset().checkDataDir(); + dataNode.checkDiskError(); // Sleep for 1 second so that datanode can interrupt and cluster clean up GenericTestUtils.waitFor(new Supplier<Boolean>() { @Override public Boolean get() { return volume.getReferenceCount() == 0; } }, 100, 10); - LocatedBlock lb = DFSTestUtil.getAllBlocks(fs, filePath).get(0); - DatanodeInfo info = lb.getLocations()[0]; + assertThat(dataNode.getFSDataset().getNumFailedVolumes(), is(1)); try { out.close(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec80de3c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java index 3d4c38c..f511dc6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java @@ -102,38 +102,6 @@ public class TestFsVolumeList { } @Test(timeout=30000) - public void testCheckDirsWithClosedVolume() throws IOException { - FsVolumeList volumeList = new FsVolumeList( - Collections.<VolumeFailureInfo>emptyList(), blockScanner, blockChooser); - final List<FsVolumeImpl> volumes = new ArrayList<>(); - for (int i = 0; i < 3; i++) { - File curDir = new File(baseDir, "volume-" + i); - curDir.mkdirs(); - FsVolumeImpl volume = new FsVolumeImpl(dataset, "storage-id", curDir, - conf, StorageType.DEFAULT); - volumes.add(volume); - volumeList.addVolume(volume.obtainReference()); - } - - // Close the 2nd volume. - volumes.get(1).setClosed(); - try { - GenericTestUtils.waitFor(new Supplier<Boolean>() { - @Override - public Boolean get() { - return volumes.get(1).checkClosed(); - } - }, 100, 3000); - } catch (TimeoutException e) { - fail("timed out while waiting for volume to be removed."); - } catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - } - // checkDirs() should ignore the 2nd volume since it is closed. - volumeList.checkDirs(); - } - - @Test(timeout=30000) public void testReleaseVolumeRefIfNoBlockScanner() throws IOException { FsVolumeList volumeList = new FsVolumeList( Collections.<VolumeFailureInfo>emptyList(), null, blockChooser); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
