HDFS-11600. Refactor TestDFSStripedOutputStreamWithFailure test classes. Contributed by Sammi Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ad1b988a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ad1b988a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ad1b988a Branch: refs/heads/HDFS-7240 Commit: ad1b988a828608b12cafb6382436cd17f95bfcc5 Parents: fea16a4 Author: Sammi Chen <[email protected]> Authored: Wed Mar 14 14:39:53 2018 +0800 Committer: Sammi Chen <[email protected]> Committed: Wed Mar 14 14:39:53 2018 +0800 ---------------------------------------------------------------------- ...edTestDFSStripedOutputStreamWithFailure.java | 71 +++ ...tputStreamWithFailureWithRandomECPolicy.java | 49 ++ .../TestDFSStripedOutputStreamWithFailure.java | 496 +------------------ ...estDFSStripedOutputStreamWithFailure000.java | 24 - ...estDFSStripedOutputStreamWithFailure010.java | 24 - ...estDFSStripedOutputStreamWithFailure020.java | 24 - ...estDFSStripedOutputStreamWithFailure030.java | 24 - ...estDFSStripedOutputStreamWithFailure040.java | 24 - ...estDFSStripedOutputStreamWithFailure050.java | 24 - ...estDFSStripedOutputStreamWithFailure060.java | 24 - ...estDFSStripedOutputStreamWithFailure070.java | 24 - ...estDFSStripedOutputStreamWithFailure080.java | 24 - ...estDFSStripedOutputStreamWithFailure090.java | 24 - ...estDFSStripedOutputStreamWithFailure100.java | 24 - ...estDFSStripedOutputStreamWithFailure110.java | 24 - ...estDFSStripedOutputStreamWithFailure120.java | 24 - ...estDFSStripedOutputStreamWithFailure130.java | 24 - ...estDFSStripedOutputStreamWithFailure140.java | 24 - ...estDFSStripedOutputStreamWithFailure150.java | 24 - ...estDFSStripedOutputStreamWithFailure160.java | 24 - ...estDFSStripedOutputStreamWithFailure170.java | 24 - ...estDFSStripedOutputStreamWithFailure180.java | 24 - ...estDFSStripedOutputStreamWithFailure190.java | 24 - ...estDFSStripedOutputStreamWithFailure200.java | 24 - ...estDFSStripedOutputStreamWithFailure210.java | 24 - ...stDFSStripedOutputStreamWithFailureBase.java | 426 ++++++++++++++++ 26 files changed, 554 insertions(+), 1016 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad1b988a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/ParameterizedTestDFSStripedOutputStreamWithFailure.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/ParameterizedTestDFSStripedOutputStreamWithFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/ParameterizedTestDFSStripedOutputStreamWithFailure.java new file mode 100644 index 0000000..284fdb7 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/ParameterizedTestDFSStripedOutputStreamWithFailure.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import static org.junit.Assume.assumeTrue; + +/** + * Test striped file write operation with data node failures with parameterized + * test cases. + */ +@RunWith(Parameterized.class) +public class ParameterizedTestDFSStripedOutputStreamWithFailure extends + TestDFSStripedOutputStreamWithFailureBase{ + public static final Logger LOG = LoggerFactory.getLogger( + ParameterizedTestDFSStripedOutputStreamWithFailure.class); + + private int base; + + @Parameterized.Parameters + public static Collection<Object[]> data() { + List<Object[]> parameters = new ArrayList<>(); + for (int i = 0; i <= 10; i++) { + parameters.add(new Object[]{RANDOM.nextInt(220)}); + } + return parameters; + } + + public ParameterizedTestDFSStripedOutputStreamWithFailure(int base) { + this.base = base; + } + + @Test(timeout = 240000) + public void runTestWithSingleFailure() { + assumeTrue(base >= 0); + if (base > lengths.size()) { + base = base % lengths.size(); + } + final int i = base; + final Integer length = getLength(i); + assumeTrue("Skip test " + i + " since length=null.", length != null); + assumeTrue("Test " + i + ", length=" + length + + ", is not chosen to run.", RANDOM.nextInt(16) != 0); + System.out.println("Run test " + i + ", length=" + length); + runTest(length); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad1b988a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/ParameterizedTestDFSStripedOutputStreamWithFailureWithRandomECPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/ParameterizedTestDFSStripedOutputStreamWithFailureWithRandomECPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/ParameterizedTestDFSStripedOutputStreamWithFailureWithRandomECPolicy.java new file mode 100644 index 0000000..a3d4ffc --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/ParameterizedTestDFSStripedOutputStreamWithFailureWithRandomECPolicy.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs; + +import org.apache.hadoop.io.erasurecode.ECSchema; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This tests write operation of DFS striped file with a random erasure code + * policy except for the default policy under Datanode failure conditions. + */ +public class + ParameterizedTestDFSStripedOutputStreamWithFailureWithRandomECPolicy extends + ParameterizedTestDFSStripedOutputStreamWithFailure { + + private final ECSchema schema; + + private static final Logger LOG = LoggerFactory.getLogger( + ParameterizedTestDFSStripedOutputStreamWithFailureWithRandomECPolicy + .class.getName()); + + public ParameterizedTestDFSStripedOutputStreamWithFailureWithRandomECPolicy( + int base) { + super(base); + schema = StripedFileTestUtil.getRandomNonDefaultECPolicy().getSchema(); + LOG.info(schema.toString()); + } + + @Override + public ECSchema getEcSchema() { + return schema; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad1b988a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java index afb4d63..800fac1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java @@ -17,234 +17,33 @@ */ package org.apache.hadoop.hdfs; -import com.google.common.base.Preconditions; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.commons.logging.impl.Log4JLogger; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; -import org.apache.hadoop.hdfs.protocol.AddErasureCodingPolicyResponse; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; -import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; -import org.apache.hadoop.hdfs.protocol.LocatedBlock; -import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; -import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager; -import org.apache.hadoop.hdfs.security.token.block.SecurityTestUtil; -import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; -import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy; import org.apache.hadoop.hdfs.server.datanode.DataNode; -import org.apache.hadoop.hdfs.server.namenode.NameNode; -import org.apache.hadoop.io.erasurecode.CodecUtil; -import org.apache.hadoop.io.erasurecode.ECSchema; -import org.apache.hadoop.io.erasurecode.ErasureCodeNative; -import org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory; -import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.test.LambdaTestUtils; -import org.apache.hadoop.util.StringUtils; -import org.apache.log4j.Level; -import org.junit.Assert; -import org.junit.Before; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; -import java.util.LinkedList; -import java.util.List; -import java.util.Random; -import java.util.Stack; -import java.util.concurrent.atomic.AtomicInteger; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import static org.junit.Assume.assumeTrue; /** - * Test striped file write operation with data node failures. + * Test striped file write operation with data node failures with fixed + * parameter test cases. */ -public class TestDFSStripedOutputStreamWithFailure { - public static final Log LOG = LogFactory.getLog( +public class TestDFSStripedOutputStreamWithFailure extends + TestDFSStripedOutputStreamWithFailureBase{ + public static final Logger LOG = LoggerFactory.getLogger( TestDFSStripedOutputStreamWithFailure.class); - static { - GenericTestUtils.setLogLevel(DFSOutputStream.LOG, Level.ALL); - GenericTestUtils.setLogLevel(DataStreamer.LOG, Level.ALL); - GenericTestUtils.setLogLevel(DFSClient.LOG, Level.ALL); - ((Log4JLogger)LogFactory.getLog(BlockPlacementPolicy.class)) - .getLogger().setLevel(Level.ALL); - } - - private final int cellSize = 64 * 1024; //64k - private final int stripesPerBlock = 4; - private ErasureCodingPolicy ecPolicy; - private int dataBlocks; - private int parityBlocks; - private int blockSize; - private int blockGroupSize; - - private static final int FLUSH_POS = - 9 * DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_DEFAULT + 1; - - public ECSchema getEcSchema() { - return StripedFileTestUtil.getDefaultECPolicy().getSchema(); - } - - /* - * Initialize erasure coding policy. - */ - @Before - public void init() { - ecPolicy = new ErasureCodingPolicy(getEcSchema(), cellSize); - dataBlocks = ecPolicy.getNumDataUnits(); - parityBlocks = ecPolicy.getNumParityUnits(); - blockSize = cellSize * stripesPerBlock; - blockGroupSize = blockSize * dataBlocks; - dnIndexSuite = getDnIndexSuite(); - lengths = newLengths(); - } - - List<Integer> newLengths() { - final List<Integer> lens = new ArrayList<>(); - lens.add(FLUSH_POS + 2); - for(int b = 0; b <= 2; b++) { - for(int c = 0; c < stripesPerBlock * dataBlocks; c++) { - for(int delta = -1; delta <= 1; delta++) { - final int length = b * blockGroupSize + c * cellSize + delta; - System.out.println(lens.size() + ": length=" + length - + ", (b, c, d) = (" + b + ", " + c + ", " + delta + ")"); - lens.add(length); - } - } - } - return lens; - } - - private int[][] dnIndexSuite; - - private int[][] getDnIndexSuite() { - final int maxNumLevel = 2; - final int maxPerLevel = 8; - List<List<Integer>> allLists = new ArrayList<>(); - int numIndex = parityBlocks; - for (int i = 0; i < maxNumLevel && numIndex > 1; i++) { - List<List<Integer>> lists = - combinations(dataBlocks + parityBlocks, numIndex); - if (lists.size() > maxPerLevel) { - Collections.shuffle(lists); - lists = lists.subList(0, maxPerLevel); - } - allLists.addAll(lists); - numIndex--; - } - int[][] dnIndexArray = new int[allLists.size()][]; - for (int i = 0; i < dnIndexArray.length; i++) { - int[] list = new int[allLists.get(i).size()]; - for (int j = 0; j < list.length; j++) { - list[j] = allLists.get(i).get(j); - } - dnIndexArray[i] = list; - } - return dnIndexArray; - } - - // get all combinations of k integers from {0,...,n-1} - private static List<List<Integer>> combinations(int n, int k) { - List<List<Integer>> res = new LinkedList<List<Integer>>(); - if (k >= 1 && n >= k) { - getComb(n, k, new Stack<Integer>(), res); - } - return res; - } - - private static void getComb(int n, int k, Stack<Integer> stack, - List<List<Integer>> res) { - if (stack.size() == k) { - List<Integer> list = new ArrayList<Integer>(stack); - res.add(list); - } else { - int next = stack.empty() ? 0 : stack.peek() + 1; - while (next < n) { - stack.push(next); - getComb(n, k, stack, res); - next++; - } - } - if (!stack.empty()) { - stack.pop(); - } - } - - private int[] getKillPositions(int fileLen, int num) { - int[] positions = new int[num]; - for (int i = 0; i < num; i++) { - positions[i] = fileLen * (i + 1) / (num + 1); - } - return positions; - } - - private List<Integer> lengths; - - Integer getLength(int i) { - return i >= 0 && i < lengths.size() ? lengths.get(i): null; - } - - private static final Random RANDOM = new Random(); - - private MiniDFSCluster cluster; - private DistributedFileSystem dfs; - private final Path dir = new Path("/" - + TestDFSStripedOutputStreamWithFailure.class.getSimpleName()); - - private void setup(Configuration conf) throws IOException { - System.out.println("NUM_DATA_BLOCKS = " + dataBlocks); - System.out.println("NUM_PARITY_BLOCKS= " + parityBlocks); - System.out.println("CELL_SIZE = " + cellSize + " (=" + - StringUtils.TraditionalBinaryPrefix.long2String(cellSize, "B", 2) - + ")"); - System.out.println("BLOCK_SIZE = " + blockSize + " (=" + - StringUtils.TraditionalBinaryPrefix.long2String(blockSize, "B", 2) - + ")"); - System.out.println("BLOCK_GROUP_SIZE = " + blockGroupSize + " (=" + - StringUtils.TraditionalBinaryPrefix.long2String(blockGroupSize, "B", 2) - + ")"); - final int numDNs = dataBlocks + parityBlocks; - if (ErasureCodeNative.isNativeCodeLoaded()) { - conf.set( - CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY, - NativeRSRawErasureCoderFactory.CODER_NAME); - } - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); - cluster.waitActive(); - dfs = cluster.getFileSystem(); - AddErasureCodingPolicyResponse[] res = - dfs.addErasureCodingPolicies(new ErasureCodingPolicy[]{ecPolicy}); - ecPolicy = res[0].getPolicy(); - dfs.enableErasureCodingPolicy(ecPolicy.getName()); - DFSTestUtil.enableAllECPolicies(dfs); - dfs.mkdirs(dir); - dfs.setErasureCodingPolicy(dir, ecPolicy.getName()); - } - - private void tearDown() { - if (cluster != null) { - cluster.shutdown(); - } - } - - private HdfsConfiguration newHdfsConfiguration() { - final HdfsConfiguration conf = new HdfsConfiguration(); - conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); - conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY, - false); - conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1); - conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0); - return conf; - } @Test(timeout=300000) public void testMultipleDatanodeFailure56() throws Exception { @@ -432,53 +231,11 @@ public class TestDFSStripedOutputStreamWithFailure { } } - void runTest(final int length) { - final HdfsConfiguration conf = newHdfsConfiguration(); - for (int dn = 0; dn < dataBlocks + parityBlocks; dn++) { - try { - LOG.info("runTest: dn=" + dn + ", length=" + length); - setup(conf); - runTest(length, new int[]{length / 2}, new int[]{dn}, false); - } catch (Throwable e) { - final String err = "failed, dn=" + dn + ", length=" + length - + StringUtils.stringifyException(e); - LOG.error(err); - Assert.fail(err); - } finally { - tearDown(); - } - } - } - - void runTestWithMultipleFailure(final int length) throws Exception { - final HdfsConfiguration conf = newHdfsConfiguration(); - for (int[] dnIndex : dnIndexSuite) { - int[] killPos = getKillPositions(length, dnIndex.length); - try { - LOG.info("runTestWithMultipleFailure: length==" + length + ", killPos=" - + Arrays.toString(killPos) + ", dnIndex=" - + Arrays.toString(dnIndex)); - setup(conf); - runTest(length, killPos, dnIndex, false); - } catch (Throwable e) { - final String err = "failed, killPos=" + Arrays.toString(killPos) - + ", dnIndex=" + Arrays.toString(dnIndex) + ", length=" + length; - LOG.error(err); - throw e; - } finally { - tearDown(); - } - } - } - /** * When the two DataNodes with partial data blocks fail. */ @Test - public void runTestWithDifferentLengths() throws Exception { - assumeTrue("Skip this test case in the subclasses. Once is enough.", - this.getClass().equals(TestDFSStripedOutputStreamWithFailure.class)); - + public void testCloseWithExceptionsInStreamer() throws Exception { final HdfsConfiguration conf = newHdfsConfiguration(); final int[] fileLengths = { @@ -515,9 +272,6 @@ public class TestDFSStripedOutputStreamWithFailure { */ @Test public void runTestWithShortStripe() throws Exception { - assumeTrue("Skip this test case in the subclasses. Once is enough.", - this.getClass().equals(TestDFSStripedOutputStreamWithFailure.class)); - final HdfsConfiguration conf = newHdfsConfiguration(); // Write a file with a 1 cell partial stripe final int length = cellSize - 123; @@ -543,238 +297,4 @@ public class TestDFSStripedOutputStreamWithFailure { tearDown(); } } - - /** - * runTest implementation. - * @param length file length - * @param killPos killing positions in ascending order - * @param dnIndex DN index to kill when meets killing positions - * @param tokenExpire wait token to expire when kill a DN - * @throws Exception - */ - private void runTest(final int length, final int[] killPos, - final int[] dnIndex, final boolean tokenExpire) throws Exception { - if (killPos[0] <= FLUSH_POS) { - LOG.warn("killPos=" + Arrays.toString(killPos) + " <= FLUSH_POS=" - + FLUSH_POS + ", length=" + length + ", dnIndex=" - + Arrays.toString(dnIndex)); - return; //skip test - } - Preconditions.checkArgument(length > killPos[0], "length=%s <= killPos=%s", - length, killPos); - Preconditions.checkArgument(killPos.length == dnIndex.length); - - final Path p = new Path(dir, "dn" + Arrays.toString(dnIndex) - + "len" + length + "kill" + Arrays.toString(killPos)); - final String fullPath = p.toString(); - LOG.info("fullPath=" + fullPath); - - if (tokenExpire) { - final NameNode nn = cluster.getNameNode(); - final BlockManager bm = nn.getNamesystem().getBlockManager(); - final BlockTokenSecretManager sm = bm.getBlockTokenSecretManager(); - - // set a short token lifetime (6 second) - SecurityTestUtil.setBlockTokenLifetime(sm, 6000L); - } - - final AtomicInteger pos = new AtomicInteger(); - final FSDataOutputStream out = dfs.create(p); - final DFSStripedOutputStream stripedOut - = (DFSStripedOutputStream)out.getWrappedStream(); - - // first GS of this block group which never proceeds blockRecovery - long firstGS = -1; - long oldGS = -1; // the old GS before bumping - List<Long> gsList = new ArrayList<>(); - final List<DatanodeInfo> killedDN = new ArrayList<>(); - int numKilled = 0; - for(; pos.get() < length;) { - final int i = pos.getAndIncrement(); - if (numKilled < killPos.length && i == killPos[numKilled]) { - assertTrue(firstGS != -1); - final long gs = getGenerationStamp(stripedOut); - if (numKilled == 0) { - assertEquals(firstGS, gs); - } else { - //TODO: implement hflush/hsync and verify gs strict greater than oldGS - assertTrue(gs >= oldGS); - } - oldGS = gs; - - if (tokenExpire) { - DFSTestUtil.flushInternal(stripedOut); - waitTokenExpires(out); - } - - killedDN.add( - killDatanode(cluster, stripedOut, dnIndex[numKilled], pos)); - numKilled++; - } - - write(out, i); - - if (i % blockGroupSize == FLUSH_POS) { - firstGS = getGenerationStamp(stripedOut); - oldGS = firstGS; - } - if (i > 0 && (i + 1) % blockGroupSize == 0) { - gsList.add(oldGS); - } - } - gsList.add(oldGS); - out.close(); - assertEquals(dnIndex.length, numKilled); - - StripedFileTestUtil.waitBlockGroupsReported(dfs, fullPath, numKilled); - - cluster.triggerBlockReports(); - StripedFileTestUtil.checkData(dfs, p, length, killedDN, gsList, - blockGroupSize); - } - - static void write(FSDataOutputStream out, int i) throws IOException { - try { - out.write(StripedFileTestUtil.getByte(i)); - } catch(IOException ioe) { - throw new IOException("Failed at i=" + i, ioe); - } - } - - static long getGenerationStamp(DFSStripedOutputStream out) - throws IOException { - final long gs = out.getBlock().getGenerationStamp(); - LOG.info("getGenerationStamp returns " + gs); - return gs; - } - - static DatanodeInfo getDatanodes(StripedDataStreamer streamer) { - for(;;) { - DatanodeInfo[] datanodes = streamer.getNodes(); - if (datanodes == null) { - // try peeking following block. - final LocatedBlock lb = streamer.peekFollowingBlock(); - if (lb != null) { - datanodes = lb.getLocations(); - } - } - - if (datanodes != null) { - Assert.assertEquals(1, datanodes.length); - Assert.assertNotNull(datanodes[0]); - return datanodes[0]; - } - - try { - Thread.sleep(100); - } catch (InterruptedException ie) { - Assert.fail(StringUtils.stringifyException(ie)); - return null; - } - } - } - - static DatanodeInfo killDatanode(MiniDFSCluster cluster, - DFSStripedOutputStream out, final int dnIndex, final AtomicInteger pos) { - final StripedDataStreamer s = out.getStripedDataStreamer(dnIndex); - final DatanodeInfo datanode = getDatanodes(s); - LOG.info("killDatanode " + dnIndex + ": " + datanode + ", pos=" + pos); - if (datanode != null) { - cluster.stopDataNode(datanode.getXferAddr()); - } - return datanode; - } - - - private void waitTokenExpires(FSDataOutputStream out) throws IOException { - Token<BlockTokenIdentifier> token = DFSTestUtil.getBlockToken(out); - while (!SecurityTestUtil.isBlockTokenExpired(token)) { - try { - Thread.sleep(10); - } catch (InterruptedException ignored) { - } - } - } - - int getBase() { - final String name = getClass().getSimpleName(); - int i = name.length() - 1; - for(; i >= 0 && Character.isDigit(name.charAt(i));){ - i--; - } - String number = name.substring(i + 1); - try { - return Integer.parseInt(number); - } catch (Exception e) { - return -1; - } - } - - private void run(int offset) { - int base = getBase(); - assumeTrue(base >= 0); - final int i = offset + base; - final Integer length = getLength(i); - if (length == null) { - System.out.println("Skip test " + i + " since length=null."); - return; - } - if (RANDOM.nextInt(16) != 0) { - System.out.println("Test " + i + ", length=" + length - + ", is not chosen to run."); - return; - } - System.out.println("Run test " + i + ", length=" + length); - runTest(length); - } - - @Test(timeout = 240000) - public void test0() { - run(0); - } - - @Test(timeout = 240000) - public void test1() { - run(1); - } - - @Test(timeout = 240000) - public void test2() { - run(2); - } - - @Test(timeout = 240000) - public void test3() { - run(3); - } - - @Test(timeout = 240000) - public void test4() { - run(4); - } - - @Test(timeout = 240000) - public void test5() { - run(5); - } - - @Test(timeout = 240000) - public void test6() { - run(6); - } - - @Test(timeout = 240000) - public void test7() { - run(7); - } - - @Test(timeout = 240000) - public void test8() { - run(8); - } - - @Test(timeout = 240000) - public void test9() { - run(9); - } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad1b988a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure000.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure000.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure000.java deleted file mode 100644 index 44dd584..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure000.java +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs; - -/** - * Test striped file write operation with data node failures. - */ -public class TestDFSStripedOutputStreamWithFailure000 - extends TestDFSStripedOutputStreamWithFailure {} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad1b988a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure010.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure010.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure010.java deleted file mode 100644 index 15261dd..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure010.java +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs; - -/** - * Test striped file write operation with data node failures. - */ -public class TestDFSStripedOutputStreamWithFailure010 - extends TestDFSStripedOutputStreamWithFailure {} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad1b988a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure020.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure020.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure020.java deleted file mode 100644 index 7c0586a..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure020.java +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs; - -/** - * Test striped file write operation with data node failures. - */ -public class TestDFSStripedOutputStreamWithFailure020 - extends TestDFSStripedOutputStreamWithFailure {} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad1b988a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure030.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure030.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure030.java deleted file mode 100644 index 0d3948a..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure030.java +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs; - -/** - * Test striped file write operation with data node failures. - */ -public class TestDFSStripedOutputStreamWithFailure030 - extends TestDFSStripedOutputStreamWithFailure {} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad1b988a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure040.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure040.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure040.java deleted file mode 100644 index 96e067a..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure040.java +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs; - -/** - * Test striped file write operation with data node failures. - */ -public class TestDFSStripedOutputStreamWithFailure040 - extends TestDFSStripedOutputStreamWithFailure {} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad1b988a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure050.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure050.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure050.java deleted file mode 100644 index c75f78f..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure050.java +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs; - -/** - * Test striped file write operation with data node failures. - */ -public class TestDFSStripedOutputStreamWithFailure050 - extends TestDFSStripedOutputStreamWithFailure {} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad1b988a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure060.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure060.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure060.java deleted file mode 100644 index ea53da2..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure060.java +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs; - -/** - * Test striped file write operation with data node failures. - */ -public class TestDFSStripedOutputStreamWithFailure060 - extends TestDFSStripedOutputStreamWithFailure {} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad1b988a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure070.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure070.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure070.java deleted file mode 100644 index 31a777a..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure070.java +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs; - -/** - * Test striped file write operation with data node failures. - */ -public class TestDFSStripedOutputStreamWithFailure070 - extends TestDFSStripedOutputStreamWithFailure {} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad1b988a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure080.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure080.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure080.java deleted file mode 100644 index a1bb14f..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure080.java +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs; - -/** - * Test striped file write operation with data node failures. - */ -public class TestDFSStripedOutputStreamWithFailure080 - extends TestDFSStripedOutputStreamWithFailure {} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad1b988a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure090.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure090.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure090.java deleted file mode 100644 index 52275d8..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure090.java +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs; - -/** - * Test striped file write operation with data node failures. - */ -public class TestDFSStripedOutputStreamWithFailure090 - extends TestDFSStripedOutputStreamWithFailure {} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad1b988a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure100.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure100.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure100.java deleted file mode 100644 index 4d08487..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure100.java +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs; - -/** - * Test striped file write operation with data node failures. - */ -public class TestDFSStripedOutputStreamWithFailure100 - extends TestDFSStripedOutputStreamWithFailure {} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad1b988a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure110.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure110.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure110.java deleted file mode 100644 index f03efa6..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure110.java +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs; - -/** - * Test striped file write operation with data node failures. - */ -public class TestDFSStripedOutputStreamWithFailure110 - extends TestDFSStripedOutputStreamWithFailure {} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad1b988a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure120.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure120.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure120.java deleted file mode 100644 index b8b8b28..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure120.java +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs; - -/** - * Test striped file write operation with data node failures. - */ -public class TestDFSStripedOutputStreamWithFailure120 - extends TestDFSStripedOutputStreamWithFailure {} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad1b988a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure130.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure130.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure130.java deleted file mode 100644 index dada985..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure130.java +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs; - -/** - * Test striped file write operation with data node failures. - */ -public class TestDFSStripedOutputStreamWithFailure130 - extends TestDFSStripedOutputStreamWithFailure {} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad1b988a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure140.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure140.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure140.java deleted file mode 100644 index 3118d72..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure140.java +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs; - -/** - * Test striped file write operation with data node failures. - */ -public class TestDFSStripedOutputStreamWithFailure140 - extends TestDFSStripedOutputStreamWithFailure {} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad1b988a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure150.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure150.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure150.java deleted file mode 100644 index 8008ad7..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure150.java +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs; - -/** - * Test striped file write operation with data node failures. - */ -public class TestDFSStripedOutputStreamWithFailure150 - extends TestDFSStripedOutputStreamWithFailure {} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad1b988a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure160.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure160.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure160.java deleted file mode 100644 index 0a19041..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure160.java +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs; - -/** - * Test striped file write operation with data node failures. - */ -public class TestDFSStripedOutputStreamWithFailure160 - extends TestDFSStripedOutputStreamWithFailure {} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad1b988a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure170.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure170.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure170.java deleted file mode 100644 index 0a669c3..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure170.java +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs; - -/** - * Test striped file write operation with data node failures. - */ -public class TestDFSStripedOutputStreamWithFailure170 - extends TestDFSStripedOutputStreamWithFailure {} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad1b988a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure180.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure180.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure180.java deleted file mode 100644 index 364781e..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure180.java +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs; - -/** - * Test striped file write operation with data node failures. - */ -public class TestDFSStripedOutputStreamWithFailure180 - extends TestDFSStripedOutputStreamWithFailure {} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad1b988a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure190.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure190.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure190.java deleted file mode 100644 index d6339d2..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure190.java +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs; - -/** - * Test striped file write operation with data node failures. - */ -public class TestDFSStripedOutputStreamWithFailure190 - extends TestDFSStripedOutputStreamWithFailure {} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad1b988a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure200.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure200.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure200.java deleted file mode 100644 index 7e419d1..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure200.java +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs; - -/** - * Test striped file write operation with data node failures. - */ -public class TestDFSStripedOutputStreamWithFailure200 - extends TestDFSStripedOutputStreamWithFailure {} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad1b988a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure210.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure210.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure210.java deleted file mode 100644 index bf7cebb..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure210.java +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs; - -/** - * Test striped file write operation with data node failures. - */ -public class TestDFSStripedOutputStreamWithFailure210 - extends TestDFSStripedOutputStreamWithFailure {} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad1b988a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailureBase.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailureBase.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailureBase.java new file mode 100644 index 0000000..769000e --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailureBase.java @@ -0,0 +1,426 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs; + +import com.google.common.base.Preconditions; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.protocol.AddErasureCodingPolicyResponse; +import org.apache.hadoop.hdfs.protocol.DatanodeInfo; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; +import org.apache.hadoop.hdfs.protocol.LocatedBlock; +import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; +import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager; +import org.apache.hadoop.hdfs.security.token.block.SecurityTestUtil; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy; +import org.apache.hadoop.hdfs.server.namenode.NameNode; +import org.apache.hadoop.io.erasurecode.CodecUtil; +import org.apache.hadoop.io.erasurecode.ECSchema; +import org.apache.hadoop.io.erasurecode.ErasureCodeNative; +import org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.util.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.event.Level; +import org.junit.Assert; +import org.junit.Before; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Random; +import java.util.Stack; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Base class for test striped file write operation. + */ +public class TestDFSStripedOutputStreamWithFailureBase { + public static final Logger LOG = LoggerFactory.getLogger( + TestDFSStripedOutputStreamWithFailureBase.class); + static { + GenericTestUtils.setLogLevel(DFSOutputStream.LOG, Level.TRACE); + GenericTestUtils.setLogLevel(DataStreamer.LOG, Level.TRACE); + GenericTestUtils.setLogLevel(DFSClient.LOG, Level.TRACE); + GenericTestUtils.setLogLevel( + LoggerFactory.getLogger(BlockPlacementPolicy.class), Level.TRACE); + } + + protected final int cellSize = 64 * 1024; // 8k + protected final int stripesPerBlock = 4; + protected ErasureCodingPolicy ecPolicy; + protected int dataBlocks; + protected int parityBlocks; + protected int blockSize; + protected int blockGroupSize; + private int[][] dnIndexSuite; + protected List<Integer> lengths; + protected static final Random RANDOM = new Random(); + MiniDFSCluster cluster; + DistributedFileSystem dfs; + final Path dir = new Path("/" + + TestDFSStripedOutputStreamWithFailureBase.class.getSimpleName()); + protected static final int FLUSH_POS = + 9 * DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_DEFAULT + 1; + + public ECSchema getEcSchema() { + return StripedFileTestUtil.getDefaultECPolicy().getSchema(); + } + + /* + * Initialize erasure coding policy. + */ + @Before + public void init() { + ecPolicy = new ErasureCodingPolicy(getEcSchema(), cellSize); + dataBlocks = ecPolicy.getNumDataUnits(); + parityBlocks = ecPolicy.getNumParityUnits(); + blockSize = cellSize * stripesPerBlock; + blockGroupSize = blockSize * dataBlocks; + dnIndexSuite = getDnIndexSuite(); + lengths = newLengths(); + } + + List<Integer> newLengths() { + final List<Integer> lens = new ArrayList<>(); + lens.add(FLUSH_POS + 2); + for(int b = 0; b <= 2; b++) { + for(int c = 0; c < stripesPerBlock * dataBlocks; c++) { + for(int delta = -1; delta <= 1; delta++) { + final int length = b * blockGroupSize + c * cellSize + delta; + System.out.println(lens.size() + ": length=" + length + + ", (b, c, d) = (" + b + ", " + c + ", " + delta + ")"); + lens.add(length); + } + } + } + return lens; + } + + private int[][] getDnIndexSuite() { + final int maxNumLevel = 2; + final int maxPerLevel = 5; + List<List<Integer>> allLists = new ArrayList<>(); + int numIndex = parityBlocks; + for (int i = 0; i < maxNumLevel && numIndex > 1; i++) { + List<List<Integer>> lists = + combinations(dataBlocks + parityBlocks, numIndex); + if (lists.size() > maxPerLevel) { + Collections.shuffle(lists); + lists = lists.subList(0, maxPerLevel); + } + allLists.addAll(lists); + numIndex--; + } + int[][] dnIndexArray = new int[allLists.size()][]; + for (int i = 0; i < dnIndexArray.length; i++) { + int[] list = new int[allLists.get(i).size()]; + for (int j = 0; j < list.length; j++) { + list[j] = allLists.get(i).get(j); + } + dnIndexArray[i] = list; + } + return dnIndexArray; + } + + // get all combinations of k integers from {0,...,n-1} + private static List<List<Integer>> combinations(int n, int k) { + List<List<Integer>> res = new LinkedList<List<Integer>>(); + if (k >= 1 && n >= k) { + getComb(n, k, new Stack<Integer>(), res); + } + return res; + } + + private static void getComb(int n, int k, Stack<Integer> stack, + List<List<Integer>> res) { + if (stack.size() == k) { + List<Integer> list = new ArrayList<Integer>(stack); + res.add(list); + } else { + int next = stack.empty() ? 0 : stack.peek() + 1; + while (next < n) { + stack.push(next); + getComb(n, k, stack, res); + next++; + } + } + if (!stack.empty()) { + stack.pop(); + } + } + + int[] getKillPositions(int fileLen, int num) { + int[] positions = new int[num]; + for (int i = 0; i < num; i++) { + positions[i] = fileLen * (i + 1) / (num + 1); + } + return positions; + } + + Integer getLength(int i) { + return i >= 0 && i < lengths.size() ? lengths.get(i): null; + } + + void setup(Configuration conf) throws IOException { + System.out.println("NUM_DATA_BLOCKS = " + dataBlocks); + System.out.println("NUM_PARITY_BLOCKS= " + parityBlocks); + System.out.println("CELL_SIZE = " + cellSize + " (=" + + StringUtils.TraditionalBinaryPrefix.long2String(cellSize, "B", 2) + + ")"); + System.out.println("BLOCK_SIZE = " + blockSize + " (=" + + StringUtils.TraditionalBinaryPrefix.long2String(blockSize, "B", 2) + + ")"); + System.out.println("BLOCK_GROUP_SIZE = " + blockGroupSize + " (=" + + StringUtils.TraditionalBinaryPrefix.long2String(blockGroupSize, "B", 2) + + ")"); + final int numDNs = dataBlocks + parityBlocks; + if (ErasureCodeNative.isNativeCodeLoaded()) { + conf.set( + CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY, + NativeRSRawErasureCoderFactory.CODER_NAME); + } + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); + cluster.waitActive(); + dfs = cluster.getFileSystem(); + AddErasureCodingPolicyResponse[] res = + dfs.addErasureCodingPolicies(new ErasureCodingPolicy[]{ecPolicy}); + ecPolicy = res[0].getPolicy(); + dfs.enableErasureCodingPolicy(ecPolicy.getName()); + DFSTestUtil.enableAllECPolicies(dfs); + dfs.mkdirs(dir); + dfs.setErasureCodingPolicy(dir, ecPolicy.getName()); + } + + void tearDown() { + if (cluster != null) { + cluster.shutdown(); + cluster = null; + } + } + + HdfsConfiguration newHdfsConfiguration() { + final HdfsConfiguration conf = new HdfsConfiguration(); + conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); + conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY, + false); + conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1); + conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0); + return conf; + } + + void runTest(final int length) { + final HdfsConfiguration conf = newHdfsConfiguration(); + for (int dn = 0; dn < dataBlocks + parityBlocks; dn++) { + try { + LOG.info("runTest: dn=" + dn + ", length=" + length); + setup(conf); + runTest(length, new int[]{length / 2}, new int[]{dn}, false); + } catch (Throwable e) { + final String err = "failed, dn=" + dn + ", length=" + length + + StringUtils.stringifyException(e); + LOG.error(err); + Assert.fail(err); + } finally { + tearDown(); + } + } + } + + void runTestWithMultipleFailure(final int length) throws Exception { + final HdfsConfiguration conf = newHdfsConfiguration(); + for (int[] dnIndex : dnIndexSuite) { + int[] killPos = getKillPositions(length, dnIndex.length); + try { + LOG.info("runTestWithMultipleFailure: length==" + length + ", killPos=" + + Arrays.toString(killPos) + ", dnIndex=" + + Arrays.toString(dnIndex)); + setup(conf); + runTest(length, killPos, dnIndex, false); + } catch (Throwable e) { + final String err = "failed, killPos=" + Arrays.toString(killPos) + + ", dnIndex=" + Arrays.toString(dnIndex) + ", length=" + length; + LOG.error(err); + throw e; + } finally { + tearDown(); + } + } + } + + /** + * runTest implementation. + * @param length file length + * @param killPos killing positions in ascending order + * @param dnIndex DN index to kill when meets killing positions + * @param tokenExpire wait token to expire when kill a DN + * @throws Exception + */ + void runTest(final int length, final int[] killPos, + final int[] dnIndex, final boolean tokenExpire) throws Exception { + if (killPos[0] <= FLUSH_POS) { + LOG.warn("killPos=" + Arrays.toString(killPos) + " <= FLUSH_POS=" + + FLUSH_POS + ", length=" + length + ", dnIndex=" + + Arrays.toString(dnIndex)); + return; //skip test + } + Preconditions.checkArgument(length > killPos[0], "length=%s <= killPos=%s", + length, killPos); + Preconditions.checkArgument(killPos.length == dnIndex.length); + + final Path p = new Path(dir, "dn" + Arrays.toString(dnIndex) + + "len" + length + "kill" + Arrays.toString(killPos)); + final String fullPath = p.toString(); + LOG.info("fullPath=" + fullPath); + + if (tokenExpire) { + final NameNode nn = cluster.getNameNode(); + final BlockManager bm = nn.getNamesystem().getBlockManager(); + final BlockTokenSecretManager sm = bm.getBlockTokenSecretManager(); + + // set a short token lifetime (6 second) + SecurityTestUtil.setBlockTokenLifetime(sm, 6000L); + } + + final AtomicInteger pos = new AtomicInteger(); + final FSDataOutputStream out = dfs.create(p); + final DFSStripedOutputStream stripedOut + = (DFSStripedOutputStream)out.getWrappedStream(); + + // first GS of this block group which never proceeds blockRecovery + long firstGS = -1; + long oldGS = -1; // the old GS before bumping + List<Long> gsList = new ArrayList<>(); + final List<DatanodeInfo> killedDN = new ArrayList<>(); + int numKilled = 0; + for(; pos.get() < length;) { + final int i = pos.getAndIncrement(); + if (numKilled < killPos.length && i == killPos[numKilled]) { + assertTrue(firstGS != -1); + final long gs = getGenerationStamp(stripedOut); + if (numKilled == 0) { + assertEquals(firstGS, gs); + } else { + //TODO: implement hflush/hsync and verify gs strict greater than oldGS + assertTrue(gs >= oldGS); + } + oldGS = gs; + + if (tokenExpire) { + DFSTestUtil.flushInternal(stripedOut); + waitTokenExpires(out); + } + + killedDN.add( + killDatanode(cluster, stripedOut, dnIndex[numKilled], pos)); + numKilled++; + } + + write(out, i); + + if (i % blockGroupSize == FLUSH_POS) { + firstGS = getGenerationStamp(stripedOut); + oldGS = firstGS; + } + if (i > 0 && (i + 1) % blockGroupSize == 0) { + gsList.add(oldGS); + } + } + gsList.add(oldGS); + out.close(); + assertEquals(dnIndex.length, numKilled); + + StripedFileTestUtil.waitBlockGroupsReported(dfs, fullPath, numKilled); + + cluster.triggerBlockReports(); + StripedFileTestUtil.checkData(dfs, p, length, killedDN, gsList, + blockGroupSize); + } + + static void write(FSDataOutputStream out, int i) throws IOException { + try { + out.write(StripedFileTestUtil.getByte(i)); + } catch(IOException ioe) { + throw new IOException("Failed at i=" + i, ioe); + } + } + + static long getGenerationStamp(DFSStripedOutputStream out) + throws IOException { + final long gs = out.getBlock().getGenerationStamp(); + LOG.info("getGenerationStamp returns " + gs); + return gs; + } + + static DatanodeInfo getDatanodes(StripedDataStreamer streamer) { + for(;;) { + DatanodeInfo[] datanodes = streamer.getNodes(); + if (datanodes == null) { + // try peeking following block. + final LocatedBlock lb = streamer.peekFollowingBlock(); + if (lb != null) { + datanodes = lb.getLocations(); + } + } + + if (datanodes != null) { + Assert.assertEquals(1, datanodes.length); + Assert.assertNotNull(datanodes[0]); + return datanodes[0]; + } + + try { + Thread.sleep(100); + } catch (InterruptedException ie) { + Assert.fail(StringUtils.stringifyException(ie)); + return null; + } + } + } + + static DatanodeInfo killDatanode(MiniDFSCluster cluster, + DFSStripedOutputStream out, final int dnIndex, final AtomicInteger pos) { + final StripedDataStreamer s = out.getStripedDataStreamer(dnIndex); + final DatanodeInfo datanode = getDatanodes(s); + LOG.info("killDatanode " + dnIndex + ": " + datanode + ", pos=" + pos); + if (datanode != null) { + cluster.stopDataNode(datanode.getXferAddr()); + } + return datanode; + } + + private void waitTokenExpires(FSDataOutputStream out) throws IOException { + Token<BlockTokenIdentifier> token = DFSTestUtil.getBlockToken(out); + while (!SecurityTestUtil.isBlockTokenExpired(token)) { + try { + Thread.sleep(10); + } catch (InterruptedException ignored) { + } + } + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
