Repository: hadoop Updated Branches: refs/heads/branch-3.0 7d9ca2e21 -> fdfbeb024
HDFS-12444. Reduce runtime of TestWriteReadStripedFile. Contributed by Huafeng Wang and Andrew Wang. (cherry picked from commit 59830ca772dfb5dcc8b3e5281ca482dea5a5fa3e) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fdfbeb02 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fdfbeb02 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fdfbeb02 Branch: refs/heads/branch-3.0 Commit: fdfbeb02432daa69fa9836b30de9315cb8c19210 Parents: 7d9ca2e Author: Andrew Wang <w...@apache.org> Authored: Tue Sep 19 13:44:42 2017 -0700 Committer: Andrew Wang <w...@apache.org> Committed: Tue Sep 19 13:44:53 2017 -0700 ---------------------------------------------------------------------- .../apache/hadoop/hdfs/StripedFileTestUtil.java | 13 +++++++---- .../hadoop/hdfs/TestWriteReadStripedFile.java | 24 ++++++++++++-------- .../hdfs/TestWriteStripedFileWithFailure.java | 3 ++- 3 files changed, 25 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdfbeb02/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java index 1489e48..c771d21 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java @@ -79,10 +79,15 @@ public class StripedFileTestUtil { assertEquals("File length should be the same", fileLength, status.getLen()); } - static void verifyPread(FileSystem fs, Path srcPath, int fileLength, - byte[] expected, byte[] buf) throws IOException { - final ErasureCodingPolicy ecPolicy = - ((DistributedFileSystem)fs).getErasureCodingPolicy(srcPath); + static void verifyPread(DistributedFileSystem fs, Path srcPath, + int fileLength, byte[] expected, byte[] buf) throws IOException { + final ErasureCodingPolicy ecPolicy = fs.getErasureCodingPolicy(srcPath); + verifyPread(fs, srcPath, fileLength, expected, buf, ecPolicy); + } + + static void verifyPread(FileSystem fs, Path srcPath, int fileLength, + byte[] expected, byte[] buf, ErasureCodingPolicy ecPolicy) + throws IOException { try (FSDataInputStream in = fs.open(srcPath)) { int[] startOffsets = {0, 1, ecPolicy.getCellSize() - 102, ecPolicy.getCellSize(), ecPolicy.getCellSize() + 102, http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdfbeb02/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java index f27c978..805bcea 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java @@ -25,6 +25,7 @@ import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; +import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.web.WebHdfsConstants; @@ -47,12 +48,13 @@ import java.util.Random; public class TestWriteReadStripedFile { public static final Log LOG = LogFactory.getLog(TestWriteReadStripedFile.class); private final ErasureCodingPolicy ecPolicy = - StripedFileTestUtil.getDefaultECPolicy(); + SystemErasureCodingPolicies.getByID( + SystemErasureCodingPolicies.RS_3_2_POLICY_ID); private final int cellSize = ecPolicy.getCellSize(); private final short dataBlocks = (short) ecPolicy.getNumDataUnits(); private final short parityBlocks = (short) ecPolicy.getNumParityUnits(); private final int numDNs = dataBlocks + parityBlocks; - private final int stripesPerBlock = 4; + private final int stripesPerBlock = 2; private final int blockSize = stripesPerBlock * cellSize; private final int blockGroupSize = blockSize * dataBlocks; @@ -78,11 +80,10 @@ public class TestWriteReadStripedFile { false); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); fs = cluster.getFileSystem(); - fs.enableErasureCodingPolicy( - StripedFileTestUtil.getDefaultECPolicy().getName()); + fs.enableErasureCodingPolicy(ecPolicy.getName()); fs.mkdirs(new Path("/ec")); cluster.getFileSystem().getClient().setErasureCodingPolicy("/ec", - StripedFileTestUtil.getDefaultECPolicy().getName()); + ecPolicy.getName()); } @After @@ -225,7 +226,8 @@ public class TestWriteReadStripedFile { byte[] smallBuf = new byte[1024]; byte[] largeBuf = new byte[fileLength + 100]; - StripedFileTestUtil.verifyPread(fs, srcPath, fileLength, expected, largeBuf); + StripedFileTestUtil.verifyPread(fs, srcPath, fileLength, expected, + largeBuf); StripedFileTestUtil.verifyStatefulRead(fs, srcPath, fileLength, expected, largeBuf); @@ -268,13 +270,15 @@ public class TestWriteReadStripedFile { byte[] smallBuf = new byte[1024]; byte[] largeBuf = new byte[fileLength + 100]; - // TODO: HDFS-8797 - //StripedFileTestUtil.verifyPread(fs, srcPath, fileLength, expected, largeBuf); + StripedFileTestUtil + .verifyPread(fs, srcPath, fileLength, expected, largeBuf, ecPolicy); - StripedFileTestUtil.verifyStatefulRead(fs, srcPath, fileLength, expected, largeBuf); + StripedFileTestUtil + .verifyStatefulRead(fs, srcPath, fileLength, expected, largeBuf); StripedFileTestUtil.verifySeek(fs, srcPath, fileLength, ecPolicy, blockGroupSize); - StripedFileTestUtil.verifyStatefulRead(fs, srcPath, fileLength, expected, smallBuf); + StripedFileTestUtil + .verifyStatefulRead(fs, srcPath, fileLength, expected, smallBuf); // webhdfs doesn't support bytebuffer read } http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdfbeb02/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java index 03e9e10..c859b71 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java @@ -160,7 +160,8 @@ public class TestWriteStripedFileWithFailure { blockSize * dataBlocks); StripedFileTestUtil.verifyStatefulRead(fs, srcPath, fileLength, expected, smallBuf); - StripedFileTestUtil.verifyPread(fs, srcPath, fileLength, expected, largeBuf); + StripedFileTestUtil.verifyPread((DistributedFileSystem)fs, srcPath, + fileLength, expected, largeBuf); // delete the file fs.delete(srcPath, true); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
