HADOOP-11706 Refine a little bit erasure coder API
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/671db987 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/671db987 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/671db987 Branch: refs/heads/HDFS-7285 Commit: 671db987ee966e015c47955f96d019602c934c0b Parents: ace0181 Author: Kai Zheng <kai.zh...@intel.com> Authored: Wed Mar 18 19:21:37 2015 +0800 Committer: Zhe Zhang <z...@apache.org> Committed: Mon Mar 30 10:11:28 2015 -0700 ---------------------------------------------------------------------- .../io/erasurecode/coder/ErasureCoder.java | 4 +++- .../erasurecode/rawcoder/RawErasureCoder.java | 4 +++- .../hadoop/io/erasurecode/TestCoderBase.java | 17 +++++++++++++--- .../erasurecode/coder/TestErasureCoderBase.java | 21 +++----------------- .../erasurecode/rawcoder/TestJRSRawCoder.java | 12 +++++------ .../erasurecode/rawcoder/TestRawCoderBase.java | 2 ++ 6 files changed, 31 insertions(+), 29 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/671db987/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java index 68875c0..c5922f3 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.io.erasurecode.coder; +import org.apache.hadoop.conf.Configurable; + /** * An erasure coder to perform encoding or decoding given a group. Generally it * involves calculating necessary internal steps according to codec logic. For @@ -31,7 +33,7 @@ package org.apache.hadoop.io.erasurecode.coder; * of multiple coding steps. * */ -public interface ErasureCoder { +public interface ErasureCoder extends Configurable { /** * Initialize with the important parameters for the code. http://git-wip-us.apache.org/repos/asf/hadoop/blob/671db987/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoder.java index 91a9abf..9af5b6c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoder.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.io.erasurecode.rawcoder; +import org.apache.hadoop.conf.Configurable; + /** * RawErasureCoder is a common interface for {@link RawErasureEncoder} and * {@link RawErasureDecoder} as both encoder and decoder share some properties. @@ -31,7 +33,7 @@ package org.apache.hadoop.io.erasurecode.rawcoder; * low level constructs, since it only takes care of the math calculation with * a group of byte buffers. */ -public interface RawErasureCoder { +public interface RawErasureCoder extends Configurable { /** * Initialize with the important parameters for the code. http://git-wip-us.apache.org/repos/asf/hadoop/blob/671db987/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java index 194413a..22fd98d 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java @@ -17,11 +17,12 @@ */ package org.apache.hadoop.io.erasurecode; +import org.apache.hadoop.conf.Configuration; + import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Random; -import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertTrue; /** @@ -31,6 +32,7 @@ import static org.junit.Assert.assertTrue; public abstract class TestCoderBase { protected static Random RAND = new Random(); + private Configuration conf; protected int numDataUnits; protected int numParityUnits; protected int chunkSize = 16 * 1024; @@ -49,8 +51,9 @@ public abstract class TestCoderBase { * @param numParityUnits * @param erasedIndexes */ - protected void prepare(int numDataUnits, int numParityUnits, - int[] erasedIndexes) { + protected void prepare(Configuration conf, int numDataUnits, + int numParityUnits, int[] erasedIndexes) { + this.conf = conf; this.numDataUnits = numDataUnits; this.numParityUnits = numParityUnits; this.erasedDataIndexes = erasedIndexes != null ? @@ -58,6 +61,14 @@ public abstract class TestCoderBase { } /** + * Get the conf the test. + * @return configuration + */ + protected Configuration getConf() { + return this.conf; + } + + /** * Compare and verify if erased chunks are equal to recovered chunks * @param erasedChunks * @param recoveredChunks http://git-wip-us.apache.org/repos/asf/hadoop/blob/671db987/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java index d911db9..b963a59 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java @@ -17,10 +17,9 @@ */ package org.apache.hadoop.io.erasurecode.coder; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.erasurecode.ECBlock; -import org.apache.hadoop.io.erasurecode.ECChunk; import org.apache.hadoop.io.erasurecode.ECBlockGroup; +import org.apache.hadoop.io.erasurecode.ECChunk; import org.apache.hadoop.io.erasurecode.TestCoderBase; /** @@ -30,7 +29,6 @@ public abstract class TestErasureCoderBase extends TestCoderBase { protected Class<? extends ErasureEncoder> encoderClass; protected Class<? extends ErasureDecoder> decoderClass; - private Configuration conf; protected int numChunksInBlock = 16; /** @@ -48,19 +46,6 @@ public abstract class TestErasureCoderBase extends TestCoderBase { } /** - * Prepare before running the case. - * @param conf - * @param numDataUnits - * @param numParityUnits - * @param erasedIndexes - */ - protected void prepare(Configuration conf, int numDataUnits, - int numParityUnits, int[] erasedIndexes) { - this.conf = conf; - super.prepare(numDataUnits, numParityUnits, erasedIndexes); - } - - /** * Generating source data, encoding, recovering and then verifying. * RawErasureCoder mainly uses ECChunk to pass input and output data buffers, * it supports two kinds of ByteBuffers, one is array backed, the other is @@ -162,7 +147,7 @@ public abstract class TestErasureCoderBase extends TestCoderBase { } encoder.initialize(numDataUnits, numParityUnits, chunkSize); - ((AbstractErasureCoder)encoder).setConf(conf); + encoder.setConf(getConf()); return encoder; } @@ -179,7 +164,7 @@ public abstract class TestErasureCoderBase extends TestCoderBase { } decoder.initialize(numDataUnits, numParityUnits, chunkSize); - ((AbstractErasureCoder)decoder).setConf(conf); + decoder.setConf(getConf()); return decoder; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/671db987/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestJRSRawCoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestJRSRawCoder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestJRSRawCoder.java index e54f647..39e5deb 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestJRSRawCoder.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestJRSRawCoder.java @@ -46,37 +46,37 @@ public class TestJRSRawCoder extends TestRawCoderBase { @Test public void testCodingNoDirectBuffer_10x4() { - prepare(10, 4, null); + prepare(null, 10, 4, null); testCoding(false); } @Test public void testCodingDirectBuffer_10x4() { - prepare(10, 4, null); + prepare(null, 10, 4, null); testCoding(true); } @Test public void testCodingDirectBuffer_10x4_erasure_of_2_4() { - prepare(10, 4, new int[] {2, 4}); + prepare(null, 10, 4, new int[] {2, 4}); testCoding(true); } @Test public void testCodingDirectBuffer_10x4_erasing_all() { - prepare(10, 4, new int[] {0, 1, 2, 3}); + prepare(null, 10, 4, new int[] {0, 1, 2, 3}); testCoding(true); } @Test public void testCodingNoDirectBuffer_3x3() { - prepare(3, 3, null); + prepare(null, 3, 3, null); testCoding(false); } @Test public void testCodingDirectBuffer_3x3() { - prepare(3, 3, null); + prepare(null, 3, 3, null); testCoding(true); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/671db987/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java index 890f632..b036eed 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java @@ -86,6 +86,7 @@ public abstract class TestRawCoderBase extends TestCoderBase { } encoder.initialize(numDataUnits, numParityUnits, chunkSize); + encoder.setConf(getConf()); return encoder; } @@ -102,6 +103,7 @@ public abstract class TestRawCoderBase extends TestCoderBase { } decoder.initialize(numDataUnits, numParityUnits, chunkSize); + decoder.setConf(getConf()); return decoder; }