HDFS-8382 Remove chunkSize and initialize from erasure coder. Contributed by Kai Zheng
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b30e96bf Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b30e96bf Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b30e96bf Branch: refs/heads/HDFS-7285 Commit: b30e96bfb4b8ce5537671c97f0c9c56cd195bfdc Parents: 0ed92e5 Author: Kai Zheng <[email protected]> Authored: Mon May 25 16:13:29 2015 +0800 Committer: Zhe Zhang <[email protected]> Committed: Tue May 26 12:07:15 2015 -0700 ---------------------------------------------------------------------- .../erasurecode/codec/AbstractErasureCodec.java | 43 ++------------------ .../io/erasurecode/codec/ErasureCodec.java | 7 ---- .../io/erasurecode/codec/RSErasureCodec.java | 13 ++++-- .../io/erasurecode/codec/XORErasureCodec.java | 13 +++--- .../erasurecode/coder/AbstractErasureCoder.java | 39 ++++++++---------- .../coder/AbstractErasureDecoder.java | 9 ++++ .../coder/AbstractErasureEncoder.java | 9 ++++ .../io/erasurecode/coder/ErasureCoder.java | 27 ++---------- .../io/erasurecode/coder/RSErasureDecoder.java | 16 ++++++-- .../io/erasurecode/coder/RSErasureEncoder.java | 16 ++++++-- .../io/erasurecode/coder/XORErasureDecoder.java | 14 +++++-- .../io/erasurecode/coder/XORErasureEncoder.java | 16 ++++++-- .../rawcoder/AbstractRawErasureCoder.java | 15 ++----- .../rawcoder/AbstractRawErasureDecoder.java | 4 ++ .../rawcoder/AbstractRawErasureEncoder.java | 4 ++ .../io/erasurecode/rawcoder/RSRawDecoder.java | 5 +-- .../io/erasurecode/rawcoder/RSRawEncoder.java | 6 +-- .../rawcoder/RSRawErasureCoderFactory.java | 8 ++-- .../erasurecode/rawcoder/RawErasureCoder.java | 14 ------- .../rawcoder/RawErasureCoderFactory.java | 8 +++- .../io/erasurecode/rawcoder/XORRawDecoder.java | 4 ++ .../io/erasurecode/rawcoder/XORRawEncoder.java | 4 ++ .../rawcoder/XORRawErasureCoderFactory.java | 8 ++-- .../erasurecode/coder/TestErasureCoderBase.java | 42 +++++++++++-------- .../erasurecode/rawcoder/TestRawCoderBase.java | 15 ++++--- .../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt | 2 + .../hadoop/hdfs/DFSStripedOutputStream.java | 3 +- .../erasurecode/ErasureCodingWorker.java | 14 +++---- .../hadoop/hdfs/util/StripedBlockUtil.java | 3 +- .../hadoop/hdfs/TestDFSStripedOutputStream.java | 4 +- .../hadoop/hdfs/TestWriteReadStripedFile.java | 3 +- 31 files changed, 187 insertions(+), 201 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b30e96bf/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/AbstractErasureCodec.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/AbstractErasureCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/AbstractErasureCodec.java index 9993786..0cacfbc 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/AbstractErasureCodec.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/AbstractErasureCodec.java @@ -19,7 +19,6 @@ package org.apache.hadoop.io.erasurecode.codec; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.io.erasurecode.ECSchema; -import org.apache.hadoop.io.erasurecode.coder.*; import org.apache.hadoop.io.erasurecode.grouper.BlockGrouper; /** @@ -28,10 +27,9 @@ import org.apache.hadoop.io.erasurecode.grouper.BlockGrouper; public abstract class AbstractErasureCodec extends Configured implements ErasureCodec { - private ECSchema schema; + private final ECSchema schema; - @Override - public void setSchema(ECSchema schema) { + public AbstractErasureCodec(ECSchema schema) { this.schema = schema; } @@ -39,7 +37,7 @@ public abstract class AbstractErasureCodec extends Configured return schema.getCodecName(); } - protected ECSchema getSchema() { + public ECSchema getSchema() { return schema; } @@ -50,39 +48,4 @@ public abstract class AbstractErasureCodec extends Configured return blockGrouper; } - - @Override - public ErasureCoder createEncoder() { - ErasureCoder encoder = doCreateEncoder(); - prepareErasureCoder(encoder); - return encoder; - } - - /** - * Create a new encoder instance to be initialized afterwards. - * @return encoder - */ - protected abstract ErasureCoder doCreateEncoder(); - - @Override - public ErasureCoder createDecoder() { - ErasureCoder decoder = doCreateDecoder(); - prepareErasureCoder(decoder); - return decoder; - } - - /** - * Create a new decoder instance to be initialized afterwards. - * @return decoder - */ - protected abstract ErasureCoder doCreateDecoder(); - - private void prepareErasureCoder(ErasureCoder erasureCoder) { - if (getSchema() == null) { - throw new RuntimeException("No schema been set yet"); - } - - erasureCoder.setConf(getConf()); - erasureCoder.initialize(getSchema()); - } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b30e96bf/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/ErasureCodec.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/ErasureCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/ErasureCodec.java index e639484..9aa3db2 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/ErasureCodec.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/ErasureCodec.java @@ -18,7 +18,6 @@ package org.apache.hadoop.io.erasurecode.codec; import org.apache.hadoop.conf.Configurable; -import org.apache.hadoop.io.erasurecode.ECSchema; import org.apache.hadoop.io.erasurecode.coder.ErasureCoder; import org.apache.hadoop.io.erasurecode.grouper.BlockGrouper; @@ -30,12 +29,6 @@ import org.apache.hadoop.io.erasurecode.grouper.BlockGrouper; public interface ErasureCodec extends Configurable { /** - * Set EC schema to be used by this codec. - * @param schema - */ - public void setSchema(ECSchema schema); - - /** * Create block grouper * @return block grouper */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/b30e96bf/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/RSErasureCodec.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/RSErasureCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/RSErasureCodec.java index 9e91b60..6edd638 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/RSErasureCodec.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/RSErasureCodec.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.io.erasurecode.codec; +import org.apache.hadoop.io.erasurecode.ECSchema; import org.apache.hadoop.io.erasurecode.coder.ErasureCoder; import org.apache.hadoop.io.erasurecode.coder.RSErasureDecoder; import org.apache.hadoop.io.erasurecode.coder.RSErasureEncoder; @@ -26,13 +27,17 @@ import org.apache.hadoop.io.erasurecode.coder.RSErasureEncoder; */ public class RSErasureCodec extends AbstractErasureCodec { + public RSErasureCodec(ECSchema schema) { + super(schema); + } + @Override - protected ErasureCoder doCreateEncoder() { - return new RSErasureEncoder(); + public ErasureCoder createEncoder() { + return new RSErasureEncoder(getSchema()); } @Override - protected ErasureCoder doCreateDecoder() { - return new RSErasureDecoder(); + public ErasureCoder createDecoder() { + return new RSErasureDecoder(getSchema()); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b30e96bf/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/XORErasureCodec.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/XORErasureCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/XORErasureCodec.java index 0f726d7..e2dcfa7 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/XORErasureCodec.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/XORErasureCodec.java @@ -27,19 +27,18 @@ import org.apache.hadoop.io.erasurecode.coder.XORErasureEncoder; */ public class XORErasureCodec extends AbstractErasureCodec { - @Override - public void setSchema(ECSchema schema) { - super.setSchema(schema); + public XORErasureCodec(ECSchema schema) { + super(schema); assert(schema.getNumParityUnits() == 1); } @Override - protected ErasureCoder doCreateEncoder() { - return new XORErasureEncoder(); + public ErasureCoder createEncoder() { + return new XORErasureEncoder(getSchema()); } @Override - protected ErasureCoder doCreateDecoder() { - return new XORErasureDecoder(); + public ErasureCoder createDecoder() { + return new XORErasureDecoder(getSchema()); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b30e96bf/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java index d491570..c572bad 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java @@ -33,18 +33,18 @@ import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder; public abstract class AbstractErasureCoder extends Configured implements ErasureCoder { - private int numDataUnits; - private int numParityUnits; - private int chunkSize; + private final int numDataUnits; + private final int numParityUnits; /** * Create raw decoder using the factory specified by rawCoderFactoryKey * @param rawCoderFactoryKey * @return raw decoder */ - protected RawErasureDecoder createRawDecoder(String rawCoderFactoryKey) { + protected RawErasureDecoder createRawDecoder( + String rawCoderFactoryKey, int dataUnitsCount, int parityUnitsCount) { RawErasureCoder rawCoder = createRawCoder(getConf(), - rawCoderFactoryKey, false); + rawCoderFactoryKey, false, dataUnitsCount, parityUnitsCount); return (RawErasureDecoder) rawCoder; } @@ -53,9 +53,10 @@ public abstract class AbstractErasureCoder * @param rawCoderFactoryKey * @return raw encoder */ - protected RawErasureEncoder createRawEncoder(String rawCoderFactoryKey) { + protected RawErasureEncoder createRawEncoder( + String rawCoderFactoryKey, int dataUnitsCount, int parityUnitsCount) { RawErasureCoder rawCoder = createRawCoder(getConf(), - rawCoderFactoryKey, true); + rawCoderFactoryKey, true, dataUnitsCount, parityUnitsCount); return (RawErasureEncoder) rawCoder; } @@ -67,7 +68,8 @@ public abstract class AbstractErasureCoder * @return raw coder */ public static RawErasureCoder createRawCoder(Configuration conf, - String rawCoderFactoryKey, boolean isEncoder) { + String rawCoderFactoryKey, boolean isEncoder, int numDataUnits, + int numParityUnits) { if (conf == null) { return null; @@ -90,21 +92,17 @@ public abstract class AbstractErasureCoder throw new RuntimeException("Failed to create raw coder", e); } - return isEncoder ? fact.createEncoder() : fact.createDecoder(); + return isEncoder ? fact.createEncoder(numDataUnits, numParityUnits) : + fact.createDecoder(numDataUnits, numParityUnits); } - @Override - public void initialize(int numDataUnits, int numParityUnits, - int chunkSize) { + public AbstractErasureCoder(int numDataUnits, int numParityUnits) { this.numDataUnits = numDataUnits; this.numParityUnits = numParityUnits; - this.chunkSize = chunkSize; } - @Override - public void initialize(ECSchema schema) { - initialize(schema.getNumDataUnits(), schema.getNumParityUnits(), - schema.getChunkSize()); + public AbstractErasureCoder(ECSchema schema) { + this(schema.getNumDataUnits(), schema.getNumParityUnits()); } @Override @@ -118,12 +116,7 @@ public abstract class AbstractErasureCoder } @Override - public int getChunkSize() { - return chunkSize; - } - - @Override - public boolean preferNativeBuffer() { + public boolean preferDirectBuffer() { return false; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b30e96bf/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java index 6437236..3ea9311 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java @@ -19,6 +19,7 @@ package org.apache.hadoop.io.erasurecode.coder; import org.apache.hadoop.io.erasurecode.ECBlock; import org.apache.hadoop.io.erasurecode.ECBlockGroup; +import org.apache.hadoop.io.erasurecode.ECSchema; /** * An abstract erasure decoder that's to be inherited by new decoders. @@ -27,6 +28,14 @@ import org.apache.hadoop.io.erasurecode.ECBlockGroup; */ public abstract class AbstractErasureDecoder extends AbstractErasureCoder { + public AbstractErasureDecoder(int numDataUnits, int numParityUnits) { + super(numDataUnits, numParityUnits); + } + + public AbstractErasureDecoder(ECSchema schema) { + super(schema); + } + @Override public ErasureCodingStep calculateCoding(ECBlockGroup blockGroup) { // We may have more than this when considering complicate cases. HADOOP-11550 http://git-wip-us.apache.org/repos/asf/hadoop/blob/b30e96bf/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureEncoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureEncoder.java index a836b75..7c887e8 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureEncoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureEncoder.java @@ -19,6 +19,7 @@ package org.apache.hadoop.io.erasurecode.coder; import org.apache.hadoop.io.erasurecode.ECBlock; import org.apache.hadoop.io.erasurecode.ECBlockGroup; +import org.apache.hadoop.io.erasurecode.ECSchema; /** * An abstract erasure encoder that's to be inherited by new encoders. @@ -27,6 +28,14 @@ import org.apache.hadoop.io.erasurecode.ECBlockGroup; */ public abstract class AbstractErasureEncoder extends AbstractErasureCoder { + public AbstractErasureEncoder(int numDataUnits, int numParityUnits) { + super(numDataUnits, numParityUnits); + } + + public AbstractErasureEncoder(ECSchema schema) { + super(schema); + } + @Override public ErasureCodingStep calculateCoding(ECBlockGroup blockGroup) { // We may have more than this when considering complicate cases. HADOOP-11550 http://git-wip-us.apache.org/repos/asf/hadoop/blob/b30e96bf/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java index 64a82ea..f05ea41 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java @@ -19,7 +19,6 @@ package org.apache.hadoop.io.erasurecode.coder; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.io.erasurecode.ECBlockGroup; -import org.apache.hadoop.io.erasurecode.ECSchema; /** * An erasure coder to perform encoding or decoding given a group. Generally it @@ -38,20 +37,6 @@ import org.apache.hadoop.io.erasurecode.ECSchema; public interface ErasureCoder extends Configurable { /** - * Initialize with the important parameters for the code. - * @param numDataUnits how many data inputs for the coding - * @param numParityUnits how many parity outputs the coding generates - * @param chunkSize the size of the input/output buffer - */ - public void initialize(int numDataUnits, int numParityUnits, int chunkSize); - - /** - * Initialize with an EC schema. - * @param schema - */ - public void initialize(ECSchema schema); - - /** * The number of data input units for the coding. A unit can be a byte, * chunk or buffer or even a block. * @return count of data input units @@ -66,12 +51,6 @@ public interface ErasureCoder extends Configurable { public int getNumParityUnits(); /** - * Chunk buffer size for the input/output - * @return chunk buffer size - */ - public int getChunkSize(); - - /** * Calculate the encoding or decoding steps given a block blockGroup. * * Note, currently only one coding step is supported. Will support complex @@ -83,13 +62,13 @@ public interface ErasureCoder extends Configurable { public ErasureCodingStep calculateCoding(ECBlockGroup blockGroup); /** - * Tell if native or off-heap buffer is preferred or not. It's for callers to + * Tell if direct or off-heap buffer is preferred or not. It's for callers to * decide how to allocate coding chunk buffers, either on heap or off heap. * It will return false by default. - * @return true if native buffer is preferred for performance consideration, + * @return true if direct buffer is preferred for performance consideration, * otherwise false. */ - public boolean preferNativeBuffer(); + public boolean preferDirectBuffer(); /** * Release the resources if any. Good chance to invoke RawErasureCoder#release. http://git-wip-us.apache.org/repos/asf/hadoop/blob/b30e96bf/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java index ec7cbb5..57f4373 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java @@ -20,6 +20,7 @@ package org.apache.hadoop.io.erasurecode.coder; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.io.erasurecode.ECBlock; import org.apache.hadoop.io.erasurecode.ECBlockGroup; +import org.apache.hadoop.io.erasurecode.ECSchema; import org.apache.hadoop.io.erasurecode.rawcoder.RSRawDecoder; import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder; @@ -31,6 +32,14 @@ import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder; public class RSErasureDecoder extends AbstractErasureDecoder { private RawErasureDecoder rsRawDecoder; + public RSErasureDecoder(int numDataUnits, int numParityUnits) { + super(numDataUnits, numParityUnits); + } + + public RSErasureDecoder(ECSchema schema) { + super(schema); + } + @Override protected ErasureCodingStep prepareDecodingStep(final ECBlockGroup blockGroup) { @@ -45,12 +54,11 @@ public class RSErasureDecoder extends AbstractErasureDecoder { private RawErasureDecoder checkCreateRSRawDecoder() { if (rsRawDecoder == null) { rsRawDecoder = createRawDecoder( - CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY); + CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY, + getNumDataUnits(), getNumParityUnits()); if (rsRawDecoder == null) { - rsRawDecoder = new RSRawDecoder(); + rsRawDecoder = new RSRawDecoder(getNumDataUnits(), getNumParityUnits()); } - rsRawDecoder.initialize(getNumDataUnits(), - getNumParityUnits(), getChunkSize()); } return rsRawDecoder; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b30e96bf/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java index 18ca5ac..ab23474 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java @@ -20,6 +20,7 @@ package org.apache.hadoop.io.erasurecode.coder; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.io.erasurecode.ECBlock; import org.apache.hadoop.io.erasurecode.ECBlockGroup; +import org.apache.hadoop.io.erasurecode.ECSchema; import org.apache.hadoop.io.erasurecode.rawcoder.RSRawEncoder; import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder; @@ -31,6 +32,14 @@ import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder; public class RSErasureEncoder extends AbstractErasureEncoder { private RawErasureEncoder rawEncoder; + public RSErasureEncoder(int numDataUnits, int numParityUnits) { + super(numDataUnits, numParityUnits); + } + + public RSErasureEncoder(ECSchema schema) { + super(schema); + } + @Override protected ErasureCodingStep prepareEncodingStep(final ECBlockGroup blockGroup) { @@ -45,12 +54,11 @@ public class RSErasureEncoder extends AbstractErasureEncoder { private RawErasureEncoder checkCreateRSRawEncoder() { if (rawEncoder == null) { rawEncoder = createRawEncoder( - CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY); + CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY, + getNumDataUnits(), getNumParityUnits()); if (rawEncoder == null) { - rawEncoder = new RSRawEncoder(); + rawEncoder = new RSRawEncoder(getNumDataUnits(), getNumParityUnits()); } - rawEncoder.initialize(getNumDataUnits(), - getNumParityUnits(), getChunkSize()); } return rawEncoder; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b30e96bf/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XORErasureDecoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XORErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XORErasureDecoder.java index 0672549..3fe8d1b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XORErasureDecoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XORErasureDecoder.java @@ -19,6 +19,7 @@ package org.apache.hadoop.io.erasurecode.coder; import org.apache.hadoop.io.erasurecode.ECBlock; import org.apache.hadoop.io.erasurecode.ECBlockGroup; +import org.apache.hadoop.io.erasurecode.ECSchema; import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder; import org.apache.hadoop.io.erasurecode.rawcoder.XORRawDecoder; @@ -29,12 +30,19 @@ import org.apache.hadoop.io.erasurecode.rawcoder.XORRawDecoder; */ public class XORErasureDecoder extends AbstractErasureDecoder { + public XORErasureDecoder(int numDataUnits, int numParityUnits) { + super(numDataUnits, numParityUnits); + } + + public XORErasureDecoder(ECSchema schema) { + super(schema); + } + @Override protected ErasureCodingStep prepareDecodingStep(final ECBlockGroup blockGroup) { // May be configured - RawErasureDecoder rawDecoder = new XORRawDecoder(); - rawDecoder.initialize(getNumDataUnits(), - getNumParityUnits(), getChunkSize()); + RawErasureDecoder rawDecoder = new XORRawDecoder( + getNumDataUnits(), getNumParityUnits()); ECBlock[] inputBlocks = getInputBlocks(blockGroup); http://git-wip-us.apache.org/repos/asf/hadoop/blob/b30e96bf/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XORErasureEncoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XORErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XORErasureEncoder.java index 9011857..5020896 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XORErasureEncoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XORErasureEncoder.java @@ -19,22 +19,30 @@ package org.apache.hadoop.io.erasurecode.coder; import org.apache.hadoop.io.erasurecode.ECBlock; import org.apache.hadoop.io.erasurecode.ECBlockGroup; +import org.apache.hadoop.io.erasurecode.ECSchema; import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder; import org.apache.hadoop.io.erasurecode.rawcoder.XORRawEncoder; /** * Xor erasure encoder that encodes a block group. * - * It implements {@link ErasureEncoder}. + * It implements {@link ErasureCoder}. */ public class XORErasureEncoder extends AbstractErasureEncoder { + public XORErasureEncoder(int numDataUnits, int numParityUnits) { + super(numDataUnits, numParityUnits); + } + + public XORErasureEncoder(ECSchema schema) { + super(schema); + } + @Override protected ErasureCodingStep prepareEncodingStep(final ECBlockGroup blockGroup) { // May be configured - RawErasureEncoder rawEncoder = new XORRawEncoder(); - rawEncoder.initialize(getNumDataUnits(), - getNumParityUnits(), getChunkSize()); + RawErasureEncoder rawEncoder = new XORRawEncoder( + getNumDataUnits(), getNumParityUnits()); ECBlock[] inputBlocks = getInputBlocks(blockGroup); http://git-wip-us.apache.org/repos/asf/hadoop/blob/b30e96bf/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java index 5268962..06ae660 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java @@ -31,16 +31,12 @@ import java.util.Arrays; public abstract class AbstractRawErasureCoder extends Configured implements RawErasureCoder { - private int numDataUnits; - private int numParityUnits; - private int chunkSize; + private final int numDataUnits; + private final int numParityUnits; - @Override - public void initialize(int numDataUnits, int numParityUnits, - int chunkSize) { + public AbstractRawErasureCoder(int numDataUnits, int numParityUnits) { this.numDataUnits = numDataUnits; this.numParityUnits = numParityUnits; - this.chunkSize = chunkSize; } @Override @@ -54,11 +50,6 @@ public abstract class AbstractRawErasureCoder } @Override - public int getChunkSize() { - return chunkSize; - } - - @Override public boolean preferDirectBuffer() { return false; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b30e96bf/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureDecoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureDecoder.java index 31f4fb8..0c1f80f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureDecoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureDecoder.java @@ -30,6 +30,10 @@ import java.nio.ByteBuffer; public abstract class AbstractRawErasureDecoder extends AbstractRawErasureCoder implements RawErasureDecoder { + public AbstractRawErasureDecoder(int numDataUnits, int numParityUnits) { + super(numDataUnits, numParityUnits); + } + @Override public void decode(ByteBuffer[] inputs, int[] erasedIndexes, ByteBuffer[] outputs) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/b30e96bf/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureEncoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureEncoder.java index 0ae54c5..c7a136b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureEncoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureEncoder.java @@ -30,6 +30,10 @@ import java.nio.ByteBuffer; public abstract class AbstractRawErasureEncoder extends AbstractRawErasureCoder implements RawErasureEncoder { + public AbstractRawErasureEncoder(int numDataUnits, int numParityUnits) { + super(numDataUnits, numParityUnits); + } + @Override public void encode(ByteBuffer[] inputs, ByteBuffer[] outputs) { checkParameters(inputs, outputs); http://git-wip-us.apache.org/repos/asf/hadoop/blob/b30e96bf/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawDecoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawDecoder.java index ff1162f..e265dce 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawDecoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawDecoder.java @@ -31,9 +31,8 @@ public class RSRawDecoder extends AbstractRawErasureDecoder { private int[] errSignature; private int[] primitivePower; - @Override - public void initialize(int numDataUnits, int numParityUnits, int chunkSize) { - super.initialize(numDataUnits, numParityUnits, chunkSize); + public RSRawDecoder(int numDataUnits, int numParityUnits) { + super(numDataUnits, numParityUnits); assert (getNumDataUnits() + getNumParityUnits() < RSUtil.GF.getFieldSize()); this.errSignature = new int[numParityUnits]; http://git-wip-us.apache.org/repos/asf/hadoop/blob/b30e96bf/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawEncoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawEncoder.java index 9136331..efeee90 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawEncoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawEncoder.java @@ -29,9 +29,9 @@ import java.nio.ByteBuffer; public class RSRawEncoder extends AbstractRawErasureEncoder { private int[] generatingPolynomial; - @Override - public void initialize(int numDataUnits, int numParityUnits, int chunkSize) { - super.initialize(numDataUnits, numParityUnits, chunkSize); + public RSRawEncoder(int numDataUnits, int numParityUnits) { + super(numDataUnits, numParityUnits); + assert (getNumDataUnits() + getNumParityUnits() < RSUtil.GF.getFieldSize()); int[] primitivePower = RSUtil.getPrimitivePower(numDataUnits, http://git-wip-us.apache.org/repos/asf/hadoop/blob/b30e96bf/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawErasureCoderFactory.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawErasureCoderFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawErasureCoderFactory.java index 19a95af..5db49e3 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawErasureCoderFactory.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawErasureCoderFactory.java @@ -23,12 +23,12 @@ package org.apache.hadoop.io.erasurecode.rawcoder; public class RSRawErasureCoderFactory implements RawErasureCoderFactory { @Override - public RawErasureEncoder createEncoder() { - return new RSRawEncoder(); + public RawErasureEncoder createEncoder(int numDataUnits, int numParityUnits) { + return new RSRawEncoder(numDataUnits, numParityUnits); } @Override - public RawErasureDecoder createDecoder() { - return new RSRawDecoder(); + public RawErasureDecoder createDecoder(int numDataUnits, int numParityUnits) { + return new RSRawDecoder(numDataUnits, numParityUnits); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b30e96bf/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoder.java index 3fb211f..0ddb460 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoder.java @@ -36,14 +36,6 @@ import org.apache.hadoop.conf.Configurable; public interface RawErasureCoder extends Configurable { /** - * Initialize with the important parameters for the code. - * @param numDataUnits how many data inputs for the coding - * @param numParityUnits how many parity outputs the coding generates - * @param chunkSize the size of the input/output buffer - */ - public void initialize(int numDataUnits, int numParityUnits, int chunkSize); - - /** * The number of data input units for the coding. A unit can be a byte, * chunk or buffer or even a block. * @return count of data input units @@ -58,12 +50,6 @@ public interface RawErasureCoder extends Configurable { public int getNumParityUnits(); /** - * Chunk buffer size for the input/output - * @return chunk buffer size - */ - public int getChunkSize(); - - /** * Tell if direct buffer is preferred or not. It's for callers to * decide how to allocate coding chunk buffers, using DirectByteBuffer or * bytes array. It will return false by default. http://git-wip-us.apache.org/repos/asf/hadoop/blob/b30e96bf/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderFactory.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderFactory.java index 95a1cfe..26eddfc 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderFactory.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderFactory.java @@ -26,13 +26,17 @@ public interface RawErasureCoderFactory { /** * Create raw erasure encoder. + * @param numDataUnits + * @param numParityUnits * @return raw erasure encoder */ - public RawErasureEncoder createEncoder(); + public RawErasureEncoder createEncoder(int numDataUnits, int numParityUnits); /** * Create raw erasure decoder. + * @param numDataUnits + * @param numParityUnits * @return raw erasure decoder */ - public RawErasureDecoder createDecoder(); + public RawErasureDecoder createDecoder(int numDataUnits, int numParityUnits); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b30e96bf/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawDecoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawDecoder.java index bf6e894..a09105c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawDecoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawDecoder.java @@ -28,6 +28,10 @@ import java.nio.ByteBuffer; */ public class XORRawDecoder extends AbstractRawErasureDecoder { + public XORRawDecoder(int numDataUnits, int numParityUnits) { + super(numDataUnits, numParityUnits); + } + @Override protected void doDecode(ByteBuffer[] inputs, int[] erasedIndexes, ByteBuffer[] outputs) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/b30e96bf/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawEncoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawEncoder.java index feffbbf..894f20c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawEncoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawEncoder.java @@ -28,6 +28,10 @@ import java.nio.ByteBuffer; */ public class XORRawEncoder extends AbstractRawErasureEncoder { + public XORRawEncoder(int numDataUnits, int numParityUnits) { + super(numDataUnits, numParityUnits); + } + protected void doEncode(ByteBuffer[] inputs, ByteBuffer[] outputs) { ByteBuffer output = outputs[0]; resetOutputBuffer(output); http://git-wip-us.apache.org/repos/asf/hadoop/blob/b30e96bf/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawErasureCoderFactory.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawErasureCoderFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawErasureCoderFactory.java index 67f45c1..de20c95 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawErasureCoderFactory.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawErasureCoderFactory.java @@ -23,12 +23,12 @@ package org.apache.hadoop.io.erasurecode.rawcoder; public class XORRawErasureCoderFactory implements RawErasureCoderFactory { @Override - public RawErasureEncoder createEncoder() { - return new XORRawEncoder(); + public RawErasureEncoder createEncoder(int numDataUnits, int numParityUnits) { + return new XORRawEncoder(numDataUnits, numParityUnits); } @Override - public RawErasureDecoder createDecoder() { - return new XORRawDecoder(); + public RawErasureDecoder createDecoder(int numDataUnits, int numParityUnits) { + return new XORRawDecoder(numDataUnits, numParityUnits); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b30e96bf/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java index 154ec18..f9666b6 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java @@ -22,6 +22,8 @@ import org.apache.hadoop.io.erasurecode.ECBlockGroup; import org.apache.hadoop.io.erasurecode.ECChunk; import org.apache.hadoop.io.erasurecode.TestCoderBase; +import java.lang.reflect.Constructor; + /** * Erasure coder test base with utilities. */ @@ -139,46 +141,50 @@ public abstract class TestErasureCoderBase extends TestCoderBase { } } + private void prepareCoders() { + if (encoder == null) { + encoder = createEncoder(); + } + + if (decoder == null) { + decoder = createDecoder(); + } + } + /** - * Create erasure encoder for test. + * Create the raw erasure encoder to test * @return */ - private ErasureCoder createEncoder() { + protected ErasureCoder createEncoder() { ErasureCoder encoder; try { - encoder = encoderClass.newInstance(); + Constructor<? extends ErasureCoder> constructor = + (Constructor<? extends ErasureCoder>) + encoderClass.getConstructor(int.class, int.class); + encoder = constructor.newInstance(numDataUnits, numParityUnits); } catch (Exception e) { throw new RuntimeException("Failed to create encoder", e); } - encoder.initialize(numDataUnits, numParityUnits, getChunkSize()); encoder.setConf(getConf()); return encoder; } - private void prepareCoders() { - if (encoder == null) { - encoder = createEncoder(); - } - - if (decoder == null) { - decoder = createDecoder(); - } - } - /** - * Create the erasure decoder for the test. + * create the raw erasure decoder to test * @return */ - private ErasureCoder createDecoder() { + protected ErasureCoder createDecoder() { ErasureCoder decoder; try { - decoder = decoderClass.newInstance(); + Constructor<? extends ErasureCoder> constructor = + (Constructor<? extends ErasureCoder>) + decoderClass.getConstructor(int.class, int.class); + decoder = constructor.newInstance(numDataUnits, numParityUnits); } catch (Exception e) { throw new RuntimeException("Failed to create decoder", e); } - decoder.initialize(numDataUnits, numParityUnits, getChunkSize()); decoder.setConf(getConf()); return decoder; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b30e96bf/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java index 4582317..cfaa2c5 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java @@ -21,6 +21,8 @@ import org.apache.hadoop.io.erasurecode.ECChunk; import org.apache.hadoop.io.erasurecode.TestCoderBase; import org.junit.Assert; +import java.lang.reflect.Constructor; + /** * Raw coder test base with utilities. */ @@ -136,12 +138,14 @@ public abstract class TestRawCoderBase extends TestCoderBase { protected RawErasureEncoder createEncoder() { RawErasureEncoder encoder; try { - encoder = encoderClass.newInstance(); + Constructor<? extends RawErasureEncoder> constructor = + (Constructor<? extends RawErasureEncoder>) + encoderClass.getConstructor(int.class, int.class); + encoder = constructor.newInstance(numDataUnits, numParityUnits); } catch (Exception e) { throw new RuntimeException("Failed to create encoder", e); } - encoder.initialize(numDataUnits, numParityUnits, getChunkSize()); encoder.setConf(getConf()); return encoder; } @@ -153,14 +157,15 @@ public abstract class TestRawCoderBase extends TestCoderBase { protected RawErasureDecoder createDecoder() { RawErasureDecoder decoder; try { - decoder = decoderClass.newInstance(); + Constructor<? extends RawErasureDecoder> constructor = + (Constructor<? extends RawErasureDecoder>) + decoderClass.getConstructor(int.class, int.class); + decoder = constructor.newInstance(numDataUnits, numParityUnits); } catch (Exception e) { throw new RuntimeException("Failed to create decoder", e); } - decoder.initialize(numDataUnits, numParityUnits, getChunkSize()); decoder.setConf(getConf()); return decoder; } - } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b30e96bf/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt index efbadaf..4609fb6 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt @@ -255,3 +255,5 @@ (waltersu4549) HDFS-7768. Change fsck to support EC files. (Takanobu Asanuma via szetszwo) + + HDFS-8382. Remove chunkSize and initialize from erasure coder. (Kai Zheng) http://git-wip-us.apache.org/repos/asf/hadoop/blob/b30e96bf/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java index 515ce0c..0935d5c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java @@ -247,8 +247,7 @@ public class DFSStripedOutputStream extends DFSOutputStream { numDataBlocks = schema.getNumDataUnits(); numAllBlocks = numDataBlocks + numParityBlocks; - encoder = new RSRawEncoder(); - encoder.initialize(numDataBlocks, numParityBlocks, cellSize); + encoder = new RSRawEncoder(numDataBlocks, numParityBlocks); coordinator = new Coordinator(dfsClient.getConf(), numDataBlocks, numAllBlocks); try { http://git-wip-us.apache.org/repos/asf/hadoop/blob/b30e96bf/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java index ded51eb..00cf0fd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java @@ -111,12 +111,12 @@ public final class ErasureCodingWorker { DFSConfigKeys.DFS_DATANODE_STRIPED_READ_BUFFER_SIZE_DEFAULT); } - private RawErasureEncoder newEncoder() { - return new RSRawEncoder(); + private RawErasureEncoder newEncoder(int numDataUnits, int numParityUnits) { + return new RSRawEncoder(numDataUnits, numParityUnits); } - private RawErasureDecoder newDecoder() { - return new RSRawDecoder(); + private RawErasureDecoder newDecoder(int numDataUnits, int numParityUnits) { + return new RSRawDecoder(numDataUnits, numParityUnits); } private void initializeStripedReadThreadPool(int num) { @@ -517,16 +517,14 @@ public final class ErasureCodingWorker { // Initialize encoder private void initEncoderIfNecessary() { if (encoder == null) { - encoder = newEncoder(); - encoder.initialize(dataBlkNum, parityBlkNum, bufferSize); + encoder = newEncoder(dataBlkNum, parityBlkNum); } } // Initialize decoder private void initDecoderIfNecessary() { if (decoder == null) { - decoder = newDecoder(); - decoder.initialize(dataBlkNum, parityBlkNum, bufferSize); + decoder = newDecoder(dataBlkNum, parityBlkNum); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b30e96bf/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java index 38dc61a..8f63236 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java @@ -287,8 +287,7 @@ public class StripedBlockUtil { } byte[][] outputs = new byte[parityBlkNum][(int) alignedStripe.getSpanInBlock()]; - RSRawDecoder rsRawDecoder = new RSRawDecoder(); - rsRawDecoder.initialize(dataBlkNum, parityBlkNum, (int) alignedStripe.getSpanInBlock()); + RSRawDecoder rsRawDecoder = new RSRawDecoder(dataBlkNum, parityBlkNum); rsRawDecoder.decode(decodeInputs, decodeIndices, outputs); for (int i = 0; i < dataBlkNum + parityBlkNum; i++) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/b30e96bf/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java index e795ea7..e041dbe 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java @@ -274,8 +274,8 @@ public class TestDFSStripedOutputStream { System.arraycopy(tmp, 0, dataBytes[i], 0, tmp.length); } } - final RawErasureEncoder encoder = new RSRawEncoder(); - encoder.initialize(dataBytes.length, parityBytes.length, cellSize); + final RawErasureEncoder encoder = + new RSRawEncoder(dataBytes.length, parityBytes.length); encoder.encode(dataBytes, expectedParityBytes); for (int i = 0; i < parityBytes.length; i++) { if (i != killedDnIndex) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/b30e96bf/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java index 1976dca..70802fb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java @@ -382,8 +382,7 @@ public class TestWriteReadStripedFile { Assert.assertEquals("The length of file should be the same to write size", length - startOffsetInFile, readLen); - RSRawDecoder rsRawDecoder = new RSRawDecoder(); - rsRawDecoder.initialize(dataBlocks, parityBlocks, 1); + RSRawDecoder rsRawDecoder = new RSRawDecoder(dataBlocks, parityBlocks); byte[] expected = new byte[readLen]; for (int i = startOffsetInFile; i < length; i++) { //TODO: workaround (filling fixed bytes), to remove after HADOOP-11938
