HADOOP-13061. Refactor erasure coders. Contributed by Kai Sasaki
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c023c748 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c023c748 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c023c748 Branch: refs/heads/trunk Commit: c023c748869063fb67d14ea996569c42578d1cea Parents: bedfec0 Author: Kai Zheng <kai.zh...@intel.com> Authored: Tue Oct 18 12:02:53 2016 +0600 Committer: Kai Zheng <kai.zh...@intel.com> Committed: Tue Oct 18 12:02:53 2016 +0600 ---------------------------------------------------------------------- .../hadoop/fs/CommonConfigurationKeys.java | 26 --- .../apache/hadoop/io/erasurecode/CodecUtil.java | 168 ++++++++++++++-- .../io/erasurecode/ErasureCodeConstants.java | 3 +- .../io/erasurecode/ErasureCodecOptions.java | 37 ++++ .../erasurecode/codec/AbstractErasureCodec.java | 53 ----- .../io/erasurecode/codec/DummyErasureCodec.java | 45 +++++ .../io/erasurecode/codec/ErasureCodec.java | 76 +++++-- .../io/erasurecode/codec/HHXORErasureCodec.java | 20 +- .../io/erasurecode/codec/RSErasureCodec.java | 20 +- .../io/erasurecode/codec/XORErasureCodec.java | 22 ++- .../io/erasurecode/codec/package-info.java | 28 +++ .../erasurecode/coder/AbstractErasureCoder.java | 64 ------ .../coder/AbstractErasureCodingStep.java | 61 ------ .../coder/AbstractErasureDecoder.java | 170 ---------------- .../coder/AbstractErasureEncoder.java | 62 ------ .../coder/AbstractHHErasureCodingStep.java | 49 ----- .../erasurecode/coder/DummyErasureDecoder.java | 46 +++++ .../erasurecode/coder/DummyErasureEncoder.java | 45 +++++ .../io/erasurecode/coder/ErasureCoder.java | 25 ++- .../io/erasurecode/coder/ErasureCodingStep.java | 8 +- .../io/erasurecode/coder/ErasureDecoder.java | 198 +++++++++++++++++++ .../erasurecode/coder/ErasureDecodingStep.java | 21 +- .../io/erasurecode/coder/ErasureEncoder.java | 91 +++++++++ .../erasurecode/coder/ErasureEncodingStep.java | 22 ++- .../erasurecode/coder/HHErasureCodingStep.java | 68 +++++++ .../erasurecode/coder/HHXORErasureDecoder.java | 24 +-- .../coder/HHXORErasureDecodingStep.java | 2 +- .../erasurecode/coder/HHXORErasureEncoder.java | 19 +- .../coder/HHXORErasureEncodingStep.java | 2 +- .../io/erasurecode/coder/RSErasureDecoder.java | 16 +- .../io/erasurecode/coder/RSErasureEncoder.java | 20 +- .../io/erasurecode/coder/XORErasureDecoder.java | 15 +- .../io/erasurecode/coder/XORErasureEncoder.java | 16 +- .../io/erasurecode/coder/package-info.java | 28 +++ .../io/erasurecode/rawcoder/CoderUtil.java | 2 +- .../conf/TestCommonConfigurationFields.java | 5 +- .../erasurecode/TestCodecRawCoderMapping.java | 3 +- .../codec/TestHHXORErasureCodec.java | 6 +- .../erasurecode/coder/TestErasureCoderBase.java | 13 +- .../coder/TestHHXORErasureCoder.java | 4 +- .../erasurecode/coder/TestRSErasureCoder.java | 4 +- .../hadoop/hdfs/TestDFSStripedInputStream.java | 3 +- .../hadoop/hdfs/TestDFSStripedOutputStream.java | 6 +- .../TestDFSStripedOutputStreamWithFailure.java | 4 +- .../hadoop/hdfs/TestReconstructStripedFile.java | 4 +- 45 files changed, 964 insertions(+), 660 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java index 2b530f0..fe522b3 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java @@ -21,9 +21,6 @@ package org.apache.hadoop.fs; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.http.lib.StaticUserWebFilter; -import org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory; -import org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactoryLegacy; -import org.apache.hadoop.io.erasurecode.rawcoder.XORRawErasureCoderFactory; /** * This class contains constants for configuration keys used @@ -160,30 +157,7 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic { public static final boolean IO_COMPRESSION_CODEC_LZ4_USELZ4HC_DEFAULT = false; - /** - * Erasure Coding configuration family - */ - /** Supported erasure codec classes */ - public static final String IO_ERASURECODE_CODECS_KEY = "io.erasurecode.codecs"; - - /** Raw coder factory for the RS default codec. */ - public static final String IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY = - "io.erasurecode.codec.rs-default.rawcoder"; - public static final String IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_DEFAULT = - RSRawErasureCoderFactory.class.getCanonicalName(); - - /** Raw coder factory for the RS legacy codec. */ - public static final String IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_KEY = - "io.erasurecode.codec.rs-legacy.rawcoder"; - public static final String IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_DEFAULT = - RSRawErasureCoderFactoryLegacy.class.getCanonicalName(); - - /** Raw coder factory for the XOR codec. */ - public static final String IO_ERASURECODE_CODEC_XOR_RAWCODER_KEY = - "io.erasurecode.codec.xor.rawcoder"; - public static final String IO_ERASURECODE_CODEC_XOR_RAWCODER_DEFAULT = - XORRawErasureCoderFactory.class.getCanonicalName(); /** * Service Authorization http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java index 9cd9561..977bacb 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java @@ -20,20 +20,108 @@ package org.apache.hadoop.io.erasurecode; import com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.io.erasurecode.codec.ErasureCodec; +import org.apache.hadoop.io.erasurecode.codec.HHXORErasureCodec; +import org.apache.hadoop.io.erasurecode.codec.RSErasureCodec; +import org.apache.hadoop.io.erasurecode.codec.XORErasureCodec; +import org.apache.hadoop.io.erasurecode.coder.ErasureDecoder; +import org.apache.hadoop.io.erasurecode.coder.ErasureEncoder; +import org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory; +import org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactoryLegacy; import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureCoderFactory; import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder; import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder; +import org.apache.hadoop.io.erasurecode.rawcoder.XORRawErasureCoderFactory; + +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; /** - * A codec & coder utility to help create raw coders conveniently. + * A codec & coder utility to help create coders conveniently. + * + * {@link CodecUtil} includes erasure coder configurations key and default + * values such as coder class name and erasure codec option values included + * by {@link ErasureCodecOptions}. {@link ErasureEncoder} and + * {@link ErasureDecoder} are created by createEncoder and createDecoder + * respectively.{@link RawErasureEncoder} and {@link RawErasureDecoder} are + * are created by createRawEncoder and createRawDecoder. */ @InterfaceAudience.Private public final class CodecUtil { + /** Erasure coder XOR codec. */ + public static final String IO_ERASURECODE_CODEC_XOR_KEY = + "io.erasurecode.codec.xor"; + public static final String IO_ERASURECODE_CODEC_XOR = + XORErasureCodec.class.getCanonicalName(); + /** Erasure coder Reed-Solomon codec. */ + public static final String IO_ERASURECODE_CODEC_RS_DEFAULT_KEY = + "io.erasurecode.codec.rs"; + public static final String IO_ERASURECODE_CODEC_RS_DEFAULT = + RSErasureCodec.class.getCanonicalName(); + /** Erasure coder hitch hiker XOR codec. */ + public static final String IO_ERASURECODE_CODEC_HHXOR_KEY = + "io.erasurecode.codec.hhxor"; + public static final String IO_ERASURECODE_CODEC_HHXOR = + HHXORErasureCodec.class.getCanonicalName(); + + /** Supported erasure codec classes. */ + + /** Raw coder factory for the RS default codec. */ + public static final String IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY = + "io.erasurecode.codec.rs-default.rawcoder"; + public static final String IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_DEFAULT = + RSRawErasureCoderFactory.class.getCanonicalName(); + + /** Raw coder factory for the RS legacy codec. */ + public static final String IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_KEY = + "io.erasurecode.codec.rs-legacy.rawcoder"; + public static final String IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_DEFAULT = + RSRawErasureCoderFactoryLegacy.class.getCanonicalName(); + + /** Raw coder factory for the XOR codec. */ + public static final String IO_ERASURECODE_CODEC_XOR_RAWCODER_KEY = + "io.erasurecode.codec.xor.rawcoder"; + public static final String IO_ERASURECODE_CODEC_XOR_RAWCODER_DEFAULT = + XORRawErasureCoderFactory.class.getCanonicalName(); + private CodecUtil() { } /** + * Create encoder corresponding to given codec. + * @param options Erasure codec options + * @return erasure encoder + */ + public static ErasureEncoder createEncoder(Configuration conf, + ErasureCodecOptions options) { + Preconditions.checkNotNull(conf); + Preconditions.checkNotNull(options); + + String codecKey = getCodecClassName(conf, + options.getSchema().getCodecName()); + + ErasureCodec codec = createCodec(conf, codecKey, options); + return codec.createEncoder(); + } + + /** + * Create decoder corresponding to given codec. + * @param options Erasure codec options + * @return erasure decoder + */ + public static ErasureDecoder createDecoder(Configuration conf, + ErasureCodecOptions options) { + Preconditions.checkNotNull(conf); + Preconditions.checkNotNull(options); + + String codecKey = getCodecClassName(conf, + options.getSchema().getCodecName()); + + ErasureCodec codec = createCodec(conf, codecKey, options); + return codec.createDecoder(); + } + + /** * Create RS raw encoder according to configuration. * @param conf configuration * @param coderOptions coder options that's used to create the coder @@ -45,7 +133,7 @@ public final class CodecUtil { Preconditions.checkNotNull(conf); Preconditions.checkNotNull(codec); - String rawCoderFactoryKey = getFactNameFromCodec(conf, codec); + String rawCoderFactoryKey = getRawCoderFactNameFromCodec(conf, codec); RawErasureCoderFactory fact = createRawCoderFactory(conf, rawCoderFactoryKey); @@ -65,7 +153,7 @@ public final class CodecUtil { Preconditions.checkNotNull(conf); Preconditions.checkNotNull(codec); - String rawCoderFactoryKey = getFactNameFromCodec(conf, codec); + String rawCoderFactoryKey = getRawCoderFactNameFromCodec(conf, codec); RawErasureCoderFactory fact = createRawCoderFactory(conf, rawCoderFactoryKey); @@ -92,22 +180,21 @@ public final class CodecUtil { return fact; } - private static String getFactNameFromCodec(Configuration conf, String codec) { + private static String getRawCoderFactNameFromCodec(Configuration conf, + String codec) { switch (codec) { case ErasureCodeConstants.RS_DEFAULT_CODEC_NAME: return conf.get( - CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY, - CommonConfigurationKeys. - IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_DEFAULT); + IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY, + IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_DEFAULT); case ErasureCodeConstants.RS_LEGACY_CODEC_NAME: return conf.get( - CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_KEY, - CommonConfigurationKeys. - IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_DEFAULT); + IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_KEY, + IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_DEFAULT); case ErasureCodeConstants.XOR_CODEC_NAME: return conf.get( - CommonConfigurationKeys.IO_ERASURECODE_CODEC_XOR_RAWCODER_KEY, - CommonConfigurationKeys.IO_ERASURECODE_CODEC_XOR_RAWCODER_DEFAULT); + IO_ERASURECODE_CODEC_XOR_RAWCODER_KEY, + IO_ERASURECODE_CODEC_XOR_RAWCODER_DEFAULT); default: // For custom codec, we throw exception if the factory is not configured String rawCoderKey = "io.erasurecode.codec." + codec + ".rawcoder"; @@ -119,4 +206,59 @@ public final class CodecUtil { return factName; } } + + private static ErasureCodec createCodec(Configuration conf, + String codecClassName, ErasureCodecOptions options) { + ErasureCodec codec = null; + try { + Class<? extends ErasureCodec> codecClass = + conf.getClassByName(codecClassName) + .asSubclass(ErasureCodec.class); + Constructor<? extends ErasureCodec> constructor + = codecClass.getConstructor(Configuration.class, + ErasureCodecOptions.class); + codec = constructor.newInstance(conf, options); + } catch (ClassNotFoundException | InstantiationException | + IllegalAccessException | NoSuchMethodException | + InvocationTargetException e) { + throw new RuntimeException("Failed to create erasure codec", e); + } + + if (codec == null) { + throw new RuntimeException("Failed to create erasure codec"); + } + + return codec; + } + + private static String getCodecClassName(Configuration conf, String codec) { + switch (codec) { + case ErasureCodeConstants.RS_DEFAULT_CODEC_NAME: + return conf.get( + CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_KEY, + CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT); + case ErasureCodeConstants.RS_LEGACY_CODEC_NAME: + //TODO:rs-legacy should be handled differently. + return conf.get( + CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_KEY, + CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT); + case ErasureCodeConstants.XOR_CODEC_NAME: + return conf.get( + CodecUtil.IO_ERASURECODE_CODEC_XOR_KEY, + CodecUtil.IO_ERASURECODE_CODEC_XOR); + case ErasureCodeConstants.HHXOR_CODEC_NAME: + return conf.get( + CodecUtil.IO_ERASURECODE_CODEC_HHXOR_KEY, + CodecUtil.IO_ERASURECODE_CODEC_HHXOR); + default: + // For custom codec, we throw exception if the factory is not configured + String codecKey = "io.erasurecode.codec." + codec + ".coder"; + String codecClass = conf.get(codecKey); + if (codecClass == null) { + throw new IllegalArgumentException("Codec not configured " + + "for custom codec " + codec); + } + return codecClass; + } + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java index 1fb9488..8d6ff85 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java @@ -22,12 +22,13 @@ package org.apache.hadoop.io.erasurecode; */ public final class ErasureCodeConstants { - private ErasureCodeConstants(){ + private ErasureCodeConstants() { } public static final String RS_DEFAULT_CODEC_NAME = "rs-default"; public static final String RS_LEGACY_CODEC_NAME = "rs-legacy"; public static final String XOR_CODEC_NAME = "xor"; + public static final String HHXOR_CODEC_NAME = "hhxor"; public static final ECSchema RS_6_3_SCHEMA = new ECSchema( RS_DEFAULT_CODEC_NAME, 6, 3); http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodecOptions.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodecOptions.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodecOptions.java new file mode 100644 index 0000000..7ae0fc2 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodecOptions.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.io.erasurecode; + +import org.apache.hadoop.classification.InterfaceAudience; + +/** + * Erasure codec options. + */ +@InterfaceAudience.Private +public class ErasureCodecOptions { + private ECSchema schema; + + public ErasureCodecOptions(ECSchema schema) { + this.schema = schema; + } + + public ECSchema getSchema() { + return schema; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/AbstractErasureCodec.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/AbstractErasureCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/AbstractErasureCodec.java deleted file mode 100644 index e2fb2cb..0000000 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/AbstractErasureCodec.java +++ /dev/null @@ -1,53 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.io.erasurecode.codec; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.conf.Configured; -import org.apache.hadoop.io.erasurecode.ECSchema; -import org.apache.hadoop.io.erasurecode.grouper.BlockGrouper; - -/** - * Abstract Erasure Codec that implements {@link ErasureCodec}. - */ -@InterfaceAudience.Private -public abstract class AbstractErasureCodec extends Configured - implements ErasureCodec { - - private final ECSchema schema; - - public AbstractErasureCodec(ECSchema schema) { - this.schema = schema; - } - - public String getName() { - return schema.getCodecName(); - } - - public ECSchema getSchema() { - return schema; - } - - @Override - public BlockGrouper createBlockGrouper() { - BlockGrouper blockGrouper = new BlockGrouper(); - blockGrouper.setSchema(getSchema()); - - return blockGrouper; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/DummyErasureCodec.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/DummyErasureCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/DummyErasureCodec.java new file mode 100644 index 0000000..3646f72 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/DummyErasureCodec.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.io.erasurecode.codec; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.erasurecode.ErasureCodecOptions; +import org.apache.hadoop.io.erasurecode.coder.DummyErasureDecoder; +import org.apache.hadoop.io.erasurecode.coder.DummyErasureEncoder; +import org.apache.hadoop.io.erasurecode.coder.ErasureDecoder; +import org.apache.hadoop.io.erasurecode.coder.ErasureEncoder; + +/** + * Dummy erasure coder does not real coding computing. This is used for only + * test or performance comparison with other erasure coders. + */ +public class DummyErasureCodec extends ErasureCodec { + public DummyErasureCodec(Configuration conf, ErasureCodecOptions options) { + super(conf, options); + } + + @Override + public ErasureEncoder createEncoder() { + return new DummyErasureEncoder(getCoderOptions()); + } + + @Override + public ErasureDecoder createDecoder() { + return new DummyErasureDecoder(getCoderOptions()); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/ErasureCodec.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/ErasureCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/ErasureCodec.java index a2ea865..c75eaea 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/ErasureCodec.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/ErasureCodec.java @@ -18,34 +18,76 @@ package org.apache.hadoop.io.erasurecode.codec; import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.conf.Configurable; -import org.apache.hadoop.io.erasurecode.coder.ErasureCoder; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.erasurecode.CodecUtil; +import org.apache.hadoop.io.erasurecode.ECSchema; +import org.apache.hadoop.io.erasurecode.ErasureCodecOptions; +import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; +import org.apache.hadoop.io.erasurecode.coder.ErasureDecoder; +import org.apache.hadoop.io.erasurecode.coder.ErasureEncoder; import org.apache.hadoop.io.erasurecode.grouper.BlockGrouper; /** - * Erasure Codec API that's to cover the essential specific aspects of a code. - * Currently it cares only block grouper and erasure coder. In future we may - * add more aspects here to make the behaviors customizable. + * Abstract Erasure Codec is defines the interface of each actual erasure + * codec classes. */ @InterfaceAudience.Private -public interface ErasureCodec extends Configurable { +public abstract class ErasureCodec { - /** - * Create block grouper - * @return block grouper - */ - public BlockGrouper createBlockGrouper(); + private ECSchema schema; + private ErasureCodecOptions codecOptions; + private ErasureCoderOptions coderOptions; + + public ErasureCodec(Configuration conf, + ErasureCodecOptions options) { + this.schema = options.getSchema(); + this.codecOptions = options; + boolean allowChangeInputs = false; + this.coderOptions = new ErasureCoderOptions(schema.getNumDataUnits(), + schema.getNumParityUnits(), allowChangeInputs, false); + } + + public String getName() { + return schema.getCodecName(); + } + + public ECSchema getSchema() { + return schema; + } /** - * Create Erasure Encoder - * @return erasure encoder + * Get a {@link ErasureCodecOptions}. + * @return erasure codec options */ - public ErasureCoder createEncoder(); + public ErasureCodecOptions getCodecOptions() { + return codecOptions; + } + + protected void setCodecOptions(ErasureCodecOptions options) { + this.codecOptions = options; + this.schema = options.getSchema(); + } /** - * Create Erasure Decoder - * @return erasure decoder + * Get a {@link ErasureCoderOptions}. + * @return erasure coder options */ - public ErasureCoder createDecoder(); + public ErasureCoderOptions getCoderOptions() { + return coderOptions; + } + + protected void setCoderOptions(ErasureCoderOptions options) { + this.coderOptions = options; + } + + public abstract ErasureEncoder createEncoder(); + + public abstract ErasureDecoder createDecoder(); + + public BlockGrouper createBlockGrouper() { + BlockGrouper blockGrouper = new BlockGrouper(); + blockGrouper.setSchema(getSchema()); + return blockGrouper; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/HHXORErasureCodec.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/HHXORErasureCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/HHXORErasureCodec.java index 3c8061d..42380f3 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/HHXORErasureCodec.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/HHXORErasureCodec.java @@ -18,8 +18,10 @@ package org.apache.hadoop.io.erasurecode.codec; import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.io.erasurecode.ECSchema; -import org.apache.hadoop.io.erasurecode.coder.ErasureCoder; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.erasurecode.ErasureCodecOptions; +import org.apache.hadoop.io.erasurecode.coder.ErasureDecoder; +import org.apache.hadoop.io.erasurecode.coder.ErasureEncoder; import org.apache.hadoop.io.erasurecode.coder.HHXORErasureDecoder; import org.apache.hadoop.io.erasurecode.coder.HHXORErasureEncoder; @@ -27,19 +29,19 @@ import org.apache.hadoop.io.erasurecode.coder.HHXORErasureEncoder; * A Hitchhiker-XOR erasure codec. */ @InterfaceAudience.Private -public class HHXORErasureCodec extends AbstractErasureCodec { +public class HHXORErasureCodec extends ErasureCodec { - public HHXORErasureCodec(ECSchema schema) { - super(schema); + public HHXORErasureCodec(Configuration conf, ErasureCodecOptions options) { + super(conf, options); } @Override - public ErasureCoder createEncoder() { - return new HHXORErasureEncoder(getSchema()); + public ErasureEncoder createEncoder() { + return new HHXORErasureEncoder(getCoderOptions()); } @Override - public ErasureCoder createDecoder() { - return new HHXORErasureDecoder(getSchema()); + public ErasureDecoder createDecoder() { + return new HHXORErasureDecoder(getCoderOptions()); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/RSErasureCodec.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/RSErasureCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/RSErasureCodec.java index 7a5786e..e57161c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/RSErasureCodec.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/RSErasureCodec.java @@ -18,8 +18,10 @@ package org.apache.hadoop.io.erasurecode.codec; import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.io.erasurecode.ECSchema; -import org.apache.hadoop.io.erasurecode.coder.ErasureCoder; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.erasurecode.ErasureCodecOptions; +import org.apache.hadoop.io.erasurecode.coder.ErasureDecoder; +import org.apache.hadoop.io.erasurecode.coder.ErasureEncoder; import org.apache.hadoop.io.erasurecode.coder.RSErasureDecoder; import org.apache.hadoop.io.erasurecode.coder.RSErasureEncoder; @@ -27,19 +29,19 @@ import org.apache.hadoop.io.erasurecode.coder.RSErasureEncoder; * A Reed-Solomon erasure codec. */ @InterfaceAudience.Private -public class RSErasureCodec extends AbstractErasureCodec { +public class RSErasureCodec extends ErasureCodec { - public RSErasureCodec(ECSchema schema) { - super(schema); + public RSErasureCodec(Configuration conf, ErasureCodecOptions options) { + super(conf, options); } @Override - public ErasureCoder createEncoder() { - return new RSErasureEncoder(getSchema()); + public ErasureEncoder createEncoder() { + return new RSErasureEncoder(getCoderOptions()); } @Override - public ErasureCoder createDecoder() { - return new RSErasureDecoder(getSchema()); + public ErasureDecoder createDecoder() { + return new RSErasureDecoder(getCoderOptions()); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/XORErasureCodec.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/XORErasureCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/XORErasureCodec.java index 9c38fc2..f9a0608 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/XORErasureCodec.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/XORErasureCodec.java @@ -18,8 +18,10 @@ package org.apache.hadoop.io.erasurecode.codec; import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.io.erasurecode.ECSchema; -import org.apache.hadoop.io.erasurecode.coder.ErasureCoder; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.erasurecode.ErasureCodecOptions; +import org.apache.hadoop.io.erasurecode.coder.ErasureDecoder; +import org.apache.hadoop.io.erasurecode.coder.ErasureEncoder; import org.apache.hadoop.io.erasurecode.coder.XORErasureDecoder; import org.apache.hadoop.io.erasurecode.coder.XORErasureEncoder; @@ -27,20 +29,20 @@ import org.apache.hadoop.io.erasurecode.coder.XORErasureEncoder; * A XOR erasure codec. */ @InterfaceAudience.Private -public class XORErasureCodec extends AbstractErasureCodec { +public class XORErasureCodec extends ErasureCodec { - public XORErasureCodec(ECSchema schema) { - super(schema); - assert(schema.getNumParityUnits() == 1); + public XORErasureCodec(Configuration conf, ErasureCodecOptions options) { + super(conf, options); + assert(options.getSchema().getNumParityUnits() == 1); } @Override - public ErasureCoder createEncoder() { - return new XORErasureEncoder(getSchema()); + public ErasureEncoder createEncoder() { + return new XORErasureEncoder(getCoderOptions()); } @Override - public ErasureCoder createDecoder() { - return new XORErasureDecoder(getSchema()); + public ErasureDecoder createDecoder() { + return new XORErasureDecoder(getCoderOptions()); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/package-info.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/package-info.java new file mode 100644 index 0000000..ddfdd4b --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/package-info.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Erasure codec framework. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +package org.apache.hadoop.io.erasurecode.codec; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java deleted file mode 100644 index 1d1641f..0000000 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java +++ /dev/null @@ -1,64 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.io.erasurecode.coder; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.conf.Configured; -import org.apache.hadoop.io.erasurecode.ECSchema; - -/** - * A common class of basic facilities to be shared by encoder and decoder - * - * It implements the {@link ErasureCoder} interface. - */ -@InterfaceAudience.Private -public abstract class AbstractErasureCoder - extends Configured implements ErasureCoder { - - private final int numDataUnits; - private final int numParityUnits; - - public AbstractErasureCoder(int numDataUnits, int numParityUnits) { - this.numDataUnits = numDataUnits; - this.numParityUnits = numParityUnits; - } - - public AbstractErasureCoder(ECSchema schema) { - this(schema.getNumDataUnits(), schema.getNumParityUnits()); - } - - @Override - public int getNumDataUnits() { - return numDataUnits; - } - - @Override - public int getNumParityUnits() { - return numParityUnits; - } - - @Override - public boolean preferDirectBuffer() { - return false; - } - - @Override - public void release() { - // Nothing to do by default - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCodingStep.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCodingStep.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCodingStep.java deleted file mode 100644 index ccff92d..0000000 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCodingStep.java +++ /dev/null @@ -1,61 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.io.erasurecode.coder; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.io.erasurecode.ECBlock; - -/** - * Abstract class for common facilities shared by {@link ErasureEncodingStep} - * and {@link ErasureDecodingStep}. - * - * It implements {@link ErasureEncodingStep}. - */ -@InterfaceAudience.Private -public abstract class AbstractErasureCodingStep implements ErasureCodingStep { - - private ECBlock[] inputBlocks; - private ECBlock[] outputBlocks; - - /** - * Constructor given input blocks and output blocks. - * @param inputBlocks - * @param outputBlocks - */ - public AbstractErasureCodingStep(ECBlock[] inputBlocks, - ECBlock[] outputBlocks) { - this.inputBlocks = inputBlocks; - this.outputBlocks = outputBlocks; - } - - @Override - public ECBlock[] getInputBlocks() { - return inputBlocks; - } - - @Override - public ECBlock[] getOutputBlocks() { - return outputBlocks; - } - - @Override - public void finish() { - // NOOP by default - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java deleted file mode 100644 index d976dd1..0000000 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java +++ /dev/null @@ -1,170 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.io.erasurecode.coder; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.io.erasurecode.ECBlock; -import org.apache.hadoop.io.erasurecode.ECBlockGroup; -import org.apache.hadoop.io.erasurecode.ECSchema; - -/** - * An abstract erasure decoder that's to be inherited by new decoders. - * - * It implements the {@link ErasureCoder} interface. - */ -@InterfaceAudience.Private -public abstract class AbstractErasureDecoder extends AbstractErasureCoder { - - public AbstractErasureDecoder(int numDataUnits, int numParityUnits) { - super(numDataUnits, numParityUnits); - } - - public AbstractErasureDecoder(ECSchema schema) { - super(schema); - } - - @Override - public ErasureCodingStep calculateCoding(ECBlockGroup blockGroup) { - // We may have more than this when considering complicate cases. HADOOP-11550 - return prepareDecodingStep(blockGroup); - } - - /** - * Perform decoding against a block blockGroup. - * @param blockGroup - * @return decoding step for caller to do the real work - */ - protected abstract ErasureCodingStep prepareDecodingStep( - ECBlockGroup blockGroup); - - /** - * We have all the data blocks and parity blocks as input blocks for - * recovering by default. It's codec specific - * @param blockGroup - * @return input blocks - */ - protected ECBlock[] getInputBlocks(ECBlockGroup blockGroup) { - ECBlock[] inputBlocks = new ECBlock[getNumDataUnits() + - getNumParityUnits()]; - - System.arraycopy(blockGroup.getDataBlocks(), 0, inputBlocks, - 0, getNumDataUnits()); - - System.arraycopy(blockGroup.getParityBlocks(), 0, inputBlocks, - getNumDataUnits(), getNumParityUnits()); - - return inputBlocks; - } - - /** - * Which blocks were erased ? - * @param blockGroup - * @return output blocks to recover - */ - protected ECBlock[] getOutputBlocks(ECBlockGroup blockGroup) { - ECBlock[] outputBlocks = new ECBlock[getNumErasedBlocks(blockGroup)]; - - int idx = 0; - - for (int i = 0; i < getNumDataUnits(); i++) { - if (blockGroup.getDataBlocks()[i].isErased()) { - outputBlocks[idx++] = blockGroup.getDataBlocks()[i]; - } - } - - for (int i = 0; i < getNumParityUnits(); i++) { - if (blockGroup.getParityBlocks()[i].isErased()) { - outputBlocks[idx++] = blockGroup.getParityBlocks()[i]; - } - } - - return outputBlocks; - } - - /** - * Get the number of erased blocks in the block group. - * @param blockGroup - * @return number of erased blocks - */ - protected int getNumErasedBlocks(ECBlockGroup blockGroup) { - int num = getNumErasedBlocks(blockGroup.getParityBlocks()); - num += getNumErasedBlocks(blockGroup.getDataBlocks()); - return num; - } - - /** - * Find out how many blocks are erased. - * @param inputBlocks all the input blocks - * @return number of erased blocks - */ - protected static int getNumErasedBlocks(ECBlock[] inputBlocks) { - int numErased = 0; - for (int i = 0; i < inputBlocks.length; i++) { - if (inputBlocks[i].isErased()) { - numErased ++; - } - } - - return numErased; - } - - /** - * Get indexes of erased blocks from inputBlocks - * @param inputBlocks - * @return indexes of erased blocks from inputBlocks - */ - protected int[] getErasedIndexes(ECBlock[] inputBlocks) { - int numErased = getNumErasedBlocks(inputBlocks); - if (numErased == 0) { - return new int[0]; - } - - int[] erasedIndexes = new int[numErased]; - int i = 0, j = 0; - for (; i < inputBlocks.length && j < erasedIndexes.length; i++) { - if (inputBlocks[i].isErased()) { - erasedIndexes[j++] = i; - } - } - - return erasedIndexes; - } - - /** - * Get erased input blocks from inputBlocks - * @param inputBlocks - * @return an array of erased blocks from inputBlocks - */ - protected ECBlock[] getErasedBlocks(ECBlock[] inputBlocks) { - int numErased = getNumErasedBlocks(inputBlocks); - if (numErased == 0) { - return new ECBlock[0]; - } - - ECBlock[] erasedBlocks = new ECBlock[numErased]; - int i = 0, j = 0; - for (; i < inputBlocks.length && j < erasedBlocks.length; i++) { - if (inputBlocks[i].isErased()) { - erasedBlocks[j++] = inputBlocks[i]; - } - } - - return erasedBlocks; - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureEncoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureEncoder.java deleted file mode 100644 index 5ec6b4e..0000000 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureEncoder.java +++ /dev/null @@ -1,62 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.io.erasurecode.coder; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.io.erasurecode.ECBlock; -import org.apache.hadoop.io.erasurecode.ECBlockGroup; -import org.apache.hadoop.io.erasurecode.ECSchema; - -/** - * An abstract erasure encoder that's to be inherited by new encoders. - * - * It implements the {@link ErasureCoder} interface. - */ -@InterfaceAudience.Private -public abstract class AbstractErasureEncoder extends AbstractErasureCoder { - - public AbstractErasureEncoder(int numDataUnits, int numParityUnits) { - super(numDataUnits, numParityUnits); - } - - public AbstractErasureEncoder(ECSchema schema) { - super(schema); - } - - @Override - public ErasureCodingStep calculateCoding(ECBlockGroup blockGroup) { - // We may have more than this when considering complicate cases. HADOOP-11550 - return prepareEncodingStep(blockGroup); - } - - /** - * Perform encoding against a block group. - * @param blockGroup - * @return encoding step for caller to do the real work - */ - protected abstract ErasureCodingStep prepareEncodingStep( - ECBlockGroup blockGroup); - - protected ECBlock[] getInputBlocks(ECBlockGroup blockGroup) { - return blockGroup.getDataBlocks(); - } - - protected ECBlock[] getOutputBlocks(ECBlockGroup blockGroup) { - return blockGroup.getParityBlocks(); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractHHErasureCodingStep.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractHHErasureCodingStep.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractHHErasureCodingStep.java deleted file mode 100644 index e577c5d..0000000 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractHHErasureCodingStep.java +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.io.erasurecode.coder; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.io.erasurecode.ECBlock; - -/** - * Abstract class for Hitchhiker common facilities shared by - * {@link HHXORErasureEncodingStep}and {@link HHXORErasureDecodingStep}. - * - * It implements {@link AbstractErasureCodingStep}. - */ -@InterfaceAudience.Private -public abstract class AbstractHHErasureCodingStep - extends AbstractErasureCodingStep { - - private static final int SUB_PACKET_SIZE = 2; - - /** - * Constructor given input blocks and output blocks. - * - * @param inputBlocks - * @param outputBlocks - */ - public AbstractHHErasureCodingStep(ECBlock[] inputBlocks, - ECBlock[] outputBlocks) { - super(inputBlocks, outputBlocks); - } - - protected int getSubPacketSize() { - return SUB_PACKET_SIZE; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/DummyErasureDecoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/DummyErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/DummyErasureDecoder.java new file mode 100644 index 0000000..8a9434e --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/DummyErasureDecoder.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.io.erasurecode.coder; + +import org.apache.hadoop.io.erasurecode.ECBlock; +import org.apache.hadoop.io.erasurecode.ECBlockGroup; +import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; +import org.apache.hadoop.io.erasurecode.rawcoder.DummyRawDecoder; +import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder; + +/** + * Dummy erasure decoder does no real computation. Instead, it just returns + * zero bytes. This decoder can be used to isolate the performance issue to + * HDFS side logic instead of codec, and is intended for test only. + */ +public class DummyErasureDecoder extends ErasureDecoder { + public DummyErasureDecoder(ErasureCoderOptions options) { + super(options); + } + + @Override + protected ErasureCodingStep prepareDecodingStep(ECBlockGroup blockGroup) { + RawErasureDecoder rawDecoder = new DummyRawDecoder(getOptions()); + + ECBlock[] inputBlocks = getInputBlocks(blockGroup); + + return new ErasureDecodingStep(inputBlocks, + getErasedIndexes(inputBlocks), + getOutputBlocks(blockGroup), rawDecoder); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/DummyErasureEncoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/DummyErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/DummyErasureEncoder.java new file mode 100644 index 0000000..fc04f5e --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/DummyErasureEncoder.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.io.erasurecode.coder; + +import org.apache.hadoop.io.erasurecode.ECBlock; +import org.apache.hadoop.io.erasurecode.ECBlockGroup; +import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; +import org.apache.hadoop.io.erasurecode.rawcoder.DummyRawEncoder; +import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder; + +/** + * Dummy erasure encoder does no real computation. Instead, it just returns + * zero bytes. This decoder can be used to isolate the performance issue to + * HDFS side logic instead of codec, and is intended for test only. + */ +public class DummyErasureEncoder extends ErasureEncoder { + public DummyErasureEncoder(ErasureCoderOptions options) { + super(options); + } + + @Override + protected ErasureCodingStep prepareEncodingStep(ECBlockGroup blockGroup) { + RawErasureEncoder rawEncoder = new DummyRawEncoder(getOptions()); + + ECBlock[] inputBlocks = getInputBlocks(blockGroup); + + return new ErasureEncodingStep(inputBlocks, + getOutputBlocks(blockGroup), rawEncoder); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java index 49cc8dd..b5ae1f1 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java @@ -20,6 +20,7 @@ package org.apache.hadoop.io.erasurecode.coder; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.io.erasurecode.ECBlockGroup; +import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; /** * An erasure coder to perform encoding or decoding given a group. Generally it @@ -39,18 +40,25 @@ import org.apache.hadoop.io.erasurecode.ECBlockGroup; public interface ErasureCoder extends Configurable { /** - * The number of data input units for the coding. A unit can be a byte, - * chunk or buffer or even a block. + * The number of data input units for the coding. A unit can be a byte, chunk + * or buffer or even a block. * @return count of data input units */ - public int getNumDataUnits(); + int getNumDataUnits(); /** * The number of parity output units for the coding. A unit can be a byte, * chunk, buffer or even a block. * @return count of parity output units */ - public int getNumParityUnits(); + int getNumParityUnits(); + + /** + * The options of erasure coder. This option is passed to + * raw erasure coder as it is. + * @return erasure coder options + */ + ErasureCoderOptions getOptions(); /** * Calculate the encoding or decoding steps given a block blockGroup. @@ -61,7 +69,7 @@ public interface ErasureCoder extends Configurable { * @param blockGroup the erasure coding block group containing all necessary * information for codec calculation */ - public ErasureCodingStep calculateCoding(ECBlockGroup blockGroup); + ErasureCodingStep calculateCoding(ECBlockGroup blockGroup); /** * Tell if direct or off-heap buffer is preferred or not. It's for callers to @@ -70,10 +78,11 @@ public interface ErasureCoder extends Configurable { * @return true if direct buffer is preferred for performance consideration, * otherwise false. */ - public boolean preferDirectBuffer(); + boolean preferDirectBuffer(); /** - * Release the resources if any. Good chance to invoke RawErasureCoder#release. + * Release the resources if any. Good chance to invoke + * RawErasureCoder#release. */ - public void release(); + void release(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCodingStep.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCodingStep.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCodingStep.java index 991eb56..9dd0aed 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCodingStep.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCodingStep.java @@ -32,14 +32,14 @@ public interface ErasureCodingStep { * or parity blocks. * @return input blocks */ - public ECBlock[] getInputBlocks(); + ECBlock[] getInputBlocks(); /** * Output blocks of writable buffers involved in this step, may be data * blocks or parity blocks. * @return output blocks */ - public ECBlock[] getOutputBlocks(); + ECBlock[] getOutputBlocks(); /** * Perform encoding or decoding given the input chunks, and generated results @@ -47,11 +47,11 @@ public interface ErasureCodingStep { * @param inputChunks * @param outputChunks */ - public void performCoding(ECChunk[] inputChunks, ECChunk[] outputChunks); + void performCoding(ECChunk[] inputChunks, ECChunk[] outputChunks); /** * Notify erasure coder that all the chunks of input blocks are processed so * the coder can be able to update internal states, considering next step. */ - public void finish(); + void finish(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecoder.java new file mode 100644 index 0000000..faf44d9 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecoder.java @@ -0,0 +1,198 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.io.erasurecode.coder; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.io.erasurecode.ECBlock; +import org.apache.hadoop.io.erasurecode.ECBlockGroup; +import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; + +/** + * An abstract erasure decoder that's to be inherited by new decoders. + * + * It implements the {@link ErasureCoder} interface. + */ +@InterfaceAudience.Private +public abstract class ErasureDecoder extends Configured + implements ErasureCoder { + private final int numDataUnits; + private final int numParityUnits; + private final ErasureCoderOptions options; + + public ErasureDecoder(ErasureCoderOptions options) { + this.options = options; + this.numDataUnits = options.getNumDataUnits(); + this.numParityUnits = options.getNumParityUnits(); + } + + @Override + public ErasureCodingStep calculateCoding(ECBlockGroup blockGroup) { + // We may have more than this when considering complicate cases. HADOOP-11550 + return prepareDecodingStep(blockGroup); + } + + @Override + public int getNumDataUnits() { + return this.numDataUnits; + } + + @Override + public int getNumParityUnits() { + return this.numParityUnits; + } + + @Override + public ErasureCoderOptions getOptions() { + return options; + } + + /** + * We have all the data blocks and parity blocks as input blocks for + * recovering by default. It's codec specific + * @param blockGroup + * @return input blocks + */ + protected ECBlock[] getInputBlocks(ECBlockGroup blockGroup) { + ECBlock[] inputBlocks = new ECBlock[getNumDataUnits() + + getNumParityUnits()]; + + System.arraycopy(blockGroup.getDataBlocks(), 0, inputBlocks, + 0, getNumDataUnits()); + + System.arraycopy(blockGroup.getParityBlocks(), 0, inputBlocks, + getNumDataUnits(), getNumParityUnits()); + + return inputBlocks; + } + + /** + * Which blocks were erased ? + * @param blockGroup + * @return output blocks to recover + */ + protected ECBlock[] getOutputBlocks(ECBlockGroup blockGroup) { + ECBlock[] outputBlocks = new ECBlock[getNumErasedBlocks(blockGroup)]; + + int idx = 0; + + for (int i = 0; i < getNumDataUnits(); i++) { + if (blockGroup.getDataBlocks()[i].isErased()) { + outputBlocks[idx++] = blockGroup.getDataBlocks()[i]; + } + } + + for (int i = 0; i < getNumParityUnits(); i++) { + if (blockGroup.getParityBlocks()[i].isErased()) { + outputBlocks[idx++] = blockGroup.getParityBlocks()[i]; + } + } + + return outputBlocks; + } + + @Override + public boolean preferDirectBuffer() { + return false; + } + + @Override + public void release() { + // Nothing to do by default + } + + /** + * Perform decoding against a block blockGroup. + * @param blockGroup + * @return decoding step for caller to do the real work + */ + protected abstract ErasureCodingStep prepareDecodingStep( + ECBlockGroup blockGroup); + + /** + * Get the number of erased blocks in the block group. + * @param blockGroup + * @return number of erased blocks + */ + protected int getNumErasedBlocks(ECBlockGroup blockGroup) { + int num = getNumErasedBlocks(blockGroup.getParityBlocks()); + num += getNumErasedBlocks(blockGroup.getDataBlocks()); + return num; + } + + /** + * Find out how many blocks are erased. + * @param inputBlocks all the input blocks + * @return number of erased blocks + */ + protected static int getNumErasedBlocks(ECBlock[] inputBlocks) { + int numErased = 0; + for (int i = 0; i < inputBlocks.length; i++) { + if (inputBlocks[i].isErased()) { + numErased ++; + } + } + + return numErased; + } + + /** + * Get indexes of erased blocks from inputBlocks + * @param inputBlocks + * @return indexes of erased blocks from inputBlocks + */ + protected int[] getErasedIndexes(ECBlock[] inputBlocks) { + int numErased = getNumErasedBlocks(inputBlocks); + if (numErased == 0) { + return new int[0]; + } + + int[] erasedIndexes = new int[numErased]; + int i = 0, j = 0; + for (; i < inputBlocks.length && j < erasedIndexes.length; i++) { + if (inputBlocks[i].isErased()) { + erasedIndexes[j++] = i; + } + } + + return erasedIndexes; + } + + /** + * Get erased input blocks from inputBlocks + * @param inputBlocks + * @return an array of erased blocks from inputBlocks + */ + protected ECBlock[] getErasedBlocks(ECBlock[] inputBlocks) { + int numErased = getNumErasedBlocks(inputBlocks); + if (numErased == 0) { + return new ECBlock[0]; + } + + ECBlock[] erasedBlocks = new ECBlock[numErased]; + int i = 0, j = 0; + for (; i < inputBlocks.length && j < erasedBlocks.length; i++) { + if (inputBlocks[i].isErased()) { + erasedBlocks[j++] = inputBlocks[i]; + } + } + + return erasedBlocks; + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecodingStep.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecodingStep.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecodingStep.java index b319cfa..ae396a2 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecodingStep.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecodingStep.java @@ -27,7 +27,9 @@ import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder; * a decoding step involved in the whole process of decoding a block group. */ @InterfaceAudience.Private -public class ErasureDecodingStep extends AbstractErasureCodingStep { +public class ErasureDecodingStep implements ErasureCodingStep { + private ECBlock[] inputBlocks; + private ECBlock[] outputBlocks; private int[] erasedIndexes; private RawErasureDecoder rawDecoder; @@ -41,7 +43,8 @@ public class ErasureDecodingStep extends AbstractErasureCodingStep { public ErasureDecodingStep(ECBlock[] inputBlocks, int[] erasedIndexes, ECBlock[] outputBlocks, RawErasureDecoder rawDecoder) { - super(inputBlocks, outputBlocks); + this.inputBlocks = inputBlocks; + this.outputBlocks = outputBlocks; this.erasedIndexes = erasedIndexes; this.rawDecoder = rawDecoder; } @@ -51,4 +54,18 @@ public class ErasureDecodingStep extends AbstractErasureCodingStep { rawDecoder.decode(inputChunks, erasedIndexes, outputChunks); } + @Override + public ECBlock[] getInputBlocks() { + return inputBlocks; + } + + @Override + public ECBlock[] getOutputBlocks() { + return outputBlocks; + } + + @Override + public void finish() { + // TODO: Finalize decoder if necessary + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncoder.java new file mode 100644 index 0000000..81666e9 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncoder.java @@ -0,0 +1,91 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.io.erasurecode.coder; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.io.erasurecode.ECBlock; +import org.apache.hadoop.io.erasurecode.ECBlockGroup; +import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; + +/** + * An abstract erasure encoder that's to be inherited by new encoders. + * + * It implements the {@link ErasureCoder} interface. + */ +@InterfaceAudience.Private +public abstract class ErasureEncoder extends Configured + implements ErasureCoder { + + private final int numDataUnits; + private final int numParityUnits; + private final ErasureCoderOptions options; + + public ErasureEncoder(ErasureCoderOptions options) { + this.options = options; + this.numDataUnits = options.getNumDataUnits(); + this.numParityUnits = options.getNumParityUnits(); + } + + @Override + public ErasureCodingStep calculateCoding(ECBlockGroup blockGroup) { + // We may have more than this when considering complicate cases. HADOOP-11550 + return prepareEncodingStep(blockGroup); + } + + @Override + public int getNumDataUnits() { + return numDataUnits; + } + + @Override + public int getNumParityUnits() { + return numParityUnits; + } + + @Override + public ErasureCoderOptions getOptions() { + return options; + } + + protected ECBlock[] getInputBlocks(ECBlockGroup blockGroup) { + return blockGroup.getDataBlocks(); + } + + protected ECBlock[] getOutputBlocks(ECBlockGroup blockGroup) { + return blockGroup.getParityBlocks(); + } + + @Override + public boolean preferDirectBuffer() { + return false; + } + + @Override + public void release() { + // Nothing to do by default + } + + /** + * Perform encoding against a block group. + * @param blockGroup + * @return encoding step for caller to do the real work + */ + protected abstract ErasureCodingStep prepareEncodingStep( + ECBlockGroup blockGroup); +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncodingStep.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncodingStep.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncodingStep.java index 311de21..f0b2be8 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncodingStep.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncodingStep.java @@ -27,8 +27,9 @@ import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder; * an encoding step involved in the whole process of encoding a block group. */ @InterfaceAudience.Private -public class ErasureEncodingStep extends AbstractErasureCodingStep { - +public class ErasureEncodingStep implements ErasureCodingStep { + private ECBlock[] inputBlocks; + private ECBlock[] outputBlocks; private RawErasureEncoder rawEncoder; /** @@ -39,7 +40,8 @@ public class ErasureEncodingStep extends AbstractErasureCodingStep { */ public ErasureEncodingStep(ECBlock[] inputBlocks, ECBlock[] outputBlocks, RawErasureEncoder rawEncoder) { - super(inputBlocks, outputBlocks); + this.inputBlocks = inputBlocks; + this.outputBlocks = outputBlocks; this.rawEncoder = rawEncoder; } @@ -48,4 +50,18 @@ public class ErasureEncodingStep extends AbstractErasureCodingStep { rawEncoder.encode(inputChunks, outputChunks); } + @Override + public ECBlock[] getInputBlocks() { + return inputBlocks; + } + + @Override + public ECBlock[] getOutputBlocks() { + return outputBlocks; + } + + @Override + public void finish() { + rawEncoder.release(); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHErasureCodingStep.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHErasureCodingStep.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHErasureCodingStep.java new file mode 100644 index 0000000..a0f5b72 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHErasureCodingStep.java @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.io.erasurecode.coder; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.io.erasurecode.ECBlock; + +/** + * Abstract class for Hitchhiker common facilities shared by + * {@link HHXORErasureEncodingStep}and {@link HHXORErasureDecodingStep}. + * + * It implements {@link ErasureCodingStep}. + */ +@InterfaceAudience.Private +public abstract class HHErasureCodingStep + implements ErasureCodingStep { + + private ECBlock[] inputBlocks; + private ECBlock[] outputBlocks; + + private static final int SUB_PACKET_SIZE = 2; + + /** + * Constructor given input blocks and output blocks. + * + * @param inputBlocks + * @param outputBlocks + */ + public HHErasureCodingStep(ECBlock[] inputBlocks, + ECBlock[] outputBlocks) { + this.inputBlocks = inputBlocks; + this.outputBlocks = outputBlocks; + } + + protected int getSubPacketSize() { + return SUB_PACKET_SIZE; + } + + @Override + public ECBlock[] getInputBlocks() { + return inputBlocks; + } + + @Override + public ECBlock[] getOutputBlocks() { + return outputBlocks; + } + + @Override + public void finish() { + // TODO: Finalize encoder/decoder if necessary + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecoder.java index 94487d8..05e9384 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecoder.java @@ -21,7 +21,6 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.io.erasurecode.CodecUtil; import org.apache.hadoop.io.erasurecode.ECBlock; import org.apache.hadoop.io.erasurecode.ECBlockGroup; -import org.apache.hadoop.io.erasurecode.ECSchema; import org.apache.hadoop.io.erasurecode.ErasureCodeConstants; import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder; @@ -39,16 +38,12 @@ import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder; * This is Hitchhiker-XOR erasure decoder that decodes a block group. */ @InterfaceAudience.Private -public class HHXORErasureDecoder extends AbstractErasureDecoder { +public class HHXORErasureDecoder extends ErasureDecoder { private RawErasureDecoder rsRawDecoder; private RawErasureEncoder xorRawEncoder; - public HHXORErasureDecoder(int numDataUnits, int numParityUnits) { - super(numDataUnits, numParityUnits); - } - - public HHXORErasureDecoder(ECSchema schema) { - super(schema); + public HHXORErasureDecoder(ErasureCoderOptions options) { + super(options); } @Override @@ -71,25 +66,26 @@ public class HHXORErasureDecoder extends AbstractErasureDecoder { private RawErasureDecoder checkCreateRSRawDecoder() { if (rsRawDecoder == null) { - ErasureCoderOptions coderOptions = new ErasureCoderOptions( - getNumDataUnits(), getNumParityUnits()); rsRawDecoder = CodecUtil.createRawDecoder(getConf(), - ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, coderOptions); + ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, getOptions()); } return rsRawDecoder; } private RawErasureEncoder checkCreateXorRawEncoder() { if (xorRawEncoder == null) { - ErasureCoderOptions coderOptions = new ErasureCoderOptions( - getNumDataUnits(), getNumParityUnits()); xorRawEncoder = CodecUtil.createRawEncoder(getConf(), - ErasureCodeConstants.XOR_CODEC_NAME, coderOptions); + ErasureCodeConstants.XOR_CODEC_NAME, getOptions()); } return xorRawEncoder; } @Override + public boolean preferDirectBuffer() { + return false; + } + + @Override public void release() { if (rsRawDecoder != null) { rsRawDecoder.release(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecodingStep.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecodingStep.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecodingStep.java index 6c81836..98b8503 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecodingStep.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecodingStep.java @@ -32,7 +32,7 @@ import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder; * decoding a block group. */ @InterfaceAudience.Private -public class HHXORErasureDecodingStep extends AbstractHHErasureCodingStep { +public class HHXORErasureDecodingStep extends HHErasureCodingStep { private int pbIndex; private int[] piggyBackIndex; private int[] piggyBackFullIndex; http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java index 219f25c..7a15a05 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java @@ -21,7 +21,6 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.io.erasurecode.CodecUtil; import org.apache.hadoop.io.erasurecode.ECBlock; import org.apache.hadoop.io.erasurecode.ECBlockGroup; -import org.apache.hadoop.io.erasurecode.ECSchema; import org.apache.hadoop.io.erasurecode.ErasureCodeConstants; import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder; @@ -38,16 +37,12 @@ import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder; * This is Hitchhiker-XOR erasure encoder that encodes a block group. */ @InterfaceAudience.Private -public class HHXORErasureEncoder extends AbstractErasureEncoder { +public class HHXORErasureEncoder extends ErasureEncoder { private RawErasureEncoder rsRawEncoder; private RawErasureEncoder xorRawEncoder; - public HHXORErasureEncoder(int numDataUnits, int numParityUnits) { - super(numDataUnits, numParityUnits); - } - - public HHXORErasureEncoder(ECSchema schema) { - super(schema); + public HHXORErasureEncoder(ErasureCoderOptions options) { + super(options); } @Override @@ -65,21 +60,17 @@ public class HHXORErasureEncoder extends AbstractErasureEncoder { private RawErasureEncoder checkCreateRSRawEncoder() { if (rsRawEncoder == null) { - ErasureCoderOptions coderOptions = new ErasureCoderOptions( - getNumDataUnits(), getNumParityUnits()); rsRawEncoder = CodecUtil.createRawEncoder(getConf(), - ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, coderOptions); + ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, getOptions()); } return rsRawEncoder; } private RawErasureEncoder checkCreateXorRawEncoder() { if (xorRawEncoder == null) { - ErasureCoderOptions erasureCoderOptions = new ErasureCoderOptions( - getNumDataUnits(), getNumParityUnits()); xorRawEncoder = CodecUtil.createRawEncoder(getConf(), ErasureCodeConstants.XOR_CODEC_NAME, - erasureCoderOptions); + getOptions()); } return xorRawEncoder; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncodingStep.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncodingStep.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncodingStep.java index f83ee26..11b1bf1 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncodingStep.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncodingStep.java @@ -31,7 +31,7 @@ import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder; * encoding a block group. */ @InterfaceAudience.Private -public class HHXORErasureEncodingStep extends AbstractHHErasureCodingStep { +public class HHXORErasureEncodingStep extends HHErasureCodingStep { private int[] piggyBackIndex; private RawErasureEncoder rsRawEncoder; private RawErasureEncoder xorRawEncoder; --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
