HADOOP-13200. Implement customizable and configurable erasure coders. Contributed by Tim Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/19c688fc Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/19c688fc Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/19c688fc Branch: refs/heads/HDFS-10467 Commit: 19c688fceb817d0562bb6244e93be2abdb52462b Parents: bd16e1f Author: Wei-Chiu Chuang <[email protected]> Authored: Thu Apr 27 11:59:33 2017 -0700 Committer: Inigo <[email protected]> Committed: Tue May 2 14:52:03 2017 -0700 ---------------------------------------------------------------------- .../apache/hadoop/io/erasurecode/CodecUtil.java | 82 ++++++-------------- .../io/erasurecode/ErasureCodeConstants.java | 1 + .../rawcoder/DummyRawErasureCoderFactory.java | 12 +++ .../NativeRSRawErasureCoderFactory.java | 13 ++++ .../NativeXORRawErasureCoderFactory.java | 13 ++++ .../RSLegacyRawErasureCoderFactory.java | 13 ++++ .../rawcoder/RSRawErasureCoderFactory.java | 13 ++++ .../rawcoder/RawErasureCoderFactory.java | 12 +++ .../rawcoder/XORRawErasureCoderFactory.java | 13 ++++ .../src/main/resources/core-default.xml | 6 +- .../erasurecode/TestCodecRawCoderMapping.java | 8 +- .../coder/TestHHXORErasureCoder.java | 2 +- .../erasurecode/coder/TestRSErasureCoder.java | 2 +- .../src/site/markdown/HDFSErasureCoding.md | 14 ++-- .../hadoop/hdfs/TestDFSStripedInputStream.java | 2 +- .../hadoop/hdfs/TestDFSStripedOutputStream.java | 2 +- .../TestDFSStripedOutputStreamWithFailure.java | 2 +- .../hadoop/hdfs/TestReconstructStripedFile.java | 2 +- .../TestUnsetAndChangeDirectoryEcPolicy.java | 2 +- 19 files changed, 138 insertions(+), 76 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/19c688fc/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java index c8b6a68..6cdbb37 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java @@ -18,8 +18,6 @@ package org.apache.hadoop.io.erasurecode; import com.google.common.base.Preconditions; -import com.google.common.base.Splitter; -import com.google.common.collect.ImmutableMap; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; @@ -30,18 +28,12 @@ import org.apache.hadoop.io.erasurecode.codec.RSErasureCodec; import org.apache.hadoop.io.erasurecode.codec.XORErasureCodec; import org.apache.hadoop.io.erasurecode.coder.ErasureDecoder; import org.apache.hadoop.io.erasurecode.coder.ErasureEncoder; -import org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory; -import org.apache.hadoop.io.erasurecode.rawcoder.NativeXORRawErasureCoderFactory; -import org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory; -import org.apache.hadoop.io.erasurecode.rawcoder.RSLegacyRawErasureCoderFactory; import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureCoderFactory; import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder; import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder; -import org.apache.hadoop.io.erasurecode.rawcoder.XORRawErasureCoderFactory; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; -import java.util.Map; /** * A codec & coder utility to help create coders conveniently. @@ -79,27 +71,12 @@ public final class CodecUtil { /** Comma separated raw codec name. The first coder is prior to the latter. */ public static final String IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODERS_KEY = IO_ERASURECODE_CODEC + "rs-legacy.rawcoders"; - public static final String IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODERS_DEFAULT = - RSLegacyRawErasureCoderFactory.class.getCanonicalName(); public static final String IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY = IO_ERASURECODE_CODEC + "rs.rawcoders"; - public static final String IO_ERASURECODE_CODEC_RS_RAWCODERS_DEFAULT = - NativeRSRawErasureCoderFactory.class.getCanonicalName() + - "," + RSRawErasureCoderFactory.class.getCanonicalName(); /** Raw coder factory for the XOR codec. */ public static final String IO_ERASURECODE_CODEC_XOR_RAWCODERS_KEY = IO_ERASURECODE_CODEC + "xor.rawcoders"; - public static final String IO_ERASURECODE_CODEC_XOR_RAWCODERS_DEFAULT = - NativeXORRawErasureCoderFactory.class.getCanonicalName() + - "," + XORRawErasureCoderFactory.class.getCanonicalName(); - - // Default coders for each codec names. - public static final Map<String, String> DEFAULT_CODERS_MAP = ImmutableMap.of( - "rs", IO_ERASURECODE_CODEC_RS_RAWCODERS_DEFAULT, - "rs-legacy", IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODERS_DEFAULT, - "xor", IO_ERASURECODE_CODEC_XOR_RAWCODERS_DEFAULT - ); private CodecUtil() { } @@ -168,70 +145,61 @@ public final class CodecUtil { } private static RawErasureCoderFactory createRawCoderFactory( - Configuration conf, String rawCoderFactoryKey) { + String coderName, String codecName) { RawErasureCoderFactory fact; - try { - Class<? extends RawErasureCoderFactory> factClass = conf.getClassByName( - rawCoderFactoryKey).asSubclass(RawErasureCoderFactory.class); - fact = factClass.newInstance(); - } catch (ClassNotFoundException | InstantiationException | - IllegalAccessException e) { - throw new RuntimeException("Failed to create raw coder factory", e); - } - - if (fact == null) { - throw new RuntimeException("Failed to create raw coder factory"); - } + fact = CodecRegistry.getInstance(). + getCoderByName(codecName, coderName); return fact; } - // Return comma separated coder names - private static String getRawCoders(Configuration conf, String codec) { - return conf.get( - IO_ERASURECODE_CODEC + codec + ".rawcoders", - DEFAULT_CODERS_MAP.getOrDefault(codec, codec) + // Return a list of coder names + private static String[] getRawCoderNames( + Configuration conf, String codecName) { + return conf.getStrings( + IO_ERASURECODE_CODEC + codecName + ".rawcoders", + CodecRegistry.getInstance().getCoderNames(codecName) ); } private static RawErasureEncoder createRawEncoderWithFallback( - Configuration conf, String codec, ErasureCoderOptions coderOptions) { - String coders = getRawCoders(conf, codec); - for (String factName : Splitter.on(",").split(coders)) { + Configuration conf, String codecName, ErasureCoderOptions coderOptions) { + String[] rawCoderNames = getRawCoderNames(conf, codecName); + for (String rawCoderName : rawCoderNames) { try { - if (factName != null) { - RawErasureCoderFactory fact = createRawCoderFactory(conf, - factName); + if (rawCoderName != null) { + RawErasureCoderFactory fact = createRawCoderFactory( + rawCoderName, codecName); return fact.createEncoder(coderOptions); } } catch (LinkageError | Exception e) { // Fallback to next coder if possible - LOG.warn("Failed to create raw erasure encoder " + factName + + LOG.warn("Failed to create raw erasure encoder " + rawCoderName + ", fallback to next codec if possible", e); } } throw new IllegalArgumentException("Fail to create raw erasure " + - "encoder with given codec: " + codec); + "encoder with given codec: " + codecName); } private static RawErasureDecoder createRawDecoderWithFallback( - Configuration conf, String codec, ErasureCoderOptions coderOptions) { - String coders = getRawCoders(conf, codec); - for (String factName : Splitter.on(",").split(coders)) { + Configuration conf, String codecName, ErasureCoderOptions coderOptions) { + String[] coders = getRawCoderNames(conf, codecName); + for (String rawCoderName : coders) { try { - if (factName != null) { - RawErasureCoderFactory fact = createRawCoderFactory(conf, - factName); + if (rawCoderName != null) { + RawErasureCoderFactory fact = createRawCoderFactory( + rawCoderName, codecName); return fact.createDecoder(coderOptions); } } catch (LinkageError | Exception e) { // Fallback to next coder if possible - LOG.warn("Failed to create raw erasure decoder " + factName + + LOG.warn("Failed to create raw erasure decoder " + rawCoderName + ", fallback to next codec if possible", e); } } throw new IllegalArgumentException("Fail to create raw erasure " + - "encoder with given codec: " + codec); + "encoder with given codec: " + codecName); } private static ErasureCodec createCodec(Configuration conf, http://git-wip-us.apache.org/repos/asf/hadoop/blob/19c688fc/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java index c830bb2..e0d7946 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java @@ -25,6 +25,7 @@ public final class ErasureCodeConstants { private ErasureCodeConstants() { } + public static final String DUMMY_CODEC_NAME = "dummy"; public static final String RS_CODEC_NAME = "rs"; public static final String RS_LEGACY_CODEC_NAME = "rs-legacy"; public static final String XOR_CODEC_NAME = "xor"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/19c688fc/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/DummyRawErasureCoderFactory.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/DummyRawErasureCoderFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/DummyRawErasureCoderFactory.java index 31ba4ef..49b36b7 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/DummyRawErasureCoderFactory.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/DummyRawErasureCoderFactory.java @@ -18,6 +18,7 @@ package org.apache.hadoop.io.erasurecode.rawcoder; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.io.erasurecode.ErasureCodeConstants; import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; /** @@ -25,6 +26,7 @@ import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; */ @InterfaceAudience.Private public class DummyRawErasureCoderFactory implements RawErasureCoderFactory { + public static final String CODER_NAME = "dummy_dummy"; @Override public RawErasureEncoder createEncoder(ErasureCoderOptions coderOptions) { @@ -35,4 +37,14 @@ public class DummyRawErasureCoderFactory implements RawErasureCoderFactory { public RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions) { return new DummyRawDecoder(coderOptions); } + + @Override + public String getCoderName() { + return CODER_NAME; + } + + @Override + public String getCodecName() { + return ErasureCodeConstants.DUMMY_CODEC_NAME; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/19c688fc/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawErasureCoderFactory.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawErasureCoderFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawErasureCoderFactory.java index 38997dd..cc98ec6 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawErasureCoderFactory.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawErasureCoderFactory.java @@ -18,6 +18,7 @@ package org.apache.hadoop.io.erasurecode.rawcoder; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.io.erasurecode.ErasureCodeConstants; import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; /** @@ -27,6 +28,8 @@ import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; @InterfaceAudience.Private public class NativeRSRawErasureCoderFactory implements RawErasureCoderFactory { + public static final String CODER_NAME = "rs_native"; + @Override public RawErasureEncoder createEncoder(ErasureCoderOptions coderOptions) { return new NativeRSRawEncoder(coderOptions); @@ -36,4 +39,14 @@ public class NativeRSRawErasureCoderFactory implements RawErasureCoderFactory { public RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions) { return new NativeRSRawDecoder(coderOptions); } + + @Override + public String getCoderName() { + return CODER_NAME; + } + + @Override + public String getCodecName() { + return ErasureCodeConstants.RS_CODEC_NAME; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/19c688fc/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawErasureCoderFactory.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawErasureCoderFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawErasureCoderFactory.java index 66b3f78..29a00dc 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawErasureCoderFactory.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawErasureCoderFactory.java @@ -18,6 +18,7 @@ package org.apache.hadoop.io.erasurecode.rawcoder; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.io.erasurecode.ErasureCodeConstants; import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; /** @@ -27,6 +28,8 @@ import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; @InterfaceAudience.Private public class NativeXORRawErasureCoderFactory implements RawErasureCoderFactory { + public static final String CODER_NAME = "xor_native"; + @Override public RawErasureEncoder createEncoder(ErasureCoderOptions coderOptions) { return new NativeXORRawEncoder(coderOptions); @@ -36,4 +39,14 @@ public class NativeXORRawErasureCoderFactory implements RawErasureCoderFactory { public RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions) { return new NativeXORRawDecoder(coderOptions); } + + @Override + public String getCoderName() { + return CODER_NAME; + } + + @Override + public String getCodecName() { + return ErasureCodeConstants.XOR_CODEC_NAME; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/19c688fc/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSLegacyRawErasureCoderFactory.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSLegacyRawErasureCoderFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSLegacyRawErasureCoderFactory.java index 45bbf40..6cbe33f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSLegacyRawErasureCoderFactory.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSLegacyRawErasureCoderFactory.java @@ -18,6 +18,7 @@ package org.apache.hadoop.io.erasurecode.rawcoder; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.io.erasurecode.ErasureCodeConstants; import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; /** @@ -26,6 +27,8 @@ import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; @InterfaceAudience.Private public class RSLegacyRawErasureCoderFactory implements RawErasureCoderFactory { + public static final String CODER_NAME = "rs-legacy_java"; + @Override public RawErasureEncoder createEncoder(ErasureCoderOptions coderOptions) { return new RSLegacyRawEncoder(coderOptions); @@ -35,4 +38,14 @@ public class RSLegacyRawErasureCoderFactory implements RawErasureCoderFactory { public RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions) { return new RSLegacyRawDecoder(coderOptions); } + + @Override + public String getCoderName() { + return CODER_NAME; + } + + @Override + public String getCodecName() { + return ErasureCodeConstants.RS_LEGACY_CODEC_NAME; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/19c688fc/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawErasureCoderFactory.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawErasureCoderFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawErasureCoderFactory.java index 8d954d5..4aadcb7 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawErasureCoderFactory.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawErasureCoderFactory.java @@ -18,6 +18,7 @@ package org.apache.hadoop.io.erasurecode.rawcoder; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.io.erasurecode.ErasureCodeConstants; import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; /** @@ -26,6 +27,8 @@ import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; @InterfaceAudience.Private public class RSRawErasureCoderFactory implements RawErasureCoderFactory { + public static final String CODER_NAME = "rs_java"; + @Override public RawErasureEncoder createEncoder(ErasureCoderOptions coderOptions) { return new RSRawEncoder(coderOptions); @@ -35,4 +38,14 @@ public class RSRawErasureCoderFactory implements RawErasureCoderFactory { public RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions) { return new RSRawDecoder(coderOptions); } + + @Override + public String getCoderName() { + return CODER_NAME; + } + + @Override + public String getCodecName() { + return ErasureCodeConstants.RS_CODEC_NAME; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/19c688fc/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderFactory.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderFactory.java index 05f228b..704e3e8 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderFactory.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderFactory.java @@ -41,4 +41,16 @@ public interface RawErasureCoderFactory { * @return raw erasure decoder */ RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions); + + /** + * Get the name of the coder. + * @return coder name + */ + String getCoderName(); + + /** + * Get the name of its codec. + * @return codec name + */ + String getCodecName(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/19c688fc/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawErasureCoderFactory.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawErasureCoderFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawErasureCoderFactory.java index 571fe12..595f022 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawErasureCoderFactory.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawErasureCoderFactory.java @@ -18,6 +18,7 @@ package org.apache.hadoop.io.erasurecode.rawcoder; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.io.erasurecode.ErasureCodeConstants; import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; /** @@ -26,6 +27,8 @@ import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; @InterfaceAudience.Private public class XORRawErasureCoderFactory implements RawErasureCoderFactory { + public static final String CODER_NAME = "xor_java"; + @Override public RawErasureEncoder createEncoder(ErasureCoderOptions coderOptions) { return new XORRawEncoder(coderOptions); @@ -35,4 +38,14 @@ public class XORRawErasureCoderFactory implements RawErasureCoderFactory { public RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions) { return new XORRawDecoder(coderOptions); } + + @Override + public String getCoderName() { + return CODER_NAME; + } + + @Override + public String getCodecName() { + return ErasureCodeConstants.XOR_CODEC_NAME; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/19c688fc/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 6fa70fd..e9a4f1a 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -668,7 +668,7 @@ <property> <name>io.erasurecode.codec.rs.rawcoders</name> - <value>org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory,org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory</value> + <value>rs_native,rs_java</value> <description> Comma separated raw coder implementations for the rs codec. The earlier factory is prior to followings in case of failure of creating raw coders. @@ -677,7 +677,7 @@ <property> <name>io.erasurecode.codec.rs-legacy.rawcoders</name> - <value>org.apache.hadoop.io.erasurecode.rawcoder.RSLegacyRawErasureCoderFactory</value> + <value>rs-legacy_java</value> <description> Comma separated raw coder implementations for the rs-legacy codec. The earlier factory is prior to followings in case of failure of creating raw coders. @@ -686,7 +686,7 @@ <property> <name>io.erasurecode.codec.xor.rawcoders</name> - <value>org.apache.hadoop.io.erasurecode.rawcoder.NativeXORRawErasureCoderFactory,org.apache.hadoop.io.erasurecode.rawcoder.XORRawErasureCoderFactory</value> + <value>xor_native,xor_java</value> <description> Comma separated raw coder implementations for the xor codec. The earlier factory is prior to followings in case of failure of creating raw coders. http://git-wip-us.apache.org/repos/asf/hadoop/blob/19c688fc/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java index b5cada6..4a6fbd4 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java @@ -28,6 +28,7 @@ import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder; import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder; import org.apache.hadoop.io.erasurecode.rawcoder.XORRawDecoder; import org.apache.hadoop.io.erasurecode.rawcoder.XORRawEncoder; +import org.apache.hadoop.io.erasurecode.rawcoder.XORRawErasureCoderFactory; import org.apache.hadoop.test.GenericTestUtils; import org.junit.Assert; import org.junit.Before; @@ -104,8 +105,8 @@ public class TestCodecRawCoderMapping { ErasureCoderOptions coderOptions = new ErasureCoderOptions( numDataUnit, numParityUnit); conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY, - RSRawErasureCoderFactory.class.getCanonicalName() + - "," + NativeRSRawErasureCoderFactory.class.getCanonicalName()); + RSRawErasureCoderFactory.CODER_NAME + + "," + NativeRSRawErasureCoderFactory.CODER_NAME); // should return default raw coder of rs codec RawErasureEncoder encoder = CodecUtil.createRawEncoder( conf, ErasureCodeConstants.RS_CODEC_NAME, coderOptions); @@ -133,8 +134,7 @@ public class TestCodecRawCoderMapping { ErasureCoderOptions coderOptions = new ErasureCoderOptions( numDataUnit, numParityUnit); conf.set(CodecUtil.IO_ERASURECODE_CODEC_XOR_RAWCODERS_KEY, - "invalid-codec," + - "org.apache.hadoop.io.erasurecode.rawcoder.XORRawErasureCoderFactory"); + "invalid-codec," + XORRawErasureCoderFactory.CODER_NAME); // should return second coder specified by IO_ERASURECODE_CODEC_CODERS RawErasureEncoder encoder = CodecUtil.createRawEncoder( conf, ErasureCodeConstants.XOR_CODEC_NAME, coderOptions); http://git-wip-us.apache.org/repos/asf/hadoop/blob/19c688fc/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java index a475a39..094ed08 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java @@ -51,7 +51,7 @@ public class TestHHXORErasureCoder extends TestHHErasureCoderBase { */ Configuration conf = new Configuration(); conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY, - RSRawErasureCoderFactory.class.getCanonicalName()); + RSRawErasureCoderFactory.CODER_NAME); prepare(conf, 10, 4, new int[]{0}, new int[0]); testCoding(true); http://git-wip-us.apache.org/repos/asf/hadoop/blob/19c688fc/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java index 11abac4..19054ce 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java @@ -58,7 +58,7 @@ public class TestRSErasureCoder extends TestErasureCoderBase { */ Configuration conf = new Configuration(); conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY, - RSRawErasureCoderFactory.class.getCanonicalName()); + RSRawErasureCoderFactory.CODER_NAME); prepare(conf, 10, 4, new int[]{0}, new int[0]); testCoding(true); http://git-wip-us.apache.org/repos/asf/hadoop/blob/19c688fc/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md index 543b5ce..dbfb111 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md @@ -117,11 +117,15 @@ Deployment be more appropriate. If the administrator only cares about node-level fault-tolerance, `RS-10-4-64k` would still be appropriate as long as there are at least 14 DataNodes in the cluster. - The codec implementation for Reed-Solomon and XOR can be configured with the following client and DataNode configuration keys: - `io.erasurecode.codec.rs.rawcoder` for the default RS codec, - `io.erasurecode.codec.rs-legacy.rawcoder` for the legacy RS codec, - `io.erasurecode.codec.xor.rawcoder` for the XOR codec. - The default implementations for all of these codecs are pure Java. For default RS codec, there is also a native implementation which leverages Intel ISA-L library to improve the performance of codec. For XOR codec, a native implementation which leverages Intel ISA-L library to improve the performance of codec is also supported. Please refer to section "Enable Intel ISA-L" for more detail information. + The codec implementations for Reed-Solomon and XOR can be configured with the following client and DataNode configuration keys: + `io.erasurecode.codec.rs.rawcoders` for the default RS codec, + `io.erasurecode.codec.rs-legacy.rawcoders` for the legacy RS codec, + `io.erasurecode.codec.xor.rawcoders` for the XOR codec. + User can also configure self-defined codec with configuration key like: + `io.erasurecode.codec.self-defined-codec.rawcoders`. + The values for these key are lists of coder names with a fall-back mechanism. + All these codecs have implementations in pure Java. For default RS codec, there is also a native implementation which leverages Intel ISA-L library to improve the performance of codec. For XOR codec, a native implementation which leverages Intel ISA-L library to improve the performance of codec is also supported. Please refer to section "Enable Intel ISA-L" for more detail information. + The default implementation for RS Legacy is pure Java, and the default implementations for default RS and XOR are native implementations using Intel ISA-L library. Erasure coding background recovery work on the DataNodes can also be tuned via the following configuration parameters: http://git-wip-us.apache.org/repos/asf/hadoop/blob/19c688fc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java index e3332fd..4f67a0a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java @@ -98,7 +98,7 @@ public class TestDFSStripedInputStream { if (ErasureCodeNative.isNativeCodeLoaded()) { conf.set( CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY, - NativeRSRawErasureCoderFactory.class.getCanonicalName()); + NativeRSRawErasureCoderFactory.CODER_NAME); } SimulatedFSDataset.setFactory(conf); cluster = new MiniDFSCluster.Builder(conf).numDataNodes( http://git-wip-us.apache.org/repos/asf/hadoop/blob/19c688fc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java index a2d5c8d..14825ca 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java @@ -85,7 +85,7 @@ public class TestDFSStripedOutputStream { if (ErasureCodeNative.isNativeCodeLoaded()) { conf.set( CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY, - NativeRSRawErasureCoderFactory.class.getCanonicalName()); + NativeRSRawErasureCoderFactory.CODER_NAME); } DFSTestUtil.enableAllECPolicies(conf); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/19c688fc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java index 66a805c..9915a2f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java @@ -214,7 +214,7 @@ public class TestDFSStripedOutputStreamWithFailure { if (ErasureCodeNative.isNativeCodeLoaded()) { conf.set( CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY, - NativeRSRawErasureCoderFactory.class.getCanonicalName()); + NativeRSRawErasureCoderFactory.CODER_NAME); } DFSTestUtil.enableAllECPolicies(conf); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/19c688fc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java index 2bd4a90..29e4028 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java @@ -100,7 +100,7 @@ public class TestReconstructStripedFile { if (ErasureCodeNative.isNativeCodeLoaded()) { conf.set( CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY, - NativeRSRawErasureCoderFactory.class.getCanonicalName()); + NativeRSRawErasureCoderFactory.CODER_NAME); } conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, StripedFileTestUtil.getDefaultECPolicy().getName()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/19c688fc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java index 6379db5..5371e20 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java @@ -68,7 +68,7 @@ public class TestUnsetAndChangeDirectoryEcPolicy { if (ErasureCodeNative.isNativeCodeLoaded()) { conf.set( CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY, - NativeRSRawErasureCoderFactory.class.getCanonicalName()); + NativeRSRawErasureCoderFactory.CODER_NAME); } DFSTestUtil.enableAllECPolicies(conf); cluster = new MiniDFSCluster.Builder(conf).numDataNodes( --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
