HDFS-11427. Rename rs-default to rs.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c1a52b04 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c1a52b04 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c1a52b04 Branch: refs/heads/YARN-5734 Commit: c1a52b04d0cc5ad5c86ae93043655f313386f7f9 Parents: b32ffa2 Author: Andrew Wang <w...@apache.org> Authored: Fri Feb 24 10:58:45 2017 -0800 Committer: Andrew Wang <w...@apache.org> Committed: Fri Feb 24 10:58:45 2017 -0800 ---------------------------------------------------------------------- .../apache/hadoop/io/erasurecode/CodecUtil.java | 28 +++++++-------- .../io/erasurecode/ErasureCodeConstants.java | 8 ++--- .../erasurecode/coder/HHXORErasureDecoder.java | 2 +- .../erasurecode/coder/HHXORErasureEncoder.java | 2 +- .../io/erasurecode/coder/RSErasureDecoder.java | 2 +- .../io/erasurecode/coder/RSErasureEncoder.java | 2 +- .../src/main/resources/core-default.xml | 4 +-- .../erasurecode/TestCodecRawCoderMapping.java | 10 +++--- .../coder/TestHHXORErasureCoder.java | 2 +- .../erasurecode/coder/TestRSErasureCoder.java | 2 +- .../src/site/markdown/HDFSErasureCoding.md | 6 ++-- .../TestDFSRSDefault10x4StripedInputStream.java | 2 +- ...TestDFSRSDefault10x4StripedOutputStream.java | 2 +- ...fault10x4StripedOutputStreamWithFailure.java | 4 +-- .../hadoop/hdfs/TestDFSStripedInputStream.java | 2 +- .../hadoop/hdfs/TestDFSStripedOutputStream.java | 2 +- .../TestDFSStripedOutputStreamWithFailure.java | 2 +- .../hadoop/hdfs/TestReconstructStripedFile.java | 2 +- .../TestUnsetAndChangeDirectoryEcPolicy.java | 2 +- .../test/resources/testErasureCodingConf.xml | 36 ++++++++++---------- 20 files changed, 61 insertions(+), 61 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java index 977bacb..861451a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java @@ -55,9 +55,9 @@ public final class CodecUtil { public static final String IO_ERASURECODE_CODEC_XOR = XORErasureCodec.class.getCanonicalName(); /** Erasure coder Reed-Solomon codec. */ - public static final String IO_ERASURECODE_CODEC_RS_DEFAULT_KEY = + public static final String IO_ERASURECODE_CODEC_RS_KEY = "io.erasurecode.codec.rs"; - public static final String IO_ERASURECODE_CODEC_RS_DEFAULT = + public static final String IO_ERASURECODE_CODEC_RS = RSErasureCodec.class.getCanonicalName(); /** Erasure coder hitch hiker XOR codec. */ public static final String IO_ERASURECODE_CODEC_HHXOR_KEY = @@ -67,10 +67,10 @@ public final class CodecUtil { /** Supported erasure codec classes. */ - /** Raw coder factory for the RS default codec. */ - public static final String IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY = - "io.erasurecode.codec.rs-default.rawcoder"; - public static final String IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_DEFAULT = + /** Raw coder factory for the RS codec. */ + public static final String IO_ERASURECODE_CODEC_RS_RAWCODER_KEY = + "io.erasurecode.codec.rs.rawcoder"; + public static final String IO_ERASURECODE_CODEC_RS_RAWCODER_DEFAULT = RSRawErasureCoderFactory.class.getCanonicalName(); /** Raw coder factory for the RS legacy codec. */ @@ -183,10 +183,10 @@ public final class CodecUtil { private static String getRawCoderFactNameFromCodec(Configuration conf, String codec) { switch (codec) { - case ErasureCodeConstants.RS_DEFAULT_CODEC_NAME: + case ErasureCodeConstants.RS_CODEC_NAME: return conf.get( - IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY, - IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_DEFAULT); + IO_ERASURECODE_CODEC_RS_RAWCODER_KEY, + IO_ERASURECODE_CODEC_RS_RAWCODER_DEFAULT); case ErasureCodeConstants.RS_LEGACY_CODEC_NAME: return conf.get( IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_KEY, @@ -233,15 +233,15 @@ public final class CodecUtil { private static String getCodecClassName(Configuration conf, String codec) { switch (codec) { - case ErasureCodeConstants.RS_DEFAULT_CODEC_NAME: + case ErasureCodeConstants.RS_CODEC_NAME: return conf.get( - CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_KEY, - CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT); + CodecUtil.IO_ERASURECODE_CODEC_RS_KEY, + CodecUtil.IO_ERASURECODE_CODEC_RS); case ErasureCodeConstants.RS_LEGACY_CODEC_NAME: //TODO:rs-legacy should be handled differently. return conf.get( - CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_KEY, - CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT); + CodecUtil.IO_ERASURECODE_CODEC_RS_KEY, + CodecUtil.IO_ERASURECODE_CODEC_RS); case ErasureCodeConstants.XOR_CODEC_NAME: return conf.get( CodecUtil.IO_ERASURECODE_CODEC_XOR_KEY, http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java index e168909..c830bb2 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java @@ -25,16 +25,16 @@ public final class ErasureCodeConstants { private ErasureCodeConstants() { } - public static final String RS_DEFAULT_CODEC_NAME = "rs-default"; + public static final String RS_CODEC_NAME = "rs"; public static final String RS_LEGACY_CODEC_NAME = "rs-legacy"; public static final String XOR_CODEC_NAME = "xor"; public static final String HHXOR_CODEC_NAME = "hhxor"; public static final ECSchema RS_6_3_SCHEMA = new ECSchema( - RS_DEFAULT_CODEC_NAME, 6, 3); + RS_CODEC_NAME, 6, 3); public static final ECSchema RS_3_2_SCHEMA = new ECSchema( - RS_DEFAULT_CODEC_NAME, 3, 2); + RS_CODEC_NAME, 3, 2); public static final ECSchema RS_6_3_LEGACY_SCHEMA = new ECSchema( RS_LEGACY_CODEC_NAME, 6, 3); @@ -43,5 +43,5 @@ public final class ErasureCodeConstants { XOR_CODEC_NAME, 2, 1); public static final ECSchema RS_10_4_SCHEMA = new ECSchema( - RS_DEFAULT_CODEC_NAME, 10, 4); + RS_CODEC_NAME, 10, 4); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecoder.java index 05e9384..4747e49 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecoder.java @@ -67,7 +67,7 @@ public class HHXORErasureDecoder extends ErasureDecoder { private RawErasureDecoder checkCreateRSRawDecoder() { if (rsRawDecoder == null) { rsRawDecoder = CodecUtil.createRawDecoder(getConf(), - ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, getOptions()); + ErasureCodeConstants.RS_CODEC_NAME, getOptions()); } return rsRawDecoder; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java index 7a15a05..6b858b6 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java @@ -61,7 +61,7 @@ public class HHXORErasureEncoder extends ErasureEncoder { private RawErasureEncoder checkCreateRSRawEncoder() { if (rsRawEncoder == null) { rsRawEncoder = CodecUtil.createRawEncoder(getConf(), - ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, getOptions()); + ErasureCodeConstants.RS_CODEC_NAME, getOptions()); } return rsRawEncoder; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java index 6e679c3..ba7f773 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java @@ -52,7 +52,7 @@ public class RSErasureDecoder extends ErasureDecoder { private RawErasureDecoder checkCreateRSRawDecoder() { if (rsRawDecoder == null) { rsRawDecoder = CodecUtil.createRawDecoder(getConf(), - ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, getOptions()); + ErasureCodeConstants.RS_CODEC_NAME, getOptions()); } return rsRawDecoder; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java index 7a09b92..193309d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java @@ -53,7 +53,7 @@ public class RSErasureEncoder extends ErasureEncoder { if (rawEncoder == null) { // TODO: we should create the raw coder according to codec. rawEncoder = CodecUtil.createRawEncoder(getConf(), - ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, getOptions()); + ErasureCodeConstants.RS_CODEC_NAME, getOptions()); } return rawEncoder; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index d8136ee..08ca05b 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -657,10 +657,10 @@ </property> <property> - <name>io.erasurecode.codec.rs-default.rawcoder</name> + <name>io.erasurecode.codec.rs.rawcoder</name> <value>org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory</value> <description> - Raw coder implementation for the rs-default codec. The default value is a + Raw coder implementation for the rs codec. The default value is a pure Java implementation. There is also a native implementation. Its value is org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory. </description> http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java index 0db001a..7f7fcf3 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java @@ -47,12 +47,12 @@ public class TestCodecRawCoderMapping { public void testRSDefaultRawCoder() { ErasureCoderOptions coderOptions = new ErasureCoderOptions( numDataUnit, numParityUnit); - // should return default raw coder of rs-default codec + // should return default raw coder of rs codec RawErasureEncoder encoder = CodecUtil.createRawEncoder( - conf, ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, coderOptions); + conf, ErasureCodeConstants.RS_CODEC_NAME, coderOptions); Assert.assertTrue(encoder instanceof RSRawEncoder); RawErasureDecoder decoder = CodecUtil.createRawDecoder( - conf, ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, coderOptions); + conf, ErasureCodeConstants.RS_CODEC_NAME, coderOptions); Assert.assertTrue(decoder instanceof RSRawDecoder); // should return default raw coder of rs-legacy codec @@ -71,11 +71,11 @@ public class TestCodecRawCoderMapping { String dummyFactName = "DummyNoneExistingFactory"; // set the dummy factory to rs-legacy and create a raw coder - // with rs-default, which is OK as the raw coder key is not used + // with rs, which is OK as the raw coder key is not used conf.set(CodecUtil. IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_KEY, dummyFactName); RawErasureEncoder encoder = CodecUtil.createRawEncoder(conf, - ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, coderOptions); + ErasureCodeConstants.RS_CODEC_NAME, coderOptions); Assert.assertTrue(encoder instanceof RSRawEncoder); // now create the raw coder with rs-legacy, which should throw exception try { http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java index 1eca1e1..91c4ba8 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java @@ -50,7 +50,7 @@ public class TestHHXORErasureCoder extends TestHHErasureCoderBase { * This tests if the configuration items work or not. */ Configuration conf = new Configuration(); - conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY, + conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY, RSRawErasureCoderFactory.class.getCanonicalName()); prepare(conf, 10, 4, new int[]{0}, new int[0]); http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java index 5f36c99..3b18347 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java @@ -57,7 +57,7 @@ public class TestRSErasureCoder extends TestErasureCoderBase { * This tests if the configuration items work or not. */ Configuration conf = new Configuration(); - conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY, + conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY, RSRawErasureCoderFactory.class.getCanonicalName()); prepare(conf, 10, 4, new int[]{0}, new int[0]); http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md index 0283e2b..36fb61d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md @@ -64,7 +64,7 @@ Architecture 2. _The size of a striping cell._ This determines the granularity of striped reads and writes, including buffer sizes and encoding work. - There are four policies currently being supported: RS-DEFAULT-3-2-64k, RS-DEFAULT-6-3-64k, RS-DEFAULT-10-4-64k and RS-LEGACY-6-3-64k. All with default cell size of 64KB. The system default policy is RS-DEFAULT-6-3-64k which use the default schema RS_6_3_SCHEMA with a cell size of 64KB. + Five policies are currently supported: RS-3-2-64k, RS-6-3-64k, RS-10-4-64k, RS-LEGACY-6-3-64k, and XOR-2-1-64k. All with default cell size of 64KB. The system default policy is RS-6-3-64k which use the default schema RS_6_3_SCHEMA with a cell size of 64KB. * **Intel ISA-L** Intel ISA-L stands for Intel Intelligent Storage Acceleration Library. ISA-L is a collection of optimized low-level functions used primarily in storage applications. It includes a fast block Reed-Solomon type erasure codes optimized for Intel AVX and AVX2 instruction sets. @@ -91,7 +91,7 @@ Deployment ### Configuration keys The codec implementation for Reed-Solomon and XOR can be configured with the following client and DataNode configuration keys: - `io.erasurecode.codec.rs-default.rawcoder` for the default RS codec, + `io.erasurecode.codec.rs.rawcoder` for the default RS codec, `io.erasurecode.codec.rs-legacy.rawcoder` for the legacy RS codec, `io.erasurecode.codec.xor.rawcoder` for the XOR codec. The default implementations for all of these codecs are pure Java. For default RS codec, there is also a native implementation which leverages Intel ISA-L library to improve the performance of codec. For XOR codec, a native implementation which leverages Intel ISA-L library to improve the performance of codec is also supported. Please refer to section "Enable Intel ISA-L" for more detail information. @@ -108,7 +108,7 @@ Deployment HDFS native implementation of default RS codec leverages Intel ISA-L library to improve the encoding and decoding calculation. To enable and use Intel ISA-L, there are three steps. 1. Build ISA-L library. Please refer to the offical site "https://github.com/01org/isa-l/"; for detail information. 2. Build Hadoop with ISA-L support. Please refer to "Intel ISA-L build options" section in "Build instructions for Hadoop"(BUILDING.txt) document. Use -Dbundle.isal to copy the contents of the isal.lib directory into the final tar file. Deploy hadoop with the tar file. Make sure ISA-L library is available on both HDFS client and DataNodes. - 3. Configure the `io.erasurecode.codec.rs-default.rawcoder` key with value `org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory` on HDFS client and DataNodes. + 3. Configure the `io.erasurecode.codec.rs.rawcoder` key with value `org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory` on HDFS client and DataNodes. To check ISA-L library enable state, try "Hadoop checknative" command. It will tell you if ISA-L library is enabled or not. http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedInputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedInputStream.java index fc0ee37..3e6d1e4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedInputStream.java @@ -22,7 +22,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; /** - * This tests read operation of DFS striped file with RS-DEFAULT-10-4-64k + * This tests read operation of DFS striped file with RS-10-4-64k * erasure code policy. */ public class TestDFSRSDefault10x4StripedInputStream extends http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStream.java index 37821c1..1ea839a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStream.java @@ -22,7 +22,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; /** - * This tests write operation of DFS striped file with RS-DEFAULT-10-4-64k + * This tests write operation of DFS striped file with RS-10-4-64k * erasure code policy. */ public class TestDFSRSDefault10x4StripedOutputStream http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStreamWithFailure.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStreamWithFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStreamWithFailure.java index 1b2ec42..340fec5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStreamWithFailure.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStreamWithFailure.java @@ -22,7 +22,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; /** - * This tests write operation of DFS striped file with RS-DEFAULT-10-4-64k + * This tests write operation of DFS striped file with RS-10-4-64k * erasure code policy under Datanode failure conditions. */ public class TestDFSRSDefault10x4StripedOutputStreamWithFailure @@ -33,4 +33,4 @@ public class TestDFSRSDefault10x4StripedOutputStreamWithFailure return ErasureCodingPolicyManager.getPolicyByPolicyID( HdfsConstants.RS_10_4_POLICY_ID); } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java index 121b9a4..9b69904 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java @@ -96,7 +96,7 @@ public class TestDFSStripedInputStream { conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0); if (ErasureCodeNative.isNativeCodeLoaded()) { conf.set( - CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY, + CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY, NativeRSRawErasureCoderFactory.class.getCanonicalName()); } SimulatedFSDataset.setFactory(conf); http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java index 5bde16e..8834e14 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java @@ -85,7 +85,7 @@ public class TestDFSStripedOutputStream { conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0); if (ErasureCodeNative.isNativeCodeLoaded()) { conf.set( - CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY, + CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY, NativeRSRawErasureCoderFactory.class.getCanonicalName()); } cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java index 0baf9cc..fabd417 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java @@ -214,7 +214,7 @@ public class TestDFSStripedOutputStreamWithFailure { final int numDNs = dataBlocks + parityBlocks; if (ErasureCodeNative.isNativeCodeLoaded()) { conf.set( - CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY, + CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY, NativeRSRawErasureCoderFactory.class.getCanonicalName()); } cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java index 4960c58..08b53c2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java @@ -100,7 +100,7 @@ public class TestReconstructStripedFile { false); if (ErasureCodeNative.isNativeCodeLoaded()) { conf.set( - CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY, + CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY, NativeRSRawErasureCoderFactory.class.getCanonicalName()); } cluster = new MiniDFSCluster.Builder(conf).numDataNodes(dnNum).build(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java index 1a4086e..f26aa06 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java @@ -69,7 +69,7 @@ public class TestUnsetAndChangeDirectoryEcPolicy { conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0); if (ErasureCodeNative.isNativeCodeLoaded()) { conf.set( - CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY, + CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY, NativeRSRawErasureCoderFactory.class.getCanonicalName()); } cluster = new MiniDFSCluster.Builder(conf).numDataNodes( http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml index 82b71def..9b9003a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml @@ -133,7 +133,7 @@ <description>setPolicy : set erasure coding policy on a directory to encode files</description> <test-commands> <command>-fs NAMENODE -mkdir /ecdir</command> - <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir</ec-admin-command> + <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command> </test-commands> <cleanup-commands> <command>-fs NAMENODE -rmdir /ecdir</command> @@ -141,7 +141,7 @@ <comparators> <comparator> <type>SubstringComparator</type> - <expected-output>Set erasure coding policy RS-DEFAULT-6-3-64k on /ecdir</expected-output> + <expected-output>Set erasure coding policy RS-6-3-64k on /ecdir</expected-output> </comparator> </comparators> </test> @@ -150,8 +150,8 @@ <description>setPolicy : set a policy twice</description> <test-commands> <command>-fs NAMENODE -mkdir /ecdir</command> - <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir</ec-admin-command> - <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir</ec-admin-command> + <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command> + <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command> </test-commands> <cleanup-commands> <command>-fs NAMENODE -rmdir /ecdir</command> @@ -159,7 +159,7 @@ <comparators> <comparator> <type>SubstringComparator</type> - <expected-output>Set erasure coding policy RS-DEFAULT-6-3-64k on /ecdir</expected-output> + <expected-output>Set erasure coding policy RS-6-3-64k on /ecdir</expected-output> </comparator> </comparators> </test> @@ -168,7 +168,7 @@ <description>unsetPolicy : unset policy and get</description> <test-commands> <command>-fs NAMENODE -mkdir /ecdir</command> - <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir</ec-admin-command> + <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command> <ec-admin-command>-fs NAMENODE -unsetPolicy -path /ecdir</ec-admin-command> <ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir</ec-admin-command> </test-commands> @@ -187,8 +187,8 @@ <description>setPolicy : change different policy and get</description> <test-commands> <command>-fs NAMENODE -mkdir /ecdir</command> - <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir</ec-admin-command> - <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-3-2-64k -path /ecdir</ec-admin-command> + <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command> + <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-3-2-64k -path /ecdir</ec-admin-command> <ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir</ec-admin-command> </test-commands> <cleanup-commands> @@ -197,7 +197,7 @@ <comparators> <comparator> <type>SubstringComparator</type> - <expected-output>RS-DEFAULT-3-2-64k</expected-output> + <expected-output>RS-3-2-64k</expected-output> </comparator> </comparators> </test> @@ -207,7 +207,7 @@ <test-commands> <command>-fs NAMENODE -mkdir /ecdir</command> <command>-fs NAMENODE -mkdir /ecdir/child</command> - <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir</ec-admin-command> + <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command> <ec-admin-command>-fs NAMENODE -unsetPolicy -path /ecdir/child</ec-admin-command> <command>-fs NAMENODE -touchz /ecdir/child/ecfile</command> <ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir/child/ecfile</ec-admin-command> @@ -220,7 +220,7 @@ <comparators> <comparator> <type>SubstringComparator</type> - <expected-output>RS-DEFAULT-6-3-64k</expected-output> + <expected-output>RS-6-3-64k</expected-output> </comparator> </comparators> </test> @@ -246,7 +246,7 @@ <description>getPolicy : get EC policy information at specified path, which doesn't have an EC policy</description> <test-commands> <command>-fs NAMENODE -mkdir /ecdir</command> - <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir</ec-admin-command> + <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command> <ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir</ec-admin-command> </test-commands> <cleanup-commands> @@ -255,7 +255,7 @@ <comparators> <comparator> <type>SubstringComparator</type> - <expected-output>RS-DEFAULT-6-3-64k</expected-output> + <expected-output>RS-6-3-64k</expected-output> </comparator> </comparators> </test> @@ -264,7 +264,7 @@ <description>getPolicy : get EC policy information at specified path, which doesn't have an EC policy</description> <test-commands> <command>-fs NAMENODE -mkdir /ecdir</command> - <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir</ec-admin-command> + <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command> <command>-fs NAMENODE -touchz /ecdir/ecfile</command> <ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir/ecfile</ec-admin-command> </test-commands> @@ -275,7 +275,7 @@ <comparators> <comparator> <type>SubstringComparator</type> - <expected-output>RS-DEFAULT-6-3-64k</expected-output> + <expected-output>RS-6-3-64k</expected-output> </comparator> </comparators> </test> @@ -290,7 +290,7 @@ <comparators> <comparator> <type>SubstringComparator</type> - <expected-output>RS-DEFAULT-6-3</expected-output> + <expected-output>RS-6-3</expected-output> </comparator> </comparators> </test> @@ -334,7 +334,7 @@ <description>setPolicy : illegal parameters - too many arguments</description> <test-commands> <command>-fs NAMENODE -mkdir /ecdir</command> - <ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir1 -policy RS-DEFAULT-3-2-64k /ecdir2</ec-admin-command> + <ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir1 -policy RS-3-2-64k /ecdir2</ec-admin-command> </test-commands> <cleanup-commands> <command>-fs NAMENODE -rmdir /ecdir</command> @@ -367,7 +367,7 @@ <test> <description>setPolicy : illegal parameters - no such file</description> <test-commands> - <ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir -policy RS-DEFAULT-3-2-64k</ec-admin-command> + <ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir -policy RS-3-2-64k</ec-admin-command> </test-commands> <cleanup-commands> </cleanup-commands> --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
