Repository: hadoop Updated Branches: refs/heads/trunk bedfec0c1 -> c023c7488
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java index afaaf24..6e679c3 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java @@ -21,7 +21,6 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.io.erasurecode.CodecUtil; import org.apache.hadoop.io.erasurecode.ECBlock; import org.apache.hadoop.io.erasurecode.ECBlockGroup; -import org.apache.hadoop.io.erasurecode.ECSchema; import org.apache.hadoop.io.erasurecode.ErasureCodeConstants; import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder; @@ -32,15 +31,11 @@ import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder; * It implements {@link ErasureCoder}. */ @InterfaceAudience.Private -public class RSErasureDecoder extends AbstractErasureDecoder { +public class RSErasureDecoder extends ErasureDecoder { private RawErasureDecoder rsRawDecoder; - public RSErasureDecoder(int numDataUnits, int numParityUnits) { - super(numDataUnits, numParityUnits); - } - - public RSErasureDecoder(ECSchema schema) { - super(schema); + public RSErasureDecoder(ErasureCoderOptions options) { + super(options); } @Override @@ -56,11 +51,8 @@ public class RSErasureDecoder extends AbstractErasureDecoder { private RawErasureDecoder checkCreateRSRawDecoder() { if (rsRawDecoder == null) { - // TODO: we should create the raw coder according to codec. - ErasureCoderOptions coderOptions = new ErasureCoderOptions( - getNumDataUnits(), getNumParityUnits()); rsRawDecoder = CodecUtil.createRawDecoder(getConf(), - ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, coderOptions); + ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, getOptions()); } return rsRawDecoder; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java index 2139113..7a09b92 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java @@ -21,7 +21,6 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.io.erasurecode.CodecUtil; import org.apache.hadoop.io.erasurecode.ECBlock; import org.apache.hadoop.io.erasurecode.ECBlockGroup; -import org.apache.hadoop.io.erasurecode.ECSchema; import org.apache.hadoop.io.erasurecode.ErasureCodeConstants; import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder; @@ -32,15 +31,11 @@ import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder; * It implements {@link ErasureCoder}. */ @InterfaceAudience.Private -public class RSErasureEncoder extends AbstractErasureEncoder { +public class RSErasureEncoder extends ErasureEncoder { private RawErasureEncoder rawEncoder; - public RSErasureEncoder(int numDataUnits, int numParityUnits) { - super(numDataUnits, numParityUnits); - } - - public RSErasureEncoder(ECSchema schema) { - super(schema); + public RSErasureEncoder(ErasureCoderOptions options) { + super(options); } @Override @@ -57,10 +52,8 @@ public class RSErasureEncoder extends AbstractErasureEncoder { private RawErasureEncoder checkCreateRSRawEncoder() { if (rawEncoder == null) { // TODO: we should create the raw coder according to codec. - ErasureCoderOptions coderOptions = new ErasureCoderOptions( - getNumDataUnits(), getNumParityUnits()); rawEncoder = CodecUtil.createRawEncoder(getConf(), - ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, coderOptions); + ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, getOptions()); } return rawEncoder; } @@ -71,4 +64,9 @@ public class RSErasureEncoder extends AbstractErasureEncoder { rawEncoder.release(); } } + + @Override + public boolean preferDirectBuffer() { + return false; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XORErasureDecoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XORErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XORErasureDecoder.java index 47fb8da..1a0e5c0 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XORErasureDecoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XORErasureDecoder.java @@ -21,7 +21,6 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.io.erasurecode.CodecUtil; import org.apache.hadoop.io.erasurecode.ECBlock; import org.apache.hadoop.io.erasurecode.ECBlockGroup; -import org.apache.hadoop.io.erasurecode.ECSchema; import org.apache.hadoop.io.erasurecode.ErasureCodeConstants; import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder; @@ -32,23 +31,17 @@ import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder; * It implements {@link ErasureCoder}. */ @InterfaceAudience.Private -public class XORErasureDecoder extends AbstractErasureDecoder { +public class XORErasureDecoder extends ErasureDecoder { - public XORErasureDecoder(int numDataUnits, int numParityUnits) { - super(numDataUnits, numParityUnits); - } - - public XORErasureDecoder(ECSchema schema) { - super(schema); + public XORErasureDecoder(ErasureCoderOptions options) { + super(options); } @Override protected ErasureCodingStep prepareDecodingStep( final ECBlockGroup blockGroup) { - ErasureCoderOptions coderOptions = new ErasureCoderOptions( - getNumDataUnits(), getNumParityUnits()); RawErasureDecoder rawDecoder = CodecUtil.createRawDecoder(getConf(), - ErasureCodeConstants.XOR_CODEC_NAME, coderOptions); + ErasureCodeConstants.XOR_CODEC_NAME, getOptions()); ECBlock[] inputBlocks = getInputBlocks(blockGroup); http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XORErasureEncoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XORErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XORErasureEncoder.java index 1735179..89da617 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XORErasureEncoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XORErasureEncoder.java @@ -21,7 +21,6 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.io.erasurecode.CodecUtil; import org.apache.hadoop.io.erasurecode.ECBlock; import org.apache.hadoop.io.erasurecode.ECBlockGroup; -import org.apache.hadoop.io.erasurecode.ECSchema; import org.apache.hadoop.io.erasurecode.ErasureCodeConstants; import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder; @@ -32,28 +31,21 @@ import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder; * It implements {@link ErasureCoder}. */ @InterfaceAudience.Private -public class XORErasureEncoder extends AbstractErasureEncoder { +public class XORErasureEncoder extends ErasureEncoder { - public XORErasureEncoder(int numDataUnits, int numParityUnits) { - super(numDataUnits, numParityUnits); - } - - public XORErasureEncoder(ECSchema schema) { - super(schema); + public XORErasureEncoder(ErasureCoderOptions options) { + super(options); } @Override protected ErasureCodingStep prepareEncodingStep( final ECBlockGroup blockGroup) { - ErasureCoderOptions coderOptions = new ErasureCoderOptions( - getNumDataUnits(), getNumParityUnits()); RawErasureEncoder rawEncoder = CodecUtil.createRawEncoder(getConf(), - ErasureCodeConstants.XOR_CODEC_NAME, coderOptions); + ErasureCodeConstants.XOR_CODEC_NAME, getOptions()); ECBlock[] inputBlocks = getInputBlocks(blockGroup); return new ErasureEncodingStep(inputBlocks, getOutputBlocks(blockGroup), rawEncoder); } - } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/package-info.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/package-info.java new file mode 100644 index 0000000..8ea40c5 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/package-info.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Erasure coders framework. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +package org.apache.hadoop.io.erasurecode.coder; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/CoderUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/CoderUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/CoderUtil.java index ef34639..6072f13 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/CoderUtil.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/CoderUtil.java @@ -28,7 +28,7 @@ import java.util.Arrays; * Helpful utilities for implementing some raw erasure coders. */ @InterfaceAudience.Private -final class CoderUtil { +public final class CoderUtil { private CoderUtil() { // No called http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java index 0727157..72316ad 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java @@ -29,6 +29,7 @@ import org.apache.hadoop.fs.local.LocalConfigKeys; import org.apache.hadoop.ha.SshFenceByTcpPort; import org.apache.hadoop.ha.ZKFailoverController; import org.apache.hadoop.http.HttpServer2; +import org.apache.hadoop.io.erasurecode.CodecUtil; import org.apache.hadoop.io.nativeio.NativeIO; import org.apache.hadoop.security.CompositeGroupsMapping; import org.apache.hadoop.security.HttpCrossOriginFilterInitializer; @@ -49,6 +50,7 @@ import org.apache.hadoop.security.ssl.SSLFactory; * {@link org.apache.hadoop.security.LdapGroupsMapping} * {@link org.apache.hadoop.security.http.CrossOriginFilter} * {@link org.apache.hadoop.security.ssl.SSLFactory} + * {@link org.apache.hadoop.io.erasurecode.rawcoder.CoderUtil} * <p></p> * against core-site.xml for missing properties. Currently only * throws an error if the class is missing a property. @@ -71,7 +73,8 @@ public class TestCommonConfigurationFields extends TestConfigurationFieldsBase { LdapGroupsMapping.class, ZKFailoverController.class, SSLFactory.class, - CompositeGroupsMapping.class + CompositeGroupsMapping.class, + CodecUtil.class }; // Initialize used variables http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java index 81dc458..0db001a 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java @@ -18,7 +18,6 @@ package org.apache.hadoop.io.erasurecode; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.io.erasurecode.rawcoder.RSRawDecoder; import org.apache.hadoop.io.erasurecode.rawcoder.RSRawDecoderLegacy; import org.apache.hadoop.io.erasurecode.rawcoder.RSRawEncoder; @@ -73,7 +72,7 @@ public class TestCodecRawCoderMapping { String dummyFactName = "DummyNoneExistingFactory"; // set the dummy factory to rs-legacy and create a raw coder // with rs-default, which is OK as the raw coder key is not used - conf.set(CommonConfigurationKeys. + conf.set(CodecUtil. IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_KEY, dummyFactName); RawErasureEncoder encoder = CodecUtil.createRawEncoder(conf, ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, coderOptions); http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/codec/TestHHXORErasureCodec.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/codec/TestHHXORErasureCodec.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/codec/TestHHXORErasureCodec.java index c980b87..d5a3384 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/codec/TestHHXORErasureCodec.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/codec/TestHHXORErasureCodec.java @@ -17,7 +17,9 @@ */ package org.apache.hadoop.io.erasurecode.codec; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.erasurecode.ECSchema; +import org.apache.hadoop.io.erasurecode.ErasureCodecOptions; import org.apache.hadoop.io.erasurecode.coder.ErasureCoder; import org.junit.Test; @@ -25,10 +27,12 @@ import static org.junit.Assert.assertEquals; public class TestHHXORErasureCodec { private ECSchema schema = new ECSchema("hhxor", 10, 4); + private ErasureCodecOptions options = new ErasureCodecOptions(schema); @Test public void testGoodCodec() { - HHXORErasureCodec codec = new HHXORErasureCodec(schema); + HHXORErasureCodec codec + = new HHXORErasureCodec(new Configuration(), options); ErasureCoder encoder = codec.createEncoder(); assertEquals(10, encoder.getNumDataUnits()); assertEquals(4, encoder.getNumParityUnits()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java index 261be57..acbc136 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java @@ -20,6 +20,7 @@ package org.apache.hadoop.io.erasurecode.coder; import org.apache.hadoop.io.erasurecode.ECBlock; import org.apache.hadoop.io.erasurecode.ECBlockGroup; import org.apache.hadoop.io.erasurecode.ECChunk; +import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; import org.apache.hadoop.io.erasurecode.TestCoderBase; import java.lang.reflect.Constructor; @@ -158,10 +159,12 @@ public abstract class TestErasureCoderBase extends TestCoderBase { protected ErasureCoder createEncoder() { ErasureCoder encoder; try { + ErasureCoderOptions options = new ErasureCoderOptions( + numDataUnits, numParityUnits, allowChangeInputs, allowDump); Constructor<? extends ErasureCoder> constructor = (Constructor<? extends ErasureCoder>) - encoderClass.getConstructor(int.class, int.class); - encoder = constructor.newInstance(numDataUnits, numParityUnits); + encoderClass.getConstructor(ErasureCoderOptions.class); + encoder = constructor.newInstance(options); } catch (Exception e) { throw new RuntimeException("Failed to create encoder", e); } @@ -177,10 +180,12 @@ public abstract class TestErasureCoderBase extends TestCoderBase { protected ErasureCoder createDecoder() { ErasureCoder decoder; try { + ErasureCoderOptions options = new ErasureCoderOptions( + numDataUnits, numParityUnits, allowChangeInputs, allowDump); Constructor<? extends ErasureCoder> constructor = (Constructor<? extends ErasureCoder>) - decoderClass.getConstructor(int.class, int.class); - decoder = constructor.newInstance(numDataUnits, numParityUnits); + decoderClass.getConstructor(ErasureCoderOptions.class); + decoder = constructor.newInstance(options); } catch (Exception e) { throw new RuntimeException("Failed to create decoder", e); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java index 09d1ec2..1eca1e1 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java @@ -18,7 +18,7 @@ package org.apache.hadoop.io.erasurecode.coder; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.io.erasurecode.CodecUtil; import org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory; import org.junit.Before; import org.junit.Test; @@ -50,7 +50,7 @@ public class TestHHXORErasureCoder extends TestHHErasureCoderBase { * This tests if the configuration items work or not. */ Configuration conf = new Configuration(); - conf.set(CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY, + conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY, RSRawErasureCoderFactory.class.getCanonicalName()); prepare(conf, 10, 4, new int[]{0}, new int[0]); http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java index b0fe8f8..5f36c99 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java @@ -18,7 +18,7 @@ package org.apache.hadoop.io.erasurecode.coder; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.io.erasurecode.CodecUtil; import org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory; import org.junit.Before; import org.junit.Rule; @@ -57,7 +57,7 @@ public class TestRSErasureCoder extends TestErasureCoderBase { * This tests if the configuration items work or not. */ Configuration conf = new Configuration(); - conf.set(CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY, + conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY, RSRawErasureCoderFactory.class.getCanonicalName()); prepare(conf, 10, 4, new int[]{0}, new int[0]); http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java index 1e27745..6af9e7c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java @@ -20,7 +20,6 @@ package org.apache.hadoop.hdfs; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; @@ -83,7 +82,7 @@ public class TestDFSStripedInputStream { conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0); if (ErasureCodeNative.isNativeCodeLoaded()) { conf.set( - CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY, + CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY, NativeRSRawErasureCoderFactory.class.getCanonicalName()); } SimulatedFSDataset.setFactory(conf); http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java index 8d54f08..4f0a36c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java @@ -19,15 +19,13 @@ package org.apache.hadoop.hdfs; import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; +import org.apache.hadoop.io.erasurecode.CodecUtil; import org.apache.hadoop.io.erasurecode.ErasureCodeNative; import org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory; import org.apache.hadoop.test.GenericTestUtils; @@ -70,7 +68,7 @@ public class TestDFSStripedOutputStream { conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0); if (ErasureCodeNative.isNativeCodeLoaded()) { conf.set( - CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY, + CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY, NativeRSRawErasureCoderFactory.class.getCanonicalName()); } cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java index 11036a7..23809c3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java @@ -22,7 +22,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.impl.Log4JLogger; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.Path; @@ -37,6 +36,7 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.namenode.NameNode; +import org.apache.hadoop.io.erasurecode.CodecUtil; import org.apache.hadoop.io.erasurecode.ErasureCodeNative; import org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory; import org.apache.hadoop.security.token.Token; @@ -188,7 +188,7 @@ public class TestDFSStripedOutputStreamWithFailure { final int numDNs = NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS; if (ErasureCodeNative.isNativeCodeLoaded()) { conf.set( - CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY, + CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY, NativeRSRawErasureCoderFactory.class.getCanonicalName()); } cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/c023c748/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java index 59e9f87..10fc43e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java @@ -33,7 +33,6 @@ import java.util.Random; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; @@ -48,6 +47,7 @@ import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; import org.apache.hadoop.hdfs.server.protocol.BlockECReconstructionCommand.BlockECReconstructionInfo; import org.apache.hadoop.hdfs.util.StripedBlockUtil; +import org.apache.hadoop.io.erasurecode.CodecUtil; import org.apache.hadoop.io.erasurecode.ErasureCodeNative; import org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory; import org.apache.hadoop.test.GenericTestUtils; @@ -97,7 +97,7 @@ public class TestReconstructStripedFile { false); if (ErasureCodeNative.isNativeCodeLoaded()) { conf.set( - CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY, + CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY, NativeRSRawErasureCoderFactory.class.getCanonicalName()); } cluster = new MiniDFSCluster.Builder(conf).numDataNodes(dnNum).build(); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
