Repository: hadoop Updated Branches: refs/heads/trunk e5766b1db -> 34ccaa836
HADOOP-11540. Raw Reed-Solomon coder using Intel ISA-L library. Contributed by Kai Zheng Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/34ccaa83 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/34ccaa83 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/34ccaa83 Branch: refs/heads/trunk Commit: 34ccaa8367f048ed9f56038efe7b3202c436b6e6 Parents: e5766b1 Author: Kai Zheng <[email protected]> Authored: Mon Aug 1 06:34:26 2016 +0800 Committer: Kai Zheng <[email protected]> Committed: Mon Aug 1 06:34:26 2016 +0800 ---------------------------------------------------------------------- .../dev-support/findbugsExcludeFile.xml | 13 ++ hadoop-common-project/hadoop-common/pom.xml | 4 + .../rawcoder/AbstractNativeRawDecoder.java | 84 +++++++++++++ .../rawcoder/AbstractNativeRawEncoder.java | 81 +++++++++++++ .../rawcoder/ByteArrayDecodingState.java | 23 ++++ .../rawcoder/ByteArrayEncodingState.java | 23 ++++ .../rawcoder/ByteBufferDecodingState.java | 14 ++- .../rawcoder/ByteBufferEncodingState.java | 10 ++ .../io/erasurecode/rawcoder/CoderUtil.java | 12 -- .../rawcoder/NativeRSRawDecoder.java | 61 ++++++++++ .../rawcoder/NativeRSRawEncoder.java | 60 +++++++++ .../NativeRSRawErasureCoderFactory.java | 39 ++++++ .../rawcoder/RawErasureCoderFactory.java | 4 +- .../erasurecode/rawcoder/util/GaloisField.java | 4 +- .../apache/hadoop/io/erasurecode/jni_common.c | 27 +++-- ...he_hadoop_io_erasurecode_ErasureCodeNative.h | 29 ----- ...io_erasurecode_rawcoder_NativeRSRawDecoder.h | 37 ------ ...io_erasurecode_rawcoder_NativeRSRawEncoder.h | 37 ------ .../rawcoder/TestNativeRSRawCoder.java | 121 +++++++++++++++++++ .../rawcoder/TestRSRawCoderInteroperable1.java | 38 ++++++ .../rawcoder/TestRSRawCoderInteroperable2.java | 38 ++++++ 21 files changed, 631 insertions(+), 128 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/34ccaa83/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml b/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml index a644aa5..b650eae 100644 --- a/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml +++ b/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml @@ -47,6 +47,19 @@ <Field name="out" /> <Bug pattern="IS2_INCONSISTENT_SYNC" /> </Match> + <!-- + The nativeCoder field is get/set and used by native codes. + --> + <Match> + <Class name="org.apache.hadoop.io.erasurecode.rawcoder.AbstractNativeRawEncoder" /> + <Field name="nativeCoder" /> + <Bug pattern="UUF_UNUSED_FIELD" /> + </Match> + <Match> + <Class name="org.apache.hadoop.io.erasurecode.rawcoder.AbstractNativeRawDecoder" /> + <Field name="nativeCoder" /> + <Bug pattern="UUF_UNUSED_FIELD" /> + </Match> <!-- Further SaslException should be ignored during cleanup and original exception should be re-thrown. http://git-wip-us.apache.org/repos/asf/hadoop/blob/34ccaa83/hadoop-common-project/hadoop-common/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index 398bb84..ef2fdf5 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -631,6 +631,8 @@ <javahClassName>org.apache.hadoop.io.compress.lz4.Lz4Compressor</javahClassName> <javahClassName>org.apache.hadoop.io.compress.lz4.Lz4Decompressor</javahClassName> <javahClassName>org.apache.hadoop.io.erasurecode.ErasureCodeNative</javahClassName> + <javahClassName>org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawEncoder</javahClassName> + <javahClassName>org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawDecoder</javahClassName> <javahClassName>org.apache.hadoop.crypto.OpensslCipher</javahClassName> <javahClassName>org.apache.hadoop.crypto.random.OpensslSecureRandom</javahClassName> <javahClassName>org.apache.hadoop.util.NativeCrc32</javahClassName> @@ -769,6 +771,8 @@ <javahClassName>org.apache.hadoop.io.compress.lz4.Lz4Compressor</javahClassName> <javahClassName>org.apache.hadoop.io.compress.lz4.Lz4Decompressor</javahClassName> <javahClassName>org.apache.hadoop.io.erasurecode.ErasureCodeNative</javahClassName> + <javahClassName>org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawEncoder</javahClassName> + <javahClassName>org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawDecoder</javahClassName> <javahClassName>org.apache.hadoop.crypto.OpensslCipher</javahClassName> <javahClassName>org.apache.hadoop.crypto.random.OpensslSecureRandom</javahClassName> <javahClassName>org.apache.hadoop.util.NativeCrc32</javahClassName> http://git-wip-us.apache.org/repos/asf/hadoop/blob/34ccaa83/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractNativeRawDecoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractNativeRawDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractNativeRawDecoder.java new file mode 100644 index 0000000..55edb64 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractNativeRawDecoder.java @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.io.erasurecode.rawcoder; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.ByteBuffer; + +/** + * Abstract native raw decoder for all native coders to extend with. + */ [email protected] +abstract class AbstractNativeRawDecoder extends RawErasureDecoder { + public static Logger LOG = + LoggerFactory.getLogger(AbstractNativeRawDecoder.class); + + public AbstractNativeRawDecoder(ErasureCoderOptions coderOptions) { + super(coderOptions); + } + + @Override + protected void doDecode(ByteBufferDecodingState decodingState) { + int[] inputOffsets = new int[decodingState.inputs.length]; + int[] outputOffsets = new int[decodingState.outputs.length]; + + ByteBuffer buffer; + for (int i = 0; i < decodingState.inputs.length; ++i) { + buffer = decodingState.inputs[i]; + if (buffer != null) { + inputOffsets[i] = buffer.position(); + } + } + + for (int i = 0; i < decodingState.outputs.length; ++i) { + buffer = decodingState.outputs[i]; + outputOffsets[i] = buffer.position(); + } + + performDecodeImpl(decodingState.inputs, inputOffsets, + decodingState.decodeLength, decodingState.erasedIndexes, + decodingState.outputs, outputOffsets); + } + + protected abstract void performDecodeImpl(ByteBuffer[] inputs, + int[] inputOffsets, int dataLen, + int[] erased, ByteBuffer[] outputs, + int[] outputOffsets); + + @Override + protected void doDecode(ByteArrayDecodingState decodingState) { + LOG.warn("convertToByteBufferState is invoked, " + + "not efficiently. Please use direct ByteBuffer inputs/outputs"); + + ByteBufferDecodingState bbdState = decodingState.convertToByteBufferState(); + doDecode(bbdState); + + for (int i = 0; i < decodingState.outputs.length; i++) { + bbdState.outputs[i].get(decodingState.outputs[i], + decodingState.outputOffsets[i], decodingState.decodeLength); + } + } + + // To link with the underlying data structure in the native layer. + // No get/set as only used by native codes. + private long nativeCoder; +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/34ccaa83/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractNativeRawEncoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractNativeRawEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractNativeRawEncoder.java new file mode 100644 index 0000000..b2c7016 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractNativeRawEncoder.java @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.io.erasurecode.rawcoder; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.ByteBuffer; + +/** + * Abstract native raw encoder for all native coders to extend with. + */ [email protected] +abstract class AbstractNativeRawEncoder extends RawErasureEncoder { + public static Logger LOG = + LoggerFactory.getLogger(AbstractNativeRawEncoder.class); + + public AbstractNativeRawEncoder(ErasureCoderOptions coderOptions) { + super(coderOptions); + } + + @Override + protected void doEncode(ByteBufferEncodingState encodingState) { + int[] inputOffsets = new int[encodingState.inputs.length]; + int[] outputOffsets = new int[encodingState.outputs.length]; + int dataLen = encodingState.inputs[0].remaining(); + + ByteBuffer buffer; + for (int i = 0; i < encodingState.inputs.length; ++i) { + buffer = encodingState.inputs[i]; + inputOffsets[i] = buffer.position(); + } + + for (int i = 0; i < encodingState.outputs.length; ++i) { + buffer = encodingState.outputs[i]; + outputOffsets[i] = buffer.position(); + } + + performEncodeImpl(encodingState.inputs, inputOffsets, dataLen, + encodingState.outputs, outputOffsets); + } + + protected abstract void performEncodeImpl( + ByteBuffer[] inputs, int[] inputOffsets, + int dataLen, ByteBuffer[] outputs, int[] outputOffsets); + + @Override + protected void doEncode(ByteArrayEncodingState encodingState) { + LOG.warn("convertToByteBufferState is invoked, " + + "not efficiently. Please use direct ByteBuffer inputs/outputs"); + + ByteBufferEncodingState bbeState = encodingState.convertToByteBufferState(); + doEncode(bbeState); + + for (int i = 0; i < encodingState.outputs.length; i++) { + bbeState.outputs[i].get(encodingState.outputs[i], + encodingState.outputOffsets[i], encodingState.encodeLength); + } + } + + // To link with the underlying data structure in the native layer. + // No get/set as only used by native codes. + private long nativeCoder; +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/34ccaa83/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/ByteArrayDecodingState.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/ByteArrayDecodingState.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/ByteArrayDecodingState.java index 69c084d..7b3828b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/ByteArrayDecodingState.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/ByteArrayDecodingState.java @@ -20,6 +20,8 @@ package org.apache.hadoop.io.erasurecode.rawcoder; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; +import java.nio.ByteBuffer; + /** * A utility class that maintains decoding state during a decode call using * byte array inputs. @@ -66,6 +68,27 @@ class ByteArrayDecodingState extends DecodingState { } /** + * Convert to a ByteBufferDecodingState when it's backed by on-heap arrays. + */ + ByteBufferDecodingState convertToByteBufferState() { + ByteBuffer[] newInputs = new ByteBuffer[inputs.length]; + ByteBuffer[] newOutputs = new ByteBuffer[outputs.length]; + + for (int i = 0; i < inputs.length; i++) { + newInputs[i] = CoderUtil.cloneAsDirectByteBuffer(inputs[i], + inputOffsets[i], decodeLength); + } + + for (int i = 0; i < outputs.length; i++) { + newOutputs[i] = ByteBuffer.allocateDirect(decodeLength); + } + + ByteBufferDecodingState bbdState = new ByteBufferDecodingState(decoder, + decodeLength, erasedIndexes, newInputs, newOutputs); + return bbdState; + } + + /** * Check and ensure the buffers are of the desired length. * @param buffers the buffers to check */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/34ccaa83/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/ByteArrayEncodingState.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/ByteArrayEncodingState.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/ByteArrayEncodingState.java index 9d861d4..518356a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/ByteArrayEncodingState.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/ByteArrayEncodingState.java @@ -20,6 +20,8 @@ package org.apache.hadoop.io.erasurecode.rawcoder; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; +import java.nio.ByteBuffer; + /** * A utility class that maintains encoding state during an encode call using * byte array inputs. @@ -62,6 +64,27 @@ class ByteArrayEncodingState extends EncodingState { } /** + * Convert to a ByteBufferEncodingState when it's backed by on-heap arrays. + */ + ByteBufferEncodingState convertToByteBufferState() { + ByteBuffer[] newInputs = new ByteBuffer[inputs.length]; + ByteBuffer[] newOutputs = new ByteBuffer[outputs.length]; + + for (int i = 0; i < inputs.length; i++) { + newInputs[i] = CoderUtil.cloneAsDirectByteBuffer(inputs[i], + inputOffsets[i], encodeLength); + } + + for (int i = 0; i < outputs.length; i++) { + newOutputs[i] = ByteBuffer.allocateDirect(encodeLength); + } + + ByteBufferEncodingState bbeState = new ByteBufferEncodingState(encoder, + encodeLength, newInputs, newOutputs); + return bbeState; + } + + /** * Check and ensure the buffers are of the desired length. * @param buffers the buffers to check */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/34ccaa83/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/ByteBufferDecodingState.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/ByteBufferDecodingState.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/ByteBufferDecodingState.java index 5c5b0f6..4970c06 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/ByteBufferDecodingState.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/ByteBufferDecodingState.java @@ -48,8 +48,20 @@ class ByteBufferDecodingState extends DecodingState { checkOutputBuffers(outputs); } + ByteBufferDecodingState(RawErasureDecoder decoder, + int decodeLength, + int[] erasedIndexes, + ByteBuffer[] inputs, + ByteBuffer[] outputs) { + this.decoder = decoder; + this.decodeLength = decodeLength; + this.erasedIndexes = erasedIndexes; + this.inputs = inputs; + this.outputs = outputs; + } + /** - * Convert to a ByteArrayEncodingState when it's backed by on-heap arrays. + * Convert to a ByteArrayDecodingState when it's backed by on-heap arrays. */ ByteArrayDecodingState convertToByteArrayState() { int[] inputOffsets = new int[inputs.length]; http://git-wip-us.apache.org/repos/asf/hadoop/blob/34ccaa83/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/ByteBufferEncodingState.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/ByteBufferEncodingState.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/ByteBufferEncodingState.java index 7a10ac2..7d3124d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/ByteBufferEncodingState.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/ByteBufferEncodingState.java @@ -46,6 +46,16 @@ class ByteBufferEncodingState extends EncodingState { checkBuffers(outputs); } + ByteBufferEncodingState(RawErasureEncoder encoder, + int encodeLength, + ByteBuffer[] inputs, + ByteBuffer[] outputs) { + this.encoder = encoder; + this.encodeLength = encodeLength; + this.inputs = inputs; + this.outputs = outputs; + } + /** * Convert to a ByteArrayEncodingState when it's backed by on-heap arrays. */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/34ccaa83/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/CoderUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/CoderUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/CoderUtil.java index aceb3c6..b22d44f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/CoderUtil.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/CoderUtil.java @@ -83,8 +83,6 @@ final class CoderUtil { /** * Initialize the output buffers with ZERO bytes. - * @param buffers - * @param dataLen */ static void resetOutputBuffers(ByteBuffer[] buffers, int dataLen) { for (ByteBuffer buffer : buffers) { @@ -94,8 +92,6 @@ final class CoderUtil { /** * Initialize the output buffers with ZERO bytes. - * @param buffers - * @param dataLen */ static void resetOutputBuffers(byte[][] buffers, int[] offsets, int dataLen) { @@ -127,10 +123,6 @@ final class CoderUtil { /** * Clone an input bytes array as direct ByteBuffer. - * @param input - * @param len - * @param offset - * @return direct ByteBuffer */ static ByteBuffer cloneAsDirectByteBuffer(byte[] input, int offset, int len) { if (input == null) { // an input can be null, if erased or not to read @@ -166,10 +158,6 @@ final class CoderUtil { * @return the first valid input */ static <T> T findFirstValidInput(T[] inputs) { - if (inputs.length > 0 && inputs[0] != null) { - return inputs[0]; - } - for (T input : inputs) { if (input != null) { return input; http://git-wip-us.apache.org/repos/asf/hadoop/blob/34ccaa83/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawDecoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawDecoder.java new file mode 100644 index 0000000..6b497cc --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawDecoder.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.io.erasurecode.rawcoder; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.io.erasurecode.ErasureCodeNative; +import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; + +import java.nio.ByteBuffer; + +/** + * A Reed-Solomon raw decoder using Intel ISA-L library. + */ [email protected] +public class NativeRSRawDecoder extends AbstractNativeRawDecoder { + + static { + ErasureCodeNative.checkNativeCodeLoaded(); + } + + public NativeRSRawDecoder(ErasureCoderOptions coderOptions) { + super(coderOptions); + initImpl(coderOptions.getNumDataUnits(), coderOptions.getNumParityUnits()); + } + + @Override + protected void performDecodeImpl(ByteBuffer[] inputs, int[] inputOffsets, + int dataLen, int[] erased, + ByteBuffer[] outputs, int[] outputOffsets) { + decodeImpl(inputs, inputOffsets, dataLen, erased, outputs, outputOffsets); + } + + @Override + public void release() { + destroyImpl(); + } + + private native void initImpl(int numDataUnits, int numParityUnits); + + private native void decodeImpl( + ByteBuffer[] inputs, int[] inputOffsets, int dataLen, int[] erased, + ByteBuffer[] outputs, int[] outputOffsets); + + private native void destroyImpl(); + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/34ccaa83/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawEncoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawEncoder.java new file mode 100644 index 0000000..297ed33 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawEncoder.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.io.erasurecode.rawcoder; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.io.erasurecode.ErasureCodeNative; +import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; + +import java.nio.ByteBuffer; + +/** + * A Reed-Solomon raw encoder using Intel ISA-L library. + */ [email protected] +public class NativeRSRawEncoder extends AbstractNativeRawEncoder { + + static { + ErasureCodeNative.checkNativeCodeLoaded(); + } + + public NativeRSRawEncoder(ErasureCoderOptions coderOptions) { + super(coderOptions); + initImpl(coderOptions.getNumDataUnits(), coderOptions.getNumParityUnits()); + } + + @Override + protected void performEncodeImpl( + ByteBuffer[] inputs, int[] inputOffsets, int dataLen, + ByteBuffer[] outputs, int[] outputOffsets) { + encodeImpl(inputs, inputOffsets, dataLen, outputs, outputOffsets); + } + + @Override + public void release() { + destroyImpl(); + } + + private native void initImpl(int numDataUnits, int numParityUnits); + + private native void encodeImpl(ByteBuffer[] inputs, int[] inputOffsets, + int dataLen, ByteBuffer[] outputs, + int[] outputOffsets); + + private native void destroyImpl(); +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/34ccaa83/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawErasureCoderFactory.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawErasureCoderFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawErasureCoderFactory.java new file mode 100644 index 0000000..38997dd --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawErasureCoderFactory.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.io.erasurecode.rawcoder; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; + +/** + * A raw coder factory for raw Reed-Solomon coder in native using Intel ISA-L. + */ + [email protected] +public class NativeRSRawErasureCoderFactory implements RawErasureCoderFactory { + + @Override + public RawErasureEncoder createEncoder(ErasureCoderOptions coderOptions) { + return new NativeRSRawEncoder(coderOptions); + } + + @Override + public RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions) { + return new NativeRSRawDecoder(coderOptions); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/34ccaa83/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderFactory.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderFactory.java index 6d94f00..05f228b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderFactory.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderFactory.java @@ -30,14 +30,14 @@ public interface RawErasureCoderFactory { /** * Create raw erasure encoder. - * @param conf the configuration used to create the encoder + * @param coderOptions the options used to create the encoder * @return raw erasure encoder */ RawErasureEncoder createEncoder(ErasureCoderOptions coderOptions); /** * Create raw erasure decoder. - * @param conf the configuration used to create the encoder + * @param coderOptions the options used to create the encoder * @return raw erasure decoder */ RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions); http://git-wip-us.apache.org/repos/asf/hadoop/blob/34ccaa83/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/util/GaloisField.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/util/GaloisField.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/util/GaloisField.java index 96a6408..fdb47be 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/util/GaloisField.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/util/GaloisField.java @@ -17,12 +17,12 @@ */ package org.apache.hadoop.io.erasurecode.rawcoder.util; -import org.apache.hadoop.classification.InterfaceAudience; - import java.nio.ByteBuffer; import java.util.HashMap; import java.util.Map; +import org.apache.hadoop.classification.InterfaceAudience; + /** * Implementation of Galois field arithmetic with 2^p elements. The input must * be unsigned integers. It's ported from HDFS-RAID, slightly adapted. http://git-wip-us.apache.org/repos/asf/hadoop/blob/34ccaa83/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_common.c ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_common.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_common.c index 8126e9a..17e05db 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_common.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_common.c @@ -34,20 +34,31 @@ void loadLib(JNIEnv *env) { void setCoder(JNIEnv* env, jobject thiz, IsalCoder* pCoder) { jclass clazz = (*env)->GetObjectClass(env, thiz); - jfieldID __coderState = (*env)->GetFieldID(env, clazz, "__native_coder", "J"); - (*env)->SetLongField(env, thiz, __coderState, (jlong) pCoder); + jfieldID fid = (*env)->GetFieldID(env, clazz, "nativeCoder", "J"); + if (fid == NULL) { + THROW(env, "java/lang/UnsatisfiedLinkError", + "Field nativeCoder not found"); + } + (*env)->SetLongField(env, thiz, fid, (jlong) pCoder); } IsalCoder* getCoder(JNIEnv* env, jobject thiz) { jclass clazz = (*env)->GetObjectClass(env, thiz); - jfieldID __verbose = (*env)->GetFieldID(env, clazz, "__native_verbose", "J"); - int verbose = (int)(*env)->GetIntField(env, thiz, __verbose); + jmethodID mid = (*env)->GetMethodID(env, clazz, "allowVerboseDump", "()Z"); + if (mid == NULL) { + THROW(env, "java/lang/UnsatisfiedLinkError", + "Method allowVerboseDump not found"); + } + jboolean verbose = (*env)->CallBooleanMethod(env, thiz, mid); - jfieldID __coderState = (*env)->GetFieldID(env, clazz, "__native_coder", "J"); - IsalCoder* pCoder = (IsalCoder*)(*env)->GetLongField(env, - thiz, __coderState); - pCoder->verbose = verbose; + jfieldID fid = (*env)->GetFieldID(env, clazz, "nativeCoder", "J"); + if (fid == NULL) { + THROW(env, "java/lang/UnsatisfiedLinkError", + "Field nativeCoder not found"); + } + IsalCoder* pCoder = (IsalCoder*)(*env)->GetLongField(env, thiz, fid); + pCoder->verbose = (verbose == JNI_TRUE) ? 1 : 0; return pCoder; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/34ccaa83/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/org_apache_hadoop_io_erasurecode_ErasureCodeNative.h ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/org_apache_hadoop_io_erasurecode_ErasureCodeNative.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/org_apache_hadoop_io_erasurecode_ErasureCodeNative.h deleted file mode 100644 index d8ff3a0..0000000 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/org_apache_hadoop_io_erasurecode_ErasureCodeNative.h +++ /dev/null @@ -1,29 +0,0 @@ -/* DO NOT EDIT THIS FILE - it is machine generated */ -#include <jni.h> -/* Header for class org_apache_hadoop_io_erasurecode_ErasureCodeNative */ - -#ifndef _Included_org_apache_hadoop_io_erasurecode_ErasureCodeNative -#define _Included_org_apache_hadoop_io_erasurecode_ErasureCodeNative -#ifdef __cplusplus -extern "C" { -#endif -/* - * Class: org_apache_hadoop_io_erasurecode_ErasureCodeNative - * Method: loadLibrary - * Signature: ()V - */ -JNIEXPORT void JNICALL Java_org_apache_hadoop_io_erasurecode_ErasureCodeNative_loadLibrary - (JNIEnv *, jclass); - -/* - * Class: org_apache_hadoop_io_erasurecode_ErasureCodeNative - * Method: getLibraryName - * Signature: ()Ljava/lang/String; - */ -JNIEXPORT jstring JNICALL Java_org_apache_hadoop_io_erasurecode_ErasureCodeNative_getLibraryName - (JNIEnv *, jclass); - -#ifdef __cplusplus -} -#endif -#endif http://git-wip-us.apache.org/repos/asf/hadoop/blob/34ccaa83/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder.h ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder.h deleted file mode 100644 index 40da4e1..0000000 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder.h +++ /dev/null @@ -1,37 +0,0 @@ -/* DO NOT EDIT THIS FILE - it is machine generated */ -#include <jni.h> -/* Header for class org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder */ - -#ifndef _Included_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder -#define _Included_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder -#ifdef __cplusplus -extern "C" { -#endif -/* - * Class: org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder - * Method: initImpl - * Signature: (II[I)V - */ -JNIEXPORT void JNICALL Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder_initImpl - (JNIEnv *, jobject, jint, jint); - -/* - * Class: org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder - * Method: decodeImpl - * Signature: ([Ljava/nio/ByteBuffer;[II[I[Ljava/nio/ByteBuffer;[I)V - */ -JNIEXPORT void JNICALL Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder_decodeImpl - (JNIEnv *, jobject, jobjectArray, jintArray, jint, jintArray, jobjectArray, jintArray); - -/* - * Class: org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder - * Method: destroyImpl - * Signature: ()V - */ -JNIEXPORT void JNICALL Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder_destroyImpl - (JNIEnv *, jobject); - -#ifdef __cplusplus -} -#endif -#endif http://git-wip-us.apache.org/repos/asf/hadoop/blob/34ccaa83/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder.h ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder.h deleted file mode 100644 index db094cf..0000000 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder.h +++ /dev/null @@ -1,37 +0,0 @@ -/* DO NOT EDIT THIS FILE - it is machine generated */ -#include <jni.h> -/* Header for class org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder */ - -#ifndef _Included_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder -#define _Included_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder -#ifdef __cplusplus -extern "C" { -#endif -/* - * Class: org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder - * Method: initImpl - * Signature: (II[I)V - */ -JNIEXPORT void JNICALL Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder_initImpl - (JNIEnv *, jobject, jint, jint); - -/* - * Class: org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder - * Method: encodeImpl - * Signature: ([Ljava/nio/ByteBuffer;[II[Ljava/nio/ByteBuffer;[I)V - */ -JNIEXPORT void JNICALL Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder_encodeImpl - (JNIEnv *, jobject, jobjectArray, jintArray, jint, jobjectArray, jintArray); - -/* - * Class: org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder - * Method: destroyImpl - * Signature: ()V - */ -JNIEXPORT void JNICALL Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder_destroyImpl - (JNIEnv *, jobject); - -#ifdef __cplusplus -} -#endif -#endif http://git-wip-us.apache.org/repos/asf/hadoop/blob/34ccaa83/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestNativeRSRawCoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestNativeRSRawCoder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestNativeRSRawCoder.java new file mode 100644 index 0000000..edbb9df --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestNativeRSRawCoder.java @@ -0,0 +1,121 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.io.erasurecode.rawcoder; + +import org.apache.hadoop.io.erasurecode.ErasureCodeNative; +import org.junit.Assume; +import org.junit.Before; +import org.junit.Test; + +/** + * Test native raw Reed-solomon encoding and decoding. + */ +public class TestNativeRSRawCoder extends TestRSRawCoderBase { + + @Before + public void setup() { + Assume.assumeTrue(ErasureCodeNative.isNativeCodeLoaded()); + this.encoderClass = NativeRSRawEncoder.class; + this.decoderClass = NativeRSRawDecoder.class; + setAllowDump(true); + } + + @Test + public void testCoding_6x3_erasing_all_d() { + prepare(null, 6, 3, new int[]{0, 1, 2}, new int[0], true); + testCodingDoMixAndTwice(); + } + + @Test + public void testCoding_6x3_erasing_d0_d2() { + prepare(null, 6, 3, new int[] {0, 2}, new int[]{}); + testCodingDoMixAndTwice(); + } + + @Test + public void testCoding_6x3_erasing_d0() { + prepare(null, 6, 3, new int[]{0}, new int[0]); + testCodingDoMixAndTwice(); + } + + @Test + public void testCoding_6x3_erasing_d2() { + prepare(null, 6, 3, new int[]{2}, new int[]{}); + testCodingDoMixAndTwice(); + } + + @Test + public void testCoding_6x3_erasing_d0_p0() { + prepare(null, 6, 3, new int[]{0}, new int[]{0}); + testCodingDoMixAndTwice(); + } + + @Test + public void testCoding_6x3_erasing_all_p() { + prepare(null, 6, 3, new int[0], new int[]{0, 1, 2}); + testCodingDoMixAndTwice(); + } + + @Test + public void testCoding_6x3_erasing_p0() { + prepare(null, 6, 3, new int[0], new int[]{0}); + testCodingDoMixAndTwice(); + } + + @Test + public void testCoding_6x3_erasing_p2() { + prepare(null, 6, 3, new int[0], new int[]{2}); + testCodingDoMixAndTwice(); + } + + @Test + public void testCoding_6x3_erasure_p0_p2() { + prepare(null, 6, 3, new int[0], new int[]{0, 2}); + testCodingDoMixAndTwice(); + } + + @Test + public void testCoding_6x3_erasing_d0_p0_p1() { + prepare(null, 6, 3, new int[]{0}, new int[]{0, 1}); + testCodingDoMixAndTwice(); + } + + @Test + public void testCoding_6x3_erasing_d0_d2_p2() { + prepare(null, 6, 3, new int[]{0, 2}, new int[]{2}); + testCodingDoMixAndTwice(); + } + + @Test + public void testCodingNegative_6x3_erasing_d2_d4() { + prepare(null, 6, 3, new int[]{2, 4}, new int[0]); + testCodingDoMixAndTwice(); + } + + @Test + public void testCodingNegative_6x3_erasing_too_many() { + prepare(null, 6, 3, new int[]{2, 4}, new int[]{0, 1}); + testCodingWithErasingTooMany(); + } + + @Test + public void testCoding_10x4_erasing_d0_p0() { + prepare(null, 10, 4, new int[] {0}, new int[] {0}); + testCodingDoMixAndTwice(); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/34ccaa83/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoderInteroperable1.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoderInteroperable1.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoderInteroperable1.java new file mode 100644 index 0000000..9ce041c --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoderInteroperable1.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.io.erasurecode.rawcoder; + +import org.apache.hadoop.io.erasurecode.ErasureCodeNative; +import org.junit.Assume; +import org.junit.Before; + +/** + * Test raw Reed-solomon coder implemented in Java. + */ +public class TestRSRawCoderInteroperable1 extends TestRSRawCoderBase { + + @Before + public void setup() { + Assume.assumeTrue(ErasureCodeNative.isNativeCodeLoaded()); + + this.encoderClass = RSRawEncoder.class; + this.decoderClass = NativeRSRawDecoder.class; + setAllowDump(true); + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/34ccaa83/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoderInteroperable2.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoderInteroperable2.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoderInteroperable2.java new file mode 100644 index 0000000..15531f3 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoderInteroperable2.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.io.erasurecode.rawcoder; + +import org.apache.hadoop.io.erasurecode.ErasureCodeNative; +import org.junit.Assume; +import org.junit.Before; + +/** + * Test raw Reed-solomon coder implemented in Java. + */ +public class TestRSRawCoderInteroperable2 extends TestRSRawCoderBase { + + @Before + public void setup() { + Assume.assumeTrue(ErasureCodeNative.isNativeCodeLoaded()); + + this.encoderClass = NativeRSRawEncoder.class; + this.decoderClass = RSRawDecoder.class; + setAllowDump(true); + } + +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
