HADOOP-14479. Erasurecode testcase failures with native enabled. Contributed by Sammi Chen
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ea1da39b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ea1da39b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ea1da39b Branch: refs/heads/YARN-5734 Commit: ea1da39b1908de7739eaecded64074355f0fd5d0 Parents: 20ba86d Author: Kai Zheng <kai.zh...@intel.com> Authored: Thu Jun 29 11:26:36 2017 +0800 Committer: Kai Zheng <kai.zh...@intel.com> Committed: Thu Jun 29 11:26:36 2017 +0800 ---------------------------------------------------------------------- .../io/erasurecode/coder/ErasureEncodingStep.java | 2 +- .../apache/hadoop/io/erasurecode/jni_xor_decoder.c | 5 ++--- .../apache/hadoop/io/erasurecode/jni_xor_encoder.c | 2 +- .../io/erasurecode/TestCodecRawCoderMapping.java | 15 +++++++++++---- 4 files changed, 15 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea1da39b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncodingStep.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncodingStep.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncodingStep.java index f0b2be8..df4ed4b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncodingStep.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncodingStep.java @@ -62,6 +62,6 @@ public class ErasureEncodingStep implements ErasureCodingStep { @Override public void finish() { - rawEncoder.release(); + // do nothing } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea1da39b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_xor_decoder.c ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_xor_decoder.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_xor_decoder.c index ff0d3c5..13f66e2 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_xor_decoder.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_xor_decoder.c @@ -31,7 +31,7 @@ typedef struct _XOREncoder { IsalCoder isalCoder; unsigned char* inputs[MMAX]; - unsigned char* outputs[1]; + unsigned char* outputs[KMAX]; } XORDecoder; JNIEXPORT void JNICALL @@ -58,8 +58,7 @@ Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeXORRawDecoder_decodeImpl( numParityUnits = ((IsalCoder*)xorDecoder)->numParityUnits; chunkSize = (int)dataLen; - getInputs(env, inputs, inputOffsets, xorDecoder->inputs, - numDataUnits + numParityUnits); + getInputs(env, inputs, inputOffsets, xorDecoder->inputs, numDataUnits); getOutputs(env, outputs, outputOffsets, xorDecoder->outputs, numParityUnits); for (i = 0; i < numDataUnits + numParityUnits; i++) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea1da39b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_xor_encoder.c ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_xor_encoder.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_xor_encoder.c index 3cfa01c..6efb5fb 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_xor_encoder.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_xor_encoder.c @@ -31,7 +31,7 @@ typedef struct _XOREncoder { IsalCoder isalCoder; unsigned char* inputs[MMAX]; - unsigned char* outputs[1]; + unsigned char* outputs[KMAX]; } XOREncoder; JNIEXPORT void JNICALL http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea1da39b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java index 4a6fbd4..c7874dc 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java @@ -18,11 +18,13 @@ package org.apache.hadoop.io.erasurecode; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawEncoder; +import org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawDecoder; import org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory; -import org.apache.hadoop.io.erasurecode.rawcoder.RSRawDecoder; import org.apache.hadoop.io.erasurecode.rawcoder.RSLegacyRawDecoder; -import org.apache.hadoop.io.erasurecode.rawcoder.RSRawEncoder; import org.apache.hadoop.io.erasurecode.rawcoder.RSLegacyRawEncoder; +import org.apache.hadoop.io.erasurecode.rawcoder.RSRawDecoder; +import org.apache.hadoop.io.erasurecode.rawcoder.RSRawEncoder; import org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory; import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder; import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder; @@ -55,10 +57,15 @@ public class TestCodecRawCoderMapping { // should return default raw coder of rs codec RawErasureEncoder encoder = CodecUtil.createRawEncoder( conf, ErasureCodeConstants.RS_CODEC_NAME, coderOptions); - Assert.assertTrue(encoder instanceof RSRawEncoder); RawErasureDecoder decoder = CodecUtil.createRawDecoder( conf, ErasureCodeConstants.RS_CODEC_NAME, coderOptions); - Assert.assertTrue(decoder instanceof RSRawDecoder); + if (ErasureCodeNative.isNativeCodeLoaded()) { + Assert.assertTrue(encoder instanceof NativeRSRawEncoder); + Assert.assertTrue(decoder instanceof NativeRSRawDecoder); + } else { + Assert.assertTrue(encoder instanceof RSRawEncoder); + Assert.assertTrue(decoder instanceof RSRawDecoder); + } // should return default raw coder of rs-legacy codec encoder = CodecUtil.createRawEncoder(conf, --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
