HADOOP-12544. Erasure Coding: create dummy raw coder to isolate performance issues in testing. Contributed by Rui Li.
Change-Id: I9856456b59ed881c5ba2acce51e4d9bd01dc6f48 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3e1745d8 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3e1745d8 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3e1745d8 Branch: refs/heads/HDFS-7240 Commit: 3e1745d8e8e5a44f7c8eab9a8234edaf389828c7 Parents: 194251c Author: Zhe Zhang <z...@apache.org> Authored: Tue Nov 3 22:26:27 2015 -0800 Committer: Zhe Zhang <z...@apache.org> Committed: Tue Nov 3 22:26:27 2015 -0800 ---------------------------------------------------------------------- hadoop-common-project/hadoop-common/CHANGES.txt | 3 + .../erasurecode/rawcoder/DummyRawDecoder.java | 47 +++++++++++ .../erasurecode/rawcoder/DummyRawEncoder.java | 46 +++++++++++ .../rawcoder/DummyRawErasureCoderFactory.java | 36 +++++++++ .../hadoop/io/erasurecode/TestCoderBase.java | 4 + .../erasurecode/rawcoder/TestDummyRawCoder.java | 83 ++++++++++++++++++++ .../erasurecode/rawcoder/TestRawCoderBase.java | 10 +-- 7 files changed, 224 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e1745d8/hadoop-common-project/hadoop-common/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 0d1bce2..453efe6 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -613,6 +613,9 @@ Trunk (Unreleased) HADOOP-12047. Indicate preference not to affect input buffers during coding in erasure coder. (Kai Zheng via waltersu4549) + HADOOP-12544. Erasure Coding: create dummy raw coder to isolate performance + issues in testing. (Rui Li via zhz) + Release 2.8.0 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e1745d8/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/DummyRawDecoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/DummyRawDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/DummyRawDecoder.java new file mode 100644 index 0000000..25dfa57 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/DummyRawDecoder.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.io.erasurecode.rawcoder; + +import org.apache.hadoop.classification.InterfaceAudience; + +import java.nio.ByteBuffer; + +/** + * A dummy raw decoder that does no real computation. + * Instead, it just returns zero bytes. + * This decoder can be used to isolate the performance issue to HDFS side logic + * instead of codec, and is intended for test only. + */ +@InterfaceAudience.Private +public class DummyRawDecoder extends AbstractRawErasureDecoder { + public DummyRawDecoder(int numDataUnits, int numParityUnits) { + super(numDataUnits, numParityUnits); + } + + @Override + protected void doDecode(ByteBuffer[] inputs, int[] erasedIndexes, + ByteBuffer[] outputs) { + // Nothing to do. Output buffers have already been reset + } + + @Override + protected void doDecode(byte[][] inputs, int[] inputOffsets, int dataLen, + int[] erasedIndexes, byte[][] outputs, int[] outputOffsets) { + // Nothing to do. Output buffers have already been reset + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e1745d8/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/DummyRawEncoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/DummyRawEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/DummyRawEncoder.java new file mode 100644 index 0000000..33e026d --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/DummyRawEncoder.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.io.erasurecode.rawcoder; + +import org.apache.hadoop.classification.InterfaceAudience; + +import java.nio.ByteBuffer; + +/** + * A dummy raw encoder that does no real computation. + * Instead, it just returns zero bytes. + * This encoder can be used to isolate the performance issue to HDFS side logic + * instead of codec, and is intended for test only. + */ +@InterfaceAudience.Private +public class DummyRawEncoder extends AbstractRawErasureEncoder { + public DummyRawEncoder(int numDataUnits, int numParityUnits) { + super(numDataUnits, numParityUnits); + } + + @Override + protected void doEncode(ByteBuffer[] inputs, ByteBuffer[] outputs) { + // Nothing to do. Output buffers have already been reset + } + + @Override + protected void doEncode(byte[][] inputs, int[] inputOffsets, int dataLen, + byte[][] outputs, int[] outputOffsets) { + // Nothing to do. Output buffers have already been reset + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e1745d8/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/DummyRawErasureCoderFactory.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/DummyRawErasureCoderFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/DummyRawErasureCoderFactory.java new file mode 100644 index 0000000..73457c2 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/DummyRawErasureCoderFactory.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.io.erasurecode.rawcoder; + +import org.apache.hadoop.classification.InterfaceAudience; + +/** + * A raw erasure coder factory for dummy raw coders. + */ +@InterfaceAudience.Private +public class DummyRawErasureCoderFactory implements RawErasureCoderFactory { + @Override + public RawErasureEncoder createEncoder(int numDataUnits, int numParityUnits) { + return new DummyRawEncoder(numDataUnits, numParityUnits); + } + + @Override + public RawErasureDecoder createDecoder(int numDataUnits, int numParityUnits) { + return new DummyRawDecoder(numDataUnits, numParityUnits); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e1745d8/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java index 62a0b9c..5721336 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java @@ -75,6 +75,10 @@ public abstract class TestCoderBase { this.zeroChunkBytes = new byte[chunkSize]; // With ZERO by default } + protected byte[] getZeroChunkBytes() { + return zeroChunkBytes; + } + protected void prepareBufferAllocator(boolean usingSlicedBuffer) { if (usingSlicedBuffer) { int roughEstimationSpace = http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e1745d8/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestDummyRawCoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestDummyRawCoder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestDummyRawCoder.java new file mode 100644 index 0000000..63a2ac8 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestDummyRawCoder.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.io.erasurecode.rawcoder; + +import org.apache.hadoop.io.erasurecode.ECChunk; +import org.junit.Before; +import org.junit.Test; + +import java.nio.ByteBuffer; + +/** + * Test dummy raw coder. + */ +public class TestDummyRawCoder extends TestRawCoderBase { + @Before + public void setup() { + encoderClass = DummyRawEncoder.class; + decoderClass = DummyRawDecoder.class; + setAllowDump(false); + setChunkSize(baseChunkSize); + } + + @Test + public void testCoding_6x3_erasing_d0_d2() { + prepare(null, 6, 3, new int[]{0, 2}, new int[0], false); + testCodingDoMixed(); + } + + @Test + public void testCoding_6x3_erasing_d0_p0() { + prepare(null, 6, 3, new int[]{0}, new int[]{0}, false); + testCodingDoMixed(); + } + + @Override + protected void testCoding(boolean usingDirectBuffer) { + this.usingDirectBuffer = usingDirectBuffer; + prepareCoders(); + + prepareBufferAllocator(true); + setAllowChangeInputs(false); + + // Generate data and encode + ECChunk[] dataChunks = prepareDataChunksForEncoding(); + markChunks(dataChunks); + ECChunk[] parityChunks = prepareParityChunksForEncoding(); + encoder.encode(dataChunks, parityChunks); + compareAndVerify(parityChunks, getEmptyChunks(parityChunks.length)); + + // Decode + restoreChunksFromMark(dataChunks); + backupAndEraseChunks(dataChunks, parityChunks); + ECChunk[] inputChunks = prepareInputChunksForDecoding( + dataChunks, parityChunks); + ensureOnlyLeastRequiredChunks(inputChunks); + ECChunk[] recoveredChunks = prepareOutputChunksForDecoding(); + decoder.decode(inputChunks, getErasedIndexesForDecoding(), recoveredChunks); + compareAndVerify(recoveredChunks, getEmptyChunks(recoveredChunks.length)); + } + + private ECChunk[] getEmptyChunks(int num) { + ECChunk[] chunks = new ECChunk[num]; + for (int i = 0; i < chunks.length; i++) { + chunks[i] = new ECChunk(ByteBuffer.wrap(getZeroChunkBytes())); + } + return chunks; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e1745d8/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java index ec93d44..9b6a196 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java @@ -30,8 +30,8 @@ import java.lang.reflect.Constructor; public abstract class TestRawCoderBase extends TestCoderBase { protected Class<? extends RawErasureEncoder> encoderClass; protected Class<? extends RawErasureDecoder> decoderClass; - private RawErasureEncoder encoder; - private RawErasureDecoder decoder; + protected RawErasureEncoder encoder; + protected RawErasureDecoder decoder; /** * Doing twice to test if the coders can be repeatedly reused. This matters @@ -187,13 +187,13 @@ public abstract class TestRawCoderBase extends TestCoderBase { compareAndVerify(backupChunks, recoveredChunks); } - private void setAllowChangeInputs(boolean allowChangeInputs) { + protected void setAllowChangeInputs(boolean allowChangeInputs) { this.allowChangeInputs = allowChangeInputs; encoder.setCoderOption(CoderOption.ALLOW_CHANGE_INPUTS, allowChangeInputs); decoder.setCoderOption(CoderOption.ALLOW_CHANGE_INPUTS, allowChangeInputs); } - private void prepareCoders() { + protected void prepareCoders() { if (encoder == null) { encoder = createEncoder(); } @@ -203,7 +203,7 @@ public abstract class TestRawCoderBase extends TestCoderBase { } } - private void ensureOnlyLeastRequiredChunks(ECChunk[] inputChunks) { + protected void ensureOnlyLeastRequiredChunks(ECChunk[] inputChunks) { int leastRequiredNum = numDataUnits; int erasedNum = erasedDataIndexes.length + erasedParityIndexes.length; int goodNum = inputChunks.length - erasedNum;
