HDFS-8813. Erasure Coding: Client no need to decode missing parity blocks. Contributed by Walter Su.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c2c26e6e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c2c26e6e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c2c26e6e Branch: refs/heads/HDFS-7285-merge Commit: c2c26e6ea7967f02c284918f68554334d4af8561 Parents: 5956d23 Author: Jing Zhao <ji...@apache.org> Authored: Thu Jul 23 11:42:02 2015 -0700 Committer: Jing Zhao <ji...@apache.org> Committed: Thu Jul 23 11:42:02 2015 -0700 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt | 3 +++ .../java/org/apache/hadoop/hdfs/DFSStripedInputStream.java | 7 ++++++- .../java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2c26e6e/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt index ad1e4e7..9741585 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt @@ -370,3 +370,6 @@ HDFS-8781. Erasure Coding: Correctly handle BlockManager#InvalidateBlocks for striped block. (Yi Liu via jing9) + + HDFS-8813. Erasure Coding: Client no need to decode missing parity blocks. + (Walter Su via jing9) http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2c26e6e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java index eecdf67..4f3a8ed 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java @@ -867,8 +867,13 @@ public class DFSStripedInputStream extends DFSInputStream { for (int i = 0; i < alignedStripe.chunks.length; i++) { if (alignedStripe.chunks[i] != null && alignedStripe.chunks[i].state == StripingChunk.MISSING) { - decodeIndices[pos++] = StripedBlockUtil.convertIndex4Decode(i, + int decodeIndex = StripedBlockUtil.convertIndex4Decode(i, dataBlkNum, parityBlkNum); + if (i < dataBlkNum) { + decodeIndices[pos++] = decodeIndex; + } else { + decodeInputs[decodeIndex] = null; + } } } decodeIndices = Arrays.copyOf(decodeIndices, pos); http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2c26e6e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java index 3e5ef43..a3ee1e8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java @@ -326,7 +326,7 @@ public class StripedBlockUtil { // Step 1: prepare indices and output buffers for missing data units int[] decodeIndices = new int[parityBlkNum]; int pos = 0; - for (int i = 0; i < alignedStripe.chunks.length; i++) { + for (int i = 0; i < dataBlkNum; i++) { if (alignedStripe.chunks[i] != null && alignedStripe.chunks[i].state == StripingChunk.MISSING){ decodeIndices[pos++] = convertIndex4Decode(i, dataBlkNum, parityBlkNum);
