HDFS-8744. Erasure Coding: the number of chunks in packet is not updated when writing parity data. Contributed by Li Bo
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f4098dfa Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f4098dfa Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f4098dfa Branch: refs/heads/HDFS-7285-merge Commit: f4098dfae44b6dd287a3054f2c04658773b4f466 Parents: e692c7d Author: boli2 <bo.b...@intel.com> Authored: Mon Jul 13 00:41:36 2015 -0400 Committer: boli2 <bo.b...@intel.com> Committed: Mon Jul 13 00:41:36 2015 -0400 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt | 3 +++ .../src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java | 2 +- .../src/main/java/org/apache/hadoop/hdfs/DFSPacket.java | 6 +++--- .../java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java | 4 +++- 4 files changed, 10 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4098dfa/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt index 90f6732..7b6d165 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt @@ -341,3 +341,6 @@ HDFS-8484. Erasure coding: Two contiguous blocks occupy IDs belong to same striped group. (Walter Su via jing9) + + HDFS-8744. Erasure Coding: the number of chunks in packet is not updated + when writing parity data. (Li Bo) http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4098dfa/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java index 9e201ad..f41044b 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java @@ -419,7 +419,7 @@ public class DFSOutputStream extends FSOutputSummer currentPacket.writeChecksum(checksum, ckoff, cklen); currentPacket.writeData(b, offset, len); - currentPacket.incNumChunks(); + currentPacket.incNumChunks(1); streamer.incBytesCurBlock(len); // If packet is full, enqueue it for transmission http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4098dfa/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSPacket.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSPacket.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSPacket.java index a26e35e..2698de3 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSPacket.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSPacket.java @@ -259,10 +259,10 @@ public class DFSPacket { } /** - * increase the number of chunks by one + * increase the number of chunks by n */ - synchronized void incNumChunks() { - numChunks++; + synchronized void incNumChunks(int n) { + numChunks += n; } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4098dfa/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java index 4234351..e6de714 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java @@ -389,11 +389,13 @@ public class DFSStripedOutputStream extends DFSOutputStream { int maxBytesToPacket = p.getMaxChunks() * bytesPerChecksum; int toWrite = byteBuffer.remaining() > maxBytesToPacket ? maxBytesToPacket: byteBuffer.remaining(); - int ckLen = ((toWrite - 1) / bytesPerChecksum + 1) * getChecksumSize(); + int chunks = (toWrite - 1) / bytesPerChecksum + 1; + int ckLen = chunks * getChecksumSize(); p.writeChecksum(checksumBuf, ckOff, ckLen); ckOff += ckLen; p.writeData(byteBuffer, toWrite); getCurrentStreamer().incBytesCurBlock(toWrite); + p.incNumChunks(chunks); packets.add(p); } return packets;
