HDFS-11541. Call RawErasureEncoder and RawErasureDecoder release() methods. Contributed by SammiChen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/84d787b9 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/84d787b9 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/84d787b9 Branch: refs/heads/HDFS-7240 Commit: 84d787b9d51196010495d51dc5ebf66c01c340ab Parents: 0e6f8e4 Author: Rakesh Radhakrishnan <[email protected]> Authored: Wed Mar 29 11:41:48 2017 +0530 Committer: Rakesh Radhakrishnan <[email protected]> Committed: Wed Mar 29 11:41:48 2017 +0530 ---------------------------------------------------------------------- .../hadoop/hdfs/DFSStripedInputStream.java | 20 ++++---- .../hadoop/hdfs/DFSStripedOutputStream.java | 1 + .../StripedBlockChecksumReconstructor.java | 48 +++++++++++--------- .../erasurecode/StripedBlockReconstructor.java | 1 + .../erasurecode/StripedReconstructor.java | 6 +++ 5 files changed, 46 insertions(+), 30 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/84d787b9/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java index 922f74e..07e6609 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java @@ -177,14 +177,18 @@ public class DFSStripedInputStream extends DFSInputStream { @Override public synchronized void close() throws IOException { - super.close(); - if (curStripeBuf != null) { - BUFFER_POOL.putBuffer(curStripeBuf); - curStripeBuf = null; - } - if (parityBuf != null) { - BUFFER_POOL.putBuffer(parityBuf); - parityBuf = null; + try { + super.close(); + } finally { + if (curStripeBuf != null) { + BUFFER_POOL.putBuffer(curStripeBuf); + curStripeBuf = null; + } + if (parityBuf != null) { + BUFFER_POOL.putBuffer(parityBuf); + parityBuf = null; + } + decoder.release(); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/84d787b9/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java index 52fc5eb..22b30e9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java @@ -1033,6 +1033,7 @@ public class DFSStripedOutputStream extends DFSOutputStream { setClosed(); // shutdown executor of flushAll tasks flushAllExecutor.shutdownNow(); + encoder.release(); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/84d787b9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockChecksumReconstructor.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockChecksumReconstructor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockChecksumReconstructor.java index 9555618..6917317 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockChecksumReconstructor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockChecksumReconstructor.java @@ -75,29 +75,33 @@ public class StripedBlockChecksumReconstructor extends StripedReconstructor { public void reconstruct() throws IOException { MessageDigest digester = MD5Hash.getDigester(); long maxTargetLength = getMaxTargetLength(); - while (requestedLen > 0 && getPositionInBlock() < maxTargetLength) { - long remaining = maxTargetLength - getPositionInBlock(); - final int toReconstructLen = (int) Math - .min(getStripedReader().getBufferSize(), remaining); - // step1: read from minimum source DNs required for reconstruction. - // The returned success list is the source DNs we do real read from - getStripedReader().readMinimumSources(toReconstructLen); - - // step2: decode to reconstruct targets - reconstructTargets(toReconstructLen); - - // step3: calculate checksum - checksumDataLen += checksumWithTargetOutput(targetBuffer.array(), - toReconstructLen, digester); - - updatePositionInBlock(toReconstructLen); - requestedLen -= toReconstructLen; - clearBuffers(); - } + try { + while (requestedLen > 0 && getPositionInBlock() < maxTargetLength) { + long remaining = maxTargetLength - getPositionInBlock(); + final int toReconstructLen = (int) Math + .min(getStripedReader().getBufferSize(), remaining); + // step1: read from minimum source DNs required for reconstruction. + // The returned success list is the source DNs we do real read from + getStripedReader().readMinimumSources(toReconstructLen); + + // step2: decode to reconstruct targets + reconstructTargets(toReconstructLen); + + // step3: calculate checksum + checksumDataLen += checksumWithTargetOutput(targetBuffer.array(), + toReconstructLen, digester); + + updatePositionInBlock(toReconstructLen); + requestedLen -= toReconstructLen; + clearBuffers(); + } - byte[] digest = digester.digest(); - md5 = new MD5Hash(digest); - md5.write(checksumWriter); + byte[] digest = digester.digest(); + md5 = new MD5Hash(digest); + md5.write(checksumWriter); + } finally { + cleanup(); + } } private long checksumWithTargetOutput(byte[] outputData, int toReconstructLen, http://git-wip-us.apache.org/repos/asf/hadoop/blob/84d787b9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java index a1da536..1119bbb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java @@ -74,6 +74,7 @@ class StripedBlockReconstructor extends StripedReconstructor metrics.incrECReconstructionBytesWritten(getBytesWritten()); getStripedReader().close(); stripedWriter.close(); + cleanup(); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/84d787b9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java index cd17864..b8433c7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java @@ -253,6 +253,12 @@ abstract class StripedReconstructor { return decoder; } + void cleanup() { + if (decoder != null) { + decoder.release(); + } + } + StripedReader getStripedReader() { return stripedReader; } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
