HADOOP-14081. S3A: Consider avoiding array copy in S3ABlockOutputStream (ByteArrayBlock). Contributed by Rajesh Balamohan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8035749c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8035749c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8035749c Branch: refs/heads/YARN-5734 Commit: 8035749c26947dc641ef87dac041050d439a16d1 Parents: 172b23a Author: Steve Loughran <[email protected]> Authored: Mon Feb 20 16:21:00 2017 +0000 Committer: Steve Loughran <[email protected]> Committed: Mon Feb 20 16:21:46 2017 +0000 ---------------------------------------------------------------------- .../org/apache/hadoop/fs/s3a/S3ADataBlocks.java | 26 +++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8035749c/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java index 0fe2af7..05f8efe 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java @@ -298,6 +298,25 @@ final class S3ADataBlocks { } + static class S3AByteArrayOutputStream extends ByteArrayOutputStream { + + S3AByteArrayOutputStream(int size) { + super(size); + } + + /** + * InputStream backed by the internal byte array + * + * @return + */ + ByteArrayInputStream getInputStream() { + ByteArrayInputStream bin = new ByteArrayInputStream(this.buf, 0, count); + this.reset(); + this.buf = null; + return bin; + } + } + /** * Stream to memory via a {@code ByteArrayOutputStream}. * @@ -310,14 +329,14 @@ final class S3ADataBlocks { */ static class ByteArrayBlock extends DataBlock { - private ByteArrayOutputStream buffer; + private S3AByteArrayOutputStream buffer; private final int limit; // cache data size so that it is consistent after the buffer is reset. private Integer dataSize; ByteArrayBlock(int limit) { this.limit = limit; - buffer = new ByteArrayOutputStream(); + buffer = new S3AByteArrayOutputStream(limit); } /** @@ -333,8 +352,7 @@ final class S3ADataBlocks { InputStream startUpload() throws IOException { super.startUpload(); dataSize = buffer.size(); - ByteArrayInputStream bufferData = new ByteArrayInputStream( - buffer.toByteArray()); + ByteArrayInputStream bufferData = buffer.getInputStream(); buffer = null; return bufferData; } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
