HDDS-732. Add read method which takes offset and length in SignedChunkInputStream. Contributed by chencan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/34b6fa7d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/34b6fa7d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/34b6fa7d Branch: refs/heads/HDDS-4 Commit: 34b6fa7d6b22439f5c703bf7e82de5c40c77994a Parents: 22867de Author: Márton Elek <[email protected]> Authored: Wed Nov 21 17:59:20 2018 +0100 Committer: Márton Elek <[email protected]> Committed: Wed Nov 21 17:59:36 2018 +0100 ---------------------------------------------------------------------- .../ozone/s3/SignedChunksInputStream.java | 40 ++++++++++++++++++++ .../ozone/s3/TestSignedChunksInputStream.java | 30 +++++++++++++++ 2 files changed, 70 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/34b6fa7d/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/SignedChunksInputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/SignedChunksInputStream.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/SignedChunksInputStream.java index a35133f..b0dfce6 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/SignedChunksInputStream.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/SignedChunksInputStream.java @@ -65,6 +65,46 @@ public class SignedChunksInputStream extends InputStream { } } + @Override + public int read(byte b[], int off, int len) throws IOException { + if (b == null) { + throw new NullPointerException(); + } else if (off < 0 || len < 0 || len > b.length - off) { + throw new IndexOutOfBoundsException(); + } else if (len == 0) { + return 0; + } + int currentOff = off; + int currentLen = len; + int totalReadBytes = 0; + int realReadLen = 0; + int maxReadLen = 0; + do { + if (remainingData > 0) { + maxReadLen = Math.min(remainingData, currentLen); + realReadLen = originalStream.read(b, currentOff, maxReadLen); + if (realReadLen == -1) { + break; + } + currentOff += realReadLen; + currentLen -= realReadLen; + totalReadBytes += realReadLen; + remainingData -= realReadLen; + if (remainingData == 0) { + //read the "\r\n" at the end of the data section + originalStream.read(); + originalStream.read(); + } + } else { + remainingData = readHeader(); + if (remainingData == -1) { + break; + } + } + } while (currentLen > 0); + return totalReadBytes > 0 ? totalReadBytes : -1; + } + private int readHeader() throws IOException { int prev = -1; int curr = 0; http://git-wip-us.apache.org/repos/asf/hadoop/blob/34b6fa7d/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestSignedChunksInputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestSignedChunksInputStream.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestSignedChunksInputStream.java index f1af7cb..3599c05 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestSignedChunksInputStream.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestSignedChunksInputStream.java @@ -49,32 +49,62 @@ public class TestSignedChunksInputStream { @Test public void singlechunk() throws IOException { + //test simple read() InputStream is = fileContent("0A;chunk-signature" + "=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40\r" + "\n1234567890\r\n"); String result = IOUtils.toString(is, Charset.forName("UTF-8")); Assert.assertEquals("1234567890", result); + + //test read(byte[],int,int) + is = fileContent("0A;chunk-signature" + + + "=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40\r" + + "\n1234567890\r\n"); + byte[] bytes = new byte[10]; + IOUtils.read(is, bytes, 0, 10); + Assert.assertEquals("1234567890", new String(bytes)); } @Test public void singlechunkwithoutend() throws IOException { + //test simple read() InputStream is = fileContent("0A;chunk-signature" + "=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40\r" + "\n1234567890"); String result = IOUtils.toString(is, Charset.forName("UTF-8")); Assert.assertEquals("1234567890", result); + + //test read(byte[],int,int) + is = fileContent("0A;chunk-signature" + + + "=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40\r" + + "\n1234567890"); + byte[] bytes = new byte[10]; + IOUtils.read(is, bytes, 0, 10); + Assert.assertEquals("1234567890", new String(bytes)); } @Test public void multichunks() throws IOException { + //test simple read() InputStream is = fileContent("0a;chunk-signature=signature\r\n" + "1234567890\r\n" + "05;chunk-signature=signature\r\n" + "abcde\r\n"); String result = IOUtils.toString(is, Charset.forName("UTF-8")); Assert.assertEquals("1234567890abcde", result); + + //test read(byte[],int,int) + is = fileContent("0a;chunk-signature=signature\r\n" + + "1234567890\r\n" + + "05;chunk-signature=signature\r\n" + + "abcde\r\n"); + byte[] bytes = new byte[15]; + IOUtils.read(is, bytes, 0, 15); + Assert.assertEquals("1234567890abcde", new String(bytes)); } private InputStream fileContent(String content) { --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
