Repository: commons-io Updated Branches: refs/heads/master 34d6eea40 -> 58b0f795b
Adding the PeekableInputStream. Project: http://git-wip-us.apache.org/repos/asf/commons-io/repo Commit: http://git-wip-us.apache.org/repos/asf/commons-io/commit/559de2c4 Tree: http://git-wip-us.apache.org/repos/asf/commons-io/tree/559de2c4 Diff: http://git-wip-us.apache.org/repos/asf/commons-io/diff/559de2c4 Branch: refs/heads/master Commit: 559de2c461e94ab636c959149c775bb27111fb48 Parents: 7b813b6 Author: Jochen Wiedmann (jwi) <jochen.wiedm...@softwareag.com> Authored: Fri Jul 27 16:43:35 2018 +0200 Committer: Jochen Wiedmann (jwi) <jochen.wiedm...@softwareag.com> Committed: Fri Jul 27 16:44:05 2018 +0200 ---------------------------------------------------------------------- src/changes/changes.xml | 3 + .../input/buffer/CircularBufferInputStream.java | 126 ++++++++++ .../io/input/buffer/CircularByteBuffer.java | 237 +++++++++++++++++++ .../io/input/buffer/PeekableInputStream.java | 87 +++++++ .../buffer/CircularBufferInputStreamTest.java | 83 +++++++ 5 files changed, 536 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/commons-io/blob/559de2c4/src/changes/changes.xml ---------------------------------------------------------------------- diff --git a/src/changes/changes.xml b/src/changes/changes.xml index 9a140c0..fecb959 100644 --- a/src/changes/changes.xml +++ b/src/changes/changes.xml @@ -47,6 +47,9 @@ The <action> type attribute can be add,update,fix,remove. <body> <!-- The release date is the date RC is cut --> <release version="2.7" date="tba" description="tba"> + <action dev="jochen" type="add"> + Adding the CircularBufferInputStream, and the PeekableInputStream. + </action> <action issue="IO-582" dev="jochen" type="fix" due-to="Bruno Palos"> Make methods in ObservableInputStream.Obsever public. </action> http://git-wip-us.apache.org/repos/asf/commons-io/blob/559de2c4/src/main/java/org/apache/commons/io/input/buffer/CircularBufferInputStream.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/io/input/buffer/CircularBufferInputStream.java b/src/main/java/org/apache/commons/io/input/buffer/CircularBufferInputStream.java new file mode 100644 index 0000000..7298e5b --- /dev/null +++ b/src/main/java/org/apache/commons/io/input/buffer/CircularBufferInputStream.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.commons.io.input.buffer; + +import java.io.BufferedInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Objects; + + +/** + * Implementation of a buffered input stream, which is internally based on the + * {@link CircularByteBuffer}. Unlike the {@link BufferedInputStream}, this one + * doesn't need to reallocate byte arrays internally. + */ +public class CircularBufferInputStream extends InputStream { + protected final InputStream in; + protected final CircularByteBuffer buffer; + protected final int bufferSize; + private boolean eofSeen; + + /** Creates a new instance, which filters the given input stream, and + * uses the given buffer size. + * @param pIn The input stream, which is being buffered. + * @param pBufferSize The size of the {@link CircularByteBuffer}, which is + * used internally. + */ + public CircularBufferInputStream(InputStream pIn, int pBufferSize) { + Objects.requireNonNull(pIn, "InputStream"); + if (pBufferSize <= 0) { + throw new IllegalArgumentException("Invalid buffer size: " + pBufferSize); + } + in = pIn; + buffer = new CircularByteBuffer(pBufferSize); + bufferSize = pBufferSize; + eofSeen = false; + } + + /** Creates a new instance, which filters the given input stream, and + * uses a reasonable default buffer size (8192). + * @param pIn The input stream, which is being buffered. + * @param pBufferSize The size of the {@link CircularByteBuffer}, which is + * used internally. + */ + public CircularBufferInputStream(InputStream pIn) { + this(pIn, 8192); + } + + protected void fillBuffer() throws IOException { + if (eofSeen) { + return; + } + int space = buffer.getSpace(); + final byte[] buf = new byte[space]; + while (space > 0) { + final int res = in.read(buf, 0, space); + if (res == -1) { + eofSeen = true; + return; + } else if (res > 0) { + buffer.add(buf, 0, res); + space -= res; + } + } + } + + protected boolean haveBytes(int pNumber) throws IOException { + if (buffer.getCurrentNumberOfBytes() < pNumber) { + fillBuffer(); + } + return buffer.hasBytes(); + } + + @Override + public int read() throws IOException { + if (!haveBytes(1)) { + return -1; + } + return buffer.read(); + } + + @Override + public int read(byte[] pBuffer) throws IOException { + return read(pBuffer, 0, pBuffer.length); + } + + @Override + public int read(byte[] pBuffer, int pOffset, int pLength) throws IOException { + Objects.requireNonNull(pBuffer, "Buffer"); + if (pOffset < 0) { + throw new IllegalArgumentException("Offset must not be negative"); + } + if (pLength < 0) { + throw new IllegalArgumentException("Length must not be negative"); + } + if (!haveBytes(pLength)) { + return -1; + } + final int result = Math.min(pLength, buffer.getCurrentNumberOfBytes()); + for (int i = 0; i < result; i++) { + pBuffer[pOffset+i] = buffer.read(); + } + return result; + } + + @Override + public void close() throws IOException { + in.close(); + eofSeen = true; + buffer.clear(); + } +} http://git-wip-us.apache.org/repos/asf/commons-io/blob/559de2c4/src/main/java/org/apache/commons/io/input/buffer/CircularByteBuffer.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/io/input/buffer/CircularByteBuffer.java b/src/main/java/org/apache/commons/io/input/buffer/CircularByteBuffer.java new file mode 100644 index 0000000..52d84bb --- /dev/null +++ b/src/main/java/org/apache/commons/io/input/buffer/CircularByteBuffer.java @@ -0,0 +1,237 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.commons.io.input.buffer; + +import java.util.Objects; + +/** + * A buffer, which doesn't need reallocation of byte arrays, because it + * reuses a single byte array. This works particularly well, if reading + * from the buffer takes place at the same time than writing to. Such is the + * case, for example, when using the buffer within a filtering input stream, + * like the {@link CircularBufferInputStream}. + */ +public class CircularByteBuffer { + private final byte[] buffer; + private int startOffset, endOffset, currentNumberOfBytes; + + /** + * Creates a new instance with the given buffer size. + */ + public CircularByteBuffer(int pSize) { + buffer = new byte[pSize]; + startOffset = 0; + endOffset = 0; + currentNumberOfBytes = 0; + } + + /** + * Creates a new instance with a reasonable default buffer size (8192). + */ + public CircularByteBuffer() { + this(8192); + } + + /** + * Returns the next byte from the buffer, removing it at the same time, so + * that following invocations won't return it again. + * @return The byte, which is being returned. + * @throws IllegalStateException The buffer is empty. Use {@link #hasBytes()}, + * or {@link #getCurrentNumberOfBytes()}, to prevent this exception. + */ + public byte read() { + if (currentNumberOfBytes <= 0) { + throw new IllegalStateException("No bytes available."); + } + final byte b = buffer[startOffset]; + --currentNumberOfBytes; + if (++startOffset == buffer.length) { + startOffset = 0; + } + return b; + } + + /** + * Returns the given number of bytes from the buffer by storing them in + * the given byte array at the given offset. + * @param pBuffer The byte array, where to add bytes. + * @param pOffset The offset, where to store bytes in the byte array. + * @param pLength The number of bytes to return. + * @throws NullPointerException The byte array {@code pBuffer} is null. + * @throws IllegalArgumentException Either of {@code pOffset}, or {@code pLength} is negative, + * or the length of the byte array {@code pBuffer} is too small. + * @throws IllegalStateException The buffer doesn't hold the given number + * of bytes. Use {@link #getCurrentNumberOfBytes()} to prevent this + * exception. + */ + public void read(byte[] pBuffer, int pOffset, int pLength) { + Objects.requireNonNull(pBuffer); + if (pOffset < 0 || pOffset >= pBuffer.length) { + throw new IllegalArgumentException("Invalid offset: " + pOffset); + } + if (pLength < 0 || pLength > buffer.length) { + throw new IllegalArgumentException("Invalid length: " + pLength); + } + if (pOffset+pLength > pBuffer.length) { + throw new IllegalArgumentException("The supplied byte array contains only " + + pBuffer.length + " bytes, but offset, and length would require " + + (pOffset+pLength-1)); + } + if (currentNumberOfBytes < pLength) { + throw new IllegalStateException("Currently, there are only " + currentNumberOfBytes + + "in the buffer, not " + pLength); + } + int offset = pOffset; + for (int i = 0; i < pLength; i++) { + pBuffer[offset++] = buffer[startOffset]; + --currentNumberOfBytes; + if (++startOffset == buffer.length) { + startOffset = 0; + } + } + } + + /** + * Adds a new byte to the buffer, which will eventually be returned by following + * invocations of {@link #read()}. + * @param pByte The byte, which is being added to the buffer. + * @throws IllegalStateException The buffer is full. Use {@link #hasSpace()}, + * or {@link #getSpace()}, to prevent this exception. + */ + public void add(byte pByte) { + if (currentNumberOfBytes >= buffer.length) { + throw new IllegalStateException("No space available"); + } + buffer[endOffset] = pByte; + ++currentNumberOfBytes; + if (++endOffset == buffer.length) { + endOffset = 0; + } + } + + /** + * Returns, whether the next bytes in the buffer are exactly those, given by + * {@code pBuffer}, {@code pOffset}, and {@code pLength}. No bytes are being + * removed from the buffer. If the result is true, then the following invocations + * of {@link #read()} are guaranteed to return exactly those bytes. + * @return True, if the next invocations of {@link #read()} will return the + * bytes at offsets {@code pOffset}+0, {@code pOffset}+1, ..., + * @code{pOffset}+@code{pLength}-1 of byte array {@code pBuffer}. + * @throws IllegalArgumentException Either of {@code pOffset}, or {@code pLength} is negative. + * @throws NullPointerException The byte array {@code pBuffer} is null. + */ + public boolean peek(byte[] pBuffer, int pOffset, int pLength) { + Objects.requireNonNull(pBuffer, "Buffer"); + if (pOffset < 0 || pOffset >= pBuffer.length) { + throw new IllegalArgumentException("Invalid offset: " + pOffset); + } + if (pLength < 0 || pLength > buffer.length) { + throw new IllegalArgumentException("Invalid length: " + pLength); + } + if (pLength < currentNumberOfBytes) { + return false; + } + int offset = startOffset; + for (int i = 0; i < pLength; i++) { + if (buffer[offset] != pBuffer[i+pOffset]) { + return false; + } + if (++offset == buffer.length) { + offset = 0; + } + } + return true; + } + + /** + * Adds the given bytes to the buffer. This is the same as invoking {@link #add(byte)} + * for the bytes at offsets {@code pOffset}+0, {@code pOffset}+1, ..., + * @code{pOffset}+@code{pLength}-1 of byte array {@code pBuffer}. + * @throws IllegalStateException The buffer doesn't have sufficient space. Use + * {@link #getSpace()} to prevent this exception. + * @throws IllegalArgumentException Either of {@code pOffset}, or {@code pLength} is negative. + * @throws NullPointerException The byte array {@code pBuffer} is null. + */ + public void add(byte[] pBuffer, int pOffset, int pLength) { + Objects.requireNonNull(pBuffer, "Buffer"); + if (pOffset < 0 || pOffset >= pBuffer.length) { + throw new IllegalArgumentException("Invalid offset: " + pOffset); + } + if (pLength < 0) { + throw new IllegalArgumentException("Invalid length: " + pLength); + } + if (currentNumberOfBytes+pLength > buffer.length) { + throw new IllegalStateException("No space available"); + } + for (int i = 0; i < pLength; i++) { + buffer[endOffset] = pBuffer[pOffset+i]; + if (++endOffset == buffer.length) { + endOffset = 0; + } + } + currentNumberOfBytes += pLength; + } + + /** + * Returns, whether there is currently room for a single byte in the buffer. + * Same as {@link #hasSpace(int) hasSpace(1)}. + * @see #hasSpace(int) + * @see #getSpace() + */ + public boolean hasSpace() { + return currentNumberOfBytes < buffer.length; + } + + /** + * Returns, whether there is currently room for the given number of bytes in the buffer. + * @see #hasSpace() + * @see #getSpace() + */ + public boolean hasSpace(int pBytes) { + return currentNumberOfBytes+pBytes <= buffer.length; + } + + /** + * Returns, whether the buffer is currently holding, at least, a single byte. + */ + public boolean hasBytes() { + return currentNumberOfBytes > 0; + } + + /** + * Returns the number of bytes, that can currently be added to the buffer. + */ + public int getSpace() { + return buffer.length - currentNumberOfBytes; + } + + /** + * Returns the number of bytes, that are currently present in the buffer. + */ + public int getCurrentNumberOfBytes() { + return currentNumberOfBytes; + } + + /** + * Removes all bytes from the buffer. + */ + public void clear() { + startOffset = 0; + endOffset = 0; + currentNumberOfBytes = 0; + } +} http://git-wip-us.apache.org/repos/asf/commons-io/blob/559de2c4/src/main/java/org/apache/commons/io/input/buffer/PeekableInputStream.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/io/input/buffer/PeekableInputStream.java b/src/main/java/org/apache/commons/io/input/buffer/PeekableInputStream.java new file mode 100644 index 0000000..e56c378 --- /dev/null +++ b/src/main/java/org/apache/commons/io/input/buffer/PeekableInputStream.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.commons.io.input.buffer; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Objects; + + +/** Implementation of a buffered input stream, which allows to peek into + * the buffers first bytes. This comes in handy when manually implementing + * scanners, lexers, parsers, or the like. + */ +public class PeekableInputStream extends CircularBufferInputStream { + /** Creates a new instance, which filters the given input stream, and + * uses the given buffer size. + * @param pIn The input stream, which is being buffered. + * @param pBufferSize The size of the {@link CircularByteBuffer}, which is + * used internally. + */ + public PeekableInputStream(InputStream pIn, int pBufferSize) { + super(pIn, pBufferSize); + } + + /** Creates a new instance, which filters the given input stream, and + * uses a reasonable default buffer size (8192). + * @param pIn The input stream, which is being buffered. + * @param pBufferSize The size of the {@link CircularByteBuffer}, which is + * used internally. + */ + public PeekableInputStream(InputStream pIn) { + super(pIn); + } + + /** + * Returns, whether the next bytes in the buffer are as given by + * {@code pBuffer}. This is equivalent to {@link #peek(byte[],int,int)} + * with {@code pOffset} == 0, and {@code pLength} == {@code pBuffer.length} + * @param pBuffer + * @return + * @throws IOException Refilling the buffer failed. + */ + public boolean peek(byte[] pBuffer) throws IOException { + Objects.requireNonNull(pBuffer, "Buffer"); + if (pBuffer.length > bufferSize) { + throw new IllegalArgumentException("Peek request size of " + pBuffer.length + + " bytes exceeds buffer size of " + bufferSize + " bytes"); + } + if (buffer.getCurrentNumberOfBytes() < pBuffer.length) { + fillBuffer(); + } + return buffer.peek(pBuffer, 0, pBuffer.length); + } + + /** + * Returns, whether the next bytes in the buffer are as given by + * {@code pBuffer}, {code pOffset}, and {@code pLength}. + * @param pBuffer + * @return + * @throws IOException + */ + public boolean peek(byte[] pBuffer, int pOffset, int pLength) throws IOException { + Objects.requireNonNull(pBuffer, "Buffer"); + if (pBuffer.length > bufferSize) { + throw new IllegalArgumentException("Peek request size of " + pBuffer.length + + " bytes exceeds buffer size of " + bufferSize + " bytes"); + } + if (buffer.getCurrentNumberOfBytes() < pBuffer.length) { + fillBuffer(); + } + return buffer.peek(pBuffer, pOffset, pLength); + } +} http://git-wip-us.apache.org/repos/asf/commons-io/blob/559de2c4/src/test/java/org/apache/commons/io/input/buffer/CircularBufferInputStreamTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/commons/io/input/buffer/CircularBufferInputStreamTest.java b/src/test/java/org/apache/commons/io/input/buffer/CircularBufferInputStreamTest.java new file mode 100644 index 0000000..311baa0 --- /dev/null +++ b/src/test/java/org/apache/commons/io/input/buffer/CircularBufferInputStreamTest.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.commons.io.input.buffer; + +import java.io.ByteArrayInputStream; +import java.util.Random; + +import org.junit.Test; + + +public class CircularBufferInputStreamTest { + private final Random rnd = new Random(1530960934483l); // System.currentTimeMillis(), when this test was written. + // Always using the same seed should ensure a reproducable test. + + @Test + public void testRandomRead() throws Exception { + final byte[] inputBuffer = newInputBuffer(); + final byte[] bufferCopy = new byte[inputBuffer.length]; + final ByteArrayInputStream bais = new ByteArrayInputStream(inputBuffer); + @SuppressWarnings("resource") + final CircularBufferInputStream cbis = new CircularBufferInputStream(bais, 253); + int offset = 0; + final byte[] readBuffer = new byte[256]; + while (offset < bufferCopy.length) { + switch (rnd.nextInt(2)) { + case 0: + { + final int res = cbis.read(); + if (res == -1) { + throw new IllegalStateException("Unexpected EOF at offset " + offset); + } + if (inputBuffer[offset] != res) { + throw new IllegalStateException("Expected " + inputBuffer[offset] + " at offset " + offset + ", got " + res); + } + ++offset; + break; + } + case 1: + { + final int res = cbis.read(readBuffer, 0, rnd.nextInt(readBuffer.length+1)); + if (res == -1) { + throw new IllegalStateException("Unexpected EOF at offset " + offset); + } else if (res == 0) { + throw new IllegalStateException("Unexpected zero-byte-result at offset " + offset); + } else { + for (int i = 0; i < res; i++) { + if (inputBuffer[offset] != readBuffer[i]) { + throw new IllegalStateException("Expected " + inputBuffer[offset] + " at offset " + offset + ", got " + readBuffer[i]); + } + ++offset; + } + } + break; + } + default: + throw new IllegalStateException("Unexpected random choice value"); + } + } + } + + /** + * Create a large, but random input buffer. + */ + private byte[] newInputBuffer() { + final byte[] buffer = new byte[16*512+rnd.nextInt(512)]; + rnd.nextBytes(buffer); + return buffer; + } +}
