Hello,
since this is my first posting to this list, first of all, let me thank all of
you for writing/helping to improve jakarta-commons httpclient.
I would like to present a tiny patch which enables the user to set a maximum
number of bytes to read from a HTTP response.
This patch has proven very useful in our project and I guess it is also
suitable for others.
I have introduced a new HttpMethod-Parameter "RESPONSE_MAX_BYTES" where you
can specify the maximum length of the HTTP Response as a "long" value.
If the limit is reached, httpclient will silently assume that the end of the
stream is reached, even if there are more bytes available.
This gives you the chance to retrieve the transferred bytes via
getResponseBody...() for further processing, if you are only interested in
the first n bytes of a possibly endless stream.
As an addition to patches for existing classes, I have created a class
"LimitedSizeInputStream" which does most of the work.
Furthermore, I have patched ChunkedInputStream, changing its behaviour when a
chunk prematurely ends before reading its size (which can happen when you
limit the response). Now, no IOException will be thrown but the chunk will
simply be discarded (chunkSize set to 0).
The usage is simple:
long limit = 1024 * 1024; // maximum response length: 1 Megabyte
HttpClient client = new HttpClient();
HttpClientParams param = (HttpClientParams)client.getParams();
param.setLongParameter(HttpMethodParams.RESPONSE_MAX_BYTES, limit);
Any subsequent calls to client.executeMethod(...) will limit the response to
1M.
The patch is included at the end of this mail. Please tell me what you think
about it.
Best regards,
Christian Kohlschütter
? diff
? src/java/org/apache/commons/httpclient/LimitedSizeInputStream.java
Index: src/java/org/apache/commons/httpclient/ChunkedInputStream.java
===================================================================
RCS file:
/home/cvspublic/jakarta-commons/httpclient/src/java/org/apache/commons/httpclient/ChunkedInputStream.java,v
retrieving revision 1.19
diff -r1.19 ChunkedInputStream.java
241c241,246
< chunkSize = getChunkSizeFromInputStream(in);
---
> try {
> chunkSize = getChunkSizeFromInputStream(in);
> } catch(IOException e) {
> LOG.debug("Cannot get chunk size - assuming EOF reached", e);
> chunkSize = 0;
> }
Index: src/java/org/apache/commons/httpclient/HttpConnection.java
===================================================================
RCS file:
/home/cvspublic/jakarta-commons/httpclient/src/java/org/apache/commons/httpclient/HttpConnection.java,v
retrieving revision 1.75
diff -r1.75 HttpConnection.java
574a575,595
>
> /**
> * Sets the maximum number of bytes readable from the HTTP Response.
> *
> * @param limit The bytes limit. 0 means no limit.
> */
> public void setResponseMaxBytes(long limit) {
> this.responseMaxBytes = limit;
> }
>
> /**
> * Resets the number of received bytes from the HTTP Response.
> * This is useful for Connection reuse.
> *
> */
> public void resetResponseByteCount() {
> if(limitedSizeInputStream != null) {
> limitedSizeInputStream.resetByteCounter();
> }
> }
>
640c661
< inputStream = new PushbackInputStream(socket.getInputStream());
---
> inputStream = new
PushbackInputStream(prepareResponseInputStream(socket.getInputStream()));
693c714
< inputStream = new PushbackInputStream(socket.getInputStream());
---
> inputStream = new
PushbackInputStream(prepareResponseInputStream(socket.getInputStream()));
1155a1177,1186
>
> protected InputStream prepareResponseInputStream(InputStream in) {
> if(this.responseMaxBytes != 0) {
> limitedSizeInputStream = new LimitedSizeInputStream(in,
this.responseMaxBytes);
> in = limitedSizeInputStream;
> } else {
> limitedSizeInputStream = null;
> }
> return in;
> }
1321a1353,1355
>
> /** The proably underlying limited-size InputStream */
> private LimitedSizeInputStream limitedSizeInputStream = null;
1364a1399,1401
>
> /** The maximum number of bytes to be read from response */
> private long responseMaxBytes;
Index: src/java/org/apache/commons/httpclient/HttpMethodDirector.java
===================================================================
RCS file:
/home/cvspublic/jakarta-commons/httpclient/src/java/org/apache/commons/httpclient/HttpMethodDirector.java,v
retrieving revision 1.5
diff -r1.5 HttpMethodDirector.java
230a231,232
>
connection.setResponseMaxBytes(this.params.getLongParameter(HttpMethodParams.RESPONSE_MAX_BYTES,
0));
> connection.resetResponseByteCount(); // if we are reusing an
existing HttpConnection object
Index: src/java/org/apache/commons/httpclient/params/HttpMethodParams.java
===================================================================
RCS file:
/home/cvspublic/jakarta-commons/httpclient/src/java/org/apache/commons/httpclient/params/HttpMethodParams.java,v
retrieving revision 1.3
diff -r1.3 HttpMethodParams.java
164a165,171
>
> /**
> * Sets the maximum number of bytes that will be read per HTTP method
response.
> * A value of zero means there is no limit. The default value is zero.
> * This parameter expects a value of type [EMAIL PROTECTED] Long}.
> */
> public static final String RESPONSE_MAX_BYTES =
"http.response.maxBytes";
package org.apache.commons.httpclient;
import java.io.FilterInputStream;
import java.io.IOException;
import java.io.InputStream;
/**
* An <code>InputStream</code> which reads only up to a specific maximum
number of bytes.
*
* @author Christian Kohlschütter
* @version $Id$
*/
public class LimitedSizeInputStream extends FilterInputStream {
private long max;
private long count = 0;
private boolean pastEOF = false;
private long mark = 0;
/**
* Creates a new <code>LimitedSizeInputStream</code> using the given
* underlying <code>InputStream</code>, reading a maximum of
<code>maxBytes</code>
* bytes from that stream.
*/
public LimitedSizeInputStream(InputStream in, long maxBytes) {
super(in);
setLimit(maxBytes);
resetByteCounter();
}
/**
* Returns the current limit.
*
* @return Maximum number of bytes to be read.
*/
public long getLimit() {
return max;
}
/**
* Sets the new limit.
*
* @param maxBytes Maximum number of bytes to be read.
*/
public void setLimit(long maxBytes) {
long oldMax = max;
this.max = Math.max(0,maxBytes);
if(max < oldMax) {
pastEOF = true;
}
}
/**
* Returns the number of bytes read/skipped.
*
* @return Number of consumed bytes.
*/
public long getByteCounter() {
return count;
}
/**
* Resets the read-bytes counter to 0.
*/
public void resetByteCounter() {
count = 0;
mark = 0;
pastEOF = false;
}
/* (non-Javadoc)
* @see java.io.InputStream#read()
*/
public int read() throws IOException {
if(pastEOF) {
return -1;
}
if(max != 0 && count >= max) {
pastEOF = true;
return -1;
}
int b = in.read();
if(b >= 0) {
count++;
} else {
pastEOF = true;
}
return b;
}
/* (non-Javadoc)
* @see java.io.InputStream#available()
*/
public int available() throws IOException {
if(max != 0 && count >= max) {
return 0;
}
return Math.min(in.available(), (int)(max-count));
}
/* (non-Javadoc)
* @see java.io.InputStream#close()
*/
public void close() throws IOException {
in.close();
}
/* (non-Javadoc)
* @see java.io.InputStream#mark(int)
*/
public synchronized void mark(int readlimit) {
in.mark(readlimit);
mark = count;
}
/* (non-Javadoc)
* @see java.io.InputStream#read(byte[], int, int)
*/
public int read(byte[] b, int off, int len) throws IOException {
if(pastEOF) {
return -1;
}
if(max != 0) {
if(count >= max) {
return -1;
}
len = Math.min(len, (int)(max-count));
}
if(len <= 0) {
return 0;
}
int r = in.read(b, off, len);
count += r;
if(count >= max || r == -1) {
pastEOF = true;
}
return r;
}
/* (non-Javadoc)
* @see java.io.InputStream#read(byte[])
*/
public int read(byte[] b) throws IOException {
return read(b, 0, b.length);
}
/* (non-Javadoc)
* @see java.io.InputStream#reset()
*/
public synchronized void reset() throws IOException {
in.reset();
count = mark;
pastEOF = (count > max);
}
/* (non-Javadoc)
* @see java.io.InputStream#skip(long)
*/
public long skip(long n) throws IOException {
if(pastEOF) {
return 0;
}
if(max != 0) {
n = Math.min(n, max-count);
}
long skipped = in.skip(n);
count += skipped;
if(count >= max) {
pastEOF = true;
}
return skipped;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [EMAIL PROTECTED]
For additional commands, e-mail: [EMAIL PROTECTED]
