Author: toad
Date: 2008-11-08 00:14:36 +0000 (Sat, 08 Nov 2008)
New Revision: 23400
Modified:
trunk/freenet/src/freenet/client/async/SingleFileInserter.java
trunk/freenet/src/freenet/keys/Key.java
trunk/freenet/src/freenet/support/compress/Bzip2Compressor.java
trunk/freenet/src/freenet/support/compress/Compressor.java
trunk/freenet/src/freenet/support/compress/GzipCompressor.java
trunk/freenet/src/freenet/support/compress/LZMACompressor.java
Log:
Limit the number of bytes read and written separately. Fixes us always using
gzip.
Modified: trunk/freenet/src/freenet/client/async/SingleFileInserter.java
===================================================================
--- trunk/freenet/src/freenet/client/async/SingleFileInserter.java
2008-11-08 00:12:08 UTC (rev 23399)
+++ trunk/freenet/src/freenet/client/async/SingleFileInserter.java
2008-11-08 00:14:36 UTC (rev 23400)
@@ -147,7 +147,7 @@
// Only produce if we are compressing
*the original data*
if(parent == cb)
ctx.eventProducer.produceEvent(new StartedCompressionEvent(comp));
- result = comp.compress(origData, new
BucketChainBucketFactory(ctx.persistentBucketFactory, CHKBlock.DATA_LENGTH),
bestCompressedDataSize);
+ result = comp.compress(origData, new
BucketChainBucketFactory(ctx.persistentBucketFactory, CHKBlock.DATA_LENGTH),
origSize, bestCompressedDataSize);
long resultSize = result.size();
if(resultSize < oneBlockCompressedSize)
{
bestCodec = comp;
Modified: trunk/freenet/src/freenet/keys/Key.java
===================================================================
--- trunk/freenet/src/freenet/keys/Key.java 2008-11-08 00:12:08 UTC (rev
23399)
+++ trunk/freenet/src/freenet/keys/Key.java 2008-11-08 00:14:36 UTC (rev
23400)
@@ -188,7 +188,7 @@
ArrayBucket compressedData;
try {
compressedData =
(ArrayBucket) comp.compress(
-
sourceData, new ArrayBucketFactory(), maxCompressedDataLength);
+
sourceData, new ArrayBucketFactory(), Long.MAX_VALUE, maxCompressedDataLength);
} catch (IOException e) {
throw new Error(e);
} catch
(CompressionOutputSizeException e) {
Modified: trunk/freenet/src/freenet/support/compress/Bzip2Compressor.java
===================================================================
--- trunk/freenet/src/freenet/support/compress/Bzip2Compressor.java
2008-11-08 00:12:08 UTC (rev 23399)
+++ trunk/freenet/src/freenet/support/compress/Bzip2Compressor.java
2008-11-08 00:14:36 UTC (rev 23400)
@@ -12,36 +12,38 @@
import freenet.support.Logger;
import freenet.support.api.Bucket;
import freenet.support.api.BucketFactory;
+import freenet.support.io.CountedOutputStream;
+
import org.apache.tools.bzip2.CBZip2InputStream;
import org.apache.tools.bzip2.CBZip2OutputStream;
public class Bzip2Compressor implements Compressor {
- public Bucket compress(Bucket data, BucketFactory bf, long maxLength)
throws IOException, CompressionOutputSizeException {
- if(maxLength <= 0)
+ public Bucket compress(Bucket data, BucketFactory bf, long
maxReadLength, long maxWriteLength) throws IOException,
CompressionOutputSizeException {
+ if(maxReadLength <= 0)
throw new IllegalArgumentException();
- Bucket output = bf.makeBucket(maxLength);
+ Bucket output = bf.makeBucket(maxWriteLength);
InputStream is = null;
OutputStream os = null;
CBZip2OutputStream bz2os = null;
try {
is = data.getInputStream();
os = output.getOutputStream();
- bz2os = new CBZip2OutputStream(os);
- long written = 0;
+ CountedOutputStream cos = new CountedOutputStream(os);
+ bz2os = new CBZip2OutputStream(cos);
+ long read = 0;
// Bigger input buffer, so can compress all at once.
// Won't hurt on I/O either, although most OSs will
only return a page at a time.
byte[] buffer = new byte[32768];
while(true) {
- int l = (int) Math.min(buffer.length, maxLength
- written);
- int x = is.read(buffer, 0, buffer.length);
- if(l < x) {
- throw new
CompressionOutputSizeException();
- }
+ int l = (int) Math.min(buffer.length,
maxReadLength - read);
+ int x = l == 0 ? -1 : is.read(buffer, 0,
buffer.length);
if(x <= -1) break;
if(x == 0) throw new IOException("Returned zero
from read()");
bz2os.write(buffer, 0, x);
- written += x;
+ read += x;
+ if(cos.written() > maxWriteLength)
+ throw new
CompressionOutputSizeException();
}
bz2os.flush();
bz2os.close();
Modified: trunk/freenet/src/freenet/support/compress/Compressor.java
===================================================================
--- trunk/freenet/src/freenet/support/compress/Compressor.java 2008-11-08
00:12:08 UTC (rev 23399)
+++ trunk/freenet/src/freenet/support/compress/Compressor.java 2008-11-08
00:14:36 UTC (rev 23400)
@@ -41,8 +41,8 @@
return null;
}
- public Bucket compress(Bucket data, BucketFactory bf, long
maxLength) throws IOException, CompressionOutputSizeException {
- return compressor.compress(data, bf, maxLength);
+ public Bucket compress(Bucket data, BucketFactory bf, long
maxReadLength, long maxWriteLength) throws IOException,
CompressionOutputSizeException {
+ return compressor.compress(data, bf, maxReadLength,
maxWriteLength);
}
public Bucket decompress(Bucket data, BucketFactory
bucketFactory, long maxLength, long maxEstimateSizeLength, Bucket preferred)
throws IOException, CompressionOutputSizeException {
@@ -81,7 +81,17 @@
}
}
- public abstract Bucket compress(Bucket data, BucketFactory bf, long
maxLength) throws IOException, CompressionOutputSizeException;
+ /**
+ * Compress the data.
+ * @param data The bucket to read from.
+ * @param bf The means to create a new bucket.
+ * @param maxReadLength The maximum number of bytes to read from the
input bucket.
+ * @param maxWriteLength The maximum number of bytes to write to the
output bucket. If this is exceeded, throw a CompressionOutputSizeException.
+ * @return The compressed data.
+ * @throws IOException If an error occurs reading or writing data.
+ * @throws CompressionOutputSizeException If the compressed data is
larger than maxWriteLength.
+ */
+ public abstract Bucket compress(Bucket data, BucketFactory bf, long
maxReadLength, long maxWriteLength) throws IOException,
CompressionOutputSizeException;
/**
* Decompress data.
Modified: trunk/freenet/src/freenet/support/compress/GzipCompressor.java
===================================================================
--- trunk/freenet/src/freenet/support/compress/GzipCompressor.java
2008-11-08 00:12:08 UTC (rev 23399)
+++ trunk/freenet/src/freenet/support/compress/GzipCompressor.java
2008-11-08 00:14:36 UTC (rev 23400)
@@ -11,34 +11,35 @@
import freenet.support.Logger;
import freenet.support.api.Bucket;
import freenet.support.api.BucketFactory;
+import freenet.support.io.CountedOutputStream;
public class GzipCompressor implements Compressor {
- public Bucket compress(Bucket data, BucketFactory bf, long maxLength)
throws IOException, CompressionOutputSizeException {
- if(maxLength <= 0)
+ public Bucket compress(Bucket data, BucketFactory bf, long
maxReadLength, long maxWriteLength) throws IOException,
CompressionOutputSizeException {
+ if(maxReadLength < 0)
throw new IllegalArgumentException();
- Bucket output = bf.makeBucket(maxLength);
+ Bucket output = bf.makeBucket(maxWriteLength);
InputStream is = null;
OutputStream os = null;
GZIPOutputStream gos = null;
try {
is = data.getInputStream();
os = output.getOutputStream();
- gos = new GZIPOutputStream(os);
- long written = 0;
+ CountedOutputStream cos = new CountedOutputStream(os);
+ gos = new GZIPOutputStream(cos);
+ long read = 0;
// Bigger input buffer, so can compress all at once.
// Won't hurt on I/O either, although most OSs will
only return a page at a time.
byte[] buffer = new byte[32768];
while(true) {
- int l = (int) Math.min(buffer.length, maxLength
- written);
- int x = is.read(buffer, 0, buffer.length);
- if(l < x) {
- throw new
CompressionOutputSizeException();
- }
+ int l = (int) Math.min(buffer.length,
maxReadLength - read);
+ int x = l == 0 ? -1 : is.read(buffer, 0, l);
if(x <= -1) break;
if(x == 0) throw new IOException("Returned zero
from read()");
gos.write(buffer, 0, x);
- written += x;
+ read += x;
+ if(cos.written() > maxWriteLength)
+ throw new
CompressionOutputSizeException();
}
gos.flush();
} finally {
Modified: trunk/freenet/src/freenet/support/compress/LZMACompressor.java
===================================================================
--- trunk/freenet/src/freenet/support/compress/LZMACompressor.java
2008-11-08 00:12:08 UTC (rev 23399)
+++ trunk/freenet/src/freenet/support/compress/LZMACompressor.java
2008-11-08 00:14:36 UTC (rev 23400)
@@ -12,36 +12,37 @@
import freenet.support.Logger;
import freenet.support.api.Bucket;
import freenet.support.api.BucketFactory;
+import freenet.support.io.CountedOutputStream;
import net.contrapunctus.lzma.LzmaInputStream;
import net.contrapunctus.lzma.LzmaOutputStream;
public class LZMACompressor implements Compressor {
- public Bucket compress(Bucket data, BucketFactory bf, long maxLength)
throws IOException, CompressionOutputSizeException {
- if(maxLength <= 0)
+ public Bucket compress(Bucket data, BucketFactory bf, long
maxReadLength, long maxWriteLength) throws IOException,
CompressionOutputSizeException {
+ if(maxReadLength <= 0)
throw new IllegalArgumentException();
- Bucket output = bf.makeBucket(maxLength);
+ Bucket output = bf.makeBucket(maxWriteLength);
InputStream is = null;
OutputStream os = null;
LzmaOutputStream lzmaOS = null;
try {
is = data.getInputStream();
os = output.getOutputStream();
- lzmaOS = new LzmaOutputStream(os);
- long written = 0;
+ CountedOutputStream cos = new CountedOutputStream(os);
+ lzmaOS = new LzmaOutputStream(cos);
+ long read = 0;
// Bigger input buffer, so can compress all at once.
// Won't hurt on I/O either, although most OSs will
only return a page at a time.
byte[] buffer = new byte[32768];
while(true) {
- int l = (int) Math.min(buffer.length, maxLength
- written);
- int x = is.read(buffer, 0, buffer.length);
- if(l < x) {
- throw new
CompressionOutputSizeException();
- }
+ int l = (int) Math.min(buffer.length,
maxReadLength - read);
+ int x = l == 0 ? -1 : is.read(buffer, 0,
buffer.length);
if(x <= -1) break;
if(x == 0) throw new IOException("Returned zero
from read()");
lzmaOS.write(buffer, 0, x);
- written += x;
+ read += x;
+ if(cos.written() > maxWriteLength)
+ throw new
CompressionOutputSizeException();
}
lzmaOS.flush();
os = null;
