Author: nextgens
Date: 2008-09-29 18:14:20 +0000 (Mon, 29 Sep 2008)
New Revision: 22883
Added:
trunk/freenet/src/freenet/support/compress/Bzip2Compressor.java
trunk/freenet/src/org/apache/
trunk/freenet/src/org/apache/tools/
trunk/freenet/test/freenet/support/compress/Bzip2CompressorTest.java
Modified:
trunk/freenet/src/org/
Log:
solves issue #70: Add transparent bzip2 support
Added: trunk/freenet/src/freenet/support/compress/Bzip2Compressor.java
===================================================================
--- trunk/freenet/src/freenet/support/compress/Bzip2Compressor.java
(rev 0)
+++ trunk/freenet/src/freenet/support/compress/Bzip2Compressor.java
2008-09-29 18:14:20 UTC (rev 22883)
@@ -0,0 +1,120 @@
+/* This code is part of Freenet. It is distributed under the GNU General
+* Public License, version 2 (or at your option any later version). See
+* http://www.gnu.org/ for further details of the GPL. */
+package freenet.support.compress;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+import freenet.support.Logger;
+import freenet.support.api.Bucket;
+import freenet.support.api.BucketFactory;
+import org.apache.tools.bzip2.CBZip2InputStream;
+import org.apache.tools.bzip2.CBZip2OutputStream;
+
+public class Bzip2Compressor implements Compressor {
+
+ @Override
+ public Bucket compress(Bucket data, BucketFactory bf, long maxLength)
throws IOException, CompressionOutputSizeException {
+ if(maxLength <= 0)
+ throw new IllegalArgumentException();
+ Bucket output = bf.makeBucket(maxLength);
+ InputStream is = null;
+ OutputStream os = null;
+ CBZip2OutputStream bz2os = null;
+ try {
+ is = data.getInputStream();
+ os = output.getOutputStream();
+ bz2os = new CBZip2OutputStream(os);
+ long written = 0;
+ // Bigger input buffer, so can compress all at once.
+ // Won't hurt on I/O either, although most OSs will
only return a page at a time.
+ byte[] buffer = new byte[32768];
+ while(true) {
+ int l = (int) Math.min(buffer.length, maxLength
- written);
+ int x = is.read(buffer, 0, buffer.length);
+ if(l < x) {
+ throw new
CompressionOutputSizeException();
+ }
+ if(x <= -1) break;
+ if(x == 0) throw new IOException("Returned zero
from read()");
+ bz2os.write(buffer, 0, x);
+ written += x;
+ }
+ bz2os.flush();
+ bz2os.close();
+ os = null;
+ } finally {
+ if(is != null) is.close();
+ if(os != null) os.close();
+ }
+ return output;
+ }
+
+ @Override
+ public Bucket decompress(Bucket data, BucketFactory bf, long maxLength,
long maxCheckSizeLength, Bucket preferred) throws IOException,
CompressionOutputSizeException {
+ Bucket output;
+ if(preferred != null)
+ output = preferred;
+ else
+ output = bf.makeBucket(maxLength);
+ InputStream is = data.getInputStream();
+ OutputStream os = output.getOutputStream();
+ decompress(is, os, maxLength, maxCheckSizeLength);
+ os.close();
+ is.close();
+ return output;
+ }
+
+ private long decompress(InputStream is, OutputStream os, long
maxLength, long maxCheckSizeBytes) throws IOException,
CompressionOutputSizeException {
+ CBZip2InputStream bz2is = new CBZip2InputStream(is);
+ long written = 0;
+ byte[] buffer = new byte[4096];
+ while(true) {
+ int l = (int) Math.min(buffer.length, maxLength -
written);
+ // We can over-read to determine whether we have
over-read.
+ // We enforce maximum size this way.
+ // FIXME there is probably a better way to do this!
+ int x = bz2is.read(buffer, 0, buffer.length);
+ if(l < x) {
+ Logger.normal(this, "l="+l+", x="+x+",
written="+written+", maxLength="+maxLength+" throwing a
CompressionOutputSizeException");
+ if(maxCheckSizeBytes > 0) {
+ written += x;
+ while(true) {
+ l = (int)
Math.min(buffer.length, maxLength + maxCheckSizeBytes - written);
+ x = bz2is.read(buffer, 0, l);
+ if(x <= -1) throw new
CompressionOutputSizeException(written);
+ if(x == 0) throw new
IOException("Returned zero from read()");
+ written += x;
+ }
+ }
+ throw new CompressionOutputSizeException();
+ }
+ if(x <= -1) return written;
+ if(x == 0) throw new IOException("Returned zero from
read()");
+ os.write(buffer, 0, x);
+ written += x;
+ }
+ }
+
+ @Override
+ public int decompress(byte[] dbuf, int i, int j, byte[] output) throws
CompressionOutputSizeException {
+ // Didn't work with Inflater.
+ // FIXME fix sometimes to use Inflater - format issue?
+ ByteArrayInputStream bais = new ByteArrayInputStream(dbuf, i,
j);
+ ByteArrayOutputStream baos = new
ByteArrayOutputStream(output.length);
+ int bytes = 0;
+ try {
+ bytes = (int)decompress(bais, baos, output.length, -1);
+ } catch (IOException e) {
+ // Impossible
+ throw new Error("Got IOException: " + e.getMessage(),
e);
+ }
+ byte[] buf = baos.toByteArray();
+ System.arraycopy(buf, 0, output, 0, bytes);
+ return bytes;
+ }
+}
Property changes on: trunk/freenet/src/org
___________________________________________________________________
Name: svn:externals
+ apache/tools/bzip2
http://svn.apache.org/repos/asf/ant/core/tags/ANT_171/src/main/org/apache/tools/bzip2/
Added: trunk/freenet/test/freenet/support/compress/Bzip2CompressorTest.java
===================================================================
--- trunk/freenet/test/freenet/support/compress/Bzip2CompressorTest.java
(rev 0)
+++ trunk/freenet/test/freenet/support/compress/Bzip2CompressorTest.java
2008-09-29 18:14:20 UTC (rev 22883)
@@ -0,0 +1,200 @@
+/* This code is part of Freenet. It is distributed under the GNU General
+* Public License, version 2 (or at your option any later version). See
+* http://www.gnu.org/ for further details of the GPL. */
+package freenet.support.compress;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+import junit.framework.TestCase;
+import freenet.support.api.Bucket;
+import freenet.support.api.BucketFactory;
+import freenet.support.io.ArrayBucket;
+import freenet.support.io.ArrayBucketFactory;
+
+/**
+ * Test case for {@link freenet.support.compress.Bzip2Compressor} class.
+ */
+public class Bzip2CompressorTest extends TestCase {
+
+ private static final String UNCOMPRESSED_DATA_1 =
GzipCompressorTest.UNCOMPRESSED_DATA_1;
+
+ private static final byte[] COMPRESSED_DATA_1 = {
+ 104,57,49,65,89,38,83,89,-18,-87,-99,-74,0,0,33,-39,-128,0,8,16,
+ 0,58,64,52,-7,-86,0,48,0,-69,65,76,38,-102,3,76,65,-92,-12,-43,
+ 61,71,-88,-51,35,76,37,52,32,19,-44,67,74,-46,-9,17,14,-35,55,
+ 100,-10,73,-75,121,-34,83,56,-125,15,32,-118,35,66,124,-120,-39,
+ 119,-104,-108,66,101,-56,94,-71,-41,-43,68,51,65,19,-44,-118,4,
+ -36,-117,33,-101,-120,-49,-10,17,-51,-19,28,76,-57,-112,-68,-50,
+ -66,-60,-43,-81,127,-51,-10,58,-92,38,18,45,102,117,-31,-116,
+ -114,-6,-87,-59,-43,-106,41,-30,-63,-34,-39,-117,-104,-114,100,
+ -115,36,-112,23,104,-110,71,-45,-116,-23,-85,-36,-24,-61,14,32,
+ 105,55,-105,-31,-4,93,-55,20,-31,66,67,-70,-90,118,-40
+ };
+
+ /**
+ * test BZIP2 compressor's identity and functionality
+ */
+ public void testBzip2Compressor() {
+ Compressor.COMPRESSOR_TYPE bz2compressor =
Compressor.COMPRESSOR_TYPE.BZIP2;
+ Compressor compressorZero =
Compressor.COMPRESSOR_TYPE.getCompressorByMetadataID((short)1);
+
+ // check BZIP2 is the second compressor
+ assertEquals(bz2compressor, compressorZero);
+ }
+
+ public void testCompress() {
+
+ // do bzip2 compression
+ byte[] compressedData =
doCompress(UNCOMPRESSED_DATA_1.getBytes());
+
+ // output size same as expected?
+ //assertEquals(compressedData.length, COMPRESSED_DATA_1.length);
+
+ // check each byte is exactly as expected
+ for (int i = 0; i < compressedData.length; i++) {
+ assertEquals(COMPRESSED_DATA_1[i], compressedData[i]);
+ }
+ }
+
+ public void testBucketDecompress() {
+
+ byte[] compressedData = COMPRESSED_DATA_1;
+
+ // do bzip2 decompression with buckets
+ byte[] uncompressedData = doBucketDecompress(compressedData);
+
+ // is the (round-tripped) uncompressed string the same as the
original?
+ String uncompressedString = new String(uncompressedData);
+ assertEquals(uncompressedString, UNCOMPRESSED_DATA_1);
+ }
+
+ public void testByteArrayDecompress() {
+
+ // build 5k array
+ byte[] originalUncompressedData = new byte[5 * 1024];
+ for(int i = 0; i < originalUncompressedData.length; i++) {
+ originalUncompressedData[i] = 1;
+ }
+
+ byte[] compressedData = doCompress(originalUncompressedData);
+ byte[] outUncompressedData = new byte[5 * 1024];
+
+ int writtenBytes = 0;
+
+ try {
+ writtenBytes =
Compressor.COMPRESSOR_TYPE.BZIP2.decompress(compressedData, 0,
compressedData.length, outUncompressedData);
+ } catch (CompressionOutputSizeException e) {
+ fail("unexpected exception thrown : " + e.getMessage());
+ }
+
+ assertEquals(writtenBytes, originalUncompressedData.length);
+ assertEquals(originalUncompressedData.length,
outUncompressedData.length);
+
+ // check each byte is exactly as expected
+ for (int i = 0; i < outUncompressedData.length; i++) {
+ assertEquals(originalUncompressedData[i],
outUncompressedData[i]);
+ }
+ }
+
+ public void testCompressException() {
+
+ byte[] uncompressedData = UNCOMPRESSED_DATA_1.getBytes();
+ Bucket inBucket = new ArrayBucket(uncompressedData);
+ BucketFactory factory = new ArrayBucketFactory();
+
+ try {
+ Compressor.COMPRESSOR_TYPE.BZIP2.compress(inBucket,
factory, 32);
+ } catch (IOException e) {
+ fail("unexpected exception thrown : " + e.getMessage());
+ } catch (CompressionOutputSizeException e) {
+ // expect this
+ }
+ }
+
+ public void testDecompressException() {
+
+ // build 5k array
+ byte[] uncompressedData = new byte[5 * 1024];
+ for(int i = 0; i < uncompressedData.length; i++) {
+ uncompressedData[i] = 1;
+ }
+
+ byte[] compressedData = doCompress(uncompressedData);
+
+ Bucket inBucket = new ArrayBucket(compressedData);
+ BucketFactory factory = new ArrayBucketFactory();
+
+ try {
+ Compressor.COMPRESSOR_TYPE.BZIP2.decompress(inBucket,
factory, 4096 + 10, 4096 + 20, null);
+ } catch (IOException e) {
+ fail("unexpected exception thrown : " + e.getMessage());
+ } catch (CompressionOutputSizeException e) {
+ // expect this
+ }
+ }
+
+ private byte[] doBucketDecompress(byte[] compressedData) {
+
+ Bucket inBucket = new ArrayBucket(compressedData);
+ BucketFactory factory = new ArrayBucketFactory();
+ Bucket outBucket = null;
+
+ try {
+ outBucket =
Compressor.COMPRESSOR_TYPE.BZIP2.decompress(inBucket, factory, 32768, 32768 *
2, null);
+ } catch (IOException e) {
+ fail("unexpected exception thrown : " + e.getMessage());
+ } catch (CompressionOutputSizeException e) {
+ fail("unexpected exception thrown : " + e.getMessage());
+ }
+
+ InputStream in = null;
+
+ try {
+ in = outBucket.getInputStream();
+ } catch (IOException e1) {
+ fail("unexpected exception thrown : " +
e1.getMessage());
+ }
+ long size = outBucket.size();
+ byte[] outBuf = new byte[(int) size];
+
+ try {
+ in.read(outBuf);
+ } catch (IOException e) {
+ fail("unexpected exception thrown : " + e.getMessage());
+ }
+
+ return outBuf;
+ }
+
+ private byte[] doCompress(byte[] uncompressedData) {
+ Bucket inBucket = new ArrayBucket(uncompressedData);
+ BucketFactory factory = new ArrayBucketFactory();
+ Bucket outBucket = null;
+
+ try {
+ outBucket =
Compressor.COMPRESSOR_TYPE.BZIP2.compress(inBucket, factory, 32768);
+ } catch (IOException e) {
+ fail("unexpected exception thrown : " + e.getMessage());
+ } catch (CompressionOutputSizeException e) {
+ fail("unexpected exception thrown : " + e.getMessage());
+ }
+
+ InputStream in = null;
+ try {
+ in = outBucket.getInputStream();
+ } catch (IOException e1) {
+ fail("unexpected exception thrown : " +
e1.getMessage());
+ }
+ long size = outBucket.size();
+ byte[] outBuf = new byte[(int) size];
+
+ try {
+ in.read(outBuf);
+ } catch (IOException e) {
+ fail("unexpected exception thrown : " + e.getMessage());
+ }
+
+ return outBuf;
+ }
+}
