Repository: commons-compress Updated Branches: refs/heads/master 043f42b65 -> ad661a8d3
allow LZ77 tuning in framed snappy/lz4 streams as well Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/ad661a8d Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/ad661a8d Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/ad661a8d Branch: refs/heads/master Commit: ad661a8d347c1c0ff1b9264cf0f8c89de7057a69 Parents: 043f42b Author: Stefan Bodewig <bode...@apache.org> Authored: Sun Mar 26 18:55:34 2017 +0200 Committer: Stefan Bodewig <bode...@apache.org> Committed: Sun Mar 26 18:55:34 2017 +0200 ---------------------------------------------------------------------- .../lz4/FramedLZ4CompressorOutputStream.java | 36 +++++++++++++++++++- .../FramedSnappyCompressorOutputStream.java | 16 ++++++++- .../lz4/FramedLZ4CompressorRoundtripTest.java | 4 +++ 3 files changed, 54 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/commons-compress/blob/ad661a8d/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStream.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStream.java b/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStream.java index 3ebcaa0..c08914f 100644 --- a/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStream.java +++ b/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStream.java @@ -88,6 +88,7 @@ public class FramedLZ4CompressorOutputStream extends CompressorOutputStream { public static class Parameters { private final BlockSize blockSize; private final boolean withContentChecksum, withBlockChecksum, withBlockDependency; + private final org.apache.commons.compress.compressors.lz77support.Parameters lz77params; /** * The default parameters of 4M block size, enabled content @@ -107,6 +108,16 @@ public class FramedLZ4CompressorOutputStream extends CompressorOutputStream { this(blockSize, true, false, false); } /** + * Sets up custom a custom block size for the LZ4 stream but + * otherwise uses the defaults of enabled content checksum, + * disabled block checksums and independent blocks. + * @param blockSize the size of a single block. + */ + public Parameters(BlockSize blockSize, + org.apache.commons.compress.compressors.lz77support.Parameters lz77params) { + this(blockSize, true, false, false, lz77params); + } + /** * Sets up custom parameters for the LZ4 stream. * @param blockSize the size of a single block. * @param withContentChecksum whether to write a content checksum @@ -120,10 +131,33 @@ public class FramedLZ4CompressorOutputStream extends CompressorOutputStream { */ public Parameters(BlockSize blockSize, boolean withContentChecksum, boolean withBlockChecksum, boolean withBlockDependency) { + this(blockSize, withContentChecksum, withBlockChecksum, withBlockDependency, + BlockLZ4CompressorOutputStream.createParameterBuilder().build()); + } + + /** + * Sets up custom parameters for the LZ4 stream. + * @param blockSize the size of a single block. + * @param withContentChecksum whether to write a content checksum + * @param withBlockChecksum whether to write a block checksum. + * Note that block checksums are not supported by the lz4 + * command line utility + * @param withBlockDependency whether a block may depend on + * the content of a previous block. Enabling this may improve + * compression ratio but makes it impossible to decompress the + * output in parallel. + * @param lz77 parameters parameters used to fine-tune + * compression, in particular to balance compression ration vs + * compression speed. + */ + public Parameters(BlockSize blockSize, boolean withContentChecksum, boolean withBlockChecksum, + boolean withBlockDependency, + org.apache.commons.compress.compressors.lz77support.Parameters lz77params) { this.blockSize = blockSize; this.withContentChecksum = withContentChecksum; this.withBlockChecksum = withBlockChecksum; this.withBlockDependency = withBlockDependency; + this.lz77params = lz77params; } @Override @@ -231,7 +265,7 @@ public class FramedLZ4CompressorOutputStream extends CompressorOutputStream { private void flushBlock() throws IOException { final boolean withBlockDependency = params.withBlockDependency; ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (BlockLZ4CompressorOutputStream o = new BlockLZ4CompressorOutputStream(baos)) { + try (BlockLZ4CompressorOutputStream o = new BlockLZ4CompressorOutputStream(baos, params.lz77params)) { if (withBlockDependency) { o.prefill(blockDependencyBuffer, blockDependencyBuffer.length - collectedBlockDependencyBytes, collectedBlockDependencyBytes); http://git-wip-us.apache.org/repos/asf/commons-compress/blob/ad661a8d/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorOutputStream.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorOutputStream.java b/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorOutputStream.java index e4d2a5e..fdb5699 100644 --- a/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorOutputStream.java +++ b/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorOutputStream.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.io.OutputStream; import org.apache.commons.compress.compressors.CompressorOutputStream; +import org.apache.commons.compress.compressors.lz77support.Parameters; import org.apache.commons.compress.utils.ByteUtils; /** @@ -42,6 +43,7 @@ public class FramedSnappyCompressorOutputStream extends CompressorOutputStream { private static final int MAX_COMPRESSED_BUFFER_SIZE = 1 << 16; private final OutputStream out; + private final Parameters params; private final PureJavaCrc32C checksum = new PureJavaCrc32C(); // used in one-arg write method private final byte[] oneByte = new byte[1]; @@ -57,7 +59,19 @@ public class FramedSnappyCompressorOutputStream extends CompressorOutputStream { * @throws IOException if writing the signature fails */ public FramedSnappyCompressorOutputStream(final OutputStream out) throws IOException { + this(out, SnappyCompressorOutputStream.createParameterBuilder(SnappyCompressorInputStream.DEFAULT_BLOCK_SIZE) + .build()); + } + + /** + * Constructs a new output stream that compresses + * snappy-framed-compressed data to the specified output stream. + * @param out the OutputStream to which to write the compressed data + * @throws IOException if writing the signature fails + */ + public FramedSnappyCompressorOutputStream(final OutputStream out, Parameters params) throws IOException { this.out = out; + this.params = params; consumer = new ByteUtils.OutputStreamByteConsumer(out); out.write(FramedSnappyCompressorInputStream.SZ_SIGNATURE); } @@ -104,7 +118,7 @@ public class FramedSnappyCompressorOutputStream extends CompressorOutputStream { private void flushBuffer() throws IOException { out.write(FramedSnappyCompressorInputStream.COMPRESSED_CHUNK_TYPE); ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (OutputStream o = new SnappyCompressorOutputStream(baos, currentIndex)) { + try (OutputStream o = new SnappyCompressorOutputStream(baos, currentIndex, params)) { o.write(buffer, 0, currentIndex); } byte[] b = baos.toByteArray(); http://git-wip-us.apache.org/repos/asf/commons-compress/blob/ad661a8d/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorRoundtripTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorRoundtripTest.java b/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorRoundtripTest.java index d54fafe..ea63c1d 100644 --- a/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorRoundtripTest.java +++ b/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorRoundtripTest.java @@ -52,6 +52,10 @@ public final class FramedLZ4CompressorRoundtripTest extends AbstractTestCase { // small blocksize (so we get enough blocks) and enabled block dependency, otherwise defaults new Object[] { new FramedLZ4CompressorOutputStream.Parameters(FramedLZ4CompressorOutputStream.BlockSize.K64, true, false, true) }, + // default, tuned for speed + new Object[] { new FramedLZ4CompressorOutputStream.Parameters(FramedLZ4CompressorOutputStream.BlockSize.M4, + true, false, false, BlockLZ4CompressorOutputStream.createParameterBuilder() + .tunedForSpeed().build()) }, }); }
