add write support for Zstandard
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/34b75c6f Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/34b75c6f Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/34b75c6f Branch: refs/heads/master Commit: 34b75c6f0ac96df0221bfbd11cafdc62dc0baa79 Parents: 9190a2d Author: Stefan Bodewig <[email protected]> Authored: Thu Dec 28 14:20:16 2017 +0100 Committer: Stefan Bodewig <[email protected]> Committed: Thu Dec 28 14:20:16 2017 +0100 ---------------------------------------------------------------------- src/changes/changes.xml | 3 + .../compressors/CompressorStreamFactory.java | 11 ++- .../zstandard/ZstdCompressorOutputStream.java | 65 +++++++++++++++++ .../compress/compressors/zstandard/package.html | 2 +- src/site/xdoc/examples.xml | 18 +++++ src/site/xdoc/index.xml | 6 +- src/site/xdoc/limitations.xml | 1 - .../zstandard/ZstdRoundtripTest.java | 76 ++++++++++++++++++++ 8 files changed, 175 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/commons-compress/blob/34b75c6f/src/changes/changes.xml ---------------------------------------------------------------------- diff --git a/src/changes/changes.xml b/src/changes/changes.xml index 6b4f1d2..9662b41 100644 --- a/src/changes/changes.xml +++ b/src/changes/changes.xml @@ -64,6 +64,9 @@ The <action> type attribute can be add,update,fix,remove. due-to="BELUGA BEHR"> Replaces instanceof checks with a type marker in LZ77 support code. </action> + <action issue="COMPRESS-426" type="add" date="2017-12-28"> + Added write-support for Zstandard compression. + </action> </release> <release version="1.15" date="2017-10-17" description="Release 1.15 http://git-wip-us.apache.org/repos/asf/commons-compress/blob/34b75c6f/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java b/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java index 234bf5e..0f03d30 100644 --- a/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java +++ b/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java @@ -56,6 +56,7 @@ import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream; import org.apache.commons.compress.compressors.xz.XZUtils; import org.apache.commons.compress.compressors.z.ZCompressorInputStream; import org.apache.commons.compress.compressors.zstandard.ZstdCompressorInputStream; +import org.apache.commons.compress.compressors.zstandard.ZstdCompressorOutputStream; import org.apache.commons.compress.compressors.zstandard.ZstdUtils; import org.apache.commons.compress.utils.IOUtils; import org.apache.commons.compress.utils.Lists; @@ -633,7 +634,7 @@ public class CompressorStreamFactory implements CompressorStreamProvider { * @param name * the compressor name, i.e. {@value #GZIP}, {@value #BZIP2}, * {@value #XZ}, {@value #PACK200}, {@value #SNAPPY_FRAMED}, - * {@value #LZ4_BLOCK}, {@value #LZ4_FRAMED} + * {@value #LZ4_BLOCK}, {@value #LZ4_FRAMED}, {@value #ZSTANDARD} * or {@value #DEFLATE} * @param out * the output stream @@ -688,6 +689,12 @@ public class CompressorStreamFactory implements CompressorStreamProvider { return new FramedLZ4CompressorOutputStream(out); } + if (ZSTANDARD.equalsIgnoreCase(name)) { + if (!ZstdUtils.isZstdCompressionAvailable()) { + throw new CompressorException("Zstandard compression is not available."); + } + return new ZstdCompressorOutputStream(out); + } } catch (final IOException e) { throw new CompressorException("Could not create CompressorOutputStream", e); } @@ -731,7 +738,7 @@ public class CompressorStreamFactory implements CompressorStreamProvider { @Override public Set<String> getOutputStreamCompressorNames() { - return Sets.newHashSet(GZIP, BZIP2, XZ, LZMA, PACK200, DEFLATE, SNAPPY_FRAMED, LZ4_BLOCK, LZ4_FRAMED); + return Sets.newHashSet(GZIP, BZIP2, XZ, LZMA, PACK200, DEFLATE, SNAPPY_FRAMED, LZ4_BLOCK, LZ4_FRAMED, ZSTANDARD); } /** http://git-wip-us.apache.org/repos/asf/commons-compress/blob/34b75c6f/src/main/java/org/apache/commons/compress/compressors/zstandard/ZstdCompressorOutputStream.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/compress/compressors/zstandard/ZstdCompressorOutputStream.java b/src/main/java/org/apache/commons/compress/compressors/zstandard/ZstdCompressorOutputStream.java new file mode 100644 index 0000000..f4c7dbe --- /dev/null +++ b/src/main/java/org/apache/commons/compress/compressors/zstandard/ZstdCompressorOutputStream.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.commons.compress.compressors.zstandard; + + +import java.io.IOException; +import java.io.OutputStream; + +import com.github.luben.zstd.ZstdOutputStream; +import org.apache.commons.compress.compressors.CompressorOutputStream; + +/** + * {@link CompressorOutputStream} implementation to create Zstandard encoded stream. + * Library relies on <a href="https://github.com/luben/zstd-jni/";>Zstandard JNI</a> + * + * @since 1.16 + */ +public class ZstdCompressorOutputStream extends CompressorOutputStream { + + private final com.github.luben.zstd.ZstdOutputStream encOS; + + public ZstdCompressorOutputStream(final OutputStream out) throws IOException { + this.encOS = new ZstdOutputStream(out); + } + + @Override + public void close() throws IOException { + encOS.close(); + } + + @Override + public void write(final int b) throws IOException { + encOS.write(b); + } + + @Override + public void write(final byte[] buf, final int off, final int len) throws IOException { + encOS.write(buf, off, len); + } + + @Override + public String toString() { + return encOS.toString(); + } + + @Override + public void flush() throws IOException { + encOS.flush(); + } +} http://git-wip-us.apache.org/repos/asf/commons-compress/blob/34b75c6f/src/main/java/org/apache/commons/compress/compressors/zstandard/package.html ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/compress/compressors/zstandard/package.html b/src/main/java/org/apache/commons/compress/compressors/zstandard/package.html index 1c105ab..6deb74f 100644 --- a/src/main/java/org/apache/commons/compress/compressors/zstandard/package.html +++ b/src/main/java/org/apache/commons/compress/compressors/zstandard/package.html @@ -18,7 +18,7 @@ --> <body> - <p>Provides stream class for decompressing streams using the + <p>Provides stream class for (de)compressing streams using the Zstandard algorithm based on <a href="https://github.com/luben/zstd-jni";>Zstandard JNI</a>.</p> http://git-wip-us.apache.org/repos/asf/commons-compress/blob/34b75c6f/src/site/xdoc/examples.xml ---------------------------------------------------------------------- diff --git a/src/site/xdoc/examples.xml b/src/site/xdoc/examples.xml index 83b1b7c..b6997b3 100644 --- a/src/site/xdoc/examples.xml +++ b/src/site/xdoc/examples.xml @@ -902,6 +902,24 @@ while (-1 != (n = zsIn.read(buffer))) { out.close(); zsIn.close(); ]]></source> + + <p>Compressing a given file using the Zstandard format (you + would certainly add exception handling and make sure all + streams get closed properly):</p> +<source><![CDATA[ +InputStream in = Files.newInputStream(Paths.get("archive.tar")); +OutputStream fout = Files.newOutputStream(Paths.get("archive.tar.zstd")); +BufferedOutputStream out = new BufferedInputStream(fout); +ZstdCompressorOutputStream zOut = new ZstdCompressorOutputStream(out); +final byte[] buffer = new byte[buffersize]; +int n = 0; +while (-1 != (n = in.read(buffer))) { + zOut.write(buffer, 0, n); +} +zOut.close(); +in.close(); +]]></source> + </subsection> <subsection name="Extending Commons Compress"> http://git-wip-us.apache.org/repos/asf/commons-compress/blob/34b75c6f/src/site/xdoc/index.xml ---------------------------------------------------------------------- diff --git a/src/site/xdoc/index.xml b/src/site/xdoc/index.xml index 6218475..ea32924 100644 --- a/src/site/xdoc/index.xml +++ b/src/site/xdoc/index.xml @@ -68,7 +68,7 @@ <subsection name="What's coming in 1.16?"> <ul> - <li>Read-only support for Zstandard compression.</li> + <li>Support for Zstandard compression.</li> </ul> </subsection> @@ -94,8 +94,8 @@ licensed <a href="https://github.com/google/brotli";>Google Brotli decoder</a>. Zstandard support is provided by the BSD licensed <a href="https://github.com/luben/zstd-jni";>Zstd-jni</a>. - As of Commons Compress 1.16 support for the Z, Zstandard - and Brotli formats is read-only.</p> + As of Commons Compress 1.16 support for the Z and Brotli + formats is read-only.</p> <p>The ar, arj, cpio, dump, tar, 7z and zip formats are supported as archivers where the <a href="zip.html">zip</a> http://git-wip-us.apache.org/repos/asf/commons-compress/blob/34b75c6f/src/site/xdoc/limitations.xml ---------------------------------------------------------------------- diff --git a/src/site/xdoc/limitations.xml b/src/site/xdoc/limitations.xml index d651204..e86b296 100644 --- a/src/site/xdoc/limitations.xml +++ b/src/site/xdoc/limitations.xml @@ -199,7 +199,6 @@ <li>the format requires the otherwise optional <a href="https://github.com/luben/zstd-jni";>Zstandard JNI</a> library.</li> - <li>read-only support</li> </ul> </section> </body> http://git-wip-us.apache.org/repos/asf/commons-compress/blob/34b75c6f/src/test/java/org/apache/commons/compress/compressors/zstandard/ZstdRoundtripTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/commons/compress/compressors/zstandard/ZstdRoundtripTest.java b/src/test/java/org/apache/commons/compress/compressors/zstandard/ZstdRoundtripTest.java new file mode 100644 index 0000000..3c1469b --- /dev/null +++ b/src/test/java/org/apache/commons/compress/compressors/zstandard/ZstdRoundtripTest.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.commons.compress.compressors.zstandard; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import org.apache.commons.compress.AbstractTestCase; +import org.apache.commons.compress.compressors.CompressorInputStream; +import org.apache.commons.compress.compressors.CompressorOutputStream; +import org.apache.commons.compress.compressors.CompressorStreamFactory; +import org.apache.commons.compress.utils.IOUtils; +import org.junit.Assert; +import org.junit.Test; + +public class ZstdRoundtripTest extends AbstractTestCase { + + @Test + public void directRoundtrip() throws Exception { + File input = getFile("bla.tar"); + long start = System.currentTimeMillis(); + final File output = new File(dir, input.getName() + ".zstd"); + try (FileInputStream is = new FileInputStream(input); + FileOutputStream os = new FileOutputStream(output); + ZstdCompressorOutputStream zos = new ZstdCompressorOutputStream(os)) { + IOUtils.copy(is, zos); + } + System.err.println(input.getName() + " written, uncompressed bytes: " + input.length() + + ", compressed bytes: " + output.length() + " after " + (System.currentTimeMillis() - start) + "ms"); + start = System.currentTimeMillis(); + try (FileInputStream is = new FileInputStream(input); + ZstdCompressorInputStream zis = new ZstdCompressorInputStream(new FileInputStream(output))) { + byte[] expected = IOUtils.toByteArray(is); + byte[] actual = IOUtils.toByteArray(zis); + Assert.assertArrayEquals(expected, actual); + } + System.err.println(output.getName() + " read after " + (System.currentTimeMillis() - start) + "ms"); + } + + @Test + public void factoryRoundtrip() throws Exception { + File input = getFile("bla.tar"); + long start = System.currentTimeMillis(); + final File output = new File(dir, input.getName() + ".zstd"); + try (FileInputStream is = new FileInputStream(input); + FileOutputStream os = new FileOutputStream(output); + CompressorOutputStream zos = new CompressorStreamFactory().createCompressorOutputStream("zstd", os)) { + IOUtils.copy(is, zos); + } + start = System.currentTimeMillis(); + try (FileInputStream is = new FileInputStream(input); + CompressorInputStream zis = new CompressorStreamFactory() + .createCompressorInputStream("zstd", new FileInputStream(output))) { + byte[] expected = IOUtils.toByteArray(is); + byte[] actual = IOUtils.toByteArray(zis); + Assert.assertArrayEquals(expected, actual); + } + } + +}
