add DEFLATE as compression format to play with
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/b74e5386 Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/b74e5386 Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/b74e5386 Branch: refs/heads/compress-2.0 Commit: b74e53866767f2323af1ac09681bf80318fe2c49 Parents: 27686ef Author: Stefan Bodewig <[email protected]> Authored: Sat Apr 29 19:54:35 2017 +0200 Committer: Stefan Bodewig <[email protected]> Committed: Sat Apr 29 19:54:35 2017 +0200 ---------------------------------------------------------------------- .../formats/deflate/DeflateCompressedInput.java | 92 +++++++++ .../deflate/DeflateCompressedOutput.java | 87 +++++++++ .../deflate/DeflateCompressionFormat.java | 89 +++++++++ .../deflate/DeflateCompressionFormatTest.java | 53 ++++++ .../formats/deflate/RoundTripTest.java | 185 +++++++++++++++++++ .../test-archives/default.tar.deflatez | Bin 0 -> 468 bytes 6 files changed, 506 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/commons-compress/blob/b74e5386/src/main/java/org/apache/commons/compress2/formats/deflate/DeflateCompressedInput.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/compress2/formats/deflate/DeflateCompressedInput.java b/src/main/java/org/apache/commons/compress2/formats/deflate/DeflateCompressedInput.java new file mode 100644 index 0000000..a7d9977 --- /dev/null +++ b/src/main/java/org/apache/commons/compress2/formats/deflate/DeflateCompressedInput.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress2.formats.deflate; + +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.channels.Channels; +import java.nio.channels.ReadableByteChannel; +import java.util.zip.Inflater; +import java.util.zip.InflaterInputStream; +import org.apache.commons.compress2.compressors.spi.AbstractCompressedInput; + +/** + * Input for DEFLATE compressed channels. + */ +public class DeflateCompressedInput extends AbstractCompressedInput { + private final InputStream in; + private final Inflater inflater; + private boolean nextCalled = false; + + public DeflateCompressedInput(ReadableByteChannel channel) { + in = Channels.newInputStream(channel); + inflater = new Inflater(false); + } + + @Override + public ReadableByteChannel next() throws IOException { + if (nextCalled) { + return null; + } + nextCalled = true; + return Channels.newChannel(new InflaterInputStream(new WrappedStream(in), inflater)); + } + + @Override + public void close() throws IOException { + try { + in.close(); + } finally { + inflater.end(); + } + } + + private class WrappedStream extends FilterInputStream { + private WrappedStream(InputStream i) { + super(i); + } + + @Override + public void close() { + inflater.reset(); + } + + @Override + public int read() throws IOException { + int r = super.read(); + count(r < 0 ? 0 : 1); + return r; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + int r = super.read(b, off, len); + count(r); + return r; + } + + @Override + public long skip(long n) throws IOException { + long r = super.skip(n); + count(r); + return r; + } + } +} http://git-wip-us.apache.org/repos/asf/commons-compress/blob/b74e5386/src/main/java/org/apache/commons/compress2/formats/deflate/DeflateCompressedOutput.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/compress2/formats/deflate/DeflateCompressedOutput.java b/src/main/java/org/apache/commons/compress2/formats/deflate/DeflateCompressedOutput.java new file mode 100644 index 0000000..f563213 --- /dev/null +++ b/src/main/java/org/apache/commons/compress2/formats/deflate/DeflateCompressedOutput.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress2.formats.deflate; + +import java.io.FilterOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.channels.Channels; +import java.nio.channels.WritableByteChannel; +import java.util.zip.Deflater; +import java.util.zip.DeflaterOutputStream; +import org.apache.commons.compress2.compressors.spi.AbstractCompressedOutput; + +/** + * Output for DEFLATE compressed channels. + */ +public class DeflateCompressedOutput extends AbstractCompressedOutput { + private WritableByteChannel channel; + private final DeflaterOutputStream out; + private final Deflater deflater; + + public DeflateCompressedOutput(WritableByteChannel channel) { + this.channel = channel; + deflater = new Deflater(Deflater.DEFAULT_COMPRESSION, false); + out = new DeflaterOutputStream(new WrappedStream(Channels.newOutputStream(channel)), deflater); + } + + @Override + public WritableByteChannel startCompressing() { + return Channels.newChannel(out); + } + + @Override + public void finish() throws IOException { + out.finish(); + } + + @Override + public void close() throws IOException { + try { + out.close(); + } finally { + try { + deflater.end(); + } finally { + channel.close(); + } + } + } + + private class WrappedStream extends FilterOutputStream { + private WrappedStream(OutputStream o) { + super(o); + } + + @Override + public void close() { } + + @Override + public void write(int b) throws IOException { + super.write(b); + count(1); + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + super.write(b, off, len); + count(len); + } + } +} http://git-wip-us.apache.org/repos/asf/commons-compress/blob/b74e5386/src/main/java/org/apache/commons/compress2/formats/deflate/DeflateCompressionFormat.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/compress2/formats/deflate/DeflateCompressionFormat.java b/src/main/java/org/apache/commons/compress2/formats/deflate/DeflateCompressionFormat.java new file mode 100644 index 0000000..ee3bed3 --- /dev/null +++ b/src/main/java/org/apache/commons/compress2/formats/deflate/DeflateCompressionFormat.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress2.formats.deflate; + +import java.nio.ByteBuffer; +import java.nio.channels.ReadableByteChannel; +import java.nio.channels.WritableByteChannel; +import org.apache.commons.compress2.compressors.spi.AbstractCompressionFormat; + +/** + * Format descriptor for the GZIP format. + */ +public class DeflateCompressionFormat extends AbstractCompressionFormat { + private static final int MAGIC_1 = 0x78; + private static final int MAGIC_2a = 0x01; + private static final int MAGIC_2b = 0x5e; + private static final int MAGIC_2c = 0x9c; + private static final int MAGIC_2d = 0xda; + + /** + * "DEFLATE" + */ + public static final String DEFLATE_FORMAT_NAME = "DEFLATE"; + + /** + * "DEFLATE" + */ + @Override + public String getName() { + return DEFLATE_FORMAT_NAME; + } + + /** + * Yes. + */ + @Override + public boolean supportsWriting() { return true; } + + /** + * Yes. + */ + @Override + public boolean supportsAutoDetection() { return true; } + + /** + * @return 2 + */ + @Override + public int getNumberOfBytesRequiredForAutodetection() { + return 2; + } + + @Override + public boolean matches(ByteBuffer probe) { + byte[] sig = new byte[2]; + probe.get(sig); + return sig[0] == MAGIC_1 && ( + sig[1] == (byte) MAGIC_2a || + sig[1] == (byte) MAGIC_2b || + sig[1] == (byte) MAGIC_2c || + sig[1] == (byte) MAGIC_2d); + } + + @Override + public DeflateCompressedInput readFrom(ReadableByteChannel channel) { + return new DeflateCompressedInput(channel); + } + + @Override + public DeflateCompressedOutput writeTo(WritableByteChannel channel) { + return new DeflateCompressedOutput(channel); + } +} http://git-wip-us.apache.org/repos/asf/commons-compress/blob/b74e5386/src/test/java/org/apache/commons/compress2/formats/deflate/DeflateCompressionFormatTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/commons/compress2/formats/deflate/DeflateCompressionFormatTest.java b/src/test/java/org/apache/commons/compress2/formats/deflate/DeflateCompressionFormatTest.java new file mode 100644 index 0000000..837f31d --- /dev/null +++ b/src/test/java/org/apache/commons/compress2/formats/deflate/DeflateCompressionFormatTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress2.formats.deflate; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import org.apache.commons.compress2.util.IOUtils; +import org.junit.Assert; +import org.junit.Test; + +public class DeflateCompressionFormatTest { + + @Test + public void shouldDetectFormat() throws IOException { + Assert.assertTrue(isAr("test-archives/default.tar.deflatez")); + } + + @Test + public void shouldRejectXMLFile() throws IOException { + Assert.assertFalse(isAr("test1.xml")); + } + + + private boolean isAr(String file) throws IOException { + File f = RoundTripTest.getFile(file); + FileInputStream c = new FileInputStream(f); + try { + byte[] b = new byte[10]; + IOUtils.readFully(c, b); + return new DeflateCompressionFormat().matches(ByteBuffer.wrap(b)); + } finally { + c.close(); + } + } +} http://git-wip-us.apache.org/repos/asf/commons-compress/blob/b74e5386/src/test/java/org/apache/commons/compress2/formats/deflate/RoundTripTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/commons/compress2/formats/deflate/RoundTripTest.java b/src/test/java/org/apache/commons/compress2/formats/deflate/RoundTripTest.java new file mode 100644 index 0000000..72f36d4 --- /dev/null +++ b/src/test/java/org/apache/commons/compress2/formats/deflate/RoundTripTest.java @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress2.formats.deflate; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.net.URI; +import java.net.URL; +import java.nio.channels.ReadableByteChannel; +import java.nio.channels.WritableByteChannel; +import java.util.Locale; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import org.apache.commons.compress2.compressors.CompressedInput; +import org.apache.commons.compress2.compressors.CompressedOutput; +import org.apache.commons.compress2.util.IOUtils; + +public class RoundTripTest { + + private File dir; + + @Before + public void createTempDir() throws Exception { + dir = mkdir("dir"); + } + + @After + public void removeTempDir() throws Exception { + rmdir(dir); + } + + @Test + public void testRoundtripUsingConstructors() throws Exception { + final File output = new File(dir, "constructors.def"); + final File file1 = getFile("test1.xml"); + try (WritableByteChannel out = new FileOutputStream(output).getChannel(); + DeflateCompressedOutput os = new DeflateCompressedOutput(out); + WritableByteChannel c = os.startCompressing(); + ReadableByteChannel in = new FileInputStream(file1).getChannel()) { + IOUtils.copy(in, c); + } + + final File input = output; + final File target = new File(dir, "test1.xml"); + try (ReadableByteChannel is = new FileInputStream(input).getChannel(); + DeflateCompressedInput in = new DeflateCompressedInput(is); + ReadableByteChannel r = in.next(); + WritableByteChannel out = new FileOutputStream(target).getChannel()) { + IOUtils.copy(r, out); + } + } + + @Test + public void testRoundtripUsingFormatInstanceAndChannels() throws Exception { + DeflateCompressionFormat format = new DeflateCompressionFormat(); + final File output = new File(dir, "format-channels.def"); + final File file1 = getFile("test1.xml"); + try (WritableByteChannel out = new FileOutputStream(output).getChannel(); + DeflateCompressedOutput os = format.writeTo(out); + WritableByteChannel c = os.startCompressing(); + ReadableByteChannel in = new FileInputStream(file1).getChannel()) { + IOUtils.copy(in, c); + } + + final File input = output; + final File target = new File(dir, "test1.xml"); + try (ReadableByteChannel is = new FileInputStream(input).getChannel(); + DeflateCompressedInput in = format.readFrom(is); + ReadableByteChannel r = in.next(); + WritableByteChannel out = new FileOutputStream(target).getChannel()) { + IOUtils.copy(r, out); + } + } + + @Test + public void testRoundtripUsingFormatInstanceAndPaths() throws Exception { + DeflateCompressionFormat format = new DeflateCompressionFormat(); + final File output = new File(dir, "format-files.def"); + final File file1 = getFile("test1.xml"); + try (CompressedOutput os = format.writeTo(output.toPath()); + WritableByteChannel c = os.startCompressing(); + ReadableByteChannel in = new FileInputStream(file1).getChannel()) { + IOUtils.copy(in, c); + } + + final File input = output; + final File target = new File(dir, "test1.xml"); + try (CompressedInput in = format.readFrom(input.toPath()); + ReadableByteChannel r = in.next(); + WritableByteChannel out = new FileOutputStream(target).getChannel()) { + IOUtils.copy(r, out); + } + } + + public static File mkdir(String name) throws IOException { + File f = File.createTempFile(name, ""); + f.delete(); + f.mkdir(); + return f; + } + + public static File getFile(String path) throws IOException { + URL url = RoundTripTest.class.getClassLoader().getResource(path); + if (url == null) { + throw new FileNotFoundException("couldn't find " + path); + } + URI uri = null; + try { + uri = url.toURI(); + } catch (java.net.URISyntaxException ex) { + throw new IOException(ex); + } + return new File(uri); + } + + public static void rmdir(File f) { + String[] s = f.list(); + if (s != null) { + for (String element : s) { + final File file = new File(f, element); + if (file.isDirectory()){ + rmdir(file); + } + boolean ok = tryHardToDelete(file); + if (!ok && file.exists()){ + System.out.println("Failed to delete "+element+" in "+f.getPath()); + } + } + } + tryHardToDelete(f); // safer to delete and check + if (f.exists()){ + throw new Error("Failed to delete "+f.getPath()); + } + } + + private static final boolean ON_WINDOWS = + System.getProperty("os.name").toLowerCase(Locale.ENGLISH) + .indexOf("windows") > -1; + + /** + * Accommodate Windows bug encountered in both Sun and IBM JDKs. + * Others possible. If the delete does not work, call System.gc(), + * wait a little and try again. + * + * @return whether deletion was successful + * @since Stolen from FileUtils in Ant 1.8.0 + */ + public static boolean tryHardToDelete(File f) { + if (f != null && f.exists() && !f.delete()) { + if (ON_WINDOWS) { + System.gc(); + } + try { + Thread.sleep(10); + } catch (InterruptedException ex) { + // Ignore Exception + } + return f.delete(); + } + return true; + } +} http://git-wip-us.apache.org/repos/asf/commons-compress/blob/b74e5386/src/test/resources/test-archives/default.tar.deflatez ---------------------------------------------------------------------- diff --git a/src/test/resources/test-archives/default.tar.deflatez b/src/test/resources/test-archives/default.tar.deflatez new file mode 100644 index 0000000..32ebe77 Binary files /dev/null and b/src/test/resources/test-archives/default.tar.deflatez differ
