This is an automated email from the ASF dual-hosted git repository. peterlee pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/commons-compress.git
commit f0aaab65408f53684efa9fc450819fff51c69436 Author: theobisproject <[email protected]> AuthorDate: Fri Nov 13 14:08:27 2020 +0100 COMPRESS-540: Fix problems with long filenames Includes fix for COMPRESS-558 If an archive contains many files with long names or a name which is larger than the buffer until know the name was wrong Add missing tests from TarArchiveInputStreamTest to TarFileTest --- .../commons/compress/archivers/tar/TarFile.java | 16 +- .../commons/compress/archivers/TarTestCase.java | 54 ++++- .../compress/archivers/tar/TarFileTest.java | 233 +++++++++++++++++++++ 3 files changed, 295 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/apache/commons/compress/archivers/tar/TarFile.java b/src/main/java/org/apache/commons/compress/archivers/tar/TarFile.java index d8e517d..f512d29 100644 --- a/src/main/java/org/apache/commons/compress/archivers/tar/TarFile.java +++ b/src/main/java/org/apache/commons/compress/archivers/tar/TarFile.java @@ -43,14 +43,14 @@ import org.apache.commons.compress.utils.BoundedSeekableByteChannelInputStream; import org.apache.commons.compress.utils.SeekableInMemoryByteChannel; /** - * The TarFile provides random access to UNIX to archives. + * The TarFile provides random access to UNIX archives. * @since 1.21 */ public class TarFile implements Closeable { private static final int SMALL_BUFFER_SIZE = 256; - private final ByteBuffer smallBuf = ByteBuffer.allocate(SMALL_BUFFER_SIZE); + private final byte[] smallBuf = new byte[SMALL_BUFFER_SIZE]; private final SeekableByteChannel archive; @@ -273,11 +273,11 @@ public class TarFile implements Closeable { } // COMPRESS-509 : the name of directories should end with '/' - String name = zipEncoding.decode(longNameData); + final String name = zipEncoding.decode(longNameData); + currEntry.setName(name); if (currEntry.isDirectory() && !name.endsWith("/")) { - name += "/"; + currEntry.setName(name + "/"); } - currEntry.setName(name); } if (currEntry.isGlobalPaxHeader()) { // Process Global Pax headers @@ -474,8 +474,10 @@ public class TarFile implements Closeable { private byte[] getLongNameData() throws IOException { final ByteArrayOutputStream longName = new ByteArrayOutputStream(); int length; - while ((length = archive.read(smallBuf)) > 0) { - longName.write(smallBuf.array(), 0, length); + try (final InputStream in = getInputStream(currEntry)) { + while ((length = in.read(smallBuf)) >= 0) { + longName.write(smallBuf, 0, length); + } } getNextTarEntry(); if (currEntry == null) { diff --git a/src/test/java/org/apache/commons/compress/archivers/TarTestCase.java b/src/test/java/org/apache/commons/compress/archivers/TarTestCase.java index f5dc104..e4fd699 100644 --- a/src/test/java/org/apache/commons/compress/archivers/TarTestCase.java +++ b/src/test/java/org/apache/commons/compress/archivers/TarTestCase.java @@ -18,14 +18,17 @@ */ package org.apache.commons.compress.archivers; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.util.List; import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; import org.apache.commons.compress.AbstractTestCase; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; @@ -534,4 +537,53 @@ public final class TarTestCase extends AbstractTestCase { } } } + + @Test + public void testLongNameLargerThanBuffer() throws IOException { + final List<Integer> nameLength = Arrays.asList(300, 4096); + + for (final Integer length : nameLength) { + final String fileName = createLongName(length); + assertEquals(length.intValue(), fileName.length()); + final byte[] data = createTarWithOneLongNameEntry(fileName); + try (final ByteArrayInputStream bis = new ByteArrayInputStream(data); + final TarArchiveInputStream tis = new TarArchiveInputStream(bis)) { + assertEquals(fileName, tis.getNextTarEntry().getName()); + } + } + } + + @Test + public void testTarFileLongNameLargerThanBuffer() throws IOException { + final List<Integer> nameLength = Arrays.asList(300, 4096); + + for (final Integer length : nameLength) { + final String fileName = createLongName(length); + assertEquals(length.intValue(), fileName.length()); + final byte[] data = createTarWithOneLongNameEntry(fileName); + try (final TarFile tarFile = new TarFile(data)) { + List<TarArchiveEntry> entries = tarFile.getEntries(); + assertEquals(fileName, entries.get(0).getName()); + } + } + } + + private String createLongName(final int nameLength) { + final StringBuffer buffer = new StringBuffer(); + for (int i = 0; i < nameLength; i++) { + buffer.append('a'); + } + return buffer.toString(); + } + + private byte[] createTarWithOneLongNameEntry(final String longName) throws IOException { + final ByteArrayOutputStream bos = new ByteArrayOutputStream(); + try (final TarArchiveOutputStream tos = new TarArchiveOutputStream(bos)) { + tos.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU); + TarArchiveEntry longFileNameEntry = new TarArchiveEntry(longName); + tos.putArchiveEntry(longFileNameEntry); + tos.closeArchiveEntry(); + } + return bos.toByteArray(); + } } diff --git a/src/test/java/org/apache/commons/compress/archivers/tar/TarFileTest.java b/src/test/java/org/apache/commons/compress/archivers/tar/TarFileTest.java index d953417..1360320 100644 --- a/src/test/java/org/apache/commons/compress/archivers/tar/TarFileTest.java +++ b/src/test/java/org/apache/commons/compress/archivers/tar/TarFileTest.java @@ -18,20 +18,221 @@ package org.apache.commons.compress.archivers.tar; import java.io.BufferedOutputStream; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Calendar; +import java.util.Date; +import java.util.List; +import java.util.TimeZone; +import java.util.zip.GZIPInputStream; import org.apache.commons.compress.AbstractTestCase; import org.apache.commons.compress.archivers.ArchiveException; +import org.apache.commons.compress.utils.CharsetNames; +import org.apache.commons.compress.utils.IOUtils; import org.junit.Test; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; public class TarFileTest extends AbstractTestCase { @Test + public void workaroundForBrokenTimeHeader() throws IOException { + try (final TarFile tarFile = new TarFile(getPath("simple-aix-native-tar.tar"))) { + final List<TarArchiveEntry> entries = tarFile.getEntries(); + assertEquals(3, entries.size()); + final TarArchiveEntry entry = entries.get(1); + assertEquals("sample/link-to-txt-file.lnk", entry.getName()); + assertEquals(new Date(0), entry.getLastModifiedDate()); + assertTrue(entry.isSymbolicLink()); + assertTrue(entry.isCheckSumOK()); + } + } + + @Test + public void datePriorToEpochInGNUFormat() throws Exception { + datePriorToEpoch("preepoch-star.tar"); + } + + @Test + public void datePriorToEpochInPAXFormat() throws Exception { + datePriorToEpoch("preepoch-posix.tar"); + } + + private void datePriorToEpoch(final String archive) throws Exception { + try (final TarFile tarFile = new TarFile(getPath(archive))) { + TarArchiveEntry entry = tarFile.getEntries().get(0); + assertEquals("foo", entry.getName()); + final Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("GMT")); + cal.set(1969, 11, 31, 23, 59, 59); + cal.set(Calendar.MILLISECOND, 0); + assertEquals(cal.getTime(), entry.getLastModifiedDate()); + assertTrue(entry.isCheckSumOK()); + } + } + + @Test + public void testCompress197() throws Exception { + try (final TarFile tarFile = new TarFile(getPath("COMPRESS-197.tar"))) { + } catch (final IOException e) { + fail("COMPRESS-197: " + e.getMessage()); + } + } + + @Test + public void shouldUseSpecifiedEncodingWhenReadingGNULongNames() + throws Exception { + final ByteArrayOutputStream bos = new ByteArrayOutputStream(); + final String encoding = CharsetNames.UTF_16; + final String name = "1234567890123456789012345678901234567890123456789" + + "01234567890123456789012345678901234567890123456789" + + "01234567890\u00e4"; + try (final TarArchiveOutputStream tos = new TarArchiveOutputStream(bos, encoding)) { + tos.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU); + TarArchiveEntry t = new TarArchiveEntry(name); + t.setSize(1); + tos.putArchiveEntry(t); + tos.write(30); + tos.closeArchiveEntry(); + } + final byte[] data = bos.toByteArray(); + try (final TarFile tarFile = new TarFile(data, encoding)) { + List<TarArchiveEntry> entries = tarFile.getEntries(); + assertEquals(1, entries.size()); + assertEquals(name, entries.get(0).getName()); + } + } + + @Test + public void readsArchiveCompletely_COMPRESS245() throws Exception { + try { + final Path tempTar = resultDir.toPath().resolve("COMPRESS-245.tar"); + try (final GZIPInputStream gin = new GZIPInputStream( + Files.newInputStream(getPath("COMPRESS-245.tar.gz")))) { + Files.copy(gin, tempTar); + } + try (final TarFile tarFile = new TarFile(tempTar)) { + assertEquals(31, tarFile.getEntries().size()); + } + } catch (final IOException e) { + fail("COMPRESS-245: " + e.getMessage()); + } + } + + @Test(expected = IOException.class) + public void shouldThrowAnExceptionOnTruncatedEntries() throws Exception { + final File dir = mkdir("COMPRESS-279"); + try (final TarFile tarFile = new TarFile(getPath("COMPRESS-279.tar"))) { + int count = 0; + for (final TarArchiveEntry entry : tarFile.getEntries()) { + Files.copy(tarFile.getInputStream(entry), dir.toPath().resolve(String.valueOf(count))); + count++; + } + } finally { + rmdir(dir); + } + } + + @Test + public void shouldReadBigGid() throws Exception { + final ByteArrayOutputStream bos = new ByteArrayOutputStream(); + try (final TarArchiveOutputStream tos = new TarArchiveOutputStream(bos)) { + tos.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_POSIX); + TarArchiveEntry t = new TarArchiveEntry("name"); + t.setGroupId(4294967294L); + t.setSize(1); + tos.putArchiveEntry(t); + tos.write(30); + tos.closeArchiveEntry(); + } + final byte[] data = bos.toByteArray(); + try (final TarFile tarFile = new TarFile(data)) { + List<TarArchiveEntry> entries = tarFile.getEntries(); + assertEquals(4294967294L, entries.get(0).getLongGroupId()); + } + } + + /** + * @link "https://issues.apache.org/jira/browse/COMPRESS-324"; + */ + @Test + public void shouldReadGNULongNameEntryWithWrongName() throws Exception { + try (final TarFile tarFile = new TarFile(getPath("COMPRESS-324.tar"))) { + List<TarArchiveEntry> entries = tarFile.getEntries(); + assertEquals("1234567890123456789012345678901234567890123456789012345678901234567890" + + "1234567890123456789012345678901234567890123456789012345678901234567890" + + "1234567890123456789012345678901234567890123456789012345678901234567890" + + "1234567890123456789012345678901234567890.txt", + entries.get(0).getName()); + } + } + + /** + * @link "https://issues.apache.org/jira/browse/COMPRESS-355"; + */ + @Test + public void survivesBlankLinesInPaxHeader() throws Exception { + try (final TarFile tarFile = new TarFile(getPath("COMPRESS-355.tar"))) { + List<TarArchiveEntry> entries = tarFile.getEntries(); + assertEquals(1, entries.size()); + assertEquals("package/package.json", entries.get(0).getName()); + } + } + + /** + * @link "https://issues.apache.org/jira/browse/COMPRESS-356"; + */ + @Test + public void survivesPaxHeaderWithNameEndingInSlash() throws Exception { + try (final TarFile tarFile = new TarFile(getPath("COMPRESS-356.tar"))) { + final List<TarArchiveEntry> entries = tarFile.getEntries(); + assertEquals(1, entries.size()); + assertEquals("package/package.json", entries.get(0).getName()); + } + } + + /** + * @link "https://issues.apache.org/jira/browse/COMPRESS-417"; + */ + @Test + public void skipsDevNumbersWhenEntryIsNoDevice() throws Exception { + try (final TarFile tarFile = new TarFile(getPath("COMPRESS-417.tar"))) { + final List<TarArchiveEntry> entries = tarFile.getEntries(); + assertEquals(2, entries.size()); + assertEquals("test1.xml", entries.get(0).getName()); + assertEquals("test2.xml", entries.get(1).getName()); + } + } + + @Test + public void singleByteReadConsistentlyReturnsMinusOneAtEof() throws Exception { + try (final TarFile tarFile = new TarFile(getPath("bla.tar")); + final InputStream input = tarFile.getInputStream(tarFile.getEntries().get(0))) { + IOUtils.toByteArray(input); + assertEquals(-1, input.read()); + assertEquals(-1, input.read()); + } + } + + @Test + public void multiByteReadConsistentlyReturnsMinusOneAtEof() throws Exception { + final byte[] buf = new byte[2]; + try (final TarFile tarFile = new TarFile(getPath("bla.tar")); + final InputStream input = tarFile.getInputStream(tarFile.getEntries().get(0))) { + IOUtils.toByteArray(input); + assertEquals(-1, input.read(buf)); + assertEquals(-1, input.read(buf)); + } + } + + @Test public void testDirectoryWithLongNameEndsWithSlash() throws IOException, ArchiveException { final String rootPath = dir.getAbsolutePath(); final String dirDirectory = "COMPRESS-509"; @@ -79,6 +280,12 @@ public class TarFileTest extends AbstractTestCase { } @Test(expected = IOException.class) + public void testParseTarWithSpecialPaxHeaders() throws IOException { + try (final TarFile tarFile = new TarFile(getPath("COMPRESS-530.tar"))) { + } + } + + @Test(expected = IOException.class) public void testParseTarWithNonNumberPaxHeaders() throws IOException { try (TarFile tarFile = new TarFile(getPath("COMPRESS-529.tar"))) { } @@ -107,4 +314,30 @@ public class TarFileTest extends AbstractTestCase { try (TarFile tarFile = new TarFile(getPath("COMPRESS-553.tar"))) { } } + + @Test + public void testCompress558() throws IOException { + final String folderName = "apache-activemq-5.16.0/examples/openwire/advanced-scenarios/jms-example-exclusive-consumer/src/main/"; + final String consumerJavaName = "apache-activemq-5.16.0/examples/openwire/advanced-scenarios/jms-example-exclusive-consumer/src/main/java/example/queue/exclusive/Consumer.java"; + final String producerJavaName = "apache-activemq-5.16.0/examples/openwire/advanced-scenarios/jms-example-exclusive-consumer/src/main/java/example/queue/exclusive/Producer.java"; + + final ByteArrayOutputStream bos = new ByteArrayOutputStream(); + try (final TarArchiveOutputStream tos = new TarArchiveOutputStream(bos)) { + tos.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU); + TarArchiveEntry rootfolder = new TarArchiveEntry(folderName); + tos.putArchiveEntry(rootfolder); + TarArchiveEntry consumerJava = new TarArchiveEntry(consumerJavaName); + tos.putArchiveEntry(consumerJava); + TarArchiveEntry producerJava = new TarArchiveEntry(producerJavaName); + tos.putArchiveEntry(producerJava); + tos.closeArchiveEntry(); + } + final byte[] data = bos.toByteArray(); + try (final TarFile tarFile = new TarFile(data)) { + List<TarArchiveEntry> entries = tarFile.getEntries(); + assertEquals(folderName, entries.get(0).getName()); + assertEquals(consumerJavaName, entries.get(1).getName()); + assertEquals(producerJavaName, entries.get(2).getName()); + } + } }
