This is an automated email from the ASF dual-hosted git repository. peterlee pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/commons-compress.git
commit 81f2084a6b2d44ff455874bbcc1a9dfb14f287d3 Author: theobisproject <[email protected]> AuthorDate: Mon Jul 6 15:12:14 2020 +0200 COMPRESS-540: Implement TarFile to allow random access to tar files Basic idea behind the implementation is to read all headers at instantiation and save the position to the entry data. This will then be accessed as a random access file when reading the entry. --- .../compress/archivers/tar/TarArchiveEntry.java | 37 ++ .../archivers/tar/TarArchiveInputStream.java | 229 +------ .../tar/TarArchiveSparseZeroInputStream.java | 49 ++ .../commons/compress/archivers/tar/TarFile.java | 712 +++++++++++++++++++++ .../commons/compress/archivers/tar/TarUtils.java | 199 ++++++ .../commons/compress/archivers/zip/ZipFile.java | 81 +-- .../compress/utils/BoundedNIOInputStream.java | 97 +++ .../BoundedSeekableByteChannelInputStream.java | 56 ++ .../commons/compress/archivers/TarTestCase.java | 232 ++++++- .../commons/compress/archivers/tar/BigFilesIT.java | 78 +-- .../compress/archivers/tar/SparseFilesTest.java | 197 +++++- .../archivers/tar/TarArchiveInputStreamTest.java | 61 -- .../compress/archivers/tar/TarFileTest.java | 87 +++ .../compress/archivers/tar/TarUtilsTest.java | 71 +- .../BoundedSeekableByteChannelInputStreamTest.java | 41 ++ src/test/resources/directory.tar | Bin 0 -> 1536 bytes 16 files changed, 1798 insertions(+), 429 deletions(-) diff --git a/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveEntry.java b/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveEntry.java index 1f373c4..d4eb2ed 100644 --- a/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveEntry.java +++ b/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveEntry.java @@ -256,6 +256,7 @@ public class TarArchiveEntry implements ArchiveEntry, TarConstants { /** Convert millis to seconds */ public static final int MILLIS_PER_SECOND = 1000; + private long dataPosition = -1; /** * Construct an empty entry and prepares the header values. @@ -554,6 +555,23 @@ public class TarArchiveEntry implements ArchiveEntry, TarConstants { } /** + * Construct an entry from an archive's header bytes for random access tar. File is set to null. + * @param headerBuf The header bytes from a tar archive entry. + * @param encoding encoding to use for file names + * @param lenient when set to true illegal values for group/userid, mode, device numbers and timestamp will be + * ignored and the fields set to {@link #UNKNOWN}. When set to false such illegal fields cause an exception instead. + * @param dataPosition Position of the entry data in the random access file + * @since 1.21 + * @throws IllegalArgumentException if any of the numeric fields have an invalid format + * @throws IOException on error + */ + public TarArchiveEntry(final byte[] headerBuf, final ZipEncoding encoding, final boolean lenient, + final long dataPosition) throws IOException { + this(headerBuf, encoding, lenient); + this.dataPosition = dataPosition; + } + + /** * Determine if the two entries are equal. Equality is determined * by the header names being equal. * @@ -1185,6 +1203,25 @@ public class TarArchiveEntry implements ArchiveEntry, TarConstants { } /** + * Data position of the archive entry in a random access tar + * @return position of the data in the tar file. If the entry is created from a stream and therefore the data + * position is unknown this will return -1. + * @since 1.21 + */ + public long getDataPosition() { + return dataPosition; + } + + /** + * Set the position of the data for the tar entry. + * @param dataPosition the position of the data in the tar + * @since 1.21 + */ + public void setDataPosition(final long dataPosition) { + this.dataPosition = dataPosition; + } + + /** * get extra PAX Headers * @return read-only map containing any extra PAX Headers * @since 1.15 diff --git a/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveInputStream.java b/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveInputStream.java index dd9dba5..3f160dc 100644 --- a/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveInputStream.java +++ b/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveInputStream.java @@ -27,7 +27,6 @@ import java.io.ByteArrayOutputStream; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Comparator; import java.util.HashMap; @@ -40,7 +39,6 @@ import org.apache.commons.compress.archivers.zip.ZipEncoding; import org.apache.commons.compress.archivers.zip.ZipEncodingHelper; import org.apache.commons.compress.utils.ArchiveUtils; import org.apache.commons.compress.utils.BoundedInputStream; -import org.apache.commons.compress.utils.CharsetNames; import org.apache.commons.compress.utils.IOUtils; /** @@ -570,7 +568,7 @@ public class TarArchiveInputStream extends ArchiveInputStream { } private void readGlobalPaxHeaders() throws IOException { - globalPaxHeaders = parsePaxHeaders(this, globalSparseHeaders); + globalPaxHeaders = TarUtils.parsePaxHeaders(this, globalSparseHeaders, globalPaxHeaders); getNextEntry(); // Get the actual file entry if (currEntry == null) { @@ -605,11 +603,11 @@ public class TarArchiveInputStream extends ArchiveInputStream { */ private void paxHeaders() throws IOException { List<TarArchiveStructSparse> sparseHeaders = new ArrayList<>(); - final Map<String, String> headers = parsePaxHeaders(this, sparseHeaders); + final Map<String, String> headers = TarUtils.parsePaxHeaders(this, sparseHeaders, globalPaxHeaders); // for 0.1 PAX Headers if (headers.containsKey("GNU.sparse.map")) { - sparseHeaders = parsePAX01SparseHeaders(headers.get("GNU.sparse.map")); + sparseHeaders = TarUtils.parsePAX01SparseHeaders(headers.get("GNU.sparse.map")); } getNextEntry(); // Get the actual file entry if (currEntry == null) { @@ -619,7 +617,7 @@ public class TarArchiveInputStream extends ArchiveInputStream { // for 1.0 PAX Format, the sparse map is stored in the file data block if (currEntry.isPaxGNU1XSparse()) { - sparseHeaders = parsePAX1XSparseHeaders(); + sparseHeaders = TarUtils.parsePAX1XSparseHeaders(inputStream, recordSize); currEntry.setSparseHeaders(sparseHeaders); } @@ -628,192 +626,6 @@ public class TarArchiveInputStream extends ArchiveInputStream { buildSparseInputStreams(); } - /** - * For PAX Format 0.1, the sparse headers are stored in a single variable : GNU.sparse.map - * GNU.sparse.map - * Map of non-null data chunks. It is a string consisting of comma-separated values "offset,size[,offset-1,size-1...]" - * - * @param sparseMap the sparse map string consisting of comma-separated values "offset,size[,offset-1,size-1...]" - * @return sparse headers parsed from sparse map - */ - private List<TarArchiveStructSparse> parsePAX01SparseHeaders(final String sparseMap) { - final List<TarArchiveStructSparse> sparseHeaders = new ArrayList<>(); - final String[] sparseHeaderStrings = sparseMap.split(","); - - for (int i = 0; i < sparseHeaderStrings.length;i += 2) { - final long sparseOffset = Long.parseLong(sparseHeaderStrings[i]); - final long sparseNumbytes = Long.parseLong(sparseHeaderStrings[i + 1]); - sparseHeaders.add(new TarArchiveStructSparse(sparseOffset, sparseNumbytes)); - } - - return sparseHeaders; - } - - /** - * For PAX Format 1.X: - * The sparse map itself is stored in the file data block, preceding the actual file data. - * It consists of a series of decimal numbers delimited by newlines. The map is padded with nulls to the nearest block boundary. - * The first number gives the number of entries in the map. Following are map entries, each one consisting of two numbers - * giving the offset and size of the data block it describes. - * @return sparse headers - * @throws IOException - */ - private List<TarArchiveStructSparse> parsePAX1XSparseHeaders() throws IOException { - // for 1.X PAX Headers - final List<TarArchiveStructSparse> sparseHeaders = new ArrayList<>(); - long bytesRead = 0; - - long[] readResult = readLineOfNumberForPax1X(inputStream); - long sparseHeadersCount = readResult[0]; - bytesRead += readResult[1]; - while (sparseHeadersCount-- > 0) { - readResult = readLineOfNumberForPax1X(inputStream); - final long sparseOffset = readResult[0]; - bytesRead += readResult[1]; - - readResult = readLineOfNumberForPax1X(inputStream); - final long sparseNumbytes = readResult[0]; - bytesRead += readResult[1]; - sparseHeaders.add(new TarArchiveStructSparse(sparseOffset, sparseNumbytes)); - } - - // skip the rest of this record data - final long bytesToSkip = recordSize - bytesRead % recordSize; - IOUtils.skip(inputStream, bytesToSkip); - return sparseHeaders; - } - - /** - * For 1.X PAX Format, the sparse headers are stored in the file data block, preceding the actual file data. - * It consists of a series of decimal numbers delimited by newlines. - * - * @param inputStream the input stream of the tar file - * @return the decimal number delimited by '\n', and the bytes read from input stream - * @throws IOException - */ - private long[] readLineOfNumberForPax1X(final InputStream inputStream) throws IOException { - int number; - long result = 0; - long bytesRead = 0; - - while((number = inputStream.read()) != '\n') { - bytesRead += 1; - if(number == -1) { - throw new IOException("Unexpected EOF when reading parse information of 1.X PAX format"); - } - result = result * 10 + (number - '0'); - } - bytesRead += 1; - - return new long[] {result, bytesRead}; - } - - /** - * For PAX Format 0.0, the sparse headers(GNU.sparse.offset and GNU.sparse.numbytes) - * may appear multi times, and they look like: - * - * GNU.sparse.size=size - * GNU.sparse.numblocks=numblocks - * repeat numblocks times - * GNU.sparse.offset=offset - * GNU.sparse.numbytes=numbytes - * end repeat - * - * For PAX Format 0.1, the sparse headers are stored in a single variable : GNU.sparse.map - * - * GNU.sparse.map - * Map of non-null data chunks. It is a string consisting of comma-separated values "offset,size[,offset-1,size-1...]" - * - * @param inputStream inputStream to read keys and values - * @param sparseHeaders used in PAX Format 0.0 & 0.1, as it may appear multi times, - * the sparse headers need to be stored in an array, not a map - * @return map of PAX headers values found inside of the current (local or global) PAX headers tar entry. - * @throws IOException - */ - Map<String, String> parsePaxHeaders(final InputStream inputStream, final List<TarArchiveStructSparse> sparseHeaders) - throws IOException { - final Map<String, String> headers = new HashMap<>(globalPaxHeaders); - Long offset = null; - // Format is "length keyword=value\n"; - while(true) { // get length - int ch; - int len = 0; - int read = 0; - while((ch = inputStream.read()) != -1) { - read++; - if (ch == '\n') { // blank line in header - break; - } else if (ch == ' '){ // End of length string - // Get keyword - final ByteArrayOutputStream coll = new ByteArrayOutputStream(); - while((ch = inputStream.read()) != -1) { - read++; - if (ch == '='){ // end of keyword - final String keyword = coll.toString(CharsetNames.UTF_8); - // Get rest of entry - final int restLen = len - read; - if (restLen <= 1) { // only NL - headers.remove(keyword); - } else { - final byte[] rest = new byte[restLen]; - final int got = IOUtils.readFully(inputStream, rest); - if (got != restLen) { - throw new IOException("Failed to read " - + "Paxheader. Expected " - + restLen - + " bytes, read " - + got); - } - // Drop trailing NL - final String value = new String(rest, 0, - restLen - 1, StandardCharsets.UTF_8); - headers.put(keyword, value); - - // for 0.0 PAX Headers - if (keyword.equals("GNU.sparse.offset")) { - if (offset != null) { - // previous GNU.sparse.offset header but but no numBytes - sparseHeaders.add(new TarArchiveStructSparse(offset, 0)); - } - offset = Long.valueOf(value); - } - - // for 0.0 PAX Headers - if (keyword.equals("GNU.sparse.numbytes")) { - if (offset == null) { - throw new IOException("Failed to read Paxheader." + - "GNU.sparse.offset is expected before GNU.sparse.numbytes shows up."); - } - sparseHeaders.add(new TarArchiveStructSparse(offset, Long.parseLong(value))); - offset = null; - } - } - break; - } - coll.write((byte) ch); - } - break; // Processed single header - } - - // COMPRESS-530 : throw if we encounter a non-number while reading length - if (ch < '0' || ch > '9') { - throw new IOException("Failed to read Paxheader. Encountered a non-number while reading length"); - } - - len *= 10; - len += ch - '0'; - } - if (ch == -1){ // EOF - break; - } - } - if (offset != null) { - // offset but no numBytes - sparseHeaders.add(new TarArchiveStructSparse(offset, 0)); - } - return headers; - } - private void applyPaxHeadersToCurrentEntry(final Map<String, String> headers, final List<TarArchiveStructSparse> sparseHeaders) { currEntry.updateEntryFromPaxHeaders(headers); currEntry.setSparseHeaders(sparseHeaders); @@ -963,11 +775,11 @@ public class TarArchiveInputStream extends ArchiveInputStream { */ private int readSparse(final byte[] buf, final int offset, final int numToRead) throws IOException { // if there are no actual input streams, just read from the original input stream - if (sparseInputStreams == null || sparseInputStreams.size() == 0) { + if (sparseInputStreams == null || sparseInputStreams.isEmpty()) { return inputStream.read(buf, offset, numToRead); } - if(currentSparseInputStreamIndex >= sparseInputStreams.size()) { + if (currentSparseInputStreamIndex >= sparseInputStreams.size()) { return -1; } @@ -1139,35 +951,8 @@ public class TarArchiveInputStream extends ArchiveInputStream { } } - if (sparseInputStreams.size() > 0) { + if (!sparseInputStreams.isEmpty()) { currentSparseInputStreamIndex = 0; } } - - /** - * This is an inputstream that always return 0, - * this is used when reading the "holes" of a sparse file - */ - private static class TarArchiveSparseZeroInputStream extends InputStream { - /** - * Just return 0 - * @return - * @throws IOException - */ - @Override - public int read() throws IOException { - return 0; - } - - /** - * these's nothing need to do when skipping - * - * @param n bytes to skip - * @return bytes actually skipped - */ - @Override - public long skip(final long n) { - return n; - } - } } diff --git a/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveSparseZeroInputStream.java b/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveSparseZeroInputStream.java new file mode 100644 index 0000000..740e44e --- /dev/null +++ b/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveSparseZeroInputStream.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.commons.compress.archivers.tar; + +import java.io.IOException; +import java.io.InputStream; + +/** + * This is an inputstream that always return 0, + * this is used when reading the "holes" of a sparse file + */ +class TarArchiveSparseZeroInputStream extends InputStream { + + /** + * Just return 0 + * @return + * @throws IOException + */ + @Override + public int read() throws IOException { + return 0; + } + + /** + * these's nothing need to do when skipping + * + * @param n bytes to skip + * @return bytes actually skipped + */ + @Override + public long skip(final long n) { + return n; + } +} diff --git a/src/main/java/org/apache/commons/compress/archivers/tar/TarFile.java b/src/main/java/org/apache/commons/compress/archivers/tar/TarFile.java new file mode 100644 index 0000000..714ff8a --- /dev/null +++ b/src/main/java/org/apache/commons/compress/archivers/tar/TarFile.java @@ -0,0 +1,712 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.commons.compress.archivers.tar; + +import java.io.ByteArrayOutputStream; +import java.io.Closeable; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.channels.SeekableByteChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import org.apache.commons.compress.archivers.zip.ZipEncoding; +import org.apache.commons.compress.archivers.zip.ZipEncodingHelper; +import org.apache.commons.compress.utils.ArchiveUtils; +import org.apache.commons.compress.utils.BoundedInputStream; +import org.apache.commons.compress.utils.BoundedNIOInputStream; +import org.apache.commons.compress.utils.BoundedSeekableByteChannelInputStream; +import org.apache.commons.compress.utils.SeekableInMemoryByteChannel; + +/** + * The TarFile provides random access to UNIX to archives. + * @since 1.21 + */ +public class TarFile implements Closeable { + + private static final int SMALL_BUFFER_SIZE = 256; + + private final ByteBuffer smallBuf = ByteBuffer.allocate(SMALL_BUFFER_SIZE); + + private final SeekableByteChannel archive; + + /** + * The encoding of the tar file + */ + private final ZipEncoding zipEncoding; + + private final LinkedList<TarArchiveEntry> entries = new LinkedList<>(); + + private final int blockSize; + + private final boolean lenient; + + private final int recordSize; + + private final ByteBuffer recordBuffer; + + // the global sparse headers, this is only used in PAX Format 0.X + private final List<TarArchiveStructSparse> globalSparseHeaders = new ArrayList<>(); + + private boolean hasHitEOF; + + /** + * The meta-data about the current entry + */ + private TarArchiveEntry currEntry; + + // the global PAX header + private Map<String, String> globalPaxHeaders = new HashMap<>(); + + private final Map<String, List<InputStream>> sparseInputStreams = new HashMap<>(); + + /** + * Constructor for TarFile. + * + * @param content the content to use + * @throws IOException when reading the tar archive fails + */ + public TarFile(final byte[] content) throws IOException { + this(new SeekableInMemoryByteChannel(content), TarConstants.DEFAULT_BLKSIZE, TarConstants.DEFAULT_RCDSIZE, null, false); + } + + /** + * Constructor for TarFile. + * + * @param content the content to use + * @param encoding the encoding to use + * @throws IOException when reading the tar archive fails + */ + public TarFile(final byte[] content, final String encoding) throws IOException { + this(new SeekableInMemoryByteChannel(content), TarConstants.DEFAULT_BLKSIZE, TarConstants.DEFAULT_RCDSIZE, encoding, false); + } + + /** + * Constructor for TarFile. + * + * @param content the content to use + * @param lenient when set to true illegal values for group/userid, mode, device numbers and timestamp will be + * ignored and the fields set to {@link TarArchiveEntry#UNKNOWN}. When set to false such illegal fields cause an + * exception instead. + * @throws IOException when reading the tar archive fails + */ + public TarFile(final byte[] content, final boolean lenient) throws IOException { + this(new SeekableInMemoryByteChannel(content), TarConstants.DEFAULT_BLKSIZE, TarConstants.DEFAULT_RCDSIZE, null, lenient); + } + + /** + * Constructor for TarFile. + * + * @param archive the file of the archive to use + * @throws IOException when reading the tar archive fails + */ + public TarFile(final File archive) throws IOException { + this(archive.toPath()); + } + + /** + * Constructor for TarFile. + * + * @param archive the file of the archive to use + * @param encoding the encoding to use + * @throws IOException when reading the tar archive fails + */ + public TarFile(final File archive, final String encoding) throws IOException { + this(archive.toPath(), encoding); + } + + /** + * Constructor for TarFile. + * + * @param archive the file of the archive to use + * @param lenient when set to true illegal values for group/userid, mode, device numbers and timestamp will be + * ignored and the fields set to {@link TarArchiveEntry#UNKNOWN}. When set to false such illegal fields cause an + * exception instead. + * @throws IOException when reading the tar archive fails + */ + public TarFile(final File archive, final boolean lenient) throws IOException { + this(archive.toPath(), lenient); + } + + /** + * Constructor for TarFile. + * + * @param archivePath the path of the archive to use + * @throws IOException when reading the tar archive fails + */ + public TarFile(final Path archivePath) throws IOException { + this(Files.newByteChannel(archivePath), TarConstants.DEFAULT_BLKSIZE, TarConstants.DEFAULT_RCDSIZE, null, false); + } + + /** + * Constructor for TarFile. + * + * @param archivePath the path of the archive to use + * @param encoding the encoding to use + * @throws IOException when reading the tar archive fails + */ + public TarFile(final Path archivePath, final String encoding) throws IOException { + this(Files.newByteChannel(archivePath), TarConstants.DEFAULT_BLKSIZE, TarConstants.DEFAULT_RCDSIZE, encoding, false); + } + + /** + * Constructor for TarFile. + * + * @param archivePath the path of the archive to use + * @param lenient when set to true illegal values for group/userid, mode, device numbers and timestamp will be + * ignored and the fields set to {@link TarArchiveEntry#UNKNOWN}. When set to false such illegal fields cause an + * exception instead. + * @throws IOException when reading the tar archive fails + */ + public TarFile(final Path archivePath, final boolean lenient) throws IOException { + this(Files.newByteChannel(archivePath), TarConstants.DEFAULT_BLKSIZE, TarConstants.DEFAULT_RCDSIZE, null, lenient); + } + + /** + * Constructor for TarFile. + * + * @param archive the seekable byte channel to use + * @param blockSize the blocks size to use + * @param recordSize the record size to use + * @param encoding the encoding to use + * @param lenient when set to true illegal values for group/userid, mode, device numbers and timestamp will be + * ignored and the fields set to {@link TarArchiveEntry#UNKNOWN}. When set to false such illegal fields cause an + * exception instead. + * @throws IOException when reading the tar archive fails + */ + public TarFile(final SeekableByteChannel archive, final int blockSize, final int recordSize, final String encoding, final boolean lenient) throws IOException { + this.archive = archive; + this.hasHitEOF = false; + this.zipEncoding = ZipEncodingHelper.getZipEncoding(encoding); + this.recordSize = recordSize; + this.recordBuffer = ByteBuffer.allocate(this.recordSize); + this.blockSize = blockSize; + this.lenient = lenient; + + TarArchiveEntry entry; + while ((entry = getNextTarEntry()) != null) { + entries.add(entry); + } + } + + /** + * Get the next entry in this tar archive. This will skip + * to the end of the current entry, if there is one, and + * place the position of the channel at the header of the + * next entry, and read the header and instantiate a new + * TarEntry from the header bytes and return that entry. + * If there are no more entries in the archive, null will + * be returned to indicate that the end of the archive has + * been reached. + * + * @return The next TarEntry in the archive, or null if there is no next entry. + * @throws IOException when reading the next TarEntry fails + */ + private TarArchiveEntry getNextTarEntry() throws IOException { + if (isAtEOF()) { + return null; + } + + if (currEntry != null) { + // Skip to the end of the entry + archive.position(currEntry.getDataPosition() + currEntry.getSize()); + + skipRecordPadding(); + } + + final ByteBuffer headerBuf = getRecord(); + if (null == headerBuf) { + return null; + } + + try { + currEntry = new TarArchiveEntry(headerBuf.array(), zipEncoding, lenient, archive.position()); + } catch (final IllegalArgumentException e) { + throw new IOException("Error detected parsing the header", e); + } + + if (currEntry.isGNULongLinkEntry()) { + final byte[] longLinkData = getLongNameData(); + if (longLinkData == null) { + // Bugzilla: 40334 + // Malformed tar file - long link entry name not followed by + // entry + return null; + } + currEntry.setLinkName(zipEncoding.decode(longLinkData)); + } + + if (currEntry.isGNULongNameEntry()) { + final byte[] longNameData = getLongNameData(); + if (longNameData == null) { + // Bugzilla: 40334 + // Malformed tar file - long entry name not followed by + // entry + return null; + } + + // COMPRESS-509 : the name of directories should end with '/' + String name = zipEncoding.decode(longNameData); + if (currEntry.isDirectory() && !name.endsWith("/")) { + name += "/"; + } + currEntry.setName(name); + } + + if (currEntry.isGlobalPaxHeader()) { // Process Global Pax headers + readGlobalPaxHeaders(); + } + + try { + if (currEntry.isPaxHeader()) { // Process Pax headers + paxHeaders(); + } else if (!globalPaxHeaders.isEmpty()) { + applyPaxHeadersToCurrentEntry(globalPaxHeaders, globalSparseHeaders); + } + } catch (NumberFormatException e) { + throw new IOException("Error detected parsing the pax header", e); + } + + if (currEntry.isOldGNUSparse()) { // Process sparse files + readOldGNUSparse(); + } + + return currEntry; + } + + /** + * Adds the sparse chunks from the current entry to the sparse chunks, + * including any additional sparse entries following the current entry. + * + * @throws IOException when reading the sparse entry fails + */ + private void readOldGNUSparse() throws IOException { + if (currEntry.isExtended()) { + TarArchiveSparseEntry entry; + do { + final ByteBuffer headerBuf = getRecord(); + if (headerBuf == null) { + currEntry = null; + break; + } + entry = new TarArchiveSparseEntry(headerBuf.array()); + currEntry.getSparseHeaders().addAll(entry.getSparseHeaders()); + currEntry.setDataPosition(currEntry.getDataPosition() + recordSize); + } while (entry.isExtended()); + } + + // sparse headers are all done reading, we need to build + // sparse input streams using these sparse headers + buildSparseInputStreams(); + } + + /** + * Build the input streams consisting of all-zero input streams and non-zero input streams. + * When reading from the non-zero input streams, the data is actually read from the original input stream. + * The size of each input stream is introduced by the sparse headers. + * + * NOTE : Some all-zero input streams and non-zero input streams have the size of 0. We DO NOT store the + * 0 size input streams because they are meaningless. + */ + private void buildSparseInputStreams() throws IOException { + List<InputStream> streams = new ArrayList<>(); + + final List<TarArchiveStructSparse> sparseHeaders = currEntry.getSparseHeaders(); + // sort the sparse headers in case they are written in wrong order + if (sparseHeaders != null && sparseHeaders.size() > 1) { + final Comparator<TarArchiveStructSparse> sparseHeaderComparator = new Comparator<TarArchiveStructSparse>() { + @Override + public int compare(final TarArchiveStructSparse p, final TarArchiveStructSparse q) { + Long pOffset = p.getOffset(); + Long qOffset = q.getOffset(); + return pOffset.compareTo(qOffset); + } + }; + Collections.sort(sparseHeaders, sparseHeaderComparator); + } + + if (sparseHeaders != null) { + // Stream doesn't need to be closed at all as it doesn't use any resources + final InputStream zeroInputStream = new TarArchiveSparseZeroInputStream(); //NOSONAR + long offset = 0; + long numberOfZeroBytesInSparseEntry = 0; + for (TarArchiveStructSparse sparseHeader : sparseHeaders) { + if (sparseHeader.getOffset() == 0 && sparseHeader.getNumbytes() == 0) { + break; + } + + if ((sparseHeader.getOffset() - offset) < 0) { + throw new IOException("Corrupted struct sparse detected"); + } + + // only store the input streams with non-zero size + if ((sparseHeader.getOffset() - offset) > 0) { + long sizeOfZeroByteStream = sparseHeader.getOffset() - offset; + streams.add(new BoundedInputStream(zeroInputStream, sizeOfZeroByteStream)); + numberOfZeroBytesInSparseEntry += sizeOfZeroByteStream; + } + + // only store the input streams with non-zero size + if (sparseHeader.getNumbytes() > 0) { + long start = + currEntry.getDataPosition() + sparseHeader.getOffset() - numberOfZeroBytesInSparseEntry; + streams.add(new BoundedSeekableByteChannelInputStream(start, sparseHeader.getNumbytes(), archive)); + } + + offset = sparseHeader.getOffset() + sparseHeader.getNumbytes(); + } + } + + sparseInputStreams.put(currEntry.getName(), streams); + } + + /** + * Update the current entry with the read pax headers + * @param headers Headers read from the pax header + * @param sparseHeaders Sparse headers read from pax header + */ + private void applyPaxHeadersToCurrentEntry(final Map<String, String> headers, final List<TarArchiveStructSparse> sparseHeaders) { + currEntry.updateEntryFromPaxHeaders(headers); + currEntry.setSparseHeaders(sparseHeaders); + } + + /** + * For PAX Format 0.0, the sparse headers(GNU.sparse.offset and GNU.sparse.numbytes) + * may appear multi times, and they look like: + * + * GNU.sparse.size=size + * GNU.sparse.numblocks=numblocks + * repeat numblocks times + * GNU.sparse.offset=offset + * GNU.sparse.numbytes=numbytes + * end repeat + * + * + * For PAX Format 0.1, the sparse headers are stored in a single variable : GNU.sparse.map + * + * GNU.sparse.map + * Map of non-null data chunks. It is a string consisting of comma-separated values "offset,size[,offset-1,size-1...]" + * + * + * For PAX Format 1.X: + * The sparse map itself is stored in the file data block, preceding the actual file data. + * It consists of a series of decimal numbers delimited by newlines. The map is padded with nulls to the nearest block boundary. + * The first number gives the number of entries in the map. Following are map entries, each one consisting of two numbers + * giving the offset and size of the data block it describes. + * @throws IOException + */ + private void paxHeaders() throws IOException { + List<TarArchiveStructSparse> sparseHeaders = new ArrayList<>(); + final Map<String, String> headers; + try (final InputStream input = getInputStream(currEntry)) { + headers = TarUtils.parsePaxHeaders(input, sparseHeaders, globalPaxHeaders); + } + + // for 0.1 PAX Headers + if (headers.containsKey("GNU.sparse.map")) { + sparseHeaders = TarUtils.parsePAX01SparseHeaders(headers.get("GNU.sparse.map")); + } + getNextTarEntry(); // Get the actual file entry + if (currEntry == null) { + throw new IOException("premature end of tar archive. Didn't find any entry after PAX header."); + } + applyPaxHeadersToCurrentEntry(headers, sparseHeaders); + + // for 1.0 PAX Format, the sparse map is stored in the file data block + if (currEntry.isPaxGNU1XSparse()) { + try (final InputStream input = getInputStream(currEntry)) { + sparseHeaders = TarUtils.parsePAX1XSparseHeaders(input, recordSize); + } + currEntry.setSparseHeaders(sparseHeaders); + // data of the entry is after the pax gnu entry. So we need to update the data position once again + currEntry.setDataPosition(currEntry.getDataPosition() + recordSize); + } + + // sparse headers are all done reading, we need to build + // sparse input streams using these sparse headers + buildSparseInputStreams(); + } + + private void readGlobalPaxHeaders() throws IOException { + try (InputStream input = getInputStream(currEntry)) { + globalPaxHeaders = TarUtils.parsePaxHeaders(input, globalSparseHeaders, globalPaxHeaders); + } + getNextTarEntry(); // Get the actual file entry + } + + /** + * Get the next entry in this tar archive as longname data. + * + * @return The next entry in the archive as longname data, or null. + * @throws IOException on error + */ + private byte[] getLongNameData() throws IOException { + final ByteArrayOutputStream longName = new ByteArrayOutputStream(); + int length; + while ((length = archive.read(smallBuf)) > 0) { + longName.write(smallBuf.array(), 0, length); + } + getNextTarEntry(); + if (currEntry == null) { + // Bugzilla: 40334 + // Malformed tar file - long entry name not followed by entry + return null; + } + byte[] longNameData = longName.toByteArray(); + // remove trailing null terminator(s) + length = longNameData.length; + while (length > 0 && longNameData[length - 1] == 0) { + --length; + } + if (length != longNameData.length) { + final byte[] l = new byte[length]; + System.arraycopy(longNameData, 0, l, 0, length); + longNameData = l; + } + return longNameData; + } + + /** + * The last record block should be written at the full size, so skip any + * additional space used to fill a record after an entry + * + * @throws IOException when skipping the padding of the record fails + */ + private void skipRecordPadding() throws IOException { + if (!isDirectory() && currEntry.getSize() > 0 && currEntry.getSize() % recordSize != 0) { + final long numRecords = (currEntry.getSize() / recordSize) + 1; + final long padding = (numRecords * recordSize) - currEntry.getSize(); + archive.position(archive.position() + padding); + } + } + + /** + * Get the next record in this tar archive. This will skip + * over any remaining data in the current entry, if there + * is one, and place the input stream at the header of the + * next entry. + * + * <p>If there are no more entries in the archive, null will be + * returned to indicate that the end of the archive has been + * reached. At the same time the {@code hasHitEOF} marker will be + * set to true.</p> + * + * @return The next TarEntry in the archive, or null if there is no next entry. + * @throws IOException when reading the next TarEntry fails + */ + private ByteBuffer getRecord() throws IOException { + ByteBuffer headerBuf = readRecord(); + setAtEOF(isEOFRecord(headerBuf)); + if (isAtEOF() && headerBuf != null) { + // Consume rest + tryToConsumeSecondEOFRecord(); + // TODO: This is present in the TarArchiveInputStream but I don't know if we need this in the random access implementation. All tests are passing ... + // consumeRemainderOfLastBlock(); + headerBuf = null; + } + return headerBuf; + } + + /** + * Tries to read the next record resetting the position in the + * archive if it is not a EOF record. + * + * <p>This is meant to protect against cases where a tar + * implementation has written only one EOF record when two are + * expected. Actually this won't help since a non-conforming + * implementation likely won't fill full blocks consisting of - by + * default - ten records either so we probably have already read + * beyond the archive anyway.</p> + * + * @throws IOException if reading the record of resetting the position in the archive fails + */ + private void tryToConsumeSecondEOFRecord() throws IOException { + boolean shouldReset = true; + try { + shouldReset = !isEOFRecord(readRecord()); + } finally { + if (shouldReset) { + archive.position(archive.position() - recordSize); + } + } + } + + /** + * Read a record from the input stream and return the data. + * + * @return The record data or null if EOF has been hit. + * @throws IOException if reading from the archive fails + */ + private ByteBuffer readRecord() throws IOException { + recordBuffer.rewind(); + final int readNow = archive.read(recordBuffer); + if (readNow != recordSize) { + return null; + } + return recordBuffer; + } + + /** + * Get all TAR Archive Entries from the TarFile + * + * @return All entries from the tar file + */ + public List<TarArchiveEntry> getEntries() { + return new ArrayList<>(entries); + } + + private boolean isEOFRecord(final ByteBuffer headerBuf) { + return headerBuf == null || ArchiveUtils.isArrayZero(headerBuf.array(), recordSize); + } + + protected final boolean isAtEOF() { + return hasHitEOF; + } + + protected final void setAtEOF(final boolean b) { + hasHitEOF = b; + } + + private boolean isDirectory() { + return currEntry != null && currEntry.isDirectory(); + } + + /** + * Get the input stream for the provided Tar Archive Entry + * @param entry Entry to get the input stream from + * @return Input stream of the provided entry + */ + public InputStream getInputStream(final TarArchiveEntry entry) { + return new BoundedTarEntryInputStream(entry, archive); + } + + @Override + public void close() throws IOException { + archive.close(); + } + + private final class BoundedTarEntryInputStream extends BoundedNIOInputStream { + + private final SeekableByteChannel channel; + + private final TarArchiveEntry entry; + + private long entryOffset; + + private int currentSparseInputStreamIndex; + + BoundedTarEntryInputStream(final TarArchiveEntry entry, final SeekableByteChannel channel) { + super(entry.getDataPosition(), entry.isSparse() ? entry.getRealSize() : entry.getSize()); + this.entry = entry; + this.channel = channel; + } + + @Override + protected int read(final long pos, final ByteBuffer buf) throws IOException { + if (entry.isSparse()) { + // for sparse entries, there are actually currEntry.getRealSize() bytes to read + if (entryOffset >= currEntry.getRealSize()) { + return -1; + } + } else { + if (entryOffset >= entry.getSize()) { + return -1; + } + } + + int totalRead = 0; + if (entry.isSparse()) { + totalRead = readSparse(entryOffset, buf, buf.limit()); + } else { + totalRead = readArchive(pos, buf); + } + + if (totalRead == -1) { + if (buf.array().length > 0) { + throw new IOException("Truncated TAR archive"); + } + setAtEOF(true); + } else { + entryOffset += totalRead; + } + return totalRead; + } + + private int readSparse(final long pos, final ByteBuffer buf, final int numToRead) throws IOException { + // if there are no actual input streams, just read from the original archive + final List<InputStream> entrySparseInputStreams = sparseInputStreams.get(entry.getName()); + if (entrySparseInputStreams == null || entrySparseInputStreams.isEmpty()) { + return readArchive(entry.getDataPosition() + pos, buf); + } + + if (currentSparseInputStreamIndex >= entrySparseInputStreams.size()) { + return -1; + } + + final InputStream currentInputStream = entrySparseInputStreams.get(currentSparseInputStreamIndex); + byte[] bufArray = new byte[numToRead]; + int readLen = currentInputStream.read(bufArray); + if (readLen != -1) { + buf.put(bufArray, 0, readLen); + } + + // if the current input stream is the last input stream, + // just return the number of bytes read from current input stream + if (currentSparseInputStreamIndex == entrySparseInputStreams.size() - 1) { + return readLen; + } + + // if EOF of current input stream is meet, open a new input stream and recursively call read + if (readLen == -1) { + currentSparseInputStreamIndex++; + return readSparse(pos, buf, numToRead); + } + + // if the rest data of current input stream is not long enough, open a new input stream + // and recursively call read + if (readLen < numToRead) { + currentSparseInputStreamIndex++; + int readLenOfNext = readSparse(pos + readLen, buf, numToRead - readLen); + if (readLenOfNext == -1) { + return readLen; + } + + return readLen + readLenOfNext; + } + + // if the rest data of current input stream is enough(which means readLen == len), just return readLen + return readLen; + } + + private int readArchive(final long pos, final ByteBuffer buf) throws IOException { + channel.position(pos); + int read = channel.read(buf); + buf.flip(); + return read; + } + } +} diff --git a/src/main/java/org/apache/commons/compress/archivers/tar/TarUtils.java b/src/main/java/org/apache/commons/compress/archivers/tar/TarUtils.java index 9f43bd7..2e5c30c 100644 --- a/src/main/java/org/apache/commons/compress/archivers/tar/TarUtils.java +++ b/src/main/java/org/apache/commons/compress/archivers/tar/TarUtils.java @@ -18,11 +18,21 @@ */ package org.apache.commons.compress.archivers.tar; +import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.InputStream; import java.math.BigInteger; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + import org.apache.commons.compress.archivers.zip.ZipEncoding; import org.apache.commons.compress.archivers.zip.ZipEncodingHelper; +import org.apache.commons.compress.utils.CharsetNames; +import org.apache.commons.compress.utils.IOUtils; import static org.apache.commons.compress.archivers.tar.TarConstants.CHKSUMLEN; import static org.apache.commons.compress.archivers.tar.TarConstants.CHKSUM_OFFSET; @@ -627,4 +637,193 @@ public class TarUtils { return storedSum == unsignedSum || storedSum == signedSum; } + /** + * For PAX Format 0.0, the sparse headers(GNU.sparse.offset and GNU.sparse.numbytes) + * may appear multi times, and they look like: + * + * GNU.sparse.size=size + * GNU.sparse.numblocks=numblocks + * repeat numblocks times + * GNU.sparse.offset=offset + * GNU.sparse.numbytes=numbytes + * end repeat + * + * For PAX Format 0.1, the sparse headers are stored in a single variable : GNU.sparse.map + * + * GNU.sparse.map + * Map of non-null data chunks. It is a string consisting of comma-separated values "offset,size[,offset-1,size-1...]" + * + * @param inputStream inputstream to read keys and values + * @param sparseHeaders used in PAX Format 0.0 & 0.1, as it may appear multi times, + * the sparse headers need to be stored in an array, not a map + * @param globalPaxHeaders global PAX headers of the tar archive + * @return map of PAX headers values found inside of the current (local or global) PAX headers tar entry. + * @throws IOException + */ + public static Map<String, String> parsePaxHeaders(final InputStream inputStream, final List<TarArchiveStructSparse> sparseHeaders, final Map<String, String> globalPaxHeaders) + throws IOException { + final Map<String, String> headers = new HashMap<>(globalPaxHeaders); + Long offset = null; + // Format is "length keyword=value\n"; + while(true) { // get length + int ch; + int len = 0; + int read = 0; + while((ch = inputStream.read()) != -1) { + read++; + if (ch == '\n') { // blank line in header + break; + } else if (ch == ' '){ // End of length string + // Get keyword + final ByteArrayOutputStream coll = new ByteArrayOutputStream(); + while((ch = inputStream.read()) != -1) { + read++; + if (ch == '='){ // end of keyword + final String keyword = coll.toString(CharsetNames.UTF_8); + // Get rest of entry + final int restLen = len - read; + if (restLen == 1) { // only NL + headers.remove(keyword); + } else { + final byte[] rest = new byte[restLen]; + final int got = IOUtils.readFully(inputStream, rest); + if (got != restLen) { + throw new IOException("Failed to read " + + "Paxheader. Expected " + + restLen + + " bytes, read " + + got); + } + // Drop trailing NL + final String value = new String(rest, 0, + restLen - 1, StandardCharsets.UTF_8); + headers.put(keyword, value); + + // for 0.0 PAX Headers + if (keyword.equals("GNU.sparse.offset")) { + if (offset != null) { + // previous GNU.sparse.offset header but but no numBytes + sparseHeaders.add(new TarArchiveStructSparse(offset, 0)); + } + offset = Long.valueOf(value); + } + + // for 0.0 PAX Headers + if (keyword.equals("GNU.sparse.numbytes")) { + if (offset == null) { + throw new IOException("Failed to read Paxheader." + + "GNU.sparse.offset is expected before GNU.sparse.numbytes shows up."); + } + sparseHeaders.add(new TarArchiveStructSparse(offset, Long.parseLong(value))); + offset = null; + } + } + break; + } + coll.write((byte) ch); + } + break; // Processed single header + } + + // COMPRESS-530 : throw if we encounter a non-number while reading length + if (ch < '0' || ch > '9') { + throw new IOException("Failed to read Paxheader. Encountered a non-number while reading length"); + } + + len *= 10; + len += ch - '0'; + } + if (ch == -1){ // EOF + break; + } + } + if (offset != null) { + // offset but no numBytes + sparseHeaders.add(new TarArchiveStructSparse(offset, 0)); + } + return headers; + } + + /** + * For PAX Format 0.1, the sparse headers are stored in a single variable : GNU.sparse.map + * GNU.sparse.map + * Map of non-null data chunks. It is a string consisting of comma-separated values "offset,size[,offset-1,size-1...]" + * + * @param sparseMap the sparse map string consisting of comma-separated values "offset,size[,offset-1,size-1...]" + * @return sparse headers parsed from sparse map + */ + public static List<TarArchiveStructSparse> parsePAX01SparseHeaders(String sparseMap) { + List<TarArchiveStructSparse> sparseHeaders = new ArrayList<>(); + String[] sparseHeaderStrings = sparseMap.split(","); + + for (int i = 0; i < sparseHeaderStrings.length; i += 2) { + long sparseOffset = Long.parseLong(sparseHeaderStrings[i]); + long sparseNumbytes = Long.parseLong(sparseHeaderStrings[i + 1]); + sparseHeaders.add(new TarArchiveStructSparse(sparseOffset, sparseNumbytes)); + } + + return sparseHeaders; + } + + /** + * For PAX Format 1.X: + * The sparse map itself is stored in the file data block, preceding the actual file data. + * It consists of a series of decimal numbers delimited by newlines. The map is padded with nulls to the nearest block boundary. + * The first number gives the number of entries in the map. Following are map entries, each one consisting of two numbers + * giving the offset and size of the data block it describes. + * @param inputStream + * @param recordSize + * @return sparse headers + * @throws IOException + */ + public static List<TarArchiveStructSparse> parsePAX1XSparseHeaders(final InputStream inputStream, final int recordSize) throws IOException { + // for 1.X PAX Headers + List<TarArchiveStructSparse> sparseHeaders = new ArrayList<>(); + long bytesRead = 0; + + long[] readResult = readLineOfNumberForPax1X(inputStream); + long sparseHeadersCount = readResult[0]; + bytesRead += readResult[1]; + while (sparseHeadersCount-- > 0) { + readResult = readLineOfNumberForPax1X(inputStream); + long sparseOffset = readResult[0]; + bytesRead += readResult[1]; + + readResult = readLineOfNumberForPax1X(inputStream); + long sparseNumbytes = readResult[0]; + bytesRead += readResult[1]; + sparseHeaders.add(new TarArchiveStructSparse(sparseOffset, sparseNumbytes)); + } + + // skip the rest of this record data + long bytesToSkip = recordSize - bytesRead % recordSize; + IOUtils.skip(inputStream, bytesToSkip); + return sparseHeaders; + } + + /** + * For 1.X PAX Format, the sparse headers are stored in the file data block, preceding the actual file data. + * It consists of a series of decimal numbers delimited by newlines. + * + * @param inputStream the input stream of the tar file + * @return the decimal number delimited by '\n', and the bytes read from input stream + * @throws IOException + */ + private static long[] readLineOfNumberForPax1X(final InputStream inputStream) throws IOException { + int number; + long result = 0; + long bytesRead = 0; + + while ((number = inputStream.read()) != '\n') { + bytesRead += 1; + if (number == -1) { + throw new IOException("Unexpected EOF when reading parse information of 1.X PAX format"); + } + result = result * 10 + (number - '0'); + } + bytesRead += 1; + + return new long[]{result, bytesRead}; + } + } diff --git a/src/main/java/org/apache/commons/compress/archivers/zip/ZipFile.java b/src/main/java/org/apache/commons/compress/archivers/zip/ZipFile.java index c1a085d..cd52fe1 100644 --- a/src/main/java/org/apache/commons/compress/archivers/zip/ZipFile.java +++ b/src/main/java/org/apache/commons/compress/archivers/zip/ZipFile.java @@ -45,6 +45,8 @@ import java.util.zip.ZipException; import org.apache.commons.compress.archivers.EntryStreamOffsets; import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; import org.apache.commons.compress.compressors.deflate64.Deflate64CompressorInputStream; +import org.apache.commons.compress.utils.BoundedNIOInputStream; +import org.apache.commons.compress.utils.BoundedSeekableByteChannelInputStream; import org.apache.commons.compress.utils.CountingInputStream; import org.apache.commons.compress.utils.IOUtils; import org.apache.commons.compress.utils.InputStreamStatistics; @@ -1314,81 +1316,10 @@ public class ZipFile implements Closeable { * Creates new BoundedInputStream, according to implementation of * underlying archive channel. */ - private BoundedInputStream createBoundedInputStream(final long start, final long remaining) { + private BoundedNIOInputStream createBoundedInputStream(final long start, final long remaining) { return archive instanceof FileChannel ? new BoundedFileChannelInputStream(start, remaining) : - new BoundedInputStream(start, remaining); - } - - /** - * InputStream that delegates requests to the underlying - * SeekableByteChannel, making sure that only bytes from a certain - * range can be read. - */ - private class BoundedInputStream extends InputStream { - private ByteBuffer singleByteBuffer; - private final long end; - private long loc; - - BoundedInputStream(final long start, final long remaining) { - this.end = start+remaining; - if (this.end < start) { - // check for potential vulnerability due to overflow - throw new IllegalArgumentException("Invalid length of stream at offset="+start+", length="+remaining); - } - loc = start; - } - - @Override - public synchronized int read() throws IOException { - if (loc >= end) { - return -1; - } - if (singleByteBuffer == null) { - singleByteBuffer = ByteBuffer.allocate(1); - } - else { - singleByteBuffer.rewind(); - } - final int read = read(loc, singleByteBuffer); - if (read < 0) { - return read; - } - loc++; - return singleByteBuffer.get() & 0xff; - } - - @Override - public synchronized int read(final byte[] b, final int off, int len) throws IOException { - if (len <= 0) { - return 0; - } - - if (len > end-loc) { - if (loc >= end) { - return -1; - } - len = (int)(end-loc); - } - - final ByteBuffer buf; - buf = ByteBuffer.wrap(b, off, len); - final int ret = read(loc, buf); - if (ret > 0) { - loc += ret; - } - return ret; - } - - protected int read(final long pos, final ByteBuffer buf) throws IOException { - final int read; - synchronized (archive) { - archive.position(pos); - read = archive.read(buf); - } - buf.flip(); - return read; - } + new BoundedSeekableByteChannelInputStream(start, remaining, archive); } /** @@ -1397,12 +1328,12 @@ public class ZipFile implements Closeable { * file channel and therefore performs significantly faster in * concurrent environment. */ - private class BoundedFileChannelInputStream extends BoundedInputStream { + private class BoundedFileChannelInputStream extends BoundedNIOInputStream { private final FileChannel archive; BoundedFileChannelInputStream(final long start, final long remaining) { super(start, remaining); - archive = (FileChannel)ZipFile.this.archive; + archive = (FileChannel) ZipFile.this.archive; } @Override diff --git a/src/main/java/org/apache/commons/compress/utils/BoundedNIOInputStream.java b/src/main/java/org/apache/commons/compress/utils/BoundedNIOInputStream.java new file mode 100644 index 0000000..6f8015a --- /dev/null +++ b/src/main/java/org/apache/commons/compress/utils/BoundedNIOInputStream.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.commons.compress.utils; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; + +/** + * NIO backed bounded input stream for reading a predefined amount of data from. + * @since 1.21 + */ +public abstract class BoundedNIOInputStream extends InputStream { + + private final long end; + private ByteBuffer singleByteBuffer; + private long loc; + + /** + * Create a new bounded input stream. + * + * @param start Position in the stream from where the reading of this bounded stream starts + * @param remaining Amount of bytes which are allowed to read from the bounded stream + */ + public BoundedNIOInputStream(final long start, final long remaining) { + this.end = start + remaining; + if (this.end < start) { + // check for potential vulnerability due to overflow + throw new IllegalArgumentException("Invalid length of stream at offset=" + start + ", length=" + remaining); + } + loc = start; + } + + @Override + public synchronized int read() throws IOException { + if (loc >= end) { + return -1; + } + if (singleByteBuffer == null) { + singleByteBuffer = ByteBuffer.allocate(1); + } else { + singleByteBuffer.rewind(); + } + int read = read(loc, singleByteBuffer); + if (read < 0) { + return read; + } + loc++; + return singleByteBuffer.get() & 0xff; + } + + @Override + public synchronized int read(final byte[] b, final int off, int len) throws IOException { + if (len <= 0) { + return 0; + } + + if (len > end - loc) { + if (loc >= end) { + return -1; + } + len = (int) (end - loc); + } + + ByteBuffer buf = ByteBuffer.wrap(b, off, len); + int ret = read(loc, buf); + if (ret > 0) { + loc += ret; + return ret; + } + return ret; + } + + /** + * Read content of the stream into a {@link ByteBuffer} + * @param pos position to start the read + * @param buf buffer to add the read content + * @return Number of read bytes + * @throws IOException If I/O fails + */ + protected abstract int read(long pos, ByteBuffer buf) throws IOException; +} diff --git a/src/main/java/org/apache/commons/compress/utils/BoundedSeekableByteChannelInputStream.java b/src/main/java/org/apache/commons/compress/utils/BoundedSeekableByteChannelInputStream.java new file mode 100644 index 0000000..f8499bd --- /dev/null +++ b/src/main/java/org/apache/commons/compress/utils/BoundedSeekableByteChannelInputStream.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.commons.compress.utils; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.SeekableByteChannel; + +/** + * InputStream that delegates requests to the underlying SeekableByteChannel, making sure that only bytes from a certain + * range can be read. + * @since 1.21 + */ +public class BoundedSeekableByteChannelInputStream extends BoundedNIOInputStream { + + private final SeekableByteChannel channel; + + /** + * Create a bounded stream on the underlying {@link SeekableByteChannel} + * + * @param start Position in the stream from where the reading of this bounded stream starts + * @param remaining Amount of bytes which are allowed to read from the bounded stream + * @param channel Channel which the reads will be delegated to + */ + public BoundedSeekableByteChannelInputStream(final long start, final long remaining, + final SeekableByteChannel channel) { + super(start, remaining); + this.channel = channel; + } + + @Override + protected int read(long pos, ByteBuffer buf) throws IOException { + int read; + synchronized (channel) { + channel.position(pos); + read = channel.read(buf); + } + buf.flip(); + return read; + } +} diff --git a/src/test/java/org/apache/commons/compress/archivers/TarTestCase.java b/src/test/java/org/apache/commons/compress/archivers/TarTestCase.java index 2363f80..f5dc104 100644 --- a/src/test/java/org/apache/commons/compress/archivers/TarTestCase.java +++ b/src/test/java/org/apache/commons/compress/archivers/TarTestCase.java @@ -18,28 +18,31 @@ */ package org.apache.commons.compress.archivers; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.util.List; import java.nio.charset.StandardCharsets; import org.apache.commons.compress.AbstractTestCase; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; +import org.apache.commons.compress.archivers.tar.TarFile; import org.apache.commons.compress.utils.CharsetNames; import org.apache.commons.compress.utils.IOUtils; import org.junit.Test; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + public final class TarTestCase extends AbstractTestCase { @Test @@ -118,26 +121,48 @@ public final class TarTestCase extends AbstractTestCase { @Test public void testTarUnarchive() throws Exception { final File input = getFile("bla.tar"); - final InputStream is = new FileInputStream(input); - final ArchiveInputStream in = ArchiveStreamFactory.DEFAULT.createArchiveInputStream("tar", is); - final TarArchiveEntry entry = (TarArchiveEntry)in.getNextEntry(); - final OutputStream out = new FileOutputStream(new File(dir, entry.getName())); - IOUtils.copy(in, out); - in.close(); - out.close(); + try (final InputStream is = new FileInputStream(input); + final ArchiveInputStream in = ArchiveStreamFactory.DEFAULT.createArchiveInputStream("tar", is)) { + final TarArchiveEntry entry = (TarArchiveEntry) in.getNextEntry(); + try (final OutputStream out = new FileOutputStream(new File(dir, entry.getName()))) { + IOUtils.copy(in, out); + } + } + } + + @Test + public void testTarFileUnarchive() throws Exception { + final File file = getFile("bla.tar"); + try (final TarFile tarFile = new TarFile(file)) { + final TarArchiveEntry entry = tarFile.getEntries().get(0); + try (final OutputStream out = new FileOutputStream(new File(dir, entry.getName()))) { + IOUtils.copy(tarFile.getInputStream(entry), out); + } + } } @Test public void testCOMPRESS114() throws Exception { final File input = getFile("COMPRESS-114.tar"); - final InputStream is = new FileInputStream(input); - final ArchiveInputStream in = new TarArchiveInputStream(is, - CharsetNames.ISO_8859_1); - TarArchiveEntry entry = (TarArchiveEntry)in.getNextEntry(); - assertEquals("3\u00b1\u00b1\u00b1F06\u00b1W2345\u00b1ZB\u00b1la\u00b1\u00b1\u00b1\u00b1\u00b1\u00b1\u00b1\u00b1BLA", entry.getName()); - entry = (TarArchiveEntry)in.getNextEntry(); - assertEquals("0302-0601-3\u00b1\u00b1\u00b1F06\u00b1W2345\u00b1ZB\u00b1la\u00b1\u00b1\u00b1\u00b1\u00b1\u00b1\u00b1\u00b1BLA",entry.getName()); - in.close(); + try (final InputStream is = new FileInputStream(input); + final ArchiveInputStream in = new TarArchiveInputStream(is, CharsetNames.ISO_8859_1)) { + TarArchiveEntry entry = (TarArchiveEntry) in.getNextEntry(); + assertEquals("3\u00b1\u00b1\u00b1F06\u00b1W2345\u00b1ZB\u00b1la\u00b1\u00b1\u00b1\u00b1\u00b1\u00b1\u00b1\u00b1BLA", entry.getName()); + entry = (TarArchiveEntry) in.getNextEntry(); + assertEquals("0302-0601-3\u00b1\u00b1\u00b1F06\u00b1W2345\u00b1ZB\u00b1la\u00b1\u00b1\u00b1\u00b1\u00b1\u00b1\u00b1\u00b1BLA", entry.getName()); + } + } + + @Test + public void testTarFileCOMPRESS114() throws Exception { + final File input = getFile("COMPRESS-114.tar"); + try (final TarFile tarFile = new TarFile(input, CharsetNames.ISO_8859_1)) { + final List<TarArchiveEntry> entries = tarFile.getEntries(); + TarArchiveEntry entry = entries.get(0); + assertEquals("3\u00b1\u00b1\u00b1F06\u00b1W2345\u00b1ZB\u00b1la\u00b1\u00b1\u00b1\u00b1\u00b1\u00b1\u00b1\u00b1BLA", entry.getName()); + entry = entries.get(1); + assertEquals("0302-0601-3\u00b1\u00b1\u00b1F06\u00b1W2345\u00b1ZB\u00b1la\u00b1\u00b1\u00b1\u00b1\u00b1\u00b1\u00b1\u00b1BLA", entry.getName()); + } } @Test @@ -181,6 +206,34 @@ public final class TarTestCase extends AbstractTestCase { } @Test + public void testTarFileDirectoryEntryFromFile() throws Exception { + final File[] tmp = createTempDirAndFile(); + File archive = File.createTempFile("test.", ".tar", tmp[0]); + archive.deleteOnExit(); + + try (final TarArchiveOutputStream tos = new TarArchiveOutputStream(new FileOutputStream(archive))) { + final long beforeArchiveWrite = tmp[0].lastModified(); + final TarArchiveEntry in = new TarArchiveEntry(tmp[0], "foo"); + tos.putArchiveEntry(in); + tos.closeArchiveEntry(); + tos.close(); + try (final TarFile tarFile = new TarFile(archive)) { + TarArchiveEntry entry = tarFile.getEntries().get(0); + assertNotNull(entry); + assertEquals("foo/", entry.getName()); + assertEquals(0, entry.getSize()); + // TAR stores time with a granularity of 1 second + assertEquals(beforeArchiveWrite / 1000, entry.getLastModifiedDate().getTime() / 1000); + assertTrue(entry.isDirectory()); + } finally { + tryHardToDelete(archive); + tryHardToDelete(tmp[1]); + rmdir(tmp[0]); + } + } + } + + @Test public void testExplicitDirectoryEntry() throws Exception { final File[] tmp = createTempDirAndFile(); File archive = null; @@ -221,6 +274,33 @@ public final class TarTestCase extends AbstractTestCase { } @Test + public void testTarFileExplicitDirectoryEntry() throws Exception { + final File[] tmp = createTempDirAndFile(); + File archive = File.createTempFile("test.", ".tar", tmp[0]); + archive.deleteOnExit(); + try (final TarArchiveOutputStream tos = new TarArchiveOutputStream(new FileOutputStream(archive))){ + final long beforeArchiveWrite = tmp[0].lastModified(); + final TarArchiveEntry in = new TarArchiveEntry("foo/"); + in.setModTime(beforeArchiveWrite); + tos.putArchiveEntry(in); + tos.closeArchiveEntry(); + tos.close(); + try (final TarFile tarFile = new TarFile(archive)) { + TarArchiveEntry entry = tarFile.getEntries().get(0); + assertNotNull(entry); + assertEquals("foo/", entry.getName()); + assertEquals(0, entry.getSize()); + assertEquals(beforeArchiveWrite / 1000, entry.getLastModifiedDate().getTime() / 1000); + assertTrue(entry.isDirectory()); + } + } finally { + tryHardToDelete(archive); + tryHardToDelete(tmp[1]); + rmdir(tmp[0]); + } + } + + @Test public void testFileEntryFromFile() throws Exception { final File[] tmp = createTempDirAndFile(); File archive = null; @@ -270,6 +350,37 @@ public final class TarTestCase extends AbstractTestCase { } @Test + public void testTarFileEntryFromFile() throws Exception { + final File[] tmp = createTempDirAndFile(); + File archive = File.createTempFile("test.", ".tar", tmp[0]); + archive.deleteOnExit(); + try (final TarArchiveOutputStream tos = new TarArchiveOutputStream(new FileOutputStream(archive))) { + final TarArchiveEntry in = new TarArchiveEntry(tmp[1], "foo"); + tos.putArchiveEntry(in); + final byte[] b = new byte[(int) tmp[1].length()]; + try (final FileInputStream fis = new FileInputStream(tmp[1])) { + while (fis.read(b) > 0) { + tos.write(b); + } + } + tos.closeArchiveEntry(); + tos.close(); + try (final TarFile tarFile = new TarFile(archive)) { + TarArchiveEntry entry = tarFile.getEntries().get(0); + assertNotNull(entry); + assertEquals("foo", entry.getName()); + assertEquals(tmp[1].length(), entry.getSize()); + assertEquals(tmp[1].lastModified() / 1000, entry.getLastModifiedDate().getTime() / 1000); + assertFalse(entry.isDirectory()); + } + } finally { + tryHardToDelete(archive); + tryHardToDelete(tmp[1]); + rmdir(tmp[0]); + } + } + + @Test public void testExplicitFileEntry() throws Exception { final File[] tmp = createTempDirAndFile(); File archive = null; @@ -321,6 +432,39 @@ public final class TarTestCase extends AbstractTestCase { } @Test + public void testTarFileExplicitFileEntry() throws Exception { + final File[] tmp = createTempDirAndFile(); + File archive = File.createTempFile("test.", ".tar", tmp[0]); + archive.deleteOnExit(); + try (final TarArchiveOutputStream tos = new TarArchiveOutputStream(new FileOutputStream(archive))){ + final TarArchiveEntry in = new TarArchiveEntry("foo"); + in.setModTime(tmp[1].lastModified()); + in.setSize(tmp[1].length()); + tos.putArchiveEntry(in); + final byte[] b = new byte[(int) tmp[1].length()]; + try (final FileInputStream fis = new FileInputStream(tmp[1])) { + while (fis.read(b) > 0) { + tos.write(b); + } + } + tos.closeArchiveEntry(); + + try (final TarFile tarFile = new TarFile(archive)) { + TarArchiveEntry entry = tarFile.getEntries().get(0); + assertNotNull(entry); + assertEquals("foo", entry.getName()); + assertEquals(tmp[1].length(), entry.getSize()); + assertEquals(tmp[1].lastModified() / 1000, entry.getLastModifiedDate().getTime() / 1000); + assertFalse(entry.isDirectory()); + } + } finally { + tryHardToDelete(archive); + tryHardToDelete(tmp[1]); + rmdir(tmp[0]); + } + } + + @Test public void testCOMPRESS178() throws Exception { final File input = getFile("COMPRESS-178.tar"); final InputStream is = new FileInputStream(input); @@ -336,6 +480,18 @@ public final class TarTestCase extends AbstractTestCase { } @Test + public void testTarFileCOMPRESS178() throws Exception { + final File input = getFile("COMPRESS-178.tar"); + try (final TarFile tarFile = new TarFile(input)) { + // Compared to the TarArchiveInputStream all entries are read when instantiating the tar file + fail("Expected IOException"); + } catch (final IOException e) { + final Throwable t = e.getCause(); + assertTrue("Expected cause = IllegalArgumentException", t instanceof IllegalArgumentException); + } + } + + @Test public void testCOMPRESS178Lenient() throws Exception { final File input = getFile("COMPRESS-178.tar"); try (final InputStream is = new FileInputStream(input); @@ -344,4 +500,38 @@ public final class TarTestCase extends AbstractTestCase { } } + @Test + public void testTarFileCOMPRESS178Lenient() throws Exception { + final File input = getFile("COMPRESS-178.tar"); + try (final TarFile tarFile = new TarFile(input, true)) { + // Compared to the TarArchiveInputStream all entries are read when instantiating the tar file + } + } + + @Test + public void testDirectoryRead() throws IOException { + final File input = getFile("directory.tar"); + try (final InputStream is = new FileInputStream(input); + final TarArchiveInputStream in = new TarArchiveInputStream(is)) { + TarArchiveEntry directoryEntry = in.getNextTarEntry(); + assertEquals("directory/", directoryEntry.getName()); + assertTrue(directoryEntry.isDirectory()); + byte[] directoryRead = IOUtils.toByteArray(in); + assertArrayEquals(new byte[0], directoryRead); + } + } + + @Test + public void testTarFileDirectoryRead() throws IOException { + final File input = getFile("directory.tar"); + try (TarFile tarFile = new TarFile(input)) { + TarArchiveEntry directoryEntry = tarFile.getEntries().get(0); + assertEquals("directory/", directoryEntry.getName()); + assertTrue(directoryEntry.isDirectory()); + try (InputStream directoryStream = tarFile.getInputStream(directoryEntry)) { + byte[] directoryRead = IOUtils.toByteArray(directoryStream); + assertArrayEquals(new byte[0], directoryRead); + } + } + } } diff --git a/src/test/java/org/apache/commons/compress/archivers/tar/BigFilesIT.java b/src/test/java/org/apache/commons/compress/archivers/tar/BigFilesIT.java index d9c50eb..9c58a48 100644 --- a/src/test/java/org/apache/commons/compress/archivers/tar/BigFilesIT.java +++ b/src/test/java/org/apache/commons/compress/archivers/tar/BigFilesIT.java @@ -18,65 +18,65 @@ package org.apache.commons.compress.archivers.tar; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; - import java.io.BufferedInputStream; import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.List; import java.util.Random; +import org.apache.commons.compress.AbstractTestCase; import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; import org.junit.Test; -public class BigFilesIT { +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + +public class BigFilesIT extends AbstractTestCase { @Test public void readFileBiggerThan8GByteStar() throws Exception { - readFileBiggerThan8GByte("/8.star.tar.gz"); + readFileBiggerThan8GByte("8.star.tar.gz"); } @Test public void readFileBiggerThan8GBytePosix() throws Exception { - readFileBiggerThan8GByte("/8.posix.tar.gz"); + readFileBiggerThan8GByte("8.posix.tar.gz"); } @Test public void readFileHeadersOfArchiveBiggerThan8GByte() throws Exception { - InputStream in = null; - GzipCompressorInputStream gzin = null; - TarArchiveInputStream tin = null; - try { - in = new BufferedInputStream(BigFilesIT.class - .getResourceAsStream("/8.posix.tar.gz") - ); - gzin = new GzipCompressorInputStream(in); - tin = new TarArchiveInputStream(gzin); + try (InputStream in = new BufferedInputStream(Files.newInputStream(getPath("8.posix.tar.gz"))); + GzipCompressorInputStream gzin = new GzipCompressorInputStream(in); + TarArchiveInputStream tin = new TarArchiveInputStream(gzin)) { final TarArchiveEntry e = tin.getNextTarEntry(); assertNotNull(e); assertNull(tin.getNextTarEntry()); - } finally { - if (tin != null) { - tin.close(); - } - if (gzin != null) { - gzin.close(); - } - if (in != null) { - in.close(); - } + } + } + + @Test + public void tarFileReadFileHeadersOfArchiveBiggerThan8GByte() throws Exception { + Path file = getPath("8.posix.tar.gz"); + Path output = resultDir.toPath().resolve("8.posix.tar"); + try (InputStream in = new BufferedInputStream(Files.newInputStream(file)); + GzipCompressorInputStream gzin = new GzipCompressorInputStream(in)) { + Files.copy(gzin, output, StandardCopyOption.REPLACE_EXISTING); + } + + try (final TarFile tarFile = new TarFile(output)) { + List<TarArchiveEntry> entries = tarFile.getEntries(); + assertEquals(1, entries.size()); + assertNotNull(entries.get(0)); } } private void readFileBiggerThan8GByte(final String name) throws Exception { - InputStream in = null; - GzipCompressorInputStream gzin = null; - TarArchiveInputStream tin = null; - try { - in = new BufferedInputStream(BigFilesIT.class - .getResourceAsStream(name)); - gzin = new GzipCompressorInputStream(in); - tin = new TarArchiveInputStream(gzin); + try (InputStream in = new BufferedInputStream(Files.newInputStream(getPath(name))); + GzipCompressorInputStream gzin = new GzipCompressorInputStream(in); + TarArchiveInputStream tin = new TarArchiveInputStream(gzin);) { final TarArchiveEntry e = tin.getNextTarEntry(); assertNotNull(e); assertEquals(8200L * 1024 * 1024, e.getSize()); @@ -96,16 +96,6 @@ public class BigFilesIT { } assertEquals(8200L * 1024 * 1024, read); assertNull(tin.getNextTarEntry()); - } finally { - if (tin != null) { - tin.close(); - } - if (gzin != null) { - gzin.close(); - } - if (in != null) { - in.close(); - } } } diff --git a/src/test/java/org/apache/commons/compress/archivers/tar/SparseFilesTest.java b/src/test/java/org/apache/commons/compress/archivers/tar/SparseFilesTest.java index 852455d..436b5a8 100644 --- a/src/test/java/org/apache/commons/compress/archivers/tar/SparseFilesTest.java +++ b/src/test/java/org/apache/commons/compress/archivers/tar/SparseFilesTest.java @@ -28,16 +28,19 @@ import static org.junit.Assume.assumeTrue; import org.apache.commons.compress.AbstractTestCase; import org.apache.commons.compress.utils.IOUtils; +import org.junit.Assume; import org.junit.Test; +import java.io.BufferedInputStream; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.List; import java.util.Locale; - public class SparseFilesTest extends AbstractTestCase { private final boolean isOnWindows = System.getProperty("os.name").toLowerCase(Locale.ENGLISH).contains("windows"); @@ -68,6 +71,32 @@ public class SparseFilesTest extends AbstractTestCase { } @Test + public void testTarFileOldGNU() throws Throwable { + final File file = getFile("oldgnu_sparse.tar"); + try (final TarFile tarFile = new TarFile(file)) { + TarArchiveEntry ae = tarFile.getEntries().get(0); + assertEquals("sparsefile", ae.getName()); + assertTrue(ae.isOldGNUSparse()); + assertTrue(ae.isGNUSparse()); + assertFalse(ae.isPaxGNUSparse()); + // TODO: Is this something which should be supported in the random access implementation. Is this even needed in the current stream implementation as it supports sparse entries now? + //assertFalse(tin.canReadEntryData(ae)); + + List<TarArchiveStructSparse> sparseHeaders = ae.getSparseHeaders(); + assertEquals(3, sparseHeaders.size()); + + assertEquals(0, sparseHeaders.get(0).getOffset()); + assertEquals(2048, sparseHeaders.get(0).getNumbytes()); + + assertEquals(1050624L, sparseHeaders.get(1).getOffset()); + assertEquals(2560, sparseHeaders.get(1).getNumbytes()); + + assertEquals(3101184L, sparseHeaders.get(2).getOffset()); + assertEquals(0, sparseHeaders.get(2).getNumbytes()); + } + } + + @Test public void testPaxGNU() throws Throwable { final File file = getFile("pax_gnu_sparse.tar"); try (TarArchiveInputStream tin = new TarArchiveInputStream(new FileInputStream(file))) { @@ -78,6 +107,38 @@ public class SparseFilesTest extends AbstractTestCase { } @Test + public void testTarFilePaxGNU() throws IOException { + final File file = getFile("pax_gnu_sparse.tar"); + try (final TarFile tarFile = new TarFile(file)) { + List<TarArchiveEntry> entries = tarFile.getEntries(); + assertPaxGNUEntry(entries.get(0), "0.0"); + assertPaxGNUEntry(entries.get(1), "0.1"); + assertPaxGNUEntry(entries.get(2), "1.0"); + } + } + + private void assertPaxGNUEntry(final TarArchiveEntry entry, final String suffix) { + assertEquals("sparsefile-" + suffix, entry.getName()); + assertTrue(entry.isGNUSparse()); + assertTrue(entry.isPaxGNUSparse()); + assertFalse(entry.isOldGNUSparse()); + // TODO: Is this something which should be supported in the random access implementation. Is this even needed in the current stream implementation as it supports sparse entries now? + //assertFalse(tin.canReadEntryData(entry)); + + List<TarArchiveStructSparse> sparseHeaders = entry.getSparseHeaders(); + assertEquals(3, sparseHeaders.size()); + + assertEquals(0, sparseHeaders.get(0).getOffset()); + assertEquals(2048, sparseHeaders.get(0).getNumbytes()); + + assertEquals(1050624L, sparseHeaders.get(1).getOffset()); + assertEquals(2560, sparseHeaders.get(1).getNumbytes()); + + assertEquals(3101184L, sparseHeaders.get(2).getOffset()); + assertEquals(0, sparseHeaders.get(2).getNumbytes()); + } + + @Test public void testExtractSparseTarsOnWindows() throws IOException { assumeTrue("This test should be ignored if not running on Windows", isOnWindows); @@ -118,6 +179,47 @@ public class SparseFilesTest extends AbstractTestCase { } @Test + public void testTarFileExtractSparseTarsOnWindows() throws IOException { + Assume.assumeTrue("Only run test on Windows", isOnWindows); + + final File oldGNUSparseTar = getFile("oldgnu_sparse.tar"); + final File paxGNUSparseTar = getFile("pax_gnu_sparse.tar"); + try (TarFile paxGnu = new TarFile(paxGNUSparseTar)) { + List<TarArchiveEntry> entries = paxGnu.getEntries(); + + // compare between old GNU and PAX 0.0 + TarArchiveEntry paxGnuEntry = entries.get(0); + try (TarFile oldGnu = new TarFile(oldGNUSparseTar)) { + TarArchiveEntry oldGnuEntry = oldGnu.getEntries().get(0); + try (InputStream old = oldGnu.getInputStream(oldGnuEntry); + InputStream pax = paxGnu.getInputStream(paxGnuEntry)) { + assertArrayEquals(IOUtils.toByteArray(old), IOUtils.toByteArray(pax)); + } + } + + // compare between old GNU and PAX 0.1 + paxGnuEntry = entries.get(1); + try (TarFile oldGnu = new TarFile(oldGNUSparseTar)) { + TarArchiveEntry oldGnuEntry = oldGnu.getEntries().get(0); + try (InputStream old = oldGnu.getInputStream(oldGnuEntry); + InputStream pax = paxGnu.getInputStream(paxGnuEntry)) { + assertArrayEquals(IOUtils.toByteArray(old), IOUtils.toByteArray(pax)); + } + } + + // compare between old GNU and PAX 1.0 + paxGnuEntry = entries.get(2); + try (TarFile oldGnu = new TarFile(oldGNUSparseTar)) { + TarArchiveEntry oldGnuEntry = oldGnu.getEntries().get(0); + try (InputStream old = oldGnu.getInputStream(oldGnuEntry); + InputStream pax = paxGnu.getInputStream(paxGnuEntry)) { + assertArrayEquals(IOUtils.toByteArray(old), IOUtils.toByteArray(pax)); + } + } + } + } + + @Test public void testExtractOldGNU() throws IOException, InterruptedException { assumeFalse("This test should be ignored on Windows", isOnWindows); @@ -137,6 +239,20 @@ public class SparseFilesTest extends AbstractTestCase { } @Test + public void testTarFileExtractOldGNU() throws IOException, InterruptedException { + Assume.assumeFalse("Don't run test on Windows", isOnWindows); + + File file = getFile("oldgnu_sparse.tar"); + try (final InputStream sparseFileInputStream = extractTarAndGetInputStream(file, "sparsefile"); + final TarFile tarFile = new TarFile(file)) { + TarArchiveEntry entry = tarFile.getEntries().get(0); + try (InputStream tarInput = tarFile.getInputStream(entry)) { + assertArrayEquals(IOUtils.toByteArray(tarInput), IOUtils.toByteArray(sparseFileInputStream)); + } + } + } + + @Test public void testExtractExtendedOldGNU() throws IOException, InterruptedException { assumeFalse("This test should be ignored on Windows", isOnWindows); @@ -176,6 +292,45 @@ public class SparseFilesTest extends AbstractTestCase { } @Test + public void testTarFileExtractExtendedOldGNU() throws IOException, InterruptedException { + Assume.assumeFalse("Don't run test on Windows", isOnWindows); + + final File file = getFile("oldgnu_extended_sparse.tar"); + try (InputStream sparseFileInputStream = extractTarAndGetInputStream(file, "sparse6"); + TarFile tarFile = new TarFile(file)) { + TarArchiveEntry ae = tarFile.getEntries().get(0); + + try (InputStream tarInput = tarFile.getInputStream(ae)) { + assertArrayEquals(IOUtils.toByteArray(tarInput), IOUtils.toByteArray(sparseFileInputStream)); + } + + List<TarArchiveStructSparse> sparseHeaders = ae.getSparseHeaders(); + assertEquals(7, sparseHeaders.size()); + + assertEquals(0, sparseHeaders.get(0).getOffset()); + assertEquals(1024, sparseHeaders.get(0).getNumbytes()); + + assertEquals(10240, sparseHeaders.get(1).getOffset()); + assertEquals(1024, sparseHeaders.get(1).getNumbytes()); + + assertEquals(16384, sparseHeaders.get(2).getOffset()); + assertEquals(1024, sparseHeaders.get(2).getNumbytes()); + + assertEquals(24576, sparseHeaders.get(3).getOffset()); + assertEquals(1024, sparseHeaders.get(3).getNumbytes()); + + assertEquals(29696, sparseHeaders.get(4).getOffset()); + assertEquals(1024, sparseHeaders.get(4).getNumbytes()); + + assertEquals(36864, sparseHeaders.get(5).getOffset()); + assertEquals(1024, sparseHeaders.get(5).getNumbytes()); + + assertEquals(51200, sparseHeaders.get(6).getOffset()); + assertEquals(0, sparseHeaders.get(6).getNumbytes()); + } + } + + @Test public void testExtractPaxGNU() throws IOException, InterruptedException { assumeFalse("This test should be ignored on Windows", isOnWindows); // GNU tar with version 1.28 has some problems reading sparsefile-0.1, @@ -210,6 +365,46 @@ public class SparseFilesTest extends AbstractTestCase { } } + @Test + public void testTarFileExtractPaxGNU() throws IOException, InterruptedException { + Assume.assumeFalse("Don't run test on Windows", isOnWindows); + + final File file = getFile("pax_gnu_sparse.tar"); + try (final TarFile paxGnu = new TarFile(file)) { + List<TarArchiveEntry> entries = paxGnu.getEntries(); + + TarArchiveEntry entry = entries.get(0); + try (InputStream sparseFileInputStream = extractTarAndGetInputStream(file, "sparsefile-0.0"); + InputStream paxInput = paxGnu.getInputStream(entry)) { + assertArrayEquals(IOUtils.toByteArray(paxInput), IOUtils.toByteArray(sparseFileInputStream)); + } + + // TODO : it's wired that I can only get a 0 size sparsefile-0.1 on my Ubuntu 16.04 + // using "tar -xf pax_gnu_sparse.tar" + entry = entries.get(1); + try (InputStream sparseFileInputStream = extractTarAndGetInputStream(file, "sparsefile-0.0"); + InputStream paxInput = paxGnu.getInputStream(entry)) { + assertArrayEquals(IOUtils.toByteArray(paxInput), IOUtils.toByteArray(sparseFileInputStream)); + } + + entry = entries.get(2); + try (InputStream sparseFileInputStream = extractTarAndGetInputStream(file, "sparsefile-1.0"); + InputStream paxInput = paxGnu.getInputStream(entry)) { + assertArrayEquals(IOUtils.toByteArray(paxInput), IOUtils.toByteArray(sparseFileInputStream)); + } + } + } + + @Test + public void compareTarArchiveInputStreamWithTarFile() throws IOException { + Path file = getPath("oldgnu_sparse.tar"); + try (TarArchiveInputStream tarIn = new TarArchiveInputStream(new BufferedInputStream(Files.newInputStream(file))); + TarFile tarFile = new TarFile(file)) { + TarArchiveEntry tarInEntry = tarIn.getNextTarEntry(); + assertArrayEquals(IOUtils.toByteArray(tarIn), IOUtils.toByteArray(tarFile.getInputStream(tarFile.getEntries().get(0)))); + } + } + private void assertPaxGNUEntry(final TarArchiveInputStream tin, final String suffix) throws Throwable { final TarArchiveEntry ae = tin.getNextTarEntry(); assertEquals("sparsefile-" + suffix, ae.getName()); diff --git a/src/test/java/org/apache/commons/compress/archivers/tar/TarArchiveInputStreamTest.java b/src/test/java/org/apache/commons/compress/archivers/tar/TarArchiveInputStreamTest.java index 3acf8fb..e7907b8 100644 --- a/src/test/java/org/apache/commons/compress/archivers/tar/TarArchiveInputStreamTest.java +++ b/src/test/java/org/apache/commons/compress/archivers/tar/TarArchiveInputStreamTest.java @@ -51,67 +51,6 @@ import org.junit.Test; public class TarArchiveInputStreamTest extends AbstractTestCase { @Test - public void readSimplePaxHeader() throws Exception { - final InputStream is = new ByteArrayInputStream(new byte[1]); - final TarArchiveInputStream tais = new TarArchiveInputStream(is); - final Map<String, String> headers = tais - .parsePaxHeaders(new ByteArrayInputStream("30 atime=1321711775.972059463\n" - .getBytes(StandardCharsets.UTF_8)), null); - assertEquals(1, headers.size()); - assertEquals("1321711775.972059463", headers.get("atime")); - tais.close(); - } - - @Test - public void secondEntryWinsWhenPaxHeaderContainsDuplicateKey() throws Exception { - final InputStream is = new ByteArrayInputStream(new byte[1]); - final TarArchiveInputStream tais = new TarArchiveInputStream(is); - final Map<String, String> headers = tais - .parsePaxHeaders(new ByteArrayInputStream("11 foo=bar\n11 foo=baz\n" - .getBytes(StandardCharsets.UTF_8)), null); - assertEquals(1, headers.size()); - assertEquals("baz", headers.get("foo")); - tais.close(); - } - - @Test - public void paxHeaderEntryWithEmptyValueRemovesKey() throws Exception { - final InputStream is = new ByteArrayInputStream(new byte[1]); - final TarArchiveInputStream tais = new TarArchiveInputStream(is); - final Map<String, String> headers = tais - .parsePaxHeaders(new ByteArrayInputStream("11 foo=bar\n7 foo=\n" - .getBytes(StandardCharsets.UTF_8)), null); - assertEquals(0, headers.size()); - tais.close(); - } - - @Test - public void readPaxHeaderWithEmbeddedNewline() throws Exception { - final InputStream is = new ByteArrayInputStream(new byte[1]); - final TarArchiveInputStream tais = new TarArchiveInputStream(is); - final Map<String, String> headers = tais - .parsePaxHeaders(new ByteArrayInputStream("28 comment=line1\nline2\nand3\n" - .getBytes(StandardCharsets.UTF_8)), null); - assertEquals(1, headers.size()); - assertEquals("line1\nline2\nand3", headers.get("comment")); - tais.close(); - } - - @Test - public void readNonAsciiPaxHeader() throws Exception { - final String ae = "\u00e4"; - final String line = "11 path="+ ae + "\n"; - assertEquals(11, line.getBytes(StandardCharsets.UTF_8).length); - final InputStream is = new ByteArrayInputStream(new byte[1]); - final TarArchiveInputStream tais = new TarArchiveInputStream(is); - final Map<String, String> headers = tais - .parsePaxHeaders(new ByteArrayInputStream(line.getBytes(StandardCharsets.UTF_8)), null); - assertEquals(1, headers.size()); - assertEquals(ae, headers.get("path")); - tais.close(); - } - - @Test public void workaroundForBrokenTimeHeader() throws Exception { try (TarArchiveInputStream in = new TarArchiveInputStream(new FileInputStream(getFile("simple-aix-native-tar.tar")))) { TarArchiveEntry tae = in.getNextTarEntry(); diff --git a/src/test/java/org/apache/commons/compress/archivers/tar/TarFileTest.java b/src/test/java/org/apache/commons/compress/archivers/tar/TarFileTest.java new file mode 100644 index 0000000..a453d3b --- /dev/null +++ b/src/test/java/org/apache/commons/compress/archivers/tar/TarFileTest.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.commons.compress.archivers.tar; + +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; + +import org.apache.commons.compress.AbstractTestCase; +import org.apache.commons.compress.archivers.ArchiveException; +import org.junit.Test; + +import static org.junit.Assert.assertTrue; + +public class TarFileTest extends AbstractTestCase { + + @Test + public void testDirectoryWithLongNameEndsWithSlash() throws IOException, ArchiveException { + final String rootPath = dir.getAbsolutePath(); + final String dirDirectory = "COMPRESS-509"; + final int count = 100; + File root = new File(rootPath + "/" + dirDirectory); + root.mkdirs(); + for (int i = 1; i < count; i++) { + // ----------------------- + // create empty dirs with incremental length + // ----------------------- + StringBuilder subDirBuilder = new StringBuilder(); + for (int j = 0; j < i; j++) { + subDirBuilder.append("a"); + } + String subDir = subDirBuilder.toString(); + File dir = new File(rootPath + "/" + dirDirectory, "/" + subDir); + dir.mkdir(); + + // ----------------------- + // tar these dirs + // ----------------------- + String fileName = "/" + dirDirectory + "/" + subDir; + File tarF = new File(rootPath + "/tar" + i + ".tar"); + try (OutputStream dest = Files.newOutputStream(tarF.toPath()); + TarArchiveOutputStream out = new TarArchiveOutputStream(new BufferedOutputStream(dest))) { + out.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_STAR); + out.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU); + + File file = new File(rootPath, fileName); + TarArchiveEntry entry = new TarArchiveEntry(file); + entry.setName(fileName); + out.putArchiveEntry(entry); + out.closeArchiveEntry(); + out.flush(); + } + // ----------------------- + // untar these tars + // ----------------------- + try (TarFile tarFile = new TarFile(tarF)) { + for (TarArchiveEntry entry : tarFile.getEntries()) { + assertTrue("Entry name: " + entry.getName(), entry.getName().endsWith("/")); + } + } + } + } + + @Test(expected = IOException.class) + public void testParseTarWithNonNumberPaxHeaders() throws IOException { + try (TarFile tarFile = new TarFile(getPath("COMPRESS-529.tar"))) { + } + } + +} diff --git a/src/test/java/org/apache/commons/compress/archivers/tar/TarUtilsTest.java b/src/test/java/org/apache/commons/compress/archivers/tar/TarUtilsTest.java index 893284b..4eef161 100644 --- a/src/test/java/org/apache/commons/compress/archivers/tar/TarUtilsTest.java +++ b/src/test/java/org/apache/commons/compress/archivers/tar/TarUtilsTest.java @@ -18,17 +18,24 @@ package org.apache.commons.compress.archivers.tar; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import java.io.ByteArrayInputStream; +import java.io.FileInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; import org.apache.commons.compress.archivers.zip.ZipEncoding; import org.apache.commons.compress.archivers.zip.ZipEncodingHelper; import org.apache.commons.compress.utils.CharsetNames; +import org.apache.commons.compress.utils.IOUtils; import org.junit.Test; -import java.nio.charset.StandardCharsets; +import static org.apache.commons.compress.AbstractTestCase.getFile; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; public class TarUtilsTest { @@ -395,4 +402,58 @@ public class TarUtilsTest { assertEquals(sparse.getNumbytes(), expectedNumbytes); } + @Test + public void readSimplePaxHeader() throws Exception { + final Map<String, String> headers = TarUtils.parsePaxHeaders( + new ByteArrayInputStream("30 atime=1321711775.972059463\n".getBytes(StandardCharsets.UTF_8)), + null, new HashMap<String, String>()); + assertEquals(1, headers.size()); + assertEquals("1321711775.972059463", headers.get("atime")); + } + + @Test + public void secondEntryWinsWhenPaxHeaderContainsDuplicateKey() throws Exception { + final Map<String, String> headers = TarUtils.parsePaxHeaders(new ByteArrayInputStream("11 foo=bar\n11 foo=baz\n" + .getBytes(StandardCharsets.UTF_8)), null, new HashMap<String, String>()); + assertEquals(1, headers.size()); + assertEquals("baz", headers.get("foo")); + } + + @Test + public void paxHeaderEntryWithEmptyValueRemovesKey() throws Exception { + final Map<String, String> headers = TarUtils + .parsePaxHeaders(new ByteArrayInputStream("11 foo=bar\n7 foo=\n" + .getBytes(StandardCharsets.UTF_8)), null, new HashMap<String, String>()); + assertEquals(0, headers.size()); + } + + @Test + public void readPaxHeaderWithEmbeddedNewline() throws Exception { + final Map<String, String> headers = TarUtils + .parsePaxHeaders(new ByteArrayInputStream("28 comment=line1\nline2\nand3\n" + .getBytes(StandardCharsets.UTF_8)), null, new HashMap<String, String>()); + assertEquals(1, headers.size()); + assertEquals("line1\nline2\nand3", headers.get("comment")); + } + + @Test + public void readNonAsciiPaxHeader() throws Exception { + final String ae = "\u00e4"; + final String line = "11 path="+ ae + "\n"; + assertEquals(11, line.getBytes(StandardCharsets.UTF_8).length); + final Map<String, String> headers = TarUtils + .parsePaxHeaders(new ByteArrayInputStream(line.getBytes(StandardCharsets.UTF_8)), null, new HashMap<String, String>()); + assertEquals(1, headers.size()); + assertEquals(ae, headers.get("path")); + } + + @Test(expected = IOException.class) + public void testParseTarWithSpecialPaxHeaders() throws IOException { + try (FileInputStream in = new FileInputStream(getFile("COMPRESS-530.tar")); + TarArchiveInputStream archive = new TarArchiveInputStream(in)) { + archive.getNextEntry(); + IOUtils.toByteArray(archive); + } + } + } diff --git a/src/test/java/org/apache/commons/compress/utils/BoundedSeekableByteChannelInputStreamTest.java b/src/test/java/org/apache/commons/compress/utils/BoundedSeekableByteChannelInputStreamTest.java new file mode 100644 index 0000000..0c97b2d --- /dev/null +++ b/src/test/java/org/apache/commons/compress/utils/BoundedSeekableByteChannelInputStreamTest.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.commons.compress.utils; + +import org.junit.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +public class BoundedSeekableByteChannelInputStreamTest { + + @Test + public void checkRestrictedRead() throws IOException { + byte[] readContent; + try (BoundedSeekableByteChannelInputStream input = new BoundedSeekableByteChannelInputStream(0, 5, + new SeekableInMemoryByteChannel("Hello World!".getBytes(StandardCharsets.UTF_8)))) { + readContent = IOUtils.toByteArray(input); + } + assertEquals(5, readContent.length); + assertArrayEquals("Hello".getBytes(StandardCharsets.UTF_8), readContent); + } + +} \ No newline at end of file diff --git a/src/test/resources/directory.tar b/src/test/resources/directory.tar new file mode 100644 index 0000000..5edccfa Binary files /dev/null and b/src/test/resources/directory.tar differ
