Author: bodewig
Date: Sat Jul 30 06:17:24 2011
New Revision: 1152430
URL: http://svn.apache.org/viewvc?rev=1152430&view=rev
Log:
ZIP64 support for writing big archive entries in the most simple of the seven
possible permutations: no compression + stream + size known upfront.
COMPRESS-150
Modified:
commons/proper/compress/branches/zip64/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveOutputStream.java
commons/proper/compress/branches/zip64/src/main/java/org/apache/commons/compress/archivers/zip/ZipEightByteInteger.java
commons/proper/compress/branches/zip64/src/test/java/org/apache/commons/compress/archivers/zip/Zip64SupportTest.java
Modified:
commons/proper/compress/branches/zip64/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveOutputStream.java
URL:
http://svn.apache.org/viewvc/commons/proper/compress/branches/zip64/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveOutputStream.java?rev=1152430&r1=1152429&r2=1152430&view=diff
==============================================================================
---
commons/proper/compress/branches/zip64/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveOutputStream.java
(original)
+++
commons/proper/compress/branches/zip64/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveOutputStream.java
Sat Jul 30 06:17:24 2011
@@ -180,11 +180,6 @@ public class ZipArchiveOutputStream exte
private static final byte[] LZERO = {0, 0, 0, 0};
/**
- * Helper, a 0 as ZipEightByteInteger.
- */
- private static final byte[] DLZERO = {0, 0, 0, 0, 0, 0, 0, 0};
-
- /**
* Holds the offsets of the LFH starts for each entry.
*/
private final Map<ZipArchiveEntry, Long> offsets =
@@ -484,6 +479,26 @@ public class ZipArchiveOutputStream exte
entry.setCompressedSize(entry.getSize());
}
+ // add a ZIP64 extended information extra field if we already
+ // know it is going to be needed
+ if (entry.getSize() >= ZIP64_MAGIC
+ || entry.getCompressedSize() >= ZIP64_MAGIC) {
+
+ Zip64ExtendedInformationExtraField z64 = getZip64Extra(entry);
+ if (entry.getMethod() == STORED) {
+ ZipEightByteInteger size =
+ new ZipEightByteInteger(entry.getSize());
+ z64.setSize(size);
+ z64.setCompressedSize(size);
+ } else {
+ // just a placeholder, real data will be in data
+ // descriptor or inserted later via RandomAccessFile
+ z64.setSize(ZipEightByteInteger.ZERO);
+ z64.setCompressedSize(ZipEightByteInteger.ZERO);
+ }
+ entry.setExtra();
+ }
+
if (entry.getMethod() == DEFLATED && hasCompressionLevelChanged) {
def.setLevel(level);
hasCompressionLevelChanged = false;
@@ -708,7 +723,7 @@ public class ZipArchiveOutputStream exte
writeVersionNeededToExtractAndGeneralPurposeBits(zipMethod,
!encodable
&& fallbackToUTF8,
- false);
+ hasZip64Extra(ze));
written += WORD;
// compression method
@@ -729,8 +744,9 @@ public class ZipArchiveOutputStream exte
writeOut(LZERO);
} else {
writeOut(ZipLong.getBytes(ze.getCrc()));
- writeOut(ZipLong.getBytes(ze.getSize()));
- writeOut(ZipLong.getBytes(ze.getSize()));
+ byte[] size = ZipLong.getBytes(Math.min(ze.getSize(),
ZIP64_MAGIC));
+ writeOut(size);
+ writeOut(size);
}
// CheckStyle:MagicNumber OFF
written += 12;
@@ -1122,8 +1138,14 @@ public class ZipArchiveOutputStream exte
ze.getExtraField(Zip64ExtendedInformationExtraField
.HEADER_ID);
if (z64 == null) {
+ /*
+ System.err.println("Adding z64 for " + ze.getName()
+ + ", method: " + ze.getMethod()
+ + " (" + (ze.getMethod() == STORED) + ")"
+ + ", raf: " + (raf != null));
+ */
z64 = new Zip64ExtendedInformationExtraField();
- ze.addExtraField(z64);
+ ze.addAsFirstExtraField(z64);
}
return z64;
}
Modified:
commons/proper/compress/branches/zip64/src/main/java/org/apache/commons/compress/archivers/zip/ZipEightByteInteger.java
URL:
http://svn.apache.org/viewvc/commons/proper/compress/branches/zip64/src/main/java/org/apache/commons/compress/archivers/zip/ZipEightByteInteger.java?rev=1152430&r1=1152429&r2=1152430&view=diff
==============================================================================
---
commons/proper/compress/branches/zip64/src/main/java/org/apache/commons/compress/archivers/zip/ZipEightByteInteger.java
(original)
+++
commons/proper/compress/branches/zip64/src/main/java/org/apache/commons/compress/archivers/zip/ZipEightByteInteger.java
Sat Jul 30 06:17:24 2011
@@ -63,6 +63,8 @@ public final class ZipEightByteInteger {
private final BigInteger value;
+ public static ZipEightByteInteger ZERO = new ZipEightByteInteger(0);
+
/**
* Create instance from a number.
* @param value the long to store as a ZipEightByteInteger
Modified:
commons/proper/compress/branches/zip64/src/test/java/org/apache/commons/compress/archivers/zip/Zip64SupportTest.java
URL:
http://svn.apache.org/viewvc/commons/proper/compress/branches/zip64/src/test/java/org/apache/commons/compress/archivers/zip/Zip64SupportTest.java?rev=1152430&r1=1152429&r2=1152430&view=diff
==============================================================================
---
commons/proper/compress/branches/zip64/src/test/java/org/apache/commons/compress/archivers/zip/Zip64SupportTest.java
(original)
+++
commons/proper/compress/branches/zip64/src/test/java/org/apache/commons/compress/archivers/zip/Zip64SupportTest.java
Sat Jul 30 06:17:24 2011
@@ -317,7 +317,6 @@ public class Zip64SupportTest {
a.close();
}
}
-
};
@Ignore
@@ -334,6 +333,165 @@ public class Zip64SupportTest {
false);
}
+ /*
+ * One entry of length 5 billion bytes, written without
+ * compression to a stream.
+ *
+ * No Compression + Stream => sizes must be known before data is
+ * written and are stored directly inside the LFH. No Data
+ * Descriptor at all.
+ *
+ * Creates a temporary archive of approx 5GB in size
+ */
+ @Test public void writeBigStoredEntryToStream() throws Throwable {
+ withTemporaryArchive("writeBigStoredEntryToStream",
+ new ZipOutputTest() {
+ public void test(File f,
+ ZipArchiveOutputStream zos)
+ throws IOException {
+ byte[] buf = new byte[1000 * 1000];
+ ZipArchiveEntry zae =
+ new ZipArchiveEntry("0");
+ zae.setSize(FIVE_BILLION);
+ zae.setMethod(ZipArchiveEntry.STORED);
+ zae.setCrc(0x5c316f50L);
+ zos.putArchiveEntry(zae);
+ for (int j = 0;
+ j < FIVE_BILLION / 1000 / 1000;
+ j++) {
+ zos.write(buf);
+ }
+ zos.closeArchiveEntry();
+ zos.close();
+
+ RandomAccessFile a =
+ new RandomAccessFile(f, "r");
+ try {
+ final long end = a.length();
+ long cdOffsetLoc = end - 22
+ - 20
+ - 56
+ + 48;
+ // seek to central directory
+ a.seek(cdOffsetLoc);
+ byte[] cdOffset = new byte[8];
+ a.readFully(cdOffset);
+ a.seek(ZipEightByteInteger
+ .getLongValue(cdOffset));
+
+ // grab first entry, verify
+ // sizes are 0xFFFFFFFF and
+ // it has a ZIP64 extended
+ // information extra field
+ byte[] header = new byte[8];
+ a.readFully(header);
+ assertArrayEquals(new byte[] {
+ // sig
+ (byte) 0x50, (byte) 0x4b, 1,
2,
+ // version made by
+ 45, 0,
+ // version needed to extract
+ 45, 0,
+ }, header);
+ // ignore GPB, method, timestamp
+ a.skipBytes(8);
+ byte[] rest = new byte[31];
+ a.readFully(rest);
+ assertArrayEquals(new byte[] {
+ // CRC
+ (byte) 0x50, (byte) 0x6F,
+ (byte) 0x31, (byte) 0x5c,
+ // Compressed Size
+ (byte) 0xFF, (byte) 0xFF,
+ (byte) 0xFF, (byte) 0xFF,
+ // Original Size
+ (byte) 0xFF, (byte) 0xFF,
+ (byte) 0xFF, (byte) 0xFF,
+ // file name length
+ 1, 0,
+ // extra field length
+ 20, 0,
+ // comment length
+ 0, 0,
+ // disk number
+ 0, 0,
+ // attributes
+ 0, 0,
+ 0, 0, 0, 0,
+ // offset
+ 0, 0, 0, 0,
+ // file name
+ (byte) '0'
+ }, rest);
+ byte[] extra = new byte[20];
+ a.readFully(extra);
+ // 5e9 == 0x12A05F200
+ assertArrayEquals(new byte[] {
+ // Header-ID
+ 1, 0,
+ // size of extra
+ 16, 0,
+ // original size
+ 0, (byte) 0xF2, 5, (byte)
0x2A,
+ 1, 0, 0, 0,
+ // compressed size
+ 0, (byte) 0xF2, 5, (byte)
0x2A,
+ 1, 0, 0, 0,
+ }, extra);
+
+ // and now validate local file header
+ a.seek(0);
+ header = new byte[6];
+ a.readFully(header);
+ assertArrayEquals(new byte[] {
+ // sig
+ (byte) 0x50, (byte) 0x4b, 3,
4,
+ // version needed to extract
+ 45, 0,
+ }, header);
+ // ignore GPB, method, timestamp
+ a.skipBytes(8);
+ rest = new byte[17];
+ a.readFully(rest);
+ assertArrayEquals(new byte[] {
+ // CRC
+ (byte) 0x50, (byte) 0x6F,
+ (byte) 0x31, (byte) 0x5c,
+ // Compressed Size
+ (byte) 0xFF, (byte) 0xFF,
+ (byte) 0xFF, (byte) 0xFF,
+ // Original Size
+ (byte) 0xFF, (byte) 0xFF,
+ (byte) 0xFF, (byte) 0xFF,
+ // file name length
+ 1, 0,
+ // extra field length
+ 20, 0,
+ // file name
+ (byte) '0'
+ }, rest);
+ a.readFully(extra);
+ // 5e9 == 0x12A05F200
+ assertArrayEquals(new byte[] {
+ // Header-ID
+ 1, 0,
+ // size of extra
+ 16, 0,
+ // original size
+ 0, (byte) 0xF2, 5, (byte)
0x2A,
+ 1, 0, 0, 0,
+ // compressed size
+ 0, (byte) 0xF2, 5, (byte)
0x2A,
+ 1, 0, 0, 0,
+ }, extra);
+ } finally {
+ a.close();
+ }
+ }
+ },
+ false);
+ }
+
static interface ZipOutputTest {
void test(File f, ZipArchiveOutputStream zos) throws IOException;
}
