Author: bodewig
Date: Sat Jul 30 14:08:56 2011
New Revision: 1152467
URL: http://svn.apache.org/viewvc?rev=1152467&view=rev
Log:
second of seven permutations: size known, compression, writing to a stream.
Lesson learned: don't trust Deflater.getBytesRead, likely don't trust any of
the related methods either, must fix later. COMPRESS-150
Modified:
commons/proper/compress/branches/zip64/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveOutputStream.java
commons/proper/compress/branches/zip64/src/test/java/org/apache/commons/compress/archivers/zip/Zip64SupportTest.java
Modified:
commons/proper/compress/branches/zip64/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveOutputStream.java
URL:
http://svn.apache.org/viewvc/commons/proper/compress/branches/zip64/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveOutputStream.java?rev=1152467&r1=1152466&r2=1152467&view=diff
==============================================================================
---
commons/proper/compress/branches/zip64/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveOutputStream.java
(original)
+++
commons/proper/compress/branches/zip64/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveOutputStream.java
Sat Jul 30 14:08:56 2011
@@ -170,6 +170,14 @@ public class ZipArchiveOutputStream exte
private long cdLength = 0;
/**
+ * Number of bytes read for the current entry (can't rely on
+ * Deflater#getBytesRead) when using DEFLATED.
+ *
+ * @since Apache Commons Compress 1.3
+ */
+ private long bytesRead = 0;
+
+ /**
* Helper, a 0 as ZipShort.
*/
private static final byte[] ZERO = {0, 0};
@@ -393,7 +401,12 @@ public class ZipArchiveOutputStream exte
deflate();
}
+ /* It turns out def.getBytesRead() returns wrong values if
+ * the size exceeds 4 GB - no idea whether one can trust
+ * def.getBytesWritten()
entry.setSize(def.getBytesRead());
+ */
+ entry.setSize(bytesRead);
entry.setCompressedSize(def.getBytesWritten());
entry.setCrc(realCrc);
@@ -438,6 +451,7 @@ public class ZipArchiveOutputStream exte
writeDataDescriptor(entry);
entry = null;
+ bytesRead = 0;
}
/**
@@ -569,6 +583,7 @@ public class ZipArchiveOutputStream exte
ZipUtil.checkRequestedFeatures(entry);
if (entry.getMethod() == DEFLATED) {
if (length > 0 && !def.finished()) {
+ bytesRead += length;
if (length <= DEFLATER_BLOCK_SIZE) {
def.setInput(b, offset, length);
deflateUntilInputIsNeeded();
@@ -782,15 +797,15 @@ public class ZipArchiveOutputStream exte
return;
}
writeOut(DD_SIG);
- writeOut(ZipLong.getBytes(entry.getCrc()));
+ writeOut(ZipLong.getBytes(ze.getCrc()));
int sizeFieldSize = WORD;
if (!hasZip64Extra(ze)) {
- writeOut(ZipLong.getBytes(entry.getCompressedSize()));
- writeOut(ZipLong.getBytes(entry.getSize()));
+ writeOut(ZipLong.getBytes(ze.getCompressedSize()));
+ writeOut(ZipLong.getBytes(ze.getSize()));
} else {
sizeFieldSize = DWORD;
- writeOut(ZipEightByteInteger.getBytes(entry.getCompressedSize()));
- writeOut(ZipEightByteInteger.getBytes(entry.getSize()));
+ writeOut(ZipEightByteInteger.getBytes(ze.getCompressedSize()));
+ writeOut(ZipEightByteInteger.getBytes(ze.getSize()));
}
written += 2 * WORD + 2 * sizeFieldSize;
}
@@ -811,16 +826,13 @@ public class ZipArchiveOutputStream exte
if (needsZip64Extra) {
Zip64ExtendedInformationExtraField z64 = getZip64Extra(ze);
- if (ze.getCompressedSize() >= ZIP64_MAGIC) {
+ if (ze.getCompressedSize() >= ZIP64_MAGIC
+ || ze.getSize() >= ZIP64_MAGIC) {
z64.setCompressedSize(new
ZipEightByteInteger(ze.getCompressedSize()));
- } else {
- // reset value that may have been set for LFH
- z64.setCompressedSize(null);
- }
- if (ze.getSize() >= ZIP64_MAGIC) {
z64.setSize(new ZipEightByteInteger(ze.getSize()));
} else {
// reset value that may have been set for LFH
+ z64.setCompressedSize(null);
z64.setSize(null);
}
if (lfhOffset >= ZIP64_MAGIC) {
@@ -856,8 +868,14 @@ public class ZipArchiveOutputStream exte
// compressed length
// uncompressed length
writeOut(ZipLong.getBytes(ze.getCrc()));
- writeOut(ZipLong.getBytes(Math.min(ze.getCompressedSize(),
ZIP64_MAGIC)));
- writeOut(ZipLong.getBytes(Math.min(ze.getSize(), ZIP64_MAGIC)));
+ if (ze.getCompressedSize() >= ZIP64_MAGIC
+ || ze.getSize() >= ZIP64_MAGIC) {
+ writeOut(ZipLong.ZIP64_MAGIC.getBytes());
+ writeOut(ZipLong.ZIP64_MAGIC.getBytes());
+ } else {
+ writeOut(ZipLong.getBytes(ze.getCompressedSize()));
+ writeOut(ZipLong.getBytes(ze.getSize()));
+ }
// CheckStyle:MagicNumber OFF
written += 12;
// CheckStyle:MagicNumber ON
@@ -1133,7 +1151,7 @@ public class ZipArchiveOutputStream exte
private Zip64ExtendedInformationExtraField
getZip64Extra(ZipArchiveEntry ze) {
hasUsedZip64 = true;
- Zip64ExtendedInformationExtraField z64 =
+ Zip64ExtendedInformationExtraField z64 =
(Zip64ExtendedInformationExtraField)
ze.getExtraField(Zip64ExtendedInformationExtraField
.HEADER_ID);
Modified:
commons/proper/compress/branches/zip64/src/test/java/org/apache/commons/compress/archivers/zip/Zip64SupportTest.java
URL:
http://svn.apache.org/viewvc/commons/proper/compress/branches/zip64/src/test/java/org/apache/commons/compress/archivers/zip/Zip64SupportTest.java?rev=1152467&r1=1152466&r2=1152467&view=diff
==============================================================================
---
commons/proper/compress/branches/zip64/src/test/java/org/apache/commons/compress/archivers/zip/Zip64SupportTest.java
(original)
+++
commons/proper/compress/branches/zip64/src/test/java/org/apache/commons/compress/archivers/zip/Zip64SupportTest.java
Sat Jul 30 14:08:56 2011
@@ -213,16 +213,7 @@ public class Zip64SupportTest {
RandomAccessFile a = new RandomAccessFile(f, "r");
try {
- final long end = a.length();
- long cdOffsetLoc = end - 22 /* eocd.length */
- - 20 /* z64 eocd locator.length */
- - 56 /* z64 eocd without extensible data sector */
- + 48 /* position in z64 eocd */;
- // seek to central directory
- a.seek(cdOffsetLoc);
- byte[] cdOffset = new byte[8];
- a.readFully(cdOffset);
- a.seek(ZipEightByteInteger.getLongValue(cdOffset));
+ final long end = getLengthAndPositionAtCentralDirectory(a);
// skip first two entries
a.skipBytes(2 * 47 /* CD entry of file with
file name length 1 and no
@@ -339,17 +330,8 @@ public class Zip64SupportTest {
RandomAccessFile a =
new RandomAccessFile(f, "r");
try {
- final long end = a.length();
- long cdOffsetLoc = end - 22
- - 20
- - 56
- + 48;
- // seek to central directory
- a.seek(cdOffsetLoc);
- byte[] cdOffset = new byte[8];
- a.readFully(cdOffset);
- a.seek(ZipEightByteInteger
- .getLongValue(cdOffset));
+ final long end =
+
getLengthAndPositionAtCentralDirectory(a);
// grab first entry, verify
// sizes are 0xFFFFFFFF and
@@ -473,7 +455,6 @@ public class Zip64SupportTest {
*
* Creates a temporary archive of approx 4MB in size
*/
- @Ignore
@Test public void writeBigDeflatedEntryKnownSizeToStream()
throws Throwable {
withTemporaryArchive("writeBigDeflatedEntryKnownSizeToStream",
@@ -498,18 +479,10 @@ public class Zip64SupportTest {
RandomAccessFile a =
new RandomAccessFile(f, "r");
try {
- final long end = a.length();
- long cdOffsetLoc = end - 22
- - 20
- - 56
- + 48;
- // seek to central directory
- a.seek(cdOffsetLoc);
- byte[] cdOffset = new byte[8];
- a.readFully(cdOffset);
- a.seek(ZipEightByteInteger
- .getLongValue(cdOffset));
+ final long end =
+
getLengthAndPositionAtCentralDirectory(a);
+ long cfhPos = a.getFilePointer();
// grab first entry, verify
// sizes are 0xFFFFFFFF and
// it has a ZIP64 extended
@@ -523,35 +496,29 @@ public class Zip64SupportTest {
45, 0,
// version needed to extract
45, 0,
- // GPB
+ // GPB (EFS + Data Descriptor)
8, 8,
// method
8, 0,
}, header);
// ignore timestamp
a.skipBytes(4);
- byte[] crc = new byte[4];
- a.readFully(crc);
+ byte[] rest = new byte[31];
+ a.readFully(rest);
assertArrayEquals(new byte[] {
+ // CRC
(byte) 0x50, (byte) 0x6F,
(byte) 0x31, (byte) 0x5c,
- }, crc);
- // ignore compressed size,
- // check it is smaller than
- // 4GB by validating it is
- // not part of the ZIP64
- // extra field
- a.skipBytes(4);
- byte[] rest = new byte[23];
- a.readFully(rest);
- assertArrayEquals(new byte[] {
+ // Compressed Size
+ (byte) 0xFF, (byte) 0xFF,
+ (byte) 0xFF, (byte) 0xFF,
// Original Size
(byte) 0xFF, (byte) 0xFF,
(byte) 0xFF, (byte) 0xFF,
// file name length
1, 0,
// extra field length
- 12, 0,
+ 20, 0,
// comment length
0, 0,
// disk number
@@ -571,12 +538,37 @@ public class Zip64SupportTest {
// Header-ID
1, 0,
// size of extra
- 8, 0,
+ 16, 0,
// original size
0, (byte) 0xF2, 5, (byte)
0x2A,
1, 0, 0, 0,
+ // don't know the
+ // compressed size,
+ // don't want to
+ // hard-code it
}, extra);
+ // validate data descriptor
+ a.seek(cfhPos - 24);
+ byte[] dd = new byte[8];
+ a.readFully(dd);
+ assertArrayEquals(new byte[] {
+ // sig
+ (byte) 0x50, (byte) 0x4b, 7,
8,
+ // CRC
+ (byte) 0x50, (byte) 0x6F,
+ (byte) 0x31, (byte) 0x5c,
+ }, dd);
+ // skip uncompressed size
+ a.skipBytes(8);
+ dd = new byte[8];
+ a.readFully(dd);
+ assertArrayEquals(new byte[] {
+ // original size
+ 0, (byte) 0xF2, 5, (byte)
0x2A,
+ 1, 0, 0, 0,
+ }, dd);
+
// and now validate local file header
a.seek(0);
header = new byte[10];
@@ -623,8 +615,6 @@ public class Zip64SupportTest {
} finally {
a.close();
}
-
- read5GBOfZerosImpl(f, "0");
}
},
false);
@@ -717,4 +707,16 @@ public class Zip64SupportTest {
}
}
}
+
+ private static long
getLengthAndPositionAtCentralDirectory(RandomAccessFile a)
+ throws IOException {
+ final long end = a.length();
+ long cdOffsetLoc = end - 22 - 20 - 56 + 48;
+ // seek to central directory locator
+ a.seek(cdOffsetLoc);
+ byte[] cdOffset = new byte[8];
+ a.readFully(cdOffset);
+ a.seek(ZipEightByteInteger.getLongValue(cdOffset));
+ return end;
+ }
}
