JAMES-2426 Store size metadata in ZIP
Project: http://git-wip-us.apache.org/repos/asf/james-project/repo Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/feb18013 Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/feb18013 Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/feb18013 Branch: refs/heads/master Commit: feb18013f50d9fad042f89f2dae63725b9372e41 Parents: 84338dc Author: Raphael Ouazana <[email protected]> Authored: Thu Jun 14 17:29:15 2018 +0200 Committer: benwa <[email protected]> Committed: Tue Jun 19 15:07:55 2018 +0700 ---------------------------------------------------------------------- .../james/mailbox/backup/SizeExtraField.java | 110 ++++++++++ .../org/apache/james/mailbox/backup/Zipper.java | 9 +- .../mailbox/backup/MailboxMessageFixture.java | 4 +- .../mailbox/backup/SizeExtraFieldTest.java | 202 +++++++++++++++++++ .../mailbox/backup/ZipArchiveEntryAssert.java | 25 +++ .../apache/james/mailbox/backup/ZipperTest.java | 13 ++ 6 files changed, 359 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/james-project/blob/feb18013/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/SizeExtraField.java ---------------------------------------------------------------------- diff --git a/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/SizeExtraField.java b/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/SizeExtraField.java new file mode 100644 index 0000000..4da7f9c --- /dev/null +++ b/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/SizeExtraField.java @@ -0,0 +1,110 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.backup; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.Objects; +import java.util.Optional; +import java.util.zip.ZipException; + +import org.apache.commons.compress.archivers.zip.ZipExtraField; +import org.apache.commons.compress.archivers.zip.ZipShort; + +public class SizeExtraField implements ZipExtraField { + public static final ZipShort ID = new ZipShort(0x6A61); // "aj" in little-endian + + private Optional<Long> size; + + public SizeExtraField() { + this(Optional.empty()); + } + + public SizeExtraField(long size) { + this(Optional.of(size)); + } + + public SizeExtraField(Optional<Long> size) { + this.size = size; + } + + @Override + public ZipShort getHeaderId() { + return ID; + } + + @Override + public ZipShort getLocalFileDataLength() { + return new ZipShort(Long.BYTES); + } + + @Override + public ZipShort getCentralDirectoryLength() { + return getLocalFileDataLength(); + } + + @Override + public byte[] getLocalFileDataData() { + long value = size.orElseThrow(() -> new RuntimeException("Value must by initialized")); + return ByteBuffer.allocate(Long.BYTES) + .order(ByteOrder.LITTLE_ENDIAN) + .putLong(value) + .array(); + } + + @Override + public byte[] getCentralDirectoryData() { + return getLocalFileDataData(); + } + + @Override + public void parseFromLocalFileData(byte[] buffer, int offset, int length) throws ZipException { + if (length != Long.BYTES) { + throw new ZipException("Unexpected data length for SizeExtraField. Expected " + Long.BYTES + " but got " + length + "."); + } + size = Optional.of(ByteBuffer + .wrap(buffer, offset, Long.BYTES) + .order(ByteOrder.LITTLE_ENDIAN) + .getLong()); + } + + @Override + public void parseFromCentralDirectoryData(byte[] buffer, int offset, int length) throws ZipException { + parseFromLocalFileData(buffer, offset, length); + } + + public Optional<Long> getSize() { + return size; + } + + @Override + public final boolean equals(Object o) { + if (o instanceof SizeExtraField) { + SizeExtraField that = (SizeExtraField) o; + + return Objects.equals(this.size, that.size); + } + return false; + } + + @Override + public final int hashCode() { + return Objects.hash(size); + } +} http://git-wip-us.apache.org/repos/asf/james-project/blob/feb18013/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/Zipper.java ---------------------------------------------------------------------- diff --git a/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/Zipper.java b/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/Zipper.java index c6d95ad..e4c2518 100644 --- a/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/Zipper.java +++ b/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/Zipper.java @@ -23,12 +23,16 @@ import java.io.IOException; import java.io.OutputStream; import java.util.List; -import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.archivers.zip.ExtraFieldUtils; +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; import org.apache.commons.io.IOUtils; import org.apache.james.mailbox.store.mail.model.MailboxMessage; public class Zipper implements Backup { + public Zipper() { + ExtraFieldUtils.register(SizeExtraField.class); + } @Override public void archive(List<MailboxMessage> messages, OutputStream destination) throws IOException { @@ -42,7 +46,8 @@ public class Zipper implements Backup { private void storeInArchive(MailboxMessage message, ZipArchiveOutputStream archiveOutputStream) throws IOException { String entryId = message.getMessageId().serialize(); - ArchiveEntry archiveEntry = archiveOutputStream.createArchiveEntry(new File(entryId), entryId); + ZipArchiveEntry archiveEntry = (ZipArchiveEntry) archiveOutputStream.createArchiveEntry(new File(entryId), entryId); + archiveEntry.addExtraField(new SizeExtraField(message.getFullContentOctets())); archiveOutputStream.putArchiveEntry(archiveEntry); IOUtils.copy(message.getFullContent(), archiveOutputStream); archiveOutputStream.closeArchiveEntry(); http://git-wip-us.apache.org/repos/asf/james-project/blob/feb18013/mailbox/backup/src/test/java/org/apache/james/mailbox/backup/MailboxMessageFixture.java ---------------------------------------------------------------------- diff --git a/mailbox/backup/src/test/java/org/apache/james/mailbox/backup/MailboxMessageFixture.java b/mailbox/backup/src/test/java/org/apache/james/mailbox/backup/MailboxMessageFixture.java index 8012e78..3d46075 100644 --- a/mailbox/backup/src/test/java/org/apache/james/mailbox/backup/MailboxMessageFixture.java +++ b/mailbox/backup/src/test/java/org/apache/james/mailbox/backup/MailboxMessageFixture.java @@ -49,8 +49,8 @@ public interface MailboxMessageFixture { SharedByteArrayInputStream CONTENT_STREAM_2 = new SharedByteArrayInputStream(MESSAGE_CONTENT_2.getBytes(MESSAGE_CHARSET)); MessageId MESSAGE_ID_1 = MESSAGE_ID_FACTORY.generate(); MessageId MESSAGE_ID_2 = MESSAGE_ID_FACTORY.generate(); - int SIZE_1 = 1000; - int SIZE_2 = 2000; + long SIZE_1 = 1000; + long SIZE_2 = 2000; SimpleMailboxMessage MESSAGE_1 = SimpleMailboxMessage.builder() .messageId(MESSAGE_ID_1) http://git-wip-us.apache.org/repos/asf/james-project/blob/feb18013/mailbox/backup/src/test/java/org/apache/james/mailbox/backup/SizeExtraFieldTest.java ---------------------------------------------------------------------- diff --git a/mailbox/backup/src/test/java/org/apache/james/mailbox/backup/SizeExtraFieldTest.java b/mailbox/backup/src/test/java/org/apache/james/mailbox/backup/SizeExtraFieldTest.java new file mode 100644 index 0000000..db2b274 --- /dev/null +++ b/mailbox/backup/src/test/java/org/apache/james/mailbox/backup/SizeExtraFieldTest.java @@ -0,0 +1,202 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.backup; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.zip.ZipException; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.shaded.org.bouncycastle.util.Arrays; + +import com.google.common.base.Charsets; + +public class SizeExtraFieldTest { + private static final byte[] ZERO_AS_BYTE_ARRAY = {0, 0, 0, 0, 0, 0, 0, 0}; + private static final byte[] _123456789ABCDEF0_AS_LE_BYTE_ARRAY = new byte[] {(byte) 0xF0, (byte) 0xDE, (byte) 0xBC, (byte) 0x9A, 0x78, 0x56, 0x34, 0x12}; + private static final byte[] FEDCBA9876543210_AS_LE_BYTE_ARRAY = new byte[] {0x10, 0x32, 0x54, 0x76, (byte) 0x98, (byte) 0xBA, (byte) 0xDC, (byte) 0xFE}; + private static final byte[] UNUSED = new byte[] {(byte) 0xDE, (byte) 0xAD}; + + private SizeExtraField testee; + + @BeforeEach + void setUp() { + testee = new SizeExtraField(); + } + + @Test + void getLocalFileDataLengthShouldReturnIntegerSize() { + assertThat(testee.getLocalFileDataLength().getValue()) + .isEqualTo(Long.BYTES); + } + + @Test + void getCentralDirectoryLengthShouldReturnIntegerSize() { + assertThat(testee.getCentralDirectoryLength().getValue()) + .isEqualTo(Long.BYTES); + } + + @Test + void getHeaderIdShouldReturnSpecificStringInLittleEndian() { + ByteBuffer byteBuffer = ByteBuffer.wrap(testee.getHeaderId().getBytes()) + .order(ByteOrder.LITTLE_ENDIAN); + assertThat(Charsets.US_ASCII.decode(byteBuffer).toString()) + .isEqualTo("aj"); + } + + @Test + void getLocalFileDataDataShouldThrowWhenNoValue() { + assertThatThrownBy(() -> testee.getLocalFileDataData()) + .isInstanceOf(RuntimeException.class); + } + + @Test + void getLocalFileDataDataShouldReturnZeroWhenZero() { + byte[] actual = new SizeExtraField(0).getLocalFileDataData(); + assertThat(actual).isEqualTo(ZERO_AS_BYTE_ARRAY); + } + + @Test + void getLocalFileDataDataShouldReturnValueInLittleIndianWhen123456789ABCDEF0() { + byte[] actual = new SizeExtraField(0x123456789ABCDEF0L).getLocalFileDataData(); + assertThat(actual).isEqualTo(_123456789ABCDEF0_AS_LE_BYTE_ARRAY); + } + + @Test + void getLocalFileDataDataShouldReturnValueInLittleIndianWhenFEDCBA9876543210() { + byte[] actual = new SizeExtraField(0xFEDCBA9876543210L).getLocalFileDataData(); + assertThat(actual).isEqualTo(FEDCBA9876543210_AS_LE_BYTE_ARRAY); + } + + @Test + void getCentralDirectoryDataShouldThrowWhenNoValue() { + assertThatThrownBy(() -> testee.getCentralDirectoryData()) + .isInstanceOf(RuntimeException.class); + } + + @Test + void getCentralDirectoryDataShouldReturnZeroWhenZero() { + byte[] actual = new SizeExtraField(0).getCentralDirectoryData(); + assertThat(actual).isEqualTo(ZERO_AS_BYTE_ARRAY); + } + + @Test + void getCentralDirectoryDataShouldReturnValueInLittleIndianWhen123456789ABCDEF0() { + byte[] actual = new SizeExtraField(0x123456789ABCDEF0L).getCentralDirectoryData(); + assertThat(actual).isEqualTo(_123456789ABCDEF0_AS_LE_BYTE_ARRAY); + } + + @Test + void getCentralDirectoryDataShouldReturnValueInLittleIndianWhenFEDCBA9876543210() { + byte[] actual = new SizeExtraField(0xFEDCBA9876543210L).getCentralDirectoryData(); + assertThat(actual).isEqualTo(FEDCBA9876543210_AS_LE_BYTE_ARRAY); + } + + @Test + void parseFromLocalFileDataShouldThrownWhenLengthIsSmallerThan8() { + byte[] input = new byte[] {0, 0, 0, 0, 0, 0, 0}; + assertThatThrownBy(() -> testee.parseFromLocalFileData(input, 0, 7)) + .isInstanceOf(ZipException.class); + } + + @Test + void parseFromLocalFileDataShouldThrownWhenLengthIsBiggerThan8() { + byte[] input = new byte[] {0, 0, 0, 0, 0, 0, 0, 0, 0}; + assertThatThrownBy(() -> testee.parseFromLocalFileData(input, 0, 9)) + .isInstanceOf(ZipException.class); + } + + @Test + void parseFromLocalFileDataShouldParseWhenZero() throws Exception { + testee.parseFromLocalFileData(ZERO_AS_BYTE_ARRAY, 0, 8); + assertThat(testee.getSize()) + .contains(0L); + } + + @Test + void parseFromLocalFileDataShouldParseWhen123456789ABCDEF0InLittleEndian() throws Exception { + testee.parseFromLocalFileData(_123456789ABCDEF0_AS_LE_BYTE_ARRAY, 0, 8); + assertThat(testee.getSize()) + .contains(0x123456789ABCDEF0L); + } + + @Test + void parseFromLocalFileDataShouldParseWhenFEDCBA9876543210InLittleEndian() throws Exception { + byte[] input = FEDCBA9876543210_AS_LE_BYTE_ARRAY; + testee.parseFromLocalFileData(input, 0, 8); + assertThat(testee.getSize()) + .contains(0xFEDCBA9876543210L); + } + + @Test + void parseFromLocalFileDataShouldHandleOffset() throws Exception { + byte[] input = Arrays.concatenate(UNUSED, _123456789ABCDEF0_AS_LE_BYTE_ARRAY); + testee.parseFromLocalFileData(input, 2, 8); + assertThat(testee.getSize()) + .contains(0x123456789ABCDEF0L); + } + + @Test + void parseFromCentralDirectoryDataShouldThrownWhenLengthIsSmallerThan8() { + byte[] input = new byte[7]; + assertThatThrownBy(() -> testee.parseFromCentralDirectoryData(input, 0, 7)) + .isInstanceOf(ZipException.class); + } + + @Test + void parseFromCentralDirectoryDataShouldThrownWhenLengthIsBiggerThan8() { + byte[] input = new byte[9]; + assertThatThrownBy(() -> testee.parseFromCentralDirectoryData(input, 0, 9)) + .isInstanceOf(ZipException.class); + } + + @Test + void parseFromCentralDirectoryDataShouldParseWhenZero() throws Exception { + testee.parseFromCentralDirectoryData(ZERO_AS_BYTE_ARRAY, 0, 8); + assertThat(testee.getSize()) + .contains(0L); + } + + @Test + void parseFromCentralDirectoryDataShouldParseWhen123456789ABCDEF0InLittleEndian() throws Exception { + testee.parseFromCentralDirectoryData(_123456789ABCDEF0_AS_LE_BYTE_ARRAY, 0, 8); + assertThat(testee.getSize()) + .contains(0x123456789ABCDEF0L); + } + + @Test + void parseFromCentralDirectoryDataShouldParseWhenFEDCBA9876543210InLittleEndian() throws Exception { + byte[] input = FEDCBA9876543210_AS_LE_BYTE_ARRAY; + testee.parseFromCentralDirectoryData(input, 0, 8); + assertThat(testee.getSize()) + .contains(0xFEDCBA9876543210L); + } + + @Test + void parseFromCentralDirectoryDataShouldHandleOffset() throws Exception { + byte[] input = Arrays.concatenate(UNUSED, _123456789ABCDEF0_AS_LE_BYTE_ARRAY); + testee.parseFromCentralDirectoryData(input, 2, 8); + assertThat(testee.getSize()) + .contains(0x123456789ABCDEF0L); + } +} http://git-wip-us.apache.org/repos/asf/james-project/blob/feb18013/mailbox/backup/src/test/java/org/apache/james/mailbox/backup/ZipArchiveEntryAssert.java ---------------------------------------------------------------------- diff --git a/mailbox/backup/src/test/java/org/apache/james/mailbox/backup/ZipArchiveEntryAssert.java b/mailbox/backup/src/test/java/org/apache/james/mailbox/backup/ZipArchiveEntryAssert.java index f3d1b89..4f268ec 100644 --- a/mailbox/backup/src/test/java/org/apache/james/mailbox/backup/ZipArchiveEntryAssert.java +++ b/mailbox/backup/src/test/java/org/apache/james/mailbox/backup/ZipArchiveEntryAssert.java @@ -22,8 +22,10 @@ package org.apache.james.mailbox.backup; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; +import java.util.Arrays; import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipExtraField; import org.apache.commons.compress.archivers.zip.ZipFile; import org.apache.commons.io.IOUtils; import org.assertj.core.api.AbstractAssert; @@ -47,6 +49,15 @@ public class ZipArchiveEntryAssert extends AbstractAssert<ZipArchiveEntryAssert, return new BasicErrorMessageFactory("%nExpecting %s to have content %s but was %s", zipArchiveEntry, expectedContent, actualContent); } + private static BasicErrorMessageFactory shouldHaveExtraFields(ZipArchiveEntry zipArchiveEntry, + ZipExtraField[] expectedExtraFields, + ZipExtraField[] actualExtraFields) { + return new BasicErrorMessageFactory("%nExpecting %s to contain exactly being %s" + + " but was containing being %s", zipArchiveEntry, + Arrays.toString(expectedExtraFields), + Arrays.toString(actualExtraFields)); + } + private final ZipFile zipFile; private final ZipArchiveEntry actual; @@ -76,4 +87,18 @@ public class ZipArchiveEntryAssert extends AbstractAssert<ZipArchiveEntryAssert, } return myself; } + + public ZipArchiveEntryAssert containsExactlyExtraFields(ZipExtraField... expectedExtraFields) { + isNotNull(); + ZipExtraField[] actualExtraFields = actual.getExtraFields(); + if (expectedExtraFields.length != actualExtraFields.length) { + throwAssertionError(shouldHaveExtraFields(actual, expectedExtraFields, actualExtraFields)); + } + for (int i = 0; i < expectedExtraFields.length; i++) { + if (!expectedExtraFields[i].equals(actualExtraFields[i])) { + throwAssertionError(shouldHaveExtraFields(actual, expectedExtraFields, actualExtraFields)); + } + } + return myself; + } } http://git-wip-us.apache.org/repos/asf/james-project/blob/feb18013/mailbox/backup/src/test/java/org/apache/james/mailbox/backup/ZipperTest.java ---------------------------------------------------------------------- diff --git a/mailbox/backup/src/test/java/org/apache/james/mailbox/backup/ZipperTest.java b/mailbox/backup/src/test/java/org/apache/james/mailbox/backup/ZipperTest.java index 888fed6..ed4e17a 100644 --- a/mailbox/backup/src/test/java/org/apache/james/mailbox/backup/ZipperTest.java +++ b/mailbox/backup/src/test/java/org/apache/james/mailbox/backup/ZipperTest.java @@ -24,6 +24,7 @@ import static org.apache.james.mailbox.backup.MailboxMessageFixture.MESSAGE_CONT import static org.apache.james.mailbox.backup.MailboxMessageFixture.MESSAGE_CONTENT_2; import static org.apache.james.mailbox.backup.MailboxMessageFixture.MESSAGE_ID_1; import static org.apache.james.mailbox.backup.MailboxMessageFixture.MESSAGE_ID_2; +import static org.apache.james.mailbox.backup.MailboxMessageFixture.SIZE_1; import static org.apache.james.mailbox.backup.ZipAssert.assertThatZip; import java.io.File; @@ -100,4 +101,16 @@ public class ZipperTest { .hasStringContent(MESSAGE_CONTENT_2)); } } + + @Test + void archiveShouldWriteSizeMetadata() throws Exception { + testee.archive(ImmutableList.of(MESSAGE_1), new FileOutputStream(destination)); + + try (ZipFile zipFile = new ZipFile(destination)) { + assertThatZip(zipFile) + .containsExactlyEntriesMatching( + zipEntryAssert -> zipEntryAssert + .containsExactlyExtraFields(new SizeExtraField(SIZE_1))); + } + } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
