This is an automated email from the ASF dual-hosted git repository. rouazana pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/james-project.git
commit aa1540fb41b2d54e3c0edba95673e6c512dcfe37 Author: Rémi Kowalski <[email protected]> AuthorDate: Mon Apr 8 12:02:11 2019 +0200 JAMES-2694 zip archive loader - read mailboxes --- .../james/mailbox/model/search/MailboxQuery.java | 7 +- .../james/mailbox/backup/MailArchiveEntry.java | 30 ++++ .../james/mailbox/backup/MailArchiveIterator.java | 26 ++++ .../james/mailbox/backup/MailArchivesLoader.java | 26 ++++ .../backup/MailboxWithAnnotationsArchiveEntry.java | 77 +++++++++++ .../james/mailbox/backup/MessageArchiveEntry.java | 72 ++++++++++ .../james/mailbox/backup/SerializedMailboxId.java | 54 ++++++++ .../james/mailbox/backup/SerializedMessageId.java | 54 ++++++++ .../james/mailbox/backup/UnknownArchiveEntry.java | 38 ++++++ .../mailbox/backup/zip/ExtraFieldExtractor.java | 56 ++++++++ .../mailbox/backup/zip/ZipArchivesLoader.java | 35 +++++ .../james/mailbox/backup/zip/ZipEntryIterator.java | 80 +++++++++++ .../backup/zip/ZippedMailAccountIterator.java | 110 +++++++++++++++ .../apache/james/mailbox/backup/zip/Zipper.java | 2 +- .../mailbox/backup/MailboxMessageFixture.java | 16 ++- .../mailbox/backup/ZipArchivesLoaderTest.java | 151 +++++++++++++++++++++ 16 files changed, 828 insertions(+), 6 deletions(-) diff --git a/mailbox/api/src/main/java/org/apache/james/mailbox/model/search/MailboxQuery.java b/mailbox/api/src/main/java/org/apache/james/mailbox/model/search/MailboxQuery.java index ed2f747..6c10eac 100644 --- a/mailbox/api/src/main/java/org/apache/james/mailbox/model/search/MailboxQuery.java +++ b/mailbox/api/src/main/java/org/apache/james/mailbox/model/search/MailboxQuery.java @@ -76,6 +76,11 @@ public final class MailboxQuery { return this; } + public Builder user(User user) { + this.username(user.asString()); + return this; + } + public Builder namespace(String namespace) { Preconditions.checkState(!this.namespace.isPresent()); @@ -89,7 +94,7 @@ public final class MailboxQuery { this.namespace = Optional.of(MailboxConstants.USER_NAMESPACE); return this; } - + public Builder expression(MailboxNameExpression expression) { this.mailboxNameExpression = Optional.of(expression); return this; diff --git a/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/MailArchiveEntry.java b/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/MailArchiveEntry.java new file mode 100644 index 0000000..672c4d2 --- /dev/null +++ b/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/MailArchiveEntry.java @@ -0,0 +1,30 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.backup; + +public interface MailArchiveEntry { + enum ArchiveEntryType { + MAILBOX, + MESSAGE, + UNKNOWN + } + + ArchiveEntryType getType(); + +} diff --git a/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/MailArchiveIterator.java b/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/MailArchiveIterator.java new file mode 100644 index 0000000..43b0966 --- /dev/null +++ b/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/MailArchiveIterator.java @@ -0,0 +1,26 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.backup; + +import java.io.Closeable; +import java.util.Iterator; + +public interface MailArchiveIterator extends Iterator<MailArchiveEntry>, Closeable { + +} \ No newline at end of file diff --git a/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/MailArchivesLoader.java b/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/MailArchivesLoader.java new file mode 100644 index 0000000..5616d16 --- /dev/null +++ b/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/MailArchivesLoader.java @@ -0,0 +1,26 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.backup; + +import java.io.IOException; +import java.io.InputStream; + +public interface MailArchivesLoader { + MailArchiveIterator load(InputStream inputStream) throws IOException; +} diff --git a/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/MailboxWithAnnotationsArchiveEntry.java b/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/MailboxWithAnnotationsArchiveEntry.java new file mode 100644 index 0000000..77c3bfc --- /dev/null +++ b/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/MailboxWithAnnotationsArchiveEntry.java @@ -0,0 +1,77 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.backup; + +import java.util.List; +import java.util.Objects; + +import org.apache.james.mailbox.model.MailboxAnnotation; + +import com.google.common.collect.ImmutableList; + +public class MailboxWithAnnotationsArchiveEntry implements MailArchiveEntry { + private final String mailboxName; + private final SerializedMailboxId mailboxId; + + private final ImmutableList<MailboxAnnotation> annotations; + + public MailboxWithAnnotationsArchiveEntry(String mailboxName, SerializedMailboxId mailboxId, List<MailboxAnnotation> annotations) { + this.mailboxName = mailboxName; + this.mailboxId = mailboxId; + this.annotations = ImmutableList.copyOf(annotations); + } + + public String getMailboxName() { + return mailboxName; + } + + public SerializedMailboxId getMailboxId() { + return mailboxId; + } + + public List<MailboxAnnotation> getAnnotations() { + return annotations; + } + + public MailboxWithAnnotationsArchiveEntry appendAnnotation(MailboxAnnotation annotation) { + ImmutableList<MailboxAnnotation> newAnnotations = ImmutableList.<MailboxAnnotation>builder().addAll(annotations).add(annotation).build(); + return new MailboxWithAnnotationsArchiveEntry(mailboxName, mailboxId, newAnnotations); + } + + @Override + public ArchiveEntryType getType() { + return ArchiveEntryType.MAILBOX; + } + + @Override + public boolean equals(Object o) { + if (o instanceof MailboxWithAnnotationsArchiveEntry) { + MailboxWithAnnotationsArchiveEntry that = (MailboxWithAnnotationsArchiveEntry) o; + return Objects.equals(mailboxName, that.mailboxName) && + Objects.equals(mailboxId, that.mailboxId) && + Objects.equals(annotations, that.annotations); + } + return false; + } + + @Override + public int hashCode() { + return Objects.hash(mailboxName, mailboxId, annotations); + } +} diff --git a/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/MessageArchiveEntry.java b/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/MessageArchiveEntry.java new file mode 100644 index 0000000..3b3a6f9 --- /dev/null +++ b/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/MessageArchiveEntry.java @@ -0,0 +1,72 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.backup; + +import java.io.InputStream; +import java.util.Date; + +import javax.mail.Flags; + +public class MessageArchiveEntry implements MailArchiveEntry { + + private final SerializedMessageId messageId; + private final SerializedMailboxId mailboxId; + private final long size; + private final Date internalDate; + private final Flags flags; + private final InputStream content; + + public MessageArchiveEntry(SerializedMessageId messageId, SerializedMailboxId mailboxId, long size, Date internalDate, Flags flags, InputStream content) { + this.messageId = messageId; + this.mailboxId = mailboxId; + this.size = size; + this.internalDate = internalDate; + this.flags = flags; + this.content = content; + } + + public SerializedMessageId getMessageId() { + return messageId; + } + + public SerializedMailboxId getMailboxId() { + return mailboxId; + } + + public long getSize() { + return size; + } + + public Date getInternalDate() { + return internalDate; + } + + public Flags getFlags() { + return flags; + } + + public InputStream getContent() { + return content; + } + + @Override + public ArchiveEntryType getType() { + return ArchiveEntryType.MESSAGE; + } +} diff --git a/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/SerializedMailboxId.java b/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/SerializedMailboxId.java new file mode 100644 index 0000000..dc0fd6b --- /dev/null +++ b/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/SerializedMailboxId.java @@ -0,0 +1,54 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.backup; + +import org.apache.james.mailbox.model.MailboxId; + +import com.google.common.base.Objects; + +public class SerializedMailboxId { + private final String value; + + public SerializedMailboxId(String value) { + this.value = value; + } + + public SerializedMailboxId(MailboxId mailboxId) { + this.value = mailboxId.serialize(); + } + + public String getValue() { + return value; + } + + @Override + public boolean equals(Object o) { + if (o instanceof SerializedMailboxId) { + SerializedMailboxId that = (SerializedMailboxId) o; + return Objects.equal(value, that.value); + } + return false; + } + + @Override + public int hashCode() { + return Objects.hashCode(value); + } + +} diff --git a/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/SerializedMessageId.java b/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/SerializedMessageId.java new file mode 100644 index 0000000..76b4c6f --- /dev/null +++ b/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/SerializedMessageId.java @@ -0,0 +1,54 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.backup; + +import org.apache.james.mailbox.model.MessageId; + +import com.google.common.base.Objects; + +public class SerializedMessageId { + private final String value; + + public SerializedMessageId(String value) { + this.value = value; + } + + public SerializedMessageId(MessageId messageId) { + this.value = messageId.serialize(); + } + + public String getValue() { + return value; + } + + @Override + public boolean equals(Object o) { + if (o instanceof SerializedMessageId) { + SerializedMessageId that = (SerializedMessageId) o; + return Objects.equal(value, that.value); + } + return false; + } + + @Override + public int hashCode() { + return Objects.hashCode(value); + } + +} diff --git a/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/UnknownArchiveEntry.java b/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/UnknownArchiveEntry.java new file mode 100644 index 0000000..88c858c --- /dev/null +++ b/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/UnknownArchiveEntry.java @@ -0,0 +1,38 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.backup; + +public class UnknownArchiveEntry implements MailArchiveEntry { + + private final String entryName; + + public UnknownArchiveEntry(String entryName) { + this.entryName = entryName; + } + + public String getEntryName() { + return entryName; + } + + @Override + public ArchiveEntryType getType() { + return ArchiveEntryType.UNKNOWN; + } + +} diff --git a/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/zip/ExtraFieldExtractor.java b/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/zip/ExtraFieldExtractor.java new file mode 100644 index 0000000..abc0c28 --- /dev/null +++ b/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/zip/ExtraFieldExtractor.java @@ -0,0 +1,56 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.backup.zip; + +import java.util.Arrays; +import java.util.Optional; +import java.util.function.Function; +import java.util.zip.ZipEntry; +import java.util.zip.ZipException; + +import org.apache.commons.compress.archivers.zip.ExtraFieldUtils; +import org.apache.commons.compress.archivers.zip.ZipExtraField; +import org.apache.commons.compress.archivers.zip.ZipShort; +import org.apache.james.util.OptionalUtils; + +public class ExtraFieldExtractor { + + public static Optional<String> getStringExtraField(ZipShort id, ZipEntry entry) throws ZipException { + ZipExtraField[] extraFields = ExtraFieldUtils.parse(entry.getExtra()); + return Arrays.stream(extraFields) + .filter(field -> field.getHeaderId().equals(id)) + .map(extraField -> ((StringExtraField) extraField).getValue()) + .findFirst() + .flatMap(Function.identity()); + } + + public static Optional<ZipEntryType> getEntryType(ZipEntry entry) { + try { + ZipExtraField[] extraFields = ExtraFieldUtils.parse(entry.getExtra()); + return Arrays.stream(extraFields) + .filter(field -> field.getHeaderId().equals(EntryTypeExtraField.ID_AQ)) + .flatMap(extraField -> + OptionalUtils.toStream(((EntryTypeExtraField) extraField).getEnumValue())) + .findFirst(); + } catch (Exception e) { + return Optional.empty(); + } + } + +} diff --git a/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/zip/ZipArchivesLoader.java b/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/zip/ZipArchivesLoader.java new file mode 100644 index 0000000..14e6838 --- /dev/null +++ b/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/zip/ZipArchivesLoader.java @@ -0,0 +1,35 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.backup.zip; + +import java.io.IOException; +import java.io.InputStream; +import java.util.zip.ZipInputStream; + +import org.apache.james.mailbox.backup.MailArchiveIterator; +import org.apache.james.mailbox.backup.MailArchivesLoader; + +public class ZipArchivesLoader implements MailArchivesLoader { + @Override + public MailArchiveIterator load(InputStream inputStream) throws IOException { + ZipInputStream zipInputStream = new ZipInputStream(inputStream); + ZipEntryIterator zipEntryIterator = new ZipEntryIterator(zipInputStream); + return new ZippedMailAccountIterator(zipEntryIterator); + } +} diff --git a/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/zip/ZipEntryIterator.java b/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/zip/ZipEntryIterator.java new file mode 100644 index 0000000..160bf74 --- /dev/null +++ b/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/zip/ZipEntryIterator.java @@ -0,0 +1,80 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.backup.zip; + +import java.io.Closeable; +import java.io.IOException; +import java.util.Iterator; +import java.util.Optional; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ZipEntryIterator implements Iterator<ZipEntry>, Closeable { + private final ZipInputStream zipInputStream; + private Optional<ZipEntry> next; + + private static final Logger LOGGER = LoggerFactory.getLogger(ZipEntryIterator.class); + + public ZipEntryIterator(ZipInputStream inputStream) { + zipInputStream = inputStream; + try { + next = Optional.ofNullable(zipInputStream.getNextEntry()); + } catch (IOException e) { + //EMPTY STREAM + next = Optional.empty(); + } + } + + @Override + public boolean hasNext() { + return next.isPresent(); + } + + @Override + public ZipEntry next() { + Optional<ZipEntry> current = next; + if (!current.isPresent()) { + return null; + } + + ZipEntry currentEntry = current.get(); + + advanceToNextEntry(); + + return currentEntry; + } + + private void advanceToNextEntry() { + try { + next = Optional.ofNullable(zipInputStream.getNextEntry()); + } catch (IOException e) { + LOGGER.error("Error when reading archive", e); + next = Optional.empty(); + } + } + + @Override + public void close() throws IOException { + zipInputStream.close(); + } + +} diff --git a/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/zip/ZippedMailAccountIterator.java b/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/zip/ZippedMailAccountIterator.java new file mode 100644 index 0000000..5b1d5c2 --- /dev/null +++ b/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/zip/ZippedMailAccountIterator.java @@ -0,0 +1,110 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.backup.zip; + +import java.io.IOException; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.Optional; +import java.util.zip.ZipEntry; +import java.util.zip.ZipException; + +import org.apache.commons.lang3.StringUtils; +import org.apache.james.mailbox.backup.MailArchiveEntry; +import org.apache.james.mailbox.backup.MailArchiveIterator; +import org.apache.james.mailbox.backup.MailboxWithAnnotationsArchiveEntry; +import org.apache.james.mailbox.backup.SerializedMailboxId; +import org.apache.james.mailbox.backup.UnknownArchiveEntry; +import org.apache.james.mailbox.model.MailboxAnnotation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.github.fge.lambdas.Throwing; +import com.google.common.collect.ImmutableList; + +public class ZippedMailAccountIterator implements MailArchiveIterator { + private static final Logger LOGGER = LoggerFactory.getLogger(ZippedMailAccountIterator.class); + private static final List<MailboxAnnotation> NO_ANNOTATION = ImmutableList.of(); + private final ZipEntryIterator zipEntryIterator; + private Optional<MailboxWithAnnotationsArchiveEntry> currentMailBox; + private Optional<ZipEntry> next; + + public ZippedMailAccountIterator(ZipEntryIterator zipEntryIterator) { + this.zipEntryIterator = zipEntryIterator; + next = Optional.ofNullable(zipEntryIterator.next()); + } + + @Override + public void close() throws IOException { + zipEntryIterator.close(); + } + + @Override + public boolean hasNext() { + return next.isPresent(); + } + + @Override + public MailArchiveEntry next() { + return next.map(this::doNext).orElseThrow(() -> new NoSuchElementException()); + } + + private MailArchiveEntry doNext(ZipEntry currentElement) { + next = Optional.ofNullable(zipEntryIterator.next()); + try { + return getMailArchiveEntry(currentElement); + } catch (Exception e) { + LOGGER.error("Error when reading archive on entry : " + currentElement.getName(), e); + next = Optional.empty(); + return new UnknownArchiveEntry(currentElement.getName()); + } + } + + private MailArchiveEntry getMailArchiveEntry(ZipEntry currentElement) throws Exception { + Optional<ZipEntryType> entryType = ExtraFieldExtractor.getEntryType(currentElement); + return entryType + .map(Throwing.<ZipEntryType, MailArchiveEntry>function(type -> + from(currentElement, type)).sneakyThrow() + ) + .orElseGet(() -> new UnknownArchiveEntry(currentElement.getName())); + } + + private Optional<SerializedMailboxId> getMailBoxId(ZipEntry entry) throws ZipException { + return ExtraFieldExtractor.getStringExtraField(MailboxIdExtraField.ID_AM, entry) + .map(SerializedMailboxId::new); + } + + private String getMailboxName(ZipEntry current) { + return StringUtils.chop(current.getName()); + } + + private MailArchiveEntry fromMailboxEntry(ZipEntry current) throws ZipException { + return new MailboxWithAnnotationsArchiveEntry(getMailboxName(current), getMailBoxId(current).get(), NO_ANNOTATION); + + } + + private MailArchiveEntry from(ZipEntry current, ZipEntryType currentEntryType) throws ZipException { + switch (currentEntryType) { + case MAILBOX: + return fromMailboxEntry(current); + default: + return new UnknownArchiveEntry(current.getName()); + } + } +} diff --git a/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/zip/Zipper.java b/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/zip/Zipper.java index b0d2e63..0600b4b 100644 --- a/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/zip/Zipper.java +++ b/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/zip/Zipper.java @@ -47,7 +47,7 @@ import com.google.common.base.Charsets; public class Zipper implements ArchiveService { - private static final String ANNOTATION_DIRECTORY = "annotations"; + public static final String ANNOTATION_DIRECTORY = "annotations"; private static final boolean AUTO_FLUSH = true; private static final Logger LOGGER = LoggerFactory.getLogger(Zipper.class); diff --git a/mailbox/backup/src/test/java/org/apache/james/mailbox/backup/MailboxMessageFixture.java b/mailbox/backup/src/test/java/org/apache/james/mailbox/backup/MailboxMessageFixture.java index 257a9f8..9be370a 100644 --- a/mailbox/backup/src/test/java/org/apache/james/mailbox/backup/MailboxMessageFixture.java +++ b/mailbox/backup/src/test/java/org/apache/james/mailbox/backup/MailboxMessageFixture.java @@ -51,6 +51,7 @@ public interface MailboxMessageFixture { String OTHER_USER = "otherUser"; User USER1 = User.fromUsername(USER); + User USER2 = User.fromUsername(OTHER_USER); String DATE_STRING_1 = "2018-02-15T15:54:02Z"; String DATE_STRING_2 = "2018-03-15T15:54:02Z"; @@ -60,16 +61,19 @@ public interface MailboxMessageFixture { MessageId.Factory MESSAGE_ID_FACTORY = new TestMessageId.Factory(); Charset MESSAGE_CHARSET = StandardCharsets.UTF_8; String MESSAGE_CONTENT_1 = "Simple message content"; - SharedByteArrayInputStream CONTENT_STREAM_1 = new SharedByteArrayInputStream(MESSAGE_CONTENT_1.getBytes(MESSAGE_CHARSET)); + byte[] MESSAGE_CONTENT_BYTES_1 = MESSAGE_CONTENT_1.getBytes(MESSAGE_CHARSET); + SharedByteArrayInputStream CONTENT_STREAM_1 = new SharedByteArrayInputStream(MESSAGE_CONTENT_BYTES_1); String MESSAGE_CONTENT_2 = "Other message content"; - SharedByteArrayInputStream CONTENT_STREAM_2 = new SharedByteArrayInputStream(MESSAGE_CONTENT_2.getBytes(MESSAGE_CHARSET)); + + byte[] MESSAGE_CONTENT_BYTES_2 = MESSAGE_CONTENT_2.getBytes(MESSAGE_CHARSET); + SharedByteArrayInputStream CONTENT_STREAM_2 = new SharedByteArrayInputStream(MESSAGE_CONTENT_BYTES_2); MessageId MESSAGE_ID_1 = MESSAGE_ID_FACTORY.generate(); MessageId MESSAGE_ID_2 = MESSAGE_ID_FACTORY.generate(); MessageId MESSAGE_ID_OTHER_USER_1 = MESSAGE_ID_FACTORY.generate(); - long SIZE_1 = 1000; - long SIZE_2 = 2000; + long SIZE_1 = MESSAGE_CONTENT_BYTES_1.length; + long SIZE_2 = MESSAGE_CONTENT_BYTES_2.length; long MESSAGE_UID_1_VALUE = 1111L; long MESSAGE_UID_2_VALUE = 2222L; long MESSAGE_UID_OTHER_USER_1_VALUE = 1111L; @@ -79,6 +83,10 @@ public interface MailboxMessageFixture { MailboxId MAILBOX_ID_1 = TestId.of(1L); MailboxId MAILBOX_ID_2 = TestId.of(2L); MailboxId MAILBOX_ID_11 = TestId.of(11L); + + SerializedMailboxId SERIALIZED_MAILBOX_ID_1 = new SerializedMailboxId(MAILBOX_ID_1); + SerializedMailboxId SERIALIZED_MAILBOX_ID_2 = new SerializedMailboxId(MAILBOX_ID_2); + Flags flags1 = new Flags("myFlags"); MailboxSession MAILBOX_SESSION = MailboxSessionUtil.create(USER); diff --git a/mailbox/backup/src/test/java/org/apache/james/mailbox/backup/ZipArchivesLoaderTest.java b/mailbox/backup/src/test/java/org/apache/james/mailbox/backup/ZipArchivesLoaderTest.java new file mode 100644 index 0000000..a4054d3 --- /dev/null +++ b/mailbox/backup/src/test/java/org/apache/james/mailbox/backup/ZipArchivesLoaderTest.java @@ -0,0 +1,151 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.backup; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.NoSuchElementException; + +import javax.mail.Flags; + +import org.apache.commons.io.IOUtils; +import org.apache.james.mailbox.MailboxManager; +import org.apache.james.mailbox.MailboxSession; +import org.apache.james.mailbox.MessageManager; +import org.apache.james.mailbox.backup.zip.ZipArchivesLoader; +import org.apache.james.mailbox.backup.zip.Zipper; +import org.apache.james.mailbox.inmemory.manager.InMemoryIntegrationResources; +import org.apache.james.mailbox.model.MailboxId; +import org.apache.james.mailbox.model.MailboxPath; +import org.apache.james.mailbox.store.mail.model.MailboxMessage; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import com.github.fge.lambdas.Throwing; + +public class ZipArchivesLoaderTest implements MailboxMessageFixture { + private static final int BUFFER_SIZE = 4096; + + private final ArchiveService archiveService = new Zipper(); + private final MailArchivesLoader archiveLoader = new ZipArchivesLoader(); + + private MailboxManager mailboxManager; + private DefaultMailboxBackup backup; + + @BeforeEach + void beforeEach() { + mailboxManager = InMemoryIntegrationResources.defaultResources().getMailboxManager(); + backup = new DefaultMailboxBackup(mailboxManager, archiveService); + } + + private void createMailBoxWithMessage(MailboxSession session, MailboxPath mailboxPath, MailboxMessage... messages) throws Exception { + MailboxId mailboxId = mailboxManager.createMailbox(mailboxPath, session).get(); + Arrays.stream(messages).forEach(Throwing.consumer(message -> + { + MessageManager.AppendCommand appendCommand = MessageManager.AppendCommand.builder() + .withFlags(message.createFlags()) + .build(message.getFullContent()); + mailboxManager.getMailbox(mailboxId, session).appendMessage(appendCommand, session); + } + ) + ); + } + + @Test + void mailAccountIteratorFromEmptyArchiveShouldThrowNoSuchElementException() throws Exception { + ByteArrayOutputStream destination = new ByteArrayOutputStream(BUFFER_SIZE); + backup.backupAccount(USER1, destination); + + InputStream source = new ByteArrayInputStream(destination.toByteArray()); + MailArchiveIterator mailArchiveIterator = archiveLoader.load(source); + + assertThat(mailArchiveIterator.hasNext()).isEqualTo(false); + assertThatThrownBy(() -> mailArchiveIterator.next()).isInstanceOf(NoSuchElementException.class); + } + + @Test + void callingNextSeveralTimeOnAnEmptyIteratorShouldThrowNoSuchElementException() throws Exception { + ByteArrayOutputStream destination = new ByteArrayOutputStream(BUFFER_SIZE); + backup.backupAccount(USER1, destination); + + InputStream source = new ByteArrayInputStream(destination.toByteArray()); + MailArchiveIterator mailArchiveIterator = archiveLoader.load(source); + + assertThat(mailArchiveIterator.hasNext()).isEqualTo(false); + assertThatThrownBy(() -> mailArchiveIterator.next()).isInstanceOf(NoSuchElementException.class); + assertThatThrownBy(() -> mailArchiveIterator.next()).isInstanceOf(NoSuchElementException.class); + assertThatThrownBy(() -> mailArchiveIterator.next()).isInstanceOf(NoSuchElementException.class); + } + + @Test + void mailAccountIteratorFromArchiveWithOneMailboxShouldContainOneMailbox() throws Exception { + MailboxSession session = mailboxManager.createSystemSession(USER); + createMailBoxWithMessage(session, MAILBOX_PATH_USER1_MAILBOX1); + + ByteArrayOutputStream destination = new ByteArrayOutputStream(BUFFER_SIZE); + backup.backupAccount(USER1, destination); + + InputStream source = new ByteArrayInputStream(destination.toByteArray()); + MailArchiveIterator mailArchiveIterator = archiveLoader.load(source); + assertThat(mailArchiveIterator.hasNext()).isEqualTo(true); + + MailboxWithAnnotationsArchiveEntry expectedMailbox = new MailboxWithAnnotationsArchiveEntry(MAILBOX_1_NAME, SERIALIZED_MAILBOX_ID_1, NO_ANNOTATION); + MailboxWithAnnotationsArchiveEntry resultMailbox = (MailboxWithAnnotationsArchiveEntry) mailArchiveIterator.next(); + verifyMailboxArchiveEntry(mailArchiveIterator, expectedMailbox, resultMailbox, false); + } + + @Test + void mailAccountIteratorFromArchiveWithTwoMailboxesShouldContainTwoMailboxes() throws Exception { + MailboxSession session = mailboxManager.createSystemSession(USER); + createMailBoxWithMessage(session, MAILBOX_PATH_USER1_MAILBOX1); + createMailBoxWithMessage(session, MAILBOX_PATH_USER1_MAILBOX2); + + ByteArrayOutputStream destination = new ByteArrayOutputStream(BUFFER_SIZE); + backup.backupAccount(USER1, destination); + + InputStream source = new ByteArrayInputStream(destination.toByteArray()); + MailArchiveIterator mailArchiveIterator = archiveLoader.load(source); + assertThat(mailArchiveIterator.hasNext()).isEqualTo(true); + + MailboxWithAnnotationsArchiveEntry expectedMailbox = new MailboxWithAnnotationsArchiveEntry(MAILBOX_1_NAME, SERIALIZED_MAILBOX_ID_1, NO_ANNOTATION); + MailboxWithAnnotationsArchiveEntry resultMailbox = (MailboxWithAnnotationsArchiveEntry) mailArchiveIterator.next(); + verifyMailboxArchiveEntry(mailArchiveIterator, expectedMailbox, resultMailbox, true); + + MailboxWithAnnotationsArchiveEntry expectedSecondMailbox = new MailboxWithAnnotationsArchiveEntry(MAILBOX_2_NAME, SERIALIZED_MAILBOX_ID_2, NO_ANNOTATION); + MailboxWithAnnotationsArchiveEntry resultSecondMailbox = (MailboxWithAnnotationsArchiveEntry) mailArchiveIterator.next(); + verifyMailboxArchiveEntry(mailArchiveIterator, expectedSecondMailbox, resultSecondMailbox, false); + } + + private void verifyMailboxArchiveEntry(MailArchiveIterator mailArchiveIterator, MailboxWithAnnotationsArchiveEntry expectedMailbox, + MailboxWithAnnotationsArchiveEntry resultMailbox, boolean iteratorHasNextElement) { + assertThat(resultMailbox.getMailboxId()).isEqualTo(expectedMailbox.getMailboxId()); + assertThat(resultMailbox.getMailboxName()).isEqualTo(expectedMailbox.getMailboxName()); + assertThat(resultMailbox.getAnnotations()).isEqualTo(expectedMailbox.getAnnotations()); + assertThat(resultMailbox).isEqualTo(expectedMailbox); + assertThat(mailArchiveIterator.hasNext()).isEqualTo(iteratorHasNextElement); + } + +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
