Repository: james-project Updated Branches: refs/heads/master 81bf0a3c8 -> b1823d38c
MAILBOX-306 Allows to retreive all messageIds with their attachmentIds Project: http://git-wip-us.apache.org/repos/asf/james-project/repo Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/5e28f4aa Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/5e28f4aa Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/5e28f4aa Branch: refs/heads/master Commit: 5e28f4aa0f1d445d9521f798d1e7513e0ce78f8d Parents: 81bf0a3 Author: Raphael Ouazana <raphael.ouaz...@linagora.com> Authored: Tue Sep 12 17:57:08 2017 +0200 Committer: benwa <btell...@linagora.com> Committed: Tue Sep 19 08:59:29 2017 +0700 ---------------------------------------------------------------------- .../cassandra/mail/CassandraMessageDAO.java | 88 ++++++++- ...estCassandraMailboxSessionMapperFactory.java | 2 +- .../cassandra/mail/CassandraMessageDAOTest.java | 182 +++++++++++++++++-- 3 files changed, 255 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/james-project/blob/5e28f4aa/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/mail/CassandraMessageDAO.java ---------------------------------------------------------------------- diff --git a/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/mail/CassandraMessageDAO.java b/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/mail/CassandraMessageDAO.java index a6165e7..b8a3d84 100644 --- a/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/mail/CassandraMessageDAO.java +++ b/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/mail/CassandraMessageDAO.java @@ -42,7 +42,9 @@ import static org.apache.james.mailbox.cassandra.table.CassandraMessageV2Table.T import java.io.IOException; import java.util.Collection; import java.util.List; +import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.function.Function; import java.util.stream.Collectors; @@ -56,6 +58,7 @@ import org.apache.commons.lang3.tuple.Pair; import org.apache.james.backends.cassandra.init.CassandraConfiguration; import org.apache.james.backends.cassandra.init.CassandraTypesProvider; import org.apache.james.backends.cassandra.utils.CassandraAsyncExecutor; +import org.apache.james.backends.cassandra.utils.CassandraUtils; import org.apache.james.mailbox.cassandra.ids.BlobId; import org.apache.james.mailbox.cassandra.ids.CassandraMessageId; import org.apache.james.mailbox.cassandra.mail.utils.Limit; @@ -68,6 +71,7 @@ import org.apache.james.mailbox.model.Cid; import org.apache.james.mailbox.model.ComposedMessageId; import org.apache.james.mailbox.model.ComposedMessageIdWithMetaData; import org.apache.james.mailbox.model.MessageAttachment; +import org.apache.james.mailbox.model.MessageId; import org.apache.james.mailbox.store.mail.MessageMapper.FetchType; import org.apache.james.mailbox.store.mail.model.MailboxMessage; import org.apache.james.mailbox.store.mail.model.impl.PropertyBuilder; @@ -85,7 +89,10 @@ import com.datastax.driver.core.UDTValue; import com.datastax.driver.core.querybuilder.QueryBuilder; import com.github.steveash.guavate.Guavate; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.MoreObjects; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.google.common.primitives.Bytes; public class CassandraMessageDAO { @@ -96,32 +103,41 @@ public class CassandraMessageDAO { private final CassandraTypesProvider typesProvider; private final CassandraBlobsDAO blobsDAO; private final CassandraConfiguration configuration; + private final CassandraUtils cassandraUtils; + private final CassandraMessageId.Factory messageIdFactory; private final PreparedStatement insert; private final PreparedStatement delete; private final PreparedStatement selectMetadata; private final PreparedStatement selectHeaders; private final PreparedStatement selectFields; private final PreparedStatement selectBody; + private final PreparedStatement selectAllMessagesWithAttachment; private final Cid.CidParser cidParser; @Inject - public CassandraMessageDAO(Session session, CassandraTypesProvider typesProvider, CassandraBlobsDAO blobsDAO, CassandraConfiguration cassandraConfiguration) { + public CassandraMessageDAO(Session session, CassandraTypesProvider typesProvider, CassandraBlobsDAO blobsDAO, CassandraConfiguration cassandraConfiguration, + CassandraUtils cassandraUtils, CassandraMessageId.Factory messageIdFactory) { this.cassandraAsyncExecutor = new CassandraAsyncExecutor(session); this.typesProvider = typesProvider; this.blobsDAO = blobsDAO; this.configuration = cassandraConfiguration; + this.cassandraUtils = cassandraUtils; + this.messageIdFactory = messageIdFactory; + this.insert = prepareInsert(session); this.delete = prepareDelete(session); this.selectMetadata = prepareSelect(session, METADATA); this.selectHeaders = prepareSelect(session, HEADERS); this.selectFields = prepareSelect(session, FIELDS); this.selectBody = prepareSelect(session, BODY); + this.selectAllMessagesWithAttachment = prepareSelectAllMessagesWithAttachment(session); this.cidParser = Cid.parser().relaxed(); } @VisibleForTesting - public CassandraMessageDAO(Session session, CassandraTypesProvider typesProvider, CassandraBlobsDAO blobsDAO) { - this(session, typesProvider, blobsDAO, CassandraConfiguration.DEFAULT_CONFIGURATION); + public CassandraMessageDAO(Session session, CassandraTypesProvider typesProvider, CassandraBlobsDAO blobsDAO, + CassandraUtils cassandraUtils, CassandraMessageId.Factory messageIdFactory) { + this(session, typesProvider, blobsDAO, CassandraConfiguration.DEFAULT_CONFIGURATION, cassandraUtils, messageIdFactory); } private PreparedStatement prepareSelect(Session session, String[] fields) { @@ -130,6 +146,11 @@ public class CassandraMessageDAO { .where(eq(MESSAGE_ID, bindMarker(MESSAGE_ID)))); } + private PreparedStatement prepareSelectAllMessagesWithAttachment(Session session) { + return session.prepare(select(MESSAGE_ID, ATTACHMENTS) + .from(TABLE_NAME)); + } + private PreparedStatement prepareInsert(Session session) { return session.prepare(insertInto(TABLE_NAME) .value(MESSAGE_ID, bindMarker(MESSAGE_ID)) @@ -381,4 +402,65 @@ public class CassandraMessageDAO { return message.get(); } } + + public CompletableFuture<Stream<MessageIdAttachmentIds>> retrieveAllMessageIdAttachmentIds() { + return cassandraAsyncExecutor.execute(selectAllMessagesWithAttachment.bind()) + .thenApply(resultSet -> cassandraUtils.convertToStream(resultSet) + .map(this::fromRow) + .filter(MessageIdAttachmentIds::hasAttachment)); + } + + private MessageIdAttachmentIds fromRow(Row row) { + MessageId messageId = messageIdFactory.of(row.getUUID(MESSAGE_ID)); + Set<AttachmentId> attachmentIds = attachmentByIds(row.getList(ATTACHMENTS, UDTValue.class)) + .map(MessageAttachmentRepresentation::getAttachmentId) + .collect(Guavate.toImmutableSet()); + return new MessageIdAttachmentIds(messageId, attachmentIds); + } + + public static class MessageIdAttachmentIds { + private final MessageId messageId; + private final Set<AttachmentId> attachmentIds; + + public MessageIdAttachmentIds(MessageId messageId, Set<AttachmentId> attachmentIds) { + Preconditions.checkNotNull(messageId); + Preconditions.checkNotNull(attachmentIds); + this.messageId = messageId; + this.attachmentIds = ImmutableSet.copyOf(attachmentIds); + } + + public MessageId getMessageId() { + return messageId; + } + + public Set<AttachmentId> getAttachmentId() { + return attachmentIds; + } + + public boolean hasAttachment() { + return ! attachmentIds.isEmpty(); + } + @Override + public final boolean equals(Object o) { + if (o instanceof MessageIdAttachmentIds) { + MessageIdAttachmentIds other = (MessageIdAttachmentIds) o; + return Objects.equals(messageId, other.messageId) + && Objects.equals(attachmentIds, other.attachmentIds); + } + return false; + } + + @Override + public final int hashCode() { + return Objects.hash(messageId, attachmentIds); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("messageId", messageId) + .add("attachmentIds", attachmentIds) + .toString(); + } + } } http://git-wip-us.apache.org/repos/asf/james-project/blob/5e28f4aa/mailbox/cassandra/src/test/java/org/apache/james/mailbox/cassandra/TestCassandraMailboxSessionMapperFactory.java ---------------------------------------------------------------------- diff --git a/mailbox/cassandra/src/test/java/org/apache/james/mailbox/cassandra/TestCassandraMailboxSessionMapperFactory.java b/mailbox/cassandra/src/test/java/org/apache/james/mailbox/cassandra/TestCassandraMailboxSessionMapperFactory.java index fcd3467..6f9d9e2 100644 --- a/mailbox/cassandra/src/test/java/org/apache/james/mailbox/cassandra/TestCassandraMailboxSessionMapperFactory.java +++ b/mailbox/cassandra/src/test/java/org/apache/james/mailbox/cassandra/TestCassandraMailboxSessionMapperFactory.java @@ -50,7 +50,7 @@ public class TestCassandraMailboxSessionMapperFactory { new CassandraUidProvider(session), new CassandraModSeqProvider(session), session, - new CassandraMessageDAO(session, typesProvider, cassandraBlobsDAO), + new CassandraMessageDAO(session, typesProvider, cassandraBlobsDAO, CassandraUtils.WITH_DEFAULT_CONFIGURATION, factory), new CassandraMessageIdDAO(session, factory), new CassandraMessageIdToImapUidDAO(session, factory), new CassandraMailboxCounterDAO(session), http://git-wip-us.apache.org/repos/asf/james-project/blob/5e28f4aa/mailbox/cassandra/src/test/java/org/apache/james/mailbox/cassandra/mail/CassandraMessageDAOTest.java ---------------------------------------------------------------------- diff --git a/mailbox/cassandra/src/test/java/org/apache/james/mailbox/cassandra/mail/CassandraMessageDAOTest.java b/mailbox/cassandra/src/test/java/org/apache/james/mailbox/cassandra/mail/CassandraMessageDAOTest.java index ae0f3cf..2ad6928 100644 --- a/mailbox/cassandra/src/test/java/org/apache/james/mailbox/cassandra/mail/CassandraMessageDAOTest.java +++ b/mailbox/cassandra/src/test/java/org/apache/james/mailbox/cassandra/mail/CassandraMessageDAOTest.java @@ -19,8 +19,11 @@ package org.apache.james.mailbox.cassandra.mail; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import java.util.Collection; import java.util.Date; import java.util.List; import java.util.concurrent.CompletableFuture; @@ -34,14 +37,18 @@ import org.apache.commons.lang3.tuple.Pair; import org.apache.james.backends.cassandra.CassandraCluster; import org.apache.james.backends.cassandra.DockerCassandraRule; import org.apache.james.backends.cassandra.init.CassandraModuleComposite; +import org.apache.james.backends.cassandra.utils.CassandraUtils; import org.apache.james.mailbox.MessageUid; import org.apache.james.mailbox.cassandra.ids.CassandraId; import org.apache.james.mailbox.cassandra.ids.CassandraMessageId; +import org.apache.james.mailbox.cassandra.mail.CassandraMessageDAO.MessageIdAttachmentIds; import org.apache.james.mailbox.cassandra.mail.utils.Limit; import org.apache.james.mailbox.cassandra.modules.CassandraBlobModule; import org.apache.james.mailbox.cassandra.modules.CassandraMessageModule; +import org.apache.james.mailbox.model.Attachment; import org.apache.james.mailbox.model.ComposedMessageId; import org.apache.james.mailbox.model.ComposedMessageIdWithMetaData; +import org.apache.james.mailbox.model.MessageAttachment; import org.apache.james.mailbox.model.MessageId; import org.apache.james.mailbox.store.mail.MessageMapper; import org.apache.james.mailbox.store.mail.model.impl.PropertyBuilder; @@ -53,25 +60,27 @@ import org.junit.Test; import com.google.common.base.Charsets; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.google.common.primitives.Bytes; +import nl.jqno.equalsverifier.EqualsVerifier; + public class CassandraMessageDAOTest { private static final int BODY_START = 16; private static final CassandraId MAILBOX_ID = CassandraId.timeBased(); private static final String CONTENT = "Subject: Test7 \n\nBody7\n.\n"; private static final MessageUid messageUid = MessageUid.of(1); + private static final List<MessageAttachment> NO_ATTACHMENT = ImmutableList.of(); @ClassRule public static DockerCassandraRule cassandraServer = new DockerCassandraRule(); private CassandraCluster cassandra; private CassandraMessageDAO testee; - private CassandraBlobsDAO blobsDAO; private CassandraMessageId.Factory messageIdFactory; private SimpleMailboxMessage message; private CassandraMessageId messageId; - private ComposedMessageId composedMessageId; private List<ComposedMessageIdWithMetaData> messageIds; @Before @@ -79,13 +88,11 @@ public class CassandraMessageDAOTest { cassandra = CassandraCluster.create(new CassandraModuleComposite(new CassandraMessageModule(), new CassandraBlobModule()), cassandraServer.getIp(), cassandraServer.getBindingPort()); messageIdFactory = new CassandraMessageId.Factory(); messageId = messageIdFactory.generate(); - blobsDAO = new CassandraBlobsDAO(cassandra.getConf()); - testee = new CassandraMessageDAO(cassandra.getConf(), cassandra.getTypesProvider(), blobsDAO); - - composedMessageId = new ComposedMessageId(MAILBOX_ID, messageId, messageUid); + CassandraBlobsDAO blobsDAO = new CassandraBlobsDAO(cassandra.getConf()); + testee = new CassandraMessageDAO(cassandra.getConf(), cassandra.getTypesProvider(), blobsDAO, CassandraUtils.WITH_DEFAULT_CONFIGURATION, new CassandraMessageId.Factory()); messageIds = ImmutableList.of(ComposedMessageIdWithMetaData.builder() - .composedMessageId(composedMessageId) + .composedMessageId(new ComposedMessageId(MAILBOX_ID, messageId, messageUid)) .flags(new Flags()) .modSeq(1) .build()); @@ -98,7 +105,7 @@ public class CassandraMessageDAOTest { @Test public void saveShouldSaveNullValueForTextualLineCountAsZero() throws Exception { - message = createMessage(messageId, CONTENT, BODY_START, new PropertyBuilder()); + message = createMessage(messageId, CONTENT, BODY_START, new PropertyBuilder(), NO_ATTACHMENT); testee.save(message).join(); @@ -114,7 +121,7 @@ public class CassandraMessageDAOTest { long textualLineCount = 10L; PropertyBuilder propertyBuilder = new PropertyBuilder(); propertyBuilder.setTextualLineCount(textualLineCount); - message = createMessage(messageId, CONTENT, BODY_START, propertyBuilder); + message = createMessage(messageId, CONTENT, BODY_START, propertyBuilder, NO_ATTACHMENT); testee.save(message).join(); @@ -126,7 +133,7 @@ public class CassandraMessageDAOTest { @Test public void saveShouldStoreMessageWithFullContent() throws Exception { - message = createMessage(messageId, CONTENT, BODY_START, new PropertyBuilder()); + message = createMessage(messageId, CONTENT, BODY_START, new PropertyBuilder(), NO_ATTACHMENT); testee.save(message).join(); @@ -139,7 +146,7 @@ public class CassandraMessageDAOTest { @Test public void saveShouldStoreMessageWithBodyContent() throws Exception { - message = createMessage(messageId, CONTENT, BODY_START, new PropertyBuilder()); + message = createMessage(messageId, CONTENT, BODY_START, new PropertyBuilder(), NO_ATTACHMENT); testee.save(message).join(); @@ -155,7 +162,7 @@ public class CassandraMessageDAOTest { @Test public void saveShouldStoreMessageWithHeaderContent() throws Exception { - message = createMessage(messageId, CONTENT, BODY_START, new PropertyBuilder()); + message = createMessage(messageId, CONTENT, BODY_START, new PropertyBuilder(), NO_ATTACHMENT); testee.save(message).join(); @@ -166,7 +173,7 @@ public class CassandraMessageDAOTest { .isEqualTo(CONTENT.substring(0, BODY_START)); } - private SimpleMailboxMessage createMessage(MessageId messageId, String content, int bodyStart, PropertyBuilder propertyBuilder) { + private SimpleMailboxMessage createMessage(MessageId messageId, String content, int bodyStart, PropertyBuilder propertyBuilder, Collection<MessageAttachment> attachments) { return SimpleMailboxMessage.builder() .messageId(messageId) .mailboxId(MAILBOX_ID) @@ -177,6 +184,7 @@ public class CassandraMessageDAOTest { .content(new SharedByteArrayInputStream(content.getBytes(Charsets.UTF_8))) .flags(new Flags()) .propertyBuilder(propertyBuilder) + .addAttachments(attachments) .build(); } @@ -188,4 +196,152 @@ public class CassandraMessageDAOTest { .orElseThrow(() -> new IllegalStateException("Collection is not supposed to be empty")); } + @Test + public void retrieveAllMessageIdAttachmentIdsShouldReturnEmptyWhenNone() { + Stream<MessageIdAttachmentIds> actual = testee.retrieveAllMessageIdAttachmentIds().join(); + + assertThat(actual).isEmpty(); + } + + @Test + public void retrieveAllMessageIdAttachmentIdsShouldReturnOneWhenStored() throws Exception { + //Given + MessageAttachment attachment = MessageAttachment.builder() + .attachment(Attachment.builder() + .bytes("content".getBytes(StandardCharsets.UTF_8)) + .type("type") + .build()) + .build(); + SimpleMailboxMessage message1 = createMessage(messageId, CONTENT, BODY_START, new PropertyBuilder(), ImmutableList.of(attachment)); + testee.save(message1).join(); + MessageIdAttachmentIds expected = new MessageIdAttachmentIds(messageId, ImmutableSet.of(attachment.getAttachmentId())); + + //When + Stream<MessageIdAttachmentIds> actual = testee.retrieveAllMessageIdAttachmentIds().join(); + + //Then + assertThat(actual).containsOnly(expected); + } + + @Test + public void retrieveAllMessageIdAttachmentIdsShouldReturnOneWhenStoredWithTwoAttachments() throws Exception { + //Given + MessageAttachment attachment1 = MessageAttachment.builder() + .attachment(Attachment.builder() + .bytes("content".getBytes(StandardCharsets.UTF_8)) + .type("type") + .build()) + .build(); + MessageAttachment attachment2 = MessageAttachment.builder() + .attachment(Attachment.builder() + .bytes("other content".getBytes(StandardCharsets.UTF_8)) + .type("type") + .build()) + .build(); + SimpleMailboxMessage message1 = createMessage(messageId, CONTENT, BODY_START, new PropertyBuilder(), ImmutableList.of(attachment1, attachment2)); + testee.save(message1).join(); + MessageIdAttachmentIds expected = new MessageIdAttachmentIds(messageId, ImmutableSet.of(attachment1.getAttachmentId(), attachment2.getAttachmentId())); + + //When + Stream<MessageIdAttachmentIds> actual = testee.retrieveAllMessageIdAttachmentIds().join(); + + //Then + assertThat(actual).containsOnly(expected); + } + + @Test + public void retrieveAllMessageIdAttachmentIdsShouldReturnAllWhenStoredWithAttachment() throws Exception { + //Given + MessageId messageId1 = messageIdFactory.generate(); + MessageId messageId2 = messageIdFactory.generate(); + MessageAttachment attachment1 = MessageAttachment.builder() + .attachment(Attachment.builder() + .bytes("content".getBytes(StandardCharsets.UTF_8)) + .type("type") + .build()) + .build(); + MessageAttachment attachment2 = MessageAttachment.builder() + .attachment(Attachment.builder() + .bytes("other content".getBytes(StandardCharsets.UTF_8)) + .type("type") + .build()) + .build(); + SimpleMailboxMessage message1 = createMessage(messageId1, CONTENT, BODY_START, new PropertyBuilder(), ImmutableList.of(attachment1)); + SimpleMailboxMessage message2 = createMessage(messageId2, CONTENT, BODY_START, new PropertyBuilder(), ImmutableList.of(attachment2)); + testee.save(message1).join(); + testee.save(message2).join(); + MessageIdAttachmentIds expected1 = new MessageIdAttachmentIds(messageId1, ImmutableSet.of(attachment1.getAttachmentId())); + MessageIdAttachmentIds expected2 = new MessageIdAttachmentIds(messageId2, ImmutableSet.of(attachment2.getAttachmentId())); + + //When + Stream<MessageIdAttachmentIds> actual = testee.retrieveAllMessageIdAttachmentIds().join(); + + //Then + assertThat(actual).containsOnly(expected1, expected2); + } + + @Test + public void retrieveAllMessageIdAttachmentIdsShouldReturnEmtpyWhenStoredWithoutAttachment() throws Exception { + //Given + SimpleMailboxMessage message1 = createMessage(messageId, CONTENT, BODY_START, new PropertyBuilder(), NO_ATTACHMENT); + testee.save(message1).join(); + + //When + Stream<MessageIdAttachmentIds> actual = testee.retrieveAllMessageIdAttachmentIds().join(); + + //Then + assertThat(actual).isEmpty(); + } + + @Test + public void retrieveAllMessageIdAttachmentIdsShouldFilterMessagesWithoutAttachment() throws Exception { + //Given + MessageId messageId1 = messageIdFactory.generate(); + MessageId messageId2 = messageIdFactory.generate(); + MessageId messageId3 = messageIdFactory.generate(); + MessageAttachment attachmentFor1 = MessageAttachment.builder() + .attachment(Attachment.builder() + .bytes("content".getBytes(StandardCharsets.UTF_8)) + .type("type") + .build()) + .build(); + MessageAttachment attachmentFor3 = MessageAttachment.builder() + .attachment(Attachment.builder() + .bytes("other content".getBytes(StandardCharsets.UTF_8)) + .type("type") + .build()) + .build(); + SimpleMailboxMessage message1 = createMessage(messageId1, CONTENT, BODY_START, new PropertyBuilder(), ImmutableList.of(attachmentFor1)); + SimpleMailboxMessage message2 = createMessage(messageId2, CONTENT, BODY_START, new PropertyBuilder(), NO_ATTACHMENT); + SimpleMailboxMessage message3 = createMessage(messageId3, CONTENT, BODY_START, new PropertyBuilder(), ImmutableList.of(attachmentFor3)); + testee.save(message1).join(); + testee.save(message2).join(); + testee.save(message3).join(); + + //When + Stream<MessageIdAttachmentIds> actual = testee.retrieveAllMessageIdAttachmentIds().join(); + + //Then + assertThat(actual).extracting(MessageIdAttachmentIds::getMessageId) + .containsOnly(messageId1, messageId3); + } + + @Test + public void messageIdAttachmentIdsShouldMatchBeanContract() { + EqualsVerifier.forClass(MessageIdAttachmentIds.class) + .allFieldsShouldBeUsed() + .verify(); + } + + @Test + public void messageIdAttachmentIdsShouldThrowOnNullMessageId() { + assertThatThrownBy(() -> new MessageIdAttachmentIds(null, ImmutableSet.of())) + .isInstanceOf(NullPointerException.class); + } + + @Test + public void messageIdAttachmentIdsShouldThrowOnNullAttachmentIds() { + assertThatThrownBy(() -> new MessageIdAttachmentIds(messageIdFactory.generate(), null)) + .isInstanceOf(NullPointerException.class); + } } \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: server-dev-unsubscr...@james.apache.org For additional commands, e-mail: server-dev-h...@james.apache.org
