Repository: james-project Updated Branches: refs/heads/master ce9321a4c -> b03574767
JAMES-2390 Avoid de-duplicating Attachment meta-data This avoids a costly byte concatenation. However the main benefit is upon attachment right setting. Given an attachment id, we will locate the containing messages, then the containing mailboxes then resolve rights. If an attachmentId is shared for several attachments being the same file, then there could be a lot of users owning this attachment. Resolving rights will mechanically be more expensive. (Think IE pictures in signatures of email) This meta-data sharing can be abandoned with no harm, and deduplication pushed down to the attachment level. The consequence is that attachment right resolution is done independently, thus for high frequency attachment, no penalty is payed. This change is motivated by analyses of production generated logs: - They show systematic failure upon specific attachments upon cassandra reads. - These failures were due to a bad paging - However, even if this issue was fixed separately, this still denotates a high cardinality in attachmentIds index table heavily penalizing reads. Please note that this change will only improve performance of newly generated attachment. Project: http://git-wip-us.apache.org/repos/asf/james-project/repo Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/bfa5e426 Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/bfa5e426 Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/bfa5e426 Branch: refs/heads/master Commit: bfa5e426ac73e1da5866e0b7e772507204b3adcb Parents: 3081d9ab Author: benwa <btell...@linagora.com> Authored: Sun May 6 20:07:58 2018 +0700 Committer: benwa <btell...@linagora.com> Committed: Tue May 8 09:15:36 2018 +0700 ---------------------------------------------------------------------- mailbox/api/pom.xml | 4 + .../apache/james/mailbox/model/Attachment.java | 4 +- .../james/mailbox/model/AttachmentId.java | 36 +--- .../org/apache/james/mailbox/model/BlobId.java | 8 +- .../james/mailbox/model/AttachmentIdTest.java | 66 +----- .../james/mailbox/model/AttachmentTest.java | 25 +-- .../apache/james/mailbox/model/BlobIdTest.java | 16 ++ .../AbstractMailboxManagerAttachmentTest.java | 29 --- .../mailbox/store/StoreBlobManagerTest.java | 2 +- .../store/mail/model/AttachmentMapperTest.java | 6 +- .../org/apache/james/utils/MessageIdProbe.java | 17 ++ .../integration/SetMessagesMethodTest.java | 204 +++++++++++-------- .../integration/cucumber/DownloadStepdefs.java | 28 ++- .../integration/cucumber/UploadStepdefs.java | 6 +- .../test/resources/cucumber/GetMessages.feature | 2 - 15 files changed, 205 insertions(+), 248 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/james-project/blob/bfa5e426/mailbox/api/pom.xml ---------------------------------------------------------------------- diff --git a/mailbox/api/pom.xml b/mailbox/api/pom.xml index c7008e1..867178e 100644 --- a/mailbox/api/pom.xml +++ b/mailbox/api/pom.xml @@ -79,6 +79,10 @@ <artifactId>commons-lang3</artifactId> </dependency> <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-text</artifactId> + </dependency> + <dependency> <groupId>org.assertj</groupId> <artifactId>assertj-core</artifactId> <scope>test</scope> http://git-wip-us.apache.org/repos/asf/james-project/blob/bfa5e426/mailbox/api/src/main/java/org/apache/james/mailbox/model/Attachment.java ---------------------------------------------------------------------- diff --git a/mailbox/api/src/main/java/org/apache/james/mailbox/model/Attachment.java b/mailbox/api/src/main/java/org/apache/james/mailbox/model/Attachment.java index cbe9113..47b2f67 100644 --- a/mailbox/api/src/main/java/org/apache/james/mailbox/model/Attachment.java +++ b/mailbox/api/src/main/java/org/apache/james/mailbox/model/Attachment.java @@ -71,7 +71,7 @@ public class Attachment { if (attachmentId != null) { return attachmentId; } - return AttachmentId.forPayloadAndType(bytes, type); + return AttachmentId.random(); } private long size() { @@ -118,7 +118,7 @@ public class Attachment { public Blob toBlob() { return Blob.builder() - .id(BlobId.fromString(attachmentId.getId())) + .id(BlobId.fromBytes(bytes)) .payload(bytes) .contentType(type) .build(); http://git-wip-us.apache.org/repos/asf/james-project/blob/bfa5e426/mailbox/api/src/main/java/org/apache/james/mailbox/model/AttachmentId.java ---------------------------------------------------------------------- diff --git a/mailbox/api/src/main/java/org/apache/james/mailbox/model/AttachmentId.java b/mailbox/api/src/main/java/org/apache/james/mailbox/model/AttachmentId.java index 21b94a7..792b588 100644 --- a/mailbox/api/src/main/java/org/apache/james/mailbox/model/AttachmentId.java +++ b/mailbox/api/src/main/java/org/apache/james/mailbox/model/AttachmentId.java @@ -19,50 +19,26 @@ package org.apache.james.mailbox.model; import java.nio.charset.StandardCharsets; -import java.util.Optional; import java.util.UUID; -import org.apache.commons.codec.digest.DigestUtils; -import org.apache.james.mime4j.codec.DecodeMonitor; -import org.apache.james.mime4j.field.ContentTypeFieldImpl; -import org.apache.james.mime4j.stream.RawField; +import org.apache.commons.text.RandomStringGenerator; -import com.google.common.annotations.VisibleForTesting; import com.google.common.base.MoreObjects; import com.google.common.base.Objects; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; -import com.google.common.primitives.Bytes; public class AttachmentId { - private static final String DEFAULT_MIME_TYPE = "application/octet-stream"; - - public static AttachmentId forPayloadAndType(byte[] payload, String contentType) { - Preconditions.checkNotNull(payload); - Preconditions.checkArgument(!Strings.isNullOrEmpty(contentType)); - - return new AttachmentId(computeRawId(payload, contentType)); - } - - private static String computeRawId(final byte[] payload, final String contentType) { - return DigestUtils.sha256Hex( - Bytes.concat( - asMimeType(contentType).getBytes(StandardCharsets.UTF_8), - DigestUtils.sha256Hex(payload).getBytes(StandardCharsets.UTF_8))); - } - - @VisibleForTesting static String asMimeType(String contentType) { - return Optional.ofNullable(ContentTypeFieldImpl.PARSER - .parse(new RawField("ContentType", contentType), DecodeMonitor.SILENT) - .getMimeType()) - .orElse(DEFAULT_MIME_TYPE); - } + public static final RandomStringGenerator RANDOM_STRING_GENERATOR = new RandomStringGenerator.Builder().withinRange('a', 'z').build(); public static AttachmentId from(BlobId blobId) { return new AttachmentId(blobId.asString()); } + public static AttachmentId random() { + return new AttachmentId(RANDOM_STRING_GENERATOR.generate(20)); + } + public static AttachmentId from(String id) { Preconditions.checkNotNull(id); Preconditions.checkArgument(!id.isEmpty()); http://git-wip-us.apache.org/repos/asf/james-project/blob/bfa5e426/mailbox/api/src/main/java/org/apache/james/mailbox/model/BlobId.java ---------------------------------------------------------------------- diff --git a/mailbox/api/src/main/java/org/apache/james/mailbox/model/BlobId.java b/mailbox/api/src/main/java/org/apache/james/mailbox/model/BlobId.java index 476eca2..34a62e3 100644 --- a/mailbox/api/src/main/java/org/apache/james/mailbox/model/BlobId.java +++ b/mailbox/api/src/main/java/org/apache/james/mailbox/model/BlobId.java @@ -21,11 +21,18 @@ package org.apache.james.mailbox.model; import java.util.Objects; +import org.apache.commons.codec.digest.DigestUtils; + import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; import com.google.common.base.Strings; public class BlobId { + public static BlobId fromBytes(byte[] bytes) { + Preconditions.checkNotNull(bytes); + return new BlobId(DigestUtils.sha256Hex(bytes)); + } + public static BlobId fromString(String raw) { Preconditions.checkArgument(!Strings.isNullOrEmpty(raw)); return new BlobId(raw); @@ -56,7 +63,6 @@ public class BlobId { return Objects.hash(id); } - @Override public String toString() { return MoreObjects.toStringHelper(this) http://git-wip-us.apache.org/repos/asf/james-project/blob/bfa5e426/mailbox/api/src/test/java/org/apache/james/mailbox/model/AttachmentIdTest.java ---------------------------------------------------------------------- diff --git a/mailbox/api/src/test/java/org/apache/james/mailbox/model/AttachmentIdTest.java b/mailbox/api/src/test/java/org/apache/james/mailbox/model/AttachmentIdTest.java index 092d739..c69e9ce 100644 --- a/mailbox/api/src/test/java/org/apache/james/mailbox/model/AttachmentIdTest.java +++ b/mailbox/api/src/test/java/org/apache/james/mailbox/model/AttachmentIdTest.java @@ -24,48 +24,18 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import java.util.UUID; -import org.apache.james.util.ClassLoaderUtils; import org.junit.Test; public class AttachmentIdTest { @Test - public void forPayloadAndTypeShouldCalculateTheUnderlyingSha256() { - AttachmentId attachmentId = AttachmentId.forPayloadAndType("payload".getBytes(), "text/plain"); - String expectedId = "d3a2642ee092a1b32c0a83cf94fc2499f7495b7b91b1bd434302a0a4c2aa4278"; - assertThat(attachmentId.getId()).isEqualTo(expectedId); - } - - @Test - public void forPayloadAndTypeShouldCalculateDifferentSha256WhenContentTypeIsDifferent() { - AttachmentId attachmentId = AttachmentId.forPayloadAndType("payload".getBytes(), "text/plain"); - AttachmentId attachmentId2 = AttachmentId.forPayloadAndType("payload".getBytes(), "text/html"); + public void randomShouldGenerateDifferentIds() { + AttachmentId attachmentId = AttachmentId.random(); + AttachmentId attachmentId2 = AttachmentId.random(); assertThat(attachmentId.getId()).isNotEqualTo(attachmentId2.getId()); } @Test - public void forPayloadAndTypeShouldCalculateSameSha256WhenMimeTypeIsSameButNotParameters() { - AttachmentId attachmentId = AttachmentId.forPayloadAndType("payload".getBytes(), "text/html; charset=UTF-8"); - AttachmentId attachmentId2 = AttachmentId.forPayloadAndType("payload".getBytes(), "text/html; charset=UTF-16"); - assertThat(attachmentId.getId()).isEqualTo(attachmentId2.getId()); - } - - @Test - public void forPayloadAndTypeShouldThrowWhenPayloadIsNull() { - assertThatThrownBy(() -> AttachmentId.forPayloadAndType(null, "text/plain")).isInstanceOf(NullPointerException.class); - } - - @Test - public void forPayloadAndTypeShouldThrowWhenTypeIsNull() { - assertThatThrownBy(() -> AttachmentId.forPayloadAndType("payload".getBytes(), null)).isInstanceOf(IllegalArgumentException.class); - } - - @Test - public void forPayloadAndTypeShouldThrowWhenTypeIsEmpty() { - assertThatThrownBy(() -> AttachmentId.forPayloadAndType("payload".getBytes(), "")).isInstanceOf(IllegalArgumentException.class); - } - - @Test public void fromShouldThrowWhenIdIsNull() { String value = null; assertThatThrownBy(() -> AttachmentId.from(value)).isInstanceOf(NullPointerException.class); @@ -103,34 +73,4 @@ public class AttachmentIdTest { assertThat(attachmentId.asUUID()) .isEqualTo(UUID.fromString("2f3a4fcc-ca64-36e3-9bcf-33e92dd93135")); } - - @Test - public void asMimeTypeShouldReturnOnlyMimeTypeFromContentTypeWhenContainingParameters() { - String mimeType = AttachmentId.asMimeType("text/html; charset=UTF-8"); - - assertThat(mimeType).isEqualTo("text/html"); - } - - @Test - public void asMimeTypeShouldReturnOnlyMimeTypeFromContentTypeWhenNoParameters() { - String mimeType = AttachmentId.asMimeType("text/html"); - - assertThat(mimeType).isEqualTo("text/html"); - } - - @Test - public void asMimeTypeShouldReturnDefaultMimeTypeWhenContentTypeIsUnparsable() { - String mimeType = AttachmentId.asMimeType("text"); - - assertThat(mimeType).isEqualTo("application/octet-stream"); - } - - @Test - public void forPayloadAndTypeShouldCalculateDifferentHashesWhenCraftedSha1Collision() throws Exception { - byte[] payload1 = ClassLoaderUtils.getSystemResourceAsByteArray("shattered-1.pdf"); - byte[] payload2 = ClassLoaderUtils.getSystemResourceAsByteArray("shattered-2.pdf"); - AttachmentId attachmentId1 = AttachmentId.forPayloadAndType(payload1, "application/pdf"); - AttachmentId attachmentId2 = AttachmentId.forPayloadAndType(payload2, "application/pdf"); - assertThat(attachmentId1).isNotEqualTo(attachmentId2); - } } http://git-wip-us.apache.org/repos/asf/james-project/blob/bfa5e426/mailbox/api/src/test/java/org/apache/james/mailbox/model/AttachmentTest.java ---------------------------------------------------------------------- diff --git a/mailbox/api/src/test/java/org/apache/james/mailbox/model/AttachmentTest.java b/mailbox/api/src/test/java/org/apache/james/mailbox/model/AttachmentTest.java index 5a7fa24..d73e073 100644 --- a/mailbox/api/src/test/java/org/apache/james/mailbox/model/AttachmentTest.java +++ b/mailbox/api/src/test/java/org/apache/james/mailbox/model/AttachmentTest.java @@ -47,7 +47,7 @@ public class AttachmentTest { } @Test - public void getByteShouldReturnByteArrayRepresentingTheAttachment() throws Exception { + public void getByteShouldReturnByteArrayRepresentingTheAttachment() { String input = "mystream"; Attachment attachment = Attachment.builder() .bytes(input.getBytes(CHARSET)) @@ -104,33 +104,20 @@ public class AttachmentTest { @Test (expected = IllegalStateException.class) public void buildShouldThrowWhenBytesIsNotProvided() { Attachment.builder() - .attachmentId(AttachmentId.forPayloadAndType("mystream".getBytes(CHARSET), "type")) + .attachmentId(AttachmentId.random()) .build(); } @Test (expected = IllegalStateException.class) public void buildShouldThrowWhenTypeIsNotProvided() { Attachment.builder() - .attachmentId(AttachmentId.forPayloadAndType("mystream".getBytes(CHARSET), "type")) + .attachmentId(AttachmentId.random()) .bytes("mystream".getBytes(CHARSET)) .build(); } @Test - public void buildShouldSetTheAttachmentId() throws Exception { - byte[] bytes = "mystream".getBytes(CHARSET); - String type = "content"; - Attachment attachment = Attachment.builder() - .bytes(bytes) - .type(type) - .build(); - AttachmentId expected = AttachmentId.forPayloadAndType(bytes, type); - - assertThat(attachment.getAttachmentId()).isEqualTo(expected); - } - - @Test - public void buildShouldSetTheSize() throws Exception { + public void buildShouldSetTheSize() { String input = "mystream"; Attachment attachment = Attachment.builder() .bytes(input.getBytes(CHARSET)) @@ -141,7 +128,7 @@ public class AttachmentTest { } @Test - public void toBlobShouldGenerateTheAttachmentBlob() throws Exception { + public void toBlobShouldGenerateTheAttachmentBlob() { byte[] bytes = "mystream".getBytes(CHARSET); String content = "content"; Attachment attachment = Attachment.builder() @@ -149,7 +136,7 @@ public class AttachmentTest { .type(content) .build(); Blob expected = Blob.builder() - .id(BlobId.fromString(attachment.getAttachmentId().getId())) + .id(BlobId.fromBytes(bytes)) .contentType(content) .payload(bytes) .build(); http://git-wip-us.apache.org/repos/asf/james-project/blob/bfa5e426/mailbox/api/src/test/java/org/apache/james/mailbox/model/BlobIdTest.java ---------------------------------------------------------------------- diff --git a/mailbox/api/src/test/java/org/apache/james/mailbox/model/BlobIdTest.java b/mailbox/api/src/test/java/org/apache/james/mailbox/model/BlobIdTest.java index fefeed0..c5a5740 100644 --- a/mailbox/api/src/test/java/org/apache/james/mailbox/model/BlobIdTest.java +++ b/mailbox/api/src/test/java/org/apache/james/mailbox/model/BlobIdTest.java @@ -22,6 +22,8 @@ package org.apache.james.mailbox.model; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import java.nio.charset.StandardCharsets; + import org.junit.Test; import nl.jqno.equalsverifier.EqualsVerifier; @@ -52,4 +54,18 @@ public class BlobIdTest { assertThat(BlobId.fromString("abc").asString()) .isEqualTo("abc"); } + + @Test + public void fromBytesShouldProduceASHA256() { + assertThat(BlobId.fromBytes("abc".getBytes(StandardCharsets.UTF_8)).asString()) + .isEqualTo("ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); + } + + @Test + public void fromBytesShouldCalculateSameSha256() { + byte[] bytes = "abc".getBytes(StandardCharsets.UTF_8); + + assertThat(BlobId.fromBytes(bytes)) + .isEqualTo(BlobId.fromBytes(bytes)); + } } http://git-wip-us.apache.org/repos/asf/james-project/blob/bfa5e426/mailbox/store/src/test/java/org/apache/james/mailbox/store/AbstractMailboxManagerAttachmentTest.java ---------------------------------------------------------------------- diff --git a/mailbox/store/src/test/java/org/apache/james/mailbox/store/AbstractMailboxManagerAttachmentTest.java b/mailbox/store/src/test/java/org/apache/james/mailbox/store/AbstractMailboxManagerAttachmentTest.java index 99e9566..562066a 100644 --- a/mailbox/store/src/test/java/org/apache/james/mailbox/store/AbstractMailboxManagerAttachmentTest.java +++ b/mailbox/store/src/test/java/org/apache/james/mailbox/store/AbstractMailboxManagerAttachmentTest.java @@ -181,34 +181,5 @@ public abstract class AbstractMailboxManagerAttachmentTest { List<MessageAttachment> attachments = messages.next().getAttachments(); assertThat(attachments).hasSize(0); } - - @Test - public void appendMessageShouldStoreOnceWhenDuplicateAttachment() throws Exception { - InputStream mailInputStream = ClassLoader.getSystemResourceAsStream("eml/oneAttachmentAndSomeTextInlined.eml"); - InputStream mailInputStream2 = ClassLoader.getSystemResourceAsStream("eml/oneAttachmentAndSomeTextInlined.eml"); - String user2 = "us...@domain.tld"; - MailboxSession user2MailboxSession = new MockMailboxSession(user2); - MessageMapper user2MessageMapper = getMailboxSessionMapperFactory().getMessageMapper(user2MailboxSession); - MailboxMapper user2MailboxMapper = getMailboxSessionMapperFactory().getMailboxMapper(user2MailboxSession); - MailboxPath user2InboxPath = MailboxPath.forUser(user2, "INBOX"); - mailboxManager.createMailbox(user2InboxPath, user2MailboxSession); - Mailbox user2Inbox = user2MailboxMapper.findMailboxByPath(user2InboxPath); - MessageManager user2InboxMessageManager = mailboxManager.getMailbox(user2InboxPath, user2MailboxSession); - - inboxMessageManager.appendMessage(MessageManager.AppendCommand.builder() - .build(mailInputStream), - mailboxSession); - user2InboxMessageManager.appendMessage(MessageManager.AppendCommand.builder() - .build(mailInputStream2), - user2MailboxSession); - Iterator<MailboxMessage> messages = messageMapper.findInMailbox(inbox, MessageRange.all(), FetchType.Full, 1); - Iterator<MailboxMessage> user2Messages = user2MessageMapper.findInMailbox(user2Inbox, MessageRange.all(), FetchType.Full, 1); - assertThat(messages.hasNext()).isTrue(); - List<MessageAttachment> attachments = messages.next().getAttachments(); - assertThat(attachments).hasSize(1); - assertThat(user2Messages.hasNext()).isTrue(); - List<MessageAttachment> user2Attachments = user2Messages.next().getAttachments(); - assertThat(attachments.equals(user2Attachments)).isTrue(); - } } http://git-wip-us.apache.org/repos/asf/james-project/blob/bfa5e426/mailbox/store/src/test/java/org/apache/james/mailbox/store/StoreBlobManagerTest.java ---------------------------------------------------------------------- diff --git a/mailbox/store/src/test/java/org/apache/james/mailbox/store/StoreBlobManagerTest.java b/mailbox/store/src/test/java/org/apache/james/mailbox/store/StoreBlobManagerTest.java index 197d8d7..fc387bb 100644 --- a/mailbox/store/src/test/java/org/apache/james/mailbox/store/StoreBlobManagerTest.java +++ b/mailbox/store/src/test/java/org/apache/james/mailbox/store/StoreBlobManagerTest.java @@ -82,7 +82,7 @@ public class StoreBlobManagerTest { assertThat(blobManager.retrieve(BLOB_ID_ATTACHMENT, session)) .isEqualTo(Blob.builder() - .id(BLOB_ID_ATTACHMENT) + .id(BlobId.fromString("ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad")) .contentType(CONTENT_TYPE) .payload(BYTES) .build()); http://git-wip-us.apache.org/repos/asf/james-project/blob/bfa5e426/mailbox/store/src/test/java/org/apache/james/mailbox/store/mail/model/AttachmentMapperTest.java ---------------------------------------------------------------------- diff --git a/mailbox/store/src/test/java/org/apache/james/mailbox/store/mail/model/AttachmentMapperTest.java b/mailbox/store/src/test/java/org/apache/james/mailbox/store/mail/model/AttachmentMapperTest.java index 03b27ce..5aea667 100644 --- a/mailbox/store/src/test/java/org/apache/james/mailbox/store/mail/model/AttachmentMapperTest.java +++ b/mailbox/store/src/test/java/org/apache/james/mailbox/store/mail/model/AttachmentMapperTest.java @@ -39,7 +39,7 @@ import org.junit.rules.ExpectedException; import com.google.common.collect.ImmutableList; public abstract class AttachmentMapperTest { - private static final AttachmentId UNKNOWN_ATTACHMENT_ID = AttachmentId.forPayloadAndType("unknown".getBytes(StandardCharsets.UTF_8), "type"); + private static final AttachmentId UNKNOWN_ATTACHMENT_ID = AttachmentId.from("unknown"); public static final Username OWNER = Username.fromRawValue("owner"); public static final Username ADDITIONAL_OWNER = Username.fromRawValue("additionalOwner"); @@ -108,13 +108,13 @@ public abstract class AttachmentMapperTest { } @Test - public void getAttachmentsShouldThrowWhenNullAttachmentId() throws Exception { + public void getAttachmentsShouldThrowWhenNullAttachmentId() { expected.expect(IllegalArgumentException.class); attachmentMapper.getAttachments(null); } @Test - public void getAttachmentsShouldReturnEmptyListWhenNonReferencedAttachmentId() throws Exception { + public void getAttachmentsShouldReturnEmptyListWhenNonReferencedAttachmentId() { List<Attachment> attachments = attachmentMapper.getAttachments(ImmutableList.of(UNKNOWN_ATTACHMENT_ID)); assertThat(attachments).isEmpty(); http://git-wip-us.apache.org/repos/asf/james-project/blob/bfa5e426/server/container/guice/protocols/jmap/src/main/java/org/apache/james/utils/MessageIdProbe.java ---------------------------------------------------------------------- diff --git a/server/container/guice/protocols/jmap/src/main/java/org/apache/james/utils/MessageIdProbe.java b/server/container/guice/protocols/jmap/src/main/java/org/apache/james/utils/MessageIdProbe.java index 66d05ae..5bfb6a7 100644 --- a/server/container/guice/protocols/jmap/src/main/java/org/apache/james/utils/MessageIdProbe.java +++ b/server/container/guice/protocols/jmap/src/main/java/org/apache/james/utils/MessageIdProbe.java @@ -29,11 +29,15 @@ import org.apache.james.mailbox.MailboxSession; import org.apache.james.mailbox.MessageIdManager; import org.apache.james.mailbox.MessageManager.FlagsUpdateMode; import org.apache.james.mailbox.exception.MailboxException; +import org.apache.james.mailbox.model.AttachmentId; import org.apache.james.mailbox.model.FetchGroupImpl; import org.apache.james.mailbox.model.MailboxId; +import org.apache.james.mailbox.model.MessageAttachment; import org.apache.james.mailbox.model.MessageId; import org.apache.james.mailbox.model.MessageResult; +import com.github.fge.lambdas.Throwing; +import com.github.steveash.guavate.Guavate; import com.google.common.collect.ImmutableList; public class MessageIdProbe implements GuiceProbe { @@ -57,4 +61,17 @@ public class MessageIdProbe implements GuiceProbe { messageIdManager.setFlags(newFlags, FlagsUpdateMode.REPLACE, messageId, mailboxIds, mailboxSession); } + + public List<AttachmentId> retrieveAttachmentIds(MessageId messageId, String username) throws MailboxException { + MailboxSession mailboxSession = mailboxManager.createSystemSession(username); + List<MessageResult> messages = messageIdManager.getMessages( + ImmutableList.of(messageId), + FetchGroupImpl.MINIMAL, + mailboxSession); + + return messages.stream() + .flatMap(Throwing.function(messageResult -> messageResult.getAttachments().stream())) + .map(MessageAttachment::getAttachmentId) + .collect(Guavate.toImmutableList()); + } } http://git-wip-us.apache.org/repos/asf/james-project/blob/bfa5e426/server/protocols/jmap-integration-testing/jmap-integration-testing-common/src/test/java/org/apache/james/jmap/methods/integration/SetMessagesMethodTest.java ---------------------------------------------------------------------- diff --git a/server/protocols/jmap-integration-testing/jmap-integration-testing-common/src/test/java/org/apache/james/jmap/methods/integration/SetMessagesMethodTest.java b/server/protocols/jmap-integration-testing/jmap-integration-testing-common/src/test/java/org/apache/james/jmap/methods/integration/SetMessagesMethodTest.java index ea5321a..8dda617 100644 --- a/server/protocols/jmap-integration-testing/jmap-integration-testing-common/src/test/java/org/apache/james/jmap/methods/integration/SetMessagesMethodTest.java +++ b/server/protocols/jmap-integration-testing/jmap-integration-testing-common/src/test/java/org/apache/james/jmap/methods/integration/SetMessagesMethodTest.java @@ -1910,7 +1910,7 @@ public abstract class SetMessagesMethodTest { .bytes("attachment".getBytes(StandardCharsets.UTF_8)) .type("application/octet-stream") .build(); - uploadAttachment(attachment); + String uploadedBlobId = uploadAttachment(attachment); String requestBody = "[" + " [" + " \"setMessages\"," + @@ -1921,7 +1921,7 @@ public abstract class SetMessagesMethodTest { " \"subject\": \"subject\"," + " \"keywords\": {\"$Draft\": true}," + " \"attachments\": [" + - " {\"blobId\" : \"" + attachment.getAttachmentId().getId() + "\", " + + " {\"blobId\" : \"" + uploadedBlobId + "\", " + " \"type\" : \"" + attachment.getType() + "\"," + " \"size\" : " + attachment.getSize() + "}" + " ]," + @@ -1949,11 +1949,6 @@ public abstract class SetMessagesMethodTest { @Test public void setMessagesShouldNotAllowDraftCreationInSomeoneElseMailbox() throws Exception { String messageCreationId = "creationId1337"; - Attachment attachment = Attachment.builder() - .bytes("attachment".getBytes(StandardCharsets.UTF_8)) - .type("application/octet-stream") - .build(); - uploadAttachment(attachment); String requestBody = "[" + " [" + " \"setMessages\"," + @@ -1987,11 +1982,6 @@ public abstract class SetMessagesMethodTest { @Test public void setMessagesShouldNotAllowDraftCreationInADelegatedMailbox() throws Exception { String messageCreationId = "creationId1337"; - Attachment attachment = Attachment.builder() - .bytes("attachment".getBytes(StandardCharsets.UTF_8)) - .type("application/octet-stream") - .build(); - uploadAttachment(attachment); jmapServer.getProbe(ACLProbeImpl.class) .addRights( @@ -3664,12 +3654,12 @@ public abstract class SetMessagesMethodTest { .bytes("attachment".getBytes(StandardCharsets.UTF_8)) .type("application/octet-stream") .build(); - uploadAttachment(attachment); + String uploadedAttachment1 = uploadAttachment(attachment); Attachment attachment2 = Attachment.builder() .bytes("attachment2".getBytes(StandardCharsets.UTF_8)) .type("application/octet-stream") .build(); - uploadAttachment(attachment2); + String uploadedAttachment2 = uploadAttachment(attachment2); String messageCreationId = "creationId"; String fromAddress = USERNAME; @@ -3685,10 +3675,10 @@ public abstract class SetMessagesMethodTest { " \"textBody\": \"Test body\"," + " \"mailboxIds\": [\"" + outboxId + "\"], " + " \"attachments\": [" + - " {\"blobId\" : \"" + attachment.getAttachmentId().getId() + "\", " + + " {\"blobId\" : \"" + uploadedAttachment1 + "\", " + " \"type\" : \"" + attachment.getType() + "\", " + " \"size\" : " + attachment.getSize() + "}," + - " {\"blobId\" : \"" + attachment2.getAttachmentId().getId() + "\", " + + " {\"blobId\" : \"" + uploadedAttachment2 + "\", " + " \"type\" : \"" + attachment2.getType() + "\", " + " \"size\" : " + attachment2.getSize() + ", " + " \"cid\" : \"123456789\", " + @@ -3715,12 +3705,12 @@ public abstract class SetMessagesMethodTest { .body(ARGUMENTS + ".notCreated", aMapWithSize(0)) .body(ARGUMENTS + ".created", aMapWithSize(1)) .body(createdPath + ".attachments", hasSize(2)) - .body(firstAttachment + ".blobId", equalTo(attachment.getAttachmentId().getId())) + .body(firstAttachment + ".blobId", equalTo(uploadedAttachment1)) .body(firstAttachment + ".type", equalTo("application/octet-stream; charset=UTF-8")) .body(firstAttachment + ".size", equalTo((int) attachment.getSize())) .body(firstAttachment + ".cid", nullValue()) .body(firstAttachment + ".isInline", equalTo(false)) - .body(secondAttachment + ".blobId", equalTo(attachment2.getAttachmentId().getId())) + .body(secondAttachment + ".blobId", equalTo(uploadedAttachment2)) .body(secondAttachment + ".type", equalTo("application/octet-stream; charset=UTF-8")) .body(secondAttachment + ".size", equalTo((int) attachment2.getSize())) .body(secondAttachment + ".cid", equalTo("123456789")) @@ -3733,17 +3723,17 @@ public abstract class SetMessagesMethodTest { .bytes("attachment".getBytes(StandardCharsets.UTF_8)) .type("application/octet-stream") .build(); - uploadAttachment(attachment); + String uploadedAttachment1 = uploadAttachment(attachment); Attachment attachment2 = Attachment.builder() .bytes("attachment2".getBytes(StandardCharsets.UTF_8)) .type("application/octet-stream") .build(); - uploadAttachment(attachment2); + String uploadedAttachment2 = uploadAttachment(attachment2); Attachment attachment3 = Attachment.builder() .bytes("attachment3".getBytes(StandardCharsets.UTF_8)) .type("application/octet-stream") .build(); - uploadAttachment(attachment3); + String uploadedAttachment3 = uploadAttachment(attachment3); String messageCreationId = "creationId"; String fromAddress = USERNAME; @@ -3764,21 +3754,21 @@ public abstract class SetMessagesMethodTest { " \"attachments\":" + " [" + " {" + - " \"blobId\" : \"" + attachment.getAttachmentId().getId() + "\", " + + " \"blobId\" : \"" + uploadedAttachment1 + "\", " + " \"type\" : \"" + attachment.getType() + "\", " + " \"size\" : " + attachment.getSize() + "," + " \"name\" : \"دÙÙاصÙر.png\", " + " \"isInline\" : false" + " }," + " {" + - " \"blobId\" : \"" + attachment2.getAttachmentId().getId() + "\", " + + " \"blobId\" : \"" + uploadedAttachment2 + "\", " + " \"type\" : \"" + attachment2.getType() + "\", " + " \"size\" : " + attachment2.getSize() + "," + " \"name\" : \"ÑволÑÑиониÑоваÑÑ.png\", " + " \"isInline\" : false" + " }," + " {" + - " \"blobId\" : \"" + attachment3.getAttachmentId().getId() + "\", " + + " \"blobId\" : \"" + uploadedAttachment3 + "\", " + " \"type\" : \"" + attachment3.getType() + "\", " + " \"size\" : " + attachment3.getSize() + "," + " \"name\" : \"è¿åè¿æ¯ä¸.png\"," + @@ -3819,19 +3809,19 @@ public abstract class SetMessagesMethodTest { .bytes("attachment".getBytes(StandardCharsets.UTF_8)) .type("application/octet-stream") .build(); - uploadAttachment(attachment); + String uploadedAttachment1 = uploadAttachment(attachment); Attachment attachment2 = Attachment.builder() .bytes("attachment2".getBytes(StandardCharsets.UTF_8)) .type("application/octet-stream") .build(); - uploadAttachment(attachment2); + String uploadedAttachment2 = uploadAttachment(attachment2); Attachment attachment3 = Attachment.builder() .bytes("attachment3".getBytes(StandardCharsets.UTF_8)) .type("application/octet-stream") .build(); - uploadAttachment(attachment3); + String uploadedAttachment3 = uploadAttachment(attachment3); String messageCreationId = "creationId"; String fromAddress = USERNAME; @@ -3852,21 +3842,21 @@ public abstract class SetMessagesMethodTest { " \"attachments\":" + " [" + " {" + - " \"blobId\" : \"" + attachment.getAttachmentId().getId() + "\", " + + " \"blobId\" : \"" + uploadedAttachment1 + "\", " + " \"type\" : \"" + attachment.getType() + "\", " + " \"size\" : " + attachment.getSize() + "," + " \"name\" : \"دÙÙاصÙر.png\", " + " \"isInline\" : false" + " }," + " {" + - " \"blobId\" : \"" + attachment2.getAttachmentId().getId() + "\", " + + " \"blobId\" : \"" + uploadedAttachment2 + "\", " + " \"type\" : \"" + attachment2.getType() + "\", " + " \"size\" : " + attachment2.getSize() + "," + " \"name\" : \"ÑволÑÑиониÑоваÑÑ.png\", " + " \"isInline\" : false" + " }," + " {" + - " \"blobId\" : \"" + attachment3.getAttachmentId().getId() + "\", " + + " \"blobId\" : \"" + uploadedAttachment3 + "\", " + " \"type\" : \"" + attachment3.getType() + "\", " + " \"size\" : " + attachment3.getSize() + "," + " \"name\" : \"è¿åè¿æ¯ä¸.png\"," + @@ -3919,20 +3909,30 @@ public abstract class SetMessagesMethodTest { .body(thirdAttachment + ".name", equalTo("è¿åè¿æ¯ä¸.png")); } - private void uploadAttachment(Attachment attachment) throws IOException { - with() + private String uploadAttachment(Attachment attachment) throws IOException { + return with() .header("Authorization", accessToken.serialize()) .contentType(attachment.getType()) .content(attachment.getStream()) - .post("/upload"); + .post("/upload") + .then() + .extract() + .body() + .jsonPath() + .getString("blobId"); } - private void uploadTextAttachment(Attachment attachment) throws IOException { - with() + private String uploadTextAttachment(Attachment attachment) throws IOException { + return with() .header("Authorization", accessToken.serialize()) .contentType(attachment.getType()) .content(new String(IOUtils.toByteArray(attachment.getStream()), StandardCharsets.UTF_8)) - .post("/upload"); + .post("/upload") + .then() + .extract() + .body() + .jsonPath() + .getString("blobId"); } @Test @@ -3948,8 +3948,7 @@ public abstract class SetMessagesMethodTest { .bytes(rawBytes) .type("application/octet-stream") .build(); - uploadAttachment(attachment); - String expectedBlobId = attachment.getAttachmentId().getId(); + String uploadedAttachment = uploadAttachment(attachment); String messageCreationId = "creationId"; String fromAddress = USERNAME; @@ -3965,7 +3964,7 @@ public abstract class SetMessagesMethodTest { " \"textBody\": \"Test body\"," + " \"mailboxIds\": [\"" + outboxId + "\"], " + " \"attachments\": [" + - " {\"blobId\" : \"" + attachment.getAttachmentId().getId() + "\", " + + " {\"blobId\" : \"" + uploadedAttachment + "\", " + " \"type\" : \"" + attachment.getType() + "\", " + " \"size\" : " + attachment.getSize() + ", " + " \"cid\" : \"123456789\", " + @@ -3996,7 +3995,7 @@ public abstract class SetMessagesMethodTest { String firstMessage = ARGUMENTS + ".list[0]"; String firstAttachment = firstMessage + ".attachments[0]"; - given() + String blobId = given() .header("Authorization", accessToken.serialize()) .body("[[\"getMessages\", {\"ids\": [\"" + receivedMessageId + "\"]}, \"#0\"]]") .when() @@ -4006,21 +4005,25 @@ public abstract class SetMessagesMethodTest { .body(NAME, equalTo("messages")) .body(ARGUMENTS + ".list", hasSize(1)) .body(firstMessage + ".attachments", hasSize(1)) - .body(firstAttachment + ".blobId", equalTo(expectedBlobId)) .body(firstAttachment + ".type", equalTo("application/octet-stream")) .body(firstAttachment + ".size", equalTo((int) attachment.getSize())) .body(firstAttachment + ".cid", equalTo("123456789")) - .body(firstAttachment + ".isInline", equalTo(true)); + .body(firstAttachment + ".isInline", equalTo(true)) + .extract() + .jsonPath() + .getString(firstAttachment + ".blobId"); + + checkBlobContent(blobId, rawBytes); } @Test public void attachmentsShouldBeRetrievedWhenChainingSetMessagesAndGetMessagesTextAttachment() throws Exception { + byte[] rawBytes = ByteStreams.toByteArray(new ZeroedInputStream(_1MB)); Attachment attachment = Attachment.builder() - .bytes(ByteStreams.toByteArray(new ZeroedInputStream(_1MB))) + .bytes(rawBytes) .type("application/octet-stream") .build(); - uploadAttachment(attachment); - String expectedBlobId = attachment.getAttachmentId().getId(); + String uploadedAttachment = uploadAttachment(attachment); String messageCreationId = "creationId"; String fromAddress = USERNAME; @@ -4036,7 +4039,7 @@ public abstract class SetMessagesMethodTest { " \"textBody\": \"Test body\"," + " \"mailboxIds\": [\"" + outboxId + "\"], " + " \"attachments\": [" + - " {\"blobId\" : \"" + attachment.getAttachmentId().getId() + "\", " + + " {\"blobId\" : \"" + uploadedAttachment + "\", " + " \"type\" : \"" + attachment.getType() + "\", " + " \"size\" : " + attachment.getSize() + ", " + " \"cid\" : \"123456789\", " + @@ -4068,7 +4071,7 @@ public abstract class SetMessagesMethodTest { String firstMessage = ARGUMENTS + ".list[0]"; String firstAttachment = firstMessage + ".attachments[0]"; - given() + String blobId = given() .header("Authorization", accessToken.serialize()) .body("[[\"getMessages\", {\"ids\": [\"" + receivedMessageId + "\"]}, \"#0\"]]") .when() @@ -4078,11 +4081,15 @@ public abstract class SetMessagesMethodTest { .body(NAME, equalTo("messages")) .body(ARGUMENTS + ".list", hasSize(1)) .body(firstMessage + ".attachments", hasSize(1)) - .body(firstAttachment + ".blobId", equalTo(expectedBlobId)) .body(firstAttachment + ".type", equalTo("application/octet-stream")) .body(firstAttachment + ".size", equalTo((int) attachment.getSize())) .body(firstAttachment + ".cid", equalTo("123456789")) - .body(firstAttachment + ".isInline", equalTo(true)); + .body(firstAttachment + ".isInline", equalTo(true)) + .extract() + .jsonPath() + .getString(firstAttachment + ".blobId"); + + checkBlobContent(blobId, rawBytes); } private boolean isAnyMessageFoundInInbox(AccessToken recipientToken) { @@ -4106,13 +4113,14 @@ public abstract class SetMessagesMethodTest { @Test public void attachmentsAndBodysShouldBeRetrievedWhenChainingSetMessagesAndGetMessagesWithMixedTextAndHtmlBodyAndHtmlAttachment() throws Exception { + byte[] rawBytes = ("<html>\n" + + " <body>attachment</body>\n" + // needed indentation, else restassured is adding some + "</html>").getBytes(StandardCharsets.UTF_8); Attachment attachment = Attachment.builder() - .bytes(("<html>\n" + - " <body>attachment</body>\n" + // needed indentation, else restassured is adding some - "</html>").getBytes(StandardCharsets.UTF_8)) + .bytes(rawBytes) .type("text/html; charset=UTF-8") .build(); - uploadTextAttachment(attachment); + String uploadedBlobId = uploadTextAttachment(attachment); String messageCreationId = "creationId"; String fromAddress = USERNAME; @@ -4129,7 +4137,7 @@ public abstract class SetMessagesMethodTest { " \"htmlBody\": \"Test <b>body</b>, HTML version\"," + " \"mailboxIds\": [\"" + outboxId + "\"], " + " \"attachments\": [" + - " {\"blobId\" : \"" + attachment.getAttachmentId().getId() + "\", " + + " {\"blobId\" : \"" + uploadedBlobId + "\", " + " \"type\" : \"" + attachment.getType() + "\", " + " \"size\" : " + attachment.getSize() + ", " + " \"isInline\" : false }" + @@ -4160,7 +4168,7 @@ public abstract class SetMessagesMethodTest { String firstMessage = ARGUMENTS + ".list[0]"; String firstAttachment = firstMessage + ".attachments[0]"; - given() + String blobId = given() .header("Authorization", accessToken.serialize()) .body("[[\"getMessages\", {\"ids\": [\"" + receivedMessageId + "\"]}, \"#0\"]]") .when() @@ -4173,20 +4181,25 @@ public abstract class SetMessagesMethodTest { .body(firstMessage + ".textBody", equalTo("Test body, plain text version")) .body(firstMessage + ".htmlBody", equalTo("Test <b>body</b>, HTML version")) .body(firstMessage + ".attachments", hasSize(1)) - .body(firstAttachment + ".blobId", equalTo(attachment.getAttachmentId().getId())) .body(firstAttachment + ".type", equalTo("text/html")) - .body(firstAttachment + ".size", equalTo((int) attachment.getSize())); + .body(firstAttachment + ".size", equalTo((int) attachment.getSize())) + .extract() + .jsonPath() + .getString(firstAttachment + ".blobId"); + + checkBlobContent(blobId, rawBytes); } @Test public void attachmentsAndBodyShouldBeRetrievedWhenChainingSetMessagesAndGetMessagesWithTextBodyAndHtmlAttachment() throws Exception { + byte[] rawBytes = ("<html>\n" + + " <body>attachment</body>\n" + // needed indentation, else restassured is adding some + "</html>").getBytes(StandardCharsets.UTF_8); Attachment attachment = Attachment.builder() - .bytes(("<html>\n" + - " <body>attachment</body>\n" + // needed indentation, else restassured is adding some - "</html>").getBytes(StandardCharsets.UTF_8)) + .bytes(rawBytes) .type("text/html; charset=UTF-8") .build(); - uploadTextAttachment(attachment); + String uploadedBlobId = uploadTextAttachment(attachment); String messageCreationId = "creationId"; String fromAddress = USERNAME; @@ -4202,7 +4215,7 @@ public abstract class SetMessagesMethodTest { " \"textBody\": \"Test body, plain text version\"," + " \"mailboxIds\": [\"" + outboxId + "\"], " + " \"attachments\": [" + - " {\"blobId\" : \"" + attachment.getAttachmentId().getId() + "\", " + + " {\"blobId\" : \"" + uploadedBlobId + "\", " + " \"type\" : \"" + attachment.getType() + "\", " + " \"size\" : " + attachment.getSize() + ", " + " \"isInline\" : false }" + @@ -4233,7 +4246,7 @@ public abstract class SetMessagesMethodTest { String firstMessage = ARGUMENTS + ".list[0]"; String firstAttachment = firstMessage + ".attachments[0]"; - given() + String blobId = given() .header("Authorization", accessToken.serialize()) .body("[[\"getMessages\", {\"ids\": [\"" + receivedMessageId + "\"]}, \"#0\"]]") .when() @@ -4246,18 +4259,35 @@ public abstract class SetMessagesMethodTest { .body(firstMessage + ".textBody", equalTo("Test body, plain text version")) .body(firstMessage + ".htmlBody", isEmptyOrNullString()) .body(firstMessage + ".attachments", hasSize(1)) - .body(firstAttachment + ".blobId", equalTo(attachment.getAttachmentId().getId())) .body(firstAttachment + ".type", equalTo("text/html")) - .body(firstAttachment + ".size", equalTo((int) attachment.getSize())); + .body(firstAttachment + ".size", equalTo((int) attachment.getSize())) + .extract() + .jsonPath() + .getString(firstAttachment + ".blobId"); + + checkBlobContent(blobId, rawBytes); } - + + public void checkBlobContent(String blobId, byte[] rawBytes) { + byte[] attachmentBytes = with() + .header("Authorization", accessToken.serialize()) + .get("/download/" + blobId) + .then() + .extract() + .body() + .asByteArray(); + + assertThat(attachmentBytes).containsExactly(rawBytes); + } + @Test public void attachmentAndEmptyBodyShouldBeRetrievedWhenChainingSetMessagesAndGetMessagesWithTextAttachmentWithoutMailBody() throws Exception { + byte[] rawBytes = ("some text").getBytes(StandardCharsets.UTF_8); Attachment attachment = Attachment.builder() - .bytes(("some text").getBytes(StandardCharsets.UTF_8)) + .bytes(rawBytes) .type("text/plain; charset=UTF-8") .build(); - uploadTextAttachment(attachment); + String uploadedBlobId = uploadTextAttachment(attachment); String messageCreationId = "creationId"; String fromAddress = USERNAME; @@ -4272,7 +4302,7 @@ public abstract class SetMessagesMethodTest { " \"subject\": \"Message with an attachment\"," + " \"mailboxIds\": [\"" + outboxId + "\"], " + " \"attachments\": [" + - " {\"blobId\" : \"" + attachment.getAttachmentId().getId() + "\", " + + " {\"blobId\" : \"" + uploadedBlobId + "\", " + " \"type\" : \"" + attachment.getType() + "\", " + " \"size\" : " + attachment.getSize() + ", " + " \"isInline\" : false }" + @@ -4303,7 +4333,7 @@ public abstract class SetMessagesMethodTest { String firstMessage = ARGUMENTS + ".list[0]"; String firstAttachment = firstMessage + ".attachments[0]"; - given() + String blobId = given() .header("Authorization", accessToken.serialize()) .body("[[\"getMessages\", {\"ids\": [\"" + receivedMessageId + "\"]}, \"#0\"]]") .when() @@ -4316,9 +4346,13 @@ public abstract class SetMessagesMethodTest { .body(firstMessage + ".textBody", isEmptyOrNullString()) .body(firstMessage + ".htmlBody", isEmptyOrNullString()) .body(firstMessage + ".attachments", hasSize(1)) - .body(firstAttachment + ".blobId", equalTo(attachment.getAttachmentId().getId())) .body(firstAttachment + ".type", equalTo("text/plain")) - .body(firstAttachment + ".size", equalTo((int) attachment.getSize())); + .body(firstAttachment + ".size", equalTo((int) attachment.getSize())) + .extract() + .jsonPath() + .getString(firstAttachment + ".blobId"); + + checkBlobContent(blobId, rawBytes); } @Test @@ -5013,7 +5047,7 @@ public abstract class SetMessagesMethodTest { .bytes(ClassLoaderUtils.getSystemResourceAsByteArray("attachment/nonIndexableAttachment.html")) .type("text/html") .build(); - uploadTextAttachment(nonIndexableAttachment); + String uploadedBlobId = uploadTextAttachment(nonIndexableAttachment); String messageCreationId = "creationId"; String fromAddress = USERNAME; @@ -5029,7 +5063,7 @@ public abstract class SetMessagesMethodTest { " \"textBody\": \"Test body\"," + " \"mailboxIds\": [\"" + outboxId + "\"], " + " \"attachments\": [" + - " {\"blobId\" : \"" + nonIndexableAttachment.getAttachmentId().getId() + "\", " + + " {\"blobId\" : \"" + uploadedBlobId + "\", " + " \"type\" : \"" + nonIndexableAttachment.getType() + "\", " + " \"name\" : \"nonIndexableAttachment.html\", " + " \"size\" : " + nonIndexableAttachment.getSize() + "}" + @@ -5054,7 +5088,7 @@ public abstract class SetMessagesMethodTest { .body(ARGUMENTS + ".notCreated", aMapWithSize(0)) .body(ARGUMENTS + ".created", aMapWithSize(1)) .body(createdPath + ".attachments", hasSize(1)) - .body(singleAttachment + ".blobId", equalTo(nonIndexableAttachment.getAttachmentId().getId())) + .body(singleAttachment + ".blobId", equalTo(uploadedBlobId)) .body(singleAttachment + ".type", equalTo("text/html; charset=UTF-8")) .body(singleAttachment + ".size", equalTo((int) nonIndexableAttachment.getSize())); } @@ -5065,7 +5099,7 @@ public abstract class SetMessagesMethodTest { .bytes(ClassLoaderUtils.getSystemResourceAsByteArray("attachment/nonIndexableAttachment.html")) .type("text/html") .build(); - uploadTextAttachment(nonIndexableAttachment); + String uploadedBlobId = uploadTextAttachment(nonIndexableAttachment); String messageCreationId = "creationId"; String fromAddress = USERNAME; @@ -5081,7 +5115,7 @@ public abstract class SetMessagesMethodTest { " \"textBody\": \"Test body\"," + " \"mailboxIds\": [\"" + outboxId + "\"], " + " \"attachments\": [" + - " {\"blobId\" : \"" + nonIndexableAttachment.getAttachmentId().getId() + "\", " + + " {\"blobId\" : \"" + uploadedBlobId + "\", " + " \"type\" : \"" + nonIndexableAttachment.getType() + "\", " + " \"name\" : \"nonIndexableAttachment.html\", " + " \"size\" : " + nonIndexableAttachment.getSize() + "}" + @@ -5125,7 +5159,7 @@ public abstract class SetMessagesMethodTest { .bytes(ClassLoaderUtils.getSystemResourceAsByteArray("attachment/nonIndexableAttachment.html")) .type("text/html") .build(); - uploadTextAttachment(nonIndexableAttachment); + String uploadedBlobId = uploadTextAttachment(nonIndexableAttachment); String messageCreationId = "creationId"; String fromAddress = USERNAME; @@ -5142,7 +5176,7 @@ public abstract class SetMessagesMethodTest { " \"textBody\": \"Test body\"," + " \"mailboxIds\": [\"" + outboxId + "\"], " + " \"attachments\": [" + - " {\"blobId\" : \"" + nonIndexableAttachment.getAttachmentId().getId() + "\", " + + " {\"blobId\" : \"" + uploadedBlobId + "\", " + " \"type\" : \"" + nonIndexableAttachment.getType() + "\", " + " \"name\" : \"nonIndexableAttachment.html\", " + " \"size\" : " + nonIndexableAttachment.getSize() + "}" + @@ -5181,12 +5215,12 @@ public abstract class SetMessagesMethodTest { .bytes("attachment".getBytes(StandardCharsets.UTF_8)) .type("application/octet-stream") .build(); - uploadAttachment(attachment); + String uploadedAttachment1 = uploadAttachment(attachment); Attachment attachment2 = Attachment.builder() .bytes("attachment2".getBytes(StandardCharsets.UTF_8)) .type("application/octet-stream") .build(); - uploadAttachment(attachment2); + String uploadedAttachment2 = uploadAttachment(attachment2); String messageCreationId = "creationId"; String fromAddress = USERNAME; @@ -5202,10 +5236,10 @@ public abstract class SetMessagesMethodTest { " \"textBody\": \"Test body\"," + " \"mailboxIds\": [\"" + outboxId + "\"], " + " \"attachments\": [" + - " {\"blobId\" : \"" + attachment.getAttachmentId().getId() + "\", " + + " {\"blobId\" : \"" + uploadedAttachment1 + "\", " + " \"type\" : \"" + attachment.getType() + "\", " + " \"size\" : " + attachment.getSize() + "}," + - " {\"blobId\" : \"" + attachment2.getAttachmentId().getId() + "\", " + + " {\"blobId\" : \"" + uploadedAttachment2 + "\", " + " \"type\" : \"" + attachment2.getType() + "\", " + " \"size\" : " + attachment2.getSize() + ", " + " \"isInline\" : true }" + @@ -5231,12 +5265,12 @@ public abstract class SetMessagesMethodTest { .body(ARGUMENTS + ".notCreated", aMapWithSize(0)) .body(ARGUMENTS + ".created", aMapWithSize(1)) .body(createdPath + ".attachments", hasSize(2)) - .body(firstAttachment + ".blobId", equalTo(attachment.getAttachmentId().getId())) + .body(firstAttachment + ".blobId", equalTo(uploadedAttachment1)) .body(firstAttachment + ".type", equalTo("application/octet-stream; charset=UTF-8")) .body(firstAttachment + ".size", equalTo((int) attachment.getSize())) .body(firstAttachment + ".cid", nullValue()) .body(firstAttachment + ".isInline", equalTo(false)) - .body(secondAttachment + ".blobId", equalTo(attachment2.getAttachmentId().getId())) + .body(secondAttachment + ".blobId", equalTo(uploadedAttachment2)) .body(secondAttachment + ".type", equalTo("application/octet-stream; charset=UTF-8")) .body(secondAttachment + ".size", equalTo((int) attachment2.getSize())) .body(secondAttachment + ".cid", nullValue()) http://git-wip-us.apache.org/repos/asf/james-project/blob/bfa5e426/server/protocols/jmap-integration-testing/jmap-integration-testing-common/src/test/java/org/apache/james/jmap/methods/integration/cucumber/DownloadStepdefs.java ---------------------------------------------------------------------- diff --git a/server/protocols/jmap-integration-testing/jmap-integration-testing-common/src/test/java/org/apache/james/jmap/methods/integration/cucumber/DownloadStepdefs.java b/server/protocols/jmap-integration-testing/jmap-integration-testing-common/src/test/java/org/apache/james/jmap/methods/integration/cucumber/DownloadStepdefs.java index 1ff185b..2abf00b 100644 --- a/server/protocols/jmap-integration-testing/jmap-integration-testing-common/src/test/java/org/apache/james/jmap/methods/integration/cucumber/DownloadStepdefs.java +++ b/server/protocols/jmap-integration-testing/jmap-integration-testing-common/src/test/java/org/apache/james/jmap/methods/integration/cucumber/DownloadStepdefs.java @@ -41,6 +41,8 @@ import org.apache.http.client.utils.URIBuilder; import org.apache.james.jmap.api.access.AccessToken; import org.apache.james.jmap.model.AttachmentAccessToken; import org.apache.james.mailbox.MessageManager.AppendCommand; +import org.apache.james.mailbox.exception.MailboxException; +import org.apache.james.mailbox.model.AttachmentId; import org.apache.james.mailbox.model.ComposedMessageId; import org.apache.james.mailbox.model.MailboxConstants; import org.apache.james.mailbox.model.MailboxPath; @@ -107,31 +109,37 @@ public class DownloadStepdefs { ComposedMessageId composedMessageId = mainStepdefs.mailboxProbe.appendMessage(user, mailboxPath, AppendCommand.from(ClassLoader.getSystemResourceAsStream("eml/oneAttachment.eml"))); - inputToMessageId.put(messageId, composedMessageId.getMessageId()); - attachmentsByMessageId.put(messageId, attachmentId); - blobIdByAttachmentId.put(attachmentId, ONE_ATTACHMENT_EML_ATTACHMENT_BLOB_ID); + retrieveAndSaveAttachmentDetails(user, messageId, attachmentId, composedMessageId); } @Given("^\"([^\"]*)\" mailbox \"([^\"]*)\" contains a message \"([^\"]*)\" with an inlined attachment \"([^\"]*)\"$") public void appendMessageWithInlinedAttachmentToMailbox(String user, String mailbox, String messageId, String attachmentId) throws Throwable { MailboxPath mailboxPath = MailboxPath.forUser(user, mailbox); - mainStepdefs.mailboxProbe.appendMessage(user, mailboxPath, - AppendCommand.from(ClassLoader.getSystemResourceAsStream("eml/oneInlinedImage.eml"))); - + ComposedMessageId composedMessageId = mainStepdefs.mailboxProbe.appendMessage(user, mailboxPath, + AppendCommand.from(ClassLoader.getSystemResourceAsStream("eml/oneInlinedImage.eml"))); + + retrieveAndSaveAttachmentDetails(user, messageId, attachmentId, composedMessageId); + } + + public void retrieveAndSaveAttachmentDetails(String user, String messageId, String attachmentId, ComposedMessageId composedMessageId) throws MailboxException { + AttachmentId mailboxAttachmentId = mainStepdefs.messageIdProbe + .retrieveAttachmentIds(composedMessageId.getMessageId(), user) + .get(0); + + inputToMessageId.put(messageId, composedMessageId.getMessageId()); attachmentsByMessageId.put(messageId, attachmentId); + blobIdByAttachmentId.put(attachmentId, mailboxAttachmentId.getId()); } @Given("^\"([^\"]*)\" mailbox \"([^\"]*)\" contains a message \"([^\"]*)\" with multiple same inlined attachments \"([^\"]*)\"$") public void appendMessageWithSameInlinedAttachmentsToMailbox(String user, String mailbox, String messageName, String attachmentId) throws Throwable { MailboxPath mailboxPath = MailboxPath.forUser(user, mailbox); - mainStepdefs.mailboxProbe.appendMessage(user, mailboxPath, + ComposedMessageId composedMessageId = mainStepdefs.mailboxProbe.appendMessage(user, mailboxPath, AppendCommand.from(ClassLoader.getSystemResourceAsStream("eml/sameInlinedImages.eml"))); - attachmentsByMessageId.put(messageName, attachmentId); - - blobIdByAttachmentId.put(attachmentId, ONE_ATTACHMENT_EML_ATTACHMENT_BLOB_ID); + retrieveAndSaveAttachmentDetails(user, messageName, attachmentId, composedMessageId); } @When("^\"([^\"]*)\" checks for the availability of the attachment endpoint$") http://git-wip-us.apache.org/repos/asf/james-project/blob/bfa5e426/server/protocols/jmap-integration-testing/jmap-integration-testing-common/src/test/java/org/apache/james/jmap/methods/integration/cucumber/UploadStepdefs.java ---------------------------------------------------------------------- diff --git a/server/protocols/jmap-integration-testing/jmap-integration-testing-common/src/test/java/org/apache/james/jmap/methods/integration/cucumber/UploadStepdefs.java b/server/protocols/jmap-integration-testing/jmap-integration-testing-common/src/test/java/org/apache/james/jmap/methods/integration/cucumber/UploadStepdefs.java index d4362cc..4e3171c 100644 --- a/server/protocols/jmap-integration-testing/jmap-integration-testing-common/src/test/java/org/apache/james/jmap/methods/integration/cucumber/UploadStepdefs.java +++ b/server/protocols/jmap-integration-testing/jmap-integration-testing-common/src/test/java/org/apache/james/jmap/methods/integration/cucumber/UploadStepdefs.java @@ -54,7 +54,6 @@ import cucumber.runtime.java.guice.ScenarioScoped; @ScenarioScoped public class UploadStepdefs { - private static final String _1M_ZEROED_FILE_BLOB_ID = "35d87cc65060b896a0541713e7868f5cb5f8be3f563ccf82b72e61c2fee67404"; private static final int _1M = 1024 * 1024; private static final int _10M = 10 * _1M; @@ -193,7 +192,7 @@ public class UploadStepdefs { .containsExactly( normalizeContentType(org.apache.http.entity.ContentType.APPLICATION_JSON.toString())); DocumentContext jsonPath = JsonPath.parse(response.getEntity().getContent()); - assertThat(jsonPath.<String>read("blobId")).isEqualTo(_1M_ZEROED_FILE_BLOB_ID); + jsonPath.<String>read("blobId"); assertThat(jsonPath.<String>read("type")).isEqualTo("application/octet-stream"); assertThat(jsonPath.<Integer>read("size")).isEqualTo(_1M); } @@ -205,7 +204,8 @@ public class UploadStepdefs { @Then("^\"([^\"]*)\" should be able to retrieve the content$") public void contentShouldBeRetrievable(String username) throws Exception { AccessToken accessToken = userStepdefs.authenticate(username); - Request request = Request.Get(baseUri(mainStepdefs.jmapServer).setPath("/download/" + _1M_ZEROED_FILE_BLOB_ID).build()); + DocumentContext jsonPath = JsonPath.parse(response.getEntity().getContent()); + Request request = Request.Get(baseUri(mainStepdefs.jmapServer).setPath("/download/" + jsonPath.<String>read("blobId")).build()); if (accessToken != null) { request.addHeader("Authorization", accessToken.serialize()); } http://git-wip-us.apache.org/repos/asf/james-project/blob/bfa5e426/server/protocols/jmap-integration-testing/jmap-integration-testing-common/src/test/resources/cucumber/GetMessages.feature ---------------------------------------------------------------------- diff --git a/server/protocols/jmap-integration-testing/jmap-integration-testing-common/src/test/resources/cucumber/GetMessages.feature b/server/protocols/jmap-integration-testing/jmap-integration-testing-common/src/test/resources/cucumber/GetMessages.feature index cb4c951..7b5f854 100644 --- a/server/protocols/jmap-integration-testing/jmap-integration-testing-common/src/test/resources/cucumber/GetMessages.feature +++ b/server/protocols/jmap-integration-testing/jmap-integration-testing-common/src/test/resources/cucumber/GetMessages.feature @@ -164,14 +164,12 @@ Feature: GetMessages method And the list of attachments of the message contains 2 attachments And the first attachment is: |key | value | - |blobId |"81dad497ef270bd4537f5b43906aa88ad2e7168744c572be9a7414707727bf58" | |type |"image/jpeg" | |size |846 | |cid |null | |isInline |false | And the second attachment is: |key | value | - |blobId |"632b5341bbe044d26e0916b82a689282cc0891b806884b4d5a2339ea90b28e85" | |type |"image/jpeg" | |size |597 | |cid |"part1.37a15c92.a7c34...@linagora.com" | --------------------------------------------------------------------- To unsubscribe, e-mail: server-dev-unsubscr...@james.apache.org For additional commands, e-mail: server-dev-h...@james.apache.org
