MAILBOX-306 Read timeout setting when retrieving all attachment ids from message
Project: http://git-wip-us.apache.org/repos/asf/james-project/repo Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/f111e623 Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/f111e623 Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/f111e623 Branch: refs/heads/master Commit: f111e6236ec18c92233a7b356d6f9951cf39dc76 Parents: 1e808fa Author: quynhn <qngu...@linagora.com> Authored: Wed Sep 13 14:14:41 2017 +0700 Committer: benwa <btell...@linagora.com> Committed: Tue Sep 19 08:59:35 2017 +0700 ---------------------------------------------------------------------- .../cassandra/init/CassandraConfiguration.java | 30 +++++++++++++++++--- .../init/CassandraConfigurationTest.java | 19 +++++++++++++ .../cassandra/mail/CassandraMessageDAO.java | 10 ++++--- .../modules/mailbox/CassandraSessionModule.java | 3 ++ .../mailbox/CassandraSessionModuleTest.java | 1 + .../modules/mailbox/cassandra.properties | 1 + src/site/xdoc/server/config-cassandra.xml | 15 ++++++++++ 7 files changed, 71 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/james-project/blob/f111e623/backends-common/cassandra/src/main/java/org/apache/james/backends/cassandra/init/CassandraConfiguration.java ---------------------------------------------------------------------- diff --git a/backends-common/cassandra/src/main/java/org/apache/james/backends/cassandra/init/CassandraConfiguration.java b/backends-common/cassandra/src/main/java/org/apache/james/backends/cassandra/init/CassandraConfiguration.java index 2121cff..f35d3b8 100644 --- a/backends-common/cassandra/src/main/java/org/apache/james/backends/cassandra/init/CassandraConfiguration.java +++ b/backends-common/cassandra/src/main/java/org/apache/james/backends/cassandra/init/CassandraConfiguration.java @@ -41,6 +41,7 @@ public class CassandraConfiguration { public static final int DEFAULT_FETCH_NEXT_PAGE_ADVANCE_IN_ROW = 100; public static final int DEFAULT_BLOB_PART_SIZE = 100 * 1024; public static final int DEFAULT_ATTACHMENT_V2_MIGRATION_READ_TIMEOUT = toIntExact(TimeUnit.DAYS.toMillis(1)); + public static final int DEFAULT_MESSAGE_ATTACHMENT_ID_MIGRATION_READ_TIMEOUT = toIntExact(TimeUnit.DAYS.toMillis(1)); public static final CassandraConfiguration DEFAULT_CONFIGURATION = builder().build(); public static class Builder { @@ -55,6 +56,7 @@ public class CassandraConfiguration { private Optional<Integer> fetchNextPageInAdvanceRow = Optional.empty(); private Optional<Integer> blobPartSize = Optional.empty(); private Optional<Integer> attachmentV2MigrationReadTimeout = Optional.empty(); + private Optional<Integer> messageAttachmentIdsReadTimeout = Optional.empty(); public Builder messageReadChunkSize(int value) { Preconditions.checkArgument(value > 0, "messageReadChunkSize needs to be strictly positive"); @@ -122,6 +124,12 @@ public class CassandraConfiguration { return this; } + public Builder messageAttachmentIdsReadTimeout(int value) { + Preconditions.checkArgument(value > 0, "messageAttachmentIdsReadTimeout needs to be strictly positive"); + this.messageAttachmentIdsReadTimeout = Optional.of(value); + return this; + } + public Builder messageReadChunkSize(Optional<Integer> value) { value.ifPresent(this::messageReadChunkSize); return this; @@ -177,6 +185,11 @@ public class CassandraConfiguration { return this; } + public Builder messageAttachmentIdsReadTimeout(Optional<Integer> value) { + value.ifPresent(this::messageAttachmentIdsReadTimeout); + return this; + } + public CassandraConfiguration build() { return new CassandraConfiguration(aclMaxRetry.orElse(DEFAULT_ACL_MAX_RETRY), messageReadChunkSize.orElse(DEFAULT_MESSAGE_CHUNK_SIZE_ON_READ), @@ -188,7 +201,8 @@ public class CassandraConfiguration { uidMaxRetry.orElse(DEFAULT_UID_MAX_RETRY), fetchNextPageInAdvanceRow.orElse(DEFAULT_FETCH_NEXT_PAGE_ADVANCE_IN_ROW), blobPartSize.orElse(DEFAULT_BLOB_PART_SIZE), - attachmentV2MigrationReadTimeout.orElse(DEFAULT_ATTACHMENT_V2_MIGRATION_READ_TIMEOUT)); + attachmentV2MigrationReadTimeout.orElse(DEFAULT_ATTACHMENT_V2_MIGRATION_READ_TIMEOUT), + messageAttachmentIdsReadTimeout.orElse(DEFAULT_MESSAGE_ATTACHMENT_ID_MIGRATION_READ_TIMEOUT)); } } @@ -207,12 +221,13 @@ public class CassandraConfiguration { private final int fetchNextPageInAdvanceRow; private final int blobPartSize; private final int attachmentV2MigrationReadTimeout; + private final int messageAttachmentIdsReadTimeout; @VisibleForTesting CassandraConfiguration(int aclMaxRetry, int messageReadChunkSize, int expungeChunkSize, int flagsUpdateChunkSize, int flagsUpdateMessageIdMaxRetry, int flagsUpdateMessageMaxRetry, int modSeqMaxRetry, int uidMaxRetry, int fetchNextPageInAdvanceRow, - int blobPartSize, final int attachmentV2MigrationReadTimeout) { + int blobPartSize, final int attachmentV2MigrationReadTimeout, int messageAttachmentIdsReadTimeout) { this.aclMaxRetry = aclMaxRetry; this.messageReadChunkSize = messageReadChunkSize; this.expungeChunkSize = expungeChunkSize; @@ -224,6 +239,7 @@ public class CassandraConfiguration { this.flagsUpdateChunkSize = flagsUpdateChunkSize; this.blobPartSize = blobPartSize; this.attachmentV2MigrationReadTimeout = attachmentV2MigrationReadTimeout; + this.messageAttachmentIdsReadTimeout = messageAttachmentIdsReadTimeout; } public int getBlobPartSize() { @@ -270,6 +286,10 @@ public class CassandraConfiguration { return attachmentV2MigrationReadTimeout; } + public int getMessageAttachmentIdsReadTimeout() { + return messageAttachmentIdsReadTimeout; + } + @Override public final boolean equals(Object o) { if (o instanceof CassandraConfiguration) { @@ -285,7 +305,8 @@ public class CassandraConfiguration { && Objects.equals(this.flagsUpdateChunkSize, that.flagsUpdateChunkSize) && Objects.equals(this.fetchNextPageInAdvanceRow, that.fetchNextPageInAdvanceRow) && Objects.equals(this.blobPartSize, that.blobPartSize) - && Objects.equals(this.attachmentV2MigrationReadTimeout, that.attachmentV2MigrationReadTimeout); + && Objects.equals(this.attachmentV2MigrationReadTimeout, that.attachmentV2MigrationReadTimeout) + && Objects.equals(this.messageAttachmentIdsReadTimeout, that.messageAttachmentIdsReadTimeout); } return false; } @@ -294,7 +315,7 @@ public class CassandraConfiguration { public final int hashCode() { return Objects.hash(aclMaxRetry, messageReadChunkSize, expungeChunkSize, flagsUpdateMessageIdMaxRetry, flagsUpdateMessageMaxRetry, modSeqMaxRetry, uidMaxRetry, fetchNextPageInAdvanceRow, flagsUpdateChunkSize, - blobPartSize, attachmentV2MigrationReadTimeout); + blobPartSize, attachmentV2MigrationReadTimeout, messageAttachmentIdsReadTimeout); } @Override @@ -311,6 +332,7 @@ public class CassandraConfiguration { .add("uidMaxRetry", uidMaxRetry) .add("blobPartSize", blobPartSize) .add("attachmentV2MigrationReadTimeout", attachmentV2MigrationReadTimeout) + .add("messageAttachmentIdsReadTimeout", messageAttachmentIdsReadTimeout) .toString(); } } http://git-wip-us.apache.org/repos/asf/james-project/blob/f111e623/backends-common/cassandra/src/test/java/org/apache/james/backends/cassandra/init/CassandraConfigurationTest.java ---------------------------------------------------------------------- diff --git a/backends-common/cassandra/src/test/java/org/apache/james/backends/cassandra/init/CassandraConfigurationTest.java b/backends-common/cassandra/src/test/java/org/apache/james/backends/cassandra/init/CassandraConfigurationTest.java index efde00e..34b6a01 100644 --- a/backends-common/cassandra/src/test/java/org/apache/james/backends/cassandra/init/CassandraConfigurationTest.java +++ b/backends-common/cassandra/src/test/java/org/apache/james/backends/cassandra/init/CassandraConfigurationTest.java @@ -210,6 +210,22 @@ public class CassandraConfigurationTest { } @Test + public void messageAttachmentIdsReadTimeoutShouldThrowOnZero() { + expectedException.expect(IllegalArgumentException.class); + + CassandraConfiguration.builder() + .messageAttachmentIdsReadTimeout(0); + } + + @Test + public void messageAttachmentIdsReadTimeoutShouldThrowOnNegativeValue() { + expectedException.expect(IllegalArgumentException.class); + + CassandraConfiguration.builder() + .messageAttachmentIdsReadTimeout(-1); + } + + @Test public void builderShouldCreateTheRightObject() { int aclMaxRetry = 1; int modSeqMaxRetry = 2; @@ -222,6 +238,7 @@ public class CassandraConfigurationTest { int expungeChunkSize = 9; int blobPartSize = 10; int attachmentV2MigrationReadTimeout = 11; + int messageAttachmentIdReadTimeout = 12; CassandraConfiguration configuration = CassandraConfiguration.builder() .aclMaxRetry(aclMaxRetry) @@ -235,6 +252,7 @@ public class CassandraConfigurationTest { .expungeChunkSize(expungeChunkSize) .blobPartSize(blobPartSize) .attachmentV2MigrationReadTimeout(attachmentV2MigrationReadTimeout) + .messageAttachmentIdsReadTimeout(messageAttachmentIdReadTimeout) .build(); softly.assertThat(configuration.getAclMaxRetry()).isEqualTo(aclMaxRetry); @@ -248,6 +266,7 @@ public class CassandraConfigurationTest { softly.assertThat(configuration.getExpungeChunkSize()).isEqualTo(expungeChunkSize); softly.assertThat(configuration.getBlobPartSize()).isEqualTo(blobPartSize); softly.assertThat(configuration.getAttachmentV2MigrationReadTimeout()).isEqualTo(attachmentV2MigrationReadTimeout); + softly.assertThat(configuration.getMessageAttachmentIdsReadTimeout()).isEqualTo(messageAttachmentIdReadTimeout); } } http://git-wip-us.apache.org/repos/asf/james-project/blob/f111e623/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/mail/CassandraMessageDAO.java ---------------------------------------------------------------------- diff --git a/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/mail/CassandraMessageDAO.java b/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/mail/CassandraMessageDAO.java index b8a3d84..9c18e73 100644 --- a/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/mail/CassandraMessageDAO.java +++ b/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/mail/CassandraMessageDAO.java @@ -404,10 +404,12 @@ public class CassandraMessageDAO { } public CompletableFuture<Stream<MessageIdAttachmentIds>> retrieveAllMessageIdAttachmentIds() { - return cassandraAsyncExecutor.execute(selectAllMessagesWithAttachment.bind()) - .thenApply(resultSet -> cassandraUtils.convertToStream(resultSet) - .map(this::fromRow) - .filter(MessageIdAttachmentIds::hasAttachment)); + return cassandraAsyncExecutor.execute( + selectAllMessagesWithAttachment.bind() + .setReadTimeoutMillis(configuration.getMessageAttachmentIdsReadTimeout())) + .thenApply(resultSet -> cassandraUtils.convertToStream(resultSet) + .map(this::fromRow) + .filter(MessageIdAttachmentIds::hasAttachment)); } private MessageIdAttachmentIds fromRow(Row row) { http://git-wip-us.apache.org/repos/asf/james-project/blob/f111e623/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/CassandraSessionModule.java ---------------------------------------------------------------------- diff --git a/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/CassandraSessionModule.java b/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/CassandraSessionModule.java index fb08814..5453df1 100644 --- a/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/CassandraSessionModule.java +++ b/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/CassandraSessionModule.java @@ -73,6 +73,7 @@ public class CassandraSessionModule extends AbstractModule { private static final String CHUNK_SIZE_EXPUNGE = "chunk.size.expunge"; private static final String BLOB_PART_SIZE = "mailbox.blob.part.size"; private static final String ATTACHMENT_V2_MIGRATION_READ_TIMEOUT = "attachment.v2.migration.read.timeout"; + private static final String MESSAGE_ATTACHMENTID_READ_TIMEOUT = "message.attachmentids.read.timeout"; private static final String CASSANDRA_NODES = "cassandra.nodes"; @Override @@ -160,6 +161,8 @@ public class CassandraSessionModule extends AbstractModule { propertiesConfiguration.getInteger(BLOB_PART_SIZE, null))) .attachmentV2MigrationReadTimeout(Optional.ofNullable( propertiesConfiguration.getInteger(ATTACHMENT_V2_MIGRATION_READ_TIMEOUT, null))) + .messageAttachmentIdsReadTimeout(Optional.ofNullable( + propertiesConfiguration.getInteger(MESSAGE_ATTACHMENTID_READ_TIMEOUT, null))) .build(); } http://git-wip-us.apache.org/repos/asf/james-project/blob/f111e623/server/container/guice/cassandra-guice/src/test/java/org/apache/james/modules/mailbox/CassandraSessionModuleTest.java ---------------------------------------------------------------------- diff --git a/server/container/guice/cassandra-guice/src/test/java/org/apache/james/modules/mailbox/CassandraSessionModuleTest.java b/server/container/guice/cassandra-guice/src/test/java/org/apache/james/modules/mailbox/CassandraSessionModuleTest.java index da0358e..39bc3ad 100644 --- a/server/container/guice/cassandra-guice/src/test/java/org/apache/james/modules/mailbox/CassandraSessionModuleTest.java +++ b/server/container/guice/cassandra-guice/src/test/java/org/apache/james/modules/mailbox/CassandraSessionModuleTest.java @@ -61,6 +61,7 @@ public class CassandraSessionModuleTest { .expungeChunkSize(9) .blobPartSize(10) .attachmentV2MigrationReadTimeout(11) + .messageAttachmentIdsReadTimeout(12) .build()); } http://git-wip-us.apache.org/repos/asf/james-project/blob/f111e623/server/container/guice/cassandra-guice/src/test/resources/modules/mailbox/cassandra.properties ---------------------------------------------------------------------- diff --git a/server/container/guice/cassandra-guice/src/test/resources/modules/mailbox/cassandra.properties b/server/container/guice/cassandra-guice/src/test/resources/modules/mailbox/cassandra.properties index d41cb75..5a37b4e 100644 --- a/server/container/guice/cassandra-guice/src/test/resources/modules/mailbox/cassandra.properties +++ b/server/container/guice/cassandra-guice/src/test/resources/modules/mailbox/cassandra.properties @@ -9,3 +9,4 @@ chunk.size.message.read=8 chunk.size.expunge=9 mailbox.blob.part.size=10 attachment.v2.migration.read.timeout=11 +message.attachmentids.read.timeout=12 http://git-wip-us.apache.org/repos/asf/james-project/blob/f111e623/src/site/xdoc/server/config-cassandra.xml ---------------------------------------------------------------------- diff --git a/src/site/xdoc/server/config-cassandra.xml b/src/site/xdoc/server/config-cassandra.xml index c2980c9..3c74beb 100644 --- a/src/site/xdoc/server/config-cassandra.xml +++ b/src/site/xdoc/server/config-cassandra.xml @@ -186,6 +186,21 @@ </subsection> + <subsection name="From V4 to V5"> + + <p>Migration tag on git repository: <a href="https://github.com/apache/james-project/releases/tag/cassandra_migration_v4_to_v5";>cassandra_migration_v4_to_v5</a></p> + + <p>Goal is to store attachment ids in the separated AttachmentMessageId table.</p> + + <p>Summary of available options for this migration:</p> + + <dl> + <dt><strong>message.attachmentids.read.timeout</strong></dt> + <dd>Optional. Defaults to one day.<br/> Controls how many milliseconds before the read attachment ids on message time out.</dd> + </dl> + + </subsection> + </section> </body> --------------------------------------------------------------------- To unsubscribe, e-mail: server-dev-unsubscr...@james.apache.org For additional commands, e-mail: server-dev-h...@james.apache.org
