JAMES-2555 Allowing per-user re-indexing
Project: http://git-wip-us.apache.org/repos/asf/james-project/repo Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/a8731817 Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/a8731817 Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/a8731817 Branch: refs/heads/master Commit: a87318174034b918115191dc90ddbe4a620952f3 Parents: 10a19a6 Author: Benoit Tellier <[email protected]> Authored: Fri Oct 12 17:41:28 2018 +0700 Committer: Benoit Tellier <[email protected]> Committed: Mon Oct 15 13:19:39 2018 +0700 ---------------------------------------------------------------------- .../apache/james/mailbox/indexer/ReIndexer.java | 3 + .../mailbox/tools/indexer/ReIndexerImpl.java | 6 ++ .../tools/indexer/ReIndexerPerformer.java | 46 ++++++++-- .../mailbox/tools/indexer/ThrowsReIndexer.java | 6 ++ .../tools/indexer/UserReindexingTask.java | 88 ++++++++++++++++++++ .../tools/indexer/ReIndexerImplTest.java | 25 ++++++ 6 files changed, 167 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/james-project/blob/a8731817/mailbox/api/src/main/java/org/apache/james/mailbox/indexer/ReIndexer.java ---------------------------------------------------------------------- diff --git a/mailbox/api/src/main/java/org/apache/james/mailbox/indexer/ReIndexer.java b/mailbox/api/src/main/java/org/apache/james/mailbox/indexer/ReIndexer.java index 94304a7..5ed156e 100644 --- a/mailbox/api/src/main/java/org/apache/james/mailbox/indexer/ReIndexer.java +++ b/mailbox/api/src/main/java/org/apache/james/mailbox/indexer/ReIndexer.java @@ -19,12 +19,15 @@ package org.apache.james.mailbox.indexer; +import org.apache.james.core.User; import org.apache.james.mailbox.exception.MailboxException; import org.apache.james.mailbox.model.MailboxPath; import org.apache.james.task.Task; public interface ReIndexer { + Task reIndex(User user) throws MailboxException; + Task reIndex(MailboxPath path) throws MailboxException; Task reIndex() throws MailboxException; http://git-wip-us.apache.org/repos/asf/james-project/blob/a8731817/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/ReIndexerImpl.java ---------------------------------------------------------------------- diff --git a/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/ReIndexerImpl.java b/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/ReIndexerImpl.java index 0ef72c4..8fd3786 100644 --- a/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/ReIndexerImpl.java +++ b/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/ReIndexerImpl.java @@ -21,6 +21,7 @@ package org.apache.mailbox.tools.indexer; import javax.inject.Inject; +import org.apache.james.core.User; import org.apache.james.mailbox.indexer.ReIndexer; import org.apache.james.mailbox.model.MailboxPath; import org.apache.james.task.Task; @@ -53,4 +54,9 @@ public class ReIndexerImpl implements ReIndexer { return new FullReindexingTask(reIndexerPerformer); } + @Override + public Task reIndex(User user) { + return new UserReindexingTask(reIndexerPerformer, user); + } + } http://git-wip-us.apache.org/repos/asf/james-project/blob/a8731817/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/ReIndexerPerformer.java ---------------------------------------------------------------------- diff --git a/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/ReIndexerPerformer.java b/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/ReIndexerPerformer.java index ce88882..fb5bf89 100644 --- a/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/ReIndexerPerformer.java +++ b/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/ReIndexerPerformer.java @@ -21,15 +21,19 @@ package org.apache.mailbox.tools.indexer; import java.util.List; import java.util.Optional; +import java.util.function.Function; import javax.inject.Inject; +import org.apache.james.core.User; import org.apache.james.mailbox.MailboxManager; import org.apache.james.mailbox.MailboxSession; import org.apache.james.mailbox.MessageUid; import org.apache.james.mailbox.exception.MailboxException; +import org.apache.james.mailbox.model.MailboxMetaData; import org.apache.james.mailbox.model.MailboxPath; import org.apache.james.mailbox.model.MessageRange; +import org.apache.james.mailbox.model.search.MailboxQuery; import org.apache.james.mailbox.store.MailboxSessionMapperFactory; import org.apache.james.mailbox.store.mail.MessageMapper; import org.apache.james.mailbox.store.mail.model.Mailbox; @@ -43,9 +47,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.github.fge.lambdas.Throwing; +import com.google.common.collect.ImmutableList; class ReIndexerPerformer { - private static final Logger LOGGER = LoggerFactory.getLogger(ReIndexerPerformer.class); private static final int NO_LIMIT = 0; @@ -73,18 +77,37 @@ class ReIndexerPerformer { MailboxSession mailboxSession = mailboxManager.createSystemSession("re-indexing"); LOGGER.info("Starting a full reindex"); List<MailboxPath> mailboxPaths = mailboxManager.list(mailboxSession); - GlobalRegistration globalRegistration = new GlobalRegistration(); - mailboxManager.addGlobalListener(globalRegistration, mailboxSession); + try { - return handleFullReindexingIterations(mailboxPaths, globalRegistration, reprocessingContext); + return reIndex(mailboxPaths, mailboxSession, reprocessingContext); } finally { - mailboxManager.removeGlobalListener(globalRegistration, mailboxSession); LOGGER.info("Full reindex finished"); } } - private Task.Result handleFullReindexingIterations(List<MailboxPath> mailboxPaths, GlobalRegistration globalRegistration, - ReprocessingContext reprocessingContext) { + Task.Result reIndex(User user, ReprocessingContext reprocessingContext) throws MailboxException { + MailboxSession mailboxSession = mailboxManager.createSystemSession(user.asString()); + LOGGER.info("Starting a reindex for user {}", user.asString()); + List<MailboxPath> mailboxPaths = mailboxManager.search(MailboxQuery.privateMailboxesBuilder(mailboxSession) + .build(), mailboxSession) + .stream() + .map(MailboxMetaData::getPath) + .collect(ImmutableList.toImmutableList()); + + try { + return reIndex(mailboxPaths, mailboxSession, reprocessingContext); + } finally { + LOGGER.info("User {} reindex finished", user.asString()); + } + } + + private Task.Result reIndex(List<MailboxPath> mailboxPaths, MailboxSession mailboxSession, ReprocessingContext reprocessingContext) throws MailboxException { + return wrapInGlobalRegistration(mailboxSession, + globalRegistration -> handleMultiMailboxesReindexingIterations(mailboxPaths, globalRegistration, reprocessingContext)); + } + + private Task.Result handleMultiMailboxesReindexingIterations(List<MailboxPath> mailboxPaths, GlobalRegistration globalRegistration, + ReprocessingContext reprocessingContext) { return mailboxPaths.stream() .map(globalRegistration::getPathToIndex) .flatMap(OptionalUtils::toStream) @@ -136,4 +159,13 @@ class ReIndexerPerformer { .findFirst(); } + private <T> T wrapInGlobalRegistration(MailboxSession session, Function<GlobalRegistration, T> function) throws MailboxException { + GlobalRegistration globalRegistration = new GlobalRegistration(); + mailboxManager.addGlobalListener(globalRegistration, session); + try { + return function.apply(globalRegistration); + } finally { + mailboxManager.removeGlobalListener(globalRegistration, session); + } + } } http://git-wip-us.apache.org/repos/asf/james-project/blob/a8731817/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/ThrowsReIndexer.java ---------------------------------------------------------------------- diff --git a/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/ThrowsReIndexer.java b/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/ThrowsReIndexer.java index 6734795..f0d86ae 100644 --- a/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/ThrowsReIndexer.java +++ b/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/ThrowsReIndexer.java @@ -19,6 +19,7 @@ package org.apache.mailbox.tools.indexer; +import org.apache.james.core.User; import org.apache.james.mailbox.exception.MailboxException; import org.apache.james.mailbox.indexer.ReIndexer; import org.apache.james.mailbox.model.MailboxPath; @@ -35,4 +36,9 @@ public class ThrowsReIndexer implements ReIndexer { public Task reIndex() throws MailboxException { throw new MailboxException("Not implemented"); } + + @Override + public Task reIndex(User user) throws MailboxException { + throw new MailboxException("Not implemented"); + } } http://git-wip-us.apache.org/repos/asf/james-project/blob/a8731817/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/UserReindexingTask.java ---------------------------------------------------------------------- diff --git a/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/UserReindexingTask.java b/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/UserReindexingTask.java new file mode 100644 index 0000000..dab53e7 --- /dev/null +++ b/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/UserReindexingTask.java @@ -0,0 +1,88 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.mailbox.tools.indexer; + +import java.util.Optional; + +import javax.inject.Inject; + +import org.apache.james.core.User; +import org.apache.james.mailbox.exception.MailboxException; +import org.apache.james.task.Task; +import org.apache.james.task.TaskExecutionDetails; + +public class UserReindexingTask implements Task { + + public static final String USER_RE_INDEXING = "userReIndexing"; + + public static class AdditionalInformation implements TaskExecutionDetails.AdditionalInformation { + private final ReprocessingContext reprocessingContext; + private final User user; + + AdditionalInformation(ReprocessingContext reprocessingContext, User user) { + this.reprocessingContext = reprocessingContext; + this.user = user; + } + + public int getSuccessfullyReprocessMailCount() { + return reprocessingContext.successfullyReprocessedMailCount(); + } + + public int getFailedReprocessedMailCount() { + return reprocessingContext.failedReprocessingMailCount(); + } + + public String getUser() { + return user.asString(); + } + } + + private final ReIndexerPerformer reIndexerPerformer; + private final User user; + private final AdditionalInformation additionalInformation; + private final ReprocessingContext reprocessingContext; + + @Inject + public UserReindexingTask(ReIndexerPerformer reIndexerPerformer, User user) { + this.reIndexerPerformer = reIndexerPerformer; + this.user = user; + this.reprocessingContext = new ReprocessingContext(); + this.additionalInformation = new AdditionalInformation(reprocessingContext, user); + } + + @Override + public Result run() { + try { + return reIndexerPerformer.reIndex(user, reprocessingContext); + } catch (MailboxException e) { + return Result.PARTIAL; + } + } + + @Override + public String type() { + return USER_RE_INDEXING; + } + + @Override + public Optional<TaskExecutionDetails.AdditionalInformation> details() { + return Optional.of(additionalInformation); + } +} http://git-wip-us.apache.org/repos/asf/james-project/blob/a8731817/mailbox/tools/indexer/src/test/java/org/apache/mailbox/tools/indexer/ReIndexerImplTest.java ---------------------------------------------------------------------- diff --git a/mailbox/tools/indexer/src/test/java/org/apache/mailbox/tools/indexer/ReIndexerImplTest.java b/mailbox/tools/indexer/src/test/java/org/apache/mailbox/tools/indexer/ReIndexerImplTest.java index 003601e..d1fed69 100644 --- a/mailbox/tools/indexer/src/test/java/org/apache/mailbox/tools/indexer/ReIndexerImplTest.java +++ b/mailbox/tools/indexer/src/test/java/org/apache/mailbox/tools/indexer/ReIndexerImplTest.java @@ -25,6 +25,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; +import org.apache.james.core.User; import org.apache.james.mailbox.MailboxSession; import org.apache.james.mailbox.MessageManager; import org.apache.james.mailbox.acl.SimpleGroupMembershipResolver; @@ -108,4 +109,28 @@ public class ReIndexerImplTest { assertThat(messageCaptor.getValue()).matches(message -> message.getMailboxId().equals(mailboxId) && message.getUid().equals(createdMessage.getUid())); } + + @Test + void userReIndexShouldBeWellPerformed() throws Exception { + MailboxSession systemSession = mailboxManager.createSystemSession(USERNAME); + MailboxId mailboxId = mailboxManager.createMailbox(INBOX, systemSession).get(); + ComposedMessageId createdMessage = mailboxManager.getMailbox(INBOX, systemSession) + .appendMessage( + MessageManager.AppendCommand.builder().build("header: value\r\n\r\nbody"), + systemSession); + + reIndexer.reIndex(User.fromUsername(USERNAME)).run(); + ArgumentCaptor<MailboxMessage> messageCaptor = ArgumentCaptor.forClass(MailboxMessage.class); + ArgumentCaptor<Mailbox> mailboxCaptor1 = ArgumentCaptor.forClass(Mailbox.class); + ArgumentCaptor<Mailbox> mailboxCaptor2 = ArgumentCaptor.forClass(Mailbox.class); + + verify(messageSearchIndex).deleteAll(any(MailboxSession.class), mailboxCaptor1.capture()); + verify(messageSearchIndex).add(any(MailboxSession.class), mailboxCaptor2.capture(), messageCaptor.capture()); + verifyNoMoreInteractions(messageSearchIndex); + + assertThat(mailboxCaptor1.getValue()).matches(mailbox -> mailbox.getMailboxId().equals(mailboxId)); + assertThat(mailboxCaptor2.getValue()).matches(mailbox -> mailbox.getMailboxId().equals(mailboxId)); + assertThat(messageCaptor.getValue()).matches(message -> message.getMailboxId().equals(mailboxId) + && message.getUid().equals(createdMessage.getUid())); + } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
