JAMES-2555 Simplify reIndexing: no more needed message event tracking Because we reindex a list of UID for a given mailbox, if: - A new message arrive it will be handled by the normal indexing process - A concurrently deleted message will be filtered out as it can not be retrieved from storage - And we will read the latest applied flag value
Note that global event tracking is still needed for mailbox path renames. Project: http://git-wip-us.apache.org/repos/asf/james-project/repo Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/9ff8a260 Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/9ff8a260 Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/9ff8a260 Branch: refs/heads/master Commit: 9ff8a260ea27993c46eb660c64a8881d9cf9f4a8 Parents: f354e3c Author: Benoit Tellier <[email protected]> Authored: Fri Oct 12 16:59:03 2018 +0700 Committer: Benoit Tellier <[email protected]> Committed: Mon Oct 15 13:18:20 2018 +0700 ---------------------------------------------------------------------- .../tools/indexer/ReIndexerPerformer.java | 33 +---- .../indexer/events/ImpactingMessageEvent.java | 144 ------------------- .../registrations/MailboxRegistration.java | 70 --------- .../registrations/MailboxRegistrationTest.java | 107 -------------- 4 files changed, 2 insertions(+), 352 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/james-project/blob/9ff8a260/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/ReIndexerPerformer.java ---------------------------------------------------------------------- diff --git a/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/ReIndexerPerformer.java b/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/ReIndexerPerformer.java index c2d471d..ce88882 100644 --- a/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/ReIndexerPerformer.java +++ b/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/ReIndexerPerformer.java @@ -38,15 +38,11 @@ import org.apache.james.mailbox.store.search.ListeningMessageSearchIndex; import org.apache.james.task.Task; import org.apache.james.util.OptionalUtils; import org.apache.james.util.streams.Iterators; -import org.apache.mailbox.tools.indexer.events.ImpactingEventType; -import org.apache.mailbox.tools.indexer.events.ImpactingMessageEvent; import org.apache.mailbox.tools.indexer.registrations.GlobalRegistration; -import org.apache.mailbox.tools.indexer.registrations.MailboxRegistration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.github.fge.lambdas.Throwing; -import com.google.common.collect.Lists; class ReIndexerPerformer { @@ -105,34 +101,27 @@ class ReIndexerPerformer { } private Task.Result reIndex(MailboxPath path, MailboxSession mailboxSession, ReprocessingContext reprocessingContext) throws MailboxException { - MailboxRegistration mailboxRegistration = new MailboxRegistration(path); LOGGER.info("Intend to reindex {}", path.asString()); Mailbox mailbox = mailboxSessionMapperFactory.getMailboxMapper(mailboxSession).findMailboxByPath(path); messageSearchIndex.deleteAll(mailboxSession, mailbox); - mailboxManager.addListener(path, mailboxRegistration, mailboxSession); try { return Iterators.toStream( mailboxSessionMapperFactory.getMessageMapper(mailboxSession) .findInMailbox(mailbox, MessageRange.all(), MessageMapper.FetchType.Metadata, NO_LIMIT)) .map(MailboxMessage::getUid) - .map(uid -> handleMessageReIndexing(mailboxSession, mailboxRegistration, mailbox, uid)) + .map(uid -> handleMessageReIndexing(mailboxSession, mailbox, uid)) .peek(reprocessingContext::updateAccordingToReprocessingResult) .reduce(Task::combine) .orElse(Task.Result.COMPLETED); } finally { LOGGER.info("Finish to reindex {}", path.asString()); - mailboxManager.removeListener(path, mailboxRegistration, mailboxSession); } } - private Task.Result handleMessageReIndexing(MailboxSession mailboxSession, MailboxRegistration mailboxRegistration, Mailbox mailbox, MessageUid uid) { + private Task.Result handleMessageReIndexing(MailboxSession mailboxSession, Mailbox mailbox, MessageUid uid) { try { - Optional<ImpactingMessageEvent> impactingMessageEvent = findMostRelevant(mailboxRegistration.getImpactingEvents(uid)); - Optional.of(uid) - .filter(x -> !wasDeleted(impactingMessageEvent)) .flatMap(Throwing.function(mUid -> fullyReadMessage(mailboxSession, mailbox, mUid))) - .map(message -> messageUpdateRegardingEvents(message, impactingMessageEvent)) .ifPresent(Throwing.consumer(message -> messageSearchIndex.add(mailboxSession, mailbox, message))); return Task.Result.COMPLETED; } catch (Exception e) { @@ -141,24 +130,6 @@ class ReIndexerPerformer { } } - private Optional<ImpactingMessageEvent> findMostRelevant(List<ImpactingMessageEvent> messageEvents) { - for (ImpactingMessageEvent impactingMessageEvent : messageEvents) { - if (impactingMessageEvent.getType().equals(ImpactingEventType.Deletion)) { - return Optional.of(impactingMessageEvent); - } - } - return Lists.reverse(messageEvents).stream().findFirst(); - } - - private boolean wasDeleted(Optional<ImpactingMessageEvent> impactingMessageEvent) { - return impactingMessageEvent.map(ImpactingMessageEvent::wasDeleted).orElse(false); - } - - private MailboxMessage messageUpdateRegardingEvents(MailboxMessage message, Optional<ImpactingMessageEvent> impactingMessageEvent) { - impactingMessageEvent.flatMap(ImpactingMessageEvent::newFlags).ifPresent(message::setFlags); - return message; - } - private Optional<MailboxMessage> fullyReadMessage(MailboxSession mailboxSession, Mailbox mailbox, MessageUid mUid) throws MailboxException { return Iterators.toStream(mailboxSessionMapperFactory.getMessageMapper(mailboxSession) .findInMailbox(mailbox, MessageRange.one(mUid), MessageMapper.FetchType.Full, SINGLE_MESSAGE)) http://git-wip-us.apache.org/repos/asf/james-project/blob/9ff8a260/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/events/ImpactingMessageEvent.java ---------------------------------------------------------------------- diff --git a/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/events/ImpactingMessageEvent.java b/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/events/ImpactingMessageEvent.java deleted file mode 100644 index 527f463..0000000 --- a/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/events/ImpactingMessageEvent.java +++ /dev/null @@ -1,144 +0,0 @@ -/**************************************************************** - * Licensed to the Apache Software Foundation (ASF) under one * - * or more contributor license agreements. See the NOTICE file * - * distributed with this work for additional information * - * regarding copyright ownership. The ASF licenses this file * - * to you under the Apache License, Version 2.0 (the * - * "License"); you may not use this file except in compliance * - * with the License. You may obtain a copy of the License at * - * * - * http://www.apache.org/licenses/LICENSE-2.0 * - * * - * Unless required by applicable law or agreed to in writing, * - * software distributed under the License is distributed on an * - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * - * KIND, either express or implied. See the License for the * - * specific language governing permissions and limitations * - * under the License. * - ****************************************************************/ - -package org.apache.mailbox.tools.indexer.events; - -import java.util.Objects; -import java.util.Optional; - -import javax.mail.Flags; - -import org.apache.james.mailbox.MessageUid; -import org.apache.james.mailbox.model.MailboxPath; - -public interface ImpactingMessageEvent extends ImpactingEvent { - - MessageUid getUid(); - - Optional<Flags> newFlags(); - - boolean wasDeleted(); - - class FlagsMessageEvent implements ImpactingMessageEvent { - - private final MailboxPath mailboxPath; - private final MessageUid uid; - private final Flags flags; - - public FlagsMessageEvent(MailboxPath mailboxPath, MessageUid uid, Flags flags) { - this.mailboxPath = mailboxPath; - this.uid = uid; - this.flags = flags; - } - - @Override - public MessageUid getUid() { - return uid; - } - - @Override - public MailboxPath getMailboxPath() { - return mailboxPath; - } - - @Override - public ImpactingEventType getType() { - return ImpactingEventType.FlagsUpdate; - } - - @Override - public Optional<Flags> newFlags() { - return Optional.of(flags); - } - - @Override - public boolean wasDeleted() { - return false; - } - - @Override - public final boolean equals(Object o) { - if (o instanceof FlagsMessageEvent) { - FlagsMessageEvent that = (FlagsMessageEvent) o; - - return Objects.equals(this.mailboxPath, that.mailboxPath) - && Objects.equals(this.uid, that.uid) - && Objects.equals(this.flags, that.flags); - } - return false; - } - - @Override - public final int hashCode() { - return Objects.hash(mailboxPath, uid, flags); - } - } - - class MessageDeletedEvent implements ImpactingMessageEvent { - - private final MailboxPath mailboxPath; - private final MessageUid uid; - - public MessageDeletedEvent(MailboxPath mailboxPath, MessageUid uid) { - this.mailboxPath = mailboxPath; - this.uid = uid; - } - - @Override - public MessageUid getUid() { - return uid; - } - - @Override - public MailboxPath getMailboxPath() { - return mailboxPath; - } - - @Override - public ImpactingEventType getType() { - return ImpactingEventType.Deletion; - } - - @Override - public Optional<Flags> newFlags() { - return Optional.empty(); - } - - @Override - public boolean wasDeleted() { - return true; - } - - @Override - public final boolean equals(Object o) { - if (o instanceof MessageDeletedEvent) { - MessageDeletedEvent that = (MessageDeletedEvent) o; - - return Objects.equals(this.mailboxPath, that.mailboxPath) - && Objects.equals(this.uid, that.uid); - } - return false; - } - - @Override - public final int hashCode() { - return Objects.hash(mailboxPath, uid); - } - } -} http://git-wip-us.apache.org/repos/asf/james-project/blob/9ff8a260/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/registrations/MailboxRegistration.java ---------------------------------------------------------------------- diff --git a/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/registrations/MailboxRegistration.java b/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/registrations/MailboxRegistration.java deleted file mode 100644 index 251fde8..0000000 --- a/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/registrations/MailboxRegistration.java +++ /dev/null @@ -1,70 +0,0 @@ -/**************************************************************** - * Licensed to the Apache Software Foundation (ASF) under one * - * or more contributor license agreements. See the NOTICE file * - * distributed with this work for additional information * - * regarding copyright ownership. The ASF licenses this file * - * to you under the Apache License, Version 2.0 (the * - * "License"); you may not use this file except in compliance * - * with the License. You may obtain a copy of the License at * - * * - * http://www.apache.org/licenses/LICENSE-2.0 * - * * - * Unless required by applicable law or agreed to in writing, * - * software distributed under the License is distributed on an * - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * - * KIND, either express or implied. See the License for the * - * specific language governing permissions and limitations * - * under the License. * - ****************************************************************/ - -package org.apache.mailbox.tools.indexer.registrations; - -import java.util.List; - -import org.apache.james.mailbox.Event; -import org.apache.james.mailbox.MailboxListener; -import org.apache.james.mailbox.MessageUid; -import org.apache.james.mailbox.model.MailboxPath; -import org.apache.james.mailbox.model.UpdatedFlags; -import org.apache.mailbox.tools.indexer.events.ImpactingMessageEvent; -import org.apache.mailbox.tools.indexer.events.ImpactingMessageEvent.FlagsMessageEvent; -import org.apache.mailbox.tools.indexer.events.ImpactingMessageEvent.MessageDeletedEvent; - -import com.google.common.collect.ArrayListMultimap; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Multimap; -import com.google.common.collect.Multimaps; - -public class MailboxRegistration implements MailboxListener { - - private final Multimap<MessageUid, ImpactingMessageEvent> impactingMessageEvents; - private final MailboxPath mailboxPath; - - public MailboxRegistration(MailboxPath mailboxPath) { - this.impactingMessageEvents = Multimaps.synchronizedMultimap(ArrayListMultimap.<MessageUid, ImpactingMessageEvent>create()); - this.mailboxPath = mailboxPath; - } - - @Override - public ListenerType getType() { - return ListenerType.MAILBOX; - } - - public List<ImpactingMessageEvent> getImpactingEvents(MessageUid uid) { - return ImmutableList.copyOf(impactingMessageEvents.get(uid)); - } - - @Override - public void event(Event event) { - if (event instanceof FlagsUpdated) { - for (UpdatedFlags updatedFlags : ((FlagsUpdated) event).getUpdatedFlags()) { - impactingMessageEvents.put(updatedFlags.getUid(), new FlagsMessageEvent(mailboxPath, updatedFlags.getUid(), updatedFlags.getNewFlags())); - } - } else if (event instanceof Expunged) { - for (MessageUid uid: ((Expunged) event).getUids()) { - impactingMessageEvents.put(uid, new MessageDeletedEvent(mailboxPath, uid)); - } - } - } - -} http://git-wip-us.apache.org/repos/asf/james-project/blob/9ff8a260/mailbox/tools/indexer/src/test/java/org/apache/mailbox/tools/indexer/registrations/MailboxRegistrationTest.java ---------------------------------------------------------------------- diff --git a/mailbox/tools/indexer/src/test/java/org/apache/mailbox/tools/indexer/registrations/MailboxRegistrationTest.java b/mailbox/tools/indexer/src/test/java/org/apache/mailbox/tools/indexer/registrations/MailboxRegistrationTest.java deleted file mode 100644 index 3b9bad2..0000000 --- a/mailbox/tools/indexer/src/test/java/org/apache/mailbox/tools/indexer/registrations/MailboxRegistrationTest.java +++ /dev/null @@ -1,107 +0,0 @@ -/**************************************************************** - * Licensed to the Apache Software Foundation (ASF) under one * - * or more contributor license agreements. See the NOTICE file * - * distributed with this work for additional information * - * regarding copyright ownership. The ASF licenses this file * - * to you under the Apache License, Version 2.0 (the * - * "License"); you may not use this file except in compliance * - * with the License. You may obtain a copy of the License at * - * * - * http://www.apache.org/licenses/LICENSE-2.0 * - * * - * Unless required by applicable law or agreed to in writing, * - * software distributed under the License is distributed on an * - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * - * KIND, either express or implied. See the License for the * - * specific language governing permissions and limitations * - * under the License. * - ****************************************************************/ - -package org.apache.mailbox.tools.indexer.registrations; - -import static org.assertj.core.api.Assertions.assertThat; - -import java.util.Date; -import java.util.TreeMap; - -import javax.mail.Flags; - -import org.apache.james.mailbox.MailboxListener; -import org.apache.james.mailbox.MessageUid; -import org.apache.james.mailbox.mock.MockMailboxSession; -import org.apache.james.mailbox.model.MailboxPath; -import org.apache.james.mailbox.model.MessageMetaData; -import org.apache.james.mailbox.model.UpdatedFlags; -import org.apache.james.mailbox.store.SimpleMessageMetaData; -import org.apache.james.mailbox.store.event.EventFactory; -import org.apache.james.mailbox.store.mail.model.DefaultMessageId; -import org.apache.james.mailbox.store.mail.model.MailboxMessage; -import org.apache.james.mailbox.store.mail.model.impl.SimpleMailbox; -import org.apache.mailbox.tools.indexer.events.ImpactingMessageEvent.FlagsMessageEvent; -import org.apache.mailbox.tools.indexer.events.ImpactingMessageEvent.MessageDeletedEvent; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; - -public class MailboxRegistrationTest { - - public static final MailboxPath INBOX = MailboxPath.forUser("[email protected]", "INBOX"); - private static final MessageUid UID = MessageUid.of(18); - private static final int UID_VALIDITY = 45; - private static final SimpleMailbox MAILBOX = new SimpleMailbox(INBOX, UID_VALIDITY); - private static final int MOD_SEQ = 21; - private static final int SIZE = 41; - private static final Flags NEW_FLAGS = new Flags(Flags.Flag.ANSWERED); - private MailboxRegistration mailboxRegistration; - private EventFactory eventFactory; - private MockMailboxSession session; - - @BeforeEach - void setUp() { - session = new MockMailboxSession("test"); - eventFactory = new EventFactory(); - mailboxRegistration = new MailboxRegistration(INBOX); - } - - @Test - void reportedEventsShouldBeInitiallyEmpty() { - assertThat(mailboxRegistration.getImpactingEvents(UID)).isEmpty(); - } - - - @Test - void addedEventsShouldNotBeReported() { - TreeMap<MessageUid, MessageMetaData> treeMap = new TreeMap<>(); - treeMap.put(UID, new SimpleMessageMetaData(UID, MOD_SEQ, new Flags(), SIZE, new Date(), new DefaultMessageId())); - MailboxListener.MailboxEvent event = eventFactory.added(session, treeMap, MAILBOX, ImmutableMap.<MessageUid, MailboxMessage>of()); - mailboxRegistration.event(event); - assertThat(mailboxRegistration.getImpactingEvents(UID)).isEmpty(); - } - - @Test - void expungedEventsShouldBeReported() { - TreeMap<MessageUid, MessageMetaData> treeMap = new TreeMap<>(); - treeMap.put(UID, new SimpleMessageMetaData(UID, MOD_SEQ, new Flags(), SIZE, new Date(), new DefaultMessageId())); - MailboxListener.MailboxEvent event = eventFactory.expunged(session, treeMap, MAILBOX); - mailboxRegistration.event(event); - assertThat(mailboxRegistration.getImpactingEvents(UID)).containsExactly(new MessageDeletedEvent(INBOX, UID)); - } - - @Test - void flagsEventsShouldBeReported() { - MailboxListener.MailboxEvent event = eventFactory.flagsUpdated(session, - Lists.newArrayList(UID), - MAILBOX, - Lists.newArrayList(UpdatedFlags.builder() - .uid(UID) - .modSeq(MOD_SEQ) - .oldFlags(new Flags()) - .newFlags(NEW_FLAGS) - .build())); - mailboxRegistration.event(event); - assertThat(mailboxRegistration.getImpactingEvents(UID)).containsExactly(new FlagsMessageEvent(INBOX, UID, NEW_FLAGS)); - } - -} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
