This is an automated email from the ASF dual-hosted git repository.
kao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
The following commit(s) were added to refs/heads/master by this push:
new 7243b69b14 JAMES-3799 Optimize memory requirements of
SimpleMessageSearchIndex
7243b69b14 is described below
commit 7243b69b147c23544792b8e8e55c5fc5dbcb10d5
Author: Karsten Otto <[email protected]>
AuthorDate: Mon Aug 8 15:23:38 2022 +0200
JAMES-3799 Optimize memory requirements of SimpleMessageSearchIndex
---
.../store/search/SimpleMessageSearchIndexTest.java | 27 +++++++++
.../store/search/SimpleMessageSearchIndex.java | 68 +++++++++++++++++++++-
2 files changed, 92 insertions(+), 3 deletions(-)
diff --git
a/mailbox/scanning-search/src/test/java/org/apache/james/mailbox/store/search/SimpleMessageSearchIndexTest.java
b/mailbox/scanning-search/src/test/java/org/apache/james/mailbox/store/search/SimpleMessageSearchIndexTest.java
index 9af14443f8..17827d17f2 100644
---
a/mailbox/scanning-search/src/test/java/org/apache/james/mailbox/store/search/SimpleMessageSearchIndexTest.java
+++
b/mailbox/scanning-search/src/test/java/org/apache/james/mailbox/store/search/SimpleMessageSearchIndexTest.java
@@ -26,7 +26,11 @@ import
org.apache.james.mailbox.inmemory.manager.InMemoryIntegrationResources;
import org.apache.james.mailbox.model.MailboxId;
import org.apache.james.mailbox.model.MessageId;
import org.apache.james.mailbox.model.SearchQuery;
+import org.apache.james.mailbox.store.mail.MessageMapper.FetchType;
import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
class SimpleMessageSearchIndexTest extends AbstractMessageSearchIndexTest {
@@ -222,4 +226,27 @@ class SimpleMessageSearchIndexTest extends
AbstractMessageSearchIndexTest {
@Override
public void headerWithDotsShouldBeIndexed() {
}
+
+ @Test
+ public void canCompareFetchTypes() {
+ assertThat(FetchType.values()).containsExactly(FetchType.METADATA,
FetchType.HEADERS, FetchType.BODY, FetchType.FULL);
+
+ assertThat(SimpleMessageSearchIndex.maxFetchType(FetchType.METADATA,
FetchType.METADATA)).isEqualTo(FetchType.METADATA);
+ assertThat(SimpleMessageSearchIndex.maxFetchType(FetchType.METADATA,
FetchType.HEADERS)).isEqualTo(FetchType.HEADERS);
+ assertThat(SimpleMessageSearchIndex.maxFetchType(FetchType.METADATA,
FetchType.BODY)).isEqualTo(FetchType.BODY);
+ assertThat(SimpleMessageSearchIndex.maxFetchType(FetchType.METADATA,
FetchType.FULL)).isEqualTo(FetchType.FULL);
+ assertThat(SimpleMessageSearchIndex.maxFetchType(FetchType.HEADERS,
FetchType.HEADERS)).isEqualTo(FetchType.HEADERS);
+ assertThat(SimpleMessageSearchIndex.maxFetchType(FetchType.HEADERS,
FetchType.BODY)).isEqualTo(FetchType.BODY);
+ assertThat(SimpleMessageSearchIndex.maxFetchType(FetchType.HEADERS,
FetchType.FULL)).isEqualTo(FetchType.FULL);
+ assertThat(SimpleMessageSearchIndex.maxFetchType(FetchType.BODY,
FetchType.BODY)).isEqualTo(FetchType.BODY);
+ assertThat(SimpleMessageSearchIndex.maxFetchType(FetchType.BODY,
FetchType.FULL)).isEqualTo(FetchType.FULL);
+ assertThat(SimpleMessageSearchIndex.maxFetchType(FetchType.FULL,
FetchType.FULL)).isEqualTo(FetchType.FULL);
+
+ assertThat(SimpleMessageSearchIndex.maxFetchType(FetchType.HEADERS,
FetchType.METADATA)).isEqualTo(FetchType.HEADERS);
+ assertThat(SimpleMessageSearchIndex.maxFetchType(FetchType.BODY,
FetchType.METADATA)).isEqualTo(FetchType.BODY);
+ assertThat(SimpleMessageSearchIndex.maxFetchType(FetchType.FULL,
FetchType.METADATA)).isEqualTo(FetchType.FULL);
+ assertThat(SimpleMessageSearchIndex.maxFetchType(FetchType.BODY,
FetchType.HEADERS)).isEqualTo(FetchType.BODY);
+ assertThat(SimpleMessageSearchIndex.maxFetchType(FetchType.FULL,
FetchType.HEADERS)).isEqualTo(FetchType.FULL);
+ assertThat(SimpleMessageSearchIndex.maxFetchType(FetchType.FULL,
FetchType.BODY)).isEqualTo(FetchType.FULL);
+ }
}
diff --git
a/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/SimpleMessageSearchIndex.java
b/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/SimpleMessageSearchIndex.java
index e1cbb0bbe0..1858d16b78 100644
---
a/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/SimpleMessageSearchIndex.java
+++
b/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/SimpleMessageSearchIndex.java
@@ -57,6 +57,7 @@ import
org.apache.james.mailbox.store.search.comparator.CombinedComparator;
import org.apache.james.util.ReactorUtils;
import org.apache.james.util.streams.Iterators;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import reactor.core.publisher.Flux;
@@ -113,6 +114,65 @@ public class SimpleMessageSearchIndex implements
MessageSearchIndex {
return null;
}
+ /**
+ * Walks down the query tree's conjunctions to find the highest necessary
mail fetch type.
+ * @param crits - list of Criterion to search from
+ * @return required fetch type - metadata, headers, or full
+ */
+ private static FetchType getFetchTypeForCriteria(List<Criterion> crits) {
+ return crits.stream()
+ .map(SimpleMessageSearchIndex::getFetchTypeForCriterion)
+ .reduce(SimpleMessageSearchIndex::maxFetchType)
+ .orElse(FetchType.METADATA);
+ }
+
+ private static FetchType getFetchTypeForCriterion(Criterion crit) {
+ if (crit instanceof ConjunctionCriterion) {
+ return getFetchTypeForCriteria(((ConjunctionCriterion)
crit).getCriteria());
+ }
+ if (crit instanceof SearchQuery.AllCriterion || crit instanceof
SearchQuery.TextCriterion) {
+ return FetchType.FULL;
+ }
+ if (crit instanceof SearchQuery.HeaderCriterion || crit instanceof
SearchQuery.MimeMessageIDCriterion) {
+ return FetchType.HEADERS;
+ }
+ return FetchType.METADATA;
+ }
+
+ /**
+ * Searches a list of query sort options for the highest necessary mail
fetch type.
+ * @param sorts - list of Sort to search
+ * @return required fetch type - metadata or headers
+ */
+ private static FetchType getFetchTypeForSorts(List<SearchQuery.Sort>
sorts) {
+ return sorts.stream()
+ .map(SimpleMessageSearchIndex::getFetchTypeForSort)
+ .reduce(FetchType.METADATA,
SimpleMessageSearchIndex::maxFetchType);
+ }
+
+ private static FetchType getFetchTypeForSort(SearchQuery.Sort sort) {
+ switch (sort.getSortClause()) {
+ case Arrival:
+ case Size:
+ case Uid:
+ case Id:
+ return FetchType.METADATA;
+ case MailboxCc:
+ case MailboxFrom:
+ case MailboxTo:
+ case BaseSubject:
+ case SentDate:
+ return FetchType.HEADERS;
+ default:
+ throw new IllegalArgumentException("cannot determine fetch
type for sort option " + sort.getSortClause());
+ }
+ }
+
+ @VisibleForTesting
+ static FetchType maxFetchType(FetchType a, FetchType b) {
+ return a.compareTo(b) >= 0 ? a : b;
+ }
+
@Override
public Flux<MessageUid> search(MailboxSession session, final Mailbox
mailbox, SearchQuery query) {
Preconditions.checkArgument(session != null, "'session' is mandatory");
@@ -130,16 +190,18 @@ public class SimpleMessageSearchIndex implements
MessageSearchIndex {
if (uidCrit != null) {
// if there is a conjugated uid range criterion in the query tree
we can optimize by
// only fetching this uid range
+ FetchType fetchType = maxFetchType(FetchType.METADATA,
getFetchTypeForSorts(query.getSorts()));
UidRange[] ranges = uidCrit.getOperator().getRange();
for (UidRange r : ranges) {
- Iterator<MailboxMessage> it = mapper.findInMailbox(mailbox,
MessageRange.range(r.getLowValue(), r.getHighValue()), FetchType.METADATA,
UNLIMITED);
+ Iterator<MailboxMessage> it = mapper.findInMailbox(mailbox,
MessageRange.range(r.getLowValue(), r.getHighValue()), fetchType, UNLIMITED);
while (it.hasNext()) {
hitSet.add(it.next());
}
}
} else {
- // we have to fetch all messages
- Iterator<MailboxMessage> messages = mapper.findInMailbox(mailbox,
MessageRange.all(), FetchType.FULL, UNLIMITED);
+ // we have to fetch all messages; try to limit their memory
requirements
+ FetchType fetchType =
maxFetchType(getFetchTypeForCriteria(query.getCriteria()),
getFetchTypeForSorts(query.getSorts()));
+ Iterator<MailboxMessage> messages = mapper.findInMailbox(mailbox,
MessageRange.all(), fetchType, UNLIMITED);
while (messages.hasNext()) {
MailboxMessage m = messages.next();
hitSet.add(m);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
