MAILBOX-292 Make Maildir Mailbox Id persistent
Project: http://git-wip-us.apache.org/repos/asf/james-project/repo Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/7e32da51 Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/7e32da51 Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/7e32da51 Branch: refs/heads/master Commit: 7e32da51a29bee1c732b2b13708bb4b986140119 Parents: d9bcebc Author: Benoit Tellier <btell...@linagora.com> Authored: Fri Nov 30 16:00:34 2018 +0700 Committer: Benoit Tellier <btell...@linagora.com> Committed: Tue Dec 4 15:47:46 2018 +0700 ---------------------------------------------------------------------- .../james/mailbox/maildir/MaildirFolder.java | 54 +++++++++++++++- .../apache/james/mailbox/maildir/MaildirId.java | 7 ++ .../james/mailbox/maildir/MaildirStore.java | 1 + .../mailbox/maildir/mail/MailboxCache.java | 68 -------------------- .../maildir/mail/MaildirMailboxMapper.java | 63 ++++++++++-------- .../DomainUserMaildirMailboxManagerTest.java | 9 --- .../FullUserMaildirMailboxManagerTest.java | 10 --- .../maildir/src/test/resources/logback-test.xml | 28 ++++++++ 8 files changed, 125 insertions(+), 115 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/james-project/blob/7e32da51/mailbox/maildir/src/main/java/org/apache/james/mailbox/maildir/MaildirFolder.java ---------------------------------------------------------------------- diff --git a/mailbox/maildir/src/main/java/org/apache/james/mailbox/maildir/MaildirFolder.java b/mailbox/maildir/src/main/java/org/apache/james/mailbox/maildir/MaildirFolder.java index 4982726..b31b522 100644 --- a/mailbox/maildir/src/main/java/org/apache/james/mailbox/maildir/MaildirFolder.java +++ b/mailbox/maildir/src/main/java/org/apache/james/mailbox/maildir/MaildirFolder.java @@ -58,6 +58,7 @@ public class MaildirFolder { public static final String VALIDITY_FILE = "james-uidvalidity"; public static final String UIDLIST_FILE = "james-uidlist"; public static final String ACL_FILE = "james-acl"; + public static final String MAILBOX_ID_FILE = "james-mailboxId"; public static final String CUR = "cur"; public static final String NEW = "new"; public static final String TMP = "tmp"; @@ -68,7 +69,8 @@ public class MaildirFolder { private final File tmpFolder; private final File uidFile; private final File aclFile; - + private final File mailboxIdFile; + private Optional<MessageUid> lastUid; private int messageCount = 0; private long uidValidity = -1; @@ -91,6 +93,7 @@ public class MaildirFolder { this.tmpFolder = new File(rootFolder, TMP); this.uidFile = new File(rootFolder, UIDLIST_FILE); this.aclFile = new File(rootFolder, ACL_FILE); + this.mailboxIdFile = new File(rootFolder, MAILBOX_ID_FILE); this.locker = locker; this.path = path; this.lastUid = Optional.empty(); @@ -163,6 +166,10 @@ public class MaildirFolder { public File getCurFolder() { return curFolder; } + + public File getMailboxIdFile() { + return mailboxIdFile; + } /** * Returns the ./new folder of this Maildir folder. @@ -294,6 +301,51 @@ public class MaildirFolder { fos.write(String.valueOf(uidValidity).getBytes()); } } + + /** + * Sets the mailboxId for this mailbox and writes it to the file system + * @param mailboxId + * @throws IOException + */ + public void setMailboxId(MaildirId mailboxId) throws IOException { + saveMailboxId(mailboxId); + } + + /** + * Read the mailboxId of the given mailbox from the file system. + * If the respective file is not yet there, it gets created and + * filled with a brand new uidValidity. + * @return The uidValidity + * @throws IOException if there are problems with the validity file + */ + public MaildirId readMailboxId() throws IOException { + if (!mailboxIdFile.exists()) { + MaildirId maildirId = MaildirId.random(); + saveMailboxId(maildirId); + return maildirId; + } + try (FileInputStream fis = new FileInputStream(mailboxIdFile); + InputStreamReader isr = new InputStreamReader(fis)) { + char[] mailboxId = new char[20]; + int len = isr.read(mailboxId); + int idAsInt = Integer.parseInt(String.valueOf(mailboxId, 0, len).trim()); + return MaildirId.of(idAsInt); + } + } + + /** + * Save the given MaildirId to the file system + * @param id + * @throws IOException + */ + private void saveMailboxId(MaildirId id) throws IOException { + if (!mailboxIdFile.createNewFile()) { + throw new IOException("Could not create file " + mailboxIdFile); + } + try (FileOutputStream fos = new FileOutputStream(mailboxIdFile)) { + fos.write(String.valueOf(id.getRawId()).getBytes()); + } + } /** * Sets and returns a new uidValidity for this folder. http://git-wip-us.apache.org/repos/asf/james-project/blob/7e32da51/mailbox/maildir/src/main/java/org/apache/james/mailbox/maildir/MaildirId.java ---------------------------------------------------------------------- diff --git a/mailbox/maildir/src/main/java/org/apache/james/mailbox/maildir/MaildirId.java b/mailbox/maildir/src/main/java/org/apache/james/mailbox/maildir/MaildirId.java index 69de83f..18df444 100644 --- a/mailbox/maildir/src/main/java/org/apache/james/mailbox/maildir/MaildirId.java +++ b/mailbox/maildir/src/main/java/org/apache/james/mailbox/maildir/MaildirId.java @@ -19,6 +19,7 @@ package org.apache.james.mailbox.maildir; import java.io.Serializable; +import java.security.SecureRandom; import org.apache.james.mailbox.model.MailboxId; @@ -31,6 +32,12 @@ public class MaildirId implements MailboxId, Serializable { } } + private static final SecureRandom RANDOM = new SecureRandom(); + + public static MaildirId random() { + return MaildirId.of(Math.abs(RANDOM.nextInt())); + } + public static MaildirId of(int id) { return new MaildirId(id); } http://git-wip-us.apache.org/repos/asf/james-project/blob/7e32da51/mailbox/maildir/src/main/java/org/apache/james/mailbox/maildir/MaildirStore.java ---------------------------------------------------------------------- diff --git a/mailbox/maildir/src/main/java/org/apache/james/mailbox/maildir/MaildirStore.java b/mailbox/maildir/src/main/java/org/apache/james/mailbox/maildir/MaildirStore.java index f7a2d8d..b0a3192 100644 --- a/mailbox/maildir/src/main/java/org/apache/james/mailbox/maildir/MaildirStore.java +++ b/mailbox/maildir/src/main/java/org/apache/james/mailbox/maildir/MaildirStore.java @@ -132,6 +132,7 @@ public class MaildirStore implements UidProvider, ModSeqProvider { folder.setMessageNameStrictParse(isMessageNameStrictParse()); try { Mailbox loadedMailbox = new SimpleMailbox(mailboxPath, folder.getUidValidity()); + loadedMailbox.setMailboxId(folder.readMailboxId()); loadedMailbox.setACL(folder.getACL(session)); return loadedMailbox; } catch (IOException e) { http://git-wip-us.apache.org/repos/asf/james-project/blob/7e32da51/mailbox/maildir/src/main/java/org/apache/james/mailbox/maildir/mail/MailboxCache.java ---------------------------------------------------------------------- diff --git a/mailbox/maildir/src/main/java/org/apache/james/mailbox/maildir/mail/MailboxCache.java b/mailbox/maildir/src/main/java/org/apache/james/mailbox/maildir/mail/MailboxCache.java deleted file mode 100644 index cf04c1e..0000000 --- a/mailbox/maildir/src/main/java/org/apache/james/mailbox/maildir/mail/MailboxCache.java +++ /dev/null @@ -1,68 +0,0 @@ -/**************************************************************** - * Licensed to the Apache Software Foundation (ASF) under one * - * or more contributor license agreements. See the NOTICE file * - * distributed with this work for additional information * - * regarding copyright ownership. The ASF licenses this file * - * to you under the Apache License, Version 2.0 (the * - * "License"); you may not use this file except in compliance * - * with the License. You may obtain a copy of the License at * - * * - * http://www.apache.org/licenses/LICENSE-2.0 * - * * - * Unless required by applicable law or agreed to in writing, * - * software distributed under the License is distributed on an * - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * - * KIND, either express or implied. See the License for the * - * specific language governing permissions and limitations * - * under the License. * - ****************************************************************/ - -package org.apache.james.mailbox.maildir.mail; - -import java.util.ArrayList; - -import org.apache.james.mailbox.exception.MailboxNotFoundException; -import org.apache.james.mailbox.maildir.MaildirId; -import org.apache.james.mailbox.store.mail.model.Mailbox; -import org.apache.james.mailbox.store.mail.model.impl.SimpleMailbox; - -public class MailboxCache { - /** - * A request-scoped list of mailboxes in order to refer to them via id - */ - private final ArrayList<Mailbox> mailboxCache = new ArrayList<>(); - - /** - * Stores a copy of a mailbox in a cache valid for one request. This is to enable - * referring to renamed mailboxes via id. - * @param mailbox The mailbox to cache - * @return the cached mailbox - */ - public synchronized Mailbox cacheMailbox(Mailbox mailbox) { - mailboxCache.add(new SimpleMailbox(mailbox)); - int id = mailboxCache.size() - 1; - mailbox.setMailboxId(MaildirId.of(id)); - return mailbox; - } - - /** - * Retrieves a mailbox from the cache - * @param mailboxId The id of the mailbox to retrieve - * @return The mailbox - * @throws MailboxNotFoundException If the mailboxId is not in the cache - */ - public synchronized Mailbox getCachedMailbox(MaildirId mailboxId) throws MailboxNotFoundException { - if (mailboxId == null) { - throw new MailboxNotFoundException("null"); - } - try { - return mailboxCache.get(mailboxId.getRawId()); - } catch (IndexOutOfBoundsException e) { - throw new MailboxNotFoundException(mailboxId); - } - } - - public synchronized void clear() { - mailboxCache.clear(); - } -} http://git-wip-us.apache.org/repos/asf/james-project/blob/7e32da51/mailbox/maildir/src/main/java/org/apache/james/mailbox/maildir/mail/MaildirMailboxMapper.java ---------------------------------------------------------------------- diff --git a/mailbox/maildir/src/main/java/org/apache/james/mailbox/maildir/mail/MaildirMailboxMapper.java b/mailbox/maildir/src/main/java/org/apache/james/mailbox/maildir/mail/MaildirMailboxMapper.java index b1b83f9..454d857 100644 --- a/mailbox/maildir/src/main/java/org/apache/james/mailbox/maildir/mail/MaildirMailboxMapper.java +++ b/mailbox/maildir/src/main/java/org/apache/james/mailbox/maildir/mail/MaildirMailboxMapper.java @@ -23,6 +23,8 @@ import java.io.FilenameFilter; import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Optional; +import java.util.Random; import java.util.regex.Pattern; import org.apache.commons.io.FileUtils; @@ -55,18 +57,12 @@ public class MaildirMailboxMapper extends NonTransactionalMapper implements Mail * The {@link MaildirStore} the mailboxes reside in */ private final MaildirStore maildirStore; - - /** - * A request-scoped list of mailboxes in order to refer to them via id - */ - private final MailboxCache mailboxCache; private final MailboxSession session; public MaildirMailboxMapper(MaildirStore maildirStore, MailboxSession session) { this.maildirStore = maildirStore; this.session = session; - this.mailboxCache = new MailboxCache(); } @Override @@ -109,14 +105,18 @@ public class MaildirMailboxMapper extends NonTransactionalMapper implements Mail @Override public Mailbox findMailboxByPath(MailboxPath mailboxPath) throws MailboxException, MailboxNotFoundException { - Mailbox mailbox = maildirStore.loadMailbox(session, mailboxPath); - return mailboxCache.cacheMailbox(mailbox); + return maildirStore.loadMailbox(session, mailboxPath); } @Override public Mailbox findMailboxById(MailboxId id) throws MailboxException, MailboxNotFoundException { - MaildirId mailboxId = (MaildirId)id; - return mailboxCache.getCachedMailbox(mailboxId); + if (id == null) { + throw new MailboxNotFoundException("null"); + } + return list().stream() + .filter(mailbox -> mailbox.getMailboxId().equals(id)) + .findAny() + .orElseThrow(() -> new MailboxNotFoundException(id)); } @Override @@ -131,13 +131,13 @@ public class MaildirMailboxMapper extends NonTransactionalMapper implements Mail for (File folder : folders) { if (folder.isDirectory()) { Mailbox mailbox = maildirStore.loadMailbox(session, root, mailboxPath.getNamespace(), mailboxPath.getUser(), folder.getName()); - mailboxList.add(mailboxCache.cacheMailbox(mailbox)); + mailboxList.add(mailbox); } } // INBOX is in the root of the folder if (Pattern.matches(mailboxPath.getName().replace(MaildirStore.WILDCARD, ".*"), MailboxConstants.INBOX)) { Mailbox mailbox = maildirStore.loadMailbox(session, root, mailboxPath.getNamespace(), mailboxPath.getUser(), ""); - mailboxList.add(0, mailboxCache.cacheMailbox(mailbox)); + mailboxList.add(0, mailbox); } return mailboxList; } @@ -152,8 +152,11 @@ public class MaildirMailboxMapper extends NonTransactionalMapper implements Mail @Override public MailboxId save(Mailbox mailbox) throws MailboxException { + MaildirId maildirId = Optional.ofNullable(mailbox.getMailboxId()) + .map(mailboxId -> (MaildirId) mailboxId) + .orElseGet(MaildirId::random); try { - Mailbox originalMailbox = mailboxCache.getCachedMailbox((MaildirId) mailbox.getMailboxId()); + Mailbox originalMailbox = findMailboxById(mailbox.getMailboxId()); MaildirFolder folder = maildirStore.createMaildirFolder(mailbox); // equals with null check if (originalMailbox.getName() == null ? mailbox.getName() != null : !originalMailbox.getName().equals(mailbox.getName())) { @@ -171,6 +174,9 @@ public class MaildirMailboxMapper extends NonTransactionalMapper implements Mail if (!originalFolder.getCurFolder().renameTo(folder.getCurFolder())) { throw new IOException("Could not rename folder " + originalFolder.getCurFolder() + " to " + folder.getCurFolder()); } + if (!originalFolder.getMailboxIdFile().renameTo(folder.getMailboxIdFile())) { + throw new IOException("Could not rename folder " + originalFolder.getCurFolder() + " to " + folder.getCurFolder()); + } if (!originalFolder.getNewFolder().renameTo(folder.getNewFolder())) { throw new IOException("Could not rename folder " + originalFolder.getNewFolder() + " to " + folder.getNewFolder()); } @@ -192,6 +198,7 @@ public class MaildirMailboxMapper extends NonTransactionalMapper implements Mail FileUtils.forceMkdir(originalFolder.getCurFolder()); FileUtils.forceMkdir(originalFolder.getNewFolder()); FileUtils.forceMkdir(originalFolder.getTmpFolder()); + originalFolder.setMailboxId(MaildirId.of(Math.abs(new Random().nextInt()))); } catch (IOException e) { throw new MailboxException("Failed to save Mailbox " + mailbox, e); } @@ -224,14 +231,14 @@ public class MaildirMailboxMapper extends NonTransactionalMapper implements Mail } try { folder.setUidValidity(mailbox.getUidValidity()); + folder.setMailboxId(maildirId); } catch (IOException ioe) { throw new MailboxException("Failed to save Mailbox " + mailbox, ioe); } folder.setACL(session, mailbox.getACL()); } - mailboxCache.cacheMailbox(mailbox); - return mailbox.getMailboxId(); + return maildirId; } @Override @@ -240,26 +247,28 @@ public class MaildirMailboxMapper extends NonTransactionalMapper implements Mail File maildirRoot = maildirStore.getMaildirRoot(); List<Mailbox> mailboxList = new ArrayList<>(); - if (maildirStore.getMaildirLocation().endsWith("/" + MaildirStore.PATH_FULLUSER)) { - File[] users = maildirRoot.listFiles(); - visitUsersForMailboxList(null, users, mailboxList); + + + if (maildirStore.getMaildirLocation().endsWith("/" + MaildirStore.PATH_DOMAIN + "/" + MaildirStore.PATH_USER)) { + File[] domains = maildirRoot.listFiles(); + for (File domain : domains) { + File[] users = domain.listFiles(); + visitUsersForMailboxList(domain, users, mailboxList); + } return mailboxList; } - - File[] domains = maildirRoot.listFiles(); - for (File domain: domains) { - File[] users = domain.listFiles(); - visitUsersForMailboxList(domain, users, mailboxList); - } - return mailboxList; + + File[] users = maildirRoot.listFiles(); + visitUsersForMailboxList(null, users, mailboxList); + return mailboxList; } @Override public void endRequest() { - mailboxCache.clear(); + } - + private void visitUsersForMailboxList(File domain, File[] users, List<Mailbox> mailboxList) throws MailboxException { String userName = null; http://git-wip-us.apache.org/repos/asf/james-project/blob/7e32da51/mailbox/maildir/src/test/java/org/apache/james/mailbox/maildir/DomainUserMaildirMailboxManagerTest.java ---------------------------------------------------------------------- diff --git a/mailbox/maildir/src/test/java/org/apache/james/mailbox/maildir/DomainUserMaildirMailboxManagerTest.java b/mailbox/maildir/src/test/java/org/apache/james/mailbox/maildir/DomainUserMaildirMailboxManagerTest.java index 508fc54..cc494f1 100644 --- a/mailbox/maildir/src/test/java/org/apache/james/mailbox/maildir/DomainUserMaildirMailboxManagerTest.java +++ b/mailbox/maildir/src/test/java/org/apache/james/mailbox/maildir/DomainUserMaildirMailboxManagerTest.java @@ -18,13 +18,9 @@ ****************************************************************/ package org.apache.james.mailbox.maildir; -import java.io.UnsupportedEncodingException; - import org.apache.james.junit.TemporaryFolderExtension; import org.apache.james.mailbox.MailboxManager; import org.apache.james.mailbox.MailboxManagerTest; -import org.apache.james.mailbox.exception.MailboxException; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.extension.RegisterExtension; public class DomainUserMaildirMailboxManagerTest extends MailboxManagerTest { @@ -41,9 +37,4 @@ public class DomainUserMaildirMailboxManagerTest extends MailboxManagerTest { } } - @Disabled("https://issues.apache.org/jira/browse/MAILBOX-292";) - @Override - public void createMailboxShouldReturnRightId() throws MailboxException, UnsupportedEncodingException { - - } } http://git-wip-us.apache.org/repos/asf/james-project/blob/7e32da51/mailbox/maildir/src/test/java/org/apache/james/mailbox/maildir/FullUserMaildirMailboxManagerTest.java ---------------------------------------------------------------------- diff --git a/mailbox/maildir/src/test/java/org/apache/james/mailbox/maildir/FullUserMaildirMailboxManagerTest.java b/mailbox/maildir/src/test/java/org/apache/james/mailbox/maildir/FullUserMaildirMailboxManagerTest.java index b69f30f..e08097f 100644 --- a/mailbox/maildir/src/test/java/org/apache/james/mailbox/maildir/FullUserMaildirMailboxManagerTest.java +++ b/mailbox/maildir/src/test/java/org/apache/james/mailbox/maildir/FullUserMaildirMailboxManagerTest.java @@ -18,13 +18,9 @@ ****************************************************************/ package org.apache.james.mailbox.maildir; -import java.io.UnsupportedEncodingException; - import org.apache.james.junit.TemporaryFolderExtension; import org.apache.james.mailbox.MailboxManager; import org.apache.james.mailbox.MailboxManagerTest; -import org.apache.james.mailbox.exception.MailboxException; -import org.junit.Ignore; import org.junit.jupiter.api.extension.RegisterExtension; public class FullUserMaildirMailboxManagerTest extends MailboxManagerTest { @@ -39,10 +35,4 @@ public class FullUserMaildirMailboxManagerTest extends MailboxManagerTest { throw new RuntimeException(e); } } - - @Ignore("https://issues.apache.org/jira/browse/MAILBOX-292";) - @Override - public void createMailboxShouldReturnRightId() throws MailboxException, UnsupportedEncodingException { - - } } http://git-wip-us.apache.org/repos/asf/james-project/blob/7e32da51/mpt/impl/imap-mailbox/maildir/src/test/resources/logback-test.xml ---------------------------------------------------------------------- diff --git a/mpt/impl/imap-mailbox/maildir/src/test/resources/logback-test.xml b/mpt/impl/imap-mailbox/maildir/src/test/resources/logback-test.xml new file mode 100644 index 0000000..b02f0d8 --- /dev/null +++ b/mpt/impl/imap-mailbox/maildir/src/test/resources/logback-test.xml @@ -0,0 +1,28 @@ +<?xml version="1.0" encoding="UTF-8"?> +<configuration> + + <contextListener class="ch.qos.logback.classic.jul.LevelChangePropagator"> + <resetJUL>true</resetJUL> + </contextListener> + + <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender"> + <encoder> + <pattern>%d{HH:mm:ss.SSS} [%-5level] %logger{15} - %msg%n%rEx</pattern> + <immediateFlush>false</immediateFlush> + </encoder> + </appender> + + <root level="WARN"> + <appender-ref ref="CONSOLE" /> + </root> + + + <logger name="org.apache.james" level="WARN" > + <appender-ref ref="CONSOLE" /> + </logger> + + <logger name="org.apache.james.backends.cassandra.DockerCassandraRule" level="WARN" > + <appender-ref ref="CONSOLE" /> + </logger> + +</configuration> --------------------------------------------------------------------- To unsubscribe, e-mail: server-dev-unsubscr...@james.apache.org For additional commands, e-mail: server-dev-h...@james.apache.org
