Author: norman Date: Wed Apr 28 20:07:29 2010 New Revision: 939071 URL: http://svn.apache.org/viewvc?rev=939071&view=rev Log: * Fix for JPA fails to persist MailboxMembership Entity on heavy load by using persimistic locking (IMAP-137) * Do not cache Mailbox instances in store api to prevent GC (IMAP-131) * Fix inmemory implementation
Removed: james/imap/trunk/jpa/src/main/java/org/apache/james/imap/jpa/mail/openjpa/OpenJPAMailboxMapper.java Modified: james/imap/trunk/deployment/src/test/java/org/apache/james/imap/functional/jpa/JPAHostSystem.java james/imap/trunk/jpa/src/main/java/org/apache/james/imap/jpa/JPAMailbox.java james/imap/trunk/jpa/src/main/java/org/apache/james/imap/jpa/JPAMailboxManager.java james/imap/trunk/jpa/src/main/java/org/apache/james/imap/jpa/MailboxSessionEntityManagerFactory.java james/imap/trunk/jpa/src/main/java/org/apache/james/imap/jpa/mail/JPAMailboxMapper.java james/imap/trunk/jpa/src/main/java/org/apache/james/imap/jpa/openjpa/OpenJPAMailbox.java james/imap/trunk/jpa/src/main/java/org/apache/james/imap/jpa/openjpa/OpenJPAMailboxManager.java james/imap/trunk/mailbox/src/main/java/org/apache/james/imap/mailbox/Content.java james/imap/trunk/memory/src/main/java/org/apache/james/imap/inmemory/InMemoryMailboxManager.java james/imap/trunk/store/src/main/java/org/apache/james/imap/store/DelegatingMailboxListener.java james/imap/trunk/store/src/main/java/org/apache/james/imap/store/StoreMailbox.java Modified: james/imap/trunk/deployment/src/test/java/org/apache/james/imap/functional/jpa/JPAHostSystem.java URL: http://svn.apache.org/viewvc/james/imap/trunk/deployment/src/test/java/org/apache/james/imap/functional/jpa/JPAHostSystem.java?rev=939071&r1=939070&r2=939071&view=diff ============================================================================== --- james/imap/trunk/deployment/src/test/java/org/apache/james/imap/functional/jpa/JPAHostSystem.java (original) +++ james/imap/trunk/deployment/src/test/java/org/apache/james/imap/functional/jpa/JPAHostSystem.java Wed Apr 28 20:07:29 2010 @@ -66,7 +66,11 @@ public class JPAHostSystem extends ImapH "org.apache.james.imap.jpa.mail.model.JPAMessage;" + "org.apache.james.imap.jpa.mail.model.JPAProperty;" + "org.apache.james.imap.jpa.user.model.JPASubscription)"); - + // persimistic locking.. + properties.put("openjpa.LockManager", "pessimistic"); + properties.put("openjpa.ReadLockLevel", "read"); + properties.put("openjpa.WriteLockLevel", "write"); + properties.put("openjpa.jdbc.TransactionIsolation", "repeatable-read"); userManager = new InMemoryUserManager(); entityManagerFactory = OpenJPAPersistence.getEntityManagerFactory(properties); MailboxSessionEntityManagerFactory factory = new MailboxSessionEntityManagerFactory(entityManagerFactory); Modified: james/imap/trunk/jpa/src/main/java/org/apache/james/imap/jpa/JPAMailbox.java URL: http://svn.apache.org/viewvc/james/imap/trunk/jpa/src/main/java/org/apache/james/imap/jpa/JPAMailbox.java?rev=939071&r1=939070&r2=939071&view=diff ============================================================================== --- james/imap/trunk/jpa/src/main/java/org/apache/james/imap/jpa/JPAMailbox.java (original) +++ james/imap/trunk/jpa/src/main/java/org/apache/james/imap/jpa/JPAMailbox.java Wed Apr 28 20:07:29 2010 @@ -25,6 +25,8 @@ import java.util.List; import javax.mail.Flags; import javax.persistence.EntityManager; +import javax.persistence.EntityTransaction; +import javax.persistence.Query; import org.apache.james.imap.jpa.mail.JPAMailboxMapper; import org.apache.james.imap.jpa.mail.JPAMessageMapper; @@ -105,12 +107,19 @@ public abstract class JPAMailbox extends } /** - * Reserve next Uid in mailbox and return the mailbox. This method needs to be synchronized - * to be sure we don't get any race-condition + * Reserve next Uid in mailbox and return the mailbox. We use a transaction here to be sure we don't get any duplicates + * */ protected Mailbox<Long> reserveNextUid(MailboxSession session) throws MailboxException { - final JPAMailboxMapper mapper = createMailboxMapper(session); - final Mailbox<Long> mailbox = mapper.consumeNextUid(getMailboxId()); + EntityManager entityManager = entityManagerFactory.getEntityManager(session); + + EntityTransaction transaction = entityManager.getTransaction(); + transaction.begin(); + Query query = entityManager.createNamedQuery("findMailboxById").setParameter("idParam", getMailboxId()); + org.apache.james.imap.jpa.mail.model.JPAMailbox mailbox = (org.apache.james.imap.jpa.mail.model.JPAMailbox) query.getSingleResult(); + mailbox.consumeUid(); + entityManager.persist(mailbox); + transaction.commit(); return mailbox; - } + } } Modified: james/imap/trunk/jpa/src/main/java/org/apache/james/imap/jpa/JPAMailboxManager.java URL: http://svn.apache.org/viewvc/james/imap/trunk/jpa/src/main/java/org/apache/james/imap/jpa/JPAMailboxManager.java?rev=939071&r1=939070&r2=939071&view=diff ============================================================================== --- james/imap/trunk/jpa/src/main/java/org/apache/james/imap/jpa/JPAMailboxManager.java (original) +++ james/imap/trunk/jpa/src/main/java/org/apache/james/imap/jpa/JPAMailboxManager.java Wed Apr 28 20:07:29 2010 @@ -19,6 +19,9 @@ package org.apache.james.imap.jpa; +import javax.persistence.EntityManager; + +import org.apache.james.imap.jpa.mail.JPAMailboxMapper; import org.apache.james.imap.mailbox.MailboxException; import org.apache.james.imap.mailbox.MailboxSession; import org.apache.james.imap.store.Authenticator; @@ -37,7 +40,6 @@ import org.apache.james.imap.store.trans public abstract class JPAMailboxManager extends StoreMailboxManager<Long> { protected final MailboxSessionEntityManagerFactory entityManagerFactory; - public JPAMailboxManager(final Authenticator authenticator, final Subscriber subscriber, final MailboxSessionEntityManagerFactory entityManagerFactory) { super(authenticator, subscriber); @@ -77,6 +79,11 @@ public abstract class JPAMailboxManager } - + @Override + protected MailboxMapper<Long> createMailboxMapper(MailboxSession session) { + EntityManager manager = entityManagerFactory.getEntityManager(session); + return new JPAMailboxMapper(manager); + } + } Modified: james/imap/trunk/jpa/src/main/java/org/apache/james/imap/jpa/MailboxSessionEntityManagerFactory.java URL: http://svn.apache.org/viewvc/james/imap/trunk/jpa/src/main/java/org/apache/james/imap/jpa/MailboxSessionEntityManagerFactory.java?rev=939071&r1=939070&r2=939071&view=diff ============================================================================== --- james/imap/trunk/jpa/src/main/java/org/apache/james/imap/jpa/MailboxSessionEntityManagerFactory.java (original) +++ james/imap/trunk/jpa/src/main/java/org/apache/james/imap/jpa/MailboxSessionEntityManagerFactory.java Wed Apr 28 20:07:29 2010 @@ -1,10 +1,31 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ package org.apache.james.imap.jpa; import javax.persistence.EntityManager; import javax.persistence.EntityManagerFactory; import org.apache.james.imap.mailbox.MailboxSession; - +/** + * Make sure the {...@link EntityManager} is always the same till the {...@link MailboxSession} change + * + */ public class MailboxSessionEntityManagerFactory { private EntityManagerFactory factory; @@ -14,16 +35,28 @@ public class MailboxSessionEntityManager this.factory = factory; } + /** + * Return the {...@link EntityManager} for this session. If not exists create one and save it in the session. If one is found in the session return it + * + * @param session + * @return manager + */ public EntityManager getEntityManager(MailboxSession session) { EntityManager manager = (EntityManager) session.getAttributes().get(ENTITYMANAGER); if (manager == null || manager.isOpen() == false) { manager = factory.createEntityManager(); session.getAttributes().put(ENTITYMANAGER, manager); + } return manager; } + /** + * Close the {...@link EntityManager} stored in the session if one exists and is open + * + * @param session + */ public void closeEntityManager(MailboxSession session) { if (session != null) { EntityManager manager = (EntityManager) session.getAttributes().remove(ENTITYMANAGER); Modified: james/imap/trunk/jpa/src/main/java/org/apache/james/imap/jpa/mail/JPAMailboxMapper.java URL: http://svn.apache.org/viewvc/james/imap/trunk/jpa/src/main/java/org/apache/james/imap/jpa/mail/JPAMailboxMapper.java?rev=939071&r1=939070&r2=939071&view=diff ============================================================================== --- james/imap/trunk/jpa/src/main/java/org/apache/james/imap/jpa/mail/JPAMailboxMapper.java (original) +++ james/imap/trunk/jpa/src/main/java/org/apache/james/imap/jpa/mail/JPAMailboxMapper.java Wed Apr 28 20:07:29 2010 @@ -36,7 +36,7 @@ import org.apache.james.imap.store.mail. /** * Data access management for mailbox. */ -public abstract class JPAMailboxMapper extends JPATransactionalMapper implements MailboxMapper<Long> { +public class JPAMailboxMapper extends JPATransactionalMapper implements MailboxMapper<Long> { private static final char SQL_WILDCARD_CHAR = '%'; @@ -138,23 +138,4 @@ public abstract class JPAMailboxMapper e throw new StorageException(HumanReadableText.SEARCH_FAILED, e); } } - - - - /* - * - */ - public Mailbox<Long> consumeNextUid(Long mailboxId) throws StorageException, MailboxNotFoundException { - try { - return doConsumeNextUid(mailboxId); - } catch (NoResultException e) { - throw new MailboxNotFoundException(mailboxId); - } catch (PersistenceException e) { - e.printStackTrace(); - throw new StorageException(HumanReadableText.COMSUME_UID_FAILED, e); - } - } - - /** Locking is required and is implementation specific */ - protected abstract JPAMailbox doConsumeNextUid(long mailboxId) throws PersistenceException; } Modified: james/imap/trunk/jpa/src/main/java/org/apache/james/imap/jpa/openjpa/OpenJPAMailbox.java URL: http://svn.apache.org/viewvc/james/imap/trunk/jpa/src/main/java/org/apache/james/imap/jpa/openjpa/OpenJPAMailbox.java?rev=939071&r1=939070&r2=939071&view=diff ============================================================================== --- james/imap/trunk/jpa/src/main/java/org/apache/james/imap/jpa/openjpa/OpenJPAMailbox.java (original) +++ james/imap/trunk/jpa/src/main/java/org/apache/james/imap/jpa/openjpa/OpenJPAMailbox.java Wed Apr 28 20:07:29 2010 @@ -33,7 +33,6 @@ import org.apache.james.imap.jpa.mail.JP import org.apache.james.imap.jpa.mail.model.AbstractJPAMailboxMembership; import org.apache.james.imap.jpa.mail.model.JPAHeader; import org.apache.james.imap.jpa.mail.model.openjpa.JPAStreamingMailboxMembership; -import org.apache.james.imap.jpa.mail.openjpa.OpenJPAMailboxMapper; import org.apache.james.imap.mailbox.MailboxException; import org.apache.james.imap.mailbox.MailboxSession; import org.apache.james.imap.mailbox.util.MailboxEventDispatcher; @@ -65,7 +64,7 @@ public class OpenJPAMailbox extends JPAM protected JPAMailboxMapper createMailboxMapper(MailboxSession session) { EntityManager manager = entityManagerFactory.getEntityManager(session); - JPAMailboxMapper mapper = new OpenJPAMailboxMapper(manager); + JPAMailboxMapper mapper = new JPAMailboxMapper(manager); return mapper; } Modified: james/imap/trunk/jpa/src/main/java/org/apache/james/imap/jpa/openjpa/OpenJPAMailboxManager.java URL: http://svn.apache.org/viewvc/james/imap/trunk/jpa/src/main/java/org/apache/james/imap/jpa/openjpa/OpenJPAMailboxManager.java?rev=939071&r1=939070&r2=939071&view=diff ============================================================================== --- james/imap/trunk/jpa/src/main/java/org/apache/james/imap/jpa/openjpa/OpenJPAMailboxManager.java (original) +++ james/imap/trunk/jpa/src/main/java/org/apache/james/imap/jpa/openjpa/OpenJPAMailboxManager.java Wed Apr 28 20:07:29 2010 @@ -19,17 +19,12 @@ package org.apache.james.imap.jpa.openjpa; -import javax.persistence.EntityManager; - import org.apache.james.imap.jpa.JPAMailboxManager; import org.apache.james.imap.jpa.MailboxSessionEntityManagerFactory; -import org.apache.james.imap.jpa.mail.openjpa.OpenJPAMailboxMapper; -import org.apache.james.imap.mailbox.MailboxSession; import org.apache.james.imap.mailbox.util.MailboxEventDispatcher; import org.apache.james.imap.store.Authenticator; import org.apache.james.imap.store.StoreMailbox; import org.apache.james.imap.store.Subscriber; -import org.apache.james.imap.store.mail.MailboxMapper; import org.apache.james.imap.store.mail.model.Mailbox; /** @@ -49,12 +44,6 @@ public class OpenJPAMailboxManager exten this(authenticator, subscriber, entityManagerFactory, false); } - @Override - protected MailboxMapper<Long> createMailboxMapper(MailboxSession session) { - EntityManager manager = entityManagerFactory.getEntityManager(session); - return new OpenJPAMailboxMapper(manager); - } - protected StoreMailbox<Long> createMailbox(MailboxEventDispatcher dispatcher, Mailbox<Long> mailboxRow) { StoreMailbox<Long> result = new OpenJPAMailbox(dispatcher,mailboxRow, entityManagerFactory, useStreaming); return result; Modified: james/imap/trunk/mailbox/src/main/java/org/apache/james/imap/mailbox/Content.java URL: http://svn.apache.org/viewvc/james/imap/trunk/mailbox/src/main/java/org/apache/james/imap/mailbox/Content.java?rev=939071&r1=939070&r2=939071&view=diff ============================================================================== --- james/imap/trunk/mailbox/src/main/java/org/apache/james/imap/mailbox/Content.java (original) +++ james/imap/trunk/mailbox/src/main/java/org/apache/james/imap/mailbox/Content.java Wed Apr 28 20:07:29 2010 @@ -33,6 +33,10 @@ public interface Content { /** * Writes content to the given channel. * + * Be aware that this operation may only be called once one the content because its possible dispose + * temp data. If you need to write the content more then one time you + * should "re-create" the content + * * @param channel * <code>Channel</code> open, not null * @throws MailboxException Modified: james/imap/trunk/memory/src/main/java/org/apache/james/imap/inmemory/InMemoryMailboxManager.java URL: http://svn.apache.org/viewvc/james/imap/trunk/memory/src/main/java/org/apache/james/imap/inmemory/InMemoryMailboxManager.java?rev=939071&r1=939070&r2=939071&view=diff ============================================================================== --- james/imap/trunk/memory/src/main/java/org/apache/james/imap/inmemory/InMemoryMailboxManager.java (original) +++ james/imap/trunk/memory/src/main/java/org/apache/james/imap/inmemory/InMemoryMailboxManager.java Wed Apr 28 20:07:29 2010 @@ -41,26 +41,38 @@ public class InMemoryMailboxManager exte private static final int INITIAL_SIZE = 128; private Map<Long, InMemoryMailbox> mailboxesById; - + private Map<String, InMemoryStoreMailbox> storeMailboxByName; + private Map<Long, String> idNameMap; + private MailboxSession session; + public InMemoryMailboxManager(Authenticator authenticator, Subscriber subscriber) { super(authenticator, subscriber); mailboxesById = new ConcurrentHashMap<Long, InMemoryMailbox>(INITIAL_SIZE); + storeMailboxByName = new ConcurrentHashMap<String, InMemoryStoreMailbox>(INITIAL_SIZE); + idNameMap = new ConcurrentHashMap<Long, String>(INITIAL_SIZE); } @Override protected StoreMailbox<Long> createMailbox(MailboxEventDispatcher dispatcher, Mailbox<Long> mailboxRow) { - final InMemoryStoreMailbox storeMailbox = new InMemoryStoreMailbox(dispatcher, (InMemoryMailbox)mailboxRow); + InMemoryStoreMailbox storeMailbox = storeMailboxByName.get(mailboxRow.getName()); + if (storeMailbox == null) { + storeMailbox = new InMemoryStoreMailbox(dispatcher, (InMemoryMailbox)mailboxRow); + storeMailboxByName.put(mailboxRow.getName(), storeMailbox); + } + return storeMailbox; } @Override protected MailboxMapper<Long> createMailboxMapper(MailboxSession session) { + this.session = session; return this; } @Override protected void doCreate(String namespaceName, MailboxSession session) throws StorageException { InMemoryMailbox mailbox = new InMemoryMailbox(randomId(), namespaceName, randomUidValidity()); + idNameMap.put(mailbox.getMailboxId(), mailbox.getName()); save(mailbox); } @@ -69,7 +81,7 @@ public class InMemoryMailboxManager exte * (non-Javadoc) * @see org.apache.james.imap.store.mail.MailboxMapper#countMailboxesWithName(java.lang.String) */ - public synchronized long countMailboxesWithName(String name) throws StorageException { + public long countMailboxesWithName(String name) throws StorageException { int total = 0; for (final InMemoryMailbox mailbox:mailboxesById.values()) { if (mailbox.getName().equals(name)) { @@ -83,7 +95,7 @@ public class InMemoryMailboxManager exte * (non-Javadoc) * @see org.apache.james.imap.store.mail.MailboxMapper#delete(org.apache.james.imap.store.mail.model.Mailbox) */ - public synchronized void delete(Mailbox<Long> mailbox) throws StorageException { + public void delete(Mailbox<Long> mailbox) throws StorageException { mailboxesById.remove(mailbox.getMailboxId()); } @@ -91,7 +103,7 @@ public class InMemoryMailboxManager exte * (non-Javadoc) * @see org.apache.james.imap.store.mail.MailboxMapper#deleteAll() */ - public synchronized void deleteAll() throws StorageException { + public void deleteAll() throws StorageException { mailboxesById.clear(); } @@ -100,7 +112,7 @@ public class InMemoryMailboxManager exte * (non-Javadoc) * @see org.apache.james.imap.store.mail.MailboxMapper#findMailboxById(java.lang.Object) */ - public synchronized Mailbox<Long> findMailboxById(Long mailboxId) throws StorageException, MailboxNotFoundException { + public Mailbox<Long> findMailboxById(Long mailboxId) throws StorageException, MailboxNotFoundException { return mailboxesById.get(mailboxesById); } @@ -123,7 +135,7 @@ public class InMemoryMailboxManager exte * (non-Javadoc) * @see org.apache.james.imap.store.mail.MailboxMapper#findMailboxWithNameLike(java.lang.String) */ - public synchronized List<Mailbox<Long>> findMailboxWithNameLike(String name) throws StorageException { + public List<Mailbox<Long>> findMailboxWithNameLike(String name) throws StorageException { final String regex = name.replace("%", ".*"); List<Mailbox<Long>> results = new ArrayList<Mailbox<Long>>(); for (final InMemoryMailbox mailbox:mailboxesById.values()) { @@ -138,7 +150,7 @@ public class InMemoryMailboxManager exte * (non-Javadoc) * @see org.apache.james.imap.store.mail.MailboxMapper#existsMailboxStartingWith(java.lang.String) */ - public synchronized boolean existsMailboxStartingWith(String mailboxName) throws StorageException { + public boolean existsMailboxStartingWith(String mailboxName) throws StorageException { boolean result = false; for (final InMemoryMailbox mailbox:mailboxesById.values()) { if (mailbox.getName().startsWith(mailboxName)) { @@ -153,8 +165,21 @@ public class InMemoryMailboxManager exte * (non-Javadoc) * @see org.apache.james.imap.store.mail.MailboxMapper#save(org.apache.james.imap.store.mail.model.Mailbox) */ - public synchronized void save(Mailbox<Long> mailbox) throws StorageException { + public void save(Mailbox<Long> mailbox) throws StorageException { mailboxesById.put(mailbox.getMailboxId(), (InMemoryMailbox) mailbox); + String name = idNameMap.remove(mailbox.getMailboxId()); + if (name != null) { + InMemoryStoreMailbox m = storeMailboxByName.remove(name); + if (m!= null) { + try { + m.getMailboxRow(session).setName(mailbox.getName()); + storeMailboxByName.put(mailbox.getName(), m); + } catch (MailboxException e) { + throw new StorageException(e.getKey(), e); + } + } + } + idNameMap.put(mailbox.getMailboxId(), mailbox.getName()); } /* @@ -180,6 +205,8 @@ public class InMemoryMailboxManager exte } }); + storeMailboxByName.clear(); + idNameMap.clear(); } } Modified: james/imap/trunk/store/src/main/java/org/apache/james/imap/store/DelegatingMailboxListener.java URL: http://svn.apache.org/viewvc/james/imap/trunk/store/src/main/java/org/apache/james/imap/store/DelegatingMailboxListener.java?rev=939071&r1=939070&r2=939071&view=diff ============================================================================== --- james/imap/trunk/store/src/main/java/org/apache/james/imap/store/DelegatingMailboxListener.java (original) +++ james/imap/trunk/store/src/main/java/org/apache/james/imap/store/DelegatingMailboxListener.java Wed Apr 28 20:07:29 2010 @@ -50,7 +50,7 @@ public class DelegatingMailboxListener i * (non-Javadoc) * @see org.apache.james.imap.mailbox.MailboxListener#event(org.apache.james.imap.mailbox.MailboxListener.Event) */ - public synchronized void event(Event event) { + public void event(Event event) { List<MailboxListener> mListeners = listeners.get(event.getMailboxName()); if (mListeners != null && mListeners.isEmpty() == false) { for (int i = 0; i < mListeners.size(); i++) { Modified: james/imap/trunk/store/src/main/java/org/apache/james/imap/store/StoreMailbox.java URL: http://svn.apache.org/viewvc/james/imap/trunk/store/src/main/java/org/apache/james/imap/store/StoreMailbox.java?rev=939071&r1=939070&r2=939071&view=diff ============================================================================== --- james/imap/trunk/store/src/main/java/org/apache/james/imap/store/StoreMailbox.java (original) +++ james/imap/trunk/store/src/main/java/org/apache/james/imap/store/StoreMailbox.java Wed Apr 28 20:07:29 2010 @@ -149,161 +149,157 @@ public abstract class StoreMailbox<Id> i final MailboxSession mailboxSession,final boolean isRecent, final Flags flagsToBeSet) throws MailboxException { // this will hold the uid after the transaction was complete - final List<Long> uidHolder = new ArrayList<Long>(); final MessageMapper<Id> mapper = createMessageMapper(mailboxSession); - mapper.execute(new TransactionalMapper.Transaction() { - - public void run() throws MailboxException { - final Mailbox<Id> mailbox = reserveNextUid(mailboxSession); + + final Mailbox<Id> mailbox = reserveNextUid(mailboxSession); + final long uid = mailbox.getLastUid(); + + + File file = null; + try { + // Create a temporary file and copy the message to it. We will work with the file as + // source for the InputStream + file = File.createTempFile("imap", ".msg"); + FileOutputStream out = new FileOutputStream(file); + + byte[] buf = new byte[1024]; + int i = 0; + while ((i = msgIn.read(buf)) != -1) { + out.write(buf, 0, i); + } + out.flush(); + out.close(); + + FileInputStream tmpMsgIn = new FileInputStream(file); + // To be thread safe, we first get our own copy and the + // exclusive + // Uid + // TODO create own message_id and assign uid later + // at the moment it could lead to the situation that uid 5 + // is + // inserted long before 4, when + // mail 4 is big and comes over a slow connection. + final int size = tmpMsgIn.available(); + final int bodyStartOctet = bodyStartOctet(new FileInputStream(file)); + + // Disable line length... This should be handled by the smtp server component and not the parser itself + // https://issues.apache.org/jira/browse/IMAP-122 + MimeEntityConfig config = new MimeEntityConfig(); + config.setMaximalBodyDescriptor(true); + config.setMaxLineLen(-1); + final ConfigurableMimeTokenStream parser = new ConfigurableMimeTokenStream(config); + + parser.setRecursionMode(MimeTokenStream.M_NO_RECURSE); + parser.parse(new FileInputStream(file)); + final List<Header> headers = new ArrayList<Header>(INITIAL_SIZE_HEADERS); + + int lineNumber = 0; + int next = parser.next(); + while (next != MimeTokenStream.T_BODY + && next != MimeTokenStream.T_END_OF_STREAM + && next != MimeTokenStream.T_START_MULTIPART) { + if (next == MimeTokenStream.T_FIELD) { + String fieldValue = parser.getField().getBody(); + if (fieldValue.endsWith("\r\f")) { + fieldValue = fieldValue.substring(0,fieldValue.length() - 2); + } + if (fieldValue.startsWith(" ")) { + fieldValue = fieldValue.substring(1); + } + final Header header + = createHeader(++lineNumber, parser.getField().getName(), + fieldValue); + headers.add(header); + } + next = parser.next(); + } + final MaximalBodyDescriptor descriptor = (MaximalBodyDescriptor) parser.getBodyDescriptor(); + final PropertyBuilder propertyBuilder = new PropertyBuilder(); + final String mediaType; + final String mediaTypeFromHeader = descriptor.getMediaType(); + final String subType; + if (mediaTypeFromHeader == null) { + mediaType = "text"; + subType = "plain"; + } else { + mediaType = mediaTypeFromHeader; + subType = descriptor.getSubType(); + } + propertyBuilder.setMediaType(mediaType); + propertyBuilder.setSubType(subType); + propertyBuilder.setContentID(descriptor.getContentId()); + propertyBuilder.setContentDescription(descriptor.getContentDescription()); + propertyBuilder.setContentLocation(descriptor.getContentLocation()); + propertyBuilder.setContentMD5(descriptor.getContentMD5Raw()); + propertyBuilder.setContentTransferEncoding(descriptor.getTransferEncoding()); + propertyBuilder.setContentLanguage(descriptor.getContentLanguage()); + propertyBuilder.setContentDispositionType(descriptor.getContentDispositionType()); + propertyBuilder.setContentDispositionParameters(descriptor.getContentDispositionParameters()); + propertyBuilder.setContentTypeParameters(descriptor.getContentTypeParameters()); + // Add missing types + final String codeset = descriptor.getCharset(); + if (codeset == null) { + if ("TEXT".equalsIgnoreCase(mediaType)) { + propertyBuilder.setCharset("us-ascii"); + } + } else { + propertyBuilder.setCharset(codeset); + } + + final String boundary = descriptor.getBoundary(); + if (boundary != null) { + propertyBuilder.setBoundary(boundary); + } + if ("text".equalsIgnoreCase(mediaType)) { + final CountingInputStream bodyStream = new CountingInputStream(parser.getInputStream()); + bodyStream.readAll(); + long lines = bodyStream.getLineCount(); - File file = null; - try { - // Create a temporary file and copy the message to it. We will work with the file as - // source for the InputStream - file = File.createTempFile("imap", ".msg"); - FileOutputStream out = new FileOutputStream(file); - - byte[] buf = new byte[1024]; - int i = 0; - while ((i = msgIn.read(buf)) != -1) { - out.write(buf, 0, i); - } - out.flush(); - out.close(); - - FileInputStream tmpMsgIn = new FileInputStream(file); - // To be thread safe, we first get our own copy and the - // exclusive - // Uid - // TODO create own message_id and assign uid later - // at the moment it could lead to the situation that uid 5 - // is - // inserted long before 4, when - // mail 4 is big and comes over a slow connection. - - final long uid = mailbox.getLastUid(); - final int size = tmpMsgIn.available(); - final int bodyStartOctet = bodyStartOctet(new FileInputStream(file)); - - // Disable line length... This should be handled by the smtp server component and not the parser itself - // https://issues.apache.org/jira/browse/IMAP-122 - MimeEntityConfig config = new MimeEntityConfig(); - config.setMaximalBodyDescriptor(true); - config.setMaxLineLen(-1); - final ConfigurableMimeTokenStream parser = new ConfigurableMimeTokenStream(config); - - parser.setRecursionMode(MimeTokenStream.M_NO_RECURSE); - parser.parse(new FileInputStream(file)); - final List<Header> headers = new ArrayList<Header>(INITIAL_SIZE_HEADERS); - - int lineNumber = 0; - int next = parser.next(); - while (next != MimeTokenStream.T_BODY - && next != MimeTokenStream.T_END_OF_STREAM - && next != MimeTokenStream.T_START_MULTIPART) { - if (next == MimeTokenStream.T_FIELD) { - String fieldValue = parser.getField().getBody(); - if (fieldValue.endsWith("\r\f")) { - fieldValue = fieldValue.substring(0,fieldValue.length() - 2); - } - if (fieldValue.startsWith(" ")) { - fieldValue = fieldValue.substring(1); - } - final Header header - = createHeader(++lineNumber, parser.getField().getName(), - fieldValue); - headers.add(header); - } - next = parser.next(); - } - final MaximalBodyDescriptor descriptor = (MaximalBodyDescriptor) parser.getBodyDescriptor(); - final PropertyBuilder propertyBuilder = new PropertyBuilder(); - final String mediaType; - final String mediaTypeFromHeader = descriptor.getMediaType(); - final String subType; - if (mediaTypeFromHeader == null) { - mediaType = "text"; - subType = "plain"; - } else { - mediaType = mediaTypeFromHeader; - subType = descriptor.getSubType(); - } - propertyBuilder.setMediaType(mediaType); - propertyBuilder.setSubType(subType); - propertyBuilder.setContentID(descriptor.getContentId()); - propertyBuilder.setContentDescription(descriptor.getContentDescription()); - propertyBuilder.setContentLocation(descriptor.getContentLocation()); - propertyBuilder.setContentMD5(descriptor.getContentMD5Raw()); - propertyBuilder.setContentTransferEncoding(descriptor.getTransferEncoding()); - propertyBuilder.setContentLanguage(descriptor.getContentLanguage()); - propertyBuilder.setContentDispositionType(descriptor.getContentDispositionType()); - propertyBuilder.setContentDispositionParameters(descriptor.getContentDispositionParameters()); - propertyBuilder.setContentTypeParameters(descriptor.getContentTypeParameters()); - // Add missing types - final String codeset = descriptor.getCharset(); - if (codeset == null) { - if ("TEXT".equalsIgnoreCase(mediaType)) { - propertyBuilder.setCharset("us-ascii"); - } - } else { - propertyBuilder.setCharset(codeset); - } - - final String boundary = descriptor.getBoundary(); - if (boundary != null) { - propertyBuilder.setBoundary(boundary); - } - if ("text".equalsIgnoreCase(mediaType)) { - final CountingInputStream bodyStream = new CountingInputStream(parser.getInputStream()); - bodyStream.readAll(); - long lines = bodyStream.getLineCount(); - - next = parser.next(); - if (next == MimeTokenStream.T_EPILOGUE) { - final CountingInputStream epilogueStream = new CountingInputStream(parser.getInputStream()); - epilogueStream.readAll(); - lines+=epilogueStream.getLineCount(); - } - propertyBuilder.setTextualLineCount(lines); - } - - final Flags flags; - if (flagsToBeSet == null) { - flags = new Flags(); - } else { - flags = flagsToBeSet; - } - if (isRecent) { - flags.add(Flags.Flag.RECENT); - } - - final MailboxMembership<Id> message = createMessage(internalDate, uid, size, bodyStartOctet, new FileInputStream(file), flags, headers, propertyBuilder); + next = parser.next(); + if (next == MimeTokenStream.T_EPILOGUE) { + final CountingInputStream epilogueStream = new CountingInputStream(parser.getInputStream()); + epilogueStream.readAll(); + lines+=epilogueStream.getLineCount(); + } + propertyBuilder.setTextualLineCount(lines); + } + + final Flags flags; + if (flagsToBeSet == null) { + flags = new Flags(); + } else { + flags = flagsToBeSet; + } + if (isRecent) { + flags.add(Flags.Flag.RECENT); + } + final MailboxMembership<Id> message = createMessage(internalDate, uid, size, bodyStartOctet, new FileInputStream(file), flags, headers, propertyBuilder); + + mapper.execute(new TransactionalMapper.Transaction() { + + public void run() throws MailboxException { mapper.save(message); - - - - dispatcher.added(uid, mailboxSession.getSessionId(), getMailboxRow(mailboxSession).getName()); - //tracker.found(uid, message.createFlags()); - uidHolder.add(uid); - } catch (IOException e) { - e.printStackTrace(); - throw new MailboxException(HumanReadableText.FAILURE_MAIL_PARSE, e); - } catch (MessagingException e) { - e.printStackTrace(); - throw new MailboxException(HumanReadableText.FAILURE_MAIL_PARSE, e); - } catch (MimeException e) { - e.printStackTrace(); - throw new MailboxException(HumanReadableText.FAILURE_MAIL_PARSE, e); - } finally { - // delete the temporary file if one was specified - if (file != null) { - file.delete(); - } } + }); + + dispatcher.added(uid, mailboxSession.getSessionId(), getMailboxRow(mailboxSession).getName()); + return uid; + } catch (IOException e) { + e.printStackTrace(); + throw new MailboxException(HumanReadableText.FAILURE_MAIL_PARSE, e); + } catch (MessagingException e) { + e.printStackTrace(); + throw new MailboxException(HumanReadableText.FAILURE_MAIL_PARSE, e); + } catch (MimeException e) { + e.printStackTrace(); + throw new MailboxException(HumanReadableText.FAILURE_MAIL_PARSE, e); + } finally { + // delete the temporary file if one was specified + if (file != null) { + file.delete(); } - }); - - return uidHolder.get(0); + } + } /** @@ -616,22 +612,27 @@ public abstract class StoreMailbox<Id> i try { final List<MailboxMembership<Id>> copiedRows = new ArrayList<MailboxMembership<Id>>(); final MessageMapper<Id> mapper = createMessageMapper(session); - mapper.execute(new TransactionalMapper.Transaction() { + - public void run() throws MailboxException { - for (MailboxMembership<Id> originalMessage:originalRows) { + for (final MailboxMembership<Id> originalMessage:originalRows) { + final Mailbox<Id> mailbox = reserveNextUid(session ); + final long uid = mailbox.getLastUid(); + + mapper.execute(new TransactionalMapper.Transaction() { + + public void run() throws MailboxException { + final MailboxMembership<Id> newRow = copyMessage(originalMessage, uid, session); + mapper.save(newRow); + copiedRows.add(newRow); + + } + + }); - final Mailbox<Id> mailbox = reserveNextUid(session ); - if (mailbox != null) { - long uid = mailbox.getLastUid(); - final MailboxMembership<Id> newRow = copyMessage(originalMessage, uid, session); - mapper.save(newRow); - copiedRows.add(newRow); - } - } - } - }); + } + + // Wait until commit before issuing events --------------------------------------------------------------------- To unsubscribe, e-mail: server-dev-unsubscr...@james.apache.org For additional commands, e-mail: server-dev-h...@james.apache.org