Bron, Ken, did you have time to take a look at this patch ? By the way, here is an updated patch. -- Cyril Servant
2009/11/19 Servant Cyril <cyril.serv...@atosorigin.com>: > Hello, > > Here is a patch for optimizing pop. Let me explain : Here we have lots > (millions) of mailboxes. Many people connect to pop every few minutes, doing > LIST, and if there are mails, they do RETR and DELE. Most of time, there is > no mail (for 138770 pop connections, there was no mail 92498 times => 66.6%). > Without the patch, the seen, index, cache and header files are opened. With > this patch, we only read statuscache.db (which is already opened) when there > is no mail. > > On the stat image joined, you can see what's happening when we empty > statuscache.db (at 10:51) : pop optimization doesn't work the first time a > client connects to pop (Lots of reads), and then, as the same clients connect > again to pop, reads slowly decrease. Without the patch, reads would stay high. > > -- > Cyril Servant > > > Ce message et les pi?ces jointes sont confidentiels et r?serv?s ? l'usage > exclusif de ses destinataires. Il peut ?galement ?tre prot?g? par le secret > professionnel. Si vous recevez ce message par erreur, merci d'en avertir > imm?diatement l'exp?diteur et de le d?truire. L'int?grit? du message ne > pouvant ?tre assur?e sur Internet, la responsabilit? du groupe Atos Origin ne > pourra ?tre recherch?e quant au contenu de ce message. Bien que les meilleurs > efforts soient faits pour maintenir cette transmission exempte de tout virus, > l'exp?diteur ne donne aucune garantie ? cet ?gard et sa responsabilit? ne > saurait ?tre recherch?e pour tout dommage r?sultant d'un virus transmis. > > This e-mail and the documents attached are confidential and intended solely > for the addressee; it may also be privileged. If you receive this e-mail in > error, please notify the sender immediately and destroy it. As its integrity > cannot be secured on the Internet, the Atos Origin group liability cannot be > triggered for the message content. Although the sender endeavours to maintain > a computer virus-free network, the sender does not warrant that this > transmission is virus-free and will not be liable for any damages resulting > from any virus transmitted. > -- Cyril
diff -u -r cyrus-imapd-2.3.15.orig/imap/index.c cyrus-imapd-2.3.15/imap/index.c --- cyrus-imapd-2.3.15.orig/imap/index.c 2009-09-09 03:22:38.000000000 +0200 +++ cyrus-imapd-2.3.15/imap/index.c 2009-11-19 11:56:57.000000000 +0100 @@ -5511,3 +5511,116 @@ l = n; } } + +int index_statuscache(char *mboxname, char *name, struct auth_state *authstate, unsigned statusitems, struct statuscache_data *scdata) +{ + int r; + struct mailbox mailbox; + int doclose = 0; + int num_recent = 0; + int num_unseen = 0; + int sepchar; + static struct seq_set seq_set = { NULL, 0, 0, 0 , NULL}; + + /* Check status cache if possible */ + if (config_getswitch(IMAPOPT_STATUSCACHE)) { + /* Do actual lookup of cache item. */ + r = statuscache_lookup(mboxname, name, statusitems, scdata); + + /* Seen/recent status uses "push" invalidation events from + * seen_db.c. This avoids needing to open cyrus.header to get + * the mailbox uniqueid to open the seen db and get the + * unseen_mtime and recentuid. */ + + if (!r) { + syslog(LOG_DEBUG, "statuscache, '%s', '%s', '0x%02x', 'yes'", + mboxname, name, statusitems); + goto statusdone; + } + + syslog(LOG_DEBUG, "statuscache, '%s', '%s', '0x%02x', 'no'", + mboxname, name, statusitems); + } + + /* Missing or invalid cache entry */ + r = mailbox_open_header(mboxname, authstate, &mailbox); + + if (!r) { + doclose = 1; + r = mailbox_open_index(&mailbox); + } + + if (!r && mailbox.exists != 0 && + (statusitems & (STATUS_RECENT | STATUS_UNSEEN))) { + /* Read \Seen state */ + struct seen *status_seendb; + time_t last_read, last_change = 0; + unsigned last_uid; + char *last_seenuids; + + r = seen_open(&mailbox, + (mailbox.options & OPT_IMAP_SHAREDSEEN) ? "anyone" : + name, + SEEN_CREATE, &status_seendb); + + if (!r) { + r = seen_lockread(status_seendb, &last_read, &last_uid, + &last_change, &last_seenuids); + seen_close(status_seendb); + } + + if (!r) { + const char *base; + unsigned long len = 0; + unsigned msg, uid; + + map_refresh(mailbox.index_fd, 0, &base, &len, + mailbox.start_offset + + mailbox.exists * mailbox.record_size, + "index", mailbox.name); + + seq_set.len = seq_set.mark = 0; + index_parse_sequence(last_seenuids, 0, &seq_set); + + for (msg = 0; msg < mailbox.exists; msg++) { + uid = ntohl(*((bit32 *)(base + mailbox.start_offset + + msg * mailbox.record_size + + OFFSET_UID))); + /* Always calculate num_recent, + * even if only need num_unseen... for caching below */ + if (uid > last_uid) num_recent++; + if ((statusitems & STATUS_UNSEEN) && + !index_insequence(uid, &seq_set, 1)) num_unseen++; + /* NB: The value of the third argument to index_insequence() + * above does not matter. */ + } + map_free(&base, &len); + free(last_seenuids); + } + } + + if (!r) { + /* We always have message count, uidnext, + * uidvalidity, and highestmodseq for cache */ + unsigned c_statusitems = statusitems | STATUS_MESSAGES | + STATUS_UIDNEXT | STATUS_UIDVALIDITY | STATUS_HIGHESTMODSEQ; + + /* If we calculated num_unseen, we implicitly calculated num_recent */ + if (c_statusitems & STATUS_UNSEEN) c_statusitems |= STATUS_RECENT; + + statuscache_fill(scdata, &mailbox, + c_statusitems, num_recent, num_unseen); + } + + if (doclose) mailbox_close(&mailbox); + if (r) return r; + + /* Upate the statuscache entry */ + if (config_getswitch(IMAPOPT_STATUSCACHE)) { + statuscache_update(mboxname, name, scdata); + } + +statusdone: + return 0; +} + diff -u -r cyrus-imapd-2.3.15.orig/imap/index.h cyrus-imapd-2.3.15/imap/index.h --- cyrus-imapd-2.3.15.orig/imap/index.h 2009-08-28 15:48:46.000000000 +0200 +++ cyrus-imapd-2.3.15/imap/index.h 2009-11-18 14:58:05.000000000 +0100 @@ -187,4 +187,6 @@ extern struct seq_set *index_parse_sequence(const char *sequence, int usinguid, struct seq_set *set); +#include "statuscache.h" +int index_statuscache(char *mboxname, char *name, struct auth_state *authstate, unsigned statusitems, struct statuscache_data *scdata); #endif /* INDEX_H */ diff -u -r cyrus-imapd-2.3.15.orig/imap/Makefile.in cyrus-imapd-2.3.15/imap/Makefile.in --- cyrus-imapd-2.3.15.orig/imap/Makefile.in 2009-03-30 18:04:56.000000000 +0200 +++ cyrus-imapd-2.3.15/imap/Makefile.in 2009-11-18 15:13:10.000000000 +0100 @@ -227,10 +227,10 @@ $(SERVICETHREAD) mupdate.o mupdate-slave.o mupdate-client.o \ mutex_pthread.o libimap.a $(DEPLIBS) $(LIBS) $(LIB_WRAP) -lpthread -pop3d: pop3d.o proxy.o backend.o tls.o mutex_fake.o libimap.a \ +pop3d: pop3d.o proxy.o backend.o tls.o mutex_fake.o index.o libimap.a \ $(DEPLIBS) $(SERVICE) $(CC) $(LDFLAGS) -o pop3d pop3d.o proxy.o backend.o tls.o $(SERVICE) \ - mutex_fake.o libimap.a $(DEPLIBS) $(LIBS) $(LIB_WRAP) + mutex_fake.o index.o libimap.a $(DEPLIBS) $(LIBS) $(LIB_WRAP) nntpd: nntpd.o proxy.o backend.o index.o smtpclient.o spool.o tls.o \ mutex_fake.o nntp_err.o libimap.a $(DEPLIBS) $(SERVICE) diff -u -r cyrus-imapd-2.3.15.orig/imap/pop3d.c cyrus-imapd-2.3.15/imap/pop3d.c --- cyrus-imapd-2.3.15.orig/imap/pop3d.c 2009-04-23 19:10:07.000000000 +0200 +++ cyrus-imapd-2.3.15/imap/pop3d.c 2009-11-18 15:00:20.000000000 +0100 @@ -89,6 +89,23 @@ #include "sync_log.h" +#include "imapd.h" +#include "index.h" + +int imapd_exists = -1; +char *imapd_userid = NULL; +int imapd_condstore_client = 0; + +struct protstream *imapd_out = NULL; +struct auth_state *imapd_authstate = 0; + +void printastring(const char *s){} + +#include "statuscache.h" + +unsigned short enterLoopNoMails; +static void cmdLoopNoMails(void); + #ifdef HAVE_KRB /* kerberos des is purported to conflict with OpenSSL DES */ #define DES_DEFS @@ -454,6 +471,10 @@ } } + if (config_getswitch(IMAPOPT_STATUSCACHE)) { + statuscache_open(NULL); + } + return 0; } @@ -562,6 +583,8 @@ cmdloop(); + /* If mailbox is empty, we enter in a new loop where all answers are predefined for an empty mailbox */ + if (enterLoopNoMails) cmdLoopNoMails(); /* QUIT executed */ /* don't bother reusing KPOP connections */ @@ -608,6 +631,10 @@ free(backend); } + if (config_getswitch(IMAPOPT_STATUSCACHE)) { + statuscache_close(); + } + mboxlist_close(); mboxlist_done(); @@ -761,6 +788,8 @@ char *p, *arg; unsigned msg = 0; + enterLoopNoMails = 0; + for (;;) { signals_poll(); @@ -1048,6 +1077,7 @@ else { prot_printf(popd_out, "-ERR Unrecognized command\r\n"); } + if ( enterLoopNoMails == 1) break; } } @@ -1559,6 +1589,7 @@ char *server = NULL, *acl; int r, log_level = LOG_ERR; const char *statusline = NULL; + struct statuscache_data scdata; /* Translate any separators in userid (use a copy since we need the original userid for AUTH to backend) */ @@ -1594,6 +1625,16 @@ goto fail; } + if (config_getswitch(IMAPOPT_STATUSCACHE)) { + //r = statuscache_lookup(inboxname, userid, STATUS_UNSEEN, &scdata); + r = index_statuscache(inboxname, popd_userid, popd_authstate, STATUS_UNSEEN|STATUS_MESSAGES|STATUS_RECENT|STATUS_UIDNEXT|STATUS_UIDVALIDITY, &scdata); + if(!r && scdata.messages == 0){ + prot_printf( popd_out, "+OK Maildrop locked and ready\r\n"); + enterLoopNoMails = 1; + return 0; + } + } + if (type & MBTYPE_REMOTE) { /* remote mailbox */ @@ -1898,3 +1939,187 @@ fatal("printstring() executed, but its not used for POP3!", EC_SOFTWARE); } + +/* Second Top-level command loop parsing used when there are no messages */ +static void cmdLoopNoMails(void) +{ + char inputbuf[8192]; + char *p, *arg; + unsigned msg = 0; + + syslog(LOG_INFO, "optipop mode for %s", popd_userid); + for (;;) + { + signals_poll(); + + if (!prot_fgets(inputbuf, sizeof(inputbuf), popd_in)) + shut_down(0); + + p = inputbuf + strlen(inputbuf); + if (p > inputbuf && p[-1] == '\n') *--p = '\0'; + if (p > inputbuf && p[-1] == '\r') *--p = '\0'; + + /* Parse into keword and argument */ + for (p = inputbuf; *p && !isspace((int) *p); p++); + if (*p) + { + *p++ = '\0'; + arg = p; + if (strcasecmp(inputbuf, "pass") != 0) + { + while (*arg && isspace((int) *arg)) + arg++; + } + if (!*arg) + { + prot_printf(popd_out, "-ERR Syntax error\r\n"); + continue; + } + } + else + arg = 0; + + lcase(inputbuf); + + if (!strcmp(inputbuf, "quit")) + { + if (!arg) + { + prot_printf(popd_out, "+OK\r\n"); + return; + } + else prot_printf(popd_out, "-ERR Unexpected extra argument\r\n"); + } + else if (!strcmp(inputbuf, "unse")) + { + if (!arg) + prot_printf(popd_out, "-ERR Missing argument\r\n"); + //else + //cmd_unseen(popd_out, popd_userid, arg); + } + else if (!strcmp(inputbuf, "capa")) + { + if (arg) + prot_printf(popd_out, "-ERR Unexpected extra argument\r\n"); + else + cmd_capa(); + } + else if (!strcmp(inputbuf, "stls")) + prot_printf(popd_out, "-ERR Unrecognized command\r\n"); + else if (!strcmp(inputbuf, "user")) + prot_printf(popd_out, "-ERR Unrecognized command\r\n"); + else if (!strcmp(inputbuf, "pass")) + prot_printf(popd_out, "-ERR Unrecognized command\r\n"); + else if (!strcmp(inputbuf, "apop")) + prot_printf(popd_out, "-ERR Unrecognized command\r\n"); + else if (!strcmp(inputbuf, "auth")) + prot_printf(popd_out, "-ERR Unrecognized command\r\n"); + else if (!strcmp(inputbuf, "stat")) + { + if (arg) + prot_printf(popd_out, "-ERR Unexpected extra argument\r\n"); + else + prot_printf(popd_out, "+OK 0 0\r\n"); + } + else if (!strcmp(inputbuf, "list")) + { + if (arg) + { + msg = parsenum(&arg); + if (arg) + prot_printf(popd_out, "-ERR Unexpected extra argument\r\n"); + else + prot_printf(popd_out, "-ERR No such message\r\n"); + } + else + { + prot_printf(popd_out, "+OK scan listing follows\r\n"); + prot_printf(popd_out, ".\r\n"); + } + } + else if (!strcmp(inputbuf, "retr")) + { + if (!arg) + prot_printf(popd_out, "-ERR Missing argument\r\n"); + else + { + msg = parsenum(&arg); + if (arg) + prot_printf(popd_out, "-ERR Unexpected extra argument\r\n"); + else + prot_printf(popd_out, "-ERR No such message\r\n"); + } + } + else if (!strcmp(inputbuf, "dele")) + { + if (!arg) + prot_printf(popd_out, "-ERR Missing argument\r\n"); + else + { + msg = parsenum(&arg); + if (arg) + prot_printf(popd_out, "-ERR Unexpected extra argument\r\n"); + else + prot_printf(popd_out, "-ERR No such message\r\n"); + } + } + else if (!strcmp(inputbuf, "noop")) + { + if (arg) + prot_printf(popd_out, "-ERR Unexpected extra argument\r\n"); + else + prot_printf(popd_out, "+OK\r\n"); + } + else if (!strcmp(inputbuf, "last")) + { + if (arg) + prot_printf(popd_out, "-ERR Unexpected extra argument\r\n"); + else + prot_printf(popd_out, "+OK 0\r\n"); + } + else if (!strcmp(inputbuf, "rset")) + { + if (arg) + prot_printf(popd_out, "-ERR Unexpected extra argument\r\n"); + else + prot_printf(popd_out, "+OK\r\n"); + } + else if (!strcmp(inputbuf, "top")) + { + if (arg) + msg = parsenum(&arg); + + if (!arg) + prot_printf(popd_out, "-ERR Missing argument\r\n"); + else + { + msg = parsenum(&arg); + if (arg) + prot_printf(popd_out, "-ERR Unexpected extra argument\r\n"); + else + prot_printf(popd_out, "-ERR No such message\r\n"); + } + } + else if (!strcmp(inputbuf, "uidl")) + { + if (arg) + { + msg = parsenum(&arg); + if (arg) + prot_printf(popd_out, "-ERR Unexpected extra argument\r\n"); + else + prot_printf(popd_out, "-ERR No such message\r\n"); + } + else + { + prot_printf(popd_out, "+OK unique-id listing follows\r\n"); + prot_printf(popd_out, ".\r\n"); + } + } + else + { + prot_printf(popd_out, "-ERR Unrecognized command\r\n"); + } + } +} +