Hello, Here is a patch for optimizing pop. Let me explain : Here we have lots (millions) of mailboxes. Many people connect to pop every few minutes, doing LIST, and if there are mails, they do RETR and DELE. Most of time, there is no mail (for 138770 pop connections, there was no mail 92498 times => 66.6%). Without the patch, the seen, index, cache and header files are opened. With this patch, we only read statuscache.db (which is already opened) when there is no mail.
On the stat image joined, you can see what's happening when we empty statuscache.db (at 10:51) : pop optimization doesn't work the first time a client connects to pop (Lots of reads), and then, as the same clients connect again to pop, reads slowly decrease. Without the patch, reads would stay high. -- Cyril Servant Ce message et les pi?ces jointes sont confidentiels et r?serv?s ? l'usage exclusif de ses destinataires. Il peut ?galement ?tre prot?g? par le secret professionnel. Si vous recevez ce message par erreur, merci d'en avertir imm?diatement l'exp?diteur et de le d?truire. L'int?grit? du message ne pouvant ?tre assur?e sur Internet, la responsabilit? du groupe Atos Origin ne pourra ?tre recherch?e quant au contenu de ce message. Bien que les meilleurs efforts soient faits pour maintenir cette transmission exempte de tout virus, l'exp?diteur ne donne aucune garantie ? cet ?gard et sa responsabilit? ne saurait ?tre recherch?e pour tout dommage r?sultant d'un virus transmis. This e-mail and the documents attached are confidential and intended solely for the addressee; it may also be privileged. If you receive this e-mail in error, please notify the sender immediately and destroy it. As its integrity cannot be secured on the Internet, the Atos Origin group liability cannot be triggered for the message content. Although the sender endeavours to maintain a computer virus-free network, the sender does not warrant that this transmission is virus-free and will not be liable for any damages resulting from any virus transmitted.
diff -u -r cyrus-imapd-2.3.15.orig/imap/index.c cyrus-imapd-2.3.15/imap/index.c
--- cyrus-imapd-2.3.15.orig/imap/index.c 2009-09-09 03:22:38.000000000 +0200
+++ cyrus-imapd-2.3.15/imap/index.c 2009-11-19 11:56:57.000000000 +0100
@@ -5511,3 +5511,116 @@
l = n;
}
}
+
+int index_statuscache(char *mboxname, char *name, struct auth_state *authstate, unsigned statusitems, struct statuscache_data *scdata)
+{
+ int r;
+ struct mailbox mailbox;
+ int doclose = 0;
+ int num_recent = 0;
+ int num_unseen = 0;
+ int sepchar;
+ static struct seq_set seq_set = { NULL, 0, 0, 0 , NULL};
+
+ /* Check status cache if possible */
+ if (config_getswitch(IMAPOPT_STATUSCACHE)) {
+ /* Do actual lookup of cache item. */
+ r = statuscache_lookup(mboxname, name, statusitems, scdata);
+
+ /* Seen/recent status uses "push" invalidation events from
+ * seen_db.c. This avoids needing to open cyrus.header to get
+ * the mailbox uniqueid to open the seen db and get the
+ * unseen_mtime and recentuid. */
+
+ if (!r) {
+ syslog(LOG_DEBUG, "statuscache, '%s', '%s', '0x%02x', 'yes'",
+ mboxname, name, statusitems);
+ goto statusdone;
+ }
+
+ syslog(LOG_DEBUG, "statuscache, '%s', '%s', '0x%02x', 'no'",
+ mboxname, name, statusitems);
+ }
+
+ /* Missing or invalid cache entry */
+ r = mailbox_open_header(mboxname, authstate, &mailbox);
+
+ if (!r) {
+ doclose = 1;
+ r = mailbox_open_index(&mailbox);
+ }
+
+ if (!r && mailbox.exists != 0 &&
+ (statusitems & (STATUS_RECENT | STATUS_UNSEEN))) {
+ /* Read \Seen state */
+ struct seen *status_seendb;
+ time_t last_read, last_change = 0;
+ unsigned last_uid;
+ char *last_seenuids;
+
+ r = seen_open(&mailbox,
+ (mailbox.options & OPT_IMAP_SHAREDSEEN) ? "anyone" :
+ name,
+ SEEN_CREATE, &status_seendb);
+
+ if (!r) {
+ r = seen_lockread(status_seendb, &last_read, &last_uid,
+ &last_change, &last_seenuids);
+ seen_close(status_seendb);
+ }
+
+ if (!r) {
+ const char *base;
+ unsigned long len = 0;
+ unsigned msg, uid;
+
+ map_refresh(mailbox.index_fd, 0, &base, &len,
+ mailbox.start_offset +
+ mailbox.exists * mailbox.record_size,
+ "index", mailbox.name);
+
+ seq_set.len = seq_set.mark = 0;
+ index_parse_sequence(last_seenuids, 0, &seq_set);
+
+ for (msg = 0; msg < mailbox.exists; msg++) {
+ uid = ntohl(*((bit32 *)(base + mailbox.start_offset +
+ msg * mailbox.record_size +
+ OFFSET_UID)));
+ /* Always calculate num_recent,
+ * even if only need num_unseen... for caching below */
+ if (uid > last_uid) num_recent++;
+ if ((statusitems & STATUS_UNSEEN) &&
+ !index_insequence(uid, &seq_set, 1)) num_unseen++;
+ /* NB: The value of the third argument to index_insequence()
+ * above does not matter. */
+ }
+ map_free(&base, &len);
+ free(last_seenuids);
+ }
+ }
+
+ if (!r) {
+ /* We always have message count, uidnext,
+ * uidvalidity, and highestmodseq for cache */
+ unsigned c_statusitems = statusitems | STATUS_MESSAGES |
+ STATUS_UIDNEXT | STATUS_UIDVALIDITY | STATUS_HIGHESTMODSEQ;
+
+ /* If we calculated num_unseen, we implicitly calculated num_recent */
+ if (c_statusitems & STATUS_UNSEEN) c_statusitems |= STATUS_RECENT;
+
+ statuscache_fill(scdata, &mailbox,
+ c_statusitems, num_recent, num_unseen);
+ }
+
+ if (doclose) mailbox_close(&mailbox);
+ if (r) return r;
+
+ /* Upate the statuscache entry */
+ if (config_getswitch(IMAPOPT_STATUSCACHE)) {
+ statuscache_update(mboxname, name, scdata);
+ }
+
+statusdone:
+ return 0;
+}
+
diff -u -r cyrus-imapd-2.3.15.orig/imap/index.h cyrus-imapd-2.3.15/imap/index.h
--- cyrus-imapd-2.3.15.orig/imap/index.h 2009-08-28 15:48:46.000000000 +0200
+++ cyrus-imapd-2.3.15/imap/index.h 2009-11-18 14:58:05.000000000 +0100
@@ -187,4 +187,6 @@
extern struct seq_set *index_parse_sequence(const char *sequence, int usinguid,
struct seq_set *set);
+#include "statuscache.h"
+int index_statuscache(char *mboxname, char *name, struct auth_state *authstate, unsigned statusitems, struct statuscache_data *scdata);
#endif /* INDEX_H */
diff -u -r cyrus-imapd-2.3.15.orig/imap/Makefile.in cyrus-imapd-2.3.15/imap/Makefile.in
--- cyrus-imapd-2.3.15.orig/imap/Makefile.in 2009-03-30 18:04:56.000000000 +0200
+++ cyrus-imapd-2.3.15/imap/Makefile.in 2009-11-18 15:13:10.000000000 +0100
@@ -227,10 +227,10 @@
$(SERVICETHREAD) mupdate.o mupdate-slave.o mupdate-client.o \
mutex_pthread.o libimap.a $(DEPLIBS) $(LIBS) $(LIB_WRAP) -lpthread
-pop3d: pop3d.o proxy.o backend.o tls.o mutex_fake.o libimap.a \
+pop3d: pop3d.o proxy.o backend.o tls.o mutex_fake.o index.o libimap.a \
$(DEPLIBS) $(SERVICE)
$(CC) $(LDFLAGS) -o pop3d pop3d.o proxy.o backend.o tls.o $(SERVICE) \
- mutex_fake.o libimap.a $(DEPLIBS) $(LIBS) $(LIB_WRAP)
+ mutex_fake.o index.o libimap.a $(DEPLIBS) $(LIBS) $(LIB_WRAP)
nntpd: nntpd.o proxy.o backend.o index.o smtpclient.o spool.o tls.o \
mutex_fake.o nntp_err.o libimap.a $(DEPLIBS) $(SERVICE)
diff -u -r cyrus-imapd-2.3.15.orig/imap/pop3d.c cyrus-imapd-2.3.15/imap/pop3d.c
--- cyrus-imapd-2.3.15.orig/imap/pop3d.c 2009-04-23 19:10:07.000000000 +0200
+++ cyrus-imapd-2.3.15/imap/pop3d.c 2009-11-18 15:00:20.000000000 +0100
@@ -89,6 +89,23 @@
#include "sync_log.h"
+#include "imapd.h"
+#include "index.h"
+
+int imapd_exists = -1;
+char *imapd_userid = NULL;
+int imapd_condstore_client = 0;
+
+struct protstream *imapd_out = NULL;
+struct auth_state *imapd_authstate = 0;
+
+void printastring(const char *s){}
+
+#include "statuscache.h"
+
+unsigned short enterLoopNoMails;
+static void cmdLoopNoMails(void);
+
#ifdef HAVE_KRB
/* kerberos des is purported to conflict with OpenSSL DES */
#define DES_DEFS
@@ -454,6 +471,10 @@
}
}
+ if (config_getswitch(IMAPOPT_STATUSCACHE)) {
+ statuscache_open(NULL);
+ }
+
return 0;
}
@@ -562,6 +583,8 @@
cmdloop();
+ /* If mailbox is empty, we enter in a new loop where all answers are predefined for an empty mailbox */
+ if (enterLoopNoMails) cmdLoopNoMails();
/* QUIT executed */
/* don't bother reusing KPOP connections */
@@ -608,6 +631,10 @@
free(backend);
}
+ if (config_getswitch(IMAPOPT_STATUSCACHE)) {
+ statuscache_close();
+ }
+
mboxlist_close();
mboxlist_done();
@@ -761,6 +788,8 @@
char *p, *arg;
unsigned msg = 0;
+ enterLoopNoMails = 0;
+
for (;;) {
signals_poll();
@@ -1048,6 +1077,7 @@
else {
prot_printf(popd_out, "-ERR Unrecognized command\r\n");
}
+ if ( enterLoopNoMails == 1) break;
}
}
@@ -1559,6 +1589,7 @@
char *server = NULL, *acl;
int r, log_level = LOG_ERR;
const char *statusline = NULL;
+ struct statuscache_data scdata;
/* Translate any separators in userid
(use a copy since we need the original userid for AUTH to backend) */
@@ -1594,6 +1625,16 @@
goto fail;
}
+ if (config_getswitch(IMAPOPT_STATUSCACHE)) {
+ //r = statuscache_lookup(inboxname, userid, STATUS_UNSEEN, &scdata);
+ r = index_statuscache(inboxname, popd_userid, popd_authstate, STATUS_UNSEEN|STATUS_MESSAGES|STATUS_RECENT|STATUS_UIDNEXT|STATUS_UIDVALIDITY, &scdata);
+ if(!r && scdata.unseen == 0){
+ prot_printf( popd_out, "+OK Maildrop locked and ready\r\n");
+ enterLoopNoMails = 1;
+ return 0;
+ }
+ }
+
if (type & MBTYPE_REMOTE) {
/* remote mailbox */
@@ -1898,3 +1939,187 @@
fatal("printstring() executed, but its not used for POP3!",
EC_SOFTWARE);
}
+
+/* Second Top-level command loop parsing used when there are no messages */
+static void cmdLoopNoMails(void)
+{
+ char inputbuf[8192];
+ char *p, *arg;
+ unsigned msg = 0;
+
+ syslog(LOG_INFO, "optipop mode for %s", popd_userid);
+ for (;;)
+ {
+ signals_poll();
+
+ if (!prot_fgets(inputbuf, sizeof(inputbuf), popd_in))
+ shut_down(0);
+
+ p = inputbuf + strlen(inputbuf);
+ if (p > inputbuf && p[-1] == '\n') *--p = '\0';
+ if (p > inputbuf && p[-1] == '\r') *--p = '\0';
+
+ /* Parse into keword and argument */
+ for (p = inputbuf; *p && !isspace((int) *p); p++);
+ if (*p)
+ {
+ *p++ = '\0';
+ arg = p;
+ if (strcasecmp(inputbuf, "pass") != 0)
+ {
+ while (*arg && isspace((int) *arg))
+ arg++;
+ }
+ if (!*arg)
+ {
+ prot_printf(popd_out, "-ERR Syntax error\r\n");
+ continue;
+ }
+ }
+ else
+ arg = 0;
+
+ lcase(inputbuf);
+
+ if (!strcmp(inputbuf, "quit"))
+ {
+ if (!arg)
+ {
+ prot_printf(popd_out, "+OK\r\n");
+ return;
+ }
+ else prot_printf(popd_out, "-ERR Unexpected extra argument\r\n");
+ }
+ else if (!strcmp(inputbuf, "unse"))
+ {
+ if (!arg)
+ prot_printf(popd_out, "-ERR Missing argument\r\n");
+ //else
+ //cmd_unseen(popd_out, popd_userid, arg);
+ }
+ else if (!strcmp(inputbuf, "capa"))
+ {
+ if (arg)
+ prot_printf(popd_out, "-ERR Unexpected extra argument\r\n");
+ else
+ cmd_capa();
+ }
+ else if (!strcmp(inputbuf, "stls"))
+ prot_printf(popd_out, "-ERR Unrecognized command\r\n");
+ else if (!strcmp(inputbuf, "user"))
+ prot_printf(popd_out, "-ERR Unrecognized command\r\n");
+ else if (!strcmp(inputbuf, "pass"))
+ prot_printf(popd_out, "-ERR Unrecognized command\r\n");
+ else if (!strcmp(inputbuf, "apop"))
+ prot_printf(popd_out, "-ERR Unrecognized command\r\n");
+ else if (!strcmp(inputbuf, "auth"))
+ prot_printf(popd_out, "-ERR Unrecognized command\r\n");
+ else if (!strcmp(inputbuf, "stat"))
+ {
+ if (arg)
+ prot_printf(popd_out, "-ERR Unexpected extra argument\r\n");
+ else
+ prot_printf(popd_out, "+OK 0 0\r\n");
+ }
+ else if (!strcmp(inputbuf, "list"))
+ {
+ if (arg)
+ {
+ msg = parsenum(&arg);
+ if (arg)
+ prot_printf(popd_out, "-ERR Unexpected extra argument\r\n");
+ else
+ prot_printf(popd_out, "-ERR No such message\r\n");
+ }
+ else
+ {
+ prot_printf(popd_out, "+OK scan listing follows\r\n");
+ prot_printf(popd_out, ".\r\n");
+ }
+ }
+ else if (!strcmp(inputbuf, "retr"))
+ {
+ if (!arg)
+ prot_printf(popd_out, "-ERR Missing argument\r\n");
+ else
+ {
+ msg = parsenum(&arg);
+ if (arg)
+ prot_printf(popd_out, "-ERR Unexpected extra argument\r\n");
+ else
+ prot_printf(popd_out, "-ERR No such message\r\n");
+ }
+ }
+ else if (!strcmp(inputbuf, "dele"))
+ {
+ if (!arg)
+ prot_printf(popd_out, "-ERR Missing argument\r\n");
+ else
+ {
+ msg = parsenum(&arg);
+ if (arg)
+ prot_printf(popd_out, "-ERR Unexpected extra argument\r\n");
+ else
+ prot_printf(popd_out, "-ERR No such message\r\n");
+ }
+ }
+ else if (!strcmp(inputbuf, "noop"))
+ {
+ if (arg)
+ prot_printf(popd_out, "-ERR Unexpected extra argument\r\n");
+ else
+ prot_printf(popd_out, "+OK\r\n");
+ }
+ else if (!strcmp(inputbuf, "last"))
+ {
+ if (arg)
+ prot_printf(popd_out, "-ERR Unexpected extra argument\r\n");
+ else
+ prot_printf(popd_out, "+OK 0\r\n");
+ }
+ else if (!strcmp(inputbuf, "rset"))
+ {
+ if (arg)
+ prot_printf(popd_out, "-ERR Unexpected extra argument\r\n");
+ else
+ prot_printf(popd_out, "+OK\r\n");
+ }
+ else if (!strcmp(inputbuf, "top"))
+ {
+ if (arg)
+ msg = parsenum(&arg);
+
+ if (!arg)
+ prot_printf(popd_out, "-ERR Missing argument\r\n");
+ else
+ {
+ msg = parsenum(&arg);
+ if (arg)
+ prot_printf(popd_out, "-ERR Unexpected extra argument\r\n");
+ else
+ prot_printf(popd_out, "-ERR No such message\r\n");
+ }
+ }
+ else if (!strcmp(inputbuf, "uidl"))
+ {
+ if (arg)
+ {
+ msg = parsenum(&arg);
+ if (arg)
+ prot_printf(popd_out, "-ERR Unexpected extra argument\r\n");
+ else
+ prot_printf(popd_out, "-ERR No such message\r\n");
+ }
+ else
+ {
+ prot_printf(popd_out, "+OK unique-id listing follows\r\n");
+ prot_printf(popd_out, ".\r\n");
+ }
+ }
+ else
+ {
+ prot_printf(popd_out, "-ERR Unrecognized command\r\n");
+ }
+ }
+}
+
<<attachment: statNFS.png>>
