Bron, Ken, did you have time to take a look at this patch ?
By the way, here is an updated patch.
--
Cyril Servant
2009/11/19 Servant Cyril <[email protected]>:
> Hello,
>
> Here is a patch for optimizing pop. Let me explain : Here we have lots
> (millions) of mailboxes. Many people connect to pop every few minutes, doing
> LIST, and if there are mails, they do RETR and DELE. Most of time, there is
> no mail (for 138770 pop connections, there was no mail 92498 times => 66.6%).
> Without the patch, the seen, index, cache and header files are opened. With
> this patch, we only read statuscache.db (which is already opened) when there
> is no mail.
>
> On the stat image joined, you can see what's happening when we empty
> statuscache.db (at 10:51) : pop optimization doesn't work the first time a
> client connects to pop (Lots of reads), and then, as the same clients connect
> again to pop, reads slowly decrease. Without the patch, reads would stay high.
>
> --
> Cyril Servant
>
>
> Ce message et les pi?ces jointes sont confidentiels et r?serv?s ? l'usage
> exclusif de ses destinataires. Il peut ?galement ?tre prot?g? par le secret
> professionnel. Si vous recevez ce message par erreur, merci d'en avertir
> imm?diatement l'exp?diteur et de le d?truire. L'int?grit? du message ne
> pouvant ?tre assur?e sur Internet, la responsabilit? du groupe Atos Origin ne
> pourra ?tre recherch?e quant au contenu de ce message. Bien que les meilleurs
> efforts soient faits pour maintenir cette transmission exempte de tout virus,
> l'exp?diteur ne donne aucune garantie ? cet ?gard et sa responsabilit? ne
> saurait ?tre recherch?e pour tout dommage r?sultant d'un virus transmis.
>
> This e-mail and the documents attached are confidential and intended solely
> for the addressee; it may also be privileged. If you receive this e-mail in
> error, please notify the sender immediately and destroy it. As its integrity
> cannot be secured on the Internet, the Atos Origin group liability cannot be
> triggered for the message content. Although the sender endeavours to maintain
> a computer virus-free network, the sender does not warrant that this
> transmission is virus-free and will not be liable for any damages resulting
> from any virus transmitted.
>
--
Cyril
diff -u -r cyrus-imapd-2.3.15.orig/imap/index.c cyrus-imapd-2.3.15/imap/index.c
--- cyrus-imapd-2.3.15.orig/imap/index.c 2009-09-09 03:22:38.000000000 +0200
+++ cyrus-imapd-2.3.15/imap/index.c 2009-11-19 11:56:57.000000000 +0100
@@ -5511,3 +5511,116 @@
l = n;
}
}
+
+int index_statuscache(char *mboxname, char *name, struct auth_state *authstate, unsigned statusitems, struct statuscache_data *scdata)
+{
+ int r;
+ struct mailbox mailbox;
+ int doclose = 0;
+ int num_recent = 0;
+ int num_unseen = 0;
+ int sepchar;
+ static struct seq_set seq_set = { NULL, 0, 0, 0 , NULL};
+
+ /* Check status cache if possible */
+ if (config_getswitch(IMAPOPT_STATUSCACHE)) {
+ /* Do actual lookup of cache item. */
+ r = statuscache_lookup(mboxname, name, statusitems, scdata);
+
+ /* Seen/recent status uses "push" invalidation events from
+ * seen_db.c. This avoids needing to open cyrus.header to get
+ * the mailbox uniqueid to open the seen db and get the
+ * unseen_mtime and recentuid. */
+
+ if (!r) {
+ syslog(LOG_DEBUG, "statuscache, '%s', '%s', '0x%02x', 'yes'",
+ mboxname, name, statusitems);
+ goto statusdone;
+ }
+
+ syslog(LOG_DEBUG, "statuscache, '%s', '%s', '0x%02x', 'no'",
+ mboxname, name, statusitems);
+ }
+
+ /* Missing or invalid cache entry */
+ r = mailbox_open_header(mboxname, authstate, &mailbox);
+
+ if (!r) {
+ doclose = 1;
+ r = mailbox_open_index(&mailbox);
+ }
+
+ if (!r && mailbox.exists != 0 &&
+ (statusitems & (STATUS_RECENT | STATUS_UNSEEN))) {
+ /* Read \Seen state */
+ struct seen *status_seendb;
+ time_t last_read, last_change = 0;
+ unsigned last_uid;
+ char *last_seenuids;
+
+ r = seen_open(&mailbox,
+ (mailbox.options & OPT_IMAP_SHAREDSEEN) ? "anyone" :
+ name,
+ SEEN_CREATE, &status_seendb);
+
+ if (!r) {
+ r = seen_lockread(status_seendb, &last_read, &last_uid,
+ &last_change, &last_seenuids);
+ seen_close(status_seendb);
+ }
+
+ if (!r) {
+ const char *base;
+ unsigned long len = 0;
+ unsigned msg, uid;
+
+ map_refresh(mailbox.index_fd, 0, &base, &len,
+ mailbox.start_offset +
+ mailbox.exists * mailbox.record_size,
+ "index", mailbox.name);
+
+ seq_set.len = seq_set.mark = 0;
+ index_parse_sequence(last_seenuids, 0, &seq_set);
+
+ for (msg = 0; msg < mailbox.exists; msg++) {
+ uid = ntohl(*((bit32 *)(base + mailbox.start_offset +
+ msg * mailbox.record_size +
+ OFFSET_UID)));
+ /* Always calculate num_recent,
+ * even if only need num_unseen... for caching below */
+ if (uid > last_uid) num_recent++;
+ if ((statusitems & STATUS_UNSEEN) &&
+ !index_insequence(uid, &seq_set, 1)) num_unseen++;
+ /* NB: The value of the third argument to index_insequence()
+ * above does not matter. */
+ }
+ map_free(&base, &len);
+ free(last_seenuids);
+ }
+ }
+
+ if (!r) {
+ /* We always have message count, uidnext,
+ * uidvalidity, and highestmodseq for cache */
+ unsigned c_statusitems = statusitems | STATUS_MESSAGES |
+ STATUS_UIDNEXT | STATUS_UIDVALIDITY | STATUS_HIGHESTMODSEQ;
+
+ /* If we calculated num_unseen, we implicitly calculated num_recent */
+ if (c_statusitems & STATUS_UNSEEN) c_statusitems |= STATUS_RECENT;
+
+ statuscache_fill(scdata, &mailbox,
+ c_statusitems, num_recent, num_unseen);
+ }
+
+ if (doclose) mailbox_close(&mailbox);
+ if (r) return r;
+
+ /* Upate the statuscache entry */
+ if (config_getswitch(IMAPOPT_STATUSCACHE)) {
+ statuscache_update(mboxname, name, scdata);
+ }
+
+statusdone:
+ return 0;
+}
+
diff -u -r cyrus-imapd-2.3.15.orig/imap/index.h cyrus-imapd-2.3.15/imap/index.h
--- cyrus-imapd-2.3.15.orig/imap/index.h 2009-08-28 15:48:46.000000000 +0200
+++ cyrus-imapd-2.3.15/imap/index.h 2009-11-18 14:58:05.000000000 +0100
@@ -187,4 +187,6 @@
extern struct seq_set *index_parse_sequence(const char *sequence, int usinguid,
struct seq_set *set);
+#include "statuscache.h"
+int index_statuscache(char *mboxname, char *name, struct auth_state *authstate, unsigned statusitems, struct statuscache_data *scdata);
#endif /* INDEX_H */
diff -u -r cyrus-imapd-2.3.15.orig/imap/Makefile.in cyrus-imapd-2.3.15/imap/Makefile.in
--- cyrus-imapd-2.3.15.orig/imap/Makefile.in 2009-03-30 18:04:56.000000000 +0200
+++ cyrus-imapd-2.3.15/imap/Makefile.in 2009-11-18 15:13:10.000000000 +0100
@@ -227,10 +227,10 @@
$(SERVICETHREAD) mupdate.o mupdate-slave.o mupdate-client.o \
mutex_pthread.o libimap.a $(DEPLIBS) $(LIBS) $(LIB_WRAP) -lpthread
-pop3d: pop3d.o proxy.o backend.o tls.o mutex_fake.o libimap.a \
+pop3d: pop3d.o proxy.o backend.o tls.o mutex_fake.o index.o libimap.a \
$(DEPLIBS) $(SERVICE)
$(CC) $(LDFLAGS) -o pop3d pop3d.o proxy.o backend.o tls.o $(SERVICE) \
- mutex_fake.o libimap.a $(DEPLIBS) $(LIBS) $(LIB_WRAP)
+ mutex_fake.o index.o libimap.a $(DEPLIBS) $(LIBS) $(LIB_WRAP)
nntpd: nntpd.o proxy.o backend.o index.o smtpclient.o spool.o tls.o \
mutex_fake.o nntp_err.o libimap.a $(DEPLIBS) $(SERVICE)
diff -u -r cyrus-imapd-2.3.15.orig/imap/pop3d.c cyrus-imapd-2.3.15/imap/pop3d.c
--- cyrus-imapd-2.3.15.orig/imap/pop3d.c 2009-04-23 19:10:07.000000000 +0200
+++ cyrus-imapd-2.3.15/imap/pop3d.c 2009-11-18 15:00:20.000000000 +0100
@@ -89,6 +89,23 @@
#include "sync_log.h"
+#include "imapd.h"
+#include "index.h"
+
+int imapd_exists = -1;
+char *imapd_userid = NULL;
+int imapd_condstore_client = 0;
+
+struct protstream *imapd_out = NULL;
+struct auth_state *imapd_authstate = 0;
+
+void printastring(const char *s){}
+
+#include "statuscache.h"
+
+unsigned short enterLoopNoMails;
+static void cmdLoopNoMails(void);
+
#ifdef HAVE_KRB
/* kerberos des is purported to conflict with OpenSSL DES */
#define DES_DEFS
@@ -454,6 +471,10 @@
}
}
+ if (config_getswitch(IMAPOPT_STATUSCACHE)) {
+ statuscache_open(NULL);
+ }
+
return 0;
}
@@ -562,6 +583,8 @@
cmdloop();
+ /* If mailbox is empty, we enter in a new loop where all answers are predefined for an empty mailbox */
+ if (enterLoopNoMails) cmdLoopNoMails();
/* QUIT executed */
/* don't bother reusing KPOP connections */
@@ -608,6 +631,10 @@
free(backend);
}
+ if (config_getswitch(IMAPOPT_STATUSCACHE)) {
+ statuscache_close();
+ }
+
mboxlist_close();
mboxlist_done();
@@ -761,6 +788,8 @@
char *p, *arg;
unsigned msg = 0;
+ enterLoopNoMails = 0;
+
for (;;) {
signals_poll();
@@ -1048,6 +1077,7 @@
else {
prot_printf(popd_out, "-ERR Unrecognized command\r\n");
}
+ if ( enterLoopNoMails == 1) break;
}
}
@@ -1559,6 +1589,7 @@
char *server = NULL, *acl;
int r, log_level = LOG_ERR;
const char *statusline = NULL;
+ struct statuscache_data scdata;
/* Translate any separators in userid
(use a copy since we need the original userid for AUTH to backend) */
@@ -1594,6 +1625,16 @@
goto fail;
}
+ if (config_getswitch(IMAPOPT_STATUSCACHE)) {
+ //r = statuscache_lookup(inboxname, userid, STATUS_UNSEEN, &scdata);
+ r = index_statuscache(inboxname, popd_userid, popd_authstate, STATUS_UNSEEN|STATUS_MESSAGES|STATUS_RECENT|STATUS_UIDNEXT|STATUS_UIDVALIDITY, &scdata);
+ if(!r && scdata.messages == 0){
+ prot_printf( popd_out, "+OK Maildrop locked and ready\r\n");
+ enterLoopNoMails = 1;
+ return 0;
+ }
+ }
+
if (type & MBTYPE_REMOTE) {
/* remote mailbox */
@@ -1898,3 +1939,187 @@
fatal("printstring() executed, but its not used for POP3!",
EC_SOFTWARE);
}
+
+/* Second Top-level command loop parsing used when there are no messages */
+static void cmdLoopNoMails(void)
+{
+ char inputbuf[8192];
+ char *p, *arg;
+ unsigned msg = 0;
+
+ syslog(LOG_INFO, "optipop mode for %s", popd_userid);
+ for (;;)
+ {
+ signals_poll();
+
+ if (!prot_fgets(inputbuf, sizeof(inputbuf), popd_in))
+ shut_down(0);
+
+ p = inputbuf + strlen(inputbuf);
+ if (p > inputbuf && p[-1] == '\n') *--p = '\0';
+ if (p > inputbuf && p[-1] == '\r') *--p = '\0';
+
+ /* Parse into keword and argument */
+ for (p = inputbuf; *p && !isspace((int) *p); p++);
+ if (*p)
+ {
+ *p++ = '\0';
+ arg = p;
+ if (strcasecmp(inputbuf, "pass") != 0)
+ {
+ while (*arg && isspace((int) *arg))
+ arg++;
+ }
+ if (!*arg)
+ {
+ prot_printf(popd_out, "-ERR Syntax error\r\n");
+ continue;
+ }
+ }
+ else
+ arg = 0;
+
+ lcase(inputbuf);
+
+ if (!strcmp(inputbuf, "quit"))
+ {
+ if (!arg)
+ {
+ prot_printf(popd_out, "+OK\r\n");
+ return;
+ }
+ else prot_printf(popd_out, "-ERR Unexpected extra argument\r\n");
+ }
+ else if (!strcmp(inputbuf, "unse"))
+ {
+ if (!arg)
+ prot_printf(popd_out, "-ERR Missing argument\r\n");
+ //else
+ //cmd_unseen(popd_out, popd_userid, arg);
+ }
+ else if (!strcmp(inputbuf, "capa"))
+ {
+ if (arg)
+ prot_printf(popd_out, "-ERR Unexpected extra argument\r\n");
+ else
+ cmd_capa();
+ }
+ else if (!strcmp(inputbuf, "stls"))
+ prot_printf(popd_out, "-ERR Unrecognized command\r\n");
+ else if (!strcmp(inputbuf, "user"))
+ prot_printf(popd_out, "-ERR Unrecognized command\r\n");
+ else if (!strcmp(inputbuf, "pass"))
+ prot_printf(popd_out, "-ERR Unrecognized command\r\n");
+ else if (!strcmp(inputbuf, "apop"))
+ prot_printf(popd_out, "-ERR Unrecognized command\r\n");
+ else if (!strcmp(inputbuf, "auth"))
+ prot_printf(popd_out, "-ERR Unrecognized command\r\n");
+ else if (!strcmp(inputbuf, "stat"))
+ {
+ if (arg)
+ prot_printf(popd_out, "-ERR Unexpected extra argument\r\n");
+ else
+ prot_printf(popd_out, "+OK 0 0\r\n");
+ }
+ else if (!strcmp(inputbuf, "list"))
+ {
+ if (arg)
+ {
+ msg = parsenum(&arg);
+ if (arg)
+ prot_printf(popd_out, "-ERR Unexpected extra argument\r\n");
+ else
+ prot_printf(popd_out, "-ERR No such message\r\n");
+ }
+ else
+ {
+ prot_printf(popd_out, "+OK scan listing follows\r\n");
+ prot_printf(popd_out, ".\r\n");
+ }
+ }
+ else if (!strcmp(inputbuf, "retr"))
+ {
+ if (!arg)
+ prot_printf(popd_out, "-ERR Missing argument\r\n");
+ else
+ {
+ msg = parsenum(&arg);
+ if (arg)
+ prot_printf(popd_out, "-ERR Unexpected extra argument\r\n");
+ else
+ prot_printf(popd_out, "-ERR No such message\r\n");
+ }
+ }
+ else if (!strcmp(inputbuf, "dele"))
+ {
+ if (!arg)
+ prot_printf(popd_out, "-ERR Missing argument\r\n");
+ else
+ {
+ msg = parsenum(&arg);
+ if (arg)
+ prot_printf(popd_out, "-ERR Unexpected extra argument\r\n");
+ else
+ prot_printf(popd_out, "-ERR No such message\r\n");
+ }
+ }
+ else if (!strcmp(inputbuf, "noop"))
+ {
+ if (arg)
+ prot_printf(popd_out, "-ERR Unexpected extra argument\r\n");
+ else
+ prot_printf(popd_out, "+OK\r\n");
+ }
+ else if (!strcmp(inputbuf, "last"))
+ {
+ if (arg)
+ prot_printf(popd_out, "-ERR Unexpected extra argument\r\n");
+ else
+ prot_printf(popd_out, "+OK 0\r\n");
+ }
+ else if (!strcmp(inputbuf, "rset"))
+ {
+ if (arg)
+ prot_printf(popd_out, "-ERR Unexpected extra argument\r\n");
+ else
+ prot_printf(popd_out, "+OK\r\n");
+ }
+ else if (!strcmp(inputbuf, "top"))
+ {
+ if (arg)
+ msg = parsenum(&arg);
+
+ if (!arg)
+ prot_printf(popd_out, "-ERR Missing argument\r\n");
+ else
+ {
+ msg = parsenum(&arg);
+ if (arg)
+ prot_printf(popd_out, "-ERR Unexpected extra argument\r\n");
+ else
+ prot_printf(popd_out, "-ERR No such message\r\n");
+ }
+ }
+ else if (!strcmp(inputbuf, "uidl"))
+ {
+ if (arg)
+ {
+ msg = parsenum(&arg);
+ if (arg)
+ prot_printf(popd_out, "-ERR Unexpected extra argument\r\n");
+ else
+ prot_printf(popd_out, "-ERR No such message\r\n");
+ }
+ else
+ {
+ prot_printf(popd_out, "+OK unique-id listing follows\r\n");
+ prot_printf(popd_out, ".\r\n");
+ }
+ }
+ else
+ {
+ prot_printf(popd_out, "-ERR Unrecognized command\r\n");
+ }
+ }
+}
+
