Some future Dovecot version will have "imap-idle" processes where
IDLEing IMAP connections get moved, so the system wouldn't waste so much
memory for all the IDLEing imap processes. A week ago I thought I'd see
how easy it would be to implement this. I got a basic proof of concept
working as a "X-STATE" command.
Save the state:
a x-state
* STATE AQDLW45QdwAAAAMAAABuAQAAAAAAAFAcffYAPHnpFctbjlDbYQAAcEmzCwAA
a OK State exported.
Restore the state:
b x-state AQDLW45QdwAAAAMAAABuAQAAAAAAAFAcffYAPHnpFctbjlDbYQAAcEmzCwAA
b OK State imported.
This could also be used to implement quick session state restoring for
webmails (as suggested by Michael Slusarz).
For getting the imap-idle process there would have to be code that:
* triggers the session saving when process is IDLEing
* figures out what filesystem paths the imap-idle should be looking at
(i.e. paths to selected mailbox's dovecot.index.log file and maybe for
e.g. maildir new/)
* send the session state string, paths and imap connection fd to
imap-idle process via UNIX socket
* implement the actual imap-idle process
* implement a way for imap-idle process to send back the state and
connection fd to restore the imap process
The patch is ugly and still missing many things. Anyway I thought I'd
include it here just in case someone was really eager to continue
implementing it. :) I'm not sure when I'll have time for it.
A full patch would probably have to have some
session_save()/session_restore() functions in lib-storage API. But a
quick and dirty way is possible to implement for v2.1 as well, as long
as some IMAP extensions aren't used (most importantly rfc5267).
diff -r 67e9cb0b06ec src/imap/Makefile.am
--- a/src/imap/Makefile.am Mon Oct 29 16:36:59 2012 +0200
+++ b/src/imap/Makefile.am Mon Oct 29 18:40:43 2012 +0200
@@ -60,7 +60,8 @@
cmd-uid.c \
cmd-unselect.c \
cmd-unsubscribe.c \
- cmd-x-cancel.c
+ cmd-x-cancel.c \
+ cmd-x-state.c
imap_SOURCES = \
$(cmds) \
@@ -74,6 +75,7 @@
imap-search-args.c \
imap-settings.c \
imap-status.c \
+ imap-state.c \
imap-sync.c \
mail-storage-callbacks.c \
main.c
@@ -90,6 +92,7 @@
imap-search-args.h \
imap-settings.h \
imap-status.h \
+ imap-state.h \
imap-sync.h
pkginc_libdir=$(pkgincludedir)
diff -r 67e9cb0b06ec src/imap/imap-commands.c
--- a/src/imap/imap-commands.c Mon Oct 29 16:36:59 2012 +0200
+++ b/src/imap/imap-commands.c Mon Oct 29 18:40:43 2012 +0200
@@ -60,6 +60,7 @@
{ "UID THREAD", cmd_thread, COMMAND_FLAG_BREAKS_SEQS },
{ "UNSELECT", cmd_unselect, COMMAND_FLAG_BREAKS_MAILBOX },
{ "X-CANCEL", cmd_x_cancel, 0 },
+ { "X-STATE", cmd_x_state, COMMAND_FLAG_REQUIRES_SYNC },
{ "XLIST", cmd_list, 0 }
};
#define IMAP_EXT_COMMANDS_COUNT N_ELEMENTS(imap_ext_commands)
diff -r 67e9cb0b06ec src/imap/imap-commands.h
--- a/src/imap/imap-commands.h Mon Oct 29 16:36:59 2012 +0200
+++ b/src/imap/imap-commands.h Mon Oct 29 18:40:43 2012 +0200
@@ -112,6 +112,7 @@
bool cmd_uid_expunge(struct client_command_context *cmd);
bool cmd_unselect(struct client_command_context *cmd);
bool cmd_x_cancel(struct client_command_context *cmd);
+bool cmd_x_state(struct client_command_context *cmd);
/* private: */
bool cmd_list_full(struct client_command_context *cmd, bool lsub);
diff -r 67e9cb0b06ec src/imap/imap-state.c
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/imap/imap-state.c Mon Oct 29 18:40:43 2012 +0200
@@ -0,0 +1,319 @@
+/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */
+
+#include "imap-common.h"
+#include "crc32.h"
+#include "str.h"
+#include "mail-search-build.h"
+#include "mail-storage-private.h"
+#include "imap-client.h"
+#include "imap-state.h"
+
+enum imap_state_type {
+ IMAP_STATE_TYPE_SESSION_ID,
+ IMAP_STATE_TYPE_MAILBOX
+};
+
+#define BUF_ADD(buf, data) \
+ buffer_append(buf, &(data), sizeof(data))
+
+static bool
+imap_state_export_mailbox(struct client *client, buffer_t *output)
+{
+ struct mailbox_status status;
+ struct mailbox_metadata metadata;
+ struct mail_namespace *ns = mailbox_get_namespace(client->mailbox);
+ struct mailbox_transaction_context *trans;
+ struct mail_search_args *search_args;
+ struct mail_search_context *search_ctx;
+ struct mail *mail;
+ uint32_t crc = 0;
+ bool ret = TRUE;
+
+ if (strcmp(client->mailbox->storage->name, "virtual") == 0 ||
+ strcmp(client->mailbox->storage->name, "imapc") == 0 ||
+ strcmp(client->mailbox->storage->name, "pop3c") == 0)
+ return FALSE;
+
+ if (mailbox_get_metadata(client->mailbox, MAILBOX_METADATA_GUID,
+ &metadata) < 0)
+ return FALSE;
+
+ mailbox_get_open_status(client->mailbox,
+ STATUS_UIDVALIDITY | STATUS_MESSAGES |
+ STATUS_HIGHESTMODSEQ, &status);
+
+ if (status.highest_modseq != client->sync_last_full_modseq)
+ return FALSE;
+
+ buffer_append_c(output, IMAP_STATE_TYPE_MAILBOX);
+ buffer_append_c(output, client->mailbox_examined ? 1 : 0);
+ BUF_ADD(output, status.uidvalidity);
+ BUF_ADD(output, status.uidnext);
+ BUF_ADD(output, status.messages);
+ BUF_ADD(output, client->sync_last_full_modseq);
+
+ /* we're now basically done, but just in case there's a bug add a
+ checksum of the currently existing UIDs and verify it when
+ importing */
+ search_args = mail_search_build_init();
+ mail_search_build_add_all(search_args);
+
+ trans = mailbox_transaction_begin(client->mailbox, 0);
+ search_ctx = mailbox_search_init(trans, search_args, NULL, 0, NULL);
+ mail_search_args_unref(&search_args);
+
+ while (mailbox_search_next(search_ctx, &mail))
+ crc = crc32_data_more(crc, &mail->uid, sizeof(mail->uid));
+ if (mailbox_search_deinit(&search_ctx) < 0)
+ ret = FALSE;
+ (void)mailbox_transaction_commit(&trans);
+
+ BUF_ADD(output, crc);
+ buffer_append(output, ns->prefix, strlen(ns->prefix)+1);
+ BUF_ADD(output, metadata.guid);
+ return ret;
+}
+
+bool imap_state_export(struct client *client, buffer_t *output)
+{
+ struct client_command_context *cmd;
+
+ for (cmd = client->command_queue; cmd != NULL; cmd = cmd->next) {
+ if (strcasecmp(cmd->name, "X-STATE") != 0 &&
+ strcasecmp(cmd->name, "IDLE") != 0)
+ return FALSE;
+ }
+
+ if (client->enabled_features != 0 ||
+ client->tls_compression)
+ return FALSE;
+ if (array_is_created(&client->search_saved_uidset) ||
+ array_is_created(&client->search_updates))
+ return FALSE;
+
+ if (client->mailbox != NULL) {
+ if (!imap_state_export_mailbox(client, output))
+ return FALSE;
+ }
+
+ buffer_append_c(output, IMAP_STATE_TYPE_SESSION_ID);
+ if (client->session_id == NULL)
+ buffer_append_c(output, '\0');
+ else {
+ buffer_append(output, client->session_id,
+ strlen(client->session_id)+1);
+ }
+
+ // FIXME: keywords, recent flags, id_logged
+ return TRUE;
+}
+
+static int
+import_string(const unsigned char **data, size_t *size, const char **str_r)
+{
+ const unsigned char *p;
+ unsigned int len;
+
+ p = memchr(*data, '\0', *size);
+ if (p == NULL)
+ return -1;
+ len = p - *data + 1;
+
+ *str_r = (const void *)*data;
+ *data += len;
+ *size -= len;
+ return 0;
+}
+
+static int
+import_uint8(const unsigned char **data, size_t *size, uint8_t *num_r)
+{
+ if (*size == 0)
+ return -1;
+ *num_r = **data;
+ *data += 1;
+ *size -= 1;
+ return 0;
+}
+
+static int
+import_uint32(const unsigned char **data, size_t *size, uint32_t *num_r)
+{
+ if (*size < sizeof(*num_r))
+ return -1;
+ memcpy(num_r, *data, sizeof(*num_r));
+ *data += sizeof(*num_r);
+ *size -= sizeof(*num_r);
+ return 0;
+}
+
+static int
+import_uint64(const unsigned char **data, size_t *size, uint64_t *num_r)
+{
+ if (*size < sizeof(*num_r))
+ return -1;
+ memcpy(num_r, *data, sizeof(*num_r));
+ *data += sizeof(*num_r);
+ *size -= sizeof(*num_r);
+ return 0;
+}
+
+static int
+send_expunges(struct client *client, uint32_t uidnext,
+ uint64_t modseq, uint32_t orig_crc)
+{
+ struct mailbox_status status;
+ struct mailbox_transaction_context *trans;
+ struct mail_search_args *search_args;
+ struct mail_search_context *search_ctx;
+ struct mail *mail;
+ uint32_t crc = 0, seq, expunged_uid;
+ ARRAY_TYPE(seq_range) uids_filter, expunged_uids;
+ ARRAY_TYPE(uint32_t) expunged_seqs;
+ struct seq_range_iter iter;
+ const uint32_t *seqs;
+ unsigned int i, count, n = 0;
+ string_t *str;
+ int ret = 0;
+
+ mailbox_get_open_status(client->mailbox,
+ STATUS_UIDVALIDITY | STATUS_UIDNEXT |
+ STATUS_MESSAGES | STATUS_KEYWORDS |
+ STATUS_HIGHESTMODSEQ, &status);
+ if (status.uidvalidity != client->uidvalidity)
+ return -1;
+
+ t_array_init(&uids_filter, 1);
+ t_array_init(&expunged_uids, 128);
+ seq_range_array_add_range(&uids_filter, 1, (uint32_t)-1);
+ if (!mailbox_get_expunged_uids(client->mailbox, modseq, &uids_filter,
+ &expunged_uids))
+ return -1;
+ seq_range_array_iter_init(&iter, &expunged_uids);
+
+ search_args = mail_search_build_init();
+ mail_search_build_add_all(search_args);
+
+ trans = mailbox_transaction_begin(client->mailbox, 0);
+ search_ctx = mailbox_search_init(trans, search_args, NULL, 0, NULL);
+ mail_search_args_unref(&search_args);
+
+ t_array_init(&expunged_seqs, array_count(&expunged_uids)+1);
+ while (mailbox_search_next(search_ctx, &mail)) {
+ while (seq_range_array_iter_nth(&iter, n, &expunged_uid) &&
+ expunged_uid < mail->uid) {
+ seq = mail->seq+n;
+ array_append(&expunged_seqs, &seq, 1);
+ crc = crc32_data_more(crc, &expunged_uid,
+ sizeof(expunged_uid));
+ n++;
+ }
+ crc = crc32_data_more(crc, &mail->uid, sizeof(mail->uid));
+ }
+ while (seq_range_array_iter_nth(&iter, n, &expunged_uid)) {
+ seq = status.messages+n;
+ array_append(&expunged_seqs, &seq, 1);
+ crc = crc32_data_more(crc, &expunged_uid,
+ sizeof(expunged_uid));
+ n++;
+ }
+
+ if (mailbox_search_deinit(&search_ctx) < 0)
+ ret = -1;
+ (void)mailbox_transaction_commit(&trans);
+ if (ret < 0)
+ return -1;
+
+ seqs = array_get(&expunged_seqs, &count);
+ // FIXME: broken if new messages have been added
+ if (client->messages_count != status.messages + count)
+ return -1;
+ if (crc != orig_crc)
+ return -1;
+
+ str = t_str_new(32);
+ for (i = count; i > 0; i--) {
+ str_truncate(str, 0);
+ str_printfa(str, "* %u EXPUNGE", seqs[i-1]);
+ client_send_line(client, str_c(str));
+ }
+ client->messages_count -= count;
+ return 0;
+}
+
+static int
+import_mailbox(struct client *client,
+ const unsigned char **data, size_t *size)
+{
+ struct mail_namespace *ns;
+ uint32_t uidvalidity, uidnext, messages_count, crc;
+ uint64_t modseq;
+ uint8_t examined;
+ const char *ns_prefix;
+ guid_128_t guid;
+ enum mailbox_flags flags = 0;
+
+ if (import_uint8(data, size, &examined) < 0 ||
+ import_uint32(data, size, &uidvalidity) < 0 ||
+ import_uint32(data, size, &uidnext) < 0 ||
+ import_uint32(data, size, &messages_count) < 0 ||
+ import_uint64(data, size, &modseq) < 0 ||
+ import_uint32(data, size, &crc) < 0 ||
+ import_string(data, size, &ns_prefix) < 0)
+ return 0;
+ if (*size < sizeof(guid))
+ return 0;
+ memcpy(guid, *data, sizeof(guid));
+ *data += sizeof(guid);
+ *size -= sizeof(guid);
+
+ ns = mail_namespace_find_prefix(client->user->namespaces, ns_prefix);
+ if (ns == NULL)
+ return 0;
+
+ if (examined)
+ flags |= MAILBOX_FLAG_READONLY;
+ else
+ flags |= MAILBOX_FLAG_DROP_RECENT;
+ client->mailbox = mailbox_alloc_guid(ns->list, guid, flags);
+ if (mailbox_sync(client->mailbox, 0) < 0)
+ return -1;
+ client->select_counter++;
+ client->mailbox_examined = examined;
+ client->messages_count = messages_count;
+ //FIXME:client->recent_count = recent_count;
+ client->uidvalidity = uidvalidity;
+
+ if (send_expunges(client, uidnext, modseq, crc) < 0)
+ return -1;
+ return 1;
+}
+
+int imap_state_import(struct client *client,
+ const unsigned char *data, size_t size)
+{
+ enum imap_state_type type;
+ const char *str;
+ int ret;
+
+ i_assert(client->mailbox == NULL);
+
+ while (size > 0) {
+ type = data[0]; data++; size--;
+ switch (type) {
+ case IMAP_STATE_TYPE_SESSION_ID:
+ // FIXME: untrusted commands shouldn't be able to set session ID
+ if (import_string(&data, &size, &str) < 0)
+ return 0;
+ client->session_id = p_strdup_empty(client->pool, str);
+ break;
+ case IMAP_STATE_TYPE_MAILBOX:
+ if ((ret = import_mailbox(client, &data, &size)) <= 0)
+ return ret;
+ break;
+ default:
+ return 0;
+ }
+ }
+ return 1;
+}
diff -r 67e9cb0b06ec src/imap/imap-state.h
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/imap/imap-state.h Mon Oct 29 18:40:43 2012 +0200
@@ -0,0 +1,11 @@
+#ifndef IMAP_STATE_H
+#define IMAP_STATE_H
+
+/* Export the IMAP client state to the given buffer. Returns TRUE if ok,
+ FALSE if state couldn't be exported. */
+bool imap_state_export(struct client *client, buffer_t *output);
+/* Returns 1 if ok, 0 if state was corrupted, -1 if other error. */
+int imap_state_import(struct client *client,
+ const unsigned char *data, size_t size);
+
+#endif
