On Tue, 21 Aug 2007 23:30:45 +1000, "Bron Gondwana" <[EMAIL PROTECTED]>
said:
> I'll be working on fast-rename in this new universe next,
> but I thought I should throw this out there for comments.
> So, what do you think (and yes, I'll be making rehash at
> least work happily with this, because we'll need to do it
> ourselves. Current plan - down the replica, rehash it,
> change the config, bring it back up. Failover, rinse,
> repeat)
And it turns out that the only change needed from David's
patch can be mostly encapsulated in this little fragment here
(also, we pass isusermbox down to mailbox_rename_copy so it
can handle the "breaks replication" issue better, *sigh* - I
am so tempted to just to return "ACCESS DENIED - if you really
want this then create a new folder and copy the damn messages
over yourself". One of the weirder namespace discontinuities
required by the IMAP spec)
+ if (isusermbox || !config_getswitch(IMAPOPT_FAST_RENAME) ||
+ ((config_hashimapspool != IMAP_ENUM_HASHIMAPSPOOL_USERID)
&&
+ (mboxlist_count_inferiors(oldname, 0, userid, auth_state)
> 0)) ||
+ (mailbox_rename_fast(&oldmailbox, newname, newpartition)
!= 0)) {
This fastrename patch works regardless of the settings of hashimapspool,
though of course due to the change from SWITCH to ENUM you still need to
have applied the previous patch. Oh well. At least it's very similar
in spirit to the virtdomains change from the past.
Bron.
--
Bron Gondwana
[EMAIL PROTECTED]
# Translate mailbox rename into filesystem rename where possible.
Index: cyrus-imapd-2.3.9/imap/mailbox.c
===================================================================
--- cyrus-imapd-2.3.9.orig/imap/mailbox.c 2007-08-21 23:59:02.000000000
-0400
+++ cyrus-imapd-2.3.9/imap/mailbox.c 2007-08-21 23:59:02.000000000 -0400
@@ -2847,6 +2847,136 @@
{ 0, 0 }
};
+/* Implement mailbox_rename as simple filesystem rename(). Not as safe as
+ mailbox_rename_copy() + mailbox_rename_cleanup() if the power fails, but
+ an awful lot faster. mboxlist_renamemailbox() will fall back to the slow
+ method if mailbox_rename_fast fails, so it is important that there is no
+ change if we throw an error */
+
+int mailbox_rename_fast(struct mailbox *mailbox,
+ const char *newname, char *newpartition)
+{
+ char oldpath[MAX_MAILBOX_PATH+1];
+ char newqroot[MAX_MAILBOX_PATH+1];
+ int newhasquota = quota_findroot(newqroot, sizeof(newqroot), newname);
+ uquota_t bytes = mailbox->quota_mailbox_used;
+ struct txn *tid = NULL;
+ char *path, *mpath, *p;
+ int save_errno;
+ int r = 0;
+ struct stat sbuf;
+
+ /* Check space in target quota root if different from source quota root */
+ if (newhasquota &&
+ (!mailbox->quota.root || strcmp(newqroot, mailbox->quota.root))) {
+ struct quota newquota;
+
+ newquota.root = newqroot;
+ r = quota_read(&newquota, NULL, 0);
+
+ if (!r && (newquota.limit >= 0) &&
+ (newquota.used + mailbox->quota_mailbox_used >
+ ((uquota_t) newquota.limit * QUOTA_UNITS))) {
+ return(IMAP_QUOTA_EXCEEDED);
+ }
+ }
+
+ /* Work out target directories */
+ r = mboxlist_getpath(newpartition, newname, &path, &mpath);
+ if (r) return r;
+
+ /* Fall back to rename_copy if different meta policies */
+ if ((mailbox->mpath && !mpath) || (!mailbox->mpath && mpath))
+ return(IMAP_IOERROR);
+
+ /* Create leading components in target path */
+ cyrus_mkdir(path, 0755);
+ if (mpath) cyrus_mkdir(mpath, 0755);
+
+ /* Use last_appenddate as a timestamp? */
+ mailbox->last_appenddate = time(NULL);
+ if ((r = mailbox_write_index_header(mailbox)))
+ return(r);
+
+ /* Point of no return */
+ if (rename(mailbox->path, path) < 0) {
+ /* Fall back to rename_copy */
+ syslog(LOG_ERR, "mailbox_fast_rename() failed: %s -> %s: %m",
+ mailbox->path, path);
+ return(IMAP_IOERROR);
+ }
+ if (mailbox->mpath && mpath && rename(mailbox->mpath, mpath) < 0) {
+ /* Fall back to rename_copy (need to undo first) */
+ rename(path, mailbox->path);
+ syslog(LOG_ERR, "mailbox_fast_rename() failed: %s -> %s: %m",
+ mailbox->path, mpath);
+ return(IMAP_IOERROR);
+ }
+ /* Can't throw any errors after this point: just log problems */
+
+ /* Remove empty parent directories */
+ strncpy(oldpath, mailbox->path, sizeof(oldpath));
+ while ((p=strrchr(oldpath, '/')) != NULL) {
+ *p = '\0';
+ if (rmdir(oldpath) < 0) break;
+ }
+ if (mailbox->mpath) {
+ strncpy(oldpath, mailbox->mpath, sizeof(oldpath));
+ while ((p=strrchr(oldpath, '/')) != NULL) {
+ *p = '\0';
+ if (rmdir(oldpath) < 0) break;
+ }
+ }
+
+ /* Small optimisation if source and target are on the same quotaroot */
+ if (mailbox->quota.root && newhasquota &&
+ !strcmp(mailbox->quota.root, newqroot))
+ return(0);
+
+ /* Release quota from old quotaroot */
+ if (mailbox->quota.root) {
+ r = quota_read(&(mailbox->quota), &tid, 1);
+ mailbox->quota.used -= bytes;
+ r = quota_write(&(mailbox->quota), &tid);
+ if (!r) quota_commit(&tid);
+ if (r) {
+ syslog(LOG_ERR,
+ "LOSTQUOTA: unable to record free of "
+ UQUOTA_T_FMT " bytes in quota %s",
+ bytes, mailbox->quota.root);
+ }
+ }
+ mailbox_close(mailbox);
+
+ /* Add quota to new quotaroot */
+ r = mailbox_open_header(newname, 0, mailbox);
+ if (!r) r = mailbox_lock_header(mailbox);
+ if (r) return(0);
+
+ if (newhasquota) {
+ mailbox->quota.root = xstrdup(newqroot);
+ r = quota_read(&(mailbox->quota), &tid, 1);
+ if (!r) {
+ mailbox->quota.used += bytes;
+ r = quota_write(&(mailbox->quota), &tid);
+ if (!r) quota_commit(&tid);
+ }
+ if (r) {
+ syslog(LOG_ERR,
+ "LOSTQUOTA: unable to record add of "
+ UQUOTA_T_FMT " bytes in quota %s",
+ bytes, mailbox->quota.root);
+ }
+ } else
+ mailbox->quota.root = NULL;
+
+ /* Need to update quotaroot in cyrus.header file */
+ mailbox_write_header(mailbox);
+
+ /* Up to parent to close the new mailbox */
+ return(0);
+}
+
/* if 'isinbox' is set, we perform the funky RENAME INBOX INBOX.old
semantics, regardless of whether or not the name of the mailbox is
'user.foo'.*/
Index: cyrus-imapd-2.3.9/imap/mailbox.h
===================================================================
--- cyrus-imapd-2.3.9.orig/imap/mailbox.h 2007-08-21 23:59:02.000000000
-0400
+++ cyrus-imapd-2.3.9/imap/mailbox.h 2007-08-21 23:59:02.000000000 -0400
@@ -364,6 +364,8 @@
struct mailbox *mailboxp);
extern int mailbox_delete(struct mailbox *mailbox, int delete_quota_root);
+extern int mailbox_rename_fast(struct mailbox *oldmailbox,
+ const char *newname, char *newparition);
extern int mailbox_rename_copy(struct mailbox *oldmailbox,
const char *newname, char *newpartition,
bit32 *olduidvalidityp, bit32 *newuidvalidityp,
Index: cyrus-imapd-2.3.9/imap/mboxlist.c
===================================================================
--- cyrus-imapd-2.3.9.orig/imap/mboxlist.c 2007-08-21 23:59:02.000000000
-0400
+++ cyrus-imapd-2.3.9/imap/mboxlist.c 2007-08-22 00:07:28.000000000 -0400
@@ -1164,6 +1164,7 @@
char *newpartition = NULL;
char *mboxent = NULL;
char *p;
+ int need_rename_cleanup = 0;
mupdate_handle *mupdate_h = NULL;
int madenew = 0;
@@ -1345,13 +1346,22 @@
/* 6. Copy mailbox */
if (!r && !(mbtype & MBTYPE_REMOTE)) {
/* Rename the actual mailbox */
- r = mailbox_rename_copy(&oldmailbox, newname, newpartition,
- NULL, NULL, &newmailbox, isusermbox);
- if (r) {
- goto done;
- } else {
- newopen = 1;
- }
+ syslog(LOG_INFO, "Rename: %s -> %s", oldname, newname);
+ need_rename_cleanup = 0;
+ if (isusermbox || !config_getswitch(IMAPOPT_FAST_RENAME) ||
+ ((config_hashimapspool != IMAP_ENUM_HASHIMAPSPOOL_USERID) &&
+ (mboxlist_count_inferiors(oldname, 0, userid, auth_state) > 0))
||
+ (mailbox_rename_fast(&oldmailbox, newname, newpartition) != 0)) {
+ /* Fall back to slow copy */
+ need_rename_cleanup = 1;
+ r = mailbox_rename_copy(&oldmailbox, newname, newpartition,
+ NULL, NULL, &newmailbox, isusermbox);
+ if (r) {
+ goto done;
+ } else {
+ newopen = 1;
+ }
+ }
}
if (!isusermbox) {
@@ -1497,7 +1507,7 @@
if(config_mupdate_server) mupdate_disconnect(&mupdate_h);
if(oldopen) {
- if(!r)
+ if (!r && need_rename_cleanup)
mailbox_rename_cleanup(&oldmailbox,isusermbox);
mailbox_close(&oldmailbox);
@@ -3398,3 +3408,60 @@
return ((!strncmp(mailboxname + domainlen, deleteprefix, deleteprefix_len)
&&
mailboxname[domainlen + deleteprefix_len] == '.') ? 1 : 0);
}
+
+struct count_tmplist {
+ int alloc;
+ int num;
+ char mb[1][MAX_MAILBOX_NAME];
+};
+
+#define COUNT_TMPLIST_INC 50
+
+/* Callback used by mboxlist_count_inferiors below */
+static int
+count_addmbox(char *name,
+ int matchlen __attribute__((unused)),
+ int maycreate __attribute__((unused)),
+ void *rock)
+{
+ struct count_tmplist **lptr = (struct count_tmplist **) rock;
+ struct count_tmplist *l = *lptr;
+
+ if (l->alloc == l->num) {
+ l->alloc += COUNT_TMPLIST_INC;
+ l = xrealloc(l, sizeof(struct count_tmplist) +
+ l->alloc * MAX_MAILBOX_NAME * (sizeof(char)));
+ *lptr = l;
+ }
+
+ strcpy(l->mb[l->num++], name);
+
+ return 0;
+}
+
+/* Check for attempt to create name which has children */
+int
+mboxlist_count_inferiors(char *mailboxname, int isadmin, char *userid,
+ struct auth_state *authstate)
+{
+ int count = 0;
+ char mailboxname2[MAX_MAILBOX_NAME+1];
+ struct count_tmplist *l = xmalloc(sizeof(struct count_tmplist));
+ char *p;
+
+ l->alloc = 0;
+ l->num = 0;
+
+ strcpy(mailboxname2, mailboxname);
+ p = mailboxname2 + strlen(mailboxname2); /* end of mailboxname */
+ strcpy(p, ".*");
+
+ /* build a list of mailboxes - we're using internal names here */
+ mboxlist_findall(NULL, mailboxname2, isadmin, userid,
+ authstate, count_addmbox, &l);
+
+ count = l->num;
+ free(l);
+
+ return(count);
+}
Index: cyrus-imapd-2.3.9/imap/mboxlist.h
===================================================================
--- cyrus-imapd-2.3.9.orig/imap/mboxlist.h 2007-08-21 23:59:01.000000000
-0400
+++ cyrus-imapd-2.3.9/imap/mboxlist.h 2007-08-21 23:59:02.000000000 -0400
@@ -199,6 +199,10 @@
/* close the database */
void mboxlist_close(void);
+/* Small utility routine for mailbox_fast_rename() */
+int mboxlist_count_inferiors(char *mailboxname, int isadmin, char *userid,
+ struct auth_state *authstate);
+
/* initialize database structures */
#define MBOXLIST_SYNC 0x02
void mboxlist_init(int flags);
Index: cyrus-imapd-2.3.9/lib/imapoptions
===================================================================
--- cyrus-imapd-2.3.9.orig/lib/imapoptions 2007-08-21 23:59:02.000000000
-0400
+++ cyrus-imapd-2.3.9/lib/imapoptions 2007-08-21 23:59:02.000000000 -0400
@@ -239,6 +239,10 @@
result in greater responsiveness for the client, especially when
expunging a large number of messages. */
+{ "fast_rename", 0, SWITCH }
+/* Use rename() to move mailbox data where possible. Faster, not quite
+ as safe if power fails part way through a rename. */
+
{ "flushseenstate", 0, SWITCH }
/* If enabled, changes to the seen state will be flushed to disk
immediately, otherwise changes will be cached and flushed when the
