On 13.01.2014 12:48, Markus Weippert wrote:
> Hi,
>
> I'm having some issues with replicating public namespaces. Everything
> seems to work fine for private namespaces, but while importing some huge
> mailboxes (many small mails) into a public namespace via imapsync,
> something goes wrong.
>
> The expected mail flow is:
> old-server (imapsync)> new-server1 (replication)> new-server2
>
> But then, dovecot seems to run into race conditions when the
> replications process tries to sync the same public mailbox under two or
> more different users at the same time. As a result, messages get
> duplicated, new-server2 sends those back to new-server1 which then
> starts to produce duplicates too. If I don't kill the processes in time
> and delete the faulty mailbox, they start to produce thousands of mails.
> In fact, server2 should not export messages at all, since it's not
> productive yet and does not get any mail except from the replication.
>
> The only thing getting logged (only few compared to the huge amount of
> duplicates produced):
> "dsync-server([email protected]): Warning: Maildir /...: Expunged message
> reappeared, giving a new UID"
>
> Is there any way to fix this?
>
> Regards,
> Markus
I looked into this a bit more. The problem seems to be, replication
locking is only done at user level. For public namespaces, this allows
two replication processes to sync the same mailbox in parallel. So I did
a (poor) implementation for mailbox level locking. It locks the mailbox
with a lock file in the control directory on both sides (not sure if
that's necessary) and skips locked mailboxes instantly, because they are
currently being synced anyway.
It actually works in my setup. The duplicate messages are gone. It logs
some warnings when two replication processes try to access the same
mailbox at once, which seems to happen quite frequently in public
namespaces.
Maybe someone more experienced can clean this up and adopt it to
upstream? I really like the replication idea and it would be nice if it
were as stable for shared/public namespaces as it is for private ones...
Regards,
Markus
P.S.:
> replication_dsync_parameters = -d -l 60 -N -x virtual -x ns_public -U
Typo, actually looks like this:
replication_dsync_parameters = -d -l 60 -N -x virtual -x legacy -U
diff -aur dovecot-2.2.10.old/src/doveadm/dsync/dsync-brain.c dovecot-2.2.10/src/doveadm/dsync/dsync-brain.c
--- dovecot-2.2.10.old/src/doveadm/dsync/dsync-brain.c 2013-11-19 21:36:30.000000000 +0100
+++ dovecot-2.2.10/src/doveadm/dsync/dsync-brain.c 2014-01-17 15:37:31.136097108 +0100
@@ -113,6 +113,7 @@
brain->ibc = ibc;
brain->sync_type = DSYNC_BRAIN_SYNC_TYPE_UNKNOWN;
brain->lock_fd = -1;
+ brain->box_lock_fd = -1;
brain->verbose_proctitle = service_set->verbose_proctitle;
hash_table_create(&brain->mailbox_states, pool, 0,
guid_128_hash, guid_128_cmp);
diff -aur dovecot-2.2.10.old/src/doveadm/dsync/dsync-brain-mailbox.c dovecot-2.2.10/src/doveadm/dsync/dsync-brain-mailbox.c
--- dovecot-2.2.10.old/src/doveadm/dsync/dsync-brain-mailbox.c 2013-12-19 22:05:18.000000000 +0100
+++ dovecot-2.2.10/src/doveadm/dsync/dsync-brain-mailbox.c 2014-01-17 17:54:53.905111604 +0100
@@ -40,6 +40,67 @@
return 1;
}
+int dsync_brain_mailbox_lock(struct dsync_brain *brain, struct mailbox *box)
+{
+ struct stat st1, st2;
+ const char *control_dir;
+
+ if (mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_CONTROL,
+ &control_dir) <= 0) {
+ i_error("Couldn't look up control dir");
+ return -1;
+ }
+
+ brain->box_lock_path = p_strconcat(brain->pool, control_dir,
+ "/"DSYNC_BOX_LOCK_FILENAME, NULL);
+ for (;;) {
+ brain->box_lock_fd = creat(brain->box_lock_path, 0600);
+ if (brain->box_lock_fd == -1) {
+ i_error("Couldn't create lock %s: %m",
+ brain->box_lock_path);
+ return -1;
+ }
+
+ if (file_try_lock(brain->box_lock_fd, brain->box_lock_path, F_WRLCK,
+ FILE_LOCK_METHOD_FCNTL, &brain->box_lock) <= 0) {
+ i_warning("Couldn't lock %s: %m", brain->box_lock_path);
+ break;
+ }
+ if (fstat(brain->box_lock_fd, &st1) < 0) {
+ if (errno != ESTALE) {
+ i_error("fstat(%s) failed: %m", brain->box_lock_path);
+ break;
+ }
+ } else if (stat(brain->box_lock_path, &st2) < 0) {
+ if (errno != ENOENT) {
+ i_error("stat(%s) failed: %m", brain->box_lock_path);
+ break;
+ }
+ } else if (st1.st_ino == st2.st_ino) {
+ /* success */
+ return 0;
+ }
+ /* file was recreated, try again */
+ i_close_fd(&brain->box_lock_fd);
+ }
+ i_close_fd(&brain->box_lock_fd);
+ brain->box_lock_fd = -1;
+ return -1;
+}
+
+void dsync_brain_mailbox_unlock(struct dsync_brain *brain)
+{
+ if (brain->box_lock_fd != -1) {
+ /* unlink the lock file before it gets unlocked */
+ if (unlink(brain->box_lock_path) < 0)
+ i_error("unlink(%s) failed: %m", brain->box_lock_path);
+ file_lock_free(&brain->box_lock);
+ i_close_fd(&brain->box_lock_fd);
+ brain->box_lock_fd = -1;
+ }
+}
+
+
int dsync_brain_mailbox_alloc(struct dsync_brain *brain, const guid_128_t guid,
struct mailbox **box_r, const char **error_r)
{
@@ -269,6 +330,7 @@
if (brain->log_scan != NULL)
dsync_transaction_log_scan_deinit(&brain->log_scan);
mailbox_free(&brain->box);
+ dsync_brain_mailbox_unlock(brain);
brain->state = brain->pre_box_state;
}
@@ -369,11 +431,17 @@
flags |= MAILBOX_FLAG_READONLY;
}
box = mailbox_alloc(node->ns->list, vname, flags);
+
+ if (dsync_brain_mailbox_lock(brain, box) < 0) {
+ return 0;
+ }
+
for (;;) {
if ((ret = dsync_box_get(box, &dsync_box)) <= 0) {
if (ret < 0)
brain->failed = TRUE;
mailbox_free(&box);
+ dsync_brain_mailbox_unlock(brain);
return ret;
}
@@ -381,6 +449,7 @@
we can skip the mailbox */
if (!dsync_brain_has_mailbox_state_changed(brain, &dsync_box)) {
mailbox_free(&box);
+ dsync_brain_mailbox_unlock(brain);
return 0;
}
if (synced) {
@@ -396,6 +465,7 @@
mailbox_get_last_error(box, NULL));
brain->failed = TRUE;
mailbox_free(&box);
+ dsync_brain_mailbox_unlock(brain);
return -1;
}
synced = TRUE;
@@ -625,6 +695,7 @@
i_assert(brain->failed);
return TRUE;
}
+
if (box == NULL) {
/* mailbox was probably deleted/renamed during sync */
if (brain->backup_send && brain->no_backup_overwrite) {
@@ -648,17 +719,23 @@
dsync_brain_slave_send_mailbox_lost(brain, dsync_box);
return TRUE;
}
+ if (dsync_brain_mailbox_lock(brain, box) < 0) {
+ brain->failed = TRUE;
+ return TRUE;
+ }
if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FULL_READ) < 0) {
i_error("Can't sync mailbox %s: %s",
mailbox_get_vname(box),
mailbox_get_last_error(box, NULL));
mailbox_free(&box);
+ dsync_brain_mailbox_unlock(brain);
brain->failed = TRUE;
return TRUE;
}
if ((ret = dsync_box_get(box, &local_dsync_box)) <= 0) {
mailbox_free(&box);
+ dsync_brain_mailbox_unlock(brain);
if (ret < 0) {
brain->failed = TRUE;
return TRUE;
@@ -677,6 +754,7 @@
if (!dsync_boxes_need_sync(brain, &local_dsync_box, dsync_box)) {
/* no fields appear to have changed, skip this mailbox */
mailbox_free(&box);
+ dsync_brain_mailbox_unlock(brain);
return TRUE;
}
diff -aur dovecot-2.2.10.old/src/doveadm/dsync/dsync-brain-private.h dovecot-2.2.10/src/doveadm/dsync/dsync-brain-private.h
--- dovecot-2.2.10.old/src/doveadm/dsync/dsync-brain-private.h 2013-11-19 21:36:30.000000000 +0100
+++ dovecot-2.2.10/src/doveadm/dsync/dsync-brain-private.h 2014-01-17 17:49:35.109122642 +0100
@@ -7,6 +7,7 @@
#include "dsync-mailbox-state.h"
#define DSYNC_LOCK_FILENAME ".dovecot-sync.lock"
+#define DSYNC_BOX_LOCK_FILENAME "dovecot-sync.lock"
struct dsync_mailbox_tree_sync_change;
@@ -60,6 +61,10 @@
const char *lock_path;
struct file_lock *lock;
+ int box_lock_fd;
+ char *box_lock_path;
+ struct file_lock *box_lock;
+
char hierarchy_sep;
struct dsync_mailbox_tree *local_mailbox_tree;
struct dsync_mailbox_tree *remote_mailbox_tree;
