Hi all,
Here are the patch I worked on when having sync issues with Exchange
(online).
Exchange is very picky with messages, rejecting messages with too long
lines or binary content without explicit error messages.
All my additions are not enabled by default (ie. defaulting to normal
mbsync behaviour)
Common cfg section:
* Either skip or fix messages with lines more than xxx bytes
(typically no more than 9900 bytes with exchange)
MaxLineLength xxx (in bytes)
CutLongLines yes|no (fix or skip message)
* Allow to rescan all mails from a folder, ignoring the last sync
latest message pulled (usefull when playing with my new settings)
IgnoreMaxPulledUid yes|no
* Skip messages with raw binary content (bytes < 0x20 except CR/LF/TAB)
SkipBinaryContent yes|no
* Allow to delete non empty folders on slave (when you are sure about
what you're doing)
DeleteNonEmpty yes|no
Drivers cfg section (imap only):
* Suppress Keyword not supported warnings
IgnoreKeywordWarnings yes|no
Those patches have been extensively tested by myself with a hundred
mailboxes migration from xxx to exchange online via imap.
Patch is against 77acc268123b8233843ca9bc3dcf90669efde08f
The only missing part is long lines cutting when there's CR/LF
convertion (I don't use maildir++)
Regards,
F.LOMBARD.
diff --git a/src/config.c b/src/config.c
index 20f09d9..3af1610 100644
--- a/src/config.c
+++ b/src/config.c
@@ -202,6 +202,16 @@ getopt_helper( conffile_t *cfile, int *cops,
channel_conf_t *conf )
conf->sync_state = expand_strdup( cfile->val );
else if (!strcasecmp( "CopyArrivalDate", cfile->cmd ))
conf->use_internal_date = parse_bool( cfile );
+ else if (!strcasecmp( "CutLongLines", cfile->cmd ))
+ conf->cut_lines = parse_bool( cfile );
+ else if (!strcasecmp( "IgnoreMaxPulledUid", cfile->cmd ))
+ conf->ignore_max_pulled_uid = parse_bool( cfile );
+ else if (!strcasecmp( "SkipBinaryContent", cfile->cmd ))
+ conf->skip_binary_content = parse_bool( cfile );
+ else if (!strcasecmp( "DeleteNonEmpty", cfile->cmd ))
+ conf->delete_nonempty = parse_bool( cfile );
+ else if (!strcasecmp( "MaxLineLength", cfile->cmd ))
+ conf->max_line_len = parse_int( cfile );
else if (!strcasecmp( "MaxMessages", cfile->cmd ))
conf->max_messages = parse_int( cfile );
else if (!strcasecmp( "ExpireUnread", cfile->cmd ))
diff --git a/src/drv_imap.c b/src/drv_imap.c
index e24c7d8..10da0cb 100644
--- a/src/drv_imap.c
+++ b/src/drv_imap.c
@@ -66,6 +66,7 @@ typedef struct imap_store_conf {
imap_server_conf_t *server;
char delimiter;
char use_namespace;
+ char ignore_keyword_warnings;
} imap_store_conf_t;
typedef struct imap_message {
@@ -1259,6 +1260,7 @@ static void
imap_socket_read( void *aux )
{
imap_store_t *ctx = (imap_store_t *)aux;
+ imap_store_conf_t *cfg = (imap_store_conf_t *)ctx->gen.conf;
struct imap_cmd *cmdp, **pcmdp;
char *cmd, *arg, *arg1, *p;
int resp, resp2, tag;
@@ -1326,7 +1328,8 @@ imap_socket_read( void *aux )
error( "IMAP error: bogus greeting response
%s\n", arg );
break;
} else if (!strcmp( "NO", arg )) {
- warn( "Warning from IMAP server: %s\n", cmd );
+ if (!strcmp( "Keywords are not supported", arg)
&& !cfg->ignore_keyword_warnings)
+ warn( "Warning from IMAP server: %s\n",
cmd );
} else if (!strcmp( "BAD", arg )) {
error( "Error from IMAP server: %s\n", cmd );
} else if (!strcmp( "CAPABILITY", arg )) {
@@ -1416,6 +1419,16 @@ imap_socket_read( void *aux )
resp = RESP_NO;
if (cmdp->param.failok)
goto doresp;
+ } else if (!strcmp( "BAD", arg )) {
+ resp = RESP_NO;
+ warn( "Warning: IMAP command '%s' returned an
error: %s %s\n",
+ starts_with( cmdp->cmd, -1, "LOGIN", 5 )
?
+ "LOGIN <user> <pass>" :
+ starts_with( cmdp->cmd, -1,
"AUTHENTICATE PLAIN", 18 ) ?
+ "AUTHENTICATE PLAIN <authdata>" :
+ cmdp->cmd,
+ arg, cmd ? cmd : "" );
+ goto doresp;
} else /*if (!strcmp( "BAD", arg ))*/
resp = RESP_CANCEL;
error( "IMAP command '%s' returned an error: %s
%s\n",
@@ -2968,6 +2981,8 @@ imap_parse_store( conffile_t *cfg, store_conf_t **storep )
store->server = srv;
} else if (!strcasecmp( "UseNamespace", cfg->cmd ))
store->use_namespace = parse_bool( cfg );
+ else if (!strcasecmp( "IgnoreKeywordWarnings", cfg->cmd
))
+ store->ignore_keyword_warnings = parse_bool(
cfg );
else if (!strcasecmp( "Path", cfg->cmd ))
store->gen.path = nfstrdup( cfg->val );
else if (!strcasecmp( "PathDelimiter", cfg->cmd )) {
diff --git a/src/sync.c b/src/sync.c
index eb48dc5..089ffa6 100644
--- a/src/sync.c
+++ b/src/sync.c
@@ -302,13 +302,15 @@ copy_msg( copy_vars_t *vars )
static void msg_stored( int sts, int uid, void *aux );
static void
-copy_msg_bytes( char **out_ptr, const char *in_buf, int *in_idx, int in_len,
int in_cr, int out_cr )
+copy_msg_bytes( char **out_ptr, const char *in_buf, int *in_idx, int in_len,
int in_cr, int out_cr, int max_line_len )
{
char *out = *out_ptr;
int idx = *in_idx;
if (out_cr != in_cr) {
+ /* message needs to be converted */
char c;
if (out_cr) {
+ /* adding CR */
for (; idx < in_len; idx++) {
if ((c = in_buf[idx]) != '\r') {
if (c == '\n')
@@ -317,15 +319,52 @@ copy_msg_bytes( char **out_ptr, const char *in_buf, int
*in_idx, int in_len, int
}
}
} else {
+ /* removing CR */
for (; idx < in_len; idx++) {
if ((c = in_buf[idx]) != '\r')
*out++ = c;
}
}
} else {
- memcpy( out, in_buf + idx, in_len - idx );
- out += in_len - idx;
- idx = in_len;
+ /* no CRLF change */
+ if (max_line_len > 0) {
+ /* there are too long lines in the message */
+ const char * curLine = in_buf + *in_idx;
+ int lines = 0;
+ while(curLine) {
+ char * nextLine = strchr(curLine, '\n');
+ int curLineLen = nextLine ? (unsigned
int)(nextLine-curLine) + 1 : strlen(curLine);
+ if (curLineLen > max_line_len) {
+ /* this line need to be cut into
smaller lines */
+ int line_idx = 0;
+ while(line_idx < curLineLen) {
+ memcpy(out, in_buf + idx +
line_idx, ((curLineLen - line_idx) < max_line_len) ? (curLineLen - line_idx) :
max_line_len);
+ out += (((curLineLen -
line_idx) < max_line_len) ? (curLineLen - line_idx) : max_line_len);
+ line_idx += ((curLineLen -
line_idx) < max_line_len) ? (curLineLen - line_idx) : max_line_len;
+ if( line_idx < curLineLen) {
+ /* add (CR)LF except
for the last line */
+ if (out_cr)
+ *out++ = '\r';
+ *out++ = '\n';
+ }
+ }
+ idx += curLineLen;
+ } else {
+ /* simple copy */
+ memcpy (out , in_buf + idx, curLineLen);
+ out += curLineLen;
+ idx += curLineLen;
+ }
+ curLine = nextLine ? (nextLine+1) : NULL;
+ lines++;
+ }
+ //debug("End index %d (message size %d), message size
should be %d\n", idx, in_len, *in_idx + out - *out_ptr);
+ } else {
+ /* simple copy */
+ memcpy( out, in_buf + idx, in_len - idx );
+ out += in_len - idx;
+ idx = in_len;
+ }
}
*out_ptr = out;
*in_idx = idx;
@@ -338,7 +377,41 @@ copy_msg_convert( int in_cr, int out_cr, copy_vars_t *vars
)
int in_len = vars->data.len;
int idx = 0, sbreak = 0, ebreak = 0;
int lines = 0, hdr_crs = 0, bdy_crs = 0, app_cr = 0, extra = 0;
+ int extra_bytes = 0;
if (vars->srec) {
+ /* check if the message has too long lines if enabled */
+ if(global_conf.max_line_len) {
+ char * curLine = in_buf;
+ while(curLine) {
+ char * nextLine = strchr(curLine, '\n');
+ uint curLineLen = nextLine ? (unsigned
int)(nextLine-curLine) + 1 : strlen(curLine);
+ if (curLineLen > global_conf.max_line_len) {
+ if (global_conf.cut_lines) {
+ /* compute the addded lines as
we are going to cut them */
+ if (out_cr)
+ extra_bytes +=
curLineLen / global_conf.max_line_len; // CR
+ extra_bytes += curLineLen /
global_conf.max_line_len; // LF
+ } else {
+ /* stop here with too long line
error */
+ free( in_buf );
+ return 2;
+ }
+ }
+ curLine = nextLine ? (nextLine+1) : NULL;
+ }
+ }
+ if(global_conf.skip_binary_content) {
+ while (idx < in_len) {
+ unsigned char c = in_buf[idx++];
+ if (c < 0x20 && c != '\r' && c != '\n' && c !=
'\t') {
+ /* binary content, skip */
+ debug("Incorrect byte %d at offset
%d/%d\n", c, idx, in_len);
+ free( in_buf );
+ return 3;
+ }
+ }
+ idx = 0;
+ }
nloop: ;
int start = idx;
int line_crs = 0;
@@ -380,11 +453,12 @@ copy_msg_convert( int in_cr, int out_cr, copy_vars_t
*vars )
extra += lines;
}
- vars->data.len = in_len + extra;
+ vars->data.len = in_len + extra + extra_bytes;
char *out_buf = vars->data.data = nfmalloc( vars->data.len );
idx = 0;
if (vars->srec) {
- copy_msg_bytes( &out_buf, in_buf, &idx, sbreak, in_cr, out_cr );
+ //debug ("Calling copy_msg_bytes for the header (0 to %d) with
%d extra bytes\n", sbreak, extra);
+ copy_msg_bytes( &out_buf, in_buf, &idx, sbreak, in_cr, out_cr,
0 );
memcpy( out_buf, "X-TUID: ", 8 );
out_buf += 8;
@@ -395,8 +469,10 @@ copy_msg_convert( int in_cr, int out_cr, copy_vars_t *vars
)
*out_buf++ = '\n';
idx = ebreak;
}
- copy_msg_bytes( &out_buf, in_buf, &idx, in_len, in_cr, out_cr );
-
+ //debug ("Calling copy_msg_bytes for the body (at %d) with %d extra
byte(s), limit is %d \n", ebreak, extra_bytes, extra_bytes > 0 ?
global_conf.max_line_len : 0);
+ copy_msg_bytes( &out_buf, in_buf, &idx, in_len, in_cr, out_cr,
extra_bytes > 0 ? global_conf.max_line_len : 0 );
+ //debug("Message after %s\n", vars->data.data);
+ //debug("Good message size should be %d + %d\n",vars->data.len-extra,
extra);
free( in_buf );
return 1;
}
@@ -422,11 +498,24 @@ msg_fetched( int sts, void *aux )
scr = (svars->drv[1-t]->flags / DRV_CRLF) & 1;
tcr = (svars->drv[t]->flags / DRV_CRLF) & 1;
if (vars->srec || scr != tcr) {
- if (!copy_msg_convert( scr, tcr, vars )) {
+ switch (copy_msg_convert( scr, tcr, vars )) {
+ case 0:
warn( "Warning: message %d from %s has
incomplete header.\n",
vars->msg->uid, str_ms[1-t] );
vars->cb( SYNC_NOGOOD, 0, vars );
return;
+ case 2:
+ warn( "Warning: message %d from %s has too long
line(s).\n",
+ vars->msg->uid, str_ms[1-t] );
+ vars->cb( SYNC_MALFORMED, 0, vars );
+ return;
+ case 3:
+ warn( "Warning: message %d from %s has raw
binary content.\n",
+ vars->msg->uid, str_ms[1-t] );
+ vars->cb( SYNC_MALFORMED, 0, vars );
+ return;
+ default:
+ break;
}
}
@@ -779,7 +868,11 @@ load_state( sync_vars_t *svars )
}
svars->existing = 0;
}
- svars->newmaxuid[M] = svars->maxuid[M];
+ if (global_conf.ignore_max_pulled_uid) {
+ svars->newmaxuid[M] = 0;
+ } else {
+ svars->newmaxuid[M] = svars->maxuid[M];
+ }
svars->newmaxuid[S] = svars->maxuid[S];
svars->mmaxxuid = INT_MAX;
line = 0;
@@ -1076,7 +1169,7 @@ box_confirmed2( sync_vars_t *svars, int t )
svars->chan->name, str_ms[t],
svars->orig_name[t] );
goto bail;
}
- if (svars->drv[1-t]->confirm_box_empty(
svars->ctx[1-t] ) != DRV_OK) {
+ if ((svars->drv[1-t]->confirm_box_empty(
svars->ctx[1-t] ) != DRV_OK) && !global_conf.delete_nonempty) {
warn( "Warning: channel %s: %s %s
cannot be opened and %s %s not empty.\n",
svars->chan->name, str_ms[t],
svars->orig_name[t], str_ms[1-t], svars->orig_name[1-t] );
goto done;
@@ -1821,6 +1914,9 @@ msg_copied( int sts, int uid, copy_vars_t *vars )
svars->state[t] |= ST_FIND_NEW;
msg_copied_p2( svars, vars->srec, t, uid );
break;
+ case SYNC_MALFORMED:
+ msg_copied_p2( svars, vars->srec, t, -2 );
+ break;
case SYNC_NOGOOD:
debug( " -> killing (%d,%d)\n", vars->srec->uid[M],
vars->srec->uid[S] );
vars->srec->status = S_DEAD;
diff --git a/src/sync.h b/src/sync.h
index be26660..1c8a673 100644
--- a/src/sync.h
+++ b/src/sync.h
@@ -55,6 +55,11 @@ typedef struct channel_conf {
uint max_messages; /* for slave only */
signed char expire_unread;
char use_internal_date;
+ uint max_line_len;
+ char cut_lines;
+ char ignore_max_pulled_uid; /* for master only */
+ char skip_binary_content; /* for master only */
+ char delete_nonempty;
} channel_conf_t;
typedef struct group_conf {
@@ -69,11 +74,12 @@ extern group_conf_t *groups;
extern const char *str_ms[2], *str_hl[2];
-#define SYNC_OK 0 /* assumed to be 0 */
-#define SYNC_FAIL 1
-#define SYNC_BAD(ms) (4<<(ms))
-#define SYNC_NOGOOD 16 /* internal */
-#define SYNC_CANCELED 32 /* internal */
+#define SYNC_OK 0 /* assumed to be 0 */
+#define SYNC_FAIL 1
+#define SYNC_BAD(ms) (4<<(ms))
+#define SYNC_NOGOOD 16 /* internal */
+#define SYNC_CANCELED 32 /* internal */
+#define SYNC_MALFORMED 64 /* internal */
#define BOX_POSSIBLE -1
#define BOX_ABSENT 0
------------------------------------------------------------------------------
Check out the vibrant tech community on one of the world's most
engaging tech sites, Slashdot.org! http://sdm.link/slashdot
_______________________________________________
isync-devel mailing list
isync-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/isync-devel
