I'm checking into the innards of rsync in order to try to figure out the
best way to add a proper "move-files" option, and I think I discovered a
potential hang when sending large numbers of files between systems. If
I understand this correctly from my preliminary reading of the code, the
generator is piping data to the sender, which is piping data to the
receiver, which can sometimes pipe data back to the generator (when it
wants to resend a file). It appears that the generator doesn't read its
input pipe until it processes all its files, so it looks like the
receiver process can hang trying to write to the generator process if
the number of "redo" items grows large enough and the generator is not
yet finished with the first pass through the list of files.
The attached patch changes generator.c to create a cache of all the ints
coming in from the receiver during the initial pass through the file
list. I chose to use select to check for data. A better approach might
be to make the input pipe non-blocking (for the reader) and just try to
read bytes into a buffer during each iteration of the initial generator
loop, and then turn the bytes into ints in the redo loop.
This patch is based on the CVS version.
..wayne..
---8<------8<------8<------8<---cut here--->8------>8------>8------>8---
Index: generator.c
--- generator.c 7 May 2001 06:59:37 -0000 1.24
+++ generator.c 30 May 2001 12:51:29 -0000
@@ -39,6 +39,8 @@
extern int modify_window;
extern char *compare_dest;
+static int *redo_list;
+static int rlist_len, rlist_alloc_len;
/* choose whether to skip a particular file */
static int skip_file(char *fname,
@@ -422,7 +424,7 @@
void generate_files(int f,struct file_list *flist,char *local_name,int f_recv)
{
- int i;
+ int i, j;
int phase=0;
if (verbose > 2)
@@ -434,6 +436,27 @@
mode_t saved_mode = file->mode;
if (!file->basename) continue;
+ if (remote_version >= 13) {
+ struct timeval tv;
+ fd_set fds;
+
+ FD_ZERO(&fds);
+ FD_SET(f_recv, &fds);
+ tv.tv_sec = 0;
+ tv.tv_usec = 0;
+
+ if (select(f_recv+1, &fds, NULL, NULL, &tv) > 0) {
+ j = read_int(f_recv);
+ if (rlist_len == rlist_alloc_len) {
+ rlist_alloc_len += 1024;
+ redo_list = (int *)Realloc(redo_list,
+sizeof(int)*rlist_alloc_len);
+ if (!redo_list)
+out_of_memory("add_delete_entry");
+ }
+
+ redo_list[rlist_len++] = j;
+ }
+ }
+
/* we need to ensure that any directories we create have writeable
permissions initially so that we can create the files within
them. This is then fixed after the files are transferred */
@@ -466,8 +489,15 @@
if (remote_version >= 13) {
/* in newer versions of the protocol the files can cycle through
the system more than once to catch initial checksum errors */
- for (i=read_int(f_recv); i != -1; i=read_int(f_recv)) {
- struct file_struct *file = flist->files[i];
+ for (j = 0;;) {
+ struct file_struct *file;
+ if (j < rlist_len)
+ i = redo_list[j++];
+ else
+ i = read_int(f_recv);
+ if (i == -1)
+ break;
+ file = flist->files[i];
recv_generator(local_name?local_name:f_name(file),
flist,i,f);
}
---8<------8<------8<------8<---cut here--->8------>8------>8------>8---
