Hi, On i386: before patch: $ dd if=/dev/urandom of=in bs=1M count=2k $ openrsync --rsync-path=/usr/bin/openrsync -av in localhost:out Transfer starting: 1 files sender.c:551: error: in: mmap: Cannot allocate memory client.c:85: error: rsync_sender receiver.c:345: error: poll: hangup server.c:145: error: rsync_receiver
With your patch: $ patch -p0 < /tmp/1 Hmm... Looks like a unified diff to me... The text leading up to this was: -------------------------- |diff --git usr.bin/rsync/uploader.c usr.bin/rsync/uploader.c |index fd07b22caeb..cce8b47a4c9 100644 |--- usr.bin/rsync/uploader.c |+++ usr.bin/rsync/uploader.c -------------------------- Patching file usr.bin/rsync/uploader.c using Plan A... Hunk #1 succeeded at 158. Hunk #2 succeeded at 741. Hunk #3 succeeded at 910. Hmm... Ignoring the trailing garbage. done $ cd usr.bin/rsync/ && make -j3 && doas make install … $ openrsync --rsync-path=/usr/bin/openrsync -av in localhost:out Transfer starting: 1 files sender.c:551: error: in: mmap: Cannot allocate memory client.c:85: error: rsync_sender receiver.c:345: error: poll: hangup server.c:145: error: rsync_receiver >From what I see, the mmap problem is on sender.c Cheers, On Sat, Aug 17, 2019 at 4:53 PM Sebastian Benoit <[email protected]> wrote: > > Joe Davis([email protected]) on 2019.08.16 12:26:36 +0100: > > By the looks of it, openrsync does attempt to map the entire file, from > > usr.bin/rsync/uploader.c: > > > > mapsz = st.st_size; > > map = mmap(NULL, mapsz, PROT_READ, MAP_SHARED, *fileinfd, 0); > > > > The likely reason for your out of memory error is the default datasize > > in login.conf. IIRC on some arches it's set to 768MB by default, which > > would allow your 300MB file to transfer, but would cause mmap to fail > > upon attempting to map the 1.6GB one. > > > > Increasing the default limits in /etc/login.conf should fix the problem. > > > > Note that rsync (not openrsync), doesn't use mmap for other reasons, > > from rsync-3.1.3/fileio.c: > > > > /* This provides functionality somewhat similar to mmap() but using read(). > > * It gives sliding window access to a file. mmap() is not used because of > > * the possibility of another program (such as a mailer) truncating the > > * file thus giving us a SIGBUS. */ > > > > Cheers, > > Joe > > Hi, > > this replaces the mmap() with pread(), please try it out. > > I dont much like the error handling here, but its a start. > > ok? > > > diff --git usr.bin/rsync/uploader.c usr.bin/rsync/uploader.c > index fd07b22caeb..cce8b47a4c9 100644 > --- usr.bin/rsync/uploader.c > +++ usr.bin/rsync/uploader.c > @@ -158,8 +158,8 @@ init_blk(struct blk *p, const struct blkset *set, off_t > offs, > p->len = idx < set->blksz - 1 ? set->len : set->rem; > p->offs = offs; > > - p->chksum_short = hash_fast(map + offs, p->len); > - hash_slow(map + offs, p->len, p->chksum_long, sess); > + p->chksum_short = hash_fast(map, p->len); > + hash_slow(map, p->len, p->chksum_long, sess); > } > > /* > @@ -741,8 +741,9 @@ rsync_uploader(struct upload *u, int *fileinfd, > { > struct blkset blk; > struct stat st; > - void *map, *bufp; > - size_t i, mapsz, pos, sz; > + void *mbuf, *bufp; > + ssize_t msz; > + size_t i, pos, sz; > off_t offs; > int c; > const struct flist *f; > @@ -909,35 +910,46 @@ rsync_uploader(struct upload *u, int *fileinfd, > blk.csum = u->csumlen; > > if (*fileinfd != -1 && st.st_size > 0) { > - mapsz = st.st_size; > - map = mmap(NULL, mapsz, PROT_READ, MAP_SHARED, *fileinfd, 0); > - if (map == MAP_FAILED) { > - ERR("%s: mmap", u->fl[u->idx].path); > - close(*fileinfd); > - *fileinfd = -1; > - return -1; > - } > - > init_blkset(&blk, st.st_size); > assert(blk.blksz); > > blk.blks = calloc(blk.blksz, sizeof(struct blk)); > if (blk.blks == NULL) { > ERR("calloc"); > - munmap(map, mapsz); > + close(*fileinfd); > + *fileinfd = -1; > + return -1; > + } > + > + if ((mbuf = calloc(1, blk.len)) == NULL) { > + ERR("calloc"); > close(*fileinfd); > *fileinfd = -1; > return -1; > } > > offs = 0; > - for (i = 0; i < blk.blksz; i++) { > - init_blk(&blk.blks[i], > - &blk, offs, i, map, sess); > + i = 0; > + do { > + msz = pread(*fileinfd, mbuf, blk.len, offs); > + if (msz < 0) { > + ERR("pread"); > + close(*fileinfd); > + *fileinfd = -1; > + return -1; > + } > + if ((size_t)msz != blk.len && (size_t)msz != blk.rem) > { > + /* short read, try again */ > + continue; > + } > + init_blk(&blk.blks[i], &blk, offs, i, mbuf, sess); > offs += blk.len; > - } > + LOG3( > + "i=%ld, offs=%lld, msz=%ld, blk.len=%lu, > blk.rem=%lu", > + i, offs, msz, blk.len, blk.rem); > + i++; > + } while (i < blk.blksz); > > - munmap(map, mapsz); > close(*fileinfd); > *fileinfd = -1; > LOG3("%s: mapped %jd B with %zu blocks", >
