Joe Davis([email protected]) on 2019.08.16 12:26:36 +0100:
> By the looks of it, openrsync does attempt to map the entire file, from
> usr.bin/rsync/uploader.c:
>
> mapsz = st.st_size;
> map = mmap(NULL, mapsz, PROT_READ, MAP_SHARED, *fileinfd, 0);
>
> The likely reason for your out of memory error is the default datasize
> in login.conf. IIRC on some arches it's set to 768MB by default, which
> would allow your 300MB file to transfer, but would cause mmap to fail
> upon attempting to map the 1.6GB one.
>
> Increasing the default limits in /etc/login.conf should fix the problem.
>
> Note that rsync (not openrsync), doesn't use mmap for other reasons,
> from rsync-3.1.3/fileio.c:
>
> /* This provides functionality somewhat similar to mmap() but using read().
> * It gives sliding window access to a file. mmap() is not used because of
> * the possibility of another program (such as a mailer) truncating the
> * file thus giving us a SIGBUS. */
>
> Cheers,
> Joe
Hi,
this replaces the mmap() with pread(), please try it out.
I dont much like the error handling here, but its a start.
ok?
diff --git usr.bin/rsync/uploader.c usr.bin/rsync/uploader.c
index fd07b22caeb..cce8b47a4c9 100644
--- usr.bin/rsync/uploader.c
+++ usr.bin/rsync/uploader.c
@@ -158,8 +158,8 @@ init_blk(struct blk *p, const struct blkset *set, off_t
offs,
p->len = idx < set->blksz - 1 ? set->len : set->rem;
p->offs = offs;
- p->chksum_short = hash_fast(map + offs, p->len);
- hash_slow(map + offs, p->len, p->chksum_long, sess);
+ p->chksum_short = hash_fast(map, p->len);
+ hash_slow(map, p->len, p->chksum_long, sess);
}
/*
@@ -741,8 +741,9 @@ rsync_uploader(struct upload *u, int *fileinfd,
{
struct blkset blk;
struct stat st;
- void *map, *bufp;
- size_t i, mapsz, pos, sz;
+ void *mbuf, *bufp;
+ ssize_t msz;
+ size_t i, pos, sz;
off_t offs;
int c;
const struct flist *f;
@@ -909,35 +910,46 @@ rsync_uploader(struct upload *u, int *fileinfd,
blk.csum = u->csumlen;
if (*fileinfd != -1 && st.st_size > 0) {
- mapsz = st.st_size;
- map = mmap(NULL, mapsz, PROT_READ, MAP_SHARED, *fileinfd, 0);
- if (map == MAP_FAILED) {
- ERR("%s: mmap", u->fl[u->idx].path);
- close(*fileinfd);
- *fileinfd = -1;
- return -1;
- }
-
init_blkset(&blk, st.st_size);
assert(blk.blksz);
blk.blks = calloc(blk.blksz, sizeof(struct blk));
if (blk.blks == NULL) {
ERR("calloc");
- munmap(map, mapsz);
+ close(*fileinfd);
+ *fileinfd = -1;
+ return -1;
+ }
+
+ if ((mbuf = calloc(1, blk.len)) == NULL) {
+ ERR("calloc");
close(*fileinfd);
*fileinfd = -1;
return -1;
}
offs = 0;
- for (i = 0; i < blk.blksz; i++) {
- init_blk(&blk.blks[i],
- &blk, offs, i, map, sess);
+ i = 0;
+ do {
+ msz = pread(*fileinfd, mbuf, blk.len, offs);
+ if (msz < 0) {
+ ERR("pread");
+ close(*fileinfd);
+ *fileinfd = -1;
+ return -1;
+ }
+ if ((size_t)msz != blk.len && (size_t)msz != blk.rem) {
+ /* short read, try again */
+ continue;
+ }
+ init_blk(&blk.blks[i], &blk, offs, i, mbuf, sess);
offs += blk.len;
- }
+ LOG3(
+ "i=%ld, offs=%lld, msz=%ld, blk.len=%lu,
blk.rem=%lu",
+ i, offs, msz, blk.len, blk.rem);
+ i++;
+ } while (i < blk.blksz);
- munmap(map, mapsz);
close(*fileinfd);
*fileinfd = -1;
LOG3("%s: mapped %jd B with %zu blocks",
