This patch is the same patch that I emailed on Aug. 13th with an
additional minor bugfix that permits large files to be transferred. As
a recap, here is what the patch does:
Adds Mehmet Balman's patch which adds the fcrc32 function that reads the
file in chunks rather than the whole file
Resumes stalled files from the point where they stopped
Fixes bugs related to serving and transferring large files.
Signed-off-by: Hugh Greenberg <[EMAIL PROTECTED]>
Index: xget.c
===================================================================
--- xget.c (revision 685)
+++ xget.c (working copy)
@@ -1,3 +1,5 @@
+#define _FILE_OFFSET_BITS 64
+
#include <stdlib.h>
#include <stdio.h>
#include <stdarg.h>
@@ -79,13 +81,12 @@
int numtransfers = 0;
int maxworkers = 20;
int maxretries = 15; /* give at least 15 attempts by default */
-int retrytime = 20; /* wait default of 20 seconds before trying again */
+int retrytime = 10; /* wait default of 20 seconds before trying again */
int singlefile = 0; /* Flag that indicates whether we are serving a single file */
char *path, *outname;
int ticksig;
u64 qpath = 1;
File *files;
-
time_t starttime;
time_t endtime;
time_t servicetime;
@@ -274,7 +275,7 @@
}
static int
-fileretry(File *f)
+fileretry(File *f, u64 offset)
{
char *tname;
@@ -297,14 +298,13 @@
spcfd_remove(f->datafd);
f->datafd = NULL;
- f->datalen = 0;
f->progress = time(NULL);
f->finished = 0;
f->datafid = spc_open(masterfs, tname, Oread);
if (!f->datafid)
goto error;
- f->datafd = spcfd_add(f->datafid, netreadcb, f);
+ f->datafd = spcfd_add(f->datafid, netreadcb, f, offset);
free(tname);
return 0;
@@ -386,7 +386,7 @@
fdone = 1;
for(f = files; f != NULL; f = f->next) {
// debug(Dbgfn, "tick\n");
- if (f->finished<0 && fileretry(f)<0)
+ if (f->finished<0 && fileretry(f, 0)<0)
return;
if (f->finished == 1)
@@ -397,7 +397,7 @@
fdone = 0;
if ((time(NULL) - f->progress) > retrytime
- && fileretry(f) < 0)
+ && fileretry(f, f->datalen) < 0)
return;
}
@@ -762,7 +762,7 @@
}
static File *
-filealloc(Spfile *parent, char *nname, char *lname, int datasize, int datalen,
+filealloc(Spfile *parent, char *nname, char *lname, u64 datasize, u64 datalen,
u32 mtime, u32 crc, u32 mode, Spuser *usr, Spgroup *grp)
{
int qp;
@@ -821,13 +821,10 @@
u32 crc, npmode;
Spuser *usr;
Spgroup *grp;
- u8 *buf;
f = NULL;
crc = 0;
ret = NULL;
- buf = NULL;
-
if ((name = strrchr(filename, '/')))
name++;
else
@@ -847,23 +844,13 @@
goto error;
}
- buf = sp_malloc(st.st_size);
- if (!buf)
- goto error;
-
- if ((read(fd, buf, st.st_size)) < st.st_size) {
- sp_uerror(errno);
- goto error;
- }
-
- crc = crc32(buf, st.st_size);
+ crc = fcrc32(fd, st.st_size);
f = filealloc(parent, name, filename, st.st_size, st.st_size,
st.st_mtime, crc, npmode, usr, grp);
if (!f)
goto error;
close(fd);
- free(buf);
debug(Dbgsrvfn, "Added file: %s\n", f->dir->name);
ret = f->dir;
} else if (S_ISDIR(st.st_mode)) {
@@ -904,18 +891,17 @@
error:
if (fd > -1)
close(fd);
- if (buf)
- free(buf);
return NULL;
}
static void
respondreqs(File *f)
{
- int count, fd;
+ int fd;
Req *req, *nreq;
Spfcall *rc;
u8 *buf;
+ u64 count;
req = f->reqs;
while (req != NULL) {
@@ -972,12 +958,12 @@
static void
netreadcb(Spcfd *fd, void *a)
{
- int n, lfd;
+ int n, lfd, readsize;
File *f;
- int crc;
+ u32 crc;
u8 *buf;
struct stat *st;
-
+
f = a;
st = NULL;
buf = NULL;
@@ -1003,15 +989,21 @@
st = NULL;
}
- buf = sp_malloc(f->datasize - f->datalen);
- n = spcfd_read(fd, buf, f->datasize - f->datalen);
- if ((lfd = open(f->lname, O_CREAT | O_RDWR | O_APPEND, 0600)) == -1) {
- sp_uerror(errno);
+ readsize = fd->iounit;
+ buf = sp_malloc(readsize);
+ n = spcfd_read(fd, buf, readsize);
+ if ((lfd = open(f->lname, O_CREAT | O_RDWR, 0600)) == -1) {
+ xget_uerror(errno);
goto error;
}
+ if ((lseek(lfd, f->datalen, SEEK_SET)) == (off_t)-1) {
+ xget_uerror(errno);
+ goto error;
+ }
+
if ((write(lfd, buf, n)) != n) {
- sp_uerror(errno);
+ xget_uerror(errno);
goto error;
}
@@ -1019,22 +1011,12 @@
if (f->datalen >= f->datasize) {
spcfd_remove(fd);
f->datafd = NULL;
- free(buf);
- buf = sp_malloc(f->datasize);
- if (!buf)
- goto error;
-
if ((lseek(lfd, 0, SEEK_SET)) == (off_t)-1) {
xget_uerror(errno);
goto error;
}
-
- if ((read(lfd, buf, f->datasize)) < f->datasize) {
- xget_uerror(errno);
- goto error;
- }
- crc = crc32(buf, f->datasize);
+ crc = fcrc32(lfd, f->datasize);
if (crc == f->crc)
f->finished = 1;
else
@@ -1058,7 +1040,7 @@
}
static int
-netfileread(Spfile *dir, char *lname, char *nname, int len, int mtime,
+netfileread(Spfile *dir, char *lname, char *nname, u64 len, int mtime,
u32 npmode, Spuser *usr, Spgroup *grp)
{
int n, blen, crc;
@@ -1155,7 +1137,7 @@
goto error;
}
- file->datafd = spcfd_add(file->datafid, netreadcb, file);
+ file->datafd = spcfd_add(file->datafid, netreadcb, file, 0);
free(buf);
return 0;
@@ -1259,12 +1241,13 @@
static int
netread(Spfile *parent, char *lprefix, char *nprefix, char *name)
{
- int i, len, mtime, ret, ecode;
+ int i, mtime, ret, ecode;
char *nname, *lname, *s, *ename;
Spwstat *st;
u32 npmode;
Spuser *usr;
Spgroup *grp;
+ u64 len;
usr = NULL;
grp = NULL;
Index: crc32.c
===================================================================
--- crc32.c (revision 685)
+++ crc32.c (working copy)
@@ -23,6 +23,8 @@
#include "spfs.h"
+#define CRC_MAX_READ 640000
+
//crc32 and crc32 were taken from: http://fxr.watson.org/fxr/source/libkern/crc32.c
static u32 crc32_tab[] = {
@@ -83,3 +85,32 @@
return crc ^ ~0U;
}
+u32
+fcrc32( int fd, size_t size)
+{
+ ssize_t bytes_read=0;
+ u8 *buf, *p=NULL;
+ u32 crc=~0U;
+
+ if (!( buf = sp_malloc(CRC_MAX_READ)))
+ goto crc_error;
+
+ while ((bytes_read = read(fd, buf, CRC_MAX_READ)) > 0){
+ p = buf;
+ while (bytes_read--)
+ crc = crc32_tab[(crc ^ *p++) & 0xFF] ^ (crc >> 8);
+ }//while
+
+ if (bytes_read < 0 ) {
+ sp_uerror(errno);
+ goto crc_error;
+ }
+
+ free(buf);
+ return crc ^ ~0U;
+
+crc_error:
+ if (buf)
+ free(buf);
+ return 0xFF ; //??
+}
Index: xget.h
===================================================================
--- xget.h (revision 685)
+++ xget.h (working copy)
@@ -25,8 +25,8 @@
struct File {
char* nname;
char* lname;
- int datasize;
- int datalen;
+ u64 datasize;
+ u64 datalen;
Spfile* dir;
Spfile* datafile;
u8* data;
@@ -86,8 +86,9 @@
static void connopen(Spconn *conn);
static void connclose(Spconn *conn);
static Spfile *localfileread(Spfile *parent, char* filename);
-static File *filealloc(Spfile *parent, char *name, char *lname, int datasize,
- int datalen, u32 mtime, u32 crc,
+static File *filealloc(Spfile *parent, char *name, char *lname,
+ u64 datasize, u64 datalen, u32 mtime, u32 crc,
u32 mode, Spuser *user, Spgroup *group);
static int file_finalize(File *f, int write);
u32 crc32(const void *buf, size_t size);
+u32 fcrc32(int fd, size_t size);
