This patch adds to my last patch about checksum calculation changes. I
have added an explicit check in xget to see if the file(s) being served
out belong to procfs. xget just skips over those files.
This patch uses the zlib adler32 function to calculate the checksum of
the files.
Instead of calculating the checksum during file setup stage, we use the
rolling hash technique to compute the checksum
while the server serves the files to the clients. This reduces the
server setup time from almost zero to a few seconds.
The clients compute their checksum while downloading the file which also
reduces the time in which the file is available
to the clients
Signed-off-by: Abhishek Kulkarni <[EMAIL PROTECTED]>
Index: xget/xget.c
===================================================================
--- xget/xget.c (revision 713)
+++ xget/xget.c (working copy)
@@ -18,10 +18,11 @@
#include <math.h>
#include <pthread.h>
#include <sys/types.h>
-#include <sys/stat.h>
+#include <sys/statfs.h>
#include <unistd.h>
#include <sys/time.h>
#include <sys/mman.h>
+#include <zlib.h>
#include "spfs.h"
#include "spclient.h"
@@ -30,6 +31,8 @@
#include "xcpu.h"
#include "xget.h"
+#define PROC_SUPER_MAGIC 0x9FA0
+
Spdirops root_ops = {
.first = dir_first,
.next = dir_next,
@@ -358,7 +361,7 @@
goto error;
}
- f->retries++;
+ f->retries++;
if ((file_finalize(f, 1)) < 0)
goto error;
@@ -368,10 +371,16 @@
f->datafd = NULL;
}
+ f->checksum = adler32(0L, Z_NULL, 0);
+ f->checksum_ptr = 0;
f->progress = time(NULL);
f->finished = 0;
if (offset == 0)
f->datalen = 0;
+
+ f->checksumfid = spc_open(masterfs, tname, Oread);
+ if (!f->checksumfid)
+ goto error;
f->datafid = spc_open(masterfs, tname, Oread);
if (!f->datafid)
@@ -475,15 +484,16 @@
fdone = 1;
for(f = files; f != NULL; f = f->next) {
// debug(Dbgfn, "tick\n");
+
+ if (f->finished == 1 && (f->finished = matchsum(f)) == 1)
+ file_finalize(f, 0);
+
if (f->finished<0) {
debug(Dbgclntfn, "File: %s checksum did not match, retrying\n",
f->nname);
if (fileretry(f, 0) < 0)
return -1;
}
-
- if (f->finished == 1)
- file_finalize(f, 0);
if (f->finished == 2)
continue;
@@ -608,6 +618,7 @@
int blen, bsize, len, ecode;
char *buf, *ename;
struct stat st;
+ struct statfs stfs;
Spfile *flist, *f, *f1;
DIR *d;
struct dirent *de;
@@ -665,10 +676,17 @@
if (lstat(buf, &st) < 0)
goto error;
+ if (statfs(buf, &stfs) < 0)
+ goto error;
+
/* skip files other than regular files */
if (!S_ISREG(st.st_mode) && !S_ISDIR(st.st_mode))
continue;
+ /* skip special filesystem files (like procfs) */
+ if (stfs.f_type == PROC_SUPER_MAGIC)
+ continue;
+
debug(Dbgfn, "Adding file: %s\n", buf);
localfileread(dir, buf);
if (sp_haserror())
@@ -900,10 +918,12 @@
f->lastworker = NULL;
f->nextworker = NULL;
f->reqs = NULL;
+ f->checksumfid = NULL;
f->datafid = NULL;
f->availfid = NULL;
f->datafd = NULL;
f->checksum = checksum;
+ f->checksum_ptr = 0;
f->finished = 0;
f->progress = time(NULL);
f->retries = 0;
@@ -930,11 +950,12 @@
{
char *name, *nextf, *ename;
struct stat st;
+ struct statfs stfs;
File *f;
Spfile *dir, *ret;
DIR *dirstr;
struct dirent *de;
- int fd, ecode;
+ int ecode;
u32 checksum, npmode;
Spuser *usr;
Spgroup *grp;
@@ -949,13 +970,22 @@
if ((name = strrchr(filename, '/')))
name++;
else
- name = filename;
+ name = filename;
- if (lstat(filename, &st) < 0) {
+ if (lstat(filename, &st) < 0) {
sp_uerror(errno);
goto error;
}
+ if (statfs(filename, &stfs) < 0) {
+ sp_uerror(errno);
+ goto error;
+ }
+
+ /* Make sure we don't serve out procfs */
+ if(stfs.f_type == PROC_SUPER_MAGIC)
+ goto error;
+
npmode = umode2npmode(st.st_mode);
if (!rootonly) {
usr = sp_unix_users->uid2user(sp_unix_users, st.st_uid);
@@ -963,23 +993,12 @@
}
if (S_ISREG(st.st_mode)) {
- if ((fd = open(filename, O_RDONLY)) == -1) {
- sp_uerror(errno);
- goto error;
- }
-
- checksum = fcrc32(fd, st.st_size);
- if (checksum == 255) {
- sp_werror("cannot compute checksum", EPERM);
- goto error;
- }
+ checksum = adler32(0L, Z_NULL, 0);
f = filealloc(parent, name, filename, st.st_size, st.st_size,
st.st_mtime, checksum, npmode, usr, grp);
if (!f)
goto error;
- close(fd);
-
debug(Dbgsrvfn, "Added file: %s\n", f->dir->name);
ret = f->dir;
@@ -1038,9 +1057,6 @@
sp_werror(NULL, 0);
}
- if (fd > -1)
- close(fd);
-
if (nextf)
free(nextf);
@@ -1083,6 +1099,11 @@
if ((n = read(fd, buf, count)) < 0)
goto error;
+
+ if (req->offset == f->checksum_ptr) {
+ f->checksum = adler32(f->checksum, (const Bytef *)buf, n);
+ f->checksum_ptr += n;
+ }
close(fd);
if (n < count)
@@ -1115,6 +1136,9 @@
free(buf);
if (fd > 0)
close(fd);
+
+ rc = sp_create_rflush();
+ sp_respond(req->req, rc);
}
static void
@@ -1122,7 +1146,6 @@
{
int n, lfd, readsize;
File *f;
- u32 checksum;
u8 *buf;
struct stat *st;
@@ -1154,9 +1177,9 @@
readsize = fd->iounit;
buf = sp_malloc(readsize);
n = spcfd_read(fd, buf, readsize);
- if (n < 0)
+ if (n < 0 || ((n == 0 && f->datasize > 0)))
goto error;
-
+
if ((lfd = open(f->lname, O_CREAT | O_RDWR, 0600)) == -1) {
xget_uerror(errno);
goto error;
@@ -1171,6 +1194,11 @@
xget_uerror(errno);
goto error;
}
+
+ if (f->datalen == f->checksum_ptr) {
+ f->checksum = adler32(f->checksum, (const Bytef *)buf, n);
+ f->checksum_ptr += n;
+ }
f->datalen += n;
if (f->datalen >= f->datasize) {
@@ -1180,12 +1208,7 @@
xget_uerror(errno);
goto error;
}
-
- checksum = fcrc32(lfd, f->datasize);
- if (checksum == f->checksum)
- f->finished = 1;
- else
- f->finished = -1;
+ f->finished = 1;
}
f->progress = time(NULL);
@@ -1208,10 +1231,45 @@
}
static int
+matchsum(File *f)
+{
+ u32 checksum;
+ char *buf;
+ int n, blen = 16;
+
+ if (!f->checksumfid)
+ return -1;
+
+ buf = sp_malloc(blen);
+ if (!buf)
+ goto error;
+
+ n = spc_read(f->checksumfid, (u8 *) buf, blen, 0);
+ if (n < 0)
+ goto error;
+
+ buf[n] = '\0';
+ checksum = strtoul(buf, NULL, 0);
+ spc_close(f->checksumfid);
+ free(buf);
+ if (f->checksum == checksum)
+ return 1;
+ else
+ return -1;
+
+error:
+ if (buf)
+ free(buf);
+
+ return -1;
+}
+
+
+static int
netfileread(Spfile *dir, char *lname, char *nname, u64 len, int mtime,
u32 npmode, Spuser *usr, Spgroup *grp)
{
- int n, blen, checksum;
+ int n, blen;
char *buf, *fname, *redirto;
Spcfid *datafid, *checksumfid, *availfid, *redirfid;
Spcfsys *redirfs;
@@ -1229,26 +1287,14 @@
buf = sp_malloc(blen);
if (!buf)
return -1;
-
- sprintf(buf, "%s/checksum", nname);
- checksumfid = spc_open(masterfs, buf, Oread);
- if (!checksumfid)
- goto error;
-
- n = spc_read(checksumfid, (u8 *) buf, blen, 0);
- if (n < 0)
- goto error;
-
- buf[n] = '\0';
- checksum = strtoul(buf, NULL, 0);
- spc_close(checksumfid);
+
fname = strrchr(nname, '/');
if (!fname)
fname = nname;
else
fname++;
- file = filealloc(dir, fname, lname, len, 0, mtime, 0, npmode,
+ file = filealloc(dir, nname, lname, len, 0, mtime, 0, npmode,
usr, grp);
if (!file)
goto error;
@@ -1282,6 +1328,11 @@
redirfs = masterfs;
}
+ sprintf(buf, "%s/checksum", nname);
+ checksumfid = spc_open(redirfs, buf, Oread);
+ if (!checksumfid)
+ goto error;
+
sprintf(buf, "%s/data", nname);
datafid = spc_open(redirfs, buf, Oread);
if (!datafid)
@@ -1294,13 +1345,14 @@
snprintf(buf, blen, "%d %s", port, redirto);
n = spc_write(availfid, (u8 *) buf, strlen(buf) + 1, 0);
- if (n < 0) {
+ if (n < 0)
goto error;
- }
file->fs = redirfs;
file->datafid = datafid;
- file->checksum = checksum;
+ file->checksum = adler32(0L, Z_NULL, 0);
+ file->checksumfid = checksumfid;
+ file->checksum_ptr = 0;
file->datafd = spcfd_add(file->datafid, netreadcb, file, 0);
file->availfid = availfid;
free(buf);
@@ -1312,9 +1364,6 @@
if (datafid)
spc_close(datafid);
- if (checksumfid)
- spc_close(checksumfid);
-
if (availfid)
spc_close(availfid);
@@ -1340,6 +1389,7 @@
char **fnames;
struct timeval tv;
+ n = 0;
fid = spc_open(masterfs, nname, Oread);
if (!fid)
return -1;
@@ -1497,7 +1547,7 @@
grp = sp_unix_users->gname2group(sp_unix_users, st->gid);
}
- ret = netdirread(parent, lname, nname, npmode, usr, grp);
+ ret = netdirread(parent, lname, nname, npmode, usr, grp);
} else {
len = st->length;
mtime = st->mtime;
@@ -1511,18 +1561,18 @@
for (i=0; i < maxretries; i++) {
sp_werror(NULL, 0);
- ret = netfileread(parent, lname, nname, len, mtime,
+ ret = netfileread(parent, lname, nname, len, mtime,
npmode, usr, grp);
+
if (!ret)
break;
}
}
-
- if (ret == -1 || sp_haserror()) {
+
+ if (ret == -1 || sp_haserror()) {
sp_rerror(&ename, &ecode);
debug(Dbgfn, "Fatal error while setting up file %s\n", nname);
sp_werror(NULL, 0);
-
}
free(st);
Index: xget/xget.h
===================================================================
--- xget/xget.h (revision 713)
+++ xget/xget.h (working copy)
@@ -38,6 +38,7 @@
Spcfsys*fs;
Spcfid* datafid; /* used while reading the file */
Spcfid* availfid;
+ Spcfid* checksumfid;
Spcfd* datafd;
int numworkers;
@@ -48,6 +49,7 @@
File* next;
File* prev;
u32 checksum;
+ u64 checksum_ptr;
int finished;
time_t progress;
int retries;
@@ -112,5 +114,4 @@
u64 datasize, u64 datalen, u32 mtime, u32 checksum,
u32 mode, Spuser *user, Spgroup *group);
static int file_finalize(File *f, int write);
-u32 crc32(const void *buf, size_t size);
-u32 fcrc32(int fd, u64 size);
+static int matchsum(File *f);
Index: xget/Makefile
===================================================================
--- xget/Makefile (revision 713)
+++ xget/Makefile (working copy)
@@ -2,13 +2,12 @@
SYSNAME!=uname
INCDIR=../include
CFLAGS=-Wall -g -I $(INCDIR) -DSYSNAME=$(SYSNAME)
-LFLAGS=-L. -L../libstrutil -lstrutil -L../libspclient -lspclient -L../libspfs -lspfs -lm
+LFLAGS=-L. -L../libstrutil -lstrutil -L../libspclient -lspclient -L../libspfs -lspfs -lm -lz
HFILES=$(INCDIR)/spfs.h $(INCDIR)/spclient.h $(INCDIR)/xcpu.h $(INCDIR)/strutil.h xget.h
CMD=xget
OFILES=\
xget.o\
- crc32.o\
all: $(CMD)
@@ -16,7 +15,7 @@
$(CC) -o xget $(CFLAGS) $(OFILES) $(LFLAGS)
xget.static: $(OFILES) $(HFILES) Makefile
- $(CC) -static -o xget.static $(CFLAGS) $(OFILES) $(LFLAGS) -lm
+ $(CC) -static -o xget.static $(CFLAGS) $(OFILES) $(LFLAGS) -lm -lz
install:
mkdir -p $(INSTALLPREFIX)/sbin
