Hello community, here is the log from the commit of package clicfs for openSUSE:Factory checked in at Mon Mar 21 15:32:46 CET 2011.
-------- --- clicfs/clicfs.changes 2010-11-30 13:38:39.000000000 +0100 +++ clicfs/clicfs.changes 2011-03-21 14:58:39.000000000 +0100 @@ -1,0 +2,8 @@ +Mon Mar 21 13:57:39 UTC 2011 - [email protected] + +- update to 1.4.0 + - rework COW file format (packed file format unchanged) + - make use of more threads again + - make COW syncing much more robust and faster too + +------------------------------------------------------------------- calling whatdependson for head-i586 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ clicfs.spec ++++++ --- /var/tmp/diff_new_pack.ppkIJl/_old 2011-03-21 15:29:36.000000000 +0100 +++ /var/tmp/diff_new_pack.ppkIJl/_new 2011-03-21 15:29:36.000000000 +0100 @@ -1,7 +1,7 @@ # -# spec file for package clicfs (Version 1.3.10) +# spec file for package clicfs # -# Copyright (c) 2010 SUSE LINUX Products GmbH, Nuernberg, Germany. +# Copyright (c) 2011 SUSE LINUX Products GmbH, Nuernberg, Germany. # # All modifications and additions to the file contributed by third parties # remain the property of their copyright owners, unless otherwise agreed @@ -22,7 +22,7 @@ BuildRequires: cmake e2fsprogs-devel fuse-devel gcc-c++ openssl-devel xz-devel Requires: fuse Summary: Compressed Loop Image Container -Version: 1.3.10 +Version: 1.4.0 Release: 1 License: GPLv2 Group: System/Filesystems @@ -36,7 +36,6 @@ creating a copy on write behaviour. - %prep %setup -c %name ++++++ clicfs.tar.bz2 ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/README new/README --- old/README 2010-11-30 13:37:15.000000000 +0100 +++ new/README 2011-03-21 14:55:37.000000000 +0100 @@ -1 +1,9 @@ This is not really worth the read (yet). + +COWFILE FORMAT: + 9 bytes: CLICCOW%02d + 8 bytes: file size (including sparse) + + num_pages*: + - 4 bytes: page index in cow (if > 0) + diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/misc/makeiso.sh new/misc/makeiso.sh --- old/misc/makeiso.sh 1970-01-01 01:00:00.000000000 +0100 +++ new/misc/makeiso.sh 2011-03-21 14:55:37.000000000 +0100 @@ -0,0 +1 @@ +make && cp src/clicfs initrd/usr/bin/clicfs && (cd initrd ; find . | cpio --create --format=newc --quiet | gzip -9 -f > ../CD1/boot/x86_64/loader/initrd) && genisoimage -R -J -f -pad -joliet-long -no-emul-boot -boot-load-size 4 -boot-info-table -b boot/x86_64/loader/isolinux.bin -o kde2.iso CD1/; isohybrid -id $(cat CD1/boot/grub/mbrid) kde2.iso && dd if=/dev/zero seek=1000 count=1 bs=1M of=kde2.iso diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/src/clicfs.c new/src/clicfs.c --- old/src/clicfs.c 2010-11-30 13:37:15.000000000 +0100 +++ new/src/clicfs.c 2011-03-21 14:55:37.000000000 +0100 @@ -17,6 +17,7 @@ */ #define FUSE_USE_VERSION 26 +#define _GNU_SOURCE #include <unistd.h> #include "clicfs.h" @@ -31,7 +32,7 @@ #include <sys/time.h> #include <sys/mman.h> -#define DEBUG 0 +//#define DEBUG 1 FILE *logger = 0; @@ -46,30 +47,31 @@ static uint32_t clic_find_next_cow() { - if (cows_index > 0) - return cows[--cows_index]; - return cow_pages + cow_index_pages; + if (cows_index > 0) { + //if (logger) fprintf(logger, "find_next (old): %d\n", cows[cows_index-1]); + return cows[--cows_index]; + } + //if (logger) fprintf(logger, "find_next (new): %d\n", cow_pages + 1); + return cow_pages + 1; } -static int clic_detach(size_t block, int islock); +static int clic_detach(size_t block); static int clic_write_cow(); pthread_mutex_t cowfile_mutex = PTHREAD_MUTEX_INITIALIZER; pthread_mutex_t cowfile_mutex_writer = PTHREAD_MUTEX_INITIALIZER; +pthread_mutex_t coms_by_part_mutex = PTHREAD_MUTEX_INITIALIZER; -static int clic_write_cow(int islocked) +static int clic_write_cow() { if (!cowfilename || cowfile_ro == 1 || !detached_allocated) return 0; int ret = 0; - //if (logger) fprintf(logger, "cow detached %dMB\n", (int)(detached_allocated / 1024)); + if (logger) fprintf(logger, "cow detached %dMB\n", (int)(detached_allocated / 1024)); if (logger) fprintf(logger, "clic_write_cow %ld\n", pthread_self()); - if (!islocked) { - pthread_mutex_lock(&cowfile_mutex_writer); - } pthread_mutex_lock(&cowfile_mutex); uint32_t i; @@ -78,95 +80,38 @@ long ptr = (long)blockmap[i]; if ( ptr && PTR_CLASS(ptr) == CLASS_MEMORY ) { // detached now off_t cowindex = clic_find_next_cow(); - off_t seeked = lseek(cowfilefd, cowindex * pagesize, SEEK_SET); - assert(seeked == (off_t)(cowindex * pagesize)); - size_t ret = write(cowfilefd, blockmap[i], pagesize); - assert(ret == pagesize); + ssize_t pret = pwrite(cowfilefd, blockmap[i], pagesize, cowindex * pagesize + cow_pages_start); + //if (logger) fprintf(logger, "pwrote %ld %ld -> %ld\n", pagesize, cowindex * pagesize + cow_pages_start, pret); + if (pret < 0) { + perror("pwrite cow"); + if (logger) fprintf(logger, "failed %s\n", strerror(errno)); + ret = -errno; + cowfile_ro = 1; + goto exit; + } + assert(pret == (ssize_t)pagesize); free(blockmap[i]); detached_allocated -= (pagesize / 1024); - blockmap[i] = (unsigned char*)(long)(cowindex << 2) + 2; - cow_pages++; - } - } - - assert(!detached_allocated); - - off_t seeked = lseek(cowfilefd, 0, SEEK_SET); - assert(seeked == 0); - uint64_t stringlen = thefilesize; - - char head[10]; - sprintf(head, "CLICCOW%02d", DOENER_MAGIC); - uint32_t index_len = write(cowfilefd, head, 9); - - index_len += write(cowfilefd, (char*)&stringlen, sizeof(uint64_t)); - stringlen = cow_pages; - index_len += write(cowfilefd, (char*)&stringlen, sizeof(uint32_t)); - stringlen = 0; - - index_len += 2 * sizeof(uint32_t) * cow_pages; - uint32_t new_cow_index_pages = index_len / pagesize + 1; - uint32_t moving; - uint32_t moved = 0; - - // should all be out - assert(cows_index == 0); - - pthread_mutex_unlock(&cowfile_mutex); - - for (moving = cow_index_pages; moving < new_cow_index_pages; ++moving) - { - // we only have a map from memory to cow, so we need to - // look up in reverse to find the page to move - // if this proves to be slow, we need even more memory - // to keep the reverse map - for (i = 0; i < num_pages; ++i) - { - long ptr = (long)blockmap[i]; - if (PTR_CLASS(ptr) == CLASS_COW) { // block - if ((uint32_t)(ptr >> 2) == moving) { - if (logger) fprintf(logger, "moving %ld %ld\n", (long)moving, (long)i); - clic_detach(i, 1); - moved++; - break; - } + blockmap[i] = (unsigned char*)(long)(cowindex << 2) + CLASS_COW; + uint32_t value = cowindex; + off_t offset = cow_index_start + i * sizeof(uint32_t); + pret = pwrite(cowfilefd, (char*)&value, sizeof(uint32_t), offset); + if (pret < 0) { + perror("pwrite2 cow"); + ret = -errno; } + //if (logger) fprintf(logger, "pwrote2 %d %d %ld -> %ld\n", i, cowindex, offset, pret); + cow_pages++; } } - assert(moved == cows_index); - - cow_index_pages = new_cow_index_pages; - - /* if we moved, we need to redetach */ - if (moved) { - cows_index = 0; - ret = clic_write_cow(1); - goto exit; - } - - pthread_mutex_lock(&cowfile_mutex); - - for (i = 0; i < num_pages; ++i) - { - long ptr = (long)blockmap[i]; - if (PTR_CLASS(ptr) == CLASS_COW) { // block - uint32_t key = i, value = ptr >> 2; - write(cowfilefd, (char*)&key, sizeof(uint32_t)); - write(cowfilefd, (char*)&value, sizeof(uint32_t)); - stringlen++; - } - } - - assert(stringlen == cow_pages); - write(cowfilefd, (char*)&index_len, sizeof(uint32_t)); + fdatasync(cowfilefd); + last_sync = time(0); + // not true for threads assert(!detached_allocated); - pthread_mutex_unlock(&cowfile_mutex); - exit: - if (!islocked) - pthread_mutex_unlock(&cowfile_mutex_writer); - + pthread_mutex_unlock(&cowfile_mutex); + if (logger) fprintf(logger, "clic_write_cow %ld done %d\n", pthread_self(), ret); return ret; } @@ -247,8 +192,6 @@ struct buffer_combo *coms_sort_by_use_last = 0; static unsigned int com_count = 0; -pthread_mutex_t picker = PTHREAD_MUTEX_INITIALIZER, seeker = PTHREAD_MUTEX_INITIALIZER;; - FILE *pack; static void clic_append_by_use(struct buffer_combo *com) @@ -280,6 +223,7 @@ return; } if (coms_sort_by_part_size == after + 1) { // just append + assert(coms_sort_by_part_size < MAX_COMS_SIZE); coms_by_part[coms_sort_by_part_size] = com; } else { // I don't like memmove @@ -287,6 +231,7 @@ for (i = coms_sort_by_part_size-1; i > after; i--) coms_by_part[i+1] = coms_by_part[i]; coms_by_part[after+1] = com; + assert(after + 1 < MAX_COMS_SIZE); } coms_sort_by_part_size++; clic_append_by_use(com); @@ -360,6 +305,7 @@ } } else free(com->out_buffer); + if (logger) fprintf(logger, "free block %d\n", com->part); memory_used -= com->out_buffer_size; int32_t res = binary_search(coms_by_part, coms_sort_by_part_size, com->part); assert(coms_by_part[res] == com); @@ -379,11 +325,13 @@ //if (logger) fprintf(logger, "clic_uncompress %d %d\n", part, parts); time_t now = time(0); + pthread_mutex_lock(&coms_by_part_mutex); + if (coms_sort_by_use_first) // clean up { if (0) clic_dump_use(); // if the oldest is 1m, drop it - while (coms_sort_by_use_first && (now - coms_sort_by_use_first->last_used > 30 || (memory_used > 1024 * 1024 * 10 && coms_sort_by_use_first->part != part))) { + while (coms_sort_by_use_first && (now - coms_sort_by_use_first->last_used > 40 || (memory_used > 1024 * 1024 * 40 && coms_sort_by_use_first->part != part))) { clic_free_com(coms_sort_by_use_first); } //clic_dump_use(); @@ -397,6 +345,7 @@ com->last_used = now; clic_remove_com_from_use(com); clic_append_by_use(com); + pthread_mutex_unlock(&coms_by_part_mutex); return buf; } @@ -409,6 +358,7 @@ } struct buffer_combo *com = malloc(sizeof(struct buffer_combo)); + assert(com); memory_used += sizeof(struct buffer_combo); if (part < largeparts) { com->out_buffer_size = blocksize_large*pagesize; @@ -422,6 +372,7 @@ } else { com->out_buffer_size = blocksize_small*pagesize; com->out_buffer = malloc(blocksize_small*pagesize); + assert(com->out_buffer); com->mmapped = 0; } memory_used += com->out_buffer_size; @@ -430,12 +381,12 @@ clic_insert_com(com, res); - pthread_mutex_lock(&seeker); + pthread_mutex_unlock(&coms_by_part_mutex); unsigned char *inbuffer = malloc(sizes[part]); + assert(inbuffer); struct timeval begin, end; gettimeofday(&begin, 0); size_t readin = clic_readpart(inbuffer, part); - pthread_mutex_unlock(&seeker); gettimeofday(&end, 0); if (!readin) { free(inbuffer); @@ -446,7 +397,7 @@ if (logger) fprintf(logger, "uncompress %d %ld-%ld %ld (read took %ld - started %ld)\n", part, (long)offs[part], (long)sizes[part], (long)readin, (end.tv_sec - begin.tv_sec) * 1000 + (end.tv_usec - begin.tv_usec) / 1000, (begin.tv_sec - start.tv_sec) * 1000 + (begin.tv_usec - start.tv_usec) / 1000 ); #endif if (!clic_decompress_part(com->out_buffer, inbuffer, readin)) { - if (logger) fprintf(logger, "uncompess of part %d failed - ignoring", part); + if (logger) fprintf(logger, "uncompess of part %d failed - ignoring\n", part); } free(inbuffer); @@ -475,81 +426,105 @@ static ssize_t clic_read_block(char *buf, size_t block); -static int clic_detach(size_t block, int islocked) +static int clic_detach(size_t block) { assert(block < num_pages); - if (!islocked) - pthread_mutex_lock(&cowfile_mutex_writer); - int ret = 0; + //if (logger) fprintf(logger, "clic_detach\n"); unsigned char *ptr = blockmap[block]; + //if (logger) fprintf(logger, "clic_detach1 %ld\n", PTR_CLASS(ptr)); if ((PTR_CLASS(ptr) == CLASS_RO ) || (PTR_CLASS(ptr) == CLASS_COW)) { if (PTR_CLASS(ptr) == CLASS_COW) { - if (cows_index == CLICFS_COW_COUNT - 1) - clic_write_cow(1); + if (logger) fprintf(logger, "detach2 cow %d index\n", cows_index); + if (cows_index == CLICFS_COW_COUNT - 1) { + ret = clic_write_cow(); + if (logger) fprintf(logger, "detach cow %d\n", ret); + } + } + + if (cowfilename && cowfile_ro == 1) { + ret = -EROFS; + goto exit; } char *newptr = malloc(pagesize); + assert(newptr); + //if (logger) fprintf(logger, "clic_detach3 %ld\n", PTR_CLASS(newptr)); detached_allocated += (pagesize / 1024); if (logger && detached_allocated % 1024 == 0 ) fprintf(logger, "detached %dMB\n", (int)(detached_allocated / 1024)); clic_read_block(newptr, block); - if (PTR_CLASS(ptr) == CLASS_COW) { // we need to mark the place in the cow obsolete - //if (logger) fprintf(logger, "detach block %ld (was %ld)\n", (long)block, (long)ptr >> 2); + if (PTR_CLASS(ptr) == CLASS_COW && !cowfile_ro) { // we need to mark the place in the cow obsolete + if (logger) fprintf(logger, "detach block %ld (was %ld)\n", (long)block, (long)ptr >> 2); + assert(cows_index < CLICFS_COW_COUNT); cows[cows_index++] = (long)ptr >> 2; cow_pages--; } blockmap[block] = (unsigned char*)newptr; - ret = 1; goto exit; } + //if (logger) fprintf(logger, "clic_detach2 %p\n", blockmap[block]); if (!blockmap[block]) { - blockmap[block] = malloc(pagesize); - assert(PTR_CLASS(ptr) == CLASS_MEMORY); + if (cowfilename && cowfile_ro == 1) { + ret = -EROFS; + goto exit; + } + + blockmap[block] = malloc(pagesize); + //if (logger) fprintf(logger, "clic_detach4 %p\n", blockmap[block]); + assert(blockmap[block]); + //assert(PTR_CLASS(ptr) == CLASS_MEMORY); detached_allocated += (pagesize / 1024); if (logger && detached_allocated % 1024 == 0 ) fprintf(logger, "detached %dMB\n", (int)(detached_allocated / 1024)); memset(blockmap[block],0,pagesize); - ret = 1; } exit: - if (!islocked) - pthread_mutex_unlock(&cowfile_mutex_writer); - + //if (logger) fprintf(logger, "clic_detach done %d\n", ret); return ret; } static size_t clic_write_block(const char *buf, off_t block, off_t ioff, size_t size) { - clic_detach(block, 1); + //if (logger) fprintf(logger, "clic_write_block %ld\n", detached_allocated); + if (clic_detach(block)) { + if (logger) fprintf(logger, "clic_detach FAILED\n"); + return -ENOSPC; + } memcpy(blockmap[block]+ioff, buf, size); + + if (detached_allocated > 40000) + clic_write_cow(); + return size; } static int clic_write(const char *path, const char *buf, size_t size, off_t offset, struct fuse_file_info *fi) { - //if (logger) fprintf(logger, "write %s %ld %ld\n", path, offset, size); + if (logger) fprintf(logger, "%lx write0 %s %ld %ld %lx\n", pthread_self(), path, offset, size, pthread_self()); (void) fi; if(path[0] == '/' && strcmp(path + 1, thefile) != 0) return -ENOENT; - if (offset >= (off_t)thefilesize) + if (offset >= (off_t)thefilesize) { + if (logger) fprintf(logger, "%lx writeF %s %ld %ld -> 0!! %lx\n", pthread_self(), path, offset, size, pthread_self()); return 0; + } - if (offset+size > (off_t)thefilesize) + if ((off_t)(offset+size) > (off_t)thefilesize) size = thefilesize-offset; - if (!size) + if (!size) { + if (logger) fprintf(logger, "%lx write %s %ld %ld -> 0!!\n", pthread_self(), path, offset, size); return 0; - - pthread_mutex_lock(&cowfile_mutex_writer); + } off_t block = offset / pagesize; off_t ioff = offset - block * pagesize; @@ -561,12 +536,16 @@ int ret = 0; if (size <= pagesize) { + if (logger) fprintf(logger, "%lx write2 %s %ld %ld\n", pthread_self(), path, offset, size); ret = clic_write_block(buf, block, ioff, size); + if (logger) fprintf(logger, "%lx write3 %s %ld %ld -> %d\n", pthread_self(), path, offset, size, ret); } else { size_t wrote = 0; do { + if (logger) fprintf(logger, "%lx write4 %s %ld %ld %lx\n", pthread_self(), path, offset, size, pthread_self()); size_t diff = clic_write_block(buf, block, ioff, size > pagesize ? pagesize : size); + if (logger) fprintf(logger, "%lx write5 %s %ld %ld -> %ld\n", pthread_self(), path, offset, size, diff); ioff = 0; size -= diff; buf += diff; @@ -576,13 +555,13 @@ ret = wrote; } - pthread_mutex_unlock(&cowfile_mutex_writer); + if (logger) fprintf(logger, "%lx writeD %s %ld %ld -> %d\n", pthread_self(), path, offset, size, ret); return ret; } static ssize_t clic_read_block(char *buf, size_t block) { - if (block >= num_pages) + if (block >= num_pages) return -EFAULT; if (!blockmap[block]) { // sparse block @@ -600,8 +579,7 @@ if (PTR_CLASS(ptr) == CLASS_COW) { off_t target = ptr >> 2; pthread_mutex_lock(&cowfile_mutex); - lseek(cowfilefd, target * pagesize, SEEK_SET); - ssize_t haveread = read(cowfilefd, buf, pagesize); + ssize_t haveread = pread(cowfilefd, buf, pagesize, target * pagesize + cow_pages_start); pthread_mutex_unlock(&cowfile_mutex); return haveread; } @@ -635,7 +613,7 @@ static int clic_read(const char *path, char *buf, size_t size, off_t offset, struct fuse_file_info *fi) { - //if (logger) fprintf(logger, "read %ld %ld %ld\n", offset, size, thefilesize); + // if (logger) fprintf(logger, "read %ld %ld %ld\n", offset, size, thefilesize); (void) fi; if(path[0] == '/' && strcmp(path + 1, thefile) != 0) return -ENOENT; @@ -645,8 +623,6 @@ assert(size % pagesize == 0); assert(offset % pagesize == 0); - pthread_mutex_lock(&cowfile_mutex_writer); - do { if (offset >= (off_t)thefilesize) @@ -654,7 +630,8 @@ off_t block = offset / pagesize; ssize_t diff = clic_read_block(buf, block); if (diff < 0) { - return diff; + //if (logger) fprintf(logger, "read %ld %ld %ld -> %ld!!\n", offset, size, thefilesize, diff); + return diff; } //if (logger) fprintf(logger, "read block %ld: %ld bytes\n", (long)block, (long)diff); if (!diff) @@ -665,8 +642,7 @@ readtotal += diff; } while (size > 0); - pthread_mutex_unlock(&cowfile_mutex_writer); - + //if (logger) fprintf(logger, "read %ld %ld %ld -> %ld\n", offset, size, thefilesize, readtotal); return readtotal; } @@ -675,9 +651,8 @@ (void)path; (void)fi; // TODO write out cow - if (logger) fflush(logger); - clic_write_cow(0); - return 0; + if (logger) { fprintf(logger, "flush\n"); fflush(logger); } + return clic_write_cow(); } static int clic_fsync(const char *path, int datasync, struct fuse_file_info *fi) @@ -686,13 +661,10 @@ (void)fi; (void)datasync; // TODO write out cow - if (logger) fflush(logger); - clic_write_cow(0); - last_sync = time(0); - pthread_mutex_lock(&cowfile_mutex); - if (cowfilefd >= 0) fsync(cowfilefd); - pthread_mutex_unlock(&cowfile_mutex); - return 0; + if (logger) { fprintf(logger, "sync\n"); fflush(logger); } + + int ret = clic_write_cow(); + return ret; } static void *clic_sync_thread(void *arg) @@ -720,6 +692,7 @@ { // avoid random reads or our profiling will be destroyed conn->max_readahead = 0; + clic_sync_tid = 0; pthread_create(&clic_sync_tid, NULL, clic_sync_thread, 0); @@ -730,9 +703,11 @@ { (void)arg; if (logger) fprintf(logger, "destroy\n"); - pthread_cancel(clic_sync_tid); - void *res; - pthread_join(clic_sync_tid, &res); + if (clic_sync_tid > 0) { + pthread_cancel(clic_sync_tid); + void *res; + pthread_join(clic_sync_tid, &res); + } } @@ -810,18 +785,21 @@ assert( DOENER_MAGIC < 100 ); int index_len = fprintf(cow, "CLICCOW%02d", DOENER_MAGIC ); - index_len += fwrite((char*)&bigfilesize, 1, sizeof(uint64_t), cow); - uint32_t stringlen = 0; - // there are 0 blocks - index_len += fwrite((char*)&stringlen, 1, sizeof(uint32_t), cow); - // the whole index is 12 bytes long - stringlen = index_len + sizeof(uint32_t); - index_len += fwrite((char*)&stringlen, 1, sizeof(uint32_t), cow); - fclose(cow); - cow_index_pages = index_len / pagesize + 1; - cow_pages = 0; + char zeros[sizeof(uint32_t)]; + memset(zeros, 0, sizeof(uint32_t)); + + size_t write_pages = bigfilesize / pagesize; + blockmap = realloc(blockmap, sizeof(unsigned char*)*write_pages); + unsigned int i; + for (i = num_pages; i < write_pages; ++i) + blockmap[i] = 0; + num_pages = write_pages; + + for (i = 0; i < num_pages; ++i) + fwrite(zeros, 1, sizeof(uint32_t), cow); + fclose(cow); return 0; } @@ -848,7 +826,8 @@ } // not sure why but multiple threads make it slower - fuse_opt_add_arg(&args, "-s"); + //fuse_opt_add_arg(&args, "-s"); + fuse_opt_add_arg(&args, "-obig_writes"); if (!packfilename) { fprintf(stderr, "usage: [-m <mb>] [-l <logfile|->] [-c <cowfile>] <packfile> <mntpoint>\n"); @@ -862,6 +841,12 @@ free(packfilename); + int procfile = open("/proc/self/om_score_adj", O_WRONLY); + if (procfile > 0) { + write(procfile, "-1000", sizeof("-1000")); + close(procfile); + } + if (cowfilename) { if (access(cowfilename, R_OK)) @@ -891,16 +876,18 @@ } for (i = 0; i < largeparts; ++i) { - posix_fadvise( fileno(packfile), offs[i], sizes[i], POSIX_FADV_SEQUENTIAL); + posix_fadvise( packfilefd, offs[i], sizes[i], POSIX_FADV_SEQUENTIAL); } coms_by_part = malloc(sizeof(struct buffer_combo*)*MAX_COMS_SIZE); + assert(coms_by_part); gettimeofday(&start, 0); /* MAIN LOOP */ int ret = fuse_main(args.argc, args.argv, &clic_oper, NULL); - clic_write_cow(0); + clic_write_cow(); // ignored if (cowfilefd >= 0) close(cowfilefd); if (logger) fclose(logger); + logger=0; while (coms_sort_by_use_first) clic_free_com(coms_sort_by_use_first); @@ -916,7 +903,7 @@ free(blockmap); free(sizes); free(offs); - fclose(packfile); + close(packfilefd); if (cowfilename) free(cowfilename); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/src/clicfs.h new/src/clicfs.h --- old/src/clicfs.h 2010-11-30 13:37:15.000000000 +0100 +++ new/src/clicfs.h 2011-03-21 14:55:37.000000000 +0100 @@ -23,7 +23,7 @@ #include <sys/types.h> extern int preset; -extern FILE *packfile; +extern int packfilefd; extern int cowfilefd; // magic 2 added large parts @@ -53,8 +53,10 @@ extern uint32_t num_pages; // the number of pages marked as CLASS_COW extern uint32_t cow_pages; -// the number of pages in the cow for index -extern unsigned int cow_index_pages; +// the offset in the cow file where the page index starts +extern off_t cow_index_start; +// the offset in the cow file where the pages start +extern off_t cow_pages_start; // support temporary changes on ro medium extern int cowfile_ro; @@ -71,7 +73,7 @@ extern int clic_decompress_part(unsigned char *out, const unsigned char *in, size_t size); extern size_t clic_readpart(unsigned char *buffer, int part); extern off_t clic_map_block(off_t block); -extern uint32_t clic_readindex_fd(int fd ); +extern uint32_t clic_readindex_fd( int fd ); extern void clic_free_lzma(); extern void clic_find_block( off_t block, off_t *part, off_t *offset ); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/src/clicfs_common.c new/src/clicfs_common.c --- old/src/clicfs_common.c 2010-11-30 13:37:15.000000000 +0100 +++ new/src/clicfs_common.c 2011-03-21 14:55:37.000000000 +0100 @@ -16,6 +16,9 @@ 02110-1301, USA */ +#define _GNU_SOURCE +//#define DEBUG 1 + #include "clicfs.h" #include <stdio.h> #include <stdlib.h> @@ -26,9 +29,10 @@ #include <sys/stat.h> #include <unistd.h> #include <fcntl.h> +#include <pthread.h> int preset = 0; -FILE *packfile = 0; +int packfilefd = -1; int cowfilefd = -1; char thefile[PATH_MAX]; @@ -44,12 +48,14 @@ unsigned char **blockmap; uint32_t num_pages = 0; uint32_t cow_pages = 0; -uint32_t cow_index_pages = 0; +off_t cow_index_start = 0; +off_t cow_pages_start = 0; uint32_t *cows = 0; unsigned int cows_index = 0; int cowfile_ro = 0; static lzma_stream strm; +static pthread_mutex_t lzma_mutex = PTHREAD_MUTEX_INITIALIZER; uint32_t clic_readindex_fd(int fd) { @@ -69,15 +75,6 @@ return stringlen; } -uint32_t clic_readindex_file(FILE * f) -{ - uint32_t stringlen = 0; - if (fread(&stringlen, 1, sizeof(uint32_t), f) != sizeof(uint32_t)) { - return 0; - } - return stringlen; -} - int clicfs_read_cow(const char *cowfilename) { cowfilefd = open(cowfilename, O_RDWR); @@ -115,33 +112,39 @@ for (i = num_pages; i < newpages; ++i) blockmap[i] = 0; num_pages = newpages; - cow_pages = clic_readindex_fd(cowfilefd); - for (i = 0; i < cow_pages; ++i) + cow_pages = 0; + for (i = 0; i < num_pages; ++i) { - uint32_t pageindex = clic_readindex_fd(cowfilefd); uint32_t page = clic_readindex_fd(cowfilefd); - assert(pageindex < num_pages); - blockmap[pageindex] = (unsigned char*)(long)(page << 2) + 2; + if (page) { + blockmap[i] = (unsigned char*)(long)(page << 2) + CLASS_COW; + cow_pages++; + } } cows = malloc(sizeof(uint32_t) * CLICFS_COW_COUNT); cows_index = 0; - uint32_t index_len = clic_readindex_fd(cowfilefd); - cow_index_pages = index_len / pagesize + 1; + cow_index_start = 9 + sizeof(uint64_t); + cow_pages_start = cow_index_start + num_pages * sizeof(uint32_t); + // we do not round up but down here as cow pages start with 1 + cow_pages_start = ( cow_pages_start / pagesize + 0) * pagesize; return 0; } int clicfs_read_pack(const char *packfilename) { - packfile = fopen(packfilename, "r"); - if (!packfile) { + packfilefd = open(packfilename, O_LARGEFILE|O_RDONLY); + if (packfilefd < 0) { fprintf(stderr, "packfile %s can't be opened\n", packfilename); return 1; } char head[7]; char expected[7]; - fread(head, 1, 6, packfile); + if (read(packfilefd, head, 6) != 6) { + perror("can't read from packfile\n"); + return 1; + } head[6] = 0; sprintf(expected, "CLIC%02d", DOENER_MAGIC); if (strcmp(head,expected)) { @@ -149,46 +152,46 @@ return 1; } - uint32_t stringlen = clic_readindex_file(packfile); + uint32_t stringlen = clic_readindex_fd(packfilefd); if (stringlen == 0 || stringlen >= PATH_MAX) { fprintf(stderr, "abnormal len %lx\n", (long)stringlen); return 1; } - if (fread(thefile, 1, stringlen, packfile) != stringlen) { + if (read(packfilefd, thefile, stringlen) != stringlen) { fprintf(stderr, "short read %ld\n", (long)stringlen); return 1; } thefile[stringlen] = 0; - uint64_t oparts = clic_readindex_file(packfile); - blocksize_small = clic_readindex_file(packfile); - blocksize_large = clic_readindex_file(packfile); - pagesize = clic_readindex_file(packfile); + uint64_t oparts = clic_readindex_fd(packfilefd); + blocksize_small = clic_readindex_fd(packfilefd); + blocksize_large = clic_readindex_fd(packfilefd); + pagesize = clic_readindex_fd(packfilefd); thefilesize = oparts * blocksize_small * pagesize; - preset = clic_readindex_file(packfile); - num_pages = clic_readindex_file(packfile); + preset = clic_readindex_fd(packfilefd); + num_pages = clic_readindex_fd(packfilefd); blockmap = malloc(sizeof(unsigned char*)*num_pages); uint32_t i; for (i = 0; i < num_pages; ++i) { // make sure it's odd to diff between pointer and block - blockmap[i] = (unsigned char*)(long)((clic_readindex_file(packfile) << 2) + 1); + blockmap[i] = (unsigned char*)(long)((clic_readindex_fd(packfilefd) << 2) + 1); } - parts = clic_readindex_file(packfile); - largeparts = clic_readindex_file(packfile); + parts = clic_readindex_fd(packfilefd); + largeparts = clic_readindex_fd(packfilefd); sizes = malloc(sizeof(uint64_t)*parts); offs = malloc(sizeof(uint64_t)*parts); for (i = 0; i < parts; ++i) { - if (fread((char*)(sizes + i), sizeof(uint64_t), 1, packfile) != 1) + if (read(packfilefd, (char*)(sizes + i), sizeof(uint64_t)) != sizeof(uint64_t)) parts = 0; if (!sizes[i]) { fprintf(stderr, "unreasonable size 0 for part %d\n", i); return 1; } - if (fread((char*)(offs + i), sizeof(uint64_t), 1, packfile) != 1) + if (read(packfilefd, (char*)(offs + i), sizeof(uint64_t)) != sizeof(uint64_t)) parts = 0; if (i > 0 && offs[i] <= offs[i-1]) { fprintf(stderr, "the offset for i is not larger than i-1: %ld\n", (long)i); @@ -199,7 +202,7 @@ fprintf(stderr, "unreasonable part number 0\n"); return 1; } - fseeko(packfile, (oparts-parts)*sizeof(uint64_t)*2, SEEK_CUR); + lseek(packfilefd, (oparts-parts)*sizeof(uint64_t)*2, SEEK_CUR); const uint32_t flags = LZMA_TELL_UNSUPPORTED_CHECK | LZMA_CONCATENATED; // C sucks @@ -224,15 +227,11 @@ size_t clic_readpart(unsigned char *buffer, int part) { - if (fseeko(packfile, offs[part], SEEK_SET)) { - fprintf(stderr, "seek failed %d %lld\n", part, (long long)offs[part]); - return 0; - } #if defined(DEBUG) fprintf(stderr, "uncompress part=%d/%d off=%ld size=%ld\n", part, parts, offs[part], sizes[part] ); #endif - size_t readin = fread(buffer, 1, sizes[part], packfile); - if (readin != sizes[part]) { + ssize_t readin = pread(packfilefd, buffer, sizes[part], offs[part]); + if (readin != (ssize_t)sizes[part]) { fprintf(stderr, "short read: %d %ld %ld %ld\n", part, (long)offs[part], (long)sizes[part], (long)readin); // returning half read blocks won't help lzma return 0; @@ -242,6 +241,7 @@ int clic_decompress_part(unsigned char *out, const unsigned char *in, size_t readin) { + pthread_mutex_lock(&lzma_mutex); strm.next_in = in; strm.avail_in = readin; strm.next_out = out; @@ -264,12 +264,14 @@ if (ret == LZMA_DATA_ERROR) { fprintf(stderr, "lzma data corrupt!\n"); + pthread_mutex_unlock(&lzma_mutex); return 0; } #if defined(DEBUG) fprintf(stderr, "ret %d\n", ret); #endif assert (ret == LZMA_OK); + pthread_mutex_unlock(&lzma_mutex); /* don't use lzma_end (will free buffers) or LZMA_FINISH (will forbid any new use) */ return 1; } ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Remember to have fun... -- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
