moved sha1_file from sheep/farm to collie/farm and made minor modification.
Signed-off-by: Kai Zhang <[email protected]> --- collie/Makefile.am | 2 +- collie/farm/farm.c | 24 ++++ collie/farm/farm.h | 23 ++++ collie/farm/sha1_file.c | 273 +++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 321 insertions(+), 1 deletions(-) create mode 100644 collie/farm/farm.c create mode 100644 collie/farm/sha1_file.c diff --git a/collie/Makefile.am b/collie/Makefile.am index 9c30495..d185512 100644 --- a/collie/Makefile.am +++ b/collie/Makefile.am @@ -23,7 +23,7 @@ INCLUDES = -I$(top_builddir)/include -I$(top_srcdir)/include sbin_PROGRAMS = collie -collie_SOURCES = farm/object_rb_tree.c \ +collie_SOURCES = farm/object_rb_tree.c farm/sha1_file.c farm/farm.c \ collie.c common.c treeview.c vdi.c node.c cluster.c if BUILD_TRACE diff --git a/collie/farm/farm.c b/collie/farm/farm.c new file mode 100644 index 0000000..283cd9b --- /dev/null +++ b/collie/farm/farm.c @@ -0,0 +1,24 @@ +/* + * Copyright (C) 2011 Taobao Inc. + * Copyright (C) 2013 Zelin Inc. + * + * Liu Yuan <[email protected]> + * Kai Zhang <[email protected]> + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License version + * 2 as published by the Free Software Foundation. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include "farm.h" + +static char farm_obj_dir[PATH_MAX]; +static char farm_dir[PATH_MAX]; + +inline char *get_object_directory(void) +{ + return farm_obj_dir; +} diff --git a/collie/farm/farm.h b/collie/farm/farm.h index 3d08396..ca108ca 100644 --- a/collie/farm/farm.h +++ b/collie/farm/farm.h @@ -20,11 +20,34 @@ #include "strbuf.h" #include "sha1.h" +#define TAG_LEN 6 +#define TAG_DATA "data\0\0" +#define TAG_TRUNK "trunk\0" +#define TAG_SNAP "snap\0\0" + enum obj_type { OBJECT = 1, VDI, }; +struct sha1_file_hdr { + char tag[TAG_LEN]; + uint64_t size; + uint64_t priv; + uint64_t reserved; +}; + +/* farm.c */ +int farm_init(const char *path); +char *get_object_directory(void); + +/* sha1_file.c */ +int sha1_file_write(unsigned char *buf, unsigned len, unsigned char *); +void *sha1_file_read(const unsigned char *sha1, struct sha1_file_hdr *); +char *sha1_to_hex(const unsigned char *sha1); +int get_sha1_hex(const char *hex, unsigned char *sha1); +int sha1_file_try_delete(const unsigned char *sha1); + /* object_rb_tree.c */ int get_obj_nr(void); void obj_tree_insert(uint64_t oid, enum obj_type type, int nr_copies); diff --git a/collie/farm/sha1_file.c b/collie/farm/sha1_file.c new file mode 100644 index 0000000..6b62914 --- /dev/null +++ b/collie/farm/sha1_file.c @@ -0,0 +1,273 @@ +/* + * Copyright (C) 2011 Taobao Inc. + * + * Liu Yuan <[email protected]> + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License version + * 2 as published by the Free Software Foundation. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +/* + * sha1_file provide us some useful features: + * + * - Regardless of object type, all objects are all in deflated with zlib, + * and have a header that not only specifies their tag, but also size + * information about the data in the object. + * + * - the general consistency of an object can always be tested independently + * of the contents or the type of the object: all objects can be validated + * by verifying that their hashes match the content of the file. + */ +#include <sys/types.h> +#include <sys/xattr.h> + +#include "farm.h" +#include "util.h" + +static void fill_sha1_path(char *pathbuf, const unsigned char *sha1) +{ + int i; + for (i = 0; i < SHA1_LEN; i++) { + static const char hex[] = "0123456789abcdef"; + unsigned int val = sha1[i]; + char *pos = pathbuf + i*2 + (i > 0); + *pos++ = hex[val >> 4]; + *pos = hex[val & 0xf]; + } +} + +static char *sha1_to_path(const unsigned char *sha1) +{ + static char buf[PATH_MAX]; + const char *objdir; + int len; + + objdir = get_object_directory(); + len = strlen(objdir); + + /* '/' + sha1(2) + '/' + sha1(38) + '\0' */ + memcpy(buf, objdir, len); + buf[len] = '/'; + buf[len+3] = '/'; + buf[len+42] = '\0'; + fill_sha1_path(buf + len + 1, sha1); + return buf; +} + +#define CNAME "user.farm.count" +#define CSIZE sizeof(uint32_t) + +static void get_sha1_file(char *name) +{ + uint32_t count; + if (getxattr(name, CNAME, &count, CSIZE) < 0) { + if (errno == ENODATA) { + count = 1; + if (setxattr(name, CNAME, &count, CSIZE, 0) < 0) + panic("%m"); + return; + } else + panic("%m"); + } + count++; + if (setxattr(name, CNAME, &count, CSIZE, 0) < 0) + panic("%m"); +} + +static int put_sha1_file(char *name) +{ + uint32_t count; + + if (getxattr(name, CNAME, &count, CSIZE) < 0) { + if (errno == ENOENT) { + sd_dprintf("sha1 file doesn't exist"); + return -1; + } else + panic("%m"); + } + + count--; + if (count == 0) { + if (unlink(name) < 0) { + sd_dprintf("%m"); + return -1; + } + sd_dprintf("%s deleted", name); + } else { + if (setxattr(name, CNAME, &count, CSIZE, 0) < 0) + panic("%m"); + } + return 0; +} + +static int sha1_buffer_write(const unsigned char *sha1, + void *buf, unsigned int size) +{ + char *filename = sha1_to_path(sha1); + int fd, ret = 0, len; + + fd = open(filename, O_WRONLY | O_CREAT | O_EXCL, 0666); + if (fd < 0) { + if (errno != EEXIST) { + sd_eprintf("failed to open file %s with error: %m", + filename); + ret = -1; + } + goto err_open; + } + len = xwrite(fd, buf, size); + if (len != size) { + sd_dprintf("%m"); + close(fd); + return -1; + } + + close(fd); + get_sha1_file(filename); +err_open: + return ret; +} + +int sha1_file_write(unsigned char *buf, unsigned len, unsigned char *outsha1) +{ + unsigned char sha1[SHA1_LEN]; + struct sha1_ctx c; + + sha1_init(&c); + sha1_update(&c, buf, len); + sha1_final(&c, sha1); + + if (sha1_buffer_write(sha1, buf, len) < 0) + return -1; + if (outsha1) + memcpy(outsha1, sha1, SHA1_LEN); + return 0; +} + +static void *map_sha1_file(const unsigned char *sha1, unsigned long *size) +{ + char *filename = sha1_to_path(sha1); + int fd = open(filename, O_RDONLY); + struct stat st; + void *map; + + if (fd < 0) { + perror(filename); + return NULL; + } + if (fstat(fd, &st) < 0) { + sd_dprintf("%m"); + close(fd); + return NULL; + } + map = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0); + close(fd); + if (map == MAP_FAILED) { + sd_dprintf("%m"); + return NULL; + } + *size = st.st_size; + return map; +} + +static void *unpack_sha1_file(void *map, unsigned long mapsize, + struct sha1_file_hdr *hdr) +{ + int hdr_len; + char *buf; + + memcpy(hdr, map, sizeof(*hdr)); + hdr_len = sizeof(*hdr); + buf = valloc(hdr->size); + if (!buf) { + sd_dprintf("%m"); + return NULL; + } + + memcpy(buf, (char *)map + hdr_len, mapsize - hdr_len); + return buf; +} + +static int verify_sha1_file(const unsigned char *sha1, + void *buf, unsigned long len) +{ + unsigned char tmp[SHA1_LEN]; + struct sha1_ctx c; + + sha1_init(&c); + sha1_update(&c, buf, len); + sha1_final(&c, tmp); + + if (memcmp((char *)tmp, (char *)sha1, SHA1_LEN) != 0) { + sd_dprintf("failed, %s != %s", sha1_to_hex(sha1), + sha1_to_hex(tmp)); + return -1; + } + return 0; +} + +void *sha1_file_read(const unsigned char *sha1, struct sha1_file_hdr *hdr) +{ + unsigned long mapsize; + void *map, *buf; + + map = map_sha1_file(sha1, &mapsize); + if (map) { + if (verify_sha1_file(sha1, map, mapsize) < 0) + return NULL; + buf = unpack_sha1_file(map, mapsize, hdr); + munmap(map, mapsize); + return buf; + } + return NULL; +} + +int sha1_file_try_delete(const unsigned char *sha1) +{ + char *filename = sha1_to_path(sha1); + + return put_sha1_file(filename); +} + +static unsigned hexval(char c) +{ + if (c >= '0' && c <= '9') + return c - '0'; + if (c >= 'a' && c <= 'f') + return c - 'a' + 10; + if (c >= 'A' && c <= 'F') + return c - 'A' + 10; + return ~0; +} + +int get_sha1_hex(const char *hex, unsigned char *sha1) +{ + int i; + for (i = 0; i < SHA1_LEN; i++) { + unsigned int val = (hexval(hex[0]) << 4) | hexval(hex[1]); + if (val & ~0xff) + return -1; + *sha1++ = val; + hex += 2; + } + return 0; +} + +char *sha1_to_hex(const unsigned char *sha1) +{ + static char buffer[50]; + static const char hex[] = "0123456789abcdef"; + char *buf = buffer; + int i; + + for (i = 0; i < SHA1_LEN; i++) { + unsigned int val = *sha1++; + *buf++ = hex[val >> 4]; + *buf++ = hex[val & 0xf]; + } + return buffer; +} -- 1.7.1 -- sheepdog mailing list [email protected] http://lists.wpkg.org/mailman/listinfo/sheepdog
