Hello community, here is the log from the commit of package s3backer for openSUSE:Factory checked in at 2019-07-11 13:17:36 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/s3backer (Old) and /work/SRC/openSUSE:Factory/.s3backer.new.4615 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "s3backer" Thu Jul 11 13:17:36 2019 rev:15 rq:714289 version:1.5.2 Changes: -------- --- /work/SRC/openSUSE:Factory/s3backer/s3backer.changes 2019-06-01 09:50:01.907313388 +0200 +++ /work/SRC/openSUSE:Factory/.s3backer.new.4615/s3backer.changes 2019-07-11 13:17:53.290776942 +0200 @@ -1,0 +2,9 @@ +Tue Jul 9 18:51:34 UTC 2019 - <[email protected]> + +- Update to release 1.5.2 + + Fixed bug where block cache would not work when run in the background (issue #112) + + Fixed bug where we were not parsing HTTP headers case-insensitively (pr #11) + + Bail out during `--listBlocks' if we see an object name past our block range + + Added `--blockHashPrefix' flag (issue #80) + +------------------------------------------------------------------- Old: ---- s3backer-1.5.1.tar.gz New: ---- s3backer-1.5.2.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ s3backer.spec ++++++ --- /var/tmp/diff_new_pack.dzFX8n/_old 2019-07-11 13:17:54.518776555 +0200 +++ /var/tmp/diff_new_pack.dzFX8n/_new 2019-07-11 13:17:54.518776555 +0200 @@ -18,7 +18,7 @@ Name: s3backer -Version: 1.5.1 +Version: 1.5.2 Release: 0 Summary: FUSE-based single file backing store via Amazon S3 License: GPL-2.0-or-later ++++++ s3backer-1.5.1.tar.gz -> s3backer-1.5.2.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/s3backer-1.5.1/CHANGES new/s3backer-1.5.2/CHANGES --- old/s3backer-1.5.1/CHANGES 2019-04-15 22:31:38.000000000 +0200 +++ new/s3backer-1.5.2/CHANGES 2019-07-09 20:44:36.000000000 +0200 @@ -1,3 +1,10 @@ +Version 1.5.2 released July 9, 2019 + + - Fixed bug where block cache would not work when run in the background (issue #112) + - Fixed bug where we were not parsing HTTP headers case-insensitively (pr #11) + - Bail out during `--listBlocks' if we see an object name past our block range + - Added `--blockHashPrefix' flag (issue #80) + Version 1.5.1 released April 15, 2019 - Fixed a few places where fixed-sized buffers were too small (issue #108) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/s3backer-1.5.1/block_cache.c new/s3backer-1.5.2/block_cache.c --- old/s3backer-1.5.1/block_cache.c 2019-03-07 02:43:01.000000000 +0100 +++ new/s3backer-1.5.2/block_cache.c 2019-07-09 20:37:22.000000000 +0200 @@ -181,6 +181,7 @@ }; /* s3backer_store functions */ +static int block_cache_create_threads(struct s3backer_store *s3b); static int block_cache_meta_data(struct s3backer_store *s3b, off_t *file_sizep, u_int *block_sizep); static int block_cache_set_mount_token(struct s3backer_store *s3b, int32_t *old_valuep, int32_t new_value); static int block_cache_read_block(struct s3backer_store *s3b, s3b_block_t block_num, void *dest, @@ -234,7 +235,6 @@ struct s3backer_store *s3b; struct block_cache_private *priv; struct cache_entry *entry; - pthread_t thread; int r; /* Initialize s3backer_store structure */ @@ -243,6 +243,7 @@ (*config->log)(LOG_ERR, "calloc(): %s", strerror(r)); goto fail0; } + s3b->create_threads = block_cache_create_threads; s3b->meta_data = block_cache_meta_data; s3b->set_mount_token = block_cache_set_mount_token; s3b->read_block = block_cache_read_block; @@ -301,22 +302,11 @@ pthread_mutex_lock(&priv->mutex); S3BCACHE_CHECK_INVARIANTS(priv); - /* Create threads */ - for (priv->num_threads = 0; priv->num_threads < config->num_threads; priv->num_threads++) { - if ((r = pthread_create(&thread, NULL, block_cache_worker_main, priv)) != 0) - goto fail10; - } - /* Done */ pthread_mutex_unlock(&priv->mutex); return s3b; -fail10: - priv->stopping = 1; - while (priv->num_threads > 0) { - pthread_cond_broadcast(&priv->worker_work); - pthread_cond_wait(&priv->worker_exit, &priv->mutex); - } +fail9: if (config->cache_file != NULL) { while ((entry = TAILQ_FIRST(&priv->cleans)) != NULL) { TAILQ_REMOVE(&priv->cleans, entry, link); @@ -324,7 +314,6 @@ } s3b_dcache_close(priv->dcache); } -fail9: s3b_hash_destroy(priv->hashtable); fail8: pthread_cond_destroy(&priv->write_complete); @@ -402,6 +391,34 @@ } static int +block_cache_create_threads(struct s3backer_store *s3b) +{ + struct block_cache_private *const priv = s3b->data; + struct block_cache_conf *const config = priv->config; + pthread_t thread; + int r; + + /* Create threads in lower layer */ + if ((r = (*priv->inner->create_threads)(priv->inner)) != 0) + return r; + + /* Grab lock */ + pthread_mutex_lock(&priv->mutex); + S3BCACHE_CHECK_INVARIANTS(priv); + + /* Create threads */ + while (priv->num_threads < config->num_threads) { + if ((r = pthread_create(&thread, NULL, block_cache_worker_main, priv)) != 0) + goto fail; + priv->num_threads++; + } + +fail: + pthread_mutex_unlock(&priv->mutex); + return r; +} + +static int block_cache_meta_data(struct s3backer_store *s3b, off_t *file_sizep, u_int *block_sizep) { struct block_cache_private *const priv = s3b->data; @@ -555,6 +572,12 @@ pthread_mutex_lock(&priv->mutex); S3BCACHE_CHECK_INVARIANTS(priv); + /* Sanity check */ + if (priv->num_threads == 0) { + (*config->log)(LOG_ERR, "block_cache_read(): no threads created yet"); + return ENOTCONN; + } + /* Update count of block(s) read sequentially by the upper layer */ if (block_num == priv->seq_last + 1) { priv->seq_count++; @@ -797,6 +820,11 @@ again: /* Sanity check */ S3BCACHE_CHECK_INVARIANTS(priv); + if (priv->num_threads == 0) { + (*config->log)(LOG_ERR, "block_cache_write(): no threads created yet"); + r = ENOTCONN; + goto fail; + } /* Find cache entry */ if ((entry = s3b_hash_get(priv->hashtable, block_num)) != NULL) { diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/s3backer-1.5.1/configure new/s3backer-1.5.2/configure --- old/s3backer-1.5.1/configure 2019-04-15 22:33:01.000000000 +0200 +++ new/s3backer-1.5.2/configure 2019-07-09 20:45:06.000000000 +0200 @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for s3backer FUSE filesystem backed by Amazon S3 1.5.1. +# Generated by GNU Autoconf 2.69 for s3backer FUSE filesystem backed by Amazon S3 1.5.2. # # Report bugs to <https://github.com/archiecobbs/s3backer>. # @@ -580,8 +580,8 @@ # Identity of this package. PACKAGE_NAME='s3backer FUSE filesystem backed by Amazon S3' PACKAGE_TARNAME='s3backer' -PACKAGE_VERSION='1.5.1' -PACKAGE_STRING='s3backer FUSE filesystem backed by Amazon S3 1.5.1' +PACKAGE_VERSION='1.5.2' +PACKAGE_STRING='s3backer FUSE filesystem backed by Amazon S3 1.5.2' PACKAGE_BUGREPORT='https://github.com/archiecobbs/s3backer' PACKAGE_URL='' @@ -1279,7 +1279,7 @@ # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures s3backer FUSE filesystem backed by Amazon S3 1.5.1 to adapt to many kinds of systems. +\`configure' configures s3backer FUSE filesystem backed by Amazon S3 1.5.2 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1345,7 +1345,7 @@ if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of s3backer FUSE filesystem backed by Amazon S3 1.5.1:";; + short | recursive ) echo "Configuration of s3backer FUSE filesystem backed by Amazon S3 1.5.2:";; esac cat <<\_ACEOF @@ -1446,7 +1446,7 @@ test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -s3backer FUSE filesystem backed by Amazon S3 configure 1.5.1 +s3backer FUSE filesystem backed by Amazon S3 configure 1.5.2 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -1794,7 +1794,7 @@ This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by s3backer FUSE filesystem backed by Amazon S3 $as_me 1.5.1, which was +It was created by s3backer FUSE filesystem backed by Amazon S3 $as_me 1.5.2, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -2657,7 +2657,7 @@ # Define the identity of the package. PACKAGE='s3backer' - VERSION='1.5.1' + VERSION='1.5.2' cat >>confdefs.h <<_ACEOF @@ -5490,7 +5490,7 @@ # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by s3backer FUSE filesystem backed by Amazon S3 $as_me 1.5.1, which was +This file was extended by s3backer FUSE filesystem backed by Amazon S3 $as_me 1.5.2, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -5556,7 +5556,7 @@ cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -s3backer FUSE filesystem backed by Amazon S3 config.status 1.5.1 +s3backer FUSE filesystem backed by Amazon S3 config.status 1.5.2 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/s3backer-1.5.1/configure.ac new/s3backer-1.5.2/configure.ac --- old/s3backer-1.5.1/configure.ac 2019-04-12 01:49:02.000000000 +0200 +++ new/s3backer-1.5.2/configure.ac 2019-07-09 20:44:44.000000000 +0200 @@ -32,7 +32,7 @@ # this exception statement from all source files in the program, then # also delete it here. -AC_INIT([s3backer FUSE filesystem backed by Amazon S3], [1.5.1], [https://github.com/archiecobbs/s3backer], [s3backer]) +AC_INIT([s3backer FUSE filesystem backed by Amazon S3], [1.5.2], [https://github.com/archiecobbs/s3backer], [s3backer]) AC_CONFIG_AUX_DIR(scripts) AM_INIT_AUTOMAKE(foreign) dnl AM_MAINTAINER_MODE diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/s3backer-1.5.1/ec_protect.c new/s3backer-1.5.2/ec_protect.c --- old/s3backer-1.5.1/ec_protect.c 2019-03-07 02:43:31.000000000 +0100 +++ new/s3backer-1.5.2/ec_protect.c 2019-07-09 20:18:00.000000000 +0200 @@ -129,6 +129,7 @@ }; /* s3backer_store functions */ +static int ec_protect_create_threads(struct s3backer_store *s3b); static int ec_protect_meta_data(struct s3backer_store *s3b, off_t *file_sizep, u_int *block_sizep); static int ec_protect_set_mount_token(struct s3backer_store *s3b, int32_t *old_valuep, int32_t new_value); static int ec_protect_read_block(struct s3backer_store *s3b, s3b_block_t block_num, void *dest, @@ -182,6 +183,7 @@ (*config->log)(LOG_ERR, "calloc(): %s", strerror(r)); goto fail0; } + s3b->create_threads = ec_protect_create_threads; s3b->meta_data = ec_protect_meta_data; s3b->set_mount_token = ec_protect_set_mount_token; s3b->read_block = ec_protect_read_block; @@ -235,6 +237,14 @@ } static int +ec_protect_create_threads(struct s3backer_store *s3b) +{ + struct ec_protect_private *const priv = s3b->data; + + return (*priv->inner->create_threads)(priv->inner); +} + +static int ec_protect_meta_data(struct s3backer_store *s3b, off_t *file_sizep, u_int *block_sizep) { struct ec_protect_private *const priv = s3b->data; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/s3backer-1.5.1/fuse_ops.c new/s3backer-1.5.2/fuse_ops.c --- old/s3backer-1.5.1/fuse_ops.c 2019-04-13 19:49:52.000000000 +0200 +++ new/s3backer-1.5.2/fuse_ops.c 2019-07-09 20:21:46.000000000 +0200 @@ -170,15 +170,27 @@ { struct s3b_config *const s3bconf = config->s3bconf; struct fuse_ops_private *const priv = the_priv; + int r; + /* Sanity check */ assert(priv != NULL); assert(priv->s3b != NULL); + + /* Initialize */ priv->block_bits = ffs(config->block_size) - 1; priv->start_time = time(NULL); priv->file_atime = priv->start_time; priv->file_mtime = priv->start_time; priv->stats_atime = priv->start_time; priv->file_size = config->num_blocks * config->block_size; + + /* Startup background threads now that we have fork()'d */ + if ((r = (*priv->s3b->create_threads)(priv->s3b)) != 0) { + (*config->log)(LOG_ERR, "fuse_op_init(): can't create threads: %s", strerror(errno)); + return NULL; + } + + /* Done */ (*config->log)(LOG_INFO, "mounting %s", s3bconf->mount); return priv; } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/s3backer-1.5.1/gitrev.c new/s3backer-1.5.2/gitrev.c --- old/s3backer-1.5.1/gitrev.c 2019-04-15 22:33:06.000000000 +0200 +++ new/s3backer-1.5.2/gitrev.c 2019-07-09 20:46:22.000000000 +0200 @@ -1 +1 @@ -const char *const s3backer_version = "1.5.1"; +const char *const s3backer_version = "1.5.2"; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/s3backer-1.5.1/http_io.c new/s3backer-1.5.2/http_io.c --- old/s3backer-1.5.1/http_io.c 2019-04-12 01:51:37.000000000 +0200 +++ new/s3backer-1.5.2/http_io.c 2019-07-09 20:18:16.000000000 +0200 @@ -82,8 +82,11 @@ #define DATE_BUF_SIZE 64 /* Size required for URL buffer */ -#define URL_BUF_SIZE(config) (strlen((config)->baseURL) + strlen((config)->bucket) \ - + strlen((config)->prefix) + S3B_BLOCK_NUM_DIGITS + 2) +#define URL_BUF_SIZE(config) (strlen((config)->baseURL) \ + + strlen((config)->bucket) + 1 \ + + strlen((config)->prefix) \ + + S3B_BLOCK_NUM_DIGITS + 1 \ + + S3B_BLOCK_NUM_DIGITS + 2) /* Bucket listing API constants */ #define LIST_PARAM_MARKER "marker" @@ -111,6 +114,9 @@ /* Enable to debug authentication stuff */ #define DEBUG_AUTHENTICATION 0 +/* Enable to debug parsing block list response */ +#define DEBUG_BLOCK_LIST 0 + /* Version 4 authentication stuff */ #define SIGNATURE_ALGORITHM "AWS4-HMAC-SHA256" #define ACCESS_KEY_PREFIX "AWS4" @@ -214,6 +220,7 @@ typedef void http_io_curl_prepper_t(CURL *curl, struct http_io *io); /* s3backer_store functions */ +static int http_io_create_threads(struct s3backer_store *s3b); static int http_io_meta_data(struct s3backer_store *s3b, off_t *file_sizep, u_int *block_sizep); static int http_io_set_mount_token(struct s3backer_store *s3b, int32_t *old_valuep, int32_t new_value); static int http_io_read_block(struct s3backer_store *s3b, s3b_block_t block_num, void *dest, @@ -273,15 +280,19 @@ static void update_hmac_from_header(HMAC_CTX *ctx, struct http_io *io, const char *name, int value_only, char *sigbuf, size_t sigbuflen); static int http_io_is_zero_block(const void *data, u_int block_size); +static s3b_block_t http_io_block_hash_prefix(s3b_block_t block_num); static int http_io_parse_hex(const char *str, u_char *buf, u_int nbytes); +static int http_io_parse_hex_block_num(const char *string, s3b_block_t *block_nump); static void http_io_prhex(char *buf, const u_char *data, size_t len); static int http_io_strcasecmp_ptr(const void *ptr1, const void *ptr2); +static int http_io_parse_header(const char *input, const char *header, const char *fmt, ...); /* Internal variables */ static pthread_mutex_t *openssl_locks; static int num_openssl_locks; static u_char zero_md5[MD5_DIGEST_LENGTH]; static u_char zero_hmac[SHA_DIGEST_LENGTH]; +static const s3b_block_t last_possible_block = (s3b_block_t)~0L; /* * Constructor @@ -309,6 +320,7 @@ r = errno; goto fail0; } + s3b->create_threads = http_io_create_threads; s3b->meta_data = http_io_meta_data; s3b->set_mount_token = http_io_set_mount_token; s3b->read_block = http_io_read_block; @@ -520,8 +532,8 @@ { struct http_io_private *const priv = s3b->data; struct http_io_conf *const config = priv->config; - char marker[sizeof("&marker=") + strlen(config->prefix) + S3B_BLOCK_NUM_DIGITS + 1]; - char urlbuf[URL_BUF_SIZE(config) + sizeof(marker) + 32]; + char url_encoded_prefix[strlen(config->prefix) * 3 + 1]; + char urlbuf[URL_BUF_SIZE(config) + sizeof("&marker=") + sizeof(url_encoded_prefix) + (S3B_BLOCK_NUM_DIGITS * 2) + 36]; struct http_io io; int r; @@ -541,7 +553,7 @@ } /* Allocate buffers for XML path and tag text content */ - io.xml_text_max = strlen(config->prefix) + S3B_BLOCK_NUM_DIGITS + 10; + io.xml_text_max = strlen(config->prefix) + (S3B_BLOCK_NUM_DIGITS * 2) + 16; if ((io.xml_text = malloc(io.xml_text_max + 1)) == NULL) { (*config->log)(LOG_ERR, "malloc: %s", strerror(errno)); goto oom; @@ -553,7 +565,6 @@ /* List blocks */ do { - char url_encoded_prefix[strlen(config->prefix) * 3 + 1]; const time_t now = time(NULL); /* Reset XML parser state */ @@ -570,8 +581,11 @@ /* Add URL parameters (note: must be in "canonical query string" format for proper authentication) */ if (io.list_truncated) { - snprintf(urlbuf + strlen(urlbuf), sizeof(urlbuf) - strlen(urlbuf), "%s=%s%0*jx&", - LIST_PARAM_MARKER, url_encoded_prefix, S3B_BLOCK_NUM_DIGITS, (uintmax_t)io.last_block); + char block_hash_buf[S3B_BLOCK_NUM_DIGITS + 2]; + + http_io_format_block_hash(config, block_hash_buf, sizeof(block_hash_buf), io.last_block); + snprintf(urlbuf + strlen(urlbuf), sizeof(urlbuf) - strlen(urlbuf), "%s=%s%s%0*jx&", + LIST_PARAM_MARKER, url_encoded_prefix, block_hash_buf, S3B_BLOCK_NUM_DIGITS, (uintmax_t)io.last_block); } snprintf(urlbuf + strlen(urlbuf), sizeof(urlbuf) - strlen(urlbuf), "%s=%u", LIST_PARAM_MAX_KEYS, LIST_BLOCKS_CHUNK); snprintf(urlbuf + strlen(urlbuf), sizeof(urlbuf) - strlen(urlbuf), "&%s=%s", LIST_PARAM_PREFIX, url_encoded_prefix); @@ -678,23 +692,62 @@ /* Reset buffer */ io->xml_text_len = 0; io->xml_text[0] = '\0'; + +#if DEBUG_BLOCK_LIST + /* Debug */ + (*io->config->log)(LOG_DEBUG, "list: new path: \"%s\"", io->xml_path); +#endif } static void http_io_list_elem_end(void *arg, const XML_Char *name) { struct http_io *const io = (struct http_io *)arg; + struct http_io_conf *const config = io->config; s3b_block_t block_num; /* Handle <Truncated> tag */ - if (strcmp(io->xml_path, "/" LIST_ELEM_LIST_BUCKET_RESLT "/" LIST_ELEM_IS_TRUNCATED) == 0) + if (strcmp(io->xml_path, "/" LIST_ELEM_LIST_BUCKET_RESLT "/" LIST_ELEM_IS_TRUNCATED) == 0) { io->list_truncated = strcmp(io->xml_text, LIST_TRUE) == 0; +#if DEBUG_BLOCK_LIST + (*config->log)(LOG_DEBUG, "list: parsed truncated=%d", io->list_truncated); +#endif + } /* Handle <Key> tag */ else if (strcmp(io->xml_path, "/" LIST_ELEM_LIST_BUCKET_RESLT "/" LIST_ELEM_CONTENTS "/" LIST_ELEM_KEY) == 0) { - if (http_io_parse_block(io->config, io->xml_text, &block_num) == 0) { +#if DEBUG_BLOCK_LIST + (*config->log)(LOG_DEBUG, "list: key=\"%s\"", io->xml_text); +#endif + + /* Attempt to parse key as a block's object name */ + if (http_io_parse_block(config, io->xml_text, &block_num) == 0) { +#if DEBUG_BLOCK_LIST + (*config->log)(LOG_DEBUG, "list: parsed key=\"%s\" -> block=%0*jx", + io->xml_text, S3B_BLOCK_NUM_DIGITS, (uintmax_t)block_num); +#endif (*io->callback_func)(io->callback_arg, block_num); io->last_block = block_num; + } else { /* object is some unrelated junk that we can ignore */ + char last_block_path[strlen(config->prefix) + S3B_BLOCK_NUM_DIGITS + 1]; + +#if DEBUG_BLOCK_LIST + (*config->log)(LOG_DEBUG, "list: can't parse key=\"%s\"", io->xml_text); +#endif + + /* + * If the object name is lexicographically after our last possible block name, we are done. + * Note that this works whether or not --blockHashPrefix is being used, because the block hash + * prefix is in the same format as the block number (i.e., 32 bit unsigned hexadecimal value). + */ + snprintf(last_block_path, sizeof(last_block_path), "%s%0*jx", + config->prefix, S3B_BLOCK_NUM_DIGITS, (uintmax_t)last_possible_block); + if (strcmp(io->xml_text, last_block_path) > 0) { +#if DEBUG_BLOCK_LIST + (*config->log)(LOG_DEBUG, "list: key=\"%s\" > last block \"%s\" -> we're done", io->xml_text, last_block_path); +#endif + io->list_truncated = 0; + } } } @@ -723,32 +776,38 @@ } /* - * Parse a block's item name (including prefix) and set the corresponding bit in the bitmap. + * Parse a block's item name (including prefix and block hash prefix if any) and returns the result in *block_nump. */ int http_io_parse_block(struct http_io_conf *config, const char *name, s3b_block_t *block_nump) { const size_t plen = strlen(config->prefix); + s3b_block_t hash_value = 0; s3b_block_t block_num = 0; - int i; - /* Check prefix */ + /* Parse prefix */ if (strncmp(name, config->prefix, plen) != 0) return -1; name += plen; - /* Parse block number */ - for (i = 0; i < S3B_BLOCK_NUM_DIGITS; i++) { - char ch = name[i]; - - if (!isxdigit(ch)) - break; - block_num <<= 4; - block_num |= ch <= '9' ? ch - '0' : tolower(ch) - 'a' + 10; + /* Parse block hash prefix followed by dash (if so configured) */ + if (config->blockHashPrefix) { + if (http_io_parse_hex_block_num(name, &hash_value) == -1) + return -1; + name += S3B_BLOCK_NUM_DIGITS; + if (*name++ != '-') + return -1; } - /* Was parse successful? */ - if (i != S3B_BLOCK_NUM_DIGITS || name[i] != '\0' || block_num >= config->num_blocks) + /* Parse block number */ + if (http_io_parse_hex_block_num(name, &block_num) == -1) + return -1; + name += S3B_BLOCK_NUM_DIGITS; + if (*name != '\0' || block_num >= config->num_blocks) + return -1; + + /* Verify hash matches what's expected */ + if (config->blockHashPrefix && hash_value != http_io_block_hash_prefix(block_num)) return -1; /* Done */ @@ -756,6 +815,75 @@ return 0; } +/* + * Parse a hexadecimal block number value, which should be S3B_BLOCK_NUM_DIGITS lowercase digits. + * + * Returns zero on success, -1 on failure. + */ +static int +http_io_parse_hex_block_num(const char *string, s3b_block_t *valuep) +{ + s3b_block_t value = 0; + int i; + + /* Parse block number */ + for (i = 0; i < S3B_BLOCK_NUM_DIGITS; i++) { + const char ch = string[i]; + + value <<= 4; + if (ch >= '0' && ch <= '9') + value |= ch - '0'; + else if (ch >= 'a' && ch <= 'f') + value |= ch - 'a' + 10; + else + return -1; + } + + /* Done */ + *valuep = value; + return 0; +} + +/* + * Append deterministic hash value based on block number for even name distribution. + * + * Ref: https://github.com/archiecobbs/s3backer/issues/80 + * Ref: https://crypto.stackexchange.com/questions/16219/cryptographic-hash-function-for-32-bit-length-input-keys + */ +void +http_io_format_block_hash(const struct http_io_conf *const config, char *buf, size_t bufsiz, s3b_block_t block_num) +{ + assert(bufsiz >= S3B_BLOCK_NUM_DIGITS + 2); + if (config->blockHashPrefix) + snprintf(buf, bufsiz, "%0*jx-", S3B_BLOCK_NUM_DIGITS, (uintmax_t)http_io_block_hash_prefix(block_num)); + else + *buf = '\0'; +} + +/* + * Calculate deterministic hash value based on block number for even name distribution. + * + * Ref: https://github.com/archiecobbs/s3backer/issues/80 + * Ref: https://crypto.stackexchange.com/questions/16219/cryptographic-hash-function-for-32-bit-length-input-keys + */ +s3b_block_t +http_io_block_hash_prefix(s3b_block_t block_num) +{ + s3b_block_t hash; + int n; + + hash = block_num; + for (n = 12; n > 0; n--) + hash = ((hash >> 8) ^ hash) * 0x6b + n; + return hash; +} + +static int +http_io_create_threads(struct s3backer_store *s3b) +{ + return 0; +} + static int http_io_meta_data(struct s3backer_store *s3b, off_t *file_sizep, u_int *block_sizep) { @@ -1950,7 +2078,7 @@ HMAC_Update(hmac_ctx, (const u_char *)config->bucket, strlen(config->bucket)); HMAC_Update(hmac_ctx, (const u_char *)resource, resource_len); #if DEBUG_AUTHENTICATION - snprintf(sigbuf + strlen(sigbuf), sizeof(sigbuf) - strlen(sigbuf), "/%s%.*s", config->bucket, resource_len, resource); + snprintf(sigbuf + strlen(sigbuf), sizeof(sigbuf) - strlen(sigbuf), "/%s%.*s", config->bucket, (int)resource_len, resource); #endif /* Finish up */ @@ -2332,13 +2460,16 @@ static void http_io_get_block_url(char *buf, size_t bufsiz, struct http_io_conf *config, s3b_block_t block_num) { + char block_hash_buf[S3B_BLOCK_NUM_DIGITS + 2]; int len; - if (config->vhost) - len = snprintf(buf, bufsiz, "%s%s%0*jx", config->baseURL, config->prefix, S3B_BLOCK_NUM_DIGITS, (uintmax_t)block_num); - else { - len = snprintf(buf, bufsiz, "%s%s/%s%0*jx", config->baseURL, - config->bucket, config->prefix, S3B_BLOCK_NUM_DIGITS, (uintmax_t)block_num); + http_io_format_block_hash(config, block_hash_buf, sizeof(block_hash_buf), block_num); + if (config->vhost) { + len = snprintf(buf, bufsiz, "%s%s%s%0*jx", config->baseURL, + config->prefix, block_hash_buf, S3B_BLOCK_NUM_DIGITS, (uintmax_t)block_num); + } else { + len = snprintf(buf, bufsiz, "%s%s/%s%s%0*jx", config->baseURL, + config->bucket, config->prefix, block_hash_buf, S3B_BLOCK_NUM_DIGITS, (uintmax_t)block_num); } (void)len; /* avoid compiler warning when NDEBUG defined */ assert(len < bufsiz); @@ -2488,7 +2619,7 @@ { struct http_io *const io = (struct http_io *)stream; const size_t total = size * nmemb; - char fmtbuf[64]; + char hashbuf[64]; char buf[1024]; u_int mtoken; @@ -2499,28 +2630,24 @@ buf[total] = '\0'; /* Check for interesting headers */ - (void)sscanf(buf, FILE_SIZE_HEADER ": %ju", &io->file_size); - (void)sscanf(buf, BLOCK_SIZE_HEADER ": %u", &io->block_size); - if (sscanf(buf, MOUNT_TOKEN_HEADER ": %x", &mtoken) == 1) + (void)http_io_parse_header(buf, FILE_SIZE_HEADER, "%ju", &io->file_size); + (void)http_io_parse_header(buf, BLOCK_SIZE_HEADER, "%u", &io->block_size); + if (http_io_parse_header(buf, MOUNT_TOKEN_HEADER, "%x", &mtoken) == 1) io->mount_token = (int32_t)mtoken; - /* ETag header requires parsing */ - if (strncasecmp(buf, ETAG_HEADER ":", sizeof(ETAG_HEADER)) == 0) { - char md5buf[MD5_DIGEST_LENGTH * 2 + 1]; - - snprintf(fmtbuf, sizeof(fmtbuf), " \"%%%uc\"", MD5_DIGEST_LENGTH * 2); - if (sscanf(buf + sizeof(ETAG_HEADER), fmtbuf, md5buf) == 1) - http_io_parse_hex(md5buf, io->md5, MD5_DIGEST_LENGTH); - } - - /* "x-amz-meta-s3backer-hmac" header requires parsing */ - if (strncasecmp(buf, HMAC_HEADER ":", sizeof(HMAC_HEADER)) == 0) { - char hmacbuf[SHA_DIGEST_LENGTH * 2 + 1]; - - snprintf(fmtbuf, sizeof(fmtbuf), " \"%%%uc\"", SHA_DIGEST_LENGTH * 2); - if (sscanf(buf + sizeof(HMAC_HEADER), fmtbuf, hmacbuf) == 1) - http_io_parse_hex(hmacbuf, io->hmac, SHA_DIGEST_LENGTH); - } + /* ETag header */ +#if MD5_DIGEST_LENGTH != 16 +#error unexpected MD5_DIGEST_LENGTH +#endif + if (http_io_parse_header(buf, ETAG_HEADER, "\"%32c\"", hashbuf) == 1) + http_io_parse_hex(hashbuf, io->md5, MD5_DIGEST_LENGTH); + + /* "x-amz-meta-s3backer-hmac" header */ +#if SHA_DIGEST_LENGTH != 20 +#error unexpected MD5_DIGEST_LENGTH +#endif + if (http_io_parse_header(buf, HMAC_HEADER, "\"%40c\"", hashbuf) == 1) + http_io_parse_hex(hashbuf, io->hmac, SHA_DIGEST_LENGTH); /* Content encoding(s) */ if (strncasecmp(buf, CONTENT_ENCODING_HEADER ":", sizeof(CONTENT_ENCODING_HEADER)) == 0) { @@ -2720,7 +2847,7 @@ if (!value_only) { HMAC_Update(ctx, (const u_char *)header->data, name_len + 1); #if DEBUG_AUTHENTICATION - snprintf(sigbuf + strlen(sigbuf), sigbuflen - strlen(sigbuf), "%.*s", name_len + 1, header->data); + snprintf(sigbuf + strlen(sigbuf), sigbuflen - strlen(sigbuf), "%.*s", (int)name_len + 1, header->data); #endif } for (value = header->data + name_len + 1; isspace(*value); value++) @@ -2793,3 +2920,28 @@ return strcasecmp(str1, str2); } +static int +http_io_parse_header(const char *const input, const char *const header, const char *const fmt, ...) +{ + + va_list args; + size_t offset; + int rtn; + + /* Initialize */ + va_start(args, fmt); + + /* Match header (case-insensitively) followed by ':' and optional whitespace */ + offset = strlen(header); + if (strncasecmp(input, header, offset) != 0 || input[offset++] != ':') + return -1; + while (isspace(input[offset])) + offset++; + + /* Parse header value */ + rtn = vsscanf(input + offset, fmt, args); + + /* Done */ + va_end(args); + return rtn; +} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/s3backer-1.5.1/http_io.h new/s3backer-1.5.2/http_io.h --- old/s3backer-1.5.1/http_io.h 2019-03-07 02:40:58.000000000 +0100 +++ new/s3backer-1.5.2/http_io.h 2019-07-07 16:36:29.000000000 +0200 @@ -76,6 +76,7 @@ int compress; // zlib compression level int vhost; // use virtual host style URL u_int *nonzero_bitmap; // is set to NULL by http_io_create() + int blockHashPrefix; int insecure; u_int block_size; off_t num_blocks; @@ -140,4 +141,5 @@ extern void http_io_get_stats(struct s3backer_store *s3b, struct http_io_stats *stats); extern void http_io_clear_stats(struct s3backer_store *s3b); extern int http_io_parse_block(struct http_io_conf *config, const char *name, s3b_block_t *block_num); +extern void http_io_format_block_hash(const struct http_io_conf *config, char *block_hash_buf, size_t bufsiz, s3b_block_t block_num); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/s3backer-1.5.1/s3b_config.c new/s3backer-1.5.2/s3b_config.c --- old/s3backer-1.5.1/s3b_config.c 2019-03-07 02:45:22.000000000 +0100 +++ new/s3backer-1.5.2/s3b_config.c 2019-07-09 20:18:43.000000000 +0200 @@ -152,6 +152,7 @@ .region= NULL, .bucket= NULL, .sse= NULL, + .blockHashPrefix= 0, .prefix= S3BACKER_DEFAULT_PREFIX, .accessType= S3BACKER_DEFAULT_ACCESS_TYPE, .authVersion= S3BACKER_DEFAULT_AUTH_VERSION, @@ -373,6 +374,11 @@ .offset= offsetof(struct s3b_config, ec_protect.min_write_delay), }, { + .templ= "--blockHashPrefix", + .offset= offsetof(struct s3b_config, http_io.blockHashPrefix), + .value= 1 + }, + { .templ= "--prefix=%s", .offset= offsetof(struct s3b_config, http_io.prefix), }, @@ -1652,6 +1658,7 @@ (*config.log)(LOG_DEBUG, "%24s: \"%s\"", "region", config.http_io.region); (*config.log)(LOG_DEBUG, "%24s: \"%s\"", config.test ? "testdir" : "bucket", config.http_io.bucket); (*config.log)(LOG_DEBUG, "%24s: \"%s\"", "prefix", config.http_io.prefix); + (*config.log)(LOG_DEBUG, "%24s: %s", "blockHashPrefix", config.http_io.blockHashPrefix ? "true" : "false"); (*config.log)(LOG_DEBUG, "%24s: \"%s\"", "defaultContentEncoding", config.http_io.default_ce != NULL ? config.http_io.default_ce : "(none)"); (*config.log)(LOG_DEBUG, "%24s: %s", "list_blocks", config.list_blocks ? "true" : "false"); @@ -1795,6 +1802,7 @@ fprintf(stderr, "\t--%-27s %s\n", "blockCacheTimeout=MILLIS", "Block cache entry timeout (zero = infinite)"); fprintf(stderr, "\t--%-27s %s\n", "blockCacheWriteDelay=MILLIS", "Block cache maximum write-back delay"); fprintf(stderr, "\t--%-27s %s\n", "blockSize=SIZE", "Block size (with optional suffix 'K', 'M', 'G', etc.)"); + fprintf(stderr, "\t--%-27s %s\n", "blockHashPrefix", "Prepend hash to block names for even distribution"); fprintf(stderr, "\t--%-27s %s\n", "cacert=FILE", "Specify SSL certificate authority file"); fprintf(stderr, "\t--%-27s %s\n", "compress[=LEVEL]", "Enable block compression, with 1=fast up to 9=small"); fprintf(stderr, "\t--%-27s %s\n", "debug", "Enable logging of debug messages"); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/s3backer-1.5.1/s3backer.1 new/s3backer-1.5.2/s3backer.1 --- old/s3backer-1.5.1/s3backer.1 2019-03-16 23:37:21.000000000 +0100 +++ new/s3backer-1.5.2/s3backer.1 2019-07-07 16:36:29.000000000 +0200 @@ -458,6 +458,13 @@ This flag requires .Fl \-blockCacheFile to be set. +.It Fl \-blockHashPrefix +Prepend random prefixes (generated deterministically from the block number) to block object names. +This spreads requests more evenly across the namespace, and prevents heavy access to a narrow range of blocks from all being directed to the same backend server. +.Pp +As with +.Fl \-prefix , +this flag must be used consistently once a disk image is established. .It Fl \-blockSize=SIZE Specify the block size. This must be a power of two and should be a multiple of the kernel's native page size. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/s3backer-1.5.1/s3backer.h new/s3backer-1.5.2/s3backer.h --- old/s3backer-1.5.1/s3backer.h 2018-05-31 00:09:31.000000000 +0200 +++ new/s3backer-1.5.2/s3backer.h 2019-07-09 20:39:21.000000000 +0200 @@ -120,6 +120,25 @@ struct s3backer_store { /* + * Create any background pthreads that may be required. + * + * This must be invoked prior to any of the following functions: + * + * o block_read + * o block_read_part + * o block_write + * o block_write_part + * + * It should be invoked after the initial process fork() because it may create pthreads. + * + * Returns: + * + * 0 Success + * Other Other error + */ + int (*create_threads)(struct s3backer_store *s3b); + + /* * Get meta-data associated with the underlying store. * * The information we acquire is: @@ -166,6 +185,7 @@ * - Return EEXIST; the block may or may not also be read normally into *dest * * Returns zero on success or a (positive) errno value on error. + * May return ENOTCONN if create_threads() has not yet been invoked. */ int (*read_block)(struct s3backer_store *s3b, s3b_block_t block_num, void *dest, u_char *actual_md5, const u_char *expect_md5, int strict); @@ -174,6 +194,7 @@ * Read part of one block. * * Returns zero on success or a (positive) errno value on error. + * May return ENOTCONN if create_threads() has not yet been invoked. */ int (*read_block_part)(struct s3backer_store *s3b, s3b_block_t block_num, u_int off, u_int len, void *dest); @@ -189,6 +210,7 @@ * parameter of read_block(); if the block is all zeroes, md5 will be zeroed. * * Returns zero on success or a (positive) errno value on error. + * May return ENOTCONN if create_threads() has not yet been invoked. */ int (*write_block)(struct s3backer_store *s3b, s3b_block_t block_num, const void *src, u_char *md5, check_cancel_t *check_cancel, void *arg); @@ -197,6 +219,7 @@ * Write part of one block. * * Returns zero on success or a (positive) errno value on error. + * May return ENOTCONN if create_threads() has not yet been invoked. */ int (*write_block_part)(struct s3backer_store *s3b, s3b_block_t block_num, u_int off, u_int len, const void *src); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/s3backer-1.5.1/s3backer.spec new/s3backer-1.5.2/s3backer.spec --- old/s3backer-1.5.1/s3backer.spec 2019-04-15 22:33:06.000000000 +0200 +++ new/s3backer-1.5.2/s3backer.spec 2019-07-09 20:46:12.000000000 +0200 @@ -29,7 +29,7 @@ # Name: s3backer -Version: 1.5.1 +Version: 1.5.2 Release: 1 License: GNU General Public License, Version 2 Summary: FUSE-based single file backing store via Amazon S3 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/s3backer-1.5.1/test_io.c new/s3backer-1.5.2/test_io.c --- old/s3backer-1.5.1/test_io.c 2018-05-31 00:09:31.000000000 +0200 +++ new/s3backer-1.5.2/test_io.c 2019-07-09 20:19:20.000000000 +0200 @@ -49,6 +49,7 @@ }; /* s3backer_store functions */ +static int test_io_create_threads(struct s3backer_store *s3b); static int test_io_meta_data(struct s3backer_store *s3b, off_t *file_sizep, u_int *block_sizep); static int test_io_set_mount_token(struct s3backer_store *s3b, int32_t *old_valuep, int32_t new_value); static int test_io_read_block(struct s3backer_store *s3b, s3b_block_t block_num, void *dest, @@ -75,6 +76,7 @@ /* Initialize structures */ if ((s3b = calloc(1, sizeof(*s3b))) == NULL) return NULL; + s3b->create_threads = test_io_create_threads; s3b->meta_data = test_io_meta_data; s3b->set_mount_token = test_io_set_mount_token; s3b->read_block = test_io_read_block; @@ -100,6 +102,12 @@ } static int +test_io_create_threads(struct s3backer_store *s3b) +{ + return 0; +} + +static int test_io_meta_data(struct s3backer_store *s3b, off_t *file_sizep, u_int *block_sizep) { return 0; @@ -237,6 +245,7 @@ { struct test_io_private *const priv = s3b->data; struct http_io_conf *const config = priv->config; + char block_hash_buf[S3B_BLOCK_NUM_DIGITS + 2]; u_char md5[MD5_DIGEST_LENGTH]; char temp[PATH_MAX]; char path[PATH_MAX]; @@ -277,7 +286,9 @@ } /* Generate path */ - snprintf(path, sizeof(path), "%s/%s%0*jx", config->bucket, config->prefix, S3B_BLOCK_NUM_DIGITS, (uintmax_t)block_num); + http_io_format_block_hash(config, block_hash_buf, sizeof(block_hash_buf), block_num); + snprintf(path, sizeof(path), "%s/%s%s%0*jx", + config->bucket, config->prefix, block_hash_buf, S3B_BLOCK_NUM_DIGITS, (uintmax_t)block_num); /* Delete zero blocks */ if (src == NULL) {
