Modified: subversion/branches/authzperf/subversion/libsvn_fs_fs/index.h URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_fs/index.h?rev=1649205&r1=1649204&r2=1649205&view=diff ============================================================================== --- subversion/branches/authzperf/subversion/libsvn_fs_fs/index.h (original) +++ subversion/branches/authzperf/subversion/libsvn_fs_fs/index.h Sat Jan 3 14:00:41 2015 @@ -48,43 +48,22 @@ #define SVN_FS_FS__ITEM_TYPE_ANY_REP 7 /* item is any representation. Only used in pre-format7. */ -/* (user visible) entry in the phys-to-log index. It describes a section - * of some packed / non-packed rev file as containing a specific item. - * There must be no overlapping / conflicting entries. - */ -typedef struct svn_fs_fs__p2l_entry_t -{ - /* offset of the first byte that belongs to the item */ - apr_off_t offset; - - /* length of the item in bytes */ - apr_off_t size; - - /* type of the item (see SVN_FS_FS__ITEM_TYPE_*) defines */ - unsigned type; - - /* modified FNV-1a checksum. 0 if unknown checksum */ - apr_uint32_t fnv1_checksum; - - /* item in that block */ - svn_fs_fs__id_part_t item; -} svn_fs_fs__p2l_entry_t; - /* Open / create a log-to-phys index file with the full file path name - * FILE_NAME. Return the open file in *PROTO_INDEX and use POOL for - * allocations. + * FILE_NAME. Return the open file in *PROTO_INDEX allocated in + * RESULT_POOL. */ svn_error_t * svn_fs_fs__l2p_proto_index_open(apr_file_t **proto_index, const char *file_name, - apr_pool_t *pool); + apr_pool_t *result_pool); /* Call this function before adding entries for the next revision to the - * log-to-phys index file in PROTO_INDEX. Use POOL for allocations. + * log-to-phys index file in PROTO_INDEX. Use SCRATCH_POOL for temporary + * allocations. */ svn_error_t * svn_fs_fs__l2p_proto_index_add_revision(apr_file_t *proto_index, - apr_pool_t *pool); + apr_pool_t *scratch_pool); /* Add a new mapping, ITEM_INDEX to the OFFSET, to log-to-phys index file * in PROTO_INDEX. Please note that mappings may be added in any order @@ -93,73 +72,83 @@ svn_fs_fs__l2p_proto_index_add_revision( * mark 'invalid' item indexes but that is already implied for all item * indexes not explicitly given a mapping. * - * Use POOL for allocations. + * Use SCRATCH_POOL for temporary allocations. */ svn_error_t * svn_fs_fs__l2p_proto_index_add_entry(apr_file_t *proto_index, apr_off_t offset, apr_uint64_t item_index, - apr_pool_t *pool); + apr_pool_t *scratch_pool); /* Use the proto index file stored at PROTO_FILE_NAME, construct the final * log-to-phys index and append it to INDEX_FILE. The first revision will * be REVISION, entries to the next revision will be assigned to REVISION+1 - * and so forth. Use POOL for allocations. + * and so forth. + * + * Return the MD5 checksum of the on-disk index data in *CHECKSUM, allocated + * in RESULT_POOL. Use SCRATCH_POOL for temporary allocations. */ svn_error_t * -svn_fs_fs__l2p_index_append(svn_fs_t *fs, +svn_fs_fs__l2p_index_append(svn_checksum_t **checksum, + svn_fs_t *fs, apr_file_t *index_file, const char *proto_file_name, svn_revnum_t revision, - apr_pool_t *pool); + apr_pool_t *result_pool, + apr_pool_t *scratch_pool); /* Open / create a phys-to-log index file with the full file path name - * FILE_NAME. Return the open file in *PROTO_INDEX and use POOL for - * allocations. + * FILE_NAME. Return the open file in *PROTO_INDEX allocated in + * RESULT_POOL. */ svn_error_t * svn_fs_fs__p2l_proto_index_open(apr_file_t **proto_index, const char *file_name, - apr_pool_t *pool); + apr_pool_t *result_pool); /* Add a new mapping ENTRY to the phys-to-log index file in PROTO_INDEX. * The entries must be added in ascending offset order and must not leave * intermittent ranges uncovered. The revision value in ENTRY may be - * SVN_INVALID_REVISION. Use POOL for allocations. + * SVN_INVALID_REVISION. Use SCRATCH_POOL for temporary allocations. */ svn_error_t * svn_fs_fs__p2l_proto_index_add_entry(apr_file_t *proto_index, svn_fs_fs__p2l_entry_t *entry, - apr_pool_t *pool); + apr_pool_t *scratch_pool); /* Set *NEXT_OFFSET to the first offset behind the last entry in the * phys-to-log proto index file PROTO_INDEX. This will be 0 for empty - * index files. Use POOL for temporary allocations. + * index files. Use SCRATCH_POOL for temporary allocations. */ svn_error_t * svn_fs_fs__p2l_proto_index_next_offset(apr_off_t *next_offset, apr_file_t *proto_index, - apr_pool_t *pool); + apr_pool_t *scratch_pool); /* Use the proto index file stored at PROTO_FILE_NAME, construct the final * phys-to-log index and append it to INDEX_FILE. Entries without a valid * revision will be assigned to the REVISION given here. - * Use POOL for allocations. + * + * Return the MD5 checksum of the on-disk index data in *CHECKSUM, allocated + * in RESULT_POOL. Use SCRATCH_POOL for temporary allocations. */ svn_error_t * -svn_fs_fs__p2l_index_append(svn_fs_t *fs, +svn_fs_fs__p2l_index_append(svn_checksum_t **checksum, + svn_fs_t *fs, apr_file_t *index_file, const char *proto_file_name, svn_revnum_t revision, - apr_pool_t *pool); + apr_pool_t *result_pool, + apr_pool_t *scratch_pool); /* Use the phys-to-log mapping files in FS to build a list of entries * that (at least partly) overlap with the range given by BLOCK_START * offset and BLOCK_SIZE in the rep / pack file containing REVISION. - * Return the array in *ENTRIES with svn_fs_fs__p2l_entry_t as elements. - * REV_FILE determines whether to access single rev or pack file data. - * If that is not available anymore (neither in cache nor on disk), - * return an error. Use POOL for allocations. + * Return the array in *ENTRIES with svn_fs_fs__p2l_entry_t as elements, + * allocated in RESULT_POOL. REV_FILE determines whether to access single + * rev or pack file data. If that is not available anymore (neither in + * cache nor on disk), return an error. Use SCRATCH_POOL for temporary + * allocations. * * Note that (only) the first and the last mapping may cross a cluster * boundary. @@ -171,14 +160,16 @@ svn_fs_fs__p2l_index_lookup(apr_array_he svn_revnum_t revision, apr_off_t block_start, apr_off_t block_size, - apr_pool_t *pool); + apr_pool_t *result_pool, + apr_pool_t *scratch_pool); /* Use the phys-to-log mapping files in FS to return the entry for the * item starting at global OFFSET in the rep file containing REVISION in - * *ENTRY. Sets *ENTRY to NULL if no item starts at exactly that offset. - * REV_FILE determines whether to access single rev or pack file data. - * If that is not available anymore (neither in cache nor on disk), - * return an error. Use POOL for allocations. + * *ENTRY, allocated in RESULT_POOL. Sets *ENTRY to NULL if no item starts + * at exactly that offset. REV_FILE determines whether to access single + * rev or pack file data. If that is not available anymore (neither in + * cache nor on disk), return an error. Use SCRATCH_POOL for temporary + * allocations. */ svn_error_t * svn_fs_fs__p2l_entry_lookup(svn_fs_fs__p2l_entry_t **entry, @@ -186,7 +177,8 @@ svn_fs_fs__p2l_entry_lookup(svn_fs_fs__p svn_fs_fs__revision_file_t *rev_file, svn_revnum_t revision, apr_off_t offset, - apr_pool_t *pool); + apr_pool_t *result_pool, + apr_pool_t *scratch_pool); /* For ITEM_INDEX within REV in FS, return the position in the respective * rev or pack file in *ABSOLUTE_POSITION. If TXN_ID is not NULL, return @@ -197,7 +189,7 @@ svn_fs_fs__p2l_entry_lookup(svn_fs_fs__p * If that is not available anymore (neither in cache nor on disk), re-open * the rev / pack file and retry to open the index file. For anything but * committed log addressed revisions, REV_FILE may be NULL. - * Use POOL for allocations. + * Use SCRATCH_POOL for temporary allocations. */ svn_error_t * svn_fs_fs__item_offset(apr_off_t *absolute_position, @@ -206,32 +198,34 @@ svn_fs_fs__item_offset(apr_off_t *absolu svn_revnum_t revision, const svn_fs_fs__id_part_t *txn_id, apr_uint64_t item_index, - apr_pool_t *pool); + apr_pool_t *scratch_pool); /* Use the log-to-phys indexes in FS to determine the maximum item indexes * assigned to revision START_REV to START_REV + COUNT - 1. That is a * close upper limit to the actual number of items in the respective revs. - * Return the results in *MAX_IDS, allocated in POOL. + * Return the results in *MAX_IDS, allocated in RESULT_POOL. + * Use SCRATCH_POOL for temporary allocations. */ svn_error_t * svn_fs_fs__l2p_get_max_ids(apr_array_header_t **max_ids, svn_fs_t *fs, svn_revnum_t start_rev, apr_size_t count, - apr_pool_t *pool); + apr_pool_t *result_pool, + apr_pool_t *scratch_pool); /* In *OFFSET, return the last OFFSET in the pack / rev file containing. * REV_FILE determines whether to access single rev or pack file data. * If that is not available anymore (neither in cache nor on disk), re-open * the rev / pack file and retry to open the index file. - * Use POOL for allocations. + * Use SCRATCH_POOL for temporary allocations. */ svn_error_t * svn_fs_fs__p2l_get_max_offset(apr_off_t *offset, svn_fs_t *fs, svn_fs_fs__revision_file_t *rev_file, svn_revnum_t revision, - apr_pool_t *pool); + apr_pool_t *scratch_pool); /* Index (re-)creation utilities. */
Modified: subversion/branches/authzperf/subversion/libsvn_fs_fs/lock.c URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_fs/lock.c?rev=1649205&r1=1649204&r2=1649205&view=diff ============================================================================== --- subversion/branches/authzperf/subversion/libsvn_fs_fs/lock.c (original) +++ subversion/branches/authzperf/subversion/libsvn_fs_fs/lock.c Sat Jan 3 14:00:41 2015 @@ -386,7 +386,8 @@ add_to_digest(const char *fs_path, const char *index_digest_path; apr_hash_t *children; svn_lock_t *lock; - int i, original_count; + int i; + unsigned int original_count; SVN_ERR(digest_path_from_path(&index_digest_path, fs_path, index_path, pool)); Modified: subversion/branches/authzperf/subversion/libsvn_fs_fs/low_level.c URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_fs/low_level.c?rev=1649205&r1=1649204&r2=1649205&view=diff ============================================================================== --- subversion/branches/authzperf/subversion/libsvn_fs_fs/low_level.c (original) +++ subversion/branches/authzperf/subversion/libsvn_fs_fs/low_level.c Sat Jan 3 14:00:41 2015 @@ -27,6 +27,7 @@ #include "private/svn_sorts_private.h" #include "private/svn_string_private.h" #include "private/svn_subr_private.h" +#include "private/svn_fspath.h" #include "../libsvn_fs/fs-loader.h" @@ -69,6 +70,36 @@ * various flags. */ #define MAX_CHANGE_LINE_LEN FSFS_MAX_PATH_LEN + 256 +/* Convert the C string in *TEXT to a revision number and return it in *REV. + * Overflows, negative values other than -1 and terminating characters other + * than 0x20 or 0x0 will cause an error. Set *TEXT to the first char after + * the initial separator or to EOS. + */ +static svn_error_t * +parse_revnum(svn_revnum_t *rev, + const char **text) +{ + const char *string = *text; + if ((string[0] == '-') && (string[1] == '1')) + { + *rev = SVN_INVALID_REVNUM; + string += 2; + } + else + { + SVN_ERR(svn_revnum_parse(rev, string, &string)); + } + + if (*string == ' ') + ++string; + else if (*string != '\0') + return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, + _("Invalid character in revision number")); + + *text = string; + return SVN_NO_ERROR; +} + svn_error_t * svn_fs_fs__parse_revision_trailer(apr_off_t *root_offset, apr_off_t *changes_offset, @@ -160,38 +191,71 @@ svn_fs_fs__unparse_revision_trailer(apr_ svn_error_t * svn_fs_fs__parse_footer(apr_off_t *l2p_offset, + svn_checksum_t **l2p_checksum, apr_off_t *p2l_offset, + svn_checksum_t **p2l_checksum, svn_stringbuf_t *footer, - svn_revnum_t rev) + svn_revnum_t rev, + apr_pool_t *result_pool) { apr_int64_t val; + char *last_str = footer->data; - /* Split the footer into the 2 number strings. */ - char *seperator = strchr(footer->data, ' '); - if (!seperator) - return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, - _("Revision file (r%ld) has corrupt footer"), - rev); - *seperator = '\0'; + /* Get the L2P offset. */ + const char *str = svn_cstring_tokenize(" ", &last_str); + if (str == NULL) + return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, + _("Invalid revision footer")); - /* Convert offset values. */ - SVN_ERR(svn_cstring_atoi64(&val, footer->data)); + SVN_ERR(svn_cstring_atoi64(&val, str)); *l2p_offset = (apr_off_t)val; - SVN_ERR(svn_cstring_atoi64(&val, seperator + 1)); + + /* Get the L2P checksum. */ + str = svn_cstring_tokenize(" ", &last_str); + if (str == NULL) + return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, + _("Invalid revision footer")); + + SVN_ERR(svn_checksum_parse_hex(l2p_checksum, svn_checksum_md5, str, + result_pool)); + + /* Get the P2L offset. */ + str = svn_cstring_tokenize(" ", &last_str); + if (str == NULL) + return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, + _("Invalid revision footer")); + + SVN_ERR(svn_cstring_atoi64(&val, str)); *p2l_offset = (apr_off_t)val; + /* Get the P2L checksum. */ + str = svn_cstring_tokenize(" ", &last_str); + if (str == NULL) + return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, + _("Invalid revision footer")); + + SVN_ERR(svn_checksum_parse_hex(p2l_checksum, svn_checksum_md5, str, + result_pool)); + return SVN_NO_ERROR; } svn_stringbuf_t * svn_fs_fs__unparse_footer(apr_off_t l2p_offset, + svn_checksum_t *l2p_checksum, apr_off_t p2l_offset, - apr_pool_t *result_pool) + svn_checksum_t *p2l_checksum, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) { return svn_stringbuf_createf(result_pool, - "%" APR_OFF_T_FMT " %" APR_OFF_T_FMT, + "%" APR_OFF_T_FMT " %s %" APR_OFF_T_FMT " %s", l2p_offset, - p2l_offset); + svn_checksum_to_cstring(l2p_checksum, + scratch_pool), + p2l_offset, + svn_checksum_to_cstring(p2l_checksum, + scratch_pool)); } /* Read the next entry in the changes record from file FILE and store @@ -228,7 +292,7 @@ read_change(change_t **change_p, return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, _("Invalid changes line in rev-file")); - info->node_rev_id = svn_fs_fs__id_parse(str, result_pool); + SVN_ERR(svn_fs_fs__id_parse(&info->node_rev_id, str, result_pool)); if (info->node_rev_id == NULL) return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, _("Invalid changes line in rev-file")); @@ -324,7 +388,9 @@ read_change(change_t **change_p, } /* Get the mergeinfo-mod flag if given. Otherwise, the next thing - is the path starting with a slash. */ + is the path starting with a slash. Also, we must initialize the + flag explicitly because 0 is not valid for a svn_tristate_t. */ + info->mergeinfo_mod = svn_tristate_unknown; if (*last_str != '/') { str = svn_cstring_tokenize(" ", &last_str); @@ -346,8 +412,12 @@ read_change(change_t **change_p, _("Invalid mergeinfo-mod flag in rev-file")); } } - + /* Get the changed path. */ + if (!svn_fspath__is_canonical(last_str)) + return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, + _("Invalid path in changes line")); + change->path.len = strlen(last_str); change->path.data = apr_pstrdup(result_pool, last_str); @@ -362,15 +432,11 @@ read_change(change_t **change_p, else { last_str = line->data; - str = svn_cstring_tokenize(" ", &last_str); - if (! str) - return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, - _("Invalid changes line in rev-file")); - info->copyfrom_rev = SVN_STR_TO_REV(str); + SVN_ERR(parse_revnum(&info->copyfrom_rev, (const char **)&last_str)); - if (! last_str) + if (!svn_fspath__is_canonical(last_str)) return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, - _("Invalid changes line in rev-file")); + _("Invalid copy-from path in changes line")); info->copyfrom_path = apr_pstrdup(result_pool, last_str); } @@ -437,11 +503,11 @@ svn_fs_fs__read_changes_incrementally(sv return SVN_NO_ERROR; } -/* Write a single change entry, path PATH, change CHANGE, and copyfrom - string COPYFROM, into the file specified by FILE. Only include the - node kind field if INCLUDE_NODE_KIND is true. Only include the - mergeinfo-mod field if INCLUDE_MERGEINFO_MODS is true. All temporary - allocations are in SCRATCH_POOL. */ +/* Write a single change entry, path PATH, change CHANGE, to STREAM. + + Only include the node kind field if INCLUDE_NODE_KIND is true. Only + include the mergeinfo-mod field if INCLUDE_MERGEINFO_MODS is true. + All temporary allocations are in SCRATCH_POOL. */ static svn_error_t * write_change_entry(svn_stream_t *stream, const char *path, @@ -534,13 +600,19 @@ svn_fs_fs__write_changes(svn_stream_t *s svn_boolean_t include_node_kinds = ffd->format >= SVN_FS_FS__MIN_KIND_IN_CHANGED_FORMAT; svn_boolean_t include_mergeinfo_mods = - ffd->format >= SVN_FS_FS__MIN_MERGEINFO_IN_CHANGES_FORMAT; + ffd->format >= SVN_FS_FS__MIN_MERGEINFO_IN_CHANGED_FORMAT; apr_array_header_t *sorted_changed_paths; int i; /* For the sake of the repository administrator sort the changes so that the final file is deterministic and repeatable, however the - rest of the FSFS code doesn't require any particular order here. */ + rest of the FSFS code doesn't require any particular order here. + + Also, this sorting is only effective in writing all entries with + a single call as write_final_changed_path_info() does. For the + list being written incrementally during transaction, we actually + *must not* change the order of entries from different calls. + */ sorted_changed_paths = svn_sort__hash(changes, svn_sort_compare_items_lexically, scratch_pool); @@ -584,8 +656,8 @@ read_header_block(apr_hash_t **headers, { svn_stringbuf_t *header_str; const char *name, *value; - apr_ssize_t i = 0; - apr_ssize_t name_len; + apr_size_t i = 0; + apr_size_t name_len; svn_boolean_t eof; SVN_ERR(svn_stream_readline(stream, &header_str, "\n", &eof, @@ -609,13 +681,10 @@ read_header_block(apr_hash_t **headers, name = header_str->data; name_len = i; - /* Skip over the NULL byte and the space following it. */ - i += 2; - - if (i > header_str->len) + /* Check if we have enough data to parse. */ + if (i + 2 > header_str->len) { /* Restore the original line for the error. */ - i -= 2; header_str->data[i] = ':'; return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, _("Found malformed header '%s' in " @@ -623,6 +692,9 @@ read_header_block(apr_hash_t **headers, header_str->data); } + /* Skip over the NULL byte and the space following it. */ + i += 2; + value = header_str->data + i; /* header_str is safely in our pool, so we can use bits of it as @@ -649,12 +721,7 @@ svn_fs_fs__parse_representation(represen rep = apr_pcalloc(result_pool, sizeof(*rep)); *rep_p = rep; - str = svn_cstring_tokenize(" ", &string); - if (str == NULL) - return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, - _("Malformed text representation offset line in node-rev")); - - rep->revision = SVN_STR_TO_REV(str); + SVN_ERR(parse_revnum(&rep->revision, (const char **)&string)); /* initialize transaction info (never stored) */ svn_fs_fs__id_txn_reset(&rep->txn_id); @@ -797,7 +864,7 @@ svn_fs_fs__read_noderev(node_revision_t SVN_ERR(svn_stream_close(stream)); - noderev->id = svn_fs_fs__id_parse(value, result_pool); + SVN_ERR(svn_fs_fs__id_parse(&noderev->id, value, result_pool)); noderev_id = value; /* for error messages later */ /* Read the type. */ @@ -848,13 +915,19 @@ svn_fs_fs__read_noderev(node_revision_t } else { + if (!svn_fspath__is_canonical(value)) + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + _("Non-canonical cpath field in node-rev '%s'"), + noderev_id); + noderev->created_path = apr_pstrdup(result_pool, value); } /* Get the predecessor ID. */ value = svn_hash_gets(headers, HEADER_PRED); if (value) - noderev->predecessor_id = svn_fs_fs__id_parse(value, result_pool); + SVN_ERR(svn_fs_fs__id_parse(&noderev->predecessor_id, value, + result_pool)); /* Get the copyroot. */ value = svn_hash_gets(headers, HEADER_COPYROOT); @@ -865,17 +938,9 @@ svn_fs_fs__read_noderev(node_revision_t } else { - char *str; - - str = svn_cstring_tokenize(" ", &value); - if (str == NULL) - return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, - _("Malformed copyroot line in node-rev '%s'"), - noderev_id); + SVN_ERR(parse_revnum(&noderev->copyroot_rev, (const char **)&value)); - noderev->copyroot_rev = SVN_STR_TO_REV(str); - - if (*value == '\0') + if (!svn_fspath__is_canonical(value)) return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, _("Malformed copyroot line in node-rev '%s'"), noderev_id); @@ -891,13 +956,7 @@ svn_fs_fs__read_noderev(node_revision_t } else { - char *str = svn_cstring_tokenize(" ", &value); - if (str == NULL) - return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, - _("Malformed copyfrom line in node-rev '%s'"), - noderev_id); - - noderev->copyfrom_rev = SVN_STR_TO_REV(str); + SVN_ERR(parse_revnum(&noderev->copyfrom_rev, (const char **)&value)); if (*value == 0) return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, @@ -1088,10 +1147,7 @@ svn_fs_fs__read_rep_header(svn_fs_fs__re if (! str || (strcmp(str, REP_DELTA) != 0)) goto error; - str = svn_cstring_tokenize(" ", &last_str); - if (! str) - goto error; - (*header)->base_revision = SVN_STR_TO_REV(str); + SVN_ERR(parse_revnum(&(*header)->base_revision, (const char **)&last_str)); str = svn_cstring_tokenize(" ", &last_str); if (! str) Modified: subversion/branches/authzperf/subversion/libsvn_fs_fs/low_level.h URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_fs/low_level.h?rev=1649205&r1=1649204&r2=1649205&view=diff ============================================================================== --- subversion/branches/authzperf/subversion/libsvn_fs_fs/low_level.h (original) +++ subversion/branches/authzperf/subversion/libsvn_fs_fs/low_level.h Sat Jan 3 14:00:41 2015 @@ -64,25 +64,35 @@ svn_fs_fs__unparse_revision_trailer(apr_ /* Given the format 7+ revision / pack FOOTER, parse it destructively * and return the start offsets of the index data in *L2P_OFFSET and - * *P2L_OFFSET, respectively. + * *P2L_OFFSET, respectively. Also, return the expected checksums in + * in *L2P_CHECKSUM and *P2L_CHECKSUM. * * Note that REV is only used to construct nicer error objects that - * mention this revision. + * mention this revision. Allocate the checksums in RESULT_POOL. */ svn_error_t * svn_fs_fs__parse_footer(apr_off_t *l2p_offset, + svn_checksum_t **l2p_checksum, apr_off_t *p2l_offset, + svn_checksum_t **p2l_checksum, svn_stringbuf_t *footer, - svn_revnum_t rev); + svn_revnum_t rev, + apr_pool_t *result_pool); -/* Given the offset of the L2P index data in L2P_OFFSET and the offset of - * the P2L index data in P2L_OFFSET, return the corresponding format 7+ - * revision / pack file footer. Allocate it in RESULT_POOL. +/* Given the offset of the L2P index data in L2P_OFFSET, the content + * checksum in L2P_CHECKSUM and the offset plus checksum of the P2L + * index data in P2L_OFFSET and P2L_CHECKSUM. + * + * Return the corresponding format 7+ revision / pack file footer. + * Allocate it in RESULT_POOL and use SCRATCH_POOL for temporary. */ svn_stringbuf_t * svn_fs_fs__unparse_footer(apr_off_t l2p_offset, + svn_checksum_t *l2p_checksum, apr_off_t p2l_offset, - apr_pool_t *result_pool); + svn_checksum_t *p2l_checksum, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool); /* Read all the changes from STREAM and store them in *CHANGES, allocated in RESULT_POOL. Do temporary allocations in SCRATCH_POOL. */ @@ -112,8 +122,9 @@ svn_fs_fs__read_changes_incrementally(sv /* Write the changed path info from CHANGES in filesystem FS to the output stream STREAM. You may call this function multiple time on - the same stream but the last call should set TERMINATE_LIST to write - an extra empty line that marks the end of the changed paths list. + the same stream. If you are writing to a (proto-)revision file, + the last call must set TERMINATE_LIST to write an extra empty line + that marks the end of the changed paths list. Perform temporary allocations in SCRATCH_POOL. */ svn_error_t * Modified: subversion/branches/authzperf/subversion/libsvn_fs_fs/pack.c URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_fs/pack.c?rev=1649205&r1=1649204&r2=1649205&view=diff ============================================================================== --- subversion/branches/authzperf/subversion/libsvn_fs_fs/pack.c (original) +++ subversion/branches/authzperf/subversion/libsvn_fs_fs/pack.c Sat Jan 3 14:00:41 2015 @@ -478,8 +478,8 @@ copy_item_to_temp(pack_context_t *contex { svn_fs_fs__p2l_entry_t *new_entry = apr_pmemdup(context->info_pool, entry, sizeof(*entry)); - new_entry->offset = 0; - SVN_ERR(svn_io_file_seek(temp_file, SEEK_CUR, &new_entry->offset, pool)); + + SVN_ERR(svn_fs_fs__get_file_offset(&new_entry->offset, temp_file, pool)); APR_ARRAY_PUSH(entries, svn_fs_fs__p2l_entry_t *) = new_entry; SVN_ERR(copy_file_data(context, temp_file, rev_file, entry->size, pool)); @@ -566,9 +566,7 @@ copy_rep_to_temp(pack_context_t *context /* create a copy of ENTRY, make it point to the copy destination and * store it in CONTEXT */ entry = apr_pmemdup(context->info_pool, entry, sizeof(*entry)); - entry->offset = 0; - SVN_ERR(svn_io_file_seek(context->reps_file, SEEK_CUR, &entry->offset, - pool)); + SVN_ERR(svn_fs_fs__get_file_offset(&entry->offset, context->reps_file, pool)); add_item_rep_mapping(context, entry); /* read & parse the representation header */ @@ -589,7 +587,7 @@ copy_rep_to_temp(pack_context_t *context } /* copy the whole rep (including header!) to our temp file */ - SVN_ERR(svn_io_file_seek(rev_file, SEEK_SET, &source_offset, pool)); + SVN_ERR(svn_io_file_seek(rev_file, APR_SET, &source_offset, pool)); SVN_ERR(copy_file_data(context, context->reps_file, rev_file, entry->size, pool)); @@ -645,12 +643,11 @@ compare_dir_entries_format6(const svn_so apr_array_header_t * svn_fs_fs__order_dir_entries(svn_fs_t *fs, apr_hash_t *directory, - svn_revnum_t revision, apr_pool_t *pool) { apr_array_header_t *ordered = svn_sort__hash(directory, - svn_fs_fs__use_log_addressing(fs, revision) + svn_fs_fs__use_log_addressing(fs) ? compare_dir_entries_format7 : compare_dir_entries_format6, pool); @@ -722,13 +719,12 @@ copy_node_to_temp(pack_context_t *contex /* create a copy of ENTRY, make it point to the copy destination and * store it in CONTEXT */ entry = apr_pmemdup(context->info_pool, entry, sizeof(*entry)); - entry->offset = 0; - SVN_ERR(svn_io_file_seek(context->reps_file, SEEK_CUR, - &entry->offset, pool)); + SVN_ERR(svn_fs_fs__get_file_offset(&entry->offset, context->reps_file, + pool)); add_item_rep_mapping(context, entry); /* copy the noderev to our temp file */ - SVN_ERR(svn_io_file_seek(rev_file, SEEK_SET, &source_offset, pool)); + SVN_ERR(svn_io_file_seek(rev_file, APR_SET, &source_offset, pool)); SVN_ERR(copy_file_data(context, context->reps_file, rev_file, entry->size, pool)); @@ -1112,7 +1108,7 @@ store_item(pack_context_t *context, /* select the item in the source file and copy it into the target * pack file */ - SVN_ERR(svn_io_file_seek(temp_file, SEEK_SET, &item->offset, pool)); + SVN_ERR(svn_io_file_seek(temp_file, APR_SET, &item->offset, pool)); SVN_ERR(copy_file_data(context, context->pack_file, temp_file, item->size, pool)); @@ -1165,7 +1161,6 @@ copy_reps_from_temp(pack_context_t *cont { apr_pool_t *iterpool = svn_pool_create(pool); apr_array_header_t *path_order = context->path_order; - apr_array_header_t *parts = apr_array_make(pool, 16, sizeof(void*)); int i; /* copy items in path order. */ @@ -1185,9 +1180,6 @@ copy_reps_from_temp(pack_context_t *cont SVN_ERR(store_item(context, temp_file, node_part, iterpool)); if (rep_part) SVN_ERR(store_item(context, temp_file, rep_part, iterpool)); - - /* processed all items */ - apr_array_clear(parts); } svn_pool_destroy(iterpool); @@ -1311,7 +1303,8 @@ pack_range(pack_context_t *context, SVN_ERR(svn_fs_fs__p2l_index_lookup(&entries, context->fs, rev_file, revision, offset, - ffd->p2l_page_size, iterpool)); + ffd->p2l_page_size, iterpool, + iterpool)); for (i = 0; i < entries->nelts; ++i) { @@ -1329,7 +1322,7 @@ pack_range(pack_context_t *context, offset = entry->offset; if (offset < rev_file->l2p_offset) { - SVN_ERR(svn_io_file_seek(rev_file->file, SEEK_SET, &offset, + SVN_ERR(svn_io_file_seek(rev_file->file, APR_SET, &offset, iterpool2)); if (entry->type == SVN_FS_FS__ITEM_TYPE_CHANGES) @@ -1445,7 +1438,8 @@ append_revision(pack_context_t *context, svn_pool_clear(iterpool); SVN_ERR(svn_fs_fs__p2l_index_lookup(&entries, context->fs, rev_file, context->start_rev, offset, - ffd->p2l_page_size, iterpool)); + ffd->p2l_page_size, iterpool, + iterpool)); for (i = 0; i < entries->nelts; ++i) { @@ -1531,7 +1525,7 @@ pack_log_addressed(svn_fs_t *fs, /* phase 1: determine the size of the revisions to pack */ SVN_ERR(svn_fs_fs__l2p_get_max_ids(&max_ids, fs, shard_rev, context.shard_end_rev - shard_rev, - pool)); + pool, pool)); /* pack revisions in ranges that don't exceed MAX_MEM */ for (i = 0; i < max_ids->nelts; ++i) @@ -1755,7 +1749,7 @@ pack_rev_shard(svn_fs_t *fs, SVN_ERR(svn_io_dir_make(pack_file_dir, APR_OS_DEFAULT, pool)); /* Index information files */ - if (svn_fs_fs__use_log_addressing(fs, shard_rev)) + if (svn_fs_fs__use_log_addressing(fs)) SVN_ERR(pack_log_addressed(fs, pack_file_dir, shard_path, shard_rev, max_mem, cancel_func, cancel_baton, pool)); else Modified: subversion/branches/authzperf/subversion/libsvn_fs_fs/pack.h URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_fs/pack.h?rev=1649205&r1=1649204&r2=1649205&view=diff ============================================================================== --- subversion/branches/authzperf/subversion/libsvn_fs_fs/pack.h (original) +++ subversion/branches/authzperf/subversion/libsvn_fs_fs/pack.h Sat Jan 3 14:00:41 2015 @@ -52,13 +52,11 @@ svn_fs_fs__get_packed_offset(apr_off_t * /* Return the svn_dir_entry_t* objects of DIRECTORY in an APR array * allocated in POOL with entries added in storage (on-disk) order. - * FS format and the directory's REVISION number will be used to pick - * the optimal ordering strategy. + * FS format will be used to pick the optimal ordering strategy. */ apr_array_header_t * svn_fs_fs__order_dir_entries(svn_fs_t *fs, apr_hash_t *directory, - svn_revnum_t revision, apr_pool_t *pool); Modified: subversion/branches/authzperf/subversion/libsvn_fs_fs/recovery.c URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_fs/recovery.c?rev=1649205&r1=1649204&r2=1649205&view=diff ============================================================================== --- subversion/branches/authzperf/subversion/libsvn_fs_fs/recovery.c (original) +++ subversion/branches/authzperf/subversion/libsvn_fs_fs/recovery.c Sat Jan 3 14:00:41 2015 @@ -216,7 +216,7 @@ recover_find_max_ids(svn_fs_t *fs, char *str_val; char *str; svn_node_kind_t kind; - svn_fs_id_t *id; + const svn_fs_id_t *id; const svn_fs_fs__id_part_t *rev_item; apr_uint64_t node_id, copy_id; apr_off_t child_dir_offset; @@ -246,7 +246,7 @@ recover_find_max_ids(svn_fs_t *fs, return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, _("Directory entry corrupt")); - id = svn_fs_fs__id_parse(str, iterpool); + SVN_ERR(svn_fs_fs__id_parse(&id, str, iterpool)); rev_item = svn_fs_fs__id_rev_item(id); if (rev_item->revision != rev) @@ -345,8 +345,10 @@ recover_body(void *baton, apr_pool_t *po svn_revnum_t youngest_rev; svn_node_kind_t youngest_revprops_kind; - /* Lose potentially corrupted data in temp files */ - SVN_ERR(svn_fs_fs__cleanup_revprop_namespace(fs)); + /* The admin may have created a plain copy of this repo before attempting + to recover it (hotcopy may or may not work with corrupted repos). + Bump the instance ID. */ + SVN_ERR(svn_fs_fs__set_uuid(fs, fs->uuid, NULL, pool)); /* We need to know the largest revision in the filesystem. */ SVN_ERR(recover_get_largest_revision(fs, &max_rev, pool)); Modified: subversion/branches/authzperf/subversion/libsvn_fs_fs/rev_file.c URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_fs/rev_file.c?rev=1649205&r1=1649204&r2=1649205&view=diff ============================================================================== --- subversion/branches/authzperf/subversion/libsvn_fs_fs/rev_file.c (original) +++ subversion/branches/authzperf/subversion/libsvn_fs_fs/rev_file.c Sat Jan 3 14:00:41 2015 @@ -31,6 +31,8 @@ #include "private/svn_io_private.h" #include "svn_private_config.h" +/* Initialize the *FILE structure for REVISION in filesystem FS. Set its + * pool member to the provided POOL. */ static void init_revision_file(svn_fs_fs__revision_file_t *file, svn_fs_t *fs, @@ -40,9 +42,7 @@ init_revision_file(svn_fs_fs__revision_f fs_fs_data_t *ffd = fs->fsap_data; file->is_packed = svn_fs_fs__is_packed_rev(fs, revision); - file->start_revision = revision < ffd->min_unpacked_rev - ? revision - (revision % ffd->max_files_per_dir) - : revision; + file->start_revision = svn_fs_fs__packed_base_rev(fs, revision); file->file = NULL; file->stream = NULL; @@ -50,7 +50,9 @@ init_revision_file(svn_fs_fs__revision_f file->l2p_stream = NULL; file->block_size = ffd->block_size; file->l2p_offset = -1; + file->l2p_checksum = NULL; file->p2l_offset = -1; + file->p2l_checksum = NULL; file->footer_offset = -1; file->pool = pool; } @@ -176,6 +178,7 @@ open_pack_or_rev_file(svn_fs_fs__revisio /* We failed for the first time. Refresh cache & retry. */ SVN_ERR(svn_fs_fs__update_min_unpacked_rev(fs, scratch_pool)); + file->start_revision = svn_fs_fs__packed_base_rev(fs, rev); retry = TRUE; } @@ -253,8 +256,10 @@ svn_fs_fs__auto_read_footer(svn_fs_fs__r footer->data[footer->len] = '\0'; /* Extract index locations. */ - SVN_ERR(svn_fs_fs__parse_footer(&file->l2p_offset, &file->p2l_offset, - footer, file->start_revision)); + SVN_ERR(svn_fs_fs__parse_footer(&file->l2p_offset, &file->l2p_checksum, + &file->p2l_offset, &file->p2l_checksum, + footer, file->start_revision, + file->pool)); file->footer_offset = filesize - footer_length - 1; } Modified: subversion/branches/authzperf/subversion/libsvn_fs_fs/rev_file.h URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_fs/rev_file.h?rev=1649205&r1=1649204&r2=1649205&view=diff ============================================================================== --- subversion/branches/authzperf/subversion/libsvn_fs_fs/rev_file.h (original) +++ subversion/branches/authzperf/subversion/libsvn_fs_fs/rev_file.h Sat Jan 3 14:00:41 2015 @@ -72,11 +72,19 @@ typedef struct svn_fs_fs__revision_file_ * has not been called, yet. */ apr_off_t l2p_offset; + /* MD5 checksum on the whole on-disk representation of the L2P index. + * NULL if svn_fs_fs__auto_read_footer has not been called, yet. */ + svn_checksum_t *l2p_checksum; + /* Offset within FILE at which the L2P index ends and the P2L index * data starts. Greater than L2P_OFFSET. -1 if svn_fs_fs__auto_read_footer * has not been called, yet. */ apr_off_t p2l_offset; + /* MD5 checksum on the whole on-disk representation of the P2L index. + * NULL if svn_fs_fs__auto_read_footer has not been called, yet. */ + svn_checksum_t *p2l_checksum; + /* Offset within FILE at which the P2L index ends and the footer starts. * Greater than P2L_OFFSET. -1 if svn_fs_fs__auto_read_footer has not * been called, yet. */ Modified: subversion/branches/authzperf/subversion/libsvn_fs_fs/revprops.c URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_fs/revprops.c?rev=1649205&r1=1649204&r2=1649205&view=diff ============================================================================== --- subversion/branches/authzperf/subversion/libsvn_fs_fs/revprops.c (original) +++ subversion/branches/authzperf/subversion/libsvn_fs_fs/revprops.c Sat Jan 3 14:00:41 2015 @@ -41,12 +41,6 @@ process got aborted and that we have re-read revprops. */ #define REVPROP_CHANGE_TIMEOUT (10 * 1000000) -/* The following are names of atomics that will be used to communicate - * revprop updates across all processes on this machine. */ -#define ATOMIC_REVPROP_GENERATION "rev-prop-generation" -#define ATOMIC_REVPROP_TIMEOUT "rev-prop-timeout" -#define ATOMIC_REVPROP_NAMESPACE "rev-prop-atomics" - svn_error_t * svn_fs_fs__upgrade_pack_revprops(svn_fs_t *fs, svn_fs_upgrade_notify_t notify_func, @@ -141,403 +135,6 @@ svn_fs_fs__upgrade_cleanup_pack_revprops return SVN_NO_ERROR; } -/* Revprop caching management. - * - * Mechanism: - * ---------- - * - * Revprop caching needs to be activated and will be deactivated for the - * respective FS instance if the necessary infrastructure could not be - * initialized. In deactivated mode, there is almost no runtime overhead - * associated with revprop caching. As long as no revprops are being read - * or changed, revprop caching imposes no overhead. - * - * When activated, we cache revprops using (revision, generation) pairs - * as keys with the generation being incremented upon every revprop change. - * Since the cache is process-local, the generation needs to be tracked - * for at least as long as the process lives but may be reset afterwards. - * - * To track the revprop generation, we use two-layer approach. On the lower - * level, we use named atomics to have a system-wide consistent value for - * the current revprop generation. However, those named atomics will only - * remain valid for as long as at least one process / thread in the system - * accesses revprops in the respective repository. The underlying shared - * memory gets cleaned up afterwards. - * - * On the second level, we will use a persistent file to track the latest - * revprop generation. It will be written upon each revprop change but - * only be read if we are the first process to initialize the named atomics - * with that value. - * - * The overhead for the second and following accesses to revprops is - * almost zero on most systems. - * - * - * Tech aspects: - * ------------- - * - * A problem is that we need to provide a globally available file name to - * back the SHM implementation on OSes that need it. We can only assume - * write access to some file within the respective repositories. Because - * a given server process may access thousands of repositories during its - * lifetime, keeping the SHM data alive for all of them is also not an - * option. - * - * So, we store the new revprop generation on disk as part of each - * setrevprop call, i.e. this write will be serialized and the write order - * be guaranteed by the repository write lock. - * - * The only racy situation occurs when the data is being read again by two - * processes concurrently but in that situation, the first process to - * finish that procedure is guaranteed to be the only one that initializes - * the SHM data. Since even writers will first go through that - * initialization phase, they will never operate on stale data. - */ - -/* Read revprop generation as stored on disk for repository FS. The result - * is returned in *CURRENT. Default to 2 if no such file is available. - */ -static svn_error_t * -read_revprop_generation_file(apr_int64_t *current, - svn_fs_t *fs, - apr_pool_t *pool) -{ - svn_error_t *err; - apr_file_t *file; - char buf[80]; - apr_size_t len; - const char *path = svn_fs_fs__path_revprop_generation(fs, pool); - - err = svn_io_file_open(&file, path, - APR_READ | APR_BUFFERED, - APR_OS_DEFAULT, pool); - if (err && APR_STATUS_IS_ENOENT(err->apr_err)) - { - svn_error_clear(err); - *current = 2; - - return SVN_NO_ERROR; - } - SVN_ERR(err); - - len = sizeof(buf); - SVN_ERR(svn_io_read_length_line(file, buf, &len, pool)); - - /* Check that the first line contains only digits. */ - SVN_ERR(svn_fs_fs__check_file_buffer_numeric(buf, 0, path, - "Revprop Generation", pool)); - SVN_ERR(svn_cstring_atoi64(current, buf)); - - return svn_io_file_close(file, pool); -} - -/* Write the CURRENT revprop generation to disk for repository FS. - */ -svn_error_t * -svn_fs_fs__write_revprop_generation_file(svn_fs_t *fs, - apr_int64_t current, - apr_pool_t *pool) -{ - char buf[SVN_INT64_BUFFER_SIZE]; - apr_size_t len = svn__i64toa(buf, current); - buf[len] = '\n'; - - SVN_ERR(svn_io_write_atomic(svn_fs_fs__path_revprop_generation(fs, pool), - buf, len + 1, - NULL /* copy_perms */, pool)); - - return SVN_NO_ERROR; -} - -/* Make sure the revprop_namespace member in FS is set. */ -static svn_error_t * -ensure_revprop_namespace(svn_fs_t *fs) -{ - fs_fs_data_t *ffd = fs->fsap_data; - - return ffd->revprop_namespace == NULL - ? svn_atomic_namespace__create(&ffd->revprop_namespace, - svn_dirent_join(fs->path, - ATOMIC_REVPROP_NAMESPACE, - fs->pool), - fs->pool) - : SVN_NO_ERROR; -} - -svn_error_t * -svn_fs_fs__cleanup_revprop_namespace(svn_fs_t *fs) -{ - const char *name = svn_dirent_join(fs->path, - ATOMIC_REVPROP_NAMESPACE, - fs->pool); - return svn_error_trace(svn_atomic_namespace__cleanup(name, fs->pool)); -} - -/* Make sure the revprop_generation member in FS is set and, if necessary, - * initialized with the latest value stored on disk. - */ -static svn_error_t * -ensure_revprop_generation(svn_fs_t *fs, apr_pool_t *pool) -{ - fs_fs_data_t *ffd = fs->fsap_data; - - SVN_ERR(ensure_revprop_namespace(fs)); - if (ffd->revprop_generation == NULL) - { - apr_int64_t current; - - SVN_ERR(svn_named_atomic__get(&ffd->revprop_generation, - ffd->revprop_namespace, - ATOMIC_REVPROP_GENERATION, - TRUE)); - - /* If the generation is at 0, we just created a new namespace - * (it would be at least 2 otherwise). Read the latest generation - * from disk and if we are the first one to initialize the atomic - * (i.e. is still 0), set it to the value just gotten. - */ - SVN_ERR(svn_named_atomic__read(¤t, ffd->revprop_generation)); - if (current == 0) - { - SVN_ERR(read_revprop_generation_file(¤t, fs, pool)); - SVN_ERR(svn_named_atomic__cmpxchg(NULL, current, 0, - ffd->revprop_generation)); - } - } - - return SVN_NO_ERROR; -} - -/* Make sure the revprop_timeout member in FS is set. */ -static svn_error_t * -ensure_revprop_timeout(svn_fs_t *fs) -{ - fs_fs_data_t *ffd = fs->fsap_data; - - SVN_ERR(ensure_revprop_namespace(fs)); - return ffd->revprop_timeout == NULL - ? svn_named_atomic__get(&ffd->revprop_timeout, - ffd->revprop_namespace, - ATOMIC_REVPROP_TIMEOUT, - TRUE) - : SVN_NO_ERROR; -} - -/* Create an error object with the given MESSAGE and pass it to the - WARNING member of FS. Clears UNDERLYING_ERR. */ -static void -log_revprop_cache_init_warning(svn_fs_t *fs, - svn_error_t *underlying_err, - const char *message, - apr_pool_t *pool) -{ - svn_error_t *err = svn_error_createf( - SVN_ERR_FS_REVPROP_CACHE_INIT_FAILURE, - underlying_err, message, - svn_dirent_local_style(fs->path, pool)); - - if (fs->warning) - (fs->warning)(fs->warning_baton, err); - - svn_error_clear(err); -} - -/* Test whether revprop cache and necessary infrastructure are - available in FS. */ -static svn_boolean_t -has_revprop_cache(svn_fs_t *fs, apr_pool_t *pool) -{ - fs_fs_data_t *ffd = fs->fsap_data; - svn_error_t *error; - - /* is the cache (still) enabled? */ - if (ffd->revprop_cache == NULL) - return FALSE; - - /* is it efficient? */ - if (!svn_named_atomic__is_efficient()) - { - /* access to it would be quite slow - * -> disable the revprop cache for good - */ - ffd->revprop_cache = NULL; - log_revprop_cache_init_warning(fs, NULL, - "Revprop caching for '%s' disabled" - " because it would be inefficient.", - pool); - - return FALSE; - } - - /* try to access our SHM-backed infrastructure */ - error = ensure_revprop_generation(fs, pool); - if (error) - { - /* failure -> disable revprop cache for good */ - - ffd->revprop_cache = NULL; - log_revprop_cache_init_warning(fs, error, - "Revprop caching for '%s' disabled " - "because SHM infrastructure for revprop " - "caching failed to initialize.", - pool); - - return FALSE; - } - - return TRUE; -} - -/* Baton structure for revprop_generation_fixup. */ -typedef struct revprop_generation_fixup_t -{ - /* revprop generation to read */ - apr_int64_t *generation; - - /* containing the revprop_generation member to query */ - fs_fs_data_t *ffd; -} revprop_generation_upgrade_t; - -/* If the revprop generation has an odd value, it means the original writer - of the revprop got killed. We don't know whether that process as able - to change the revprop data but we assume that it was. Therefore, we - increase the generation in that case to basically invalidate everyone's - cache content. - Execute this only while holding the write lock to the repo in baton->FFD. - */ -static svn_error_t * -revprop_generation_fixup(void *void_baton, - apr_pool_t *pool) -{ - revprop_generation_upgrade_t *baton = void_baton; - assert(baton->ffd->has_write_lock); - - /* Maybe, either the original revprop writer or some other reader has - already corrected / bumped the revprop generation. Thus, we need - to read it again. */ - SVN_ERR(svn_named_atomic__read(baton->generation, - baton->ffd->revprop_generation)); - - /* Cause everyone to re-read revprops upon their next access, if the - last revprop write did not complete properly. */ - while (*baton->generation % 2) - SVN_ERR(svn_named_atomic__add(baton->generation, - 1, - baton->ffd->revprop_generation)); - - return SVN_NO_ERROR; -} - -/* Read the current revprop generation and return it in *GENERATION. - Also, detect aborted / crashed writers and recover from that. - Use the access object in FS to set the shared mem values. */ -static svn_error_t * -read_revprop_generation(apr_int64_t *generation, - svn_fs_t *fs, - apr_pool_t *pool) -{ - apr_int64_t current = 0; - fs_fs_data_t *ffd = fs->fsap_data; - - /* read the current revprop generation number */ - SVN_ERR(ensure_revprop_generation(fs, pool)); - SVN_ERR(svn_named_atomic__read(¤t, ffd->revprop_generation)); - - /* is an unfinished revprop write under the way? */ - if (current % 2) - { - apr_int64_t timeout = 0; - - /* read timeout for the write operation */ - SVN_ERR(ensure_revprop_timeout(fs)); - SVN_ERR(svn_named_atomic__read(&timeout, ffd->revprop_timeout)); - - /* has the writer process been aborted, - * i.e. has the timeout been reached? - */ - if (apr_time_now() > timeout) - { - revprop_generation_upgrade_t baton; - baton.generation = ¤t; - baton.ffd = ffd; - - /* Ensure that the original writer process no longer exists by - * acquiring the write lock to this repository. Then, fix up - * the revprop generation. - */ - if (ffd->has_write_lock) - SVN_ERR(revprop_generation_fixup(&baton, pool)); - else - SVN_ERR(svn_fs_fs__with_write_lock(fs, revprop_generation_fixup, - &baton, pool)); - } - } - - /* return the value we just got */ - *generation = current; - return SVN_NO_ERROR; -} - -/* Set the revprop generation to the next odd number to indicate that - there is a revprop write process under way. If that times out, - readers shall recover from that state & re-read revprops. - Use the access object in FS to set the shared mem value. */ -static svn_error_t * -begin_revprop_change(svn_fs_t *fs, apr_pool_t *pool) -{ - apr_int64_t current; - fs_fs_data_t *ffd = fs->fsap_data; - - /* set the timeout for the write operation */ - SVN_ERR(ensure_revprop_timeout(fs)); - SVN_ERR(svn_named_atomic__write(NULL, - apr_time_now() + REVPROP_CHANGE_TIMEOUT, - ffd->revprop_timeout)); - - /* set the revprop generation to an odd value to indicate - * that a write is in progress - */ - SVN_ERR(ensure_revprop_generation(fs, pool)); - do - { - SVN_ERR(svn_named_atomic__add(¤t, - 1, - ffd->revprop_generation)); - } - while (current % 2 == 0); - - return SVN_NO_ERROR; -} - -/* Set the revprop generation to the next even number to indicate that - a) readers shall re-read revprops, and - b) the write process has been completed (no recovery required) - Use the access object in FS to set the shared mem value. */ -static svn_error_t * -end_revprop_change(svn_fs_t *fs, apr_pool_t *pool) -{ - apr_int64_t current = 1; - fs_fs_data_t *ffd = fs->fsap_data; - - /* set the revprop generation to an even value to indicate - * that a write has been completed - */ - SVN_ERR(ensure_revprop_generation(fs, pool)); - do - { - SVN_ERR(svn_named_atomic__add(¤t, - 1, - ffd->revprop_generation)); - } - while (current % 2); - - /* Save the latest generation to disk. FS is currently in a "locked" - * state such that we can be sure the be the only ones to write that - * file. - */ - return svn_fs_fs__write_revprop_generation_file(fs, current, pool); -} - /* Container for all data required to access the packed revprop file * for a given REVISION. This structure will be filled incrementally * by read_pack_revprops() its sub-routines. @@ -612,16 +209,6 @@ parse_revprop(apr_hash_t **properties, *properties = apr_hash_make(pool); SVN_ERR(svn_hash_read2(*properties, stream, SVN_HASH_TERMINATOR, pool)); - if (has_revprop_cache(fs, pool)) - { - fs_fs_data_t *ffd = fs->fsap_data; - pair_cache_key_t key = { 0 }; - - key.revision = revision; - key.second = generation; - SVN_ERR(svn_cache__set(ffd->revprop_cache, &key, *properties, - scratch_pool)); - } return SVN_NO_ERROR; } @@ -669,6 +256,19 @@ read_non_packed_revprop(apr_hash_t **pro return SVN_NO_ERROR; } +/* Return the minimum length of any packed revprop file name in REVPROPS. */ +static apr_size_t +get_min_filename_len(packed_revprops_t *revprops) +{ + char number_buffer[SVN_INT64_BUFFER_SIZE]; + + /* The revprop filenames have the format <REV>.<COUNT> - with <REV> being + * at least the first rev in the shard and <COUNT> having at least one + * digit. Thus, the minimum is 2 + #decimal places in the start rev. + */ + return svn__i64toa(number_buffer, revprops->manifest_start) + 2; +} + /* Given FS and REVPROPS->REVISION, fill the FILENAME, FOLDER and MANIFEST * members. Use POOL for allocating results and SCRATCH_POOL for temporaries. */ @@ -681,9 +281,27 @@ get_revprop_packname(svn_fs_t *fs, fs_fs_data_t *ffd = fs->fsap_data; svn_stringbuf_t *content = NULL; const char *manifest_file_path; - int idx; + int idx, rev_count; + char *buffer, *buffer_end; + const char **filenames, **filenames_end; + apr_size_t min_filename_len; + + /* Determine the dimensions. Rev 0 is excluded from the first shard. */ + rev_count = ffd->max_files_per_dir; + revprops->manifest_start + = revprops->revision - (revprops->revision % rev_count); + if (revprops->manifest_start == 0) + { + ++revprops->manifest_start; + --rev_count; + } + + revprops->manifest = apr_array_make(pool, rev_count, sizeof(const char*)); + + /* No line in the file can be less than this number of chars long. */ + min_filename_len = get_min_filename_len(revprops); - /* read content of the manifest file */ + /* Read the content of the manifest file */ revprops->folder = svn_fs_fs__path_revprops_pack_shard(fs, revprops->revision, pool); manifest_file_path @@ -691,37 +309,68 @@ get_revprop_packname(svn_fs_t *fs, SVN_ERR(svn_fs_fs__read_content(&content, manifest_file_path, pool)); - /* parse the manifest. Every line is a file name */ - revprops->manifest = apr_array_make(pool, ffd->max_files_per_dir, - sizeof(const char*)); - - /* Read all lines. Since the last line ends with a newline, we will - end up with a valid but empty string after the last entry. */ - while (content->data && *content->data) - { - APR_ARRAY_PUSH(revprops->manifest, const char*) = content->data; - content->data = strchr(content->data, '\n'); - if (content->data) - { - *content->data = 0; - content->data++; - } + /* There CONTENT must have a certain minimal size and there no + * unterminated lines at the end of the file. Both guarantees also + * simplify the parser loop below. + */ + if ( content->len < rev_count * (min_filename_len + 1) + || content->data[content->len - 1] != '\n') + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + _("Packed revprop manifest for r%ld not " + "properly terminated"), revprops->revision); + + /* Chop (parse) the manifest CONTENT into filenames, one per line. + * We only have to replace all newlines with NUL and add all line + * starts to REVPROPS->MANIFEST. + * + * There must be exactly REV_COUNT lines and that is the number of + * lines we parse from BUFFER to FILENAMES. Set the end pointer for + * the source BUFFER such that BUFFER+MIN_FILENAME_LEN is still valid + * BUFFER_END is always valid due to CONTENT->LEN > MIN_FILENAME_LEN. + * + * Please note that this loop is performance critical for e.g. 'svn log'. + * It is run 1000x per revprop access, i.e. per revision and about + * 50 million times per sec (and CPU core). + */ + for (filenames = (const char **)revprops->manifest->elts, + filenames_end = filenames + rev_count, + buffer = content->data, + buffer_end = buffer + content->len - min_filename_len; + (filenames < filenames_end) && (buffer < buffer_end); + ++filenames) + { + /* BUFFER always points to the start of the next line / filename. */ + *filenames = buffer; + + /* Find the next EOL. This is guaranteed to stay within the CONTENT + * buffer because we left enough room after BUFFER_END and we know + * we will always see a newline as the last non-NUL char. */ + buffer += min_filename_len; + while (*buffer != '\n') + ++buffer; + + /* Found EOL. Turn it into the filename terminator and move BUFFER + * to the start of the next line or CONTENT buffer end. */ + *buffer = '\0'; + ++buffer; } - content = NULL; /* No longer a valid stringbuf. */ - /* Index for our revision. Rev 0 is excluded from the first shard. */ - revprops->manifest_start = revprops->revision - - (revprops->revision % ffd->max_files_per_dir); - if (revprops->manifest_start == 0) - ++revprops->manifest_start; - idx = (int)(revprops->revision - revprops->manifest_start); + /* We must have reached the end of both buffers. */ + if (buffer < content->data + content->len) + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + _("Packed revprop manifest for r%ld " + "has too many entries"), revprops->revision); - if (revprops->manifest->nelts <= idx) + if (filenames < filenames_end) return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, - _("Packed revprop manifest for r%ld too " - "small"), revprops->revision); + _("Packed revprop manifest for r%ld " + "has too few entries"), revprops->revision); + + /* The target array has now exactly one entry per revision. */ + revprops->manifest->nelts = rev_count; /* Now get the file name */ + idx = (int)(revprops->revision - revprops->manifest_start); revprops->filename = APR_ARRAY_IDX(revprops->manifest, idx, const char*); return SVN_NO_ERROR; @@ -739,8 +388,9 @@ same_shard(svn_fs_t *fs, } /* Given FS and the full packed file content in REVPROPS->PACKED_REVPROPS, - * fill the START_REVISION, SIZES, OFFSETS members. Also, make - * PACKED_REVPROPS point to the first serialized revprop. + * fill the START_REVISION member, and make PACKED_REVPROPS point to the + * first serialized revprop. If READ_ALL is set, initialize the SIZES + * and OFFSETS members as well. * * Parse the revprops for REVPROPS->REVISION and set the PROPERTIES as * well as the SERIALIZED_SIZE member. If revprop caching has been @@ -749,6 +399,7 @@ same_shard(svn_fs_t *fs, static svn_error_t * parse_packed_revprops(svn_fs_t *fs, packed_revprops_t *revprops, + svn_boolean_t read_all, apr_pool_t *pool, apr_pool_t *scratch_pool) { @@ -805,11 +456,14 @@ parse_packed_revprops(svn_fs_t *fs, revprops->packed_revprops->len = (apr_size_t)(uncompressed->len - offset); revprops->packed_revprops->blocksize = (apr_size_t)(uncompressed->blocksize - offset); - /* STREAM still points to the first entry in the sizes list. - * Init / construct REVPROPS members. */ + /* STREAM still points to the first entry in the sizes list. */ revprops->start_revision = (svn_revnum_t)first_rev; - revprops->sizes = apr_array_make(pool, (int)count, sizeof(offset)); - revprops->offsets = apr_array_make(pool, (int)count, sizeof(offset)); + if (read_all) + { + /* Init / construct REVPROPS members. */ + revprops->sizes = apr_array_make(pool, (int)count, sizeof(offset)); + revprops->offsets = apr_array_make(pool, (int)count, sizeof(offset)); + } /* Now parse, revision by revision, the size and content of each * revisions' revprops. */ @@ -817,7 +471,6 @@ parse_packed_revprops(svn_fs_t *fs, { apr_int64_t size; svn_string_t serialized; - apr_hash_t *properties; svn_revnum_t revision = (svn_revnum_t)(first_rev + i); svn_pool_clear(iterpool); @@ -838,20 +491,18 @@ parse_packed_revprops(svn_fs_t *fs, revprops->generation, &serialized, pool, iterpool)); revprops->serialized_size = serialized.len; + + /* If we only wanted the revprops for REVISION then we are done. */ + if (!read_all) + break; } - else + + if (read_all) { - /* If revprop caching is enabled, parse any revprops. - * They will get cached as a side-effect of this. */ - if (has_revprop_cache(fs, pool)) - SVN_ERR(parse_revprop(&properties, fs, revision, - revprops->generation, &serialized, - iterpool, iterpool)); + /* fill REVPROPS data structures */ + APR_ARRAY_PUSH(revprops->sizes, apr_off_t) = serialized.len; + APR_ARRAY_PUSH(revprops->offsets, apr_off_t) = offset; } - - /* fill REVPROPS data structures */ - APR_ARRAY_PUSH(revprops->sizes, apr_off_t) = serialized.len; - APR_ARRAY_PUSH(revprops->offsets, apr_off_t) = offset; revprops->total_size += serialized.len; offset += serialized.len; @@ -862,6 +513,8 @@ parse_packed_revprops(svn_fs_t *fs, /* In filesystem FS, read the packed revprops for revision REV into * *REVPROPS. Use GENERATION to populate the revprop cache, if enabled. + * If you want to modify revprop contents / update REVPROPS, READ_ALL + * must be set. Otherwise, only the properties of REV are being provided. * Allocate data in POOL. */ static svn_error_t * @@ -869,6 +522,7 @@ read_pack_revprop(packed_revprops_t **re svn_fs_t *fs, svn_revnum_t rev, apr_int64_t generation, + svn_boolean_t read_all, apr_pool_t *pool) { apr_pool_t *iterpool = svn_pool_create(pool); @@ -911,14 +565,6 @@ read_pack_revprop(packed_revprops_t **re file_path, i + 1 < SVN_FS_FS__RECOVERABLE_RETRY_COUNT, pool)); - - /* If we could not find the file, there was a write. - * So, we should refresh our revprop generation info as well such - * that others may find data we will put into the cache. They would - * consider it outdated, otherwise. - */ - if (missing && has_revprop_cache(fs, pool)) - SVN_ERR(read_revprop_generation(&result->generation, fs, pool)); } /* the file content should be available now */ @@ -927,7 +573,7 @@ read_pack_revprop(packed_revprops_t **re _("Failed to read revprop pack file for r%ld"), rev); /* parse it. RESULT will be complete afterwards. */ - err = parse_packed_revprops(fs, result, pool, iterpool); + err = parse_packed_revprops(fs, result, read_all, pool, iterpool); svn_pool_destroy(iterpool); if (err) return svn_error_createf(SVN_ERR_FS_CORRUPT, err, @@ -957,22 +603,6 @@ svn_fs_fs__get_revision_proplist(apr_has /* should they be available at all? */ SVN_ERR(svn_fs_fs__ensure_revision_exists(rev, fs, pool)); - /* Try cache lookup first. */ - if (has_revprop_cache(fs, pool)) - { - svn_boolean_t is_cached; - pair_cache_key_t key = { 0 }; - - SVN_ERR(read_revprop_generation(&generation, fs, pool)); - - key.revision = rev; - key.second = generation; - SVN_ERR(svn_cache__get((void **) proplist_p, &is_cached, - ffd->revprop_cache, &key, pool)); - if (is_cached) - return SVN_NO_ERROR; - } - /* if REV had not been packed when we began, try reading it from the * non-packed shard. If that fails, we will fall through to packed * shard reads. */ @@ -997,7 +627,7 @@ svn_fs_fs__get_revision_proplist(apr_has if (ffd->format >= SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT && !*proplist_p) { packed_revprops_t *revprops; - SVN_ERR(read_pack_revprop(&revprops, fs, rev, generation, pool)); + SVN_ERR(read_pack_revprop(&revprops, fs, rev, generation, FALSE, pool)); *proplist_p = revprops->properties; } @@ -1043,7 +673,6 @@ write_non_packed_revprop(const char **fi * file at TMP_PATH to FINAL_PATH and give it the permissions from * PERMS_REFERENCE. * - * If indicated in BUMP_GENERATION, increase FS' revprop generation. * Finally, delete all the temporary files given in FILES_TO_DELETE. * The latter may be NULL. * @@ -1055,21 +684,11 @@ switch_to_new_revprop(svn_fs_t *fs, const char *tmp_path, const char *perms_reference, apr_array_header_t *files_to_delete, - svn_boolean_t bump_generation, apr_pool_t *pool) { - /* Now, we may actually be replacing revprops. Make sure that all other - threads and processes will know about this. */ - if (bump_generation) - SVN_ERR(begin_revprop_change(fs, pool)); - SVN_ERR(svn_fs_fs__move_into_place(tmp_path, final_path, perms_reference, pool)); - /* Indicate that the update (if relevant) has been completed. */ - if (bump_generation) - SVN_ERR(end_revprop_change(fs, pool)); - /* Clean up temporary files, if necessary. */ if (files_to_delete) { @@ -1286,13 +905,8 @@ write_packed_revprop(const char **final_ apr_off_t new_total_size; int changed_index; - /* read the current revprop generation. This value will not change - * while we hold the global write lock to this FS. */ - if (has_revprop_cache(fs, pool)) - SVN_ERR(read_revprop_generation(&generation, fs, pool)); - /* read contents of the current pack file */ - SVN_ERR(read_pack_revprop(&revprops, fs, rev, generation, pool)); + SVN_ERR(read_pack_revprop(&revprops, fs, rev, generation, TRUE, pool)); /* serialize the new revprops */ serialized = svn_stringbuf_create_empty(pool); @@ -1425,7 +1039,6 @@ svn_fs_fs__set_revision_proplist(svn_fs_ apr_pool_t *pool) { svn_boolean_t is_packed; - svn_boolean_t bump_generation = FALSE; const char *final_path; const char *tmp_path; const char *perms_reference; @@ -1436,24 +1049,6 @@ svn_fs_fs__set_revision_proplist(svn_fs_ /* this info will not change while we hold the global FS write lock */ is_packed = svn_fs_fs__is_packed_revprop(fs, rev); - /* Test whether revprops already exist for this revision. - * Only then will we need to bump the revprop generation. */ - if (has_revprop_cache(fs, pool)) - { - if (is_packed) - { - bump_generation = TRUE; - } - else - { - svn_node_kind_t kind; - SVN_ERR(svn_io_check_path(svn_fs_fs__path_revprops(fs, rev, pool), - &kind, - pool)); - bump_generation = kind != svn_node_none; - } - } - /* Serialize the new revprop data */ if (is_packed) SVN_ERR(write_packed_revprop(&final_path, &tmp_path, &files_to_delete, @@ -1471,7 +1066,7 @@ svn_fs_fs__set_revision_proplist(svn_fs_ /* Now, switch to the new revprop data. */ SVN_ERR(switch_to_new_revprop(fs, final_path, tmp_path, perms_reference, - files_to_delete, bump_generation, pool)); + files_to_delete, pool)); return SVN_NO_ERROR; } Modified: subversion/branches/authzperf/subversion/libsvn_fs_fs/revprops.h URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_fs/revprops.h?rev=1649205&r1=1649204&r2=1649205&view=diff ============================================================================== --- subversion/branches/authzperf/subversion/libsvn_fs_fs/revprops.h (original) +++ subversion/branches/authzperf/subversion/libsvn_fs_fs/revprops.h Sat Jan 3 14:00:41 2015 @@ -22,17 +22,6 @@ #include "svn_fs.h" -/* Write the CURRENT revprop generation to disk for repository FS. - */ -svn_error_t * -svn_fs_fs__write_revprop_generation_file(svn_fs_t *fs, - apr_int64_t current, - apr_pool_t *pool); - -/* Make sure the revprop_namespace member in FS is set. */ -svn_error_t * -svn_fs_fs__cleanup_revprop_namespace(svn_fs_t *fs); - /* In the filesystem FS, pack all revprop shards up to min_unpacked_rev. * * NOTE: Keep the old non-packed shards around until after the format bump. Modified: subversion/branches/authzperf/subversion/libsvn_fs_fs/structure URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_fs/structure?rev=1649205&r1=1649204&r2=1649205&view=diff ============================================================================== --- subversion/branches/authzperf/subversion/libsvn_fs_fs/structure (original) +++ subversion/branches/authzperf/subversion/libsvn_fs_fs/structure Sat Jan 3 14:00:41 2015 @@ -43,7 +43,7 @@ repository) is: <shard>.pack/ Pack directory, if the repo has been packed (see below) <rev>.<count> Pack file, if the repository has been packed (see below) manifest Pack manifest file, if a pack file exists (see below) - revprops.db SQLite database of the packed revision properties + revprops.db SQLite database of the packed revprops (format 5 only) transactions/ Subdirectory containing transactions <txnid>.txn/ Directory containing transaction <txnid> txn-protorevs/ Subdirectory containing transaction proto-revision files @@ -64,7 +64,7 @@ repository) is: format File containing the format number of this filesystem fsfs.conf Configuration file min-unpacked-rev File containing the oldest revision not in a pack file - min-unpacked-revprop File containing the oldest revision of unpacked revprop + min-unpacked-revprop Same for revision properties (format 5 only) rep-cache.db SQLite database mapping rep checksums to locations Files in the revprops directory are in the hash dump format used by @@ -214,8 +214,8 @@ Filesystem format options Currently, the only recognised format options are "layout" and "addressing". The first specifies the paths that will be used to store the revision -files and revision property files. The second specifies for which -revisions address translation is required. +files and revision property files. The second specifies that logical to +physical address translation is required. The "layout" option is followed by the name of the filesystem layout and any required parameters. The default layout, if no "layout" @@ -255,11 +255,10 @@ The supported modes, and the parameters pairs with "offset" being the byte offset relative to the beginning of the revision in the respective rev or pack file. -"logical <first-revision-to-use-it>" - 'first-revision-to-use-it' specifies the first revision to use logical - addressing, must coincide with the beginning of a shard and may be a - future revision. All earlier revisions use physical addressing. It is - illegal to use logical addressing on non-sharded repositories. +"logical" + All existing and future revision files will use logical + addressing. It is illegal to use logical addressing on non-sharded + repositories. Addressing modes @@ -613,11 +612,14 @@ lines containing "\n<root-offset> <cp-of the offset of the root directory node revision and <cp-offset> is the offset of the changed-path data. -In logical addressing mode, the revision footer is pair of offsets, -separated by a space and the whole is terminated a single byte. That -byte contains the length (as plain 8 bit value) of the footer excluding -the length byte. The first offset is the start of the log-to-phys index, -the other is the start offset of the phys-to-log index. +In logical addressing mode, the revision footer has the form + + <l2p offset> <l2p checksum> <p2l offset> <p2l checksum><terminal byte> + +The terminal byte contains the length (as plain 8 bit value) of the footer +excluding that length byte. The first offset is the start of the log-to- +phys index, followed by the digest of the MD5 checksum over its content. +The other pair gives the same of for the phys-to-log index. All numbers in the rev file format are unsigned and are represented as ASCII decimal. Modified: subversion/branches/authzperf/subversion/libsvn_fs_fs/structure-indexes URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_fs/structure-indexes?rev=1649205&r1=1649204&r2=1649205&view=diff ============================================================================== --- subversion/branches/authzperf/subversion/libsvn_fs_fs/structure-indexes (original) +++ subversion/branches/authzperf/subversion/libsvn_fs_fs/structure-indexes Sat Jan 3 14:00:41 2015 @@ -80,6 +80,14 @@ Most data is unsigned by nature but will signed integers. +Encoding in proto-index files +----------------------------- + +These have a much simpler encoding. Throughout the files, all records have +the same length (but different between L2P and P2L). All records contain +unsigned 64 bit integers only, stored in little endian byte order. + + Log-to-phys index ================= @@ -143,7 +151,7 @@ page: Index on-disk format -------------------- - index := header revisions pages offsets + index := "L2P-INDEX\n" header revisions pages offsets header := u(<header>.<first revision>) \ u(<header>.<page size>) \ @@ -158,11 +166,13 @@ Index on-disk format u(<header>.<page table>[k].<entry count>), for k in 0 .. s(<header>.<page table>)-1 - offsets := i(<header>.<page table>[k].<offsets>[0]) \ + offsets := page(k), + for k in 0 .. s(<header>.<page table>)-1 + + page(k) := i(<header>.<page table>[k].<offsets>[0]) \ i( <header>.<page table>[k].<offsets>[l] \ - <header>.<page table>[k].<offsets>[l - 1]), - for l in 1 .. s(<header>.<page table>[k].<entry count>)-1, - for k in 0 .. s(<header>.<page table>)-1 + for l in 1 .. s(<header>.<page table>[k].<entry count>)-1 u(x) ... unsigned int x in 7b/8b encoding i(x) ... signed int x in 7b/8b encoding @@ -190,7 +200,7 @@ at the beginning of the file is optional ... <eof> /* end of file. */ -All entries are pairs of 64 bit unsigned integers in machine endianess. +All entries are pairs of 64 bit unsigned integers in little endian order. Phys-to-log index @@ -256,7 +266,7 @@ entry: Index on-disk format -------------------- - index := header pages items + index := "P2L-INDEX\n" header pages items header := u(<header>.<first revision>) \ u(<header>.<file size>) \ @@ -286,14 +296,25 @@ Index on-disk format started after the begin of a given page and overlap with the next page will not be stored in the start page. The runtime representation will duplicate items overlapping page boundaries; the on-disk representation - will not. + will not. Thus, pages inside large items will have zero entries on disk. Proto index file format ----------------------- The index will be created from a proto index file containing simple -instances of the in-memory representation of svn_fs_fs__p2l_entry_t. +instances of svn_fs_fs__p2l_entry_t with the following element order: + + item offset as uint64 + item size as uint64 + item type as uint64 + modified FNV1a checksum as uint64 + revision as uint64, with SVN_INVALID_REVNUM mapped to 0 + and revisions >= 0 stored as rev+1 + item index as uint64 + +All values are stored in little endian order. + Page table and header information, except start revision and page size, can easily be derived from that information. Modified: subversion/branches/authzperf/subversion/libsvn_fs_fs/temp_serializer.c URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_fs/temp_serializer.c?rev=1649205&r1=1649204&r2=1649205&view=diff ============================================================================== --- subversion/branches/authzperf/subversion/libsvn_fs_fs/temp_serializer.c (original) +++ subversion/branches/authzperf/subversion/libsvn_fs_fs/temp_serializer.c Sat Jan 3 14:00:41 2015 @@ -205,7 +205,7 @@ static svn_temp_serializer__context_t * serialize_dir(apr_array_header_t *entries, apr_pool_t *pool) { dir_data_t dir_data; - apr_size_t i = 0; + int i = 0; svn_temp_serializer__context_t *context; /* calculate sizes */
