Modified: subversion/branches/authzperf/subversion/libsvn_fs_fs/index.c URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_fs/index.c?rev=1649205&r1=1649204&r2=1649205&view=diff ============================================================================== --- subversion/branches/authzperf/subversion/libsvn_fs_fs/index.c (original) +++ subversion/branches/authzperf/subversion/libsvn_fs_fs/index.c Sat Jan 3 14:00:41 2015 @@ -43,6 +43,33 @@ /* maximum length of a uint64 in an 7/8b encoding */ #define ENCODED_INT_LENGTH 10 +/* APR is missing an APR_OFF_T_MAX. So, define one. We will use it to + * limit file offsets stored in the indexes. + * + * We assume that everything shorter than 64 bits, it is at least 32 bits. + * We also assume that the type is always signed meaning we only have an + * effective positive range of 63 or 31 bits, respectively. + */ +static +const apr_uint64_t off_t_max = (sizeof(apr_off_t) == sizeof(apr_int64_t)) + ? APR_INT64_MAX + : APR_INT32_MAX; + +/* We store P2L proto-index entries as 6 values, 64 bits each on disk. + * See also svn_fs_fs__p2l_proto_index_add_entry(). + */ +#define P2L_PROTO_INDEX_ENTRY_SIZE (6 * sizeof(apr_uint64_t)) + +/* We put this string in front of the L2P index header. */ +#define L2P_STREAM_PREFIX "L2P-INDEX\n" + +/* We put this string in front of the P2L index header. */ +#define P2L_STREAM_PREFIX "P2L-INDEX\n" + +/* Size of the buffer that will fit the index header prefixes. */ +#define STREAM_PREFIX_LEN MAX(sizeof(L2P_STREAM_PREFIX), \ + sizeof(P2L_STREAM_PREFIX)) + /* Page tables in the log-to-phys index file exclusively contain entries * of this type to describe position and size of a given page. */ @@ -201,13 +228,15 @@ stream_error_create(svn_fs_fs__packed_nu const char *message) { const char *file_name; - apr_off_t offset = 0; + apr_off_t offset; SVN_ERR(svn_io_file_name_get(&file_name, stream->file, stream->pool)); - SVN_ERR(svn_io_file_seek(stream->file, SEEK_CUR, &offset, stream->pool)); + SVN_ERR(svn_fs_fs__get_file_offset(&offset, stream->file, stream->pool)); return svn_error_createf(err, NULL, message, file_name, - (apr_uint64_t)offset); + apr_psprintf(stream->pool, + "%" APR_UINT64_T_HEX_FMT, + (apr_uint64_t)offset)); } /* Read up to MAX_NUMBER_PREFETCH numbers from the STREAM->NEXT_OFFSET in @@ -247,16 +276,16 @@ packed_stream_read(svn_fs_fs__packed_num read = sizeof(buffer); block_left = stream->block_size - (stream->next_offset - block_start); if (block_left >= 10 && block_left < read) - read = block_left; + read = (apr_size_t)block_left; /* Don't read beyond the end of the file section that belongs to this * index / stream. */ - read = MIN(read, stream->stream_end - stream->next_offset); + read = (apr_size_t)MIN(read, stream->stream_end - stream->next_offset); err = apr_file_read(stream->file, buffer, &read); if (err && !APR_STATUS_IS_EOF(err)) return stream_error_create(stream, err, - _("Can't read index file '%s' at offset 0x%" APR_UINT64_T_HEX_FMT)); + _("Can't read index file '%s' at offset 0x%s")); /* if the last number is incomplete, trim it from the buffer */ while (read > 0 && buffer[read-1] >= 0x80) @@ -266,7 +295,7 @@ packed_stream_read(svn_fs_fs__packed_num * at least *one* further number. */ if SVN__PREDICT_FALSE(read == 0) return stream_error_create(stream, err, - _("Unexpected end of index file %s at offset 0x%"APR_UINT64_T_HEX_FMT)); + _("Unexpected end of index file %s at offset 0x%s")); /* parse file buffer and expand into stream buffer */ target = stream->buffer; @@ -300,7 +329,7 @@ packed_stream_read(svn_fs_fs__packed_num /* let's catch corrupted data early. It would surely cause * havoc further down the line. */ if SVN__PREDICT_FALSE(shift > 8 * sizeof(value)) - return svn_error_createf(SVN_ERR_FS_ITEM_INDEX_CORRUPTION, NULL, + return svn_error_createf(SVN_ERR_FS_INDEX_CORRUPTION, NULL, _("Corrupt index: number too large")); } } @@ -315,22 +344,45 @@ packed_stream_read(svn_fs_fs__packed_num /* Create and open a packed number stream reading from offsets START to * END in FILE and return it in *STREAM. Access the file in chunks of - * BLOCK_SIZE bytes. Use POOL for allocations. + * BLOCK_SIZE bytes. Expect the stream to be prefixed by STREAM_PREFIX. + * Allocate *STREAM in RESULT_POOL and use SCRATCH_POOL for temporaries. */ static svn_error_t * packed_stream_open(svn_fs_fs__packed_number_stream_t **stream, apr_file_t *file, apr_off_t start, apr_off_t end, + const char *stream_prefix, apr_size_t block_size, - apr_pool_t *pool) + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) { - svn_fs_fs__packed_number_stream_t *result - = apr_palloc(pool, sizeof(*result)); + char buffer[STREAM_PREFIX_LEN + 1] = { 0 }; + apr_size_t len = strlen(stream_prefix); + svn_fs_fs__packed_number_stream_t *result; + + /* If this is violated, we forgot to adjust STREAM_PREFIX_LEN after + * changing the index header prefixes. */ + SVN_ERR_ASSERT(len < sizeof(buffer)); + + /* Read the header prefix and compare it with the expected prefix */ + SVN_ERR(svn_io_file_aligned_seek(file, block_size, NULL, start, + scratch_pool)); + SVN_ERR(svn_io_file_read_full2(file, buffer, len, NULL, NULL, + scratch_pool)); + + if (strncmp(buffer, stream_prefix, len)) + return svn_error_createf(SVN_ERR_FS_INDEX_CORRUPTION, NULL, + _("Index stream header prefix mismatch.\n" + " expected: %s" + " found: %s"), stream_prefix, buffer); + + /* Construct the actual stream object. */ + result = apr_palloc(result_pool, sizeof(*result)); - result->pool = pool; + result->pool = result_pool; result->file = file; - result->stream_start = start; + result->stream_start = start + len; result->stream_end = end; result->used = 0; @@ -439,6 +491,18 @@ encode_int(unsigned char *p, apr_int64_t return encode_uint(p, (apr_uint64_t)(value < 0 ? -1 - 2*value : 2*value)); } +/* Append VALUE to STREAM in 7/8b encoding. + */ +static svn_error_t * +stream_write_encoded(svn_stream_t *stream, + apr_uint64_t value) +{ + unsigned char encoded[ENCODED_INT_LENGTH]; + + apr_size_t len = encode_uint(encoded, value); + return svn_error_trace(svn_stream_write(stream, (char *)encoded, &len)); +} + /* Map unsigned VALUE back to signed integer. */ static apr_int64_t @@ -447,37 +511,194 @@ decode_int(apr_uint64_t value) return (apr_int64_t)(value % 2 ? -1 - value / 2 : value / 2); } +/* Write VALUE to the PROTO_INDEX file, using SCRATCH_POOL for temporary + * allocations. + * + * The point of this function is to ensure an architecture-independent + * proto-index file format. All data is written as unsigned 64 bits ints + * in little endian byte order. 64 bits is the largest portable integer + * we have and unsigned values have well-defined conversions in C. + */ +static svn_error_t * +write_uint64_to_proto_index(apr_file_t *proto_index, + apr_uint64_t value, + apr_pool_t *scratch_pool) +{ + apr_byte_t buffer[sizeof(value)]; + int i; + apr_size_t written; + + /* Split VALUE into 8 bytes using LE ordering. */ + for (i = 0; i < sizeof(buffer); ++i) + { + /* Unsigned conversions are well-defined ... */ + buffer[i] = (apr_byte_t)value; + value >>= CHAR_BIT; + } + + /* Write it all to disk. */ + SVN_ERR(svn_io_file_write_full(proto_index, buffer, sizeof(buffer), + &written, scratch_pool)); + SVN_ERR_ASSERT(written == sizeof(buffer)); + + return SVN_NO_ERROR; +} + +/* Read one unsigned 64 bit value from PROTO_INDEX file and return it in + * *VALUE_P. If EOF is NULL, error out when trying to read beyond EOF. + * Use SCRATCH_POOL for temporary allocations. + * + * This function is the inverse to write_uint64_to_proto_index (see there), + * reading the external LE byte order and convert it into host byte order. + */ +static svn_error_t * +read_uint64_from_proto_index(apr_file_t *proto_index, + apr_uint64_t *value_p, + svn_boolean_t *eof, + apr_pool_t *scratch_pool) +{ + apr_byte_t buffer[sizeof(*value_p)]; + apr_size_t read; + + /* Read the full 8 bytes or our 64 bit value, unless we hit EOF. + * Assert that we never read partial values. */ + SVN_ERR(svn_io_file_read_full2(proto_index, buffer, sizeof(buffer), + &read, eof, scratch_pool)); + SVN_ERR_ASSERT((eof && *eof) || read == sizeof(buffer)); + + /* If we did not hit EOF, reconstruct the uint64 value and return it. */ + if (!eof || !*eof) + { + int i; + apr_uint64_t value; + + /* This could only overflow if CHAR_BIT had a value that is not + * a divisor of 64. */ + value = 0; + for (i = sizeof(buffer) - 1; i >= 0; --i) + value = (value << CHAR_BIT) + buffer[i]; + + *value_p = value; + } + + return SVN_NO_ERROR; +} + +/* Convenience function similar to read_uint64_from_proto_index, but returns + * an uint32 value in VALUE_P. Return an error if the value does not fit. + */ +static svn_error_t * +read_uint32_from_proto_index(apr_file_t *proto_index, + apr_uint32_t *value_p, + svn_boolean_t *eof, + apr_pool_t *scratch_pool) +{ + apr_uint64_t value; + SVN_ERR(read_uint64_from_proto_index(proto_index, &value, eof, + scratch_pool)); + if (!eof || !*eof) + { + if (value > APR_UINT32_MAX) + return svn_error_createf(SVN_ERR_FS_INDEX_OVERFLOW, NULL, + _("UINT32 0x%s too large, max = 0x%s"), + apr_psprintf(scratch_pool, + "%" APR_UINT64_T_HEX_FMT, + value), + apr_psprintf(scratch_pool, + "%" APR_UINT64_T_HEX_FMT, + (apr_uint64_t)APR_UINT32_MAX)); + + /* This conversion is not lossy because the value can be represented + * in the target type. */ + *value_p = (apr_uint32_t)value; + } + + return SVN_NO_ERROR; +} + +/* Convenience function similar to read_uint64_from_proto_index, but returns + * an off_t value in VALUE_P. Return an error if the value does not fit. + */ +static svn_error_t * +read_off_t_from_proto_index(apr_file_t *proto_index, + apr_off_t *value_p, + svn_boolean_t *eof, + apr_pool_t *scratch_pool) +{ + apr_uint64_t value; + SVN_ERR(read_uint64_from_proto_index(proto_index, &value, eof, + scratch_pool)); + if (!eof || !*eof) + { + if (value > off_t_max) + return svn_error_createf(SVN_ERR_FS_INDEX_OVERFLOW, NULL, + _("File offset 0x%s too large, max = 0x%s"), + apr_psprintf(scratch_pool, + "%" APR_UINT64_T_HEX_FMT, + value), + apr_psprintf(scratch_pool, + "%" APR_UINT64_T_HEX_FMT, + off_t_max)); + + /* Shortening conversion from unsigned to signed int is well-defined + * and not lossy in C because the value can be represented in the + * target type. */ + *value_p = (apr_off_t)value; + } + + return SVN_NO_ERROR; +} + /* * log-to-phys index */ -/* Write ENTRY to log-to-phys PROTO_INDEX file and verify the results. - * Use POOL for allocations. +/* Append ENTRY to log-to-phys PROTO_INDEX file. + * Use SCRATCH_POOL for temporary allocations. */ static svn_error_t * -write_entry_to_proto_index(apr_file_t *proto_index, - l2p_proto_entry_t entry, - apr_pool_t *pool) +write_l2p_entry_to_proto_index(apr_file_t *proto_index, + l2p_proto_entry_t entry, + apr_pool_t *scratch_pool) { - apr_size_t written = sizeof(entry); + SVN_ERR(write_uint64_to_proto_index(proto_index, entry.offset, + scratch_pool)); + SVN_ERR(write_uint64_to_proto_index(proto_index, entry.item_index, + scratch_pool)); + + return SVN_NO_ERROR; +} - SVN_ERR(svn_io_file_write(proto_index, &entry, &written, pool)); - SVN_ERR_ASSERT(written == sizeof(entry)); +/* Read *ENTRY from log-to-phys PROTO_INDEX file and indicate end-of-file + * in *EOF, or error out in that case if EOF is NULL. *ENTRY is in an + * undefined state if an end-of-file occurred. + * Use SCRATCH_POOL for temporary allocations. + */ +static svn_error_t * +read_l2p_entry_from_proto_index(apr_file_t *proto_index, + l2p_proto_entry_t *entry, + svn_boolean_t *eof, + apr_pool_t *scratch_pool) +{ + SVN_ERR(read_uint64_from_proto_index(proto_index, &entry->offset, eof, + scratch_pool)); + SVN_ERR(read_uint64_from_proto_index(proto_index, &entry->item_index, eof, + scratch_pool)); return SVN_NO_ERROR; } /* Write the log-2-phys index page description for the l2p_page_entry_t * array ENTRIES, starting with element START up to but not including END. - * Write the resulting representation into BUFFER. Use POOL for temporary - * allocations. + * Write the resulting representation into BUFFER. Use SCRATCH_POOL for + * temporary allocations. */ static svn_error_t * encode_l2p_page(apr_array_header_t *entries, int start, int end, svn_spillbuf_t *buffer, - apr_pool_t *pool) + apr_pool_t *scratch_pool) { unsigned char encoded[ENCODED_INT_LENGTH]; int i; @@ -490,7 +711,7 @@ encode_l2p_page(apr_array_header_t *entr apr_int64_t diff = values[i] - last_value; last_value = values[i]; SVN_ERR(svn_spillbuf__write(buffer, (const char *)encoded, - encode_int(encoded, diff), pool)); + encode_int(encoded, diff), scratch_pool)); } return SVN_NO_ERROR; @@ -499,32 +720,32 @@ encode_l2p_page(apr_array_header_t *entr svn_error_t * svn_fs_fs__l2p_proto_index_open(apr_file_t **proto_index, const char *file_name, - apr_pool_t *pool) + apr_pool_t *result_pool) { SVN_ERR(svn_io_file_open(proto_index, file_name, APR_READ | APR_WRITE | APR_CREATE | APR_APPEND | APR_BUFFERED, - APR_OS_DEFAULT, pool)); + APR_OS_DEFAULT, result_pool)); return SVN_NO_ERROR; } svn_error_t * svn_fs_fs__l2p_proto_index_add_revision(apr_file_t *proto_index, - apr_pool_t *pool) + apr_pool_t *scratch_pool) { l2p_proto_entry_t entry; entry.offset = 0; entry.item_index = 0; - return svn_error_trace(write_entry_to_proto_index(proto_index, entry, - pool)); + return svn_error_trace(write_l2p_entry_to_proto_index(proto_index, entry, + scratch_pool)); } svn_error_t * svn_fs_fs__l2p_proto_index_add_entry(apr_file_t *proto_index, apr_off_t offset, apr_uint64_t item_index, - apr_pool_t *pool) + apr_pool_t *scratch_pool) { l2p_proto_entry_t entry; @@ -539,30 +760,32 @@ svn_fs_fs__l2p_proto_index_add_entry(apr SVN_ERR_ASSERT(item_index < UINT_MAX / 2); entry.item_index = item_index; - return svn_error_trace(write_entry_to_proto_index(proto_index, entry, - pool)); + return svn_error_trace(write_l2p_entry_to_proto_index(proto_index, entry, + scratch_pool)); } svn_error_t * -svn_fs_fs__l2p_index_append(svn_fs_t *fs, +svn_fs_fs__l2p_index_append(svn_checksum_t **checksum, + svn_fs_t *fs, apr_file_t *index_file, const char *proto_file_name, svn_revnum_t revision, - apr_pool_t *pool) + apr_pool_t * result_pool, + apr_pool_t *scratch_pool) { fs_fs_data_t *ffd = fs->fsap_data; apr_file_t *proto_index = NULL; + svn_stream_t *stream; int i; apr_uint64_t entry; svn_boolean_t eof = FALSE; - unsigned char encoded[ENCODED_INT_LENGTH]; int last_page_count = 0; /* total page count at the start of the current revision */ /* temporary data structures that collect the data which will be moved to the target file in a second step */ - apr_pool_t *local_pool = svn_pool_create(pool); + apr_pool_t *local_pool = svn_pool_create(scratch_pool); apr_pool_t *iterpool = svn_pool_create(local_pool); apr_array_header_t *page_counts = apr_array_make(local_pool, 16, sizeof(apr_uint64_t)); @@ -582,7 +805,7 @@ svn_fs_fs__l2p_index_append(svn_fs_t *fs /* Paranoia check that makes later casting to int32 safe. * The current implementation is limited to 2G entries per page. */ if (ffd->l2p_page_size > APR_INT32_MAX) - return svn_error_createf(SVN_ERR_FS_ITEM_INDEX_OVERFLOW , NULL, + return svn_error_createf(SVN_ERR_FS_INDEX_OVERFLOW , NULL, _("L2P index page size %s" " exceeds current limit of 2G entries"), apr_psprintf(local_pool, "%" APR_UINT64_T_FMT, @@ -591,19 +814,16 @@ svn_fs_fs__l2p_index_append(svn_fs_t *fs /* start at the beginning of the source file */ SVN_ERR(svn_io_file_open(&proto_index, proto_file_name, APR_READ | APR_CREATE | APR_BUFFERED, - APR_OS_DEFAULT, pool)); + APR_OS_DEFAULT, scratch_pool)); /* process all entries until we fail due to EOF */ for (entry = 0; !eof; ++entry) { l2p_proto_entry_t proto_entry; - apr_size_t read = 0; /* (attempt to) read the next entry from the source */ - SVN_ERR(svn_io_file_read_full2(proto_index, - &proto_entry, sizeof(proto_entry), - &read, &eof, local_pool)); - SVN_ERR_ASSERT(eof || read == sizeof(proto_entry)); + SVN_ERR(read_l2p_entry_from_proto_index(proto_index, &proto_entry, + &eof, local_pool)); /* handle new revision */ if ((entry > 0 && proto_entry.offset == 0) || eof) @@ -613,8 +833,11 @@ svn_fs_fs__l2p_index_append(svn_fs_t *fs int entry_count = 0; for (i = 0; i < entries->nelts; i += entry_count) { - /* 1 page with up to 8k entries */ - apr_size_t last_buffer_size = svn_spillbuf__get_size(buffer); + /* 1 page with up to L2P_PAGE_SIZE entries. + * fsfs.conf settings validation guarantees this to fit into + * our address space. */ + apr_size_t last_buffer_size + = (apr_size_t)svn_spillbuf__get_size(buffer); svn_pool_clear(iterpool); @@ -643,7 +866,7 @@ svn_fs_fs__l2p_index_append(svn_fs_t *fs /* store the mapping in our array */ if (proto_entry.item_index > APR_INT32_MAX) - return svn_error_createf(SVN_ERR_FS_ITEM_INDEX_OVERFLOW , NULL, + return svn_error_createf(SVN_ERR_FS_INDEX_OVERFLOW , NULL, _("Item index %s too large " "in l2p proto index for revision %ld"), apr_psprintf(local_pool, "%" APR_UINT64_T_FMT, @@ -664,52 +887,44 @@ svn_fs_fs__l2p_index_append(svn_fs_t *fs /* Paranoia check that makes later casting to int32 safe. * The current implementation is limited to 2G pages per index. */ if (page_counts->nelts > APR_INT32_MAX) - return svn_error_createf(SVN_ERR_FS_ITEM_INDEX_OVERFLOW , NULL, + return svn_error_createf(SVN_ERR_FS_INDEX_OVERFLOW , NULL, _("L2P index page count %d" " exceeds current limit of 2G pages"), page_counts->nelts); + /* open target stream. */ + stream = svn_stream_checksummed2(svn_stream_from_aprfile2(index_file, TRUE, + local_pool), + NULL, checksum, svn_checksum_md5, FALSE, + result_pool); + + /* write header info */ - SVN_ERR(svn_io_file_write_full(index_file, encoded, - encode_uint(encoded, revision), - NULL, local_pool)); - SVN_ERR(svn_io_file_write_full(index_file, encoded, - encode_uint(encoded, ffd->l2p_page_size), - NULL, local_pool)); - SVN_ERR(svn_io_file_write_full(index_file, encoded, - encode_uint(encoded, page_counts->nelts), - NULL, local_pool)); - SVN_ERR(svn_io_file_write_full(index_file, encoded, - encode_uint(encoded, page_sizes->nelts), - NULL, local_pool)); + SVN_ERR(svn_stream_puts(stream, L2P_STREAM_PREFIX)); + SVN_ERR(stream_write_encoded(stream, revision)); + SVN_ERR(stream_write_encoded(stream, ffd->l2p_page_size)); + SVN_ERR(stream_write_encoded(stream, page_counts->nelts)); + SVN_ERR(stream_write_encoded(stream, page_sizes->nelts)); /* write the revision table */ for (i = 0; i < page_counts->nelts; ++i) { apr_uint64_t value = APR_ARRAY_IDX(page_counts, i, apr_uint64_t); - SVN_ERR(svn_io_file_write_full(index_file, encoded, - encode_uint(encoded, value), - NULL, local_pool)); + SVN_ERR(stream_write_encoded(stream, value)); } - + /* write the page table */ for (i = 0; i < page_sizes->nelts; ++i) { apr_uint64_t value = APR_ARRAY_IDX(page_sizes, i, apr_uint64_t); - SVN_ERR(svn_io_file_write_full(index_file, encoded, - encode_uint(encoded, value), - NULL, local_pool)); + SVN_ERR(stream_write_encoded(stream, value)); value = APR_ARRAY_IDX(entry_counts, i, apr_uint64_t); - SVN_ERR(svn_io_file_write_full(index_file, encoded, - encode_uint(encoded, value), - NULL, local_pool)); + SVN_ERR(stream_write_encoded(stream, value)); } - /* append page contents */ + /* append page contents and implicitly close STREAM */ SVN_ERR(svn_stream_copy3(svn_stream__from_spillbuf(buffer, local_pool), - svn_stream_from_aprfile2(index_file, TRUE, - local_pool), - NULL, NULL, local_pool)); + stream, NULL, NULL, local_pool)); svn_pool_destroy(local_pool); @@ -733,7 +948,9 @@ auto_open_l2p_index(svn_fs_fs__revision_ rev_file->file, rev_file->l2p_offset, rev_file->p2l_offset, - ffd->block_size, + L2P_STREAM_PREFIX, + (apr_size_t)ffd->block_size, + rev_file->pool, rev_file->pool)); } @@ -741,23 +958,25 @@ auto_open_l2p_index(svn_fs_fs__revision_ } /* Read the header data structure of the log-to-phys index for REVISION - * in FS and return it in *HEADER. Use REV_FILE to access on-disk data. - * Use POOL for allocations. + * in FS and return it in *HEADER, allocated in RESULT_POOL. Use REV_FILE + * to access on-disk data. Use SCRATCH_POOL for temporary allocations. */ static svn_error_t * get_l2p_header_body(l2p_header_t **header, svn_fs_fs__revision_file_t *rev_file, svn_fs_t *fs, svn_revnum_t revision, - apr_pool_t *pool) + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) { fs_fs_data_t *ffd = fs->fsap_data; apr_uint64_t value; - int i; + apr_size_t i; apr_size_t page, page_count; apr_off_t offset; - l2p_header_t *result = apr_pcalloc(pool, sizeof(*result)); + l2p_header_t *result = apr_pcalloc(result_pool, sizeof(*result)); apr_size_t page_table_index; + svn_revnum_t next_rev; pair_cache_key_t key; key.revision = rev_file->start_revision; @@ -766,29 +985,48 @@ get_l2p_header_body(l2p_header_t **heade SVN_ERR(auto_open_l2p_index(rev_file, fs, revision)); packed_stream_seek(rev_file->l2p_stream, 0); - /* read the table sizes */ + /* Read the table sizes. Check the data for plausibility and + * consistency with other bits. */ SVN_ERR(packed_stream_get(&value, rev_file->l2p_stream)); result->first_revision = (svn_revnum_t)value; + if (result->first_revision != rev_file->start_revision) + return svn_error_create(SVN_ERR_FS_INDEX_CORRUPTION, NULL, + _("Index rev / pack file revision numbers do not match")); + SVN_ERR(packed_stream_get(&value, rev_file->l2p_stream)); result->page_size = (apr_uint32_t)value; + if (!result->page_size || (result->page_size & (result->page_size - 1))) + return svn_error_create(SVN_ERR_FS_INDEX_CORRUPTION, NULL, + _("L2P index page size is not a power of two")); + SVN_ERR(packed_stream_get(&value, rev_file->l2p_stream)); result->revision_count = (int)value; + if ( result->revision_count != 1 + && result->revision_count != (apr_uint64_t)ffd->max_files_per_dir) + return svn_error_create(SVN_ERR_FS_INDEX_CORRUPTION, NULL, + _("Invalid number of revisions in L2P index")); + SVN_ERR(packed_stream_get(&value, rev_file->l2p_stream)); page_count = (apr_size_t)value; - - if (result->first_revision > revision - || result->first_revision + result->revision_count <= revision) - return svn_error_createf(SVN_ERR_FS_ITEM_INDEX_CORRUPTION, NULL, + if (page_count < result->revision_count) + return svn_error_create(SVN_ERR_FS_INDEX_CORRUPTION, NULL, + _("Fewer L2P index pages than revisions")); + if (page_count > (rev_file->p2l_offset - rev_file->l2p_offset) / 2) + return svn_error_create(SVN_ERR_FS_INDEX_CORRUPTION, NULL, + _("L2P index page count implausibly large")); + + next_rev = result->first_revision + (svn_revnum_t)result->revision_count; + if (result->first_revision > revision || next_rev <= revision) + return svn_error_createf(SVN_ERR_FS_INDEX_CORRUPTION, NULL, _("Corrupt L2P index for r%ld only covers r%ld:%ld"), - revision, result->first_revision, - result->first_revision + result->revision_count); + revision, result->first_revision, next_rev); /* allocate the page tables */ result->page_table - = apr_pcalloc(pool, page_count * sizeof(*result->page_table)); + = apr_pcalloc(result_pool, page_count * sizeof(*result->page_table)); result->page_table_index - = apr_pcalloc(pool, (result->revision_count + 1) - * sizeof(*result->page_table_index)); + = apr_pcalloc(result_pool, (result->revision_count + 1) + * sizeof(*result->page_table_index)); /* read per-revision page table sizes (i.e. number of pages per rev) */ page_table_index = 0; @@ -797,16 +1035,36 @@ get_l2p_header_body(l2p_header_t **heade for (i = 0; i < result->revision_count; ++i) { SVN_ERR(packed_stream_get(&value, rev_file->l2p_stream)); + if (value == 0) + return svn_error_create(SVN_ERR_FS_INDEX_CORRUPTION, NULL, + _("Revision with no L2P index pages")); + page_table_index += (apr_size_t)value; + if (page_table_index > page_count) + return svn_error_create(SVN_ERR_FS_INDEX_CORRUPTION, NULL, + _("L2P page table exceeded")); + result->page_table_index[i+1] = page_table_index; } + if (page_table_index != page_count) + return svn_error_create(SVN_ERR_FS_INDEX_CORRUPTION, NULL, + _("Revisions do not cover the full L2P index page table")); + /* read actual page tables */ for (page = 0; page < page_count; ++page) { SVN_ERR(packed_stream_get(&value, rev_file->l2p_stream)); + if (value == 0) + return svn_error_create(SVN_ERR_FS_INDEX_CORRUPTION, NULL, + _("Empty L2P index page")); + result->page_table[page].size = (apr_uint32_t)value; SVN_ERR(packed_stream_get(&value, rev_file->l2p_stream)); + if (value > result->page_size) + return svn_error_create(SVN_ERR_FS_INDEX_CORRUPTION, NULL, + _("Page exceeds L2P index page size")); + result->page_table[page].entry_count = (apr_uint32_t)value; } @@ -820,7 +1078,7 @@ get_l2p_header_body(l2p_header_t **heade /* return and cache the header */ *header = result; - SVN_ERR(svn_cache__set(ffd->l2p_header_cache, &key, result, pool)); + SVN_ERR(svn_cache__set(ffd->l2p_header_cache, &key, result, scratch_pool)); return SVN_NO_ERROR; } @@ -863,7 +1121,7 @@ l2p_page_info_copy(l2p_page_info_baton_t /* revision offset within the index file */ apr_size_t rel_revision = baton->revision - header->first_revision; if (rel_revision >= header->revision_count) - return svn_error_createf(SVN_ERR_FS_ITEM_INDEX_REVISION , NULL, + return svn_error_createf(SVN_ERR_FS_INDEX_REVISION , NULL, _("Revision %ld not covered by item index"), baton->revision); @@ -889,7 +1147,7 @@ l2p_page_info_copy(l2p_page_info_baton_t max_item_index = (apr_uint64_t)header->page_size * (last_entry - first_entry); if (baton->item_index >= max_item_index) - return svn_error_createf(SVN_ERR_FS_ITEM_INDEX_OVERFLOW , NULL, + return svn_error_createf(SVN_ERR_FS_INDEX_OVERFLOW , NULL, _("Item index %s exceeds l2p limit " "of %s for revision %ld"), apr_psprintf(scratch_pool, @@ -938,13 +1196,13 @@ l2p_page_info_access_func(void **out, /* Get the page info requested in *BATON from FS and set the output fields * in *BATON. Use REV_FILE for on-disk file access. - * Use POOL for allocations. + * Use SCRATCH_POOL for temporary allocations. */ static svn_error_t * get_l2p_page_info(l2p_page_info_baton_t *baton, svn_fs_fs__revision_file_t *rev_file, svn_fs_t *fs, - apr_pool_t *pool) + apr_pool_t *scratch_pool) { fs_fs_data_t *ffd = fs->fsap_data; l2p_header_t *result; @@ -958,14 +1216,15 @@ get_l2p_page_info(l2p_page_info_baton_t SVN_ERR(svn_cache__get_partial((void**)&dummy, &is_cached, ffd->l2p_header_cache, &key, l2p_page_info_access_func, baton, - pool)); + scratch_pool)); if (is_cached) return SVN_NO_ERROR; /* read from disk, cache and copy the result */ - SVN_ERR(get_l2p_header_body(&result, rev_file, fs, baton->revision, pool)); + SVN_ERR(get_l2p_header_body(&result, rev_file, fs, baton->revision, + scratch_pool, scratch_pool)); SVN_ERR(l2p_page_info_copy(baton, result, result->page_table, - result->page_table_index, pool)); + result->page_table_index, scratch_pool)); return SVN_NO_ERROR; } @@ -1028,14 +1287,14 @@ l2p_page_table_access_func(void **out, * If the data cannot be found in the cache, the result will be empty * (it never can be empty for a valid REVISION if the data is cached). * Use the info from REV_FILE to determine pack / rev file properties. - * Use POOL for temporary allocations. + * Use SCRATCH_POOL for temporary allocations. */ static svn_error_t * get_l2p_page_table(apr_array_header_t *pages, svn_fs_t *fs, svn_fs_fs__revision_file_t *rev_file, svn_revnum_t revision, - apr_pool_t *pool) + apr_pool_t *scratch_pool) { fs_fs_data_t *ffd = fs->fsap_data; svn_boolean_t is_cached = FALSE; @@ -1050,7 +1309,8 @@ get_l2p_page_table(apr_array_header_t *p baton.pages = pages; SVN_ERR(svn_cache__get_partial((void**)&pages, &is_cached, ffd->l2p_header_cache, &key, - l2p_page_table_access_func, &baton, pool)); + l2p_page_table_access_func, &baton, + scratch_pool)); return SVN_NO_ERROR; } @@ -1058,7 +1318,7 @@ get_l2p_page_table(apr_array_header_t *p /* From the log-to-phys index file starting at START_REVISION in FS, read * the mapping page identified by TABLE_ENTRY and return it in *PAGE. * Use REV_FILE to access on-disk files. - * Use POOL for allocations. + * Use RESULT_POOL for allocations. */ static svn_error_t * get_l2p_page(l2p_page_t **page, @@ -1066,10 +1326,10 @@ get_l2p_page(l2p_page_t **page, svn_fs_t *fs, svn_revnum_t start_revision, l2p_page_table_entry_t *table_entry, - apr_pool_t *pool) + apr_pool_t *result_pool) { apr_uint32_t i; - l2p_page_t *result = apr_pcalloc(pool, sizeof(*result)); + l2p_page_t *result = apr_pcalloc(result_pool, sizeof(*result)); apr_uint64_t last_value = 0; /* open index file and select page */ @@ -1078,8 +1338,8 @@ get_l2p_page(l2p_page_t **page, /* initialize the page content */ result->entry_count = table_entry->entry_count; - result->offsets = apr_pcalloc(pool, result->entry_count - * sizeof(*result->offsets)); + result->offsets = apr_pcalloc(result_pool, result->entry_count + * sizeof(*result->offsets)); /* read all page entries (offsets in rev file and container sub-items) */ for (i = 0; i < result->entry_count; ++i) @@ -1090,6 +1350,13 @@ get_l2p_page(l2p_page_t **page, result->offsets[i] = last_value - 1; } + /* After reading all page entries, the read cursor must have moved by + * TABLE_ENTRY->SIZE bytes. */ + if ( packed_stream_offset(rev_file->l2p_stream) + != table_entry->offset + table_entry->size) + return svn_error_create(SVN_ERR_FS_INDEX_CORRUPTION, NULL, + _("L2P actual page size does not match page table value.")); + *page = result; return SVN_NO_ERROR; @@ -1121,6 +1388,17 @@ prefetch_l2p_pages(svn_boolean_t *end, apr_pool_t *iterpool; svn_fs_fs__page_cache_key_t key = { 0 }; + /* Parameter check. */ + if (min_offset < 0) + min_offset = 0; + + if (max_offset <= 0) + { + /* Nothing to do */ + *end = TRUE; + return SVN_NO_ERROR; + } + /* get the page table for REVISION from cache */ *end = FALSE; SVN_ERR(get_l2p_page_table(pages, fs, rev_file, revision, scratch_pool)); @@ -1150,8 +1428,8 @@ prefetch_l2p_pages(svn_boolean_t *end, continue; /* skip pages outside the specified index file range */ - if ( entry->offset < min_offset - || entry->offset + entry->size > max_offset) + if ( entry->offset < (apr_uint64_t)min_offset + || entry->offset + entry->size > (apr_uint64_t)max_offset) { *end = TRUE; continue; @@ -1209,7 +1487,7 @@ l2p_page_get_entry(l2p_entry_baton_t *ba { /* overflow check */ if (page->entry_count <= baton->page_offset) - return svn_error_createf(SVN_ERR_FS_ITEM_INDEX_OVERFLOW , NULL, + return svn_error_createf(SVN_ERR_FS_INDEX_OVERFLOW , NULL, _("Item index %s" " too large in revision %ld"), apr_psprintf(scratch_pool, "%" APR_UINT64_T_FMT, @@ -1244,7 +1522,7 @@ l2p_entry_access_func(void **out, /* Using the log-to-phys indexes in FS, find the absolute offset in the * rev file for (REVISION, ITEM_INDEX) and return it in *OFFSET. - * Use POOL for allocations. + * Use SCRATCH_POOL for temporary allocations. */ static svn_error_t * l2p_index_lookup(apr_off_t *offset, @@ -1252,7 +1530,7 @@ l2p_index_lookup(apr_off_t *offset, svn_fs_fs__revision_file_t *rev_file, svn_revnum_t revision, apr_uint64_t item_index, - apr_pool_t *pool) + apr_pool_t *scratch_pool) { fs_fs_data_t *ffd = fs->fsap_data; l2p_page_info_baton_t info_baton; @@ -1266,7 +1544,7 @@ l2p_index_lookup(apr_off_t *offset, * access the l2p index page for (REVISION,ITEM_INDEX)*/ info_baton.revision = revision; info_baton.item_index = item_index; - SVN_ERR(get_l2p_page_info(&info_baton, rev_file, fs, pool)); + SVN_ERR(get_l2p_page_info(&info_baton, rev_file, fs, scratch_pool)); /* try to find the page in the cache and get the OFFSET from it */ page_baton.revision = revision; @@ -1280,7 +1558,8 @@ l2p_index_lookup(apr_off_t *offset, SVN_ERR(svn_cache__get_partial(&dummy, &is_cached, ffd->l2p_page_cache, &key, - l2p_entry_access_func, &page_baton, pool)); + l2p_entry_access_func, &page_baton, + scratch_pool)); if (!is_cached) { @@ -1299,18 +1578,20 @@ l2p_index_lookup(apr_off_t *offset, /* read the relevant page */ SVN_ERR(get_l2p_page(&page, rev_file, fs, info_baton.first_revision, - &info_baton.entry, pool)); + &info_baton.entry, scratch_pool)); /* cache the page and extract the result we need */ - SVN_ERR(svn_cache__set(ffd->l2p_page_cache, &key, page, pool)); - SVN_ERR(l2p_page_get_entry(&page_baton, page, page->offsets, pool)); + SVN_ERR(svn_cache__set(ffd->l2p_page_cache, &key, page, scratch_pool)); + SVN_ERR(l2p_page_get_entry(&page_baton, page, page->offsets, + scratch_pool)); if (ffd->use_block_read) { - apr_pool_t *iterpool = svn_pool_create(pool); + apr_pool_t *iterpool = svn_pool_create(scratch_pool); /* prefetch pages from following and preceding revisions */ - pages = apr_array_make(pool, 16, sizeof(l2p_page_table_entry_t)); + pages = apr_array_make(scratch_pool, 16, + sizeof(l2p_page_table_entry_t)); end = FALSE; for (prefetch_revision = revision; prefetch_revision < last_revision && !end; @@ -1352,32 +1633,32 @@ l2p_index_lookup(apr_off_t *offset, /* Using the log-to-phys proto index in transaction TXN_ID in FS, find the * absolute offset in the proto rev file for the given ITEM_INDEX and return - * it in *OFFSET. Use POOL for allocations. + * it in *OFFSET. Use SCRATCH_POOL for temporary allocations. */ static svn_error_t * l2p_proto_index_lookup(apr_off_t *offset, svn_fs_t *fs, const svn_fs_fs__id_part_t *txn_id, apr_uint64_t item_index, - apr_pool_t *pool) + apr_pool_t *scratch_pool) { svn_boolean_t eof = FALSE; apr_file_t *file = NULL; SVN_ERR(svn_io_file_open(&file, - svn_fs_fs__path_l2p_proto_index(fs, txn_id, pool), - APR_READ | APR_BUFFERED, APR_OS_DEFAULT, pool)); + svn_fs_fs__path_l2p_proto_index(fs, txn_id, + scratch_pool), + APR_READ | APR_BUFFERED, APR_OS_DEFAULT, + scratch_pool)); /* process all entries until we fail due to EOF */ *offset = -1; while (!eof) { l2p_proto_entry_t entry; - apr_size_t read = 0; /* (attempt to) read the next entry from the source */ - SVN_ERR(svn_io_file_read_full2(file, &entry, sizeof(entry), - &read, &eof, pool)); - SVN_ERR_ASSERT(eof || read == sizeof(entry)); + SVN_ERR(read_l2p_entry_from_proto_index(file, &entry, &eof, + scratch_pool)); /* handle new revision */ if (!eof && entry.item_index == item_index) @@ -1387,21 +1668,23 @@ l2p_proto_index_lookup(apr_off_t *offset } } - SVN_ERR(svn_io_file_close(file, pool)); + SVN_ERR(svn_io_file_close(file, scratch_pool)); return SVN_NO_ERROR; } /* Read the log-to-phys header info of the index covering REVISION from FS * and return it in *HEADER. REV_FILE provides the pack / rev status. - * Use POOL for allocations. + * Allocate *HEADER in RESULT_POOL, use SCRATCH_POOL for temporary + * allocations. */ static svn_error_t * get_l2p_header(l2p_header_t **header, svn_fs_fs__revision_file_t *rev_file, svn_fs_t *fs, svn_revnum_t revision, - apr_pool_t *pool) + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) { fs_fs_data_t *ffd = fs->fsap_data; svn_boolean_t is_cached = FALSE; @@ -1411,12 +1694,13 @@ get_l2p_header(l2p_header_t **header, key.revision = rev_file->start_revision; key.second = rev_file->is_packed; SVN_ERR(svn_cache__get((void**)header, &is_cached, ffd->l2p_header_cache, - &key, pool)); + &key, result_pool)); if (is_cached) return SVN_NO_ERROR; /* read from disk and cache the result */ - SVN_ERR(get_l2p_header_body(header, rev_file, fs, revision, pool)); + SVN_ERR(get_l2p_header_body(header, rev_file, fs, revision, result_pool, + scratch_pool)); return SVN_NO_ERROR; } @@ -1426,23 +1710,25 @@ svn_fs_fs__l2p_get_max_ids(apr_array_hea svn_fs_t *fs, svn_revnum_t start_rev, apr_size_t count, - apr_pool_t *pool) + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) { l2p_header_t *header = NULL; svn_revnum_t revision; svn_revnum_t last_rev = (svn_revnum_t)(start_rev + count); svn_fs_fs__revision_file_t *rev_file; - apr_pool_t *header_pool = svn_pool_create(pool); + apr_pool_t *header_pool = svn_pool_create(scratch_pool); /* read index master data structure for the index covering START_REV */ SVN_ERR(svn_fs_fs__open_pack_or_rev_file(&rev_file, fs, start_rev, header_pool, header_pool)); - SVN_ERR(get_l2p_header(&header, rev_file, fs, start_rev, header_pool)); + SVN_ERR(get_l2p_header(&header, rev_file, fs, start_rev, header_pool, + header_pool)); SVN_ERR(svn_fs_fs__close_revision_file(rev_file)); /* Determine the length of the item index list for each rev. * Read new index headers as required. */ - *max_ids = apr_array_make(pool, (int)count, sizeof(apr_uint64_t)); + *max_ids = apr_array_make(result_pool, (int)count, sizeof(apr_uint64_t)); for (revision = start_rev; revision < last_rev; ++revision) { apr_uint64_t full_page_count; @@ -1459,7 +1745,7 @@ svn_fs_fs__l2p_get_max_ids(apr_array_hea SVN_ERR(svn_fs_fs__open_pack_or_rev_file(&rev_file, fs, revision, header_pool, header_pool)); SVN_ERR(get_l2p_header(&header, rev_file, fs, revision, - header_pool)); + header_pool, header_pool)); SVN_ERR(svn_fs_fs__close_revision_file(rev_file)); } @@ -1487,17 +1773,17 @@ svn_fs_fs__item_offset(apr_off_t *absolu svn_revnum_t revision, const svn_fs_fs__id_part_t *txn_id, apr_uint64_t item_index, - apr_pool_t *pool) + apr_pool_t *scratch_pool) { svn_error_t *err = SVN_NO_ERROR; if (txn_id) { - if (svn_fs_fs__use_log_addressing(fs, txn_id->revision + 1)) + if (svn_fs_fs__use_log_addressing(fs)) { /* the txn is going to produce a rev with logical addressing. So, we need to get our info from the (proto) index file. */ SVN_ERR(l2p_proto_index_lookup(absolute_position, fs, txn_id, - item_index, pool)); + item_index, scratch_pool)); } else { @@ -1505,17 +1791,18 @@ svn_fs_fs__item_offset(apr_off_t *absolu *absolute_position = item_index; } } - else if (svn_fs_fs__use_log_addressing(fs, revision)) + else if (svn_fs_fs__use_log_addressing(fs)) { /* ordinary index lookup */ SVN_ERR(l2p_index_lookup(absolute_position, fs, rev_file, revision, - item_index, pool)); + item_index, scratch_pool)); } else if (rev_file->is_packed) { /* pack file with physical addressing */ apr_off_t rev_offset; - SVN_ERR(svn_fs_fs__get_packed_offset(&rev_offset, fs, revision, pool)); + SVN_ERR(svn_fs_fs__get_packed_offset(&rev_offset, fs, revision, + scratch_pool)); *absolute_position = rev_offset + item_index; } else @@ -1534,11 +1821,11 @@ svn_fs_fs__item_offset(apr_off_t *absolu svn_error_t * svn_fs_fs__p2l_proto_index_open(apr_file_t **proto_index, const char *file_name, - apr_pool_t *pool) + apr_pool_t *result_pool) { SVN_ERR(svn_io_file_open(proto_index, file_name, APR_READ | APR_WRITE | APR_CREATE | APR_APPEND | APR_BUFFERED, - APR_OS_DEFAULT, pool)); + APR_OS_DEFAULT, result_pool)); return SVN_NO_ERROR; } @@ -1547,13 +1834,97 @@ svn_fs_fs__p2l_proto_index_open(apr_file svn_error_t * svn_fs_fs__p2l_proto_index_add_entry(apr_file_t *proto_index, svn_fs_fs__p2l_entry_t *entry, - apr_pool_t *pool) + apr_pool_t *scratch_pool) { - apr_size_t written = sizeof(*entry); + apr_uint64_t revision; - SVN_ERR(svn_io_file_write_full(proto_index, entry, sizeof(*entry), - &written, pool)); - SVN_ERR_ASSERT(written == sizeof(*entry)); + /* Make sure all signed elements of ENTRY have non-negative values. + * + * For file offsets and sizes, this is a given as we use them to describe + * absolute positions and sizes. For revisions, SVN_INVALID_REVNUM is + * valid, hence we have to shift it by 1. + */ + SVN_ERR_ASSERT(entry->offset >= 0); + SVN_ERR_ASSERT(entry->size >= 0); + SVN_ERR_ASSERT( entry->item.revision >= 0 + || entry->item.revision == SVN_INVALID_REVNUM); + + revision = entry->item.revision == SVN_INVALID_REVNUM + ? 0 + : ((apr_uint64_t)entry->item.revision + 1); + + /* Now, all values will nicely convert to uint64. */ + /* Make sure to keep P2L_PROTO_INDEX_ENTRY_SIZE consistent with this: */ + + SVN_ERR(write_uint64_to_proto_index(proto_index, entry->offset, + scratch_pool)); + SVN_ERR(write_uint64_to_proto_index(proto_index, entry->size, + scratch_pool)); + SVN_ERR(write_uint64_to_proto_index(proto_index, entry->type, + scratch_pool)); + SVN_ERR(write_uint64_to_proto_index(proto_index, entry->fnv1_checksum, + scratch_pool)); + SVN_ERR(write_uint64_to_proto_index(proto_index, revision, + scratch_pool)); + SVN_ERR(write_uint64_to_proto_index(proto_index, entry->item.number, + scratch_pool)); + + return SVN_NO_ERROR; +} + +/* Read *ENTRY from log-to-phys PROTO_INDEX file and indicate end-of-file + * in *EOF, or error out in that case if EOF is NULL. *ENTRY is in an + * undefined state if an end-of-file occurred. + * Use SCRATCH_POOL for temporary allocations. + */ +static svn_error_t * +read_p2l_entry_from_proto_index(apr_file_t *proto_index, + svn_fs_fs__p2l_entry_t *entry, + svn_boolean_t *eof, + apr_pool_t *scratch_pool) +{ + apr_uint64_t revision; + + SVN_ERR(read_off_t_from_proto_index(proto_index, &entry->offset, + eof, scratch_pool)); + SVN_ERR(read_off_t_from_proto_index(proto_index, &entry->size, + eof, scratch_pool)); + SVN_ERR(read_uint32_from_proto_index(proto_index, &entry->type, + eof, scratch_pool)); + SVN_ERR(read_uint32_from_proto_index(proto_index, &entry->fnv1_checksum, + eof, scratch_pool)); + SVN_ERR(read_uint64_from_proto_index(proto_index, &revision, + eof, scratch_pool)); + SVN_ERR(read_uint64_from_proto_index(proto_index, &entry->item.number, + eof, scratch_pool)); + + /* Do the inverse REVSION number conversion (see + * svn_fs_fs__p2l_proto_index_add_entry), if we actually read a complete + * record. + */ + if (!eof || !*eof) + { + /* Be careful with the arithmetics here (overflows and wrap-around): */ + if (revision > 0 && revision - 1 > LONG_MAX) + return svn_error_createf(SVN_ERR_FS_INDEX_OVERFLOW, NULL, + _("Revision 0x%s too large, max = 0x%s"), + apr_psprintf(scratch_pool, + "%" APR_UINT64_T_HEX_FMT, + revision), + apr_psprintf(scratch_pool, + "%" APR_UINT64_T_HEX_FMT, + (apr_uint64_t)LONG_MAX)); + + /* Shortening conversion from unsigned to signed int is well-defined + * and not lossy in C because the value can be represented in the + * target type. Also, cast to 'long' instead of 'svn_revnum_t' here + * to provoke a compiler warning if those types should differ and we + * would need to change the overflow checking logic. + */ + entry->item.revision = revision == 0 + ? SVN_INVALID_REVNUM + : (long)(revision - 1); + } return SVN_NO_ERROR; } @@ -1561,12 +1932,12 @@ svn_fs_fs__p2l_proto_index_add_entry(apr svn_error_t * svn_fs_fs__p2l_proto_index_next_offset(apr_off_t *next_offset, apr_file_t *proto_index, - apr_pool_t *pool) + apr_pool_t *scratch_pool) { apr_off_t offset = 0; /* Empty index file? */ - SVN_ERR(svn_io_file_seek(proto_index, APR_END, &offset, pool)); + SVN_ERR(svn_io_file_seek(proto_index, APR_END, &offset, scratch_pool)); if (offset == 0) { *next_offset = 0; @@ -1575,11 +1946,11 @@ svn_fs_fs__p2l_proto_index_next_offset(a { /* At least one entry. Read last entry. */ svn_fs_fs__p2l_entry_t entry; - offset -= sizeof(entry); + offset -= P2L_PROTO_INDEX_ENTRY_SIZE; - SVN_ERR(svn_io_file_seek(proto_index, APR_SET, &offset, pool)); - SVN_ERR(svn_io_file_read_full2(proto_index, &entry, sizeof(entry), - NULL, NULL, pool)); + SVN_ERR(svn_io_file_seek(proto_index, APR_SET, &offset, scratch_pool)); + SVN_ERR(read_p2l_entry_from_proto_index(proto_index, &entry, + NULL, scratch_pool)); /* Return next offset. */ *next_offset = entry.offset + entry.size; @@ -1589,15 +1960,18 @@ svn_fs_fs__p2l_proto_index_next_offset(a } svn_error_t * -svn_fs_fs__p2l_index_append(svn_fs_t *fs, +svn_fs_fs__p2l_index_append(svn_checksum_t **checksum, + svn_fs_t *fs, apr_file_t *index_file, const char *proto_file_name, svn_revnum_t revision, - apr_pool_t *pool) + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) { fs_fs_data_t *ffd = fs->fsap_data; apr_uint64_t page_size = ffd->p2l_page_size; apr_file_t *proto_index = NULL; + svn_stream_t *stream; int i; svn_boolean_t eof = FALSE; unsigned char encoded[ENCODED_INT_LENGTH]; @@ -1612,7 +1986,7 @@ svn_fs_fs__p2l_index_append(svn_fs_t *fs /* temporary data structures that collect the data which will be moved to the target file in a second step */ - apr_pool_t *local_pool = svn_pool_create(pool); + apr_pool_t *local_pool = svn_pool_create(scratch_pool); apr_array_header_t *table_sizes = apr_array_make(local_pool, 16, sizeof(apr_uint64_t)); @@ -1621,29 +1995,27 @@ svn_fs_fs__p2l_index_append(svn_fs_t *fs = svn_spillbuf__create(0x10000, 0x1000000, local_pool); /* for loop temps ... */ - apr_pool_t *iter_pool = svn_pool_create(pool); + apr_pool_t *iterpool = svn_pool_create(scratch_pool); /* start at the beginning of the source file */ SVN_ERR(svn_io_file_open(&proto_index, proto_file_name, APR_READ | APR_CREATE | APR_BUFFERED, - APR_OS_DEFAULT, pool)); + APR_OS_DEFAULT, scratch_pool)); /* process all entries until we fail due to EOF */ while (!eof) { svn_fs_fs__p2l_entry_t entry; - apr_size_t read = 0; apr_uint64_t entry_end; svn_boolean_t new_page = svn_spillbuf__get_size(buffer) == 0; apr_uint64_t compound; apr_int64_t rev_diff, compound_diff; - svn_pool_clear(iter_pool); + svn_pool_clear(iterpool); /* (attempt to) read the next entry from the source */ - SVN_ERR(svn_io_file_read_full2(proto_index, &entry, sizeof(entry), - &read, &eof, iter_pool)); - SVN_ERR_ASSERT(eof || read == sizeof(entry)); + SVN_ERR(read_p2l_entry_from_proto_index(proto_index, &entry, + &eof, iterpool)); /* "unused" (and usually non-existent) section to cover the offsets at the end the of the last page. */ @@ -1684,7 +2056,7 @@ svn_fs_fs__p2l_index_append(svn_fs_t *fs { SVN_ERR(svn_spillbuf__write(buffer, (const char *)encoded, encode_uint(encoded, entry.offset), - iter_pool)); + iterpool)); last_revision = revision; last_compound = 0; } @@ -1692,7 +2064,7 @@ svn_fs_fs__p2l_index_append(svn_fs_t *fs /* write simple item entry */ SVN_ERR(svn_spillbuf__write(buffer, (const char *)encoded, encode_uint(encoded, entry.size), - iter_pool)); + iterpool)); rev_diff = entry.item.revision - last_revision; last_revision = entry.item.revision; @@ -1703,13 +2075,13 @@ svn_fs_fs__p2l_index_append(svn_fs_t *fs SVN_ERR(svn_spillbuf__write(buffer, (const char *)encoded, encode_int(encoded, compound_diff), - iter_pool)); + iterpool)); SVN_ERR(svn_spillbuf__write(buffer, (const char *)encoded, encode_int(encoded, rev_diff), - iter_pool)); + iterpool)); SVN_ERR(svn_spillbuf__write(buffer, (const char *)encoded, encode_uint(encoded, entry.fnv1_checksum), - iter_pool)); + iterpool)); last_entry_end = entry_end; } @@ -1721,36 +2093,31 @@ svn_fs_fs__p2l_index_append(svn_fs_t *fs APR_ARRAY_PUSH(table_sizes, apr_uint64_t) = svn_spillbuf__get_size(buffer) - last_buffer_size; + /* Open target stream. */ + stream = svn_stream_checksummed2(svn_stream_from_aprfile2(index_file, TRUE, + local_pool), + NULL, checksum, svn_checksum_md5, FALSE, + result_pool); + /* write the start revision, file size and page size */ - SVN_ERR(svn_io_file_write_full(index_file, encoded, - encode_uint(encoded, revision), - NULL, local_pool)); - SVN_ERR(svn_io_file_write_full(index_file, encoded, - encode_uint(encoded, file_size), - NULL, local_pool)); - SVN_ERR(svn_io_file_write_full(index_file, encoded, - encode_uint(encoded, page_size), - NULL, local_pool)); + SVN_ERR(svn_stream_puts(stream, P2L_STREAM_PREFIX)); + SVN_ERR(stream_write_encoded(stream, revision)); + SVN_ERR(stream_write_encoded(stream, file_size)); + SVN_ERR(stream_write_encoded(stream, page_size)); /* write the page table (actually, the sizes of each page description) */ - SVN_ERR(svn_io_file_write_full(index_file, encoded, - encode_uint(encoded, table_sizes->nelts), - NULL, local_pool)); + SVN_ERR(stream_write_encoded(stream, table_sizes->nelts)); for (i = 0; i < table_sizes->nelts; ++i) { apr_uint64_t value = APR_ARRAY_IDX(table_sizes, i, apr_uint64_t); - SVN_ERR(svn_io_file_write_full(index_file, encoded, - encode_uint(encoded, value), - NULL, local_pool)); + SVN_ERR(stream_write_encoded(stream, value)); } - /* append page contents */ + /* append page contents and implicitly close STREAM */ SVN_ERR(svn_stream_copy3(svn_stream__from_spillbuf(buffer, local_pool), - svn_stream_from_aprfile2(index_file, TRUE, - local_pool), - NULL, NULL, local_pool)); + stream, NULL, NULL, local_pool)); - svn_pool_destroy(iter_pool); + svn_pool_destroy(iterpool); svn_pool_destroy(local_pool); return SVN_NO_ERROR; @@ -1773,7 +2140,10 @@ auto_open_p2l_index(svn_fs_fs__revision_ rev_file->file, rev_file->p2l_offset, rev_file->footer_offset, - ffd->block_size, rev_file->pool)); + P2L_STREAM_PREFIX, + (apr_size_t)ffd->block_size, + rev_file->pool, + rev_file->pool)); } return SVN_NO_ERROR; @@ -1781,15 +2151,16 @@ auto_open_p2l_index(svn_fs_fs__revision_ /* Read the header data structure of the phys-to-log index for REVISION in - * FS and return it in *HEADER. Use REV_FILE to access on-disk data. - * Use POOL for allocations. + * FS and return it in *HEADER, allocated in RESULT_POOL. Use REV_FILE to + * access on-disk data. Use SCRATCH_POOL for temporary allocations. */ static svn_error_t * get_p2l_header(p2l_header_t **header, svn_fs_fs__revision_file_t *rev_file, svn_fs_t *fs, svn_revnum_t revision, - apr_pool_t *pool) + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) { fs_fs_data_t *ffd = fs->fsap_data; apr_uint64_t value; @@ -1804,31 +2175,47 @@ get_p2l_header(p2l_header_t **header, key.second = rev_file->is_packed; SVN_ERR(svn_cache__get((void**)header, &is_cached, ffd->p2l_header_cache, - &key, pool)); + &key, result_pool)); if (is_cached) return SVN_NO_ERROR; /* not found -> must read it from disk. * Open index file or position read pointer to the begin of the file */ if (rev_file->p2l_stream == NULL) - SVN_ERR(auto_open_p2l_index(rev_file, fs, key.revision)); + SVN_ERR(auto_open_p2l_index(rev_file, fs, rev_file->start_revision)); else packed_stream_seek(rev_file->p2l_stream, 0); /* allocate result data structure */ - result = apr_pcalloc(pool, sizeof(*result)); + result = apr_pcalloc(result_pool, sizeof(*result)); - /* read table sizes and allocate page array */ + /* Read table sizes, check them for plausibility and allocate page array. */ SVN_ERR(packed_stream_get(&value, rev_file->p2l_stream)); result->first_revision = (svn_revnum_t)value; + if (result->first_revision != rev_file->start_revision) + return svn_error_create(SVN_ERR_FS_INDEX_CORRUPTION, NULL, + _("Index rev / pack file revision numbers do not match")); + SVN_ERR(packed_stream_get(&value, rev_file->p2l_stream)); result->file_size = value; + if (result->file_size != (apr_uint64_t)rev_file->l2p_offset) + return svn_error_create(SVN_ERR_FS_INDEX_CORRUPTION, NULL, + _("Index offset and rev / pack file size do not match")); + SVN_ERR(packed_stream_get(&value, rev_file->p2l_stream)); result->page_size = value; + if (!result->page_size || (result->page_size & (result->page_size - 1))) + return svn_error_create(SVN_ERR_FS_INDEX_CORRUPTION, NULL, + _("P2L index page size is not a power of two")); + SVN_ERR(packed_stream_get(&value, rev_file->p2l_stream)); result->page_count = (apr_size_t)value; + if (result->page_count != (result->file_size - 1) / result->page_size + 1) + return svn_error_create(SVN_ERR_FS_INDEX_CORRUPTION, NULL, + _("P2L page count does not match rev / pack file size")); + result->offsets - = apr_pcalloc(pool, (result->page_count + 1) * sizeof(*result->offsets)); + = apr_pcalloc(result_pool, (result->page_count + 1) * sizeof(*result->offsets)); /* read page sizes and derive page description offsets from them */ result->offsets[0] = 0; @@ -1844,7 +2231,7 @@ get_p2l_header(p2l_header_t **header, result->offsets[i] += offset; /* cache the header data */ - SVN_ERR(svn_cache__set(ffd->p2l_header_cache, &key, result, pool)); + SVN_ERR(svn_cache__set(ffd->p2l_header_cache, &key, result, scratch_pool)); /* return the result */ *header = result; @@ -1877,7 +2264,7 @@ typedef struct p2l_page_info_baton_t /* offset within the p2l index file describing the following page */ apr_off_t next_offset; - /* PAGE_NO * PAGE_SIZE (is <= OFFSET) */ + /* PAGE_NO * PAGE_SIZE (if <= OFFSET) */ apr_off_t page_start; /* total number of pages indexed */ @@ -1900,13 +2287,15 @@ p2l_page_info_copy(p2l_page_info_baton_t */ if (baton->offset / header->page_size < header->page_count) { - baton->page_no = baton->offset / header->page_size; + /* This cast is safe because the value is < header->page_count. */ + baton->page_no = (apr_size_t)(baton->offset / header->page_size); baton->start_offset = offsets[baton->page_no]; baton->next_offset = offsets[baton->page_no + 1]; baton->page_size = header->page_size; } else { + /* Beyond the last page. */ baton->page_no = header->page_count; baton->start_offset = offsets[baton->page_no]; baton->next_offset = offsets[baton->page_no]; @@ -1941,14 +2330,14 @@ p2l_page_info_func(void **out, /* Read the header data structure of the phys-to-log index for revision * BATON->REVISION in FS. Return in *BATON all info relevant to read the - * index page for the rev / pack file offset BATON->OFFSET. - * Use REV_FILE to access on-disk data. Use POOL for allocations. + * index page for the rev / pack file offset BATON->OFFSET. Use REV_FILE + * to access on-disk data. Use SCRATCH_POOL for temporary allocations. */ static svn_error_t * get_p2l_page_info(p2l_page_info_baton_t *baton, svn_fs_fs__revision_file_t *rev_file, svn_fs_t *fs, - apr_pool_t *pool) + apr_pool_t *scratch_pool) { fs_fs_data_t *ffd = fs->fsap_data; p2l_header_t *header; @@ -1961,11 +2350,13 @@ get_p2l_page_info(p2l_page_info_baton_t key.second = rev_file->is_packed; SVN_ERR(svn_cache__get_partial(&dummy, &is_cached, ffd->p2l_header_cache, - &key, p2l_page_info_func, baton, pool)); + &key, p2l_page_info_func, baton, + scratch_pool)); if (is_cached) return SVN_NO_ERROR; - SVN_ERR(get_p2l_header(&header, rev_file, fs, baton->revision, pool)); + SVN_ERR(get_p2l_header(&header, rev_file, fs, baton->revision, + scratch_pool, scratch_pool)); /* copy the requested info into *BATON */ p2l_page_info_copy(baton, header, header->offsets); @@ -1975,15 +2366,14 @@ get_p2l_page_info(p2l_page_info_baton_t /* Read a mapping entry from the phys-to-log index STREAM and append it to * RESULT. *ITEM_INDEX contains the phys offset for the entry and will - * be moved forward by the size of entry. Use POOL for allocations. + * be moved forward by the size of entry. */ static svn_error_t * read_entry(svn_fs_fs__packed_number_stream_t *stream, apr_off_t *item_offset, svn_revnum_t *last_revision, apr_uint64_t *last_compound, - apr_array_header_t *result, - apr_pool_t *pool) + apr_array_header_t *result) { apr_uint64_t value; @@ -1995,10 +2385,19 @@ read_entry(svn_fs_fs__packed_number_stre SVN_ERR(packed_stream_get(&value, stream)); *last_compound += decode_int(value); - - entry.type = (int)(*last_compound & 7); + + entry.type = *last_compound & 7; entry.item.number = *last_compound / 8; + /* Verify item type. */ + if (entry.type > SVN_FS_FS__ITEM_TYPE_CHANGES) + return svn_error_create(SVN_ERR_FS_INDEX_CORRUPTION, NULL, + _("Invalid item type in P2L index")); + if ( entry.type == SVN_FS_FS__ITEM_TYPE_CHANGES + && entry.item.number != SVN_FS_FS__ITEM_INDEX_CHANGES) + return svn_error_create(SVN_ERR_FS_INDEX_CORRUPTION, NULL, + _("Changed path list must have item number 1")); + SVN_ERR(packed_stream_get(&value, stream)); *last_revision += (svn_revnum_t)decode_int(value); entry.item.revision = *last_revision; @@ -2006,6 +2405,22 @@ read_entry(svn_fs_fs__packed_number_stre SVN_ERR(packed_stream_get(&value, stream)); entry.fnv1_checksum = (apr_uint32_t)value; + /* Truncating the checksum to 32 bits may have hidden random data in the + * unused extra bits of the on-disk representation (7/8 bit representation + * uses 5 bytes on disk for the 32 bit value, leaving 3 bits unused). */ + if (value > APR_UINT32_MAX) + return svn_error_create(SVN_ERR_FS_INDEX_CORRUPTION, NULL, + _("Invalid FNV1 checksum in P2L index")); + + /* Some of the index data for empty rev / pack file sections will not be + * used during normal operation. Thus, we have strict rules for the + * contents of those unused fields. */ + if (entry.type == SVN_FS_FS__ITEM_TYPE_UNUSED) + if ( entry.item.number != SVN_FS_FS__ITEM_INDEX_UNUSED + || entry.fnv1_checksum != 0) + return svn_error_create(SVN_ERR_FS_INDEX_CORRUPTION, NULL, + _("Empty regions must have item number 0 and checksum 0")); + APR_ARRAY_PUSH(result, svn_fs_fs__p2l_entry_t) = entry; *item_offset += entry.size; @@ -2015,9 +2430,9 @@ read_entry(svn_fs_fs__packed_number_stre /* Read the phys-to-log mappings for the cluster beginning at rev file * offset PAGE_START from the index for START_REVISION in FS. The data * can be found in the index page beginning at START_OFFSET with the next - * page beginning at NEXT_OFFSET. Return the relevant index entries in - * *ENTRIES. Use REV_FILE to access on-disk data. - * Use POOL for other allocations. + * page beginning at NEXT_OFFSET. PAGE_SIZE is the L2P index page size. + * Return the relevant index entries in *ENTRIES. Use REV_FILE to access + * on-disk data. Allocate *ENTRIES in RESULT_POOL. */ static svn_error_t * get_p2l_page(apr_array_header_t **entries, @@ -2028,11 +2443,11 @@ get_p2l_page(apr_array_header_t **entrie apr_off_t next_offset, apr_off_t page_start, apr_uint64_t page_size, - apr_pool_t *pool) + apr_pool_t *result_pool) { apr_uint64_t value; apr_array_header_t *result - = apr_array_make(pool, 16, sizeof(svn_fs_fs__p2l_entry_t)); + = apr_array_make(result_pool, 16, sizeof(svn_fs_fs__p2l_entry_t)); apr_off_t item_offset; apr_off_t offset; svn_revnum_t last_revision; @@ -2050,24 +2465,43 @@ get_p2l_page(apr_array_header_t **entrie /* read all entries of this page */ last_revision = start_revision; last_compound = 0; - do + + /* Special case: empty pages. */ + if (start_offset == next_offset) { - SVN_ERR(read_entry(rev_file->p2l_stream, &item_offset, &last_revision, - &last_compound, result, pool)); - offset = packed_stream_offset(rev_file->p2l_stream); + /* Empty page. This only happens if the first entry of the next page + * also covers this page (and possibly more) completely. */ + SVN_ERR(read_entry(rev_file->p2l_stream, &item_offset, + &last_revision, &last_compound, result)); } - while (offset < next_offset); - - /* if we haven't covered the cluster end yet, we must read the first - * entry of the next page */ - if (item_offset < page_start + page_size) + else { - SVN_ERR(packed_stream_get(&value, rev_file->p2l_stream)); - item_offset = (apr_off_t)value; - last_revision = start_revision; - last_compound = 0; - SVN_ERR(read_entry(rev_file->p2l_stream, &item_offset, &last_revision, - &last_compound, result, pool)); + /* Read non-empty page. */ + do + { + SVN_ERR(read_entry(rev_file->p2l_stream, &item_offset, + &last_revision, &last_compound, result)); + offset = packed_stream_offset(rev_file->p2l_stream); + } + while (offset < next_offset); + + /* We should now be exactly at the next offset, i.e. the numbers in + * the stream cannot overlap into the next page description. */ + if (offset != next_offset) + return svn_error_create(SVN_ERR_FS_INDEX_CORRUPTION, NULL, + _("P2L page description overlaps with next page description")); + + /* if we haven't covered the cluster end yet, we must read the first + * entry of the next page */ + if (item_offset < page_start + page_size) + { + SVN_ERR(packed_stream_get(&value, rev_file->p2l_stream)); + item_offset = (apr_off_t)value; + last_revision = start_revision; + last_compound = 0; + SVN_ERR(read_entry(rev_file->p2l_stream, &item_offset, + &last_revision, &last_compound, result)); + } } *entries = result; @@ -2148,7 +2582,8 @@ prefetch_p2l_page(svn_boolean_t *end, /* Lookup & construct the baton and key information that we will need for * a P2L page cache lookup. We want the page covering OFFSET in the rev / * pack file containing REVSION in FS. Return the results in *PAGE_INFO_P - * and *KEY_P. Read data through REV_FILE. Use POOL for allocations. + * and *KEY_P. Read data through REV_FILE. Use SCRATCH_POOL for temporary + * allocations. */ static svn_error_t * get_p2l_keys(p2l_page_info_baton_t *page_info_p, @@ -2157,7 +2592,7 @@ get_p2l_keys(p2l_page_info_baton_t *page svn_fs_t *fs, svn_revnum_t revision, apr_off_t offset, - apr_pool_t *pool) + apr_pool_t *scratch_pool) { p2l_page_info_baton_t page_info; @@ -2165,13 +2600,13 @@ get_p2l_keys(p2l_page_info_baton_t *page * contents at pack / rev file position OFFSET. */ page_info.offset = offset; page_info.revision = revision; - SVN_ERR(get_p2l_page_info(&page_info, rev_file, fs, pool)); + SVN_ERR(get_p2l_page_info(&page_info, rev_file, fs, scratch_pool)); /* if the offset refers to a non-existent page, bail out */ if (page_info.page_count <= page_info.page_no) - return svn_error_createf(SVN_ERR_FS_ITEM_INDEX_OVERFLOW , NULL, + return svn_error_createf(SVN_ERR_FS_INDEX_OVERFLOW , NULL, _("Offset %s too large in revision %ld"), - apr_off_t_toa(pool, offset), revision); + apr_off_t_toa(scratch_pool, offset), revision); /* return results */ if (page_info_p) @@ -2284,7 +2719,7 @@ p2l_index_lookup(apr_array_header_t *ent svn_revnum_t revision, apr_off_t block_start, apr_off_t block_end, - apr_pool_t *pool) + apr_pool_t *scratch_pool) { fs_fs_data_t *ffd = fs->fsap_data; svn_fs_fs__page_cache_key_t key; @@ -2302,15 +2737,15 @@ p2l_index_lookup(apr_array_header_t *ent /* look for the fist page of the range in our cache */ SVN_ERR(get_p2l_keys(&page_info, &key, rev_file, fs, revision, block_start, - pool)); + scratch_pool)); SVN_ERR(svn_cache__get_partial((void**)&local_result, &is_cached, ffd->p2l_page_cache, &key, p2l_entries_func, - &block, pool)); + &block, scratch_pool)); if (!is_cached) { svn_boolean_t end; - apr_pool_t *iterpool = svn_pool_create(pool); + apr_pool_t *iterpool = svn_pool_create(scratch_pool); apr_off_t original_page_start = page_info.page_start; int leaking_bucket = 4; p2l_page_info_baton_t prefetch_info = page_info; @@ -2350,6 +2785,21 @@ p2l_index_lookup(apr_array_header_t *ent page_info.page_start, page_info.page_size, iterpool)); + /* The last cache entry must not end beyond the range covered by + * this index. The same applies for any subset of entries. */ + if (page_entries->nelts) + { + const svn_fs_fs__p2l_entry_t *entry + = &APR_ARRAY_IDX(page_entries, page_entries->nelts - 1, + svn_fs_fs__p2l_entry_t); + if ( entry->offset + entry->size + > page_info.page_size * page_info.page_count) + return svn_error_createf(SVN_ERR_FS_INDEX_OVERFLOW , NULL, + _("Last P2L index entry extends beyond " + "the last page in revision %ld."), + revision); + } + SVN_ERR(svn_cache__set(ffd->p2l_page_cache, &key, page_entries, iterpool)); @@ -2424,13 +2874,14 @@ svn_fs_fs__p2l_index_lookup(apr_array_he svn_revnum_t revision, apr_off_t block_start, apr_off_t block_size, - apr_pool_t *pool) + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) { apr_off_t block_end = block_start + block_size; /* the receiving container */ int last_count = 0; - apr_array_header_t *result = apr_array_make(pool, 16, + apr_array_header_t *result = apr_array_make(result_pool, 16, sizeof(svn_fs_fs__p2l_entry_t)); /* Fetch entries page-by-page. Since the p2l index is supposed to cover @@ -2440,7 +2891,7 @@ svn_fs_fs__p2l_index_lookup(apr_array_he { svn_fs_fs__p2l_entry_t *entry; SVN_ERR(p2l_index_lookup(result, rev_file, fs, revision, block_start, - block_end, pool)); + block_end, scratch_pool)); SVN_ERR_ASSERT(result->nelts > 0); /* continue directly behind last item */ @@ -2478,17 +2929,20 @@ compare_p2l_entry_offsets(const void *lh } /* Cached data extraction utility. DATA is a P2L index page, e.g. an APR - * array of svn_fs_fs__p2l_entry_t elements. Return the entry for the item - * starting at OFFSET or NULL if that's not an the start offset of any item. + * array of svn_fs_fs__p2l_entry_t elements. Return the entry for the item, + * allocated in RESULT_POOL, starting at OFFSET or NULL if that's not an + * the start offset of any item. Use SCRATCH_POOL for temporary allocations. */ static svn_fs_fs__p2l_entry_t * get_p2l_entry_from_cached_page(const void *data, apr_uint64_t offset, - apr_pool_t *pool) + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) { /* resolve all pointer values of in-cache data */ const apr_array_header_t *page = data; - apr_array_header_t *entries = apr_pmemdup(pool, page, sizeof(*page)); + apr_array_header_t *entries = apr_pmemdup(scratch_pool, page, + sizeof(*page)); svn_fs_fs__p2l_entry_t *entry; entries->elts = (char *)svn_temp_deserializer__ptr(page, @@ -2499,7 +2953,7 @@ get_p2l_entry_from_cached_page(const voi (int (*)(const void *, const void *))compare_p2l_entry_offsets); /* return it, if it is a perfect match */ - return entry ? apr_pmemdup(pool, entry, sizeof(*entry)) : NULL; + return entry ? apr_pmemdup(result_pool, entry, sizeof(*entry)) : NULL; } /* Implements svn_cache__partial_getter_func_t for P2L index pages, copying @@ -2514,7 +2968,8 @@ p2l_entry_lookup_func(void **out, apr_pool_t *result_pool) { svn_fs_fs__p2l_entry_t *entry - = get_p2l_entry_from_cached_page(data, *(apr_off_t *)baton, result_pool); + = get_p2l_entry_from_cached_page(data, *(apr_off_t *)baton, result_pool, + result_pool); *out = entry && entry->offset == *(apr_off_t *)baton ? apr_pmemdup(result_pool, entry, sizeof(*entry)) @@ -2529,7 +2984,8 @@ svn_fs_fs__p2l_entry_lookup(svn_fs_fs__p svn_fs_fs__revision_file_t *rev_file, svn_revnum_t revision, apr_off_t offset, - apr_pool_t *pool) + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) { fs_fs_data_t *ffd = fs->fsap_data; svn_fs_fs__page_cache_key_t key = { 0 }; @@ -2540,17 +2996,19 @@ svn_fs_fs__p2l_entry_lookup(svn_fs_fs__p /* look for this info in our cache */ SVN_ERR(get_p2l_keys(&page_info, &key, rev_file, fs, revision, offset, - pool)); + scratch_pool)); SVN_ERR(svn_cache__get_partial((void**)entry_p, &is_cached, ffd->p2l_page_cache, &key, - p2l_entry_lookup_func, &offset, pool)); + p2l_entry_lookup_func, &offset, + result_pool)); if (!is_cached) { /* do a standard index lookup. This is will automatically prefetch * data to speed up future lookups. */ - apr_array_header_t *entries = apr_array_make(pool, 1, sizeof(**entry_p)); + apr_array_header_t *entries = apr_array_make(result_pool, 1, + sizeof(**entry_p)); SVN_ERR(p2l_index_lookup(entries, rev_file, fs, revision, offset, - offset + 1, pool)); + offset + 1, scratch_pool)); /* Find the entry that we want. */ *entry_p = svn_sort__array_lookup(entries, &offset, NULL, @@ -2584,7 +3042,7 @@ p2l_get_max_offset(apr_off_t *offset, svn_fs_t *fs, svn_fs_fs__revision_file_t *rev_file, svn_revnum_t revision, - apr_pool_t *pool) + apr_pool_t *scratch_pool) { fs_fs_data_t *ffd = fs->fsap_data; p2l_header_t *header; @@ -2598,14 +3056,16 @@ p2l_get_max_offset(apr_off_t *offset, SVN_ERR(svn_cache__get_partial((void **)&offset_p, &is_cached, ffd->p2l_header_cache, &key, - p2l_get_max_offset_func, NULL, pool)); + p2l_get_max_offset_func, NULL, + scratch_pool)); if (is_cached) { *offset = *offset_p; return SVN_NO_ERROR; } - SVN_ERR(get_p2l_header(&header, rev_file, fs, revision, pool)); + SVN_ERR(get_p2l_header(&header, rev_file, fs, revision, scratch_pool, + scratch_pool)); *offset = header->file_size; return SVN_NO_ERROR; @@ -2616,24 +3076,24 @@ svn_fs_fs__p2l_get_max_offset(apr_off_t svn_fs_t *fs, svn_fs_fs__revision_file_t *rev_file, svn_revnum_t revision, - apr_pool_t *pool) + apr_pool_t *scratch_pool) { return svn_error_trace(p2l_get_max_offset(offset, fs, rev_file, revision, - pool)); + scratch_pool)); } /* Calculate the FNV1 checksum over the offset range in REV_FILE, covered by - * ENTRY. Store the result in ENTRY->FNV1_CHECKSUM. Use POOL for temporary - * allocations. */ + * ENTRY. Store the result in ENTRY->FNV1_CHECKSUM. Use SCRATCH_POOL for + * temporary allocations. */ static svn_error_t * calc_fnv1(svn_fs_fs__p2l_entry_t *entry, svn_fs_fs__revision_file_t *rev_file, - apr_pool_t *pool) + apr_pool_t *scratch_pool) { unsigned char buffer[4096]; svn_checksum_t *checksum; svn_checksum_ctx_t *context - = svn_checksum_ctx_create(svn_checksum_fnv1a_32x4, pool); + = svn_checksum_ctx_create(svn_checksum_fnv1a_32x4, scratch_pool); apr_off_t size = entry->size; /* Special rules apply to unused sections / items. The data must be a @@ -2646,20 +3106,21 @@ calc_fnv1(svn_fs_fs__p2l_entry_t *entry, } /* Read the block and feed it to the checksum calculator. */ - SVN_ERR(svn_io_file_seek(rev_file->file, APR_SET, &entry->offset, pool)); + SVN_ERR(svn_io_file_seek(rev_file->file, APR_SET, &entry->offset, + scratch_pool)); while (size > 0) { apr_size_t to_read = size > sizeof(buffer) ? sizeof(buffer) : (apr_size_t)size; SVN_ERR(svn_io_file_read_full2(rev_file->file, buffer, to_read, NULL, - NULL, pool)); + NULL, scratch_pool)); SVN_ERR(svn_checksum_update(context, buffer, to_read)); size -= to_read; } /* Store final checksum in ENTRY. */ - SVN_ERR(svn_checksum_final(&checksum, context, pool)); + SVN_ERR(svn_checksum_final(&checksum, context, scratch_pool)); entry->fnv1_checksum = ntohl(*(const apr_uint32_t *)checksum->digest); return SVN_NO_ERROR;
