Modified: subversion/branches/authzperf/subversion/libsvn_fs_fs/verify.c URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_fs/verify.c?rev=1649205&r1=1649204&r2=1649205&view=diff ============================================================================== --- subversion/branches/authzperf/subversion/libsvn_fs_fs/verify.c (original) +++ subversion/branches/authzperf/subversion/libsvn_fs_fs/verify.c Sat Jan 3 14:00:41 2015 @@ -22,6 +22,7 @@ #include "svn_sorts.h" #include "svn_checksum.h" +#include "svn_time.h" #include "private/svn_subr_private.h" #include "verify.h" @@ -159,6 +160,93 @@ verify_rep_cache(svn_fs_t *fs, return SVN_NO_ERROR; } +/* Verify that the MD5 checksum of the data between offsets START and END + * in FILE matches the EXPECTED checksum. If there is a mismatch use the + * indedx NAME in the error message. Supports cancellation with CANCEL_FUNC + * and CANCEL_BATON. SCRATCH_POOL is for temporary allocations. */ +static svn_error_t * +verify_index_checksum(apr_file_t *file, + const char *name, + apr_off_t start, + apr_off_t end, + svn_checksum_t *expected, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *scratch_pool) +{ + unsigned char buffer[SVN__STREAM_CHUNK_SIZE]; + apr_off_t size = end - start; + svn_checksum_t *actual; + svn_checksum_ctx_t *context + = svn_checksum_ctx_create(svn_checksum_md5, scratch_pool); + + /* Calculate the index checksum. */ + SVN_ERR(svn_io_file_seek(file, APR_SET, &start, scratch_pool)); + while (size > 0) + { + apr_size_t to_read = size > sizeof(buffer) + ? sizeof(buffer) + : (apr_size_t)size; + SVN_ERR(svn_io_file_read_full2(file, buffer, to_read, NULL, NULL, + scratch_pool)); + SVN_ERR(svn_checksum_update(context, buffer, to_read)); + size -= to_read; + + if (cancel_func) + SVN_ERR(cancel_func(cancel_baton)); + } + + SVN_ERR(svn_checksum_final(&actual, context, scratch_pool)); + + /* Verify that it matches the expected checksum. */ + if (!svn_checksum_match(expected, actual)) + { + const char *file_name; + + SVN_ERR(svn_io_file_name_get(&file_name, file, scratch_pool)); + SVN_ERR(svn_checksum_mismatch_err(expected, actual, scratch_pool, + _("%s checksum mismatch in file %s"), + name, file_name)); + } + + return SVN_NO_ERROR; +} + +/* Verify the MD5 checksums of the index data in the rev / pack file + * containing revision START in FS. If given, invoke CANCEL_FUNC with + * CANCEL_BATON at regular intervals. Use SCRATCH_POOL for temporary + * allocations. + */ +static svn_error_t * +verify_index_checksums(svn_fs_t *fs, + svn_revnum_t start, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *scratch_pool) +{ + svn_fs_fs__revision_file_t *rev_file; + + /* Open the rev / pack file and read the footer */ + SVN_ERR(svn_fs_fs__open_pack_or_rev_file(&rev_file, fs, start, + scratch_pool, scratch_pool)); + SVN_ERR(svn_fs_fs__auto_read_footer(rev_file)); + + /* Verify the index contents against the checksum from the footer. */ + SVN_ERR(verify_index_checksum(rev_file->file, "L2P index", + rev_file->l2p_offset, rev_file->p2l_offset, + rev_file->l2p_checksum, + cancel_func, cancel_baton, scratch_pool)); + SVN_ERR(verify_index_checksum(rev_file->file, "P2L index", + rev_file->p2l_offset, rev_file->footer_offset, + rev_file->p2l_checksum, + cancel_func, cancel_baton, scratch_pool)); + + /* Done. */ + SVN_ERR(svn_fs_fs__close_revision_file(rev_file)); + + return SVN_NO_ERROR; +} + /* Verify that for all log-to-phys index entries for revisions START to * START + COUNT-1 in FS there is a consistent entry in the phys-to-log * index. If given, invoke CANCEL_FUNC with CANCEL_BATON at regular @@ -182,7 +270,8 @@ compare_l2p_to_p2l_index(svn_fs_t *fs, iterpool)); /* determine the range of items to check for each revision */ - SVN_ERR(svn_fs_fs__l2p_get_max_ids(&max_ids, fs, start, count, pool)); + SVN_ERR(svn_fs_fs__l2p_get_max_ids(&max_ids, fs, start, count, pool, + iterpool)); /* check all items in all revisions if the given range */ for (i = 0; i < max_ids->nelts; ++i) @@ -205,10 +294,11 @@ compare_l2p_to_p2l_index(svn_fs_t *fs, /* find the corresponding P2L entry */ SVN_ERR(svn_fs_fs__p2l_entry_lookup(&p2l_entry, fs, rev_file, - revision, offset, iterpool)); + revision, offset, iterpool, + iterpool)); if (p2l_entry == NULL) - return svn_error_createf(SVN_ERR_FS_ITEM_INDEX_INCONSISTENT, + return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT, NULL, _("p2l index entry not found for " "PHYS %s returned by " @@ -218,7 +308,7 @@ compare_l2p_to_p2l_index(svn_fs_t *fs, if ( p2l_entry->item.number != k || p2l_entry->item.revision != revision) - return svn_error_createf(SVN_ERR_FS_ITEM_INDEX_INCONSISTENT, + return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT, NULL, _("p2l index info LOG r%ld:i%ld" " does not match " @@ -282,9 +372,9 @@ compare_p2l_to_l2p_index(svn_fs_t *fs, /* get all entries for the current block */ SVN_ERR(svn_fs_fs__p2l_index_lookup(&entries, fs, rev_file, start, offset, ffd->p2l_page_size, - iterpool)); + iterpool, iterpool)); if (entries->nelts == 0) - return svn_error_createf(SVN_ERR_FS_ITEM_INDEX_CORRUPTION, + return svn_error_createf(SVN_ERR_FS_INDEX_CORRUPTION, NULL, _("p2l does not cover offset %s" " for revision %ld"), @@ -301,7 +391,24 @@ compare_p2l_to_l2p_index(svn_fs_t *fs, = &APR_ARRAY_IDX(entries, i, svn_fs_fs__p2l_entry_t); /* check all sub-items for consist entries in the L2P index */ - if (entry->type != SVN_FS_FS__ITEM_TYPE_UNUSED) + if (entry->type == SVN_FS_FS__ITEM_TYPE_UNUSED) + { + /* There is no L2P entry for unused rev file sections. + * And its P2L index data is hardly ever used. But we + * should still check whether someone tempered with it. */ + if ( entry->item.revision != SVN_INVALID_REVNUM + && ( entry->item.revision < start + || entry->item.revision >= start + count)) + return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT, + NULL, + _("Empty P2L entry for PHYS %s " + "refers to revision %ld outside " + "the rev / pack file (%ld-%ld)"), + apr_off_t_toa(pool, entry->offset), + (long)entry->item.number, + start, start + count - 1); + } + else { apr_off_t l2p_offset; SVN_ERR(svn_fs_fs__item_offset(&l2p_offset, fs, rev_file, @@ -309,7 +416,7 @@ compare_p2l_to_l2p_index(svn_fs_t *fs, entry->item.number, iterpool)); if (l2p_offset != entry->offset) - return svn_error_createf(SVN_ERR_FS_ITEM_INDEX_INCONSISTENT, + return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT, NULL, _("l2p index entry PHYS %s" "does not match p2l index value " @@ -416,7 +523,7 @@ expected_checksum(apr_file_t *file, SVN_ERR(svn_io_file_name_get(&file_name, file, pool)); return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, - _("Checksum mismatch item at offset %s of " + _("Checksum mismatch in item at offset %s of " "length %s bytes in file %s"), apr_off_t_toa(pool, entry->offset), apr_off_t_toa(pool, entry->size), file_name); @@ -511,7 +618,7 @@ compare_p2l_to_rev(svn_fs_t *fs, pool)); if (rev_file->l2p_offset != max_offset) - return svn_error_createf(SVN_ERR_FS_ITEM_INDEX_INCONSISTENT, NULL, + return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT, NULL, _("File size of %s for revision r%ld does " "not match p2l index size of %s"), apr_off_t_toa(pool, rev_file->l2p_offset), start, @@ -532,7 +639,12 @@ compare_p2l_to_rev(svn_fs_t *fs, /* get all entries for the current block */ SVN_ERR(svn_fs_fs__p2l_index_lookup(&entries, fs, rev_file, start, offset, ffd->p2l_page_size, - iterpool)); + iterpool, iterpool)); + + /* The above might have moved the file pointer. + * Ensure we actually start reading at OFFSET. */ + SVN_ERR(svn_io_file_aligned_seek(rev_file->file, ffd->block_size, + NULL, offset, iterpool)); /* process all entries (and later continue with the next block) */ for (i = 0; i < entries->nelts; ++i) @@ -550,7 +662,7 @@ compare_p2l_to_rev(svn_fs_t *fs, /* p2l index must cover all rev / pack file offsets exactly once */ if (entry->offset != offset) - return svn_error_createf(SVN_ERR_FS_ITEM_INDEX_INCONSISTENT, + return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT, NULL, _("p2l index entry for revision r%ld" " is non-contiguous between offsets " @@ -566,7 +678,7 @@ compare_p2l_to_rev(svn_fs_t *fs, if (entry->offset != max_offset) SVN_ERR(read_all_nul(rev_file->file, entry->size, pool)); } - else if (entry->fnv1_checksum) + else { if (entry->size < STREAM_THRESHOLD) SVN_ERR(expected_buffered_checksum(rev_file->file, entry, @@ -591,14 +703,47 @@ compare_p2l_to_rev(svn_fs_t *fs, return SVN_NO_ERROR; } -static svn_revnum_t -packed_base_rev(svn_fs_t *fs, svn_revnum_t rev) +/* Verify that the revprops of the revisions START to END in FS can be + * accessed. Invoke CANCEL_FUNC with CANCEL_BATON at regular intervals. + * + * The values of START and END have already been auto-selected and + * verified. + */ +static svn_error_t * +verify_revprops(svn_fs_t *fs, + svn_revnum_t start, + svn_revnum_t end, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *pool) { - fs_fs_data_t *ffd = fs->fsap_data; + svn_revnum_t revision; + apr_pool_t *iterpool = svn_pool_create(pool); + + for (revision = start; revision < end; ++revision) + { + svn_string_t *date; + apr_time_t timetemp; - return rev < ffd->min_unpacked_rev - ? rev - (rev % ffd->max_files_per_dir) - : rev; + svn_pool_clear(iterpool); + + /* Access the svn:date revprop. + * This implies parsing all revprops for that revision. */ + SVN_ERR(svn_fs_fs__revision_prop(&date, fs, revision, + SVN_PROP_REVISION_DATE, iterpool)); + + /* The time stamp is the only revprop that, if given, needs to + * have a valid content. */ + if (date) + SVN_ERR(svn_time_from_cstring(&timetemp, date->data, iterpool)); + + if (cancel_func) + SVN_ERR(cancel_func(cancel_baton)); + } + + svn_pool_destroy(iterpool); + + return SVN_NO_ERROR; } static svn_revnum_t @@ -609,22 +754,24 @@ pack_size(svn_fs_t *fs, svn_revnum_t rev return rev < ffd->min_unpacked_rev ? ffd->max_files_per_dir : 1; } -/* Verify that the log-to-phys indexes and phys-to-log indexes are - * consistent with each other. The function signature is similar to - * svn_fs_fs__verify. +/* Verify that on-disk representation has not been tempered with (in a way + * that leaves the repository in a corrupted state). This compares log-to- + * phys with phys-to-log indexes, verifies the low-level checksums and + * checks that all revprops are available. The function signature is + * similar to svn_fs_fs__verify. * * The values of START and END have already been auto-selected and * verified. You may call this for format7 or higher repos. */ static svn_error_t * -verify_index_consistency(svn_fs_t *fs, - svn_revnum_t start, - svn_revnum_t end, - svn_fs_progress_notify_func_t notify_func, - void *notify_baton, - svn_cancel_func_t cancel_func, - void *cancel_baton, - apr_pool_t *pool) +verify_f7_metadata_consistency(svn_fs_t *fs, + svn_revnum_t start, + svn_revnum_t end, + svn_fs_progress_notify_func_t notify_func, + void *notify_baton, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *pool) { fs_fs_data_t *ffd = fs->fsap_data; svn_revnum_t revision, next_revision; @@ -635,7 +782,7 @@ verify_index_consistency(svn_fs_t *fs, svn_error_t *err = SVN_NO_ERROR; svn_revnum_t count = pack_size(fs, revision); - svn_revnum_t pack_start = packed_base_rev(fs, revision); + svn_revnum_t pack_start = svn_fs_fs__packed_base_rev(fs, revision); svn_revnum_t pack_end = pack_start + count; svn_pool_clear(iterpool); @@ -643,9 +790,14 @@ verify_index_consistency(svn_fs_t *fs, if (notify_func && (pack_start % ffd->max_files_per_dir == 0)) notify_func(pack_start, notify_baton, iterpool); + /* Check for external corruption to the indexes. */ + err = verify_index_checksums(fs, pack_start, cancel_func, + cancel_baton, iterpool); + /* two-way index check */ - err = compare_l2p_to_p2l_index(fs, pack_start, pack_end - pack_start, - cancel_func, cancel_baton, iterpool); + if (!err) + err = compare_l2p_to_p2l_index(fs, pack_start, pack_end - pack_start, + cancel_func, cancel_baton, iterpool); if (!err) err = compare_p2l_to_l2p_index(fs, pack_start, pack_end - pack_start, cancel_func, cancel_baton, iterpool); @@ -655,6 +807,11 @@ verify_index_consistency(svn_fs_t *fs, err = compare_p2l_to_rev(fs, pack_start, pack_end - pack_start, cancel_func, cancel_baton, iterpool); + /* ensure that revprops are available and accessible */ + if (!err) + err = verify_revprops(fs, pack_start, pack_end, + cancel_func, cancel_baton, iterpool); + /* concurrent packing is one of the reasons why verification may fail. Make sure, we operate on up-to-date information. */ if (err) @@ -668,7 +825,7 @@ verify_index_consistency(svn_fs_t *fs, /* We could simply assign revision here but the code below is more intuitive to maintainers. */ - next_revision = packed_base_rev(fs, revision); + next_revision = svn_fs_fs__packed_base_rev(fs, revision); } else { @@ -705,11 +862,10 @@ svn_fs_fs__verify(svn_fs_t *fs, /* log/phys index consistency. We need to check them first to make sure we can access the rev / pack files in format7. */ - if (svn_fs_fs__use_log_addressing(fs, end)) - SVN_ERR(verify_index_consistency(fs, - MAX(start, ffd->min_log_addressing_rev), - end, notify_func, notify_baton, - cancel_func, cancel_baton, pool)); + if (svn_fs_fs__use_log_addressing(fs)) + SVN_ERR(verify_f7_metadata_consistency(fs, start, end, + notify_func, notify_baton, + cancel_func, cancel_baton, pool)); /* rep cache consistency */ if (ffd->format >= SVN_FS_FS__MIN_REP_SHARING_FORMAT)
Modified: subversion/branches/authzperf/subversion/libsvn_fs_util/fs-util.c URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_util/fs-util.c?rev=1649205&r1=1649204&r2=1649205&view=diff ============================================================================== --- subversion/branches/authzperf/subversion/libsvn_fs_util/fs-util.c (original) +++ subversion/branches/authzperf/subversion/libsvn_fs_util/fs-util.c Sat Jan 3 14:00:41 2015 @@ -199,6 +199,7 @@ svn_fs__path_change_create_internal(cons change->node_rev_id = node_rev_id; change->change_kind = change_kind; change->mergeinfo_mod = svn_tristate_unknown; + change->copyfrom_rev = SVN_INVALID_REVNUM; return change; } Propchange: subversion/branches/authzperf/subversion/libsvn_fs_x/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Sat Jan 3 14:00:41 2015 @@ -20,6 +20,7 @@ /subversion/branches/fsfs-lock-many/subversion/libsvn_fs_x:1571740-1577217 /subversion/branches/fsfs-pack/subversion/libsvn_fs_x:873717-874575 /subversion/branches/fsx/subversion/libsvn_fs_x:1507845-1508040 +/subversion/branches/fsx-id/subversion/libsvn_fs_x:1645603-1649011 /subversion/branches/gnome-keyring/subversion/libsvn_fs_x:870558-871410 /subversion/branches/gpg-agent-password-store/subversion/libsvn_fs_x:1005036-1150766 /subversion/branches/gtest_addition/subversion/libsvn_fs_x:1452117-1502138 @@ -50,6 +51,7 @@ /subversion/branches/issue-4194-dev/subversion/libsvn_fs_x:1410507-1414880 /subversion/branches/javahl-ra/subversion/libsvn_fs_x:991978-1494640 /subversion/branches/kwallet/subversion/libsvn_fs_x:870785-871314 +/subversion/branches/log-addressing/subversion/libsvn_fs_fs:1531982 /subversion/branches/log-addressing/subversion/libsvn_fs_x:1511324 /subversion/branches/log-g-performance/subversion/libsvn_fs_x:870941-871032 /subversion/branches/merge-skips-obstructions/subversion/libsvn_fs_x:874525-874615 @@ -62,6 +64,8 @@ /subversion/branches/reintegrate-improvements/subversion/libsvn_fs_x:873853-874164 /subversion/branches/remote-only-status/subversion/libsvn_fs_x:1581845-1586090 /subversion/branches/revprop-cache/subversion/libsvn_fs_x:1298521-1326293 +/subversion/branches/revprop-caching-ng/subversion/libsvn_fs_fs:1619782-1620595 +/subversion/branches/revprop-caching-ng/subversion/libsvn_fs_x:1620597 /subversion/branches/revprop-packing/subversion/libsvn_fs_x:1143907,1143971,1143997,1144017,1144499,1144568,1146145 /subversion/branches/subtree-mergeinfo/subversion/libsvn_fs_x:876734-878766 /subversion/branches/svn-mergeinfo-enhancements/subversion/libsvn_fs_x:870119-870195,870197-870288 @@ -82,5 +86,5 @@ /subversion/branches/verify-at-commit/subversion/libsvn_fs_x:1462039-1462408 /subversion/branches/verify-keep-going/subversion/libsvn_fs_x:1439280-1492639,1546002-1546110 /subversion/branches/wc-collate-path/subversion/libsvn_fs_x:1402685-1480384 -/subversion/trunk/subversion/libsvn_fs_fs:1415133-1596500 -/subversion/trunk/subversion/libsvn_fs_x:1414756-1509914,1613053-1621376 +/subversion/trunk/subversion/libsvn_fs_fs:1415133-1596500,1596567,1597414,1597989,1598273,1599140,1600872,1601633,1603485-1603487,1603499,1603605,1604128,1604188,1604413-1604414,1604416-1604417,1604421,1604442,1604700,1604717,1604720,1604726,1604755,1604794,1604802,1604824,1604836,1604844,1604902-1604903,1604911,1604925,1604933,1604947,1605059-1605060,1605064-1605065,1605068,1605071-1605073,1605075,1605123,1605188-1605189,1605191,1605197,1605444,1605633,1606132,1606142,1606144,1606514,1606526,1606528,1606551,1606554,1606564,1606598-1606599,1606656,1606658,1606662,1606744,1606840,1607085,1607572,1612407,1612810,1613339,1613872,1614611,1615348,1615351-1615352,1615356,1616338-1616339,1616613,1617586,1617688,1618138,1618151,1618153,1618226,1618641,1618653,1618662,1619068,1619358,1619413,1619769,1619774,1620602,1620909,1620912,1620928,1620930,1621275,1621635,1622931,1622937,1622942,1622946,1622959-1622960,1622963,1622987,1623007,1623368,1623373,1623377,1623379,1623381,1623398,1623402,162 4011,1624265,1624512,1626246,1626871,1626873,1626886,1627497-1627498,1627502,1627947-1627949,1627966,1628083,1628093,1628158-1628159,1628161,1628392-1628393,1628415,1628427,1628676,1628738,1628762,1628764,1629854-1629855,1629857,1629865,1629873,1629875,1629879,1630067,1630070,1631049-1631051,1631075,1631115,1631171,1631180,1631185-1631186,1631196-1631197,1631239-1631240,1631548,1631550,1631563,1631567,1631588,1631598,1632646,1632776,1632849,1632851-1632853,1632856-1632857,1632868,1632908,1632926,1633232,1633617-1633618,1634872,1634875,1634879-1634880,1634920,1636478,1636483,1636629,1636644,1637184,1637186,1637330,1637358,1637363,1637393,1639319,1639322,1639335,1639348,1639352,1639355,1639358,1639414,1639419,1639426,1639430,1639436,1639440,1639549,1640061-1640062,1640197,1640915,1640966,1641013,1643139,1643233,1645567,1646021,1646712,1646716,1647537,1647540-1647541,1647820,1647905,1648230,1648238,1648241-1648243,1648253,1648272,1648532,1648537-1648539,1648542,1648591,1648612 +/subversion/trunk/subversion/libsvn_fs_x:1414756-1509914,1613053-1649188
