Modified: subversion/branches/authzperf/subversion/libsvn_fs_x/fs.h URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_x/fs.h?rev=1649205&r1=1649204&r2=1649205&view=diff ============================================================================== --- subversion/branches/authzperf/subversion/libsvn_fs_x/fs.h (original) +++ subversion/branches/authzperf/subversion/libsvn_fs_x/fs.h Sat Jan 3 14:00:41 2015 @@ -36,7 +36,6 @@ #include "private/svn_fs_private.h" #include "private/svn_sqlite.h" #include "private/svn_mutex.h" -#include "private/svn_named_atomic.h" #include "id.h" @@ -127,15 +126,25 @@ extern "C" { */ #define SVN_FS_X__FORMAT_NUMBER 1 +/* On most operating systems apr implements file locks per process, not + per file. On Windows apr implements the locking as per file handle + locks, so we don't have to add our own mutex for just in-process + synchronization. */ +#if APR_HAS_THREADS && !defined(WIN32) +#define SVN_FS_X__USE_LOCK_MUTEX 1 +#else +#define SVN_FS_X__USE_LOCK_MUTEX 0 +#endif + /* Private FSX-specific data shared between all svn_txn_t objects that relate to a particular transaction in a filesystem (as identified by transaction id and filesystem UUID). Objects of this type are allocated in their own subpool of the common pool. */ -typedef struct fs_x_shared_txn_data_t +typedef struct svn_fs_x__shared_txn_data_t { /* The next transaction in the list, or NULL if there is no following transaction. */ - struct fs_x_shared_txn_data_t *next; + struct svn_fs_x__shared_txn_data_t *next; /* ID of this transaction. */ svn_fs_x__txn_id_t txn_id; @@ -149,22 +158,22 @@ typedef struct fs_x_shared_txn_data_t /* The pool in which this object has been allocated; a subpool of the common pool. */ apr_pool_t *pool; -} fs_x_shared_txn_data_t; +} svn_fs_x__shared_txn_data_t; /* Private FSX-specific data shared between all svn_fs_t objects that relate to a particular filesystem, as identified by filesystem UUID. Objects of this type are allocated in the common pool. */ -typedef struct fs_x_shared_data_t +typedef struct svn_fs_x__shared_data_t { /* A list of shared transaction objects for each transaction that is currently active, or NULL if none are. All access to this list, including the contents of the objects stored in it, is synchronised under TXN_LIST_LOCK. */ - fs_x_shared_txn_data_t *txns; + svn_fs_x__shared_txn_data_t *txns; /* A free transaction object, or NULL if there is no free object. Access to this object is synchronised under TXN_LIST_LOCK. */ - fs_x_shared_txn_data_t *free_txn; + svn_fs_x__shared_txn_data_t *free_txn; /* The following lock must be taken out in reverse order of their declaration here. Any subset may be acquired and held at any given @@ -190,26 +199,26 @@ typedef struct fs_x_shared_data_t /* The common pool, under which this object is allocated, subpools of which are used to allocate the transaction objects. */ apr_pool_t *common_pool; -} fs_x_shared_data_t; +} svn_fs_x__shared_data_t; /* Data structure for the 1st level DAG node cache. */ -typedef struct fs_x_dag_cache_t fs_x_dag_cache_t; +typedef struct svn_fs_x__dag_cache_t svn_fs_x__dag_cache_t; /* Key type for all caches that use revision + offset / counter as key. Note: Cache keys should be 16 bytes for best performance and there should be no padding. */ -typedef struct pair_cache_key_t +typedef struct svn_fs_x__pair_cache_key_t { /* The object's revision. Use the 64 data type to prevent padding. */ apr_int64_t revision; /* Sub-address: item index, revprop generation, packed flag, etc. */ apr_int64_t second; -} pair_cache_key_t; +} svn_fs_x__pair_cache_key_t; /* Key type that identifies a representation / rep header. */ -typedef struct representation_cache_key_t +typedef struct svn_fs_x__representation_cache_key_t { /* Revision that contains the representation */ svn_revnum_t revision; @@ -219,12 +228,12 @@ typedef struct representation_cache_key_ /* Item index of the representation */ apr_uint64_t item_index; -} representation_cache_key_t; +} svn_fs_x__representation_cache_key_t; /* Key type that identifies a txdelta window. Note: Cache keys should require no padding. */ -typedef struct window_cache_key_t +typedef struct svn_fs_x__window_cache_key_t { /* The object's revision. Use the 64 data type to prevent padding. */ apr_int64_t revision; @@ -234,11 +243,11 @@ typedef struct window_cache_key_t /* Item index of the representation */ apr_uint64_t item_index; -} window_cache_key_t; +} svn_fs_x__window_cache_key_t; /* Private (non-shared) FSX-specific data for each svn_fs_t object. Any caches in here may be NULL. */ -typedef struct fs_x_data_t +typedef struct svn_fs_x__data_t { /* The format number of this FS. */ int format; @@ -246,9 +255,11 @@ typedef struct fs_x_data_t /* The maximum number of files to store per directory. */ int max_files_per_dir; - /* Rev / pack file read granularity. */ + /* Rev / pack file read granularity in bytes. */ apr_int64_t block_size; + /* Rev / pack file granularity (in bytes) covered by a single phys-to-log + * index page. */ /* Capacity in entries of log-to-phys index pages */ apr_int64_t l2p_page_size; @@ -268,12 +279,8 @@ typedef struct fs_x_data_t e.g. memcached may be ignored as caching is an optional feature. */ svn_boolean_t fail_stop; - /* A cache of revision root IDs, mapping from (svn_revnum_t *) to - (svn_fs_id_t *). (Not threadsafe.) */ - svn_cache__t *rev_root_id_cache; - /* Caches native dag_node_t* instances and acts as a 1st level cache */ - fs_x_dag_cache_t *dag_node_cache; + svn_fs_x__dag_cache_t *dag_node_cache; /* DAG node cache for immutable nodes. Maps (revision, fspath) to (dag_node_t *). This is the 2nd level cache for DAG nodes. */ @@ -281,24 +288,16 @@ typedef struct fs_x_data_t /* A cache of the contents of immutable directories; maps from unparsed FS ID to a apr_hash_t * mapping (const char *) dirent - names to (svn_fs_dirent_t *). */ + names to (svn_fs_x__dirent_t *). */ svn_cache__t *dir_cache; /* Fulltext cache; currently only used with memcached. Maps from rep key (revision/offset) to svn_stringbuf_t. */ svn_cache__t *fulltext_cache; - /* Access object to the atomics namespace used by revprop caching. - Will be NULL until the first access. */ - svn_atomic_namespace__t *revprop_namespace; - /* Access object to the revprop "generation". Will be NULL until - the first access. */ - svn_named_atomic__t *revprop_generation; - - /* Access object to the revprop update timeout. Will be NULL until - the first access. */ - svn_named_atomic__t *revprop_timeout; + the first access. May be also get closed and set to NULL again. */ + apr_file_t *revprop_generation_file; /* Revision property cache. Maps from (rev,generation) to apr_hash_t. */ svn_cache__t *revprop_cache; @@ -312,11 +311,12 @@ typedef struct fs_x_data_t respective pack file. */ svn_cache__t *packed_offset_cache; - /* Cache for txdelta_window_t objects; the key is window_cache_key_t */ + /* Cache for txdelta_window_t objects; + * the key is svn_fs_x__window_cache_key_t */ svn_cache__t *txdelta_window_cache; /* Cache for combined windows as svn_stringbuf_t objects; - the key is window_cache_key_t */ + the key is svn_fs_x__window_cache_key_t */ svn_cache__t *combined_window_cache; /* Cache for svn_fs_x__rep_header_t objects; @@ -327,8 +327,8 @@ typedef struct fs_x_data_t the key is a (pack file revision, file offset) pair */ svn_cache__t *noderevs_container_cache; - /* Cache for change lists as APR arrays of change_t * objects; the key - is the revision */ + /* Cache for change lists as APR arrays of svn_fs_x__change_t * objects; + the key is the revision */ svn_cache__t *changes_cache; /* Cache for change_list_t containers; @@ -373,7 +373,7 @@ typedef struct fs_x_data_t svn_boolean_t has_write_lock; /* Data shared between all svn_fs_t objects for a given filesystem. */ - fs_x_shared_data_t *shared; + svn_fs_x__shared_data_t *shared; /* The sqlite database used for rep caching. */ svn_sqlite__db_t *rep_cache_db; @@ -409,37 +409,39 @@ typedef struct fs_x_data_t /* Pack after every commit. */ svn_boolean_t pack_after_commit; + /* Per-instance filesystem ID, which provides an additional level of + uniqueness for filesystems that share the same UUID, but should + still be distinguishable (e.g. backups produced by svn_fs_hotcopy() + or dump / load cycles). */ + const char *instance_id; + /* Pointer to svn_fs_open. */ svn_error_t *(*svn_fs_open_)(svn_fs_t **, const char *, apr_hash_t *, apr_pool_t *, apr_pool_t *); -} fs_x_data_t; +} svn_fs_x__data_t; /*** Filesystem Transaction ***/ -typedef struct transaction_t +typedef struct svn_fs_x__transaction_t { /* property list (const char * name, svn_string_t * value). may be NULL if there are no properties. */ apr_hash_t *proplist; - /* node revision id of the root node. */ - const svn_fs_id_t *root_id; - - /* node revision id of the node which is the root of the revision - upon which this txn is base. (unfinished only) */ - const svn_fs_id_t *base_id; + /* revision upon which this txn is base. (unfinished only) */ + svn_revnum_t base_rev; /* copies list (const char * copy_ids), or NULL if there have been no copies in this transaction. */ apr_array_header_t *copies; -} transaction_t; +} svn_fs_x__transaction_t; /*** Representation ***/ /* If you add fields to this, check to see if you need to change * svn_fs_x__rep_copy. */ -typedef struct representation_t +typedef struct svn_fs_x__representation_t { /* Checksums digests for the contents produced by this representation. This checksum is for the contents the rep shows to consumers, @@ -457,7 +459,7 @@ typedef struct representation_t unsigned char md5_digest[APR_MD5_DIGESTSIZE]; /* Change set and item number where this representation is located. */ - svn_fs_x__id_part_t id; + svn_fs_x__id_t id; /* The size of the representation in bytes as seen in the revision file. */ @@ -466,23 +468,29 @@ typedef struct representation_t /* The size of the fulltext of the representation. */ svn_filesize_t expanded_size; -} representation_t; +} svn_fs_x__representation_t; /*** Node-Revision ***/ /* If you add fields to this, check to see if you need to change * copy_node_revision in dag.c. */ -typedef struct node_revision_t +typedef struct svn_fs_x__noderev_t { /* node kind */ svn_node_kind_t kind; - /* The node-id for this node-rev. */ - const svn_fs_id_t *id; + /* Predecessor node revision id. Will be "unused" if there is no + predecessor for this node revision. */ + svn_fs_x__id_t predecessor_id; + + /* The ID of this noderev */ + svn_fs_x__id_t noderev_id; + + /* Identifier of the node that this noderev belongs to. */ + svn_fs_x__id_t node_id; - /* predecessor node revision id, or NULL if there is no predecessor - for this node revision */ - const svn_fs_id_t *predecessor_id; + /* Copy identifier of this line of history. */ + svn_fs_x__id_t copy_id; /* If this node-rev is a copy, where was it copied from? */ const char *copyfrom_path; @@ -499,11 +507,11 @@ typedef struct node_revision_t /* representation key for this node's properties. may be NULL if there are no properties. */ - representation_t *prop_rep; + svn_fs_x__representation_t *prop_rep; /* representation for this node's data. may be NULL if there is no data. */ - representation_t *data_rep; + svn_fs_x__representation_t *data_rep; /* path at which this node first came into existence. */ const char *created_path; @@ -518,18 +526,47 @@ typedef struct node_revision_t /* Does this node itself have svn:mergeinfo? */ svn_boolean_t has_mergeinfo; -} node_revision_t; +} svn_fs_x__noderev_t; + + +/** The type of a directory entry. */ +typedef struct svn_fs_x__dirent_t +{ + + /** The name of this directory entry. */ + const char *name; + + /** The node revision ID it names. */ + svn_fs_x__id_t id; + + /** The node kind. */ + svn_node_kind_t kind; +} svn_fs_x__dirent_t; /*** Change ***/ -typedef struct change_t +typedef struct svn_fs_x__change_t { /* Path of the change. */ svn_string_t path; - /* API compatible change description */ - svn_fs_path_change2_t info; -} change_t; + /* node revision id of changed path */ + svn_fs_x__id_t noderev_id; + + /* See svn_fs_path_change2_t for a description for the remaining elements. + */ + svn_fs_path_change_kind_t change_kind; + + svn_boolean_t text_mod; + svn_boolean_t prop_mod; + svn_node_kind_t node_kind; + + svn_boolean_t copyfrom_known; + svn_revnum_t copyfrom_rev; + const char *copyfrom_path; + + svn_tristate_t mergeinfo_mod; +} svn_fs_x__change_t; #ifdef __cplusplus
Modified: subversion/branches/authzperf/subversion/libsvn_fs_x/fs_x.c URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_x/fs_x.c?rev=1649205&r1=1649204&r2=1649205&view=diff ============================================================================== --- subversion/branches/authzperf/subversion/libsvn_fs_x/fs_x.c (original) +++ subversion/branches/authzperf/subversion/libsvn_fs_x/fs_x.c Sat Jan 3 14:00:41 2015 @@ -157,7 +157,7 @@ svn_fs_x__write_format(svn_fs_t *fs, { svn_stringbuf_t *sb; const char *path = svn_fs_x__path_format(fs, pool); - fs_x_data_t *ffd = fs->fsap_data; + svn_fs_x__data_t *ffd = fs->fsap_data; SVN_ERR_ASSERT(1 <= ffd->format && ffd->format <= SVN_FS_X__FORMAT_NUMBER); @@ -183,30 +183,55 @@ svn_fs_x__write_format(svn_fs_t *fs, return svn_io_set_file_read_only(path, FALSE, pool); } -/* Find the youngest revision in a repository at path FS_PATH and - return it in *YOUNGEST_P. Perform temporary allocations in - POOL. */ +/* Check that BLOCK_SIZE is a valid block / page size, i.e. it is within + * the range of what the current system may address in RAM and it is a + * power of 2. Assume that the element size within the block is ITEM_SIZE. + * Use SCRATCH_POOL for temporary allocations. + */ static svn_error_t * -get_youngest(svn_revnum_t *youngest_p, - const char *fs_path, - apr_pool_t *pool) -{ - svn_stringbuf_t *buf; - SVN_ERR(svn_fs_x__read_content(&buf, - svn_dirent_join(fs_path, PATH_CURRENT, pool), - pool)); - - *youngest_p = SVN_STR_TO_REV(buf->data); +verify_block_size(apr_int64_t block_size, + apr_size_t item_size, + const char *name, + apr_pool_t *scratch_pool + ) +{ + /* Limit range. */ + if (block_size <= 0) + return svn_error_createf(SVN_ERR_BAD_CONFIG_VALUE, NULL, + _("%s is too small for fsfs.conf setting '%s'."), + apr_psprintf(scratch_pool, + "%" APR_INT64_T_FMT, + block_size), + name); + + if (block_size > SVN_MAX_OBJECT_SIZE / item_size) + return svn_error_createf(SVN_ERR_BAD_CONFIG_VALUE, NULL, + _("%s is too large for fsfs.conf setting '%s'."), + apr_psprintf(scratch_pool, + "%" APR_INT64_T_FMT, + block_size), + name); + + /* Ensure it is a power of two. + * For positive X, X & (X-1) will reset the lowest bit set. + * If the result is 0, at most one bit has been set. */ + if (0 != (block_size & (block_size - 1))) + return svn_error_createf(SVN_ERR_BAD_CONFIG_VALUE, NULL, + _("%s is invalid for fsfs.conf setting '%s' " + "because it is not a power of 2."), + apr_psprintf(scratch_pool, + "%" APR_INT64_T_FMT, + block_size), + name); return SVN_NO_ERROR; } - /* Read the configuration information of the file system at FS_PATH * and set the respective values in FFD. Use pools as usual. */ static svn_error_t * -read_config(fs_x_data_t *ffd, +read_config(svn_fs_x__data_t *ffd, const char *fs_path, apr_pool_t *result_pool, apr_pool_t *scratch_pool) @@ -268,8 +293,20 @@ read_config(fs_x_data_t *ffd, CONFIG_OPTION_P2L_PAGE_SIZE, 0x400)); + /* Don't accept unreasonable or illegal values. + * Block size and P2L page size are in kbytes; + * L2P blocks are arrays of apr_off_t. */ + SVN_ERR(verify_block_size(ffd->block_size, 0x400, + CONFIG_OPTION_BLOCK_SIZE, scratch_pool)); + SVN_ERR(verify_block_size(ffd->p2l_page_size, 0x400, + CONFIG_OPTION_P2L_PAGE_SIZE, scratch_pool)); + SVN_ERR(verify_block_size(ffd->l2p_page_size, sizeof(apr_off_t), + CONFIG_OPTION_L2P_PAGE_SIZE, scratch_pool)); + + /* convert kBytes to bytes */ ffd->block_size *= 0x400; ffd->p2l_page_size *= 0x400; + /* L2P pages are in entries - not in (k)Bytes */ /* Debug options. */ SVN_ERR(svn_config_get_bool(config, &ffd->pack_after_commit, @@ -435,23 +472,18 @@ write_config(svn_fs_t *fs, "### For SSD-based storage systems, slightly lower values around 16 kB" NL "### may improve latency while still maximizing throughput." NL "### Can be changed at any time but must be a power of 2." NL -"### block-size is 64 kBytes by default." NL +"### block-size is given in kBytes and with a default of 64 kBytes." NL "# " CONFIG_OPTION_BLOCK_SIZE " = 64" NL "###" NL "### The log-to-phys index maps data item numbers to offsets within the" NL -"### rev or pack file. A revision typically contains 2 .. 5 such items" NL -"### per changed path. For each revision, at least one page is being" NL -"### allocated in the l2p index with unused parts resulting in no wasted" NL -"### space." NL -"### Changing this parameter only affects larger revisions with thousands" NL -"### of changed paths. A smaller value means that more pages need to be" NL -"### allocated for such revisions, increasing the size of the page table" NL -"### meaning it takes longer to read that table (once). Access to each" NL -"### page is then faster because less data has to read. So, if you have" NL -"### several extremely large revisions (approaching 1 mio changes), think" NL +"### rev or pack file. This index is organized in pages of a fixed maximum" NL +"### capacity. To access an item, the page table and the respective page" NL +"### must be read." NL +"### This parameter only affects revisions with thousands of changed paths." NL +"### If you have several extremely large revisions (~1 mio changes), think" NL "### about increasing this setting. Reducing the value will rarely result" NL "### in a net speedup." NL -"### This is an expert setting. Any non-zero value is possible." NL +"### This is an expert setting. Must be a power of 2." NL "### l2p-page-size is 8192 entries by default." NL "# " CONFIG_OPTION_L2P_PAGE_SIZE " = 8192" NL "###" NL @@ -460,7 +492,7 @@ write_config(svn_fs_t *fs, "### stored at any particular offset. The index describes the rev file" NL "### in chunks (pages) and keeps a global list of all those pages. Large" NL "### pages mean a shorter page table but a larger per-page description of" NL -"### data items in it. The latency sweetspot depends on the change size" NL +"### data items in it. The latency sweet spot depends on the change size" NL "### distribution but covers a relatively wide range." NL "### If the repository contains very large files, i.e. individual changes" NL "### of tens of MB each, increasing the page size will shorten the index" NL @@ -468,7 +500,7 @@ write_config(svn_fs_t *fs, "### smaller changes." NL "### For source code repositories, this should be about 16x the block-size." NL "### Must be a power of 2." NL -"### p2l-page-size is 1024 kBytes by default." NL +"### p2l-page-size is given in kBytes and with a default of 1024 kBytes." NL "# " CONFIG_OPTION_P2L_PAGE_SIZE " = 1024" NL ; #undef NL @@ -476,34 +508,65 @@ write_config(svn_fs_t *fs, fsx_conf_contents, pool); } -svn_error_t * -svn_fs_x__open(svn_fs_t *fs, const char *path, apr_pool_t *pool) +/* Read FS's UUID file and store the data in the FS struct. */ +static svn_error_t * +read_uuid(svn_fs_t *fs, + apr_pool_t *scratch_pool) { - fs_x_data_t *ffd = fs->fsap_data; + svn_fs_x__data_t *ffd = fs->fsap_data; apr_file_t *uuid_file; - int format, max_files_per_dir; char buf[APR_UUID_FORMATTED_LENGTH + 2]; apr_size_t limit; - fs->path = apr_pstrdup(fs->pool, path); + /* Read the repository uuid. */ + SVN_ERR(svn_io_file_open(&uuid_file, svn_fs_x__path_uuid(fs, scratch_pool), + APR_READ | APR_BUFFERED, APR_OS_DEFAULT, + scratch_pool)); + + limit = sizeof(buf); + SVN_ERR(svn_io_read_length_line(uuid_file, buf, &limit, scratch_pool)); + fs->uuid = apr_pstrdup(fs->pool, buf); + + /* Read the instance ID. */ + limit = sizeof(buf); + SVN_ERR(svn_io_read_length_line(uuid_file, buf, &limit, + scratch_pool)); + ffd->instance_id = apr_pstrdup(fs->pool, buf); + + SVN_ERR(svn_io_file_close(uuid_file, scratch_pool)); - /* Read the FS format number. */ + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__read_format_file(svn_fs_t *fs, + apr_pool_t *scratch_pool) +{ + svn_fs_x__data_t *ffd = fs->fsap_data; + int format, max_files_per_dir; + + /* Read info from format file. */ SVN_ERR(read_format(&format, &max_files_per_dir, - svn_fs_x__path_format(fs, pool), pool)); + svn_fs_x__path_format(fs, scratch_pool), scratch_pool)); - /* Now we've got a format number no matter what. */ + /* Now that we've got *all* info, store / update values in FFD. */ ffd->format = format; ffd->max_files_per_dir = max_files_per_dir; - /* Read in and cache the repository uuid. */ - SVN_ERR(svn_io_file_open(&uuid_file, svn_fs_x__path_uuid(fs, pool), - APR_READ | APR_BUFFERED, APR_OS_DEFAULT, pool)); + return SVN_NO_ERROR; +} - limit = sizeof(buf); - SVN_ERR(svn_io_read_length_line(uuid_file, buf, &limit, pool)); - fs->uuid = apr_pstrdup(fs->pool, buf); +svn_error_t * +svn_fs_x__open(svn_fs_t *fs, const char *path, apr_pool_t *pool) +{ + svn_fs_x__data_t *ffd = fs->fsap_data; + fs->path = apr_pstrdup(fs->pool, path); - SVN_ERR(svn_io_file_close(uuid_file, pool)); + /* Read the FS format file. */ + SVN_ERR(svn_fs_x__read_format_file(fs, pool)); + + /* Read in and cache the repository uuid. */ + SVN_ERR(read_uuid(fs, pool)); /* Read the min unpacked revision. */ SVN_ERR(svn_fs_x__update_min_unpacked_rev(fs, pool)); @@ -511,7 +574,8 @@ svn_fs_x__open(svn_fs_t *fs, const char /* Read the configuration file. */ SVN_ERR(read_config(ffd, fs->path, fs->pool, pool)); - return get_youngest(&(ffd->youngest_rev_cache), path, pool); + return svn_error_trace(svn_fs_x__read_current(&ffd->youngest_rev_cache, + fs, pool)); } /* Baton type bridging svn_fs_x__upgrade and upgrade_body carrying @@ -566,12 +630,11 @@ svn_fs_x__upgrade(svn_fs_t *fs, svn_error_t * svn_fs_x__youngest_rev(svn_revnum_t *youngest_p, - svn_fs_t *fs, - apr_pool_t *pool) + svn_fs_t *fs, + apr_pool_t *pool) { - fs_x_data_t *ffd = fs->fsap_data; - - SVN_ERR(get_youngest(youngest_p, fs->path, pool)); + svn_fs_x__data_t *ffd = fs->fsap_data; + SVN_ERR(svn_fs_x__read_current(youngest_p, fs, pool)); ffd->youngest_rev_cache = *youngest_p; return SVN_NO_ERROR; @@ -582,7 +645,7 @@ svn_fs_x__ensure_revision_exists(svn_rev svn_fs_t *fs, apr_pool_t *pool) { - fs_x_data_t *ffd = fs->fsap_data; + svn_fs_x__data_t *ffd = fs->fsap_data; if (! SVN_IS_VALID_REVNUM(rev)) return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL, @@ -594,7 +657,7 @@ svn_fs_x__ensure_revision_exists(svn_rev if (rev <= ffd->youngest_rev_cache) return SVN_NO_ERROR; - SVN_ERR(get_youngest(&(ffd->youngest_rev_cache), fs->path, pool)); + SVN_ERR(svn_fs_x__read_current(&ffd->youngest_rev_cache, fs, pool)); /* Check again. */ if (rev <= ffd->youngest_rev_cache) @@ -604,67 +667,13 @@ svn_fs_x__ensure_revision_exists(svn_rev _("No such revision %ld"), rev); } -/* Open the correct revision file for REV. If the filesystem FS has - been packed, *FILE will be set to the packed file; otherwise, set *FILE - to the revision file for REV. Return SVN_ERR_FS_NO_SUCH_REVISION if the - file doesn't exist. - - TODO: Consider returning an indication of whether this is a packed rev - file, so the caller need not rely on is_packed_rev() which in turn - relies on the cached FFD->min_unpacked_rev value not having changed - since the rev file was opened. - - Use POOL for allocations. */ -svn_error_t * -svn_fs_x__open_pack_or_rev_file(apr_file_t **file, - svn_fs_t *fs, - svn_revnum_t rev, - apr_pool_t *pool) -{ - svn_error_t *err; - svn_boolean_t retry = FALSE; - - do - { - const char *path = svn_fs_x__path_rev_absolute(fs, rev, pool); - - /* open the revision file in buffered r/o mode */ - err = svn_io_file_open(file, path, - APR_READ | APR_BUFFERED, APR_OS_DEFAULT, pool); - if (err && APR_STATUS_IS_ENOENT(err->apr_err)) - { - /* Could not open the file. This may happen if the - * file once existed but got packed later. */ - svn_error_clear(err); - - /* if that was our 2nd attempt, leave it at that. */ - if (retry) - return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL, - _("No such revision %ld"), rev); - - /* We failed for the first time. Refresh cache & retry. */ - SVN_ERR(svn_fs_x__update_min_unpacked_rev(fs, pool)); - - retry = TRUE; - } - else - { - retry = FALSE; - } - } - while (retry); - - return svn_error_trace(err); -} - - svn_error_t * svn_fs_x__revision_proplist(apr_hash_t **proplist_p, svn_fs_t *fs, svn_revnum_t rev, apr_pool_t *pool) { - SVN_ERR(svn_fs_x__get_revision_proplist(proplist_p, fs, rev, pool)); + SVN_ERR(svn_fs_x__get_revision_proplist(proplist_p, fs, rev, FALSE, pool)); return SVN_NO_ERROR; } @@ -672,7 +681,7 @@ svn_fs_x__revision_proplist(apr_hash_t * svn_error_t * svn_fs_x__file_length(svn_filesize_t *length, - node_revision_t *noderev, + svn_fs_x__noderev_t *noderev, apr_pool_t *pool) { if (noderev->data_rep) @@ -684,8 +693,8 @@ svn_fs_x__file_length(svn_filesize_t *le } svn_boolean_t -svn_fs_x__file_text_rep_equal(representation_t *a, - representation_t *b) +svn_fs_x__file_text_rep_equal(svn_fs_x__representation_t *a, + svn_fs_x__representation_t *b) { svn_boolean_t a_empty = a == NULL || a->expanded_size == 0; svn_boolean_t b_empty = b == NULL || b->expanded_size == 0; @@ -699,7 +708,7 @@ svn_fs_x__file_text_rep_equal(representa /* Same physical representation? Note that these IDs are always up-to-date instead of e.g. being set lazily. */ - if (svn_fs_x__id_part_eq(&a->id, &b->id)) + if (svn_fs_x__id_eq(&a->id, &b->id)) return TRUE; /* Contents are equal if the checksums match. These are also always known. @@ -711,13 +720,13 @@ svn_fs_x__file_text_rep_equal(representa svn_error_t * svn_fs_x__prop_rep_equal(svn_boolean_t *equal, svn_fs_t *fs, - node_revision_t *a, - node_revision_t *b, + svn_fs_x__noderev_t *a, + svn_fs_x__noderev_t *b, svn_boolean_t strict, apr_pool_t *scratch_pool) { - representation_t *rep_a = a->prop_rep; - representation_t *rep_b = b->prop_rep; + svn_fs_x__representation_t *rep_a = a->prop_rep; + svn_fs_x__representation_t *rep_b = b->prop_rep; apr_hash_t *proplist_a; apr_hash_t *proplist_b; @@ -741,7 +750,7 @@ svn_fs_x__prop_rep_equal(svn_boolean_t * } /* Same path in same txn? */ - if (svn_fs_x__id_eq(a->id, b->id)) + if (svn_fs_x__id_eq(&a->noderev_id, &b->noderev_id)) { *equal = TRUE; return SVN_NO_ERROR; @@ -767,7 +776,7 @@ svn_fs_x__prop_rep_equal(svn_boolean_t * svn_error_t * svn_fs_x__file_checksum(svn_checksum_t **checksum, - node_revision_t *noderev, + svn_fs_x__noderev_t *noderev, svn_checksum_kind_t kind, apr_pool_t *pool) { @@ -801,11 +810,11 @@ svn_fs_x__file_checksum(svn_checksum_t * return SVN_NO_ERROR; } -representation_t * -svn_fs_x__rep_copy(representation_t *rep, +svn_fs_x__representation_t * +svn_fs_x__rep_copy(svn_fs_x__representation_t *rep, apr_pool_t *pool) { - representation_t *rep_new; + svn_fs_x__representation_t *rep_new; if (rep == NULL) return NULL; @@ -818,62 +827,93 @@ svn_fs_x__rep_copy(representation_t *rep } -/* Write out the zeroth revision for filesystem FS. */ +/* Write out the zeroth revision for filesystem FS. + Perform temporary allocations in SCRATCH_POOL. */ static svn_error_t * -write_revision_zero(svn_fs_t *fs) +write_revision_zero(svn_fs_t *fs, + apr_pool_t *scratch_pool) { - const char *path_revision_zero = svn_fs_x__path_rev(fs, 0, fs->pool); + /* Use an explicit sub-pool to have full control over temp file lifetimes. + * Since we have it, use it for everything else as well. */ + apr_pool_t *subpool = svn_pool_create(scratch_pool); + const char *path_revision_zero = svn_fs_x__path_rev(fs, 0, subpool); apr_hash_t *proplist; svn_string_t date; - const char *path; - /* Write out a rev file for revision 0. */ + apr_array_header_t *index_entries; + svn_fs_x__p2l_entry_t *entry; + svn_fs_x__revision_file_t *rev_file; + const char *l2p_proto_index, *p2l_proto_index; + + /* Write a skeleton r0 with no indexes. */ SVN_ERR(svn_io_file_create_binary (path_revision_zero, -/* "DELTA\nSVN\4END\nENDREP\n"*/ "DELTA\nSVN\1" /* txdelta v1 */ "\0\0\4\2\5" /* sview offset, sview len, tview len, instr len, newlen */ "\1\x84" /* 1 instr byte, new 4 bytes */ "\4END\n" /* 4 new bytes, E, N, D, \n */ "ENDREP\n" - "id: 0+0.0+0.2+0\n" + "id: 2+0\n" + "node: 0+0\n" + "copy: 0+0\n" "type: dir\n" "count: 0\n" "text: 0 3 16 4 " "2d2977d1c96f487abe4a1e202dd03b4e\n" "cpath: /\n" "\n\n", - 0x7b, fs->pool)); - - SVN_ERR(svn_io_set_file_read_only(path_revision_zero, FALSE, fs->pool)); + 0x8b, subpool)); - path = svn_fs_x__path_l2p_index(fs, 0, fs->pool); - SVN_ERR(svn_io_file_create_binary - (path, - "\0\1\x80\x40\1\1" /* rev 0, single page */ - "\5\4" /* page size: bytes, count */ - "\0" /* 0 container offsets in list */ - "\0\x7b\x1e\1", /* phys offsets + 1 */ - 13, - fs->pool)); - SVN_ERR(svn_io_set_file_read_only(path, FALSE, fs->pool)); + /* Construct the index P2L contents: describe the 3 items we have. + Be sure to create them in on-disk order. */ + index_entries = apr_array_make(subpool, 3, sizeof(entry)); + + entry = apr_pcalloc(subpool, sizeof(*entry)); + entry->offset = 0; + entry->size = 0x1d; + entry->type = SVN_FS_X__ITEM_TYPE_DIR_REP; + entry->item_count = 1; + entry->items = apr_pcalloc(subpool, sizeof(*entry->items)); + entry->items[0].change_set = 0; + entry->items[0].number = SVN_FS_X__ITEM_INDEX_FIRST_USER; + APR_ARRAY_PUSH(index_entries, svn_fs_x__p2l_entry_t *) = entry; + + entry = apr_pcalloc(subpool, sizeof(*entry)); + entry->offset = 0x1d; + entry->size = 0x6d; + entry->type = SVN_FS_X__ITEM_TYPE_NODEREV; + entry->item_count = 1; + entry->items = apr_pcalloc(subpool, sizeof(*entry->items)); + entry->items[0].change_set = 0; + entry->items[0].number = SVN_FS_X__ITEM_INDEX_ROOT_NODE; + APR_ARRAY_PUSH(index_entries, svn_fs_x__p2l_entry_t *) = entry; + + entry = apr_pcalloc(subpool, sizeof(*entry)); + entry->offset = 0x1d + 0x6d; + entry->size = 1; + entry->type = SVN_FS_X__ITEM_TYPE_CHANGES; + entry->item_count = 1; + entry->items = apr_pcalloc(subpool, sizeof(*entry->items)); + entry->items[0].change_set = 0; + entry->items[0].number = SVN_FS_X__ITEM_INDEX_CHANGES; + APR_ARRAY_PUSH(index_entries, svn_fs_x__p2l_entry_t *) = entry; + + /* Now re-open r0, create proto-index files from our entries and + rewrite the index section of r0. */ + SVN_ERR(svn_fs_x__open_pack_or_rev_file_writable(&rev_file, fs, 0, + subpool, subpool)); + SVN_ERR(svn_fs_x__p2l_index_from_p2l_entries(&p2l_proto_index, fs, + rev_file, index_entries, + subpool, subpool)); + SVN_ERR(svn_fs_x__l2p_index_from_p2l_entries(&l2p_proto_index, fs, + index_entries, + subpool, subpool)); + SVN_ERR(svn_fs_x__add_index_data(fs, rev_file->file, l2p_proto_index, + p2l_proto_index, 0, subpool)); + SVN_ERR(svn_fs_x__close_revision_file(rev_file)); - path = svn_fs_x__path_p2l_index(fs, 0, fs->pool); - SVN_ERR(svn_io_file_create_binary - (path, - "\0\x7b" /* start rev, rev file size */ - "\x80\x80\4\1\x21" /* 64k pages, 1 page using 33 bytes */ - "\0" /* offset entry 0 page 1 */ - /* len, type & count, checksum, - (rev, 2*item)* */ - "\x1d\x11\x8e\xef\xf2\xd6\x01\0\6" - "\x5d\x15\xb6\xea\x97\xe0\x0f\0\4" - "\1\x16\x9d\x9e\xa9\x94\x0f\0\2" - "\x85\xff\3\0\0", /* last entry fills up 64k page */ - 40, - fs->pool)); - SVN_ERR(svn_io_set_file_read_only(path, FALSE, fs->pool)); + SVN_ERR(svn_io_set_file_read_only(path_revision_zero, FALSE, fs->pool)); /* Set a date on revision 0. */ date.data = svn_time_to_cstring(apr_time_now(), fs->pool); @@ -884,40 +924,19 @@ write_revision_zero(svn_fs_t *fs) } svn_error_t * -svn_fs_x__create(svn_fs_t *fs, - const char *path, - apr_pool_t *pool) +svn_fs_x__create_file_tree(svn_fs_t *fs, + const char *path, + int format, + int shard_size, + apr_pool_t *pool) { - int format = SVN_FS_X__FORMAT_NUMBER; - fs_x_data_t *ffd = fs->fsap_data; + svn_fs_x__data_t *ffd = fs->fsap_data; - fs->path = apr_pstrdup(pool, path); - /* See if compatibility with older versions was explicitly requested. */ - if (fs->config) - { - svn_version_t *compatible_version; - SVN_ERR(svn_fs__compatible_version(&compatible_version, fs->config, - pool)); - - /* select format number */ - switch(compatible_version->minor) - { - case 0: - case 1: - case 2: - case 3: - case 4: - case 5: - case 6: - case 7: - case 8: return svn_error_create(SVN_ERR_FS_UNSUPPORTED_FORMAT, NULL, - _("FSX is not compatible with Subversion prior to 1.9")); - - default:format = SVN_FS_X__FORMAT_NUMBER; - } - } + fs->path = apr_pstrdup(fs->pool, path); ffd->format = format; - ffd->max_files_per_dir = SVN_FS_X_DEFAULT_MAX_FILES_PER_DIR; + + /* Use an appropriate sharding mode if supported by the format. */ + ffd->max_files_per_dir = shard_size; /* Create the revision data directories. */ SVN_ERR(svn_io_make_dir_recursively(svn_fs_x__path_rev_shard(fs, 0, pool), @@ -929,29 +948,31 @@ svn_fs_x__create(svn_fs_t *fs, pool)); /* Create the transaction directory. */ - SVN_ERR(svn_io_make_dir_recursively(svn_dirent_join(path, PATH_TXNS_DIR, - pool), + SVN_ERR(svn_io_make_dir_recursively(svn_fs_x__path_txns_dir(fs, pool), pool)); /* Create the protorevs directory. */ - SVN_ERR(svn_io_make_dir_recursively(svn_dirent_join(path, PATH_TXN_PROTOS_DIR, - pool), + SVN_ERR(svn_io_make_dir_recursively(svn_fs_x__path_txn_proto_revs(fs, pool), pool)); /* Create the 'current' file. */ - SVN_ERR(svn_io_file_create(svn_fs_x__path_current(fs, pool), "0\n", pool)); - SVN_ERR(svn_io_file_create_empty(svn_fs_x__path_lock(fs, pool), pool)); - SVN_ERR(svn_fs_x__set_uuid(fs, NULL, pool)); + SVN_ERR(svn_io_file_create_empty(svn_fs_x__path_current(fs, pool), pool)); + SVN_ERR(svn_fs_x__write_current(fs, 0, pool)); - SVN_ERR(write_revision_zero(fs)); + /* Create the 'uuid' file. */ + SVN_ERR(svn_io_file_create_empty(svn_fs_x__path_lock(fs, pool), pool)); + SVN_ERR(svn_fs_x__set_uuid(fs, NULL, NULL, pool)); + /* Create the fsfs.conf file. */ SVN_ERR(write_config(fs, pool)); - SVN_ERR(read_config(ffd, fs->path, fs->pool, pool)); + /* Add revision 0. */ + SVN_ERR(write_revision_zero(fs, pool)); + /* Create the min unpacked rev file. */ SVN_ERR(svn_io_file_create(svn_fs_x__path_min_unpacked_rev(fs, pool), - "0\n", pool)); + "0\n", pool)); /* Create the txn-current file if the repository supports the transaction sequence file. */ @@ -960,6 +981,52 @@ svn_fs_x__create(svn_fs_t *fs, SVN_ERR(svn_io_file_create_empty(svn_fs_x__path_txn_current_lock(fs, pool), pool)); + /* Initialize the revprop caching info. */ + SVN_ERR(svn_fs_x__reset_revprop_generation_file(fs, pool)); + + ffd->youngest_rev_cache = 0; + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__create(svn_fs_t *fs, + const char *path, + apr_pool_t *pool) +{ + int format = SVN_FS_X__FORMAT_NUMBER; + svn_fs_x__data_t *ffd = fs->fsap_data; + + fs->path = apr_pstrdup(fs->pool, path); + /* See if compatibility with older versions was explicitly requested. */ + if (fs->config) + { + svn_version_t *compatible_version; + SVN_ERR(svn_fs__compatible_version(&compatible_version, fs->config, + pool)); + + /* select format number */ + switch(compatible_version->minor) + { + case 0: + case 1: + case 2: + case 3: + case 4: + case 5: + case 6: + case 7: + case 8: return svn_error_create(SVN_ERR_FS_UNSUPPORTED_FORMAT, NULL, + _("FSX is not compatible with Subversion prior to 1.9")); + + default:format = SVN_FS_X__FORMAT_NUMBER; + } + } + + /* Actual FS creation. */ + SVN_ERR(svn_fs_x__create_file_tree(fs, path, format, + SVN_FS_X_DEFAULT_MAX_FILES_PER_DIR, + pool)); + /* This filesystem is ready. Stamp it with a format number. */ SVN_ERR(svn_fs_x__write_format(fs, FALSE, pool)); @@ -970,28 +1037,32 @@ svn_fs_x__create(svn_fs_t *fs, svn_error_t * svn_fs_x__set_uuid(svn_fs_t *fs, const char *uuid, + const char *instance_id, apr_pool_t *pool) { - char *my_uuid; - apr_size_t my_uuid_len; + svn_fs_x__data_t *ffd = fs->fsap_data; const char *uuid_path = svn_fs_x__path_uuid(fs, pool); + svn_stringbuf_t *contents = svn_stringbuf_create_empty(pool); if (! uuid) uuid = svn_uuid_generate(pool); - /* Make sure we have a copy in FS->POOL, and append a newline. */ - my_uuid = apr_pstrcat(fs->pool, uuid, "\n", SVN_VA_NULL); - my_uuid_len = strlen(my_uuid); + if (! instance_id) + instance_id = svn_uuid_generate(pool); + + svn_stringbuf_appendcstr(contents, uuid); + svn_stringbuf_appendcstr(contents, "\n"); + svn_stringbuf_appendcstr(contents, instance_id); + svn_stringbuf_appendcstr(contents, "\n"); /* We use the permissions of the 'current' file, because the 'uuid' file does not exist during repository creation. */ - SVN_ERR(svn_io_write_atomic(uuid_path, my_uuid, my_uuid_len, + SVN_ERR(svn_io_write_atomic(uuid_path, contents->data, contents->len, svn_fs_x__path_current(fs, pool) /* perms */, pool)); - /* Remove the newline we added, and stash the UUID. */ - my_uuid[my_uuid_len - 1] = '\0'; - fs->uuid = my_uuid; + fs->uuid = apr_pstrdup(fs->pool, uuid); + ffd->instance_id = apr_pstrdup(fs->pool, instance_id); return SVN_NO_ERROR; } @@ -1109,7 +1180,7 @@ svn_fs_x__info_format(int *fs_format, apr_pool_t *result_pool, apr_pool_t *scratch_pool) { - fs_x_data_t *ffd = fs->fsap_data; + svn_fs_x__data_t *ffd = fs->fsap_data; *fs_format = ffd->format; *supports_version = apr_palloc(result_pool, sizeof(svn_version_t)); Modified: subversion/branches/authzperf/subversion/libsvn_fs_x/fs_x.h URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_x/fs_x.h?rev=1649205&r1=1649204&r2=1649205&view=diff ============================================================================== --- subversion/branches/authzperf/subversion/libsvn_fs_x/fs_x.h (original) +++ subversion/branches/authzperf/subversion/libsvn_fs_x/fs_x.h Sat Jan 3 14:00:41 2015 @@ -25,6 +25,11 @@ #include "fs.h" +/* Read the 'format' file of fsx filesystem FS and store its info in FS. + * Use SCRATCH_POOL for temporary allocations. */ +svn_error_t * +svn_fs_x__read_format_file(svn_fs_t *fs, apr_pool_t *scratch_pool); + /* Open the fsx filesystem pointed to by PATH and associate it with filesystem object FS. Use POOL for temporary allocations. @@ -51,15 +56,6 @@ svn_error_t *svn_fs_x__youngest_rev(svn_ svn_fs_t *fs, apr_pool_t *pool); -/* For revision REV in fileysystem FS, open the revision (or packed rev) - file and seek to the start of the revision. Return it in *FILE, and - use POOL for allocations. */ -svn_error_t * -svn_fs_x__open_pack_or_rev_file(apr_file_t **file, - svn_fs_t *fs, - svn_revnum_t rev, - apr_pool_t *pool); - /* Return SVN_ERR_FS_NO_SUCH_REVISION if the given revision REV is newer than the current youngest revision in FS or is simply not a valid revision number, else return success. */ @@ -85,13 +81,13 @@ svn_error_t *svn_fs_x__revision_proplist /* Set *LENGTH to the be fulltext length of the node revision specified by NODEREV. Use POOL for temporary allocations. */ svn_error_t *svn_fs_x__file_length(svn_filesize_t *length, - node_revision_t *noderev, + svn_fs_x__noderev_t *noderev, apr_pool_t *pool); /* Return TRUE if the representations in A and B have equal contents, else return FALSE. */ -svn_boolean_t svn_fs_x__file_text_rep_equal(representation_t *a, - representation_t *b); +svn_boolean_t svn_fs_x__file_text_rep_equal(svn_fs_x__representation_t *a, + svn_fs_x__representation_t *b); /* Set *EQUAL to TRUE if the property representations in A and B within FS have equal contents, else set it to FALSE. If STRICT is not set, allow @@ -99,25 +95,43 @@ svn_boolean_t svn_fs_x__file_text_rep_eq Use SCRATCH_POOL for temporary allocations. */ svn_error_t *svn_fs_x__prop_rep_equal(svn_boolean_t *equal, svn_fs_t *fs, - node_revision_t *a, - node_revision_t *b, + svn_fs_x__noderev_t *a, + svn_fs_x__noderev_t *b, svn_boolean_t strict, apr_pool_t *scratch_pool); /* Return a copy of the representation REP allocated from POOL. */ -representation_t *svn_fs_x__rep_copy(representation_t *rep, - apr_pool_t *pool); +svn_fs_x__representation_t * +svn_fs_x__rep_copy(svn_fs_x__representation_t *rep, + apr_pool_t *pool); /* Return the recorded checksum of type KIND for the text representation of NODREV into CHECKSUM, allocating from POOL. If no stored checksum is available, put all NULL into CHECKSUM. */ svn_error_t *svn_fs_x__file_checksum(svn_checksum_t **checksum, - node_revision_t *noderev, + svn_fs_x__noderev_t *noderev, svn_checksum_kind_t kind, apr_pool_t *pool); +/* Under the repository db PATH, create a FSFS repository with FORMAT, + * the given SHARD_SIZE. If not supported by the respective format, + * the latter two parameters will be ignored. FS will be updated. + * + * The only file not being written is the 'format' file. This allows + * callers such as hotcopy to modify the contents before turning the + * tree into an accessible repository. + * + * Use POOL for temporary allocations. + */ +svn_error_t * +svn_fs_x__create_file_tree(svn_fs_t *fs, + const char *path, + int format, + int shard_size, + apr_pool_t *pool); + /* Create a fs_x fileysystem referenced by FS at path PATH. Get any temporary allocations from POOL. @@ -127,11 +141,12 @@ svn_error_t *svn_fs_x__create(svn_fs_t * const char *path, apr_pool_t *pool); -/* Set the uuid of repository FS to UUID, if UUID is not NULL; - otherwise, set the uuid of FS to a newly generated UUID. Perform - temporary allocations in POOL. */ +/* Set the uuid of repository FS to UUID and the instance ID to INSTANCE_ID. + If any of them is NULL, use a newly generated UUID / ID instead. + Perform temporary allocations in POOL. */ svn_error_t *svn_fs_x__set_uuid(svn_fs_t *fs, const char *uuid, + const char *instance_id, apr_pool_t *pool); /* Set *PATH to the path of REV in FS, whether in a pack file or not. Modified: subversion/branches/authzperf/subversion/libsvn_fs_x/hotcopy.c URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_x/hotcopy.c?rev=1649205&r1=1649204&r2=1649205&view=diff ============================================================================== --- subversion/branches/authzperf/subversion/libsvn_fs_x/hotcopy.c (original) +++ subversion/branches/authzperf/subversion/libsvn_fs_x/hotcopy.c Sat Jan 3 14:00:41 2015 @@ -36,9 +36,13 @@ #include "svn_private_config.h" /* Like svn_io_dir_file_copy(), but doesn't copy files that exist at - * the destination and do not differ in terms of kind, size, and mtime. */ + * the destination and do not differ in terms of kind, size, and mtime. + * Set *SKIPPED_P to FALSE only if the file was copied, do not change + * the value in *SKIPPED_P otherwise. SKIPPED_P may be NULL if not + * required. */ static svn_error_t * -hotcopy_io_dir_file_copy(const char *src_path, +hotcopy_io_dir_file_copy(svn_boolean_t *skipped_p, + const char *src_path, const char *dst_path, const char *file, apr_pool_t *scratch_pool) @@ -66,6 +70,9 @@ hotcopy_io_dir_file_copy(const char *src return SVN_NO_ERROR; } + if (skipped_p) + *skipped_p = FALSE; + return svn_error_trace(svn_io_dir_file_copy(src_path, dst_path, file, scratch_pool)); } @@ -107,9 +114,12 @@ entry_name_to_utf8(const char **name_p, /* Like svn_io_copy_dir_recursively() but doesn't copy regular files that * exist in the destination and do not differ from the source in terms of - * kind, size, and mtime. */ + * kind, size, and mtime. Set *SKIPPED_P to FALSE only if at least one + * file was copied, do not change the value in *SKIPPED_P otherwise. + * SKIPPED_P may be NULL if not required. */ static svn_error_t * -hotcopy_io_copy_dir_recursively(const char *src, +hotcopy_io_copy_dir_recursively(svn_boolean_t *skipped_p, + const char *src, const char *dst_parent, const char *dst_basename, svn_boolean_t copy_perms, @@ -175,8 +185,8 @@ hotcopy_io_copy_dir_recursively(const ch src, subpool)); if (this_entry.filetype == APR_REG) /* regular file */ { - SVN_ERR(hotcopy_io_dir_file_copy(src, dst_path, entryname_utf8, - subpool)); + SVN_ERR(hotcopy_io_dir_file_copy(skipped_p, src, dst_path, + entryname_utf8, subpool)); } else if (this_entry.filetype == APR_LNK) /* symlink */ { @@ -199,7 +209,8 @@ hotcopy_io_copy_dir_recursively(const ch continue; src_target = svn_dirent_join(src, entryname_utf8, subpool); - SVN_ERR(hotcopy_io_copy_dir_recursively(src_target, + SVN_ERR(hotcopy_io_copy_dir_recursively(skipped_p, + src_target, dst_path, entryname_utf8, copy_perms, @@ -229,14 +240,15 @@ hotcopy_io_copy_dir_recursively(const ch /* Copy an un-packed revision or revprop file for revision REV from SRC_SUBDIR * to DST_SUBDIR. Assume a sharding layout based on MAX_FILES_PER_DIR. - * If INCLUDE_INDEXES is set, copy rev index files as well. + * Set *SKIPPED_P to FALSE only if the file was copied, do not change the + * value in *SKIPPED_P otherwise. SKIPPED_P may be NULL if not required. * Use SCRATCH_POOL for temporary allocations. */ static svn_error_t * -hotcopy_copy_shard_file(const char *src_subdir, +hotcopy_copy_shard_file(svn_boolean_t *skipped_p, + const char *src_subdir, const char *dst_subdir, svn_revnum_t rev, int max_files_per_dir, - svn_boolean_t include_indexes, apr_pool_t *scratch_pool) { const char *src_subdir_shard = src_subdir, @@ -254,21 +266,10 @@ hotcopy_copy_shard_file(const char *src_ scratch_pool)); } - SVN_ERR(hotcopy_io_dir_file_copy(src_subdir_shard, dst_subdir_shard, + SVN_ERR(hotcopy_io_dir_file_copy(skipped_p, + src_subdir_shard, dst_subdir_shard, apr_psprintf(scratch_pool, "%ld", rev), scratch_pool)); - if (include_indexes) - { - SVN_ERR(hotcopy_io_dir_file_copy(src_subdir_shard, dst_subdir_shard, - apr_psprintf(scratch_pool, "%ld.l2p", - rev), - scratch_pool)); - SVN_ERR(hotcopy_io_dir_file_copy(src_subdir_shard, dst_subdir_shard, - apr_psprintf(scratch_pool, "%ld.p2l", - rev), - scratch_pool)); - } - return SVN_NO_ERROR; } @@ -277,9 +278,13 @@ hotcopy_copy_shard_file(const char *src_ * MAX_FILES_PER_DIR revisions, from SRC_FS to DST_FS. * Update *DST_MIN_UNPACKED_REV in case the shard is new in DST_FS. * Do not re-copy data which already exists in DST_FS. + * Set *SKIPPED_P to FALSE only if at least one part of the shard + * was copied, do not change the value in *SKIPPED_P otherwise. + * SKIPPED_P may be NULL if not required. * Use SCRATCH_POOL for temporary allocations. */ static svn_error_t * -hotcopy_copy_packed_shard(svn_revnum_t *dst_min_unpacked_rev, +hotcopy_copy_packed_shard(svn_boolean_t *skipped_p, + svn_revnum_t *dst_min_unpacked_rev, svn_fs_t *src_fs, svn_fs_t *dst_fs, svn_revnum_t rev, @@ -292,7 +297,7 @@ hotcopy_copy_packed_shard(svn_revnum_t * const char *src_subdir_packed_shard; svn_revnum_t revprop_rev; apr_pool_t *iterpool; - fs_x_data_t *src_ffd = src_fs->fsap_data; + svn_fs_x__data_t *src_ffd = src_fs->fsap_data; /* Copy the packed shard. */ src_subdir = svn_dirent_join(src_fs->path, PATH_REVS_DIR, scratch_pool); @@ -301,7 +306,7 @@ hotcopy_copy_packed_shard(svn_revnum_t * rev / max_files_per_dir); src_subdir_packed_shard = svn_dirent_join(src_subdir, packed_shard, scratch_pool); - SVN_ERR(hotcopy_io_copy_dir_recursively(src_subdir_packed_shard, + SVN_ERR(hotcopy_io_copy_dir_recursively(skipped_p, src_subdir_packed_shard, dst_subdir, packed_shard, TRUE /* copy_perms */, NULL /* cancel_func */, NULL, @@ -321,9 +326,9 @@ hotcopy_copy_packed_shard(svn_revnum_t * { svn_pool_clear(iterpool); - SVN_ERR(hotcopy_copy_shard_file(src_subdir, dst_subdir, + SVN_ERR(hotcopy_copy_shard_file(skipped_p, src_subdir, dst_subdir, revprop_rev, max_files_per_dir, - FALSE, iterpool)); + iterpool)); } svn_pool_destroy(iterpool); } @@ -331,8 +336,8 @@ hotcopy_copy_packed_shard(svn_revnum_t * { /* revprop for revision 0 will never be packed */ if (rev == 0) - SVN_ERR(hotcopy_copy_shard_file(src_subdir, dst_subdir, - 0, max_files_per_dir, FALSE, + SVN_ERR(hotcopy_copy_shard_file(skipped_p, src_subdir, dst_subdir, + 0, max_files_per_dir, scratch_pool)); /* packed revprops folder */ @@ -340,7 +345,8 @@ hotcopy_copy_packed_shard(svn_revnum_t * rev / max_files_per_dir); src_subdir_packed_shard = svn_dirent_join(src_subdir, packed_shard, scratch_pool); - SVN_ERR(hotcopy_io_copy_dir_recursively(src_subdir_packed_shard, + SVN_ERR(hotcopy_io_copy_dir_recursively(skipped_p, + src_subdir_packed_shard, dst_subdir, packed_shard, TRUE /* copy_perms */, NULL /* cancel_func */, NULL, @@ -359,53 +365,30 @@ hotcopy_copy_packed_shard(svn_revnum_t * return SVN_NO_ERROR; } -/* If NEW_YOUNGEST is younger than *DST_YOUNGEST, update the 'current' - * file in DST_FS and set *DST_YOUNGEST to NEW_YOUNGEST. - * Use SCRATCH_POOL for temporary allocations. */ -static svn_error_t * -hotcopy_update_current(svn_revnum_t *dst_youngest, - svn_fs_t *dst_fs, - svn_revnum_t new_youngest, - apr_pool_t *scratch_pool) -{ - if (*dst_youngest >= new_youngest) - return SVN_NO_ERROR; - - /* Update 'current'. */ - SVN_ERR(svn_fs_x__write_current(dst_fs, new_youngest, scratch_pool)); - - *dst_youngest = new_youngest; - - return SVN_NO_ERROR; -} - -/* Remove FILE in SHARD folder. Use POOL for temporary allocations. */ +/* Remove file PATH, if it exists - even if it is read-only. + * Use POOL for temporary allocations. */ static svn_error_t * -hotcopy_remove_file(const char *shard, - const char *file, +hotcopy_remove_file(const char *path, apr_pool_t *pool) { - const char *rev_path = svn_dirent_join(shard, file, pool); - /* Make the rev file writable and remove it. */ - SVN_ERR(svn_io_set_file_read_write(rev_path, TRUE, pool)); - SVN_ERR(svn_io_remove_file2(rev_path, TRUE, pool)); + SVN_ERR(svn_io_set_file_read_write(path, TRUE, pool)); + SVN_ERR(svn_io_remove_file2(path, TRUE, pool)); return SVN_NO_ERROR; } /* Remove revision or revprop files between START_REV (inclusive) and - * END_REV (non-inclusive) from folder DST_SUBDIR in DST_FS. Also, - * remove index files if REMOVE_INDEXES is set. Assume sharding as per - * MAX_FILES_PER_DIR. Use SCRATCH_POOL for temporary allocations. */ + * END_REV (non-inclusive) from folder DST_SUBDIR in DST_FS. Assume + * sharding as per MAX_FILES_PER_DIR. + * Use SCRATCH_POOL for temporary allocations. */ static svn_error_t * hotcopy_remove_files(svn_fs_t *dst_fs, const char *dst_subdir, svn_revnum_t start_rev, svn_revnum_t end_rev, int max_files_per_dir, - svn_boolean_t remove_indexes, apr_pool_t *scratch_pool) { const char *shard; @@ -430,18 +413,11 @@ hotcopy_remove_files(svn_fs_t *dst_fs, } /* remove files for REV */ - SVN_ERR(hotcopy_remove_file(dst_subdir_shard, - apr_psprintf(iterpool, "%ld", rev), + SVN_ERR(hotcopy_remove_file(svn_dirent_join(dst_subdir_shard, + apr_psprintf(iterpool, + "%ld", rev), + iterpool), iterpool)); - if (remove_indexes) - { - SVN_ERR(hotcopy_remove_file(dst_subdir_shard, - apr_psprintf(iterpool, "%ld.p2l", rev), - iterpool)); - SVN_ERR(hotcopy_remove_file(dst_subdir_shard, - apr_psprintf(iterpool, "%ld.l2p", rev), - iterpool)); - } } svn_pool_destroy(iterpool); @@ -465,7 +441,7 @@ hotcopy_remove_rev_files(svn_fs_t *dst_f PATH_REVS_DIR, scratch_pool), start_rev, end_rev, - max_files_per_dir, TRUE, scratch_pool)); + max_files_per_dir, scratch_pool)); return SVN_NO_ERROR; } @@ -489,7 +465,7 @@ hotcopy_remove_revprop_files(svn_fs_t *d PATH_REVPROPS_DIR, scratch_pool), start_rev ? start_rev : 1, end_rev, - max_files_per_dir, FALSE, scratch_pool)); + max_files_per_dir, scratch_pool)); return SVN_NO_ERROR; } @@ -501,8 +477,8 @@ hotcopy_incremental_check_preconditions( svn_fs_t *dst_fs, apr_pool_t *pool) { - fs_x_data_t *src_ffd = src_fs->fsap_data; - fs_x_data_t *dst_ffd = dst_fs->fsap_data; + svn_fs_x__data_t *src_ffd = src_fs->fsap_data; + svn_fs_x__data_t *dst_ffd = dst_fs->fsap_data; /* We only support incremental hotcopy between the same format. */ if (src_ffd->format != dst_ffd->format) @@ -552,135 +528,36 @@ remove_folder(const char *path, return svn_error_trace(err); } -/* Baton for hotcopy_body(). */ -struct hotcopy_body_baton { - svn_fs_t *src_fs; - svn_fs_t *dst_fs; - svn_boolean_t incremental; - svn_cancel_func_t cancel_func; - void *cancel_baton; -}; - -/* Perform a hotcopy, either normal or incremental. - * - * Normal hotcopy assumes that the destination exists as an empty - * directory. It behaves like an incremental hotcopy except that - * none of the copied files already exist in the destination. - * - * An incremental hotcopy copies only changed or new files to the destination, - * and removes files from the destination no longer present in the source. - * While the incremental hotcopy is running, readers should still be able - * to access the destintation repository without error and should not see - * revisions currently in progress of being copied. Readers are able to see - * new fully copied revisions even if the entire incremental hotcopy procedure - * has not yet completed. - * - * Writers are blocked out completely during the entire incremental hotcopy - * process to ensure consistency. This function assumes that the repository - * write-lock is held. +/* Copy the revision and revprop files (possibly sharded / packed) from + * SRC_FS to DST_FS. Do not re-copy data which already exists in DST_FS. + * When copying packed or unpacked shards, checkpoint the result in DST_FS + * for every shard by updating the 'current' file if necessary. Assume + * the >= SVN_FS_FS__MIN_NO_GLOBAL_IDS_FORMAT filesystem format without + * global next-ID counters. Indicate progress via the optional NOTIFY_FUNC + * callback using NOTIFY_BATON. Use POOL for temporary allocations. */ static svn_error_t * -hotcopy_body(void *baton, apr_pool_t *pool) +hotcopy_revisions(svn_fs_t *src_fs, + svn_fs_t *dst_fs, + svn_revnum_t src_youngest, + svn_revnum_t dst_youngest, + svn_boolean_t incremental, + const char *src_revs_dir, + const char *dst_revs_dir, + const char *src_revprops_dir, + const char *dst_revprops_dir, + svn_fs_hotcopy_notify_t notify_func, + void* notify_baton, + svn_cancel_func_t cancel_func, + void* cancel_baton, + apr_pool_t *pool) { - struct hotcopy_body_baton *hbb = baton; - svn_fs_t *src_fs = hbb->src_fs; - fs_x_data_t *src_ffd = src_fs->fsap_data; - svn_fs_t *dst_fs = hbb->dst_fs; + svn_fs_x__data_t *src_ffd = src_fs->fsap_data; int max_files_per_dir = src_ffd->max_files_per_dir; - svn_boolean_t incremental = hbb->incremental; - svn_cancel_func_t cancel_func = hbb->cancel_func; - void* cancel_baton = hbb->cancel_baton; - svn_revnum_t src_youngest; - svn_revnum_t dst_youngest; - svn_revnum_t rev; svn_revnum_t src_min_unpacked_rev; svn_revnum_t dst_min_unpacked_rev; - const char *src_subdir; - const char *dst_subdir; - const char *revprop_src_subdir; - const char *revprop_dst_subdir; + svn_revnum_t rev; apr_pool_t *iterpool; - svn_node_kind_t kind; - - /* Try to copy the config. - * - * ### We try copying the config file before doing anything else, - * ### because higher layers will abort the hotcopy if we throw - * ### an error from this function, and that renders the hotcopy - * ### unusable anyway. */ - svn_error_t *err; - - err = svn_io_dir_file_copy(src_fs->path, dst_fs->path, PATH_CONFIG, - pool); - if (err) - { - if (APR_STATUS_IS_ENOENT(err->apr_err)) - { - /* 1.6.0 to 1.6.11 did not copy the configuration file during - * hotcopy. So if we're hotcopying a repository which has been - * created as a hotcopy itself, it's possible that fsx.conf - * does not exist. Ask the user to re-create it. - * - * ### It would be nice to make this a non-fatal error, - * ### but this function does not get an svn_fs_t object - * ### so we have no way of just printing a warning via - * ### the fs->warning() callback. */ - - const char *msg; - const char *src_abspath; - const char *dst_abspath; - const char *config_relpath; - svn_error_t *err2; - - config_relpath = svn_dirent_join(src_fs->path, PATH_CONFIG, pool); - err2 = svn_dirent_get_absolute(&src_abspath, src_fs->path, pool); - if (err2) - return svn_error_trace(svn_error_compose_create(err, err2)); - err2 = svn_dirent_get_absolute(&dst_abspath, dst_fs->path, pool); - if (err2) - return svn_error_trace(svn_error_compose_create(err, err2)); - - /* ### hack: strip off the 'db/' directory from paths so - * ### they make sense to the user */ - src_abspath = svn_dirent_dirname(src_abspath, pool); - dst_abspath = svn_dirent_dirname(dst_abspath, pool); - - msg = apr_psprintf(pool, - _("Failed to create hotcopy at '%s'. " - "The file '%s' is missing from the source " - "repository. Please create this file, for " - "instance by running 'svnadmin upgrade %s'"), - dst_abspath, config_relpath, src_abspath); - return svn_error_quick_wrap(err, msg); - } - else - return svn_error_trace(err); - } - - if (cancel_func) - SVN_ERR(cancel_func(cancel_baton)); - - /* Find the youngest revision in the source and destination. - * We only support hotcopies from sources with an equal or greater amount - * of revisions than the destination. - * This also catches the case where users accidentally swap the - * source and destination arguments. */ - SVN_ERR(svn_fs_x__youngest_rev(&src_youngest, src_fs, pool)); - if (incremental) - { - SVN_ERR(svn_fs_x__youngest_rev(&dst_youngest, dst_fs, pool)); - if (src_youngest < dst_youngest) - return svn_error_createf(SVN_ERR_UNSUPPORTED_FEATURE, NULL, - _("The hotcopy destination already contains more revisions " - "(%lu) than the hotcopy source contains (%lu); are source " - "and destination swapped?"), - dst_youngest, src_youngest); - } - else - dst_youngest = 0; - - if (cancel_func) - SVN_ERR(cancel_func(cancel_baton)); /* Copy the min unpacked rev, and read its value. */ SVN_ERR(svn_fs_x__read_min_unpacked_rev(&src_min_unpacked_rev, src_fs, @@ -710,30 +587,41 @@ hotcopy_body(void *baton, apr_pool_t *po * Copy the necessary rev files. */ - src_subdir = svn_dirent_join(src_fs->path, PATH_REVS_DIR, pool); - dst_subdir = svn_dirent_join(dst_fs->path, PATH_REVS_DIR, pool); - SVN_ERR(svn_io_make_dir_recursively(dst_subdir, pool)); - iterpool = svn_pool_create(pool); /* First, copy packed shards. */ for (rev = 0; rev < src_min_unpacked_rev; rev += max_files_per_dir) { + svn_boolean_t skipped = TRUE; + svn_revnum_t pack_end_rev; + svn_pool_clear(iterpool); if (cancel_func) SVN_ERR(cancel_func(cancel_baton)); /* Copy the packed shard. */ - SVN_ERR(hotcopy_copy_packed_shard(&dst_min_unpacked_rev, + SVN_ERR(hotcopy_copy_packed_shard(&skipped, &dst_min_unpacked_rev, src_fs, dst_fs, rev, max_files_per_dir, iterpool)); - /* If necessary, update 'current' to the most recent packed rev, - * so readers can see new revisions which arrived in this pack. */ - SVN_ERR(hotcopy_update_current(&dst_youngest, dst_fs, - rev + max_files_per_dir - 1, - iterpool)); + pack_end_rev = rev + max_files_per_dir - 1; + + /* Whenever this pack did not previously exist in the destination, + * update 'current' to the most recent packed rev (so readers can see + * new revisions which arrived in this pack). */ + if (pack_end_rev > dst_youngest) + { + SVN_ERR(svn_fs_x__write_current(dst_fs, pack_end_rev, iterpool)); + } + + /* When notifying about packed shards, make things simpler by either + * reporting a full revision range, i.e [pack start, pack end] or + * reporting nothing. There is one case when this approach might not + * be exact (incremental hotcopy with a pack replacing last unpacked + * revisions), but generally this is good enough. */ + if (notify_func && !skipped) + notify_func(notify_baton, rev, pack_end_rev, iterpool); /* Remove revision files which are now packed. */ if (incremental) @@ -760,90 +648,170 @@ hotcopy_body(void *baton, apr_pool_t *po if (cancel_func) SVN_ERR(cancel_func(cancel_baton)); - /* Now, copy pairs of non-packed revisions and revprop files. - * If necessary, update 'current' after copying all files from a shard. */ SVN_ERR_ASSERT(rev == src_min_unpacked_rev); SVN_ERR_ASSERT(src_min_unpacked_rev == dst_min_unpacked_rev); - revprop_src_subdir = svn_dirent_join(src_fs->path, PATH_REVPROPS_DIR, pool); - revprop_dst_subdir = svn_dirent_join(dst_fs->path, PATH_REVPROPS_DIR, pool); - SVN_ERR(svn_io_make_dir_recursively(revprop_dst_subdir, pool)); + + /* Now, copy pairs of non-packed revisions and revprop files. + * If necessary, update 'current' after copying all files from a shard. */ for (; rev <= src_youngest; rev++) { + svn_boolean_t skipped = TRUE; + svn_pool_clear(iterpool); if (cancel_func) SVN_ERR(cancel_func(cancel_baton)); + /* Copying non-packed revisions is racy in case the source repository is + * being packed concurrently with this hotcopy operation. With the pack + * lock, however, the race is impossible, because hotcopy and pack + * operations block each other. + * + * We assume that all revisions coming after 'min-unpacked-rev' really + * are unpacked and that's not necessarily true with concurrent packing. + * Don't try to be smart in this edge case, because handling it properly + * might require copying *everything* from the start. Just abort the + * hotcopy with an ENOENT (revision file moved to a pack, so it is no + * longer where we expect it to be). */ + /* Copy the rev file. */ - err = hotcopy_copy_shard_file(src_subdir, dst_subdir, - rev, max_files_per_dir, TRUE, - iterpool); - if (err) + SVN_ERR(hotcopy_copy_shard_file(&skipped, src_revs_dir, dst_revs_dir, + rev, max_files_per_dir, + iterpool)); + + /* Copy the revprop file. */ + SVN_ERR(hotcopy_copy_shard_file(&skipped, src_revprops_dir, + dst_revprops_dir, + rev, max_files_per_dir, + iterpool)); + + /* Whenever this revision did not previously exist in the destination, + * checkpoint the progress via 'current' (do that once per full shard + * in order not to slow things down). */ + if (rev > dst_youngest) { - if (APR_STATUS_IS_ENOENT(err->apr_err)) + if (max_files_per_dir && (rev % max_files_per_dir == 0)) { - svn_error_clear(err); - - /* The source rev file does not exist. This can happen if the - * source repository is being packed concurrently with this - * hotcopy operation. - * - * If the new revision is now packed, and the youngest revision - * we're interested in is not inside this pack, try to copy the - * pack instead. - * - * If the youngest revision ended up being packed, don't try - * to be smart and work around this. Just abort the hotcopy. */ - SVN_ERR(svn_fs_x__update_min_unpacked_rev(src_fs, pool)); - if (svn_fs_x__is_packed_rev(src_fs, rev)) - { - if (svn_fs_x__is_packed_rev(src_fs, src_youngest)) - return svn_error_createf( - SVN_ERR_FS_NO_SUCH_REVISION, NULL, - _("The assumed HEAD revision (%lu) of the " - "hotcopy source has been packed while the " - "hotcopy was in progress; please restart " - "the hotcopy operation"), - src_youngest); - - SVN_ERR(hotcopy_copy_packed_shard(&dst_min_unpacked_rev, - src_fs, dst_fs, - rev, max_files_per_dir, - iterpool)); - rev = dst_min_unpacked_rev; - continue; - } - else - return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL, - _("Revision %lu disappeared from the " - "hotcopy source while hotcopy was " - "in progress"), rev); + SVN_ERR(svn_fs_x__write_current(dst_fs, rev, iterpool)); } - else - return svn_error_trace(err); } - /* Copy the revprop file. */ - SVN_ERR(hotcopy_copy_shard_file(revprop_src_subdir, - revprop_dst_subdir, - rev, max_files_per_dir, FALSE, - iterpool)); - - /* After completing a full shard, update 'current'. */ - if (max_files_per_dir && rev % max_files_per_dir == 0) - SVN_ERR(hotcopy_update_current(&dst_youngest, dst_fs, rev, iterpool)); + if (notify_func && !skipped) + notify_func(notify_baton, rev, rev, iterpool); } svn_pool_destroy(iterpool); - if (cancel_func) - SVN_ERR(cancel_func(cancel_baton)); - /* We assume that all revisions were copied now, i.e. we didn't exit the * above loop early. 'rev' was last incremented during exit of the loop. */ SVN_ERR_ASSERT(rev == src_youngest + 1); - /* All revisions were copied. Update 'current'. */ - SVN_ERR(hotcopy_update_current(&dst_youngest, dst_fs, src_youngest, pool)); + return SVN_NO_ERROR; +} + +/* Baton for hotcopy_body(). */ +struct hotcopy_body_baton { + svn_fs_t *src_fs; + svn_fs_t *dst_fs; + svn_boolean_t incremental; + svn_fs_hotcopy_notify_t notify_func; + void *notify_baton; + svn_cancel_func_t cancel_func; + void *cancel_baton; +}; + +/* Perform a hotcopy, either normal or incremental. + * + * Normal hotcopy assumes that the destination exists as an empty + * directory. It behaves like an incremental hotcopy except that + * none of the copied files already exist in the destination. + * + * An incremental hotcopy copies only changed or new files to the destination, + * and removes files from the destination no longer present in the source. + * While the incremental hotcopy is running, readers should still be able + * to access the destintation repository without error and should not see + * revisions currently in progress of being copied. Readers are able to see + * new fully copied revisions even if the entire incremental hotcopy procedure + * has not yet completed. + * + * Writers are blocked out completely during the entire incremental hotcopy + * process to ensure consistency. This function assumes that the repository + * write-lock is held. + */ +static svn_error_t * +hotcopy_body(void *baton, apr_pool_t *pool) +{ + struct hotcopy_body_baton *hbb = baton; + svn_fs_t *src_fs = hbb->src_fs; + svn_fs_t *dst_fs = hbb->dst_fs; + svn_boolean_t incremental = hbb->incremental; + svn_fs_hotcopy_notify_t notify_func = hbb->notify_func; + void* notify_baton = hbb->notify_baton; + svn_cancel_func_t cancel_func = hbb->cancel_func; + void* cancel_baton = hbb->cancel_baton; + svn_revnum_t src_youngest; + svn_revnum_t dst_youngest; + const char *src_revprops_dir; + const char *dst_revprops_dir; + const char *src_revs_dir; + const char *dst_revs_dir; + const char *src_subdir; + const char *dst_subdir; + svn_node_kind_t kind; + + /* Try to copy the config. + * + * ### We try copying the config file before doing anything else, + * ### because higher layers will abort the hotcopy if we throw + * ### an error from this function, and that renders the hotcopy + * ### unusable anyway. */ + SVN_ERR(svn_io_dir_file_copy(src_fs->path, dst_fs->path, PATH_CONFIG, + pool)); + + if (cancel_func) + SVN_ERR(cancel_func(cancel_baton)); + + /* Find the youngest revision in the source and destination. + * We only support hotcopies from sources with an equal or greater amount + * of revisions than the destination. + * This also catches the case where users accidentally swap the + * source and destination arguments. */ + SVN_ERR(svn_fs_x__read_current(&src_youngest, src_fs, pool)); + if (incremental) + { + SVN_ERR(svn_fs_x__youngest_rev(&dst_youngest, dst_fs, pool)); + if (src_youngest < dst_youngest) + return svn_error_createf(SVN_ERR_UNSUPPORTED_FEATURE, NULL, + _("The hotcopy destination already contains more revisions " + "(%lu) than the hotcopy source contains (%lu); are source " + "and destination swapped?"), + dst_youngest, src_youngest); + } + else + dst_youngest = 0; + + src_revs_dir = svn_dirent_join(src_fs->path, PATH_REVS_DIR, pool); + dst_revs_dir = svn_dirent_join(dst_fs->path, PATH_REVS_DIR, pool); + src_revprops_dir = svn_dirent_join(src_fs->path, PATH_REVPROPS_DIR, pool); + dst_revprops_dir = svn_dirent_join(dst_fs->path, PATH_REVPROPS_DIR, pool); + + /* Ensure that the required folders exist in the destination + * before actually copying the revisions and revprops. */ + SVN_ERR(svn_io_make_dir_recursively(dst_revs_dir, pool)); + SVN_ERR(svn_io_make_dir_recursively(dst_revprops_dir, pool)); + + if (cancel_func) + SVN_ERR(cancel_func(cancel_baton)); + + /* Split the logic for new and old FS formats. The latter is much simpler + * due to the absense of sharding and packing. However, it requires special + * care when updating the 'current' file (which contains not just the + * revision number, but also the next-ID counters). */ + SVN_ERR(hotcopy_revisions(src_fs, dst_fs, src_youngest, dst_youngest, + incremental, src_revs_dir, dst_revs_dir, + src_revprops_dir, dst_revprops_dir, + notify_func, notify_baton, + cancel_func, cancel_baton, pool)); + SVN_ERR(svn_fs_x__write_current(dst_fs, src_youngest, pool)); /* Replace the locks tree. * This is racy in case readers are currently trying to list locks in @@ -863,7 +831,7 @@ hotcopy_body(void *baton, apr_pool_t *po src_subdir = svn_dirent_join(src_fs->path, PATH_NODE_ORIGINS_DIR, pool); SVN_ERR(svn_io_check_path(src_subdir, &kind, pool)); if (kind == svn_node_dir) - SVN_ERR(hotcopy_io_copy_dir_recursively(src_subdir, dst_fs->path, + SVN_ERR(hotcopy_io_copy_dir_recursively(NULL, src_subdir, dst_fs->path, PATH_NODE_ORIGINS_DIR, TRUE, cancel_func, cancel_baton, pool)); @@ -879,8 +847,10 @@ hotcopy_body(void *baton, apr_pool_t *po SVN_ERR(svn_io_check_path(src_subdir, &kind, pool)); if (kind == svn_node_file) { + /* Copy the rep cache and then remove entries for revisions + * that did not make it into the destination. */ SVN_ERR(svn_sqlite__hotcopy(src_subdir, dst_subdir, pool)); - SVN_ERR(svn_fs_x__del_rep_reference(dst_fs, dst_youngest, pool)); + SVN_ERR(svn_fs_x__del_rep_reference(dst_fs, src_youngest, pool)); } /* Copy the txn-current file. */ @@ -891,17 +861,13 @@ hotcopy_body(void *baton, apr_pool_t *po * reset it to zero (since this is on a different path, it will not * overlap with data already in cache). Also, clean up stale files * used for the named atomics implementation. */ - SVN_ERR(svn_io_check_path(svn_fs_x__path_revprop_generation(src_fs, pool), - &kind, pool)); - if (kind == svn_node_file) - SVN_ERR(svn_fs_x__write_revprop_generation_file(dst_fs, 0, pool)); - - SVN_ERR(svn_fs_x__cleanup_revprop_namespace(dst_fs)); + SVN_ERR(svn_fs_x__reset_revprop_generation_file(dst_fs, pool)); return SVN_NO_ERROR; } -/* Wrapper around hotcopy_body taking out all necessary source repositories. +/* Wrapper around hotcopy_body taking out all necessary source repository + * locks. */ static svn_error_t * hotcopy_locking_src_body(void *baton, apr_pool_t *pool) @@ -921,57 +887,25 @@ hotcopy_create_empty_dest(svn_fs_t *src_ const char *dst_path, apr_pool_t *pool) { - fs_x_data_t *src_ffd = src_fs->fsap_data; - fs_x_data_t *dst_ffd = dst_fs->fsap_data; - - dst_fs->path = apr_pstrdup(pool, dst_path); - - dst_ffd->max_files_per_dir = src_ffd->max_files_per_dir; - dst_ffd->format = src_ffd->format; + svn_fs_x__data_t *src_ffd = src_fs->fsap_data; - /* Create the revision data directories. */ - SVN_ERR(svn_io_make_dir_recursively(svn_fs_x__path_rev_shard(dst_fs, 0, - pool), - pool)); - - /* Create the revprops directory. */ - SVN_ERR(svn_io_make_dir_recursively(svn_fs_x__path_revprops_shard(dst_fs, - 0, pool), - pool)); - - /* Create the transaction directory. */ - SVN_ERR(svn_io_make_dir_recursively(svn_dirent_join(dst_path, PATH_TXNS_DIR, - pool), - pool)); - - /* Create the protorevs directory. */ - SVN_ERR(svn_io_make_dir_recursively(svn_dirent_join(dst_path, - PATH_TXN_PROTOS_DIR, - pool), - pool)); - - /* Create the 'current' file. */ - SVN_ERR(svn_io_file_create(svn_fs_x__path_current(dst_fs, pool), "0\n", - pool)); - - /* Create lock file and UUID. */ - SVN_ERR(svn_io_file_create_empty(svn_fs_x__path_lock(dst_fs, pool), pool)); - SVN_ERR(svn_fs_x__set_uuid(dst_fs, src_fs->uuid, pool)); - - /* Create the min unpacked rev file. */ - SVN_ERR(svn_io_file_create(svn_fs_x__path_min_unpacked_rev(dst_fs, pool), - "0\n", pool)); - - /* Create the txn-current file if the repository supports - the transaction sequence file. */ - SVN_ERR(svn_io_file_create(svn_fs_x__path_txn_current(dst_fs, pool), - "0\n", pool)); - SVN_ERR(svn_io_file_create_empty(svn_fs_x__path_txn_current_lock(dst_fs, - pool), - pool)); - - /* FS creation is complete. Stamp it with a format file. */ - SVN_ERR(svn_fs_x__write_format(dst_fs, TRUE, pool)); + /* Create the DST_FS repository with the same layout as SRC_FS. */ + SVN_ERR(svn_fs_x__create_file_tree(dst_fs, dst_path, src_ffd->format, + src_ffd->max_files_per_dir, pool)); + + /* Copy the UUID. Hotcopy destination receives a new instance ID, but + * has the same filesystem UUID as the source. */ + SVN_ERR(svn_fs_x__set_uuid(dst_fs, src_fs->uuid, NULL, pool)); + + /* Remove revision 0 contents. Otherwise, it may not get overwritten + * due to having a newer timestamp. */ + SVN_ERR(hotcopy_remove_file(svn_fs_x__path_rev(dst_fs, 0, pool), pool)); + SVN_ERR(hotcopy_remove_file(svn_fs_x__path_revprops(dst_fs, 0, pool), + pool)); + + /* This filesystem is ready. Stamp it with a format number. Fail if + * the 'format' file should already exist. */ + SVN_ERR(svn_fs_x__write_format(dst_fs, FALSE, pool)); return SVN_NO_ERROR; } @@ -1020,6 +954,8 @@ svn_error_t * svn_fs_x__hotcopy(svn_fs_t *src_fs, svn_fs_t *dst_fs, svn_boolean_t incremental, + svn_fs_hotcopy_notify_t notify_func, + void *notify_baton, svn_cancel_func_t cancel_func, void *cancel_baton, apr_pool_t *pool) @@ -1029,10 +965,12 @@ svn_fs_x__hotcopy(svn_fs_t *src_fs, hbb.src_fs = src_fs; hbb.dst_fs = dst_fs; hbb.incremental = incremental; + hbb.notify_func = notify_func; + hbb.notify_baton = notify_baton; hbb.cancel_func = cancel_func; hbb.cancel_baton = cancel_baton; - SVN_ERR(svn_fs_x__with_write_lock(dst_fs, hotcopy_locking_src_body, &hbb, - pool)); + SVN_ERR(svn_fs_x__with_all_locks(dst_fs, hotcopy_locking_src_body, &hbb, + pool)); return SVN_NO_ERROR; }
