Author: hwright Date: Thu Aug 11 02:25:10 2011 New Revision: 1156447 URL: http://svn.apache.org/viewvc?rev=1156447&view=rev Log: On the fs-py branch: Implement the "set_revision_proplist" functionality in Python, rather than C. This is something of a checkpoint commit, as most of the tests start failing after it.
* subversion/python/svn/fs.py (): A number of new constants. (FS.__path_rev_shard, FS.__path_revprops_shard, FS.__path_rev_absolute, FS.__path_rev, FS.__is_pathed_rev, FS.__read_current, FS._get_youngest, FS.__ensure_revision_exists, FS._set_revision_proplist, FS.__read_format, FS.__update_min_unpacked_rev): New. (FS._create_fs): Set a couple of values, determine the right format to create. (FS._open_fs): Read the format and a couple other values upon open. (FS.__setup_paths): Initialize a couple more paths. * subversion/python/svn/__init__.py (is_valid_revnum): New. * subversion/python/svn/hash.py: New. * subversion/libsvn_fs_py/fs_fs.c (set_revision_proplist): Rip out the implementation in favor of a python one. (svn_fs_py__create): Track a rename in a helper function. * subversion/libsvn_fs_py/py_util.c (convert_hash): New. (convert_svn_string_t): New. (svn_fs_py__convert_hash): Renamed to... (svn_fs_py__convert_cstring_hash): ...this. (svn_fs_py__convert_proplist): New. * subversion/libsvn_fs_py/py_util.h (svn_fs_py__convert_hash): Renamed to... (svn_fs_py__convert_cstring_hash): ...this. (svn_fs_py__convert_proplist): New. Added: subversion/branches/fs-py/subversion/python/svn/hash.py (with props) Modified: subversion/branches/fs-py/subversion/libsvn_fs_py/fs_fs.c subversion/branches/fs-py/subversion/libsvn_fs_py/py_util.c subversion/branches/fs-py/subversion/libsvn_fs_py/py_util.h subversion/branches/fs-py/subversion/python/svn/__init__.py subversion/branches/fs-py/subversion/python/svn/fs.py Modified: subversion/branches/fs-py/subversion/libsvn_fs_py/fs_fs.c URL: http://svn.apache.org/viewvc/subversion/branches/fs-py/subversion/libsvn_fs_py/fs_fs.c?rev=1156447&r1=1156446&r2=1156447&view=diff ============================================================================== --- subversion/branches/fs-py/subversion/libsvn_fs_py/fs_fs.c (original) +++ subversion/branches/fs-py/subversion/libsvn_fs_py/fs_fs.c Thu Aug 11 02:25:10 2011 @@ -2850,32 +2850,11 @@ set_revision_proplist(svn_fs_t *fs, apr_hash_t *proplist, apr_pool_t *pool) { - SVN_ERR(ensure_revision_exists(fs, rev, pool)); - - if (1) - { - const char *final_path = path_revprops(fs, rev, pool); - const char *tmp_path; - const char *perms_reference; - svn_stream_t *stream; - - /* ### do we have a directory sitting around already? we really shouldn't - ### have to get the dirname here. */ - SVN_ERR(svn_stream_open_unique(&stream, &tmp_path, - svn_dirent_dirname(final_path, pool), - svn_io_file_del_none, pool, pool)); - SVN_ERR(svn_hash_write2(proplist, stream, SVN_HASH_TERMINATOR, pool)); - SVN_ERR(svn_stream_close(stream)); - - /* We use the rev file of this revision as the perms reference, - because when setting revprops for the first time, the revprop - file won't exist and therefore can't serve as its own reference. - (Whereas the rev file should already exist at this point.) */ - SVN_ERR(svn_fs_py__path_rev_absolute(&perms_reference, fs, rev, pool)); - SVN_ERR(move_into_place(tmp_path, final_path, perms_reference, pool)); + fs_fs_data_t *ffd = fs->fsap_data; - return SVN_NO_ERROR; - } + SVN_ERR(svn_fs_py__call_method(NULL, ffd->p_fs, "_set_revision_proplist", + "(lO&)", rev, + svn_fs_py__convert_proplist, proplist)); return SVN_NO_ERROR; } @@ -6257,7 +6236,7 @@ svn_fs_py__create(svn_fs_t *fs, SVN_ERR(svn_fs_py__call_method(&ffd->p_fs, ffd->p_module, "_create_fs", "(sO&)", path, - svn_fs_py__convert_hash, fs->config)); + svn_fs_py__convert_cstring_hash, fs->config)); apr_pool_cleanup_register(fs->pool, ffd->p_fs, svn_fs_py__destroy_py_object, apr_pool_cleanup_null); Modified: subversion/branches/fs-py/subversion/libsvn_fs_py/py_util.c URL: http://svn.apache.org/viewvc/subversion/branches/fs-py/subversion/libsvn_fs_py/py_util.c?rev=1156447&r1=1156446&r2=1156447&view=diff ============================================================================== --- subversion/branches/fs-py/subversion/libsvn_fs_py/py_util.c (original) +++ subversion/branches/fs-py/subversion/libsvn_fs_py/py_util.c Thu Aug 11 02:25:10 2011 @@ -378,17 +378,18 @@ svn_fs_py__call_method(PyObject **p_resu return svn_error_trace(err); } -PyObject * -svn_fs_py__convert_hash(void *object) + +static PyObject * +convert_hash(apr_hash_t *hash, + PyObject *(*converter_func)(void *value)) { - apr_hash_t *hash = object; apr_hash_index_t *hi; - PyObject *p_dict; + PyObject *dict; if (hash == NULL) Py_RETURN_NONE; - if ((p_dict = PyDict_New()) == NULL) + if ((dict = PyDict_New()) == NULL) return NULL; for (hi = apr_hash_first(NULL, hash); hi; hi = apr_hash_next(hi)) @@ -398,23 +399,44 @@ svn_fs_py__convert_hash(void *object) PyObject *value; apr_hash_this(hi, &key, NULL, &val); - value = PyString_FromString(val); + value = (*converter_func)(val); if (value == NULL) { - Py_DECREF(p_dict); + Py_DECREF(dict); return NULL; } /* ### gotta cast this thing cuz Python doesn't use "const" */ - if (PyDict_SetItemString(p_dict, (char *)key, value) == -1) + if (PyDict_SetItemString(dict, (char *)key, value) == -1) { Py_DECREF(value); - Py_DECREF(p_dict); + Py_DECREF(dict); return NULL; } Py_DECREF(value); } - return p_dict; + return dict; +} + +static PyObject * +convert_svn_string_t(void *value) +{ + const svn_string_t *s = value; + + /* ### gotta cast this thing cuz Python doesn't use "const" */ + return PyString_FromStringAndSize((void *)s->data, s->len); +} + +PyObject * +svn_fs_py__convert_cstring_hash(void *object) +{ + return convert_hash(object, PyString_FromString); +} + +PyObject * +svn_fs_py__convert_proplist(void *object) +{ + return convert_hash(object, convert_svn_string_t); } svn_error_t * Modified: subversion/branches/fs-py/subversion/libsvn_fs_py/py_util.h URL: http://svn.apache.org/viewvc/subversion/branches/fs-py/subversion/libsvn_fs_py/py_util.h?rev=1156447&r1=1156446&r2=1156447&view=diff ============================================================================== --- subversion/branches/fs-py/subversion/libsvn_fs_py/py_util.h (original) +++ subversion/branches/fs-py/subversion/libsvn_fs_py/py_util.h Thu Aug 11 02:25:10 2011 @@ -41,7 +41,10 @@ svn_fs_py__call_method(PyObject **p_resu ...); PyObject * -svn_fs_py__convert_hash(void *object); +svn_fs_py__convert_cstring_hash(void *object); + +PyObject * +svn_fs_py__convert_proplist(void *object); /* Load a reference to the FS Python module into the shared data. */ svn_error_t * Modified: subversion/branches/fs-py/subversion/python/svn/__init__.py URL: http://svn.apache.org/viewvc/subversion/branches/fs-py/subversion/python/svn/__init__.py?rev=1156447&r1=1156446&r2=1156447&view=diff ============================================================================== --- subversion/branches/fs-py/subversion/python/svn/__init__.py (original) +++ subversion/branches/fs-py/subversion/python/svn/__init__.py Thu Aug 11 02:25:10 2011 @@ -26,3 +26,7 @@ class SubversionException(Exception): def __str__(self): return self.message + + +def is_valid_revnum(rev): + return rev >= 0 Modified: subversion/branches/fs-py/subversion/python/svn/fs.py URL: http://svn.apache.org/viewvc/subversion/branches/fs-py/subversion/python/svn/fs.py?rev=1156447&r1=1156446&r2=1156447&view=diff ============================================================================== --- subversion/branches/fs-py/subversion/python/svn/fs.py (original) +++ subversion/branches/fs-py/subversion/python/svn/fs.py Thu Aug 11 02:25:10 2011 @@ -18,15 +18,74 @@ # specific language governing permissions and limitations # under the License. -import os, uuid, shutil +import os, uuid, shutil, tempfile +import svn +import svn.hash # Some constants +CONFIG_PRE_1_4_COMPATIBLE = "pre-1.4-compatible" +CONFIG_PRE_1_5_COMPATIBLE = "pre-1.5-compatible" +CONFIG_PRE_1_6_COMPATIBLE = "pre-1.6-compatible" + +PATH_FORMAT = "format" # Contains format number PATH_UUID = "uuid" # Contains UUID PATH_CURRENT = "current" # Youngest revision +PATH_REVS_DIR = "revs" # Directory of revisions +PATH_REVPROPS_DIR = "revprops" # Directory of revprops +PATH_MIN_UNPACKED_REV = "min-unpacked-rev" # Oldest revision which + # has not been packed. + +FORMAT_NUMBER = 4 + +MIN_LAYOUT_FORMAT_OPTION_FORMAT = 3 +MIN_PROTOREVS_DIR_FORMAT = 3 +MIN_NO_GLOBAL_IDS_FORMAT = 3 +MIN_PACKED_FORMAT = 4 + +_DEFAULT_MAX_FILES_PER_DIR = 1000 class FS(object): + def __path_rev_shard(self, rev): + assert self.max_files_per_dir + return os.path.join(self.path, PATH_REVS_DIR, + '%d' % (rev // self.max_files_per_dir) ) + + def __path_revprops_shard(self, rev): + assert self.max_files_per_dir + return os.path.join(self.path, PATH_REVPROPS_DIR, + '%d' % (rev // self.max_files_per_dir) ) + + def __path_revprops(self, rev): + if self.max_files_per_dir: + return os.path.join(self.__path_revprops_shard(rev), str(rev)) + else: + return os.path.join(self.path, PATH_REVPROPS_DIR, str(rev)) + + def __path_rev_absolute(self, rev): + if self.format < MIN_PACKED_FORMAT or not self.__is_packed_rev(rev): + return self.__path_rev(rev) + else: + return self.__path_rev_packed(rev, "pack") + + def __path_rev(self, rev): + assert not self.__is_packed_rev(rev) + if self.max_files_per_dir: + return os.path.join(self.__path_rev_shard(rev), str(rev)) + else: + return os.path.join(self.path, PATH_REVS_DIR, str(rev)) + + def __is_packed_rev(self, rev): + return rev < self.__min_unpacked_rev + + def __read_current(self): + with open(self.__path_current, 'rb') as f: + return int(f.readline()) + + def _get_youngest(self): + return self.__read_current() + def set_uuid(self, uuid_in = None): '''Set the UUID for the filesystem. If UUID_IN is not given, generate a new one a la RFC 4122.''' @@ -43,8 +102,80 @@ class FS(object): shutil.copymode(self.__path_current, self.__path_uuid) + def __ensure_revision_exists(self, rev): + if not svn.is_valid_revnum(rev): + raise svn.SubversionException(svn.err.FS_NO_SUCH_REVISION, + _("Invalid revision number '%ld'") % rev) + + # Did the revision exist the last time we checked the current file? + if rev <= self.__youngest_rev_cache: + return + + # Check again + self.__youngest_rev_cache = self._get_youngest() + if rev <= self.__youngest_rev_cache: + return + + raise svn.SubversionException(svn.err.FS_NO_SUCH_REVISION, + _("No such revision %ld") % rev) + + def _set_revision_proplist(self, rev, props): + self.__ensure_revision_exists(rev) + + final_path = self.__path_revprops(rev) + (fd, fn) = tempfile.mkstemp(dir=os.path.dirname(final_path)) + os.write(fd, svn.hash.encode(props, svn.hash.TERMINATOR)) + os.close(fd) + shutil.copystat(self.__path_rev_absolute(rev), fn) + + os.rename(fn, final_path) + + + def __read_format(self): + try: + with open(self.__path_format, 'rb') as f: + self.format = int(f.readline()) + self.max_files_per_dir = 0 + for l in f: + l = l.split() + if self.format > MIN_LAYOUT_FORMAT_OPTION_FORMAT \ + and l[0] == 'layout': + if l[1] == 'linear': + self.max_files_per_dir = 0 + elif l[1] == 'sharded': + self.max_files_per_dir = int(l[2]) + except IOError: + # Treat an absent format file as format 1. + self.format = 1 + self.max_files_per_dir = 0 + + + def __update_min_unpacked_rev(self): + assert self.format >= MIN_PACKED_FORMAT + with open(self.__path_min_unpacked_rev, 'rb') as f: + self.__min_unpacked_rev = int(f.readline()) + + def _create_fs(self): 'Create a new Subversion filesystem' + self.__youngest_rev_cache = 0 + self.__min_unpacked_rev = 0 + + # See if compatibility with older versions was explicitly requested. + if CONFIG_PRE_1_4_COMPATIBLE in self._config: + self.format = 1 + elif CONFIG_PRE_1_5_COMPATIBLE in self._config: + self.format = 2 + elif CONFIG_PRE_1_6_COMPATIBLE in self._config: + self.format = 3 + else: + self.format = FORMAT_NUMBER + + # Override the default linear layout if this is a new-enough format. + if self.format >= MIN_LAYOUT_FORMAT_OPTION_FORMAT: + self.max_files_per_dir = _DEFAULT_MAX_FILES_PER_DIR + else: + self.max_files_per_dir = 0 def _open_fs(self): @@ -52,10 +183,18 @@ class FS(object): with open(self.__path_uuid, 'rb') as f: self.uuid = uuid.UUID(f.readline().rstrip()) + self.__youngest_rev_cache = self.__read_current() + self.__read_format() + if self.format >= MIN_PACKED_FORMAT: + self.__update_min_unpacked_rev() + def __setup_paths(self): self.__path_uuid = os.path.join(self.path, PATH_UUID) self.__path_current = os.path.join(self.path, PATH_CURRENT) + self.__path_format = os.path.join(self.path, PATH_FORMAT) + self.__path_min_unpacked_rev = os.path.join(self.path, + PATH_MIN_UNPACKED_REV) def __init__(self, path, create=False, config=None): Added: subversion/branches/fs-py/subversion/python/svn/hash.py URL: http://svn.apache.org/viewvc/subversion/branches/fs-py/subversion/python/svn/hash.py?rev=1156447&view=auto ============================================================================== --- subversion/branches/fs-py/subversion/python/svn/hash.py (added) +++ subversion/branches/fs-py/subversion/python/svn/hash.py Thu Aug 11 02:25:10 2011 @@ -0,0 +1,31 @@ +#!/usr/bin/env python +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +TERMINATOR = "END" + +def encode(h, terminator): + output = [] + for k in sorted(h.keys()): + output.append('K %d\n%s\nV %s\n%s\n' % (len(k), k, len(h[k]), h[k])) + + if terminator: + output.append('%s\n' % terminator) + + return ''.join(output) Propchange: subversion/branches/fs-py/subversion/python/svn/hash.py ------------------------------------------------------------------------------ svn:eol-style = native