Author: hwright
Date: Thu Aug 11 02:25:10 2011
New Revision: 1156447

URL: http://svn.apache.org/viewvc?rev=1156447&view=rev
Log:
On the fs-py branch:
Implement the "set_revision_proplist" functionality in Python, rather than C.
This is something of a checkpoint commit, as most of the tests start failing
after it.

* subversion/python/svn/fs.py
  (): A number of new constants.
  (FS.__path_rev_shard, FS.__path_revprops_shard, FS.__path_rev_absolute,
   FS.__path_rev, FS.__is_pathed_rev, FS.__read_current, FS._get_youngest,
   FS.__ensure_revision_exists, FS._set_revision_proplist, FS.__read_format,
   FS.__update_min_unpacked_rev): New.
  (FS._create_fs): Set a couple of values, determine the right format to
    create.
  (FS._open_fs): Read the format and a couple other values upon open.
  (FS.__setup_paths): Initialize a couple more paths.

* subversion/python/svn/__init__.py
  (is_valid_revnum): New.

* subversion/python/svn/hash.py:
  New.

* subversion/libsvn_fs_py/fs_fs.c
  (set_revision_proplist): Rip out the implementation in favor of a python one.
  (svn_fs_py__create): Track a rename in a helper function.
 
* subversion/libsvn_fs_py/py_util.c
  (convert_hash): New.
  (convert_svn_string_t): New.
  (svn_fs_py__convert_hash): Renamed to...
  (svn_fs_py__convert_cstring_hash): ...this.
  (svn_fs_py__convert_proplist): New.

* subversion/libsvn_fs_py/py_util.h
  (svn_fs_py__convert_hash): Renamed to...
  (svn_fs_py__convert_cstring_hash): ...this.
  (svn_fs_py__convert_proplist): New.

Added:
    subversion/branches/fs-py/subversion/python/svn/hash.py   (with props)
Modified:
    subversion/branches/fs-py/subversion/libsvn_fs_py/fs_fs.c
    subversion/branches/fs-py/subversion/libsvn_fs_py/py_util.c
    subversion/branches/fs-py/subversion/libsvn_fs_py/py_util.h
    subversion/branches/fs-py/subversion/python/svn/__init__.py
    subversion/branches/fs-py/subversion/python/svn/fs.py

Modified: subversion/branches/fs-py/subversion/libsvn_fs_py/fs_fs.c
URL: 
http://svn.apache.org/viewvc/subversion/branches/fs-py/subversion/libsvn_fs_py/fs_fs.c?rev=1156447&r1=1156446&r2=1156447&view=diff
==============================================================================
--- subversion/branches/fs-py/subversion/libsvn_fs_py/fs_fs.c (original)
+++ subversion/branches/fs-py/subversion/libsvn_fs_py/fs_fs.c Thu Aug 11 
02:25:10 2011
@@ -2850,32 +2850,11 @@ set_revision_proplist(svn_fs_t *fs,
                       apr_hash_t *proplist,
                       apr_pool_t *pool)
 {
-  SVN_ERR(ensure_revision_exists(fs, rev, pool));
-
-  if (1)
-    {
-      const char *final_path = path_revprops(fs, rev, pool);
-      const char *tmp_path;
-      const char *perms_reference;
-      svn_stream_t *stream;
-
-      /* ### do we have a directory sitting around already? we really shouldn't
-         ### have to get the dirname here. */
-      SVN_ERR(svn_stream_open_unique(&stream, &tmp_path,
-                                     svn_dirent_dirname(final_path, pool),
-                                     svn_io_file_del_none, pool, pool));
-      SVN_ERR(svn_hash_write2(proplist, stream, SVN_HASH_TERMINATOR, pool));
-      SVN_ERR(svn_stream_close(stream));
-
-      /* We use the rev file of this revision as the perms reference,
-         because when setting revprops for the first time, the revprop
-         file won't exist and therefore can't serve as its own reference.
-         (Whereas the rev file should already exist at this point.) */
-      SVN_ERR(svn_fs_py__path_rev_absolute(&perms_reference, fs, rev, pool));
-      SVN_ERR(move_into_place(tmp_path, final_path, perms_reference, pool));
+  fs_fs_data_t *ffd = fs->fsap_data;
 
-      return SVN_NO_ERROR;
-    }
+  SVN_ERR(svn_fs_py__call_method(NULL, ffd->p_fs, "_set_revision_proplist",
+                                 "(lO&)", rev,
+                                 svn_fs_py__convert_proplist, proplist));
 
   return SVN_NO_ERROR;
 }
@@ -6257,7 +6236,7 @@ svn_fs_py__create(svn_fs_t *fs,
 
   SVN_ERR(svn_fs_py__call_method(&ffd->p_fs, ffd->p_module, "_create_fs",
                                  "(sO&)", path,
-                                 svn_fs_py__convert_hash, fs->config));
+                                 svn_fs_py__convert_cstring_hash, fs->config));
   apr_pool_cleanup_register(fs->pool, ffd->p_fs, svn_fs_py__destroy_py_object,
                             apr_pool_cleanup_null);
 

Modified: subversion/branches/fs-py/subversion/libsvn_fs_py/py_util.c
URL: 
http://svn.apache.org/viewvc/subversion/branches/fs-py/subversion/libsvn_fs_py/py_util.c?rev=1156447&r1=1156446&r2=1156447&view=diff
==============================================================================
--- subversion/branches/fs-py/subversion/libsvn_fs_py/py_util.c (original)
+++ subversion/branches/fs-py/subversion/libsvn_fs_py/py_util.c Thu Aug 11 
02:25:10 2011
@@ -378,17 +378,18 @@ svn_fs_py__call_method(PyObject **p_resu
   return svn_error_trace(err);
 }
 
-PyObject *
-svn_fs_py__convert_hash(void *object)
+
+static PyObject *
+convert_hash(apr_hash_t *hash,
+             PyObject *(*converter_func)(void *value))
 {
-  apr_hash_t *hash = object;
   apr_hash_index_t *hi;
-  PyObject *p_dict;
+  PyObject *dict;
 
   if (hash == NULL)
     Py_RETURN_NONE;
 
-  if ((p_dict = PyDict_New()) == NULL)
+  if ((dict = PyDict_New()) == NULL)
     return NULL;
 
   for (hi = apr_hash_first(NULL, hash); hi; hi = apr_hash_next(hi))
@@ -398,23 +399,44 @@ svn_fs_py__convert_hash(void *object)
       PyObject *value;
 
       apr_hash_this(hi, &key, NULL, &val);
-      value = PyString_FromString(val);
+      value = (*converter_func)(val);
       if (value == NULL)
         {
-          Py_DECREF(p_dict);
+          Py_DECREF(dict);
           return NULL;
         }
       /* ### gotta cast this thing cuz Python doesn't use "const" */
-      if (PyDict_SetItemString(p_dict, (char *)key, value) == -1)
+      if (PyDict_SetItemString(dict, (char *)key, value) == -1)
         {
           Py_DECREF(value);
-          Py_DECREF(p_dict);
+          Py_DECREF(dict);
           return NULL;
         }
       Py_DECREF(value);
     }
 
-  return p_dict;
+  return dict;
+}
+
+static PyObject *
+convert_svn_string_t(void *value)
+{
+  const svn_string_t *s = value;
+
+  /* ### gotta cast this thing cuz Python doesn't use "const" */
+  return PyString_FromStringAndSize((void *)s->data, s->len);
+}
+
+PyObject *
+svn_fs_py__convert_cstring_hash(void *object)
+{
+  return convert_hash(object, PyString_FromString);
+}
+
+PyObject *
+svn_fs_py__convert_proplist(void *object)
+{
+  return convert_hash(object, convert_svn_string_t);
 }
 
 svn_error_t *

Modified: subversion/branches/fs-py/subversion/libsvn_fs_py/py_util.h
URL: 
http://svn.apache.org/viewvc/subversion/branches/fs-py/subversion/libsvn_fs_py/py_util.h?rev=1156447&r1=1156446&r2=1156447&view=diff
==============================================================================
--- subversion/branches/fs-py/subversion/libsvn_fs_py/py_util.h (original)
+++ subversion/branches/fs-py/subversion/libsvn_fs_py/py_util.h Thu Aug 11 
02:25:10 2011
@@ -41,7 +41,10 @@ svn_fs_py__call_method(PyObject **p_resu
                        ...);
 
 PyObject *
-svn_fs_py__convert_hash(void *object);
+svn_fs_py__convert_cstring_hash(void *object);
+
+PyObject *
+svn_fs_py__convert_proplist(void *object);
 
 /* Load a reference to the FS Python module into the shared data. */
 svn_error_t *

Modified: subversion/branches/fs-py/subversion/python/svn/__init__.py
URL: 
http://svn.apache.org/viewvc/subversion/branches/fs-py/subversion/python/svn/__init__.py?rev=1156447&r1=1156446&r2=1156447&view=diff
==============================================================================
--- subversion/branches/fs-py/subversion/python/svn/__init__.py (original)
+++ subversion/branches/fs-py/subversion/python/svn/__init__.py Thu Aug 11 
02:25:10 2011
@@ -26,3 +26,7 @@ class SubversionException(Exception):
 
     def __str__(self):
         return self.message
+
+
+def is_valid_revnum(rev):
+    return rev >= 0

Modified: subversion/branches/fs-py/subversion/python/svn/fs.py
URL: 
http://svn.apache.org/viewvc/subversion/branches/fs-py/subversion/python/svn/fs.py?rev=1156447&r1=1156446&r2=1156447&view=diff
==============================================================================
--- subversion/branches/fs-py/subversion/python/svn/fs.py (original)
+++ subversion/branches/fs-py/subversion/python/svn/fs.py Thu Aug 11 02:25:10 
2011
@@ -18,15 +18,74 @@
 # specific language governing permissions and limitations
 # under the License.
 
-import os, uuid, shutil
+import os, uuid, shutil, tempfile
 
+import svn
+import svn.hash
 
 # Some constants
+CONFIG_PRE_1_4_COMPATIBLE   = "pre-1.4-compatible"
+CONFIG_PRE_1_5_COMPATIBLE   = "pre-1.5-compatible"
+CONFIG_PRE_1_6_COMPATIBLE   = "pre-1.6-compatible"
+
+PATH_FORMAT                 = "format"          # Contains format number
 PATH_UUID                   = "uuid"            # Contains UUID
 PATH_CURRENT                = "current"         # Youngest revision
+PATH_REVS_DIR               = "revs"            # Directory of revisions
+PATH_REVPROPS_DIR           = "revprops"        # Directory of revprops
+PATH_MIN_UNPACKED_REV       = "min-unpacked-rev" # Oldest revision which
+                                                 # has not been packed.
+
+FORMAT_NUMBER                       = 4
+
+MIN_LAYOUT_FORMAT_OPTION_FORMAT     = 3
+MIN_PROTOREVS_DIR_FORMAT            = 3
+MIN_NO_GLOBAL_IDS_FORMAT            = 3
+MIN_PACKED_FORMAT                   = 4
+
+_DEFAULT_MAX_FILES_PER_DIR = 1000
 
 
 class FS(object):
+    def __path_rev_shard(self, rev):
+        assert self.max_files_per_dir
+        return os.path.join(self.path, PATH_REVS_DIR,
+                            '%d' % (rev // self.max_files_per_dir) )
+
+    def __path_revprops_shard(self, rev):
+        assert self.max_files_per_dir
+        return os.path.join(self.path, PATH_REVPROPS_DIR,
+                            '%d' % (rev // self.max_files_per_dir) )
+
+    def __path_revprops(self, rev):
+        if self.max_files_per_dir:
+            return os.path.join(self.__path_revprops_shard(rev), str(rev))
+        else:
+            return os.path.join(self.path, PATH_REVPROPS_DIR, str(rev))
+            
+    def __path_rev_absolute(self, rev):
+        if self.format < MIN_PACKED_FORMAT or not self.__is_packed_rev(rev):
+            return self.__path_rev(rev)
+        else:
+            return self.__path_rev_packed(rev, "pack")
+
+    def __path_rev(self, rev):
+        assert not self.__is_packed_rev(rev)
+        if self.max_files_per_dir:
+            return os.path.join(self.__path_rev_shard(rev), str(rev))
+        else:
+            return os.path.join(self.path, PATH_REVS_DIR, str(rev))
+
+    def __is_packed_rev(self, rev):
+        return rev < self.__min_unpacked_rev
+
+    def __read_current(self):
+        with open(self.__path_current, 'rb') as f:
+            return int(f.readline())
+
+    def _get_youngest(self):
+        return self.__read_current()
+
     def set_uuid(self, uuid_in = None):
         '''Set the UUID for the filesystem.  If UUID_IN is not given, generate
            a new one a la RFC 4122.'''
@@ -43,8 +102,80 @@ class FS(object):
         shutil.copymode(self.__path_current, self.__path_uuid)
 
 
+    def __ensure_revision_exists(self, rev):
+        if not svn.is_valid_revnum(rev):
+            raise svn.SubversionException(svn.err.FS_NO_SUCH_REVISION,
+                                    _("Invalid revision number '%ld'") % rev)
+
+        # Did the revision exist the last time we checked the current file?
+        if rev <= self.__youngest_rev_cache:
+            return
+
+        # Check again
+        self.__youngest_rev_cache = self._get_youngest()
+        if rev <= self.__youngest_rev_cache:
+            return
+
+        raise svn.SubversionException(svn.err.FS_NO_SUCH_REVISION,
+                                      _("No such revision %ld") % rev)
+
+    def _set_revision_proplist(self, rev, props):
+        self.__ensure_revision_exists(rev)
+
+        final_path = self.__path_revprops(rev)
+        (fd, fn) = tempfile.mkstemp(dir=os.path.dirname(final_path))
+        os.write(fd, svn.hash.encode(props, svn.hash.TERMINATOR))
+        os.close(fd)
+        shutil.copystat(self.__path_rev_absolute(rev), fn)
+
+        os.rename(fn, final_path)
+
+
+    def __read_format(self):
+        try:
+            with open(self.__path_format, 'rb') as f:
+                self.format = int(f.readline())
+                self.max_files_per_dir = 0
+                for l in f:
+                    l = l.split()
+                    if self.format > MIN_LAYOUT_FORMAT_OPTION_FORMAT \
+                            and l[0] == 'layout':
+                        if l[1] == 'linear':
+                            self.max_files_per_dir = 0
+                        elif l[1] == 'sharded':
+                            self.max_files_per_dir = int(l[2])
+        except IOError:
+            # Treat an absent format file as format 1.
+            self.format = 1
+            self.max_files_per_dir = 0
+
+
+    def __update_min_unpacked_rev(self):
+        assert self.format >= MIN_PACKED_FORMAT
+        with open(self.__path_min_unpacked_rev, 'rb') as f:
+            self.__min_unpacked_rev = int(f.readline())
+
+
     def _create_fs(self):
         'Create a new Subversion filesystem'
+        self.__youngest_rev_cache = 0
+        self.__min_unpacked_rev = 0
+
+        # See if compatibility with older versions was explicitly requested.
+        if CONFIG_PRE_1_4_COMPATIBLE in self._config:
+            self.format = 1
+        elif CONFIG_PRE_1_5_COMPATIBLE in self._config:
+            self.format = 2
+        elif CONFIG_PRE_1_6_COMPATIBLE in self._config:
+            self.format = 3
+        else:
+            self.format = FORMAT_NUMBER
+
+        # Override the default linear layout if this is a new-enough format.
+        if self.format >= MIN_LAYOUT_FORMAT_OPTION_FORMAT:
+            self.max_files_per_dir = _DEFAULT_MAX_FILES_PER_DIR
+        else:
+            self.max_files_per_dir = 0
 
 
     def _open_fs(self):
@@ -52,10 +183,18 @@ class FS(object):
         with open(self.__path_uuid, 'rb') as f:
             self.uuid = uuid.UUID(f.readline().rstrip())
 
+        self.__youngest_rev_cache = self.__read_current()
+        self.__read_format()
+        if self.format >= MIN_PACKED_FORMAT:
+            self.__update_min_unpacked_rev()
+
 
     def __setup_paths(self):
         self.__path_uuid = os.path.join(self.path, PATH_UUID)
         self.__path_current = os.path.join(self.path, PATH_CURRENT)
+        self.__path_format = os.path.join(self.path, PATH_FORMAT)
+        self.__path_min_unpacked_rev = os.path.join(self.path,
+                                                    PATH_MIN_UNPACKED_REV)
 
 
     def __init__(self, path, create=False, config=None):

Added: subversion/branches/fs-py/subversion/python/svn/hash.py
URL: 
http://svn.apache.org/viewvc/subversion/branches/fs-py/subversion/python/svn/hash.py?rev=1156447&view=auto
==============================================================================
--- subversion/branches/fs-py/subversion/python/svn/hash.py (added)
+++ subversion/branches/fs-py/subversion/python/svn/hash.py Thu Aug 11 02:25:10 
2011
@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+TERMINATOR = "END"
+
+def encode(h, terminator):
+    output = []
+    for k in sorted(h.keys()):
+        output.append('K %d\n%s\nV %s\n%s\n' % (len(k), k, len(h[k]), h[k]))
+
+    if terminator:
+        output.append('%s\n' % terminator)
+
+    return ''.join(output)

Propchange: subversion/branches/fs-py/subversion/python/svn/hash.py
------------------------------------------------------------------------------
    svn:eol-style = native


Reply via email to