Script 'mail_helper' called by obssrc
Hello community,
here is the log from the commit of package python-devpi-server for
openSUSE:Factory checked in at 2021-11-17 01:13:42
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Comparing /work/SRC/openSUSE:Factory/python-devpi-server (Old)
and /work/SRC/openSUSE:Factory/.python-devpi-server.new.1890 (New)
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "python-devpi-server"
Wed Nov 17 01:13:42 2021 rev:7 rq:931517 version:6.2.0
Changes:
--------
--- /work/SRC/openSUSE:Factory/python-devpi-server/python-devpi-server.changes
2021-08-03 22:49:32.148439328 +0200
+++
/work/SRC/openSUSE:Factory/.python-devpi-server.new.1890/python-devpi-server.changes
2021-11-17 01:14:40.662181641 +0100
@@ -1,0 +2,11 @@
+Mon Nov 8 11:58:00 UTC 2021 - Dirk M??ller <[email protected]>
+
+- update to 6.2.0:
+ * Optimized some database access patterns. A new index is added to the
+ database on first startup. For large databases that can take a while.
+ * Improved performance of loads from database.
+ * Optimized memory and cache use for database access.
+ * Use frozenset for project name cache of mirror indexes. This mitigates
+ memory fragmentation on some Linux distributions.
+
+-------------------------------------------------------------------
Old:
----
devpi-server-6.1.0.tar.gz
New:
----
devpi-server-6.2.0.tar.gz
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Other differences:
------------------
++++++ python-devpi-server.spec ++++++
--- /var/tmp/diff_new_pack.2lriwT/_old 2021-11-17 01:14:41.222181851 +0100
+++ /var/tmp/diff_new_pack.2lriwT/_new 2021-11-17 01:14:41.226181853 +0100
@@ -20,7 +20,7 @@
%define commands export fsck gen-config import init passwd server gen-secret
%define skip_python2 1
Name: python-devpi-server
-Version: 6.1.0
+Version: 6.2.0
Release: 0
Summary: Private PyPI caching server
License: MIT
++++++ devpi-server-6.1.0.tar.gz -> devpi-server-6.2.0.tar.gz ++++++
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/devpi-server-6.1.0/CHANGELOG
new/devpi-server-6.2.0/CHANGELOG
--- old/devpi-server-6.1.0/CHANGELOG 2021-07-11 17:55:40.000000000 +0200
+++ new/devpi-server-6.2.0/CHANGELOG 2021-08-12 11:05:00.000000000 +0200
@@ -2,6 +2,21 @@
.. towncrier release notes start
+6.2.0 (2021-08-12)
+==================
+
+Bug Fixes
+---------
+
+- Optimized some database access patterns. A new index is added to the
database on first startup. For large databases that can take a while.
+
+- Improved performance of loads from database.
+
+- Optimized memory and cache use for database access.
+
+- Use frozenset for project name cache of mirror indexes. This mitigates
memory fragmentation on some Linux distributions.
+
+
6.1.0 (2021-07-11)
==================
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/devpi-server-6.1.0/PKG-INFO
new/devpi-server-6.2.0/PKG-INFO
--- old/devpi-server-6.1.0/PKG-INFO 2021-07-11 17:55:43.020612000 +0200
+++ new/devpi-server-6.2.0/PKG-INFO 2021-08-12 11:05:02.939989600 +0200
@@ -1,6 +1,6 @@
Metadata-Version: 2.1
Name: devpi-server
-Version: 6.1.0
+Version: 6.2.0
Summary: devpi-server: reliable private and pypi.org caching server
Home-page: https://devpi.net
Maintainer: Holger Krekel, Florian Schulze
@@ -102,6 +102,21 @@
.. towncrier release notes start
+6.2.0 (2021-08-12)
+==================
+
+Bug Fixes
+---------
+
+- Optimized some database access patterns. A new index is added to the
database on first startup. For large databases that can take a while.
+
+- Improved performance of loads from database.
+
+- Optimized memory and cache use for database access.
+
+- Use frozenset for project name cache of mirror indexes. This mitigates
memory fragmentation on some Linux distributions.
+
+
6.1.0 (2021-07-11)
==================
@@ -222,13 +237,4 @@
- Pin to pyramid<2.
-5.5.0 (2020-05-04)
-==================
-
-Features
---------
-
-- Proxy requests from replica to master are now streamed if possible. This
improves reliability of large uploads through replicas and reduces RAM usage on
the replica.
-
-
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/devpi-server-6.1.0/devpi_server/__init__.py
new/devpi-server-6.2.0/devpi_server/__init__.py
--- old/devpi-server-6.1.0/devpi_server/__init__.py 2021-07-11
17:55:40.000000000 +0200
+++ new/devpi-server-6.2.0/devpi_server/__init__.py 2021-08-12
11:05:00.000000000 +0200
@@ -1 +1 @@
-__version__ = '6.1.0'
+__version__ = '6.2.0'
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/devpi-server-6.1.0/devpi_server/exceptions.py
new/devpi-server-6.2.0/devpi_server/exceptions.py
--- old/devpi-server-6.1.0/devpi_server/exceptions.py 1970-01-01
01:00:00.000000000 +0100
+++ new/devpi-server-6.2.0/devpi_server/exceptions.py 2021-08-12
11:05:00.000000000 +0200
@@ -0,0 +1,33 @@
+import traceback
+
+
+class LazyExceptionFormatter:
+ __slots__ = ('e',)
+
+ def __init__(self, e):
+ self.e = e
+
+ def __str__(self):
+ return "%s:%s:%s %s" % (
+ *traceback.extract_tb(self.e.__traceback__, 2)[-1][:3],
+ ''.join(traceback.format_exception_only(
+ self.e.__class__, self.e)).strip())
+
+
+class LazyExceptionOnlyFormatter:
+ __slots__ = ('e',)
+
+ def __init__(self, e):
+ self.e = e
+
+ def __str__(self):
+ return ''.join(traceback.format_exception_only(
+ self.e.__class__, self.e)).strip()
+
+
+def lazy_format_exception(e):
+ return LazyExceptionFormatter(e)
+
+
+def lazy_format_exception_only(e):
+ return LazyExceptionOnlyFormatter(e)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/devpi-server-6.1.0/devpi_server/extpypi.py
new/devpi-server-6.2.0/devpi_server/extpypi.py
--- old/devpi-server-6.1.0/devpi_server/extpypi.py 2021-07-11
17:55:40.000000000 +0200
+++ new/devpi-server-6.2.0/devpi_server/extpypi.py 2021-08-12
11:05:00.000000000 +0200
@@ -96,8 +96,8 @@
@property
def releaselinks(self):
- l = map(BasenameMeta, self.basename2link.values())
- return [x.obj for x in l]
+ # the BasenameMeta wrapping essentially does link validation
+ return [BasenameMeta(x).obj for x in self.basename2link.values()]
def parse_index(self, disturl, html):
p = HTMLPage(html, disturl.url)
@@ -366,7 +366,7 @@
self.cache_retrieve_times.refresh(project)
# make project appear in projects list even
# before we next check up the full list with remote
- self.cache_projectnames.get_inplace().add(project)
+ self.cache_projectnames.add(project)
self.keyfs.tx.on_commit_success(on_commit)
@@ -480,7 +480,7 @@
# parse simple index's link
assert response.text is not None, response.text
result = parse_index(response.url, response.text)
- releaselinks = list(result.releaselinks)
+ releaselinks = result.releaselinks
# first we try to process mirror links without an explicit write
transaction.
# if all links already exist in storage we might then return our
already
@@ -600,7 +600,7 @@
""" Helper class for maintaining project names from a mirror. """
def __init__(self):
self._timestamp = -1
- self._data = set()
+ self._data = frozenset()
def exists(self):
return self._timestamp != -1
@@ -610,16 +610,20 @@
def get(self):
""" Get a copy of the cached data. """
- return set(self._data)
-
- def get_inplace(self):
- """ Get cached data in-place. """
return self._data
+ def add(self, project):
+ """ Add project to cache. """
+ self._data = self._data.union({project})
+
+ def discard(self, project):
+ """ Remove project from cache. """
+ self._data = self._data.difference({project})
+
def set(self, data):
""" Set data and update timestamp. """
if data is not self._data:
- self._data = data.copy()
+ self._data = frozenset(data)
self.mark_current()
def mark_current(self):
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/devpi-server-6.1.0/devpi_server/filestore.py
new/devpi-server-6.2.0/devpi_server/filestore.py
--- old/devpi-server-6.1.0/devpi_server/filestore.py 2021-07-11
17:55:40.000000000 +0200
+++ new/devpi-server-6.2.0/devpi_server/filestore.py 2021-08-12
11:05:00.000000000 +0200
@@ -9,17 +9,12 @@
from wsgiref.handlers import format_date_time
import py
import re
-import sys
from devpi_common.metadata import splitbasename
from devpi_common.types import cached_property, parse_hash_spec
from .log import threadlog
+from urllib.parse import unquote
-if sys.version_info >= (3, 0):
- from urllib.parse import unquote
-else:
- from urllib import unquote
-
_nodefault = object()
@@ -35,6 +30,27 @@
return hash_value[:3], hash_value[3:16]
+def key_from_link(keyfs, link, user, index):
+ if link.hash_spec:
+ # we can only create 32K entries per directory
+ # so let's take the first 3 bytes which gives
+ # us a maximum of 16^3 = 4096 entries in the root dir
+ a, b = make_splitdir(link.hash_spec)
+ return keyfs.STAGEFILE(
+ user=user, index=index,
+ hashdir_a=a, hashdir_b=b,
+ filename=link.basename)
+ else:
+ parts = link.torelpath().split("/")
+ assert parts
+ dirname = "_".join(parts[:-1])
+ dirname = re.sub('[^a-zA-Z0-9_.-]', '_', dirname)
+ return keyfs.PYPIFILE_NOMD5(
+ user=user, index=index,
+ dirname=unquote(dirname),
+ basename=link.basename)
+
+
def unicode_if_bytes(val):
if isinstance(val, py.builtin.bytes):
val = py.builtin._totext(val)
@@ -48,24 +64,7 @@
self.keyfs = keyfs
def maplink(self, link, user, index, project):
- parts = link.torelpath().split("/")
- assert parts
- basename = unquote(parts[-1])
- if link.hash_spec:
- # we can only create 32K entries per directory
- # so let's take the first 3 bytes which gives
- # us a maximum of 16^3 = 4096 entries in the root dir
- a, b = make_splitdir(link.hash_spec)
- key = self.keyfs.STAGEFILE(user=user, index=index,
- hashdir_a=a, hashdir_b=b,
- filename=link.basename)
- else:
- dirname = "_".join(parts[:-1])
- dirname = re.sub('[^a-zA-Z0-9_.-]', '_', dirname)
- key = self.keyfs.PYPIFILE_NOMD5(
- user=user, index=index,
- dirname=unquote(dirname),
- basename=basename)
+ key = key_from_link(self.keyfs, link, user, index)
entry = FileEntry(key, readonly=False)
entry.url = link.geturl_nofragment().url
# verify checksum if the entry is fresh, a file exists
@@ -82,7 +81,7 @@
entry.project = project
version = None
try:
- (projectname, version, ext) = splitbasename(basename)
+ (projectname, version, ext) = splitbasename(link.basename)
except ValueError:
pass
# only store version on entry if we can determine it
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/devpi-server-6.1.0/devpi_server/fileutil.py
new/devpi-server-6.2.0/devpi_server/fileutil.py
--- old/devpi-server-6.1.0/devpi_server/fileutil.py 2021-07-11
17:55:40.000000000 +0200
+++ new/devpi-server-6.2.0/devpi_server/fileutil.py 2021-08-12
11:05:00.000000000 +0200
@@ -1,8 +1,10 @@
import errno
import os.path
import sys
-from execnet.gateway_base import Unserializer, _Serializer
+from execnet.gateway_base import LoadError, Unserializer, _Serializer
+from functools import partial
from io import BytesIO
+from struct import unpack
_nodefault = object()
@@ -20,9 +22,77 @@
def loads(data):
- return Unserializer(
- BytesIO(data),
- strconfig=(False, False)).load(versioned=False)
+ read = BytesIO(data).read
+ _unpack_int4 = partial(unpack, "!i")
+ _unpack_float8 = partial(unpack, "!d")
+ stack = []
+ stack_append = stack.append
+ stack_pop = stack.pop
+
+ def _load_collection(type_):
+ length = _unpack_int4(read(4))[0]
+ if length:
+ res = type_(stack[-length:])
+ del stack[-length:]
+ stack_append(res)
+ else:
+ stack_append(type_())
+
+ stopped = False
+ while True:
+ opcode = read(1)
+ if not opcode:
+ raise EOFError
+ if opcode == b'@': # tuple
+ _load_collection(tuple)
+ elif opcode == b'A': # bytes
+ stack_append(read(_unpack_int4(read(4))[0]))
+ elif opcode == b'B': # Channel
+ raise NotImplementedError("%s" % Unserializer.num2func[opcode])
+ elif opcode == b'C': # False
+ stack_append(False)
+ elif opcode == b'D': # float
+ stack_append(_unpack_float8(read(8))[0])
+ elif opcode == b'E': # frozenset
+ _load_collection(frozenset)
+ elif opcode in (b'F', b'G'): # int, long
+ stack_append(_unpack_int4(read(4))[0])
+ elif opcode in (b'H', b'I'): # longint, longlong
+ stack_append(int(read(_unpack_int4(read(4))[0])))
+ elif opcode == b'J': # dict
+ stack_append({})
+ elif opcode == b'K': # list
+ stack_append([None] * _unpack_int4(read(4))[0])
+ elif opcode == b'L': # None
+ stack_append(None)
+ elif opcode == b'M': # Python 2 string
+ stack_append(read(_unpack_int4(read(4))[0]))
+ elif opcode in (b'N', b'S'): # Python 3 string, unicode
+ stack_append(read(_unpack_int4(read(4))[0]).decode('utf-8'))
+ elif opcode == b'O': # set
+ _load_collection(set)
+ elif opcode == b'P': # setitem
+ try:
+ value = stack_pop()
+ key = stack_pop()
+ except IndexError:
+ raise LoadError("not enough items for setitem")
+ stack[-1][key] = value
+ elif opcode == b'Q': # stop
+ stopped = True
+ break
+ elif opcode == b'R': # True
+ stack_append(True)
+ elif opcode == b'T': # complex
+ stack_append(complex(_unpack_float8(read(8))[0],
_unpack_float8(read(8))[0]))
+ else:
+ raise LoadError(
+ "unknown opcode %r - wire protocol corruption?" % opcode)
+ if not stopped:
+ raise LoadError("didn't get STOP")
+ if len(stack) != 1:
+ raise LoadError("internal unserialization error")
+ return stack_pop(0)
def dumps(obj):
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/devpi-server-6.1.0/devpi_server/interfaces.py
new/devpi-server-6.2.0/devpi_server/interfaces.py
--- old/devpi-server-6.1.0/devpi_server/interfaces.py 1970-01-01
01:00:00.000000000 +0100
+++ new/devpi-server-6.2.0/devpi_server/interfaces.py 2021-08-12
11:05:00.000000000 +0200
@@ -0,0 +1,82 @@
+from contextlib import closing
+from zope.interface import Interface
+from zope.interface import classImplements
+from zope.interface.interface import adapter_hooks
+from zope.interface.verify import verifyObject
+
+
+class IStorageConnection(Interface):
+ def db_read_last_changelog_serial():
+ """ Return last stored serial.
+ Returns -1 if nothing is stored yet. """
+
+ def db_read_typedkey(relpath):
+ """ Return key name and serial for given relpath.
+ Raises KeyError if not found. """
+
+ def get_changes(serial):
+ """ Returns deserialized readonly changes for given serial. """
+
+ def get_raw_changelog_entry(serial):
+ """ Returns serializes changes for given serial. """
+
+
+class IStorageConnection2(IStorageConnection):
+ def get_relpath_at(relpath, serial):
+ """ Get tuple of (last_serial, back_serial, value) for given relpath
+ at given serial.
+ Raises KeyError if not found. """
+
+ def iter_relpaths_at(typedkeys, at_serial):
+ """ Iterate over all relpaths of the given typed keys starting
+ from at_serial until the first serial in the database.
+ Yields RelpathInfo objects."""
+
+
+# some adapters for legacy plugins
+
+
+def unwrap_connection_obj(obj):
+ if isinstance(obj, closing):
+ obj = obj.thing
+ return obj
+
+
+def get_connection_class(obj):
+ return unwrap_connection_obj(obj).__class__
+
+
+def verify_connection_interface(obj):
+ verifyObject(IStorageConnection2, unwrap_connection_obj(obj))
+
+
+@adapter_hooks.append
+def adapt(iface, obj):
+ # this is not traditional adaption which would return a new object,
+ # but for performance reasons we directly patch the class, so the next
+ # time no adaption call is necessary
+ if iface is IStorageConnection:
+ _obj = unwrap_connection_obj(obj)
+ cls = get_connection_class(_obj)
+ # any storage connection which needs to be adapted to this
+ # interface is a legacy one and we can say that it provides
+ # the original interface directly
+ classImplements(cls, IStorageConnection)
+ # make sure the object now actually provides this interface
+ verifyObject(IStorageConnection, _obj)
+ return obj
+ elif iface is IStorageConnection2:
+ from .keyfs import get_relpath_at
+ from .keyfs import iter_relpaths_at
+ # first make sure the old connection interface is implemented
+ obj = IStorageConnection(obj)
+ _obj = unwrap_connection_obj(obj)
+ cls = get_connection_class(_obj)
+ # now add fallback method directly to the class
+ cls.get_relpath_at = get_relpath_at
+ cls.iter_relpaths_at = iter_relpaths_at
+ # and add the interface
+ classImplements(cls, IStorageConnection2)
+ # make sure the object now actually provides this interface
+ verifyObject(IStorageConnection2, _obj)
+ return obj
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/devpi-server-6.1.0/devpi_server/keyfs.py
new/devpi-server-6.2.0/devpi_server/keyfs.py
--- old/devpi-server-6.1.0/devpi_server/keyfs.py 2021-07-11
17:55:40.000000000 +0200
+++ new/devpi-server-6.2.0/devpi_server/keyfs.py 2021-08-12
11:05:00.000000000 +0200
@@ -12,6 +12,7 @@
import py
from . import mythread
from .fileutil import loads
+from .interfaces import IStorageConnection2
from .log import threadlog, thread_push_log, thread_pop_log
from .readonly import get_mutable_deepcopy, ensure_deeply_readonly, \
is_deeply_readonly
@@ -141,7 +142,7 @@
def _execute_hooks(self, event_serial, log, raising=False):
log.debug("calling hooks for tx%s", event_serial)
- with self.keyfs._storage.get_connection() as conn:
+ with self.keyfs.get_connection() as conn:
changes = conn.get_changes(event_serial)
# we first check for missing files before we call subscribers
for relpath, (keyname, back_serial, val) in changes.items():
@@ -229,12 +230,19 @@
cache_size=cache_size)
self._readonly = readonly
+ def get_connection(self, closing=True, write=False):
+ conn = IStorageConnection2(
+ self._storage.get_connection(closing=False, write=write))
+ if closing:
+ return contextlib.closing(conn)
+ return conn
+
def finalize_init(self):
self._storage.perform_crash_recovery()
def import_changes(self, serial, changes):
subscriber_task_infos = []
- with self._storage.get_connection(write=True) as conn:
+ with self.get_connection(write=True) as conn:
with conn.write_transaction() as fswriter:
next_serial = conn.last_changelog_serial + 1
assert next_serial == serial, (next_serial, serial)
@@ -292,7 +300,7 @@
recheck = timeout
with threadlog.around("debug", "waiting for tx-serial %s", serial):
with self._cv_new_transaction:
- with self._storage.get_connection() as conn:
+ with self.get_connection() as conn:
while serial > conn.db_read_last_changelog_serial():
if timeout is not None and time_spent >= timeout:
return False
@@ -304,7 +312,7 @@
return self.get_current_serial() + 1
def get_current_serial(self):
- with self._storage.get_connection() as conn:
+ with self.get_connection() as conn:
return conn.last_changelog_serial
def get_last_commit_timestamp(self):
@@ -501,6 +509,54 @@
value = attr.ib()
+def get_relpath_at(self, relpath, serial):
+ """ Fallback method for legacy storage connections. """
+ (keyname, last_serial) = self.db_read_typedkey(relpath)
+ serials_and_values = iter_serial_and_value_backwards(
+ self, relpath, last_serial)
+ try:
+ (last_serial, back_serial, val) = next(serials_and_values)
+ while last_serial >= 0:
+ if last_serial > serial:
+ (last_serial, back_serial, val) = next(serials_and_values)
+ continue
+ return (last_serial, back_serial, val)
+ except StopIteration:
+ pass
+ raise KeyError(relpath)
+
+
+def iter_serial_and_value_backwards(conn, relpath, last_serial):
+ while last_serial >= 0:
+ tup = conn.get_changes(last_serial).get(relpath)
+ if tup is None:
+ raise RuntimeError("no transaction entry at %s" % (last_serial))
+ keyname, back_serial, val = tup
+ yield (last_serial, back_serial, val)
+ last_serial = back_serial
+
+ # we could not find any change below at_serial which means
+ # the key didn't exist at that point in time
+ return
+
+
+def iter_relpaths_at(self, typedkeys, at_serial):
+ keynames = frozenset(k.name for k in typedkeys)
+ seen = set()
+ for serial in range(at_serial, -1, -1):
+ raw_entry = self.get_raw_changelog_entry(serial)
+ changes = loads(raw_entry)[0]
+ for relpath, (keyname, back_serial, val) in changes.items():
+ if keyname not in keynames:
+ continue
+ if relpath not in seen:
+ seen.add(relpath)
+ yield RelpathInfo(
+ relpath=relpath, keyname=keyname,
+ serial=serial, back_serial=back_serial,
+ value=val)
+
+
class Transaction(object):
def __init__(self, keyfs, at_serial=None, write=False):
self.keyfs = keyfs
@@ -521,64 +577,30 @@
@cached_property
def conn(self):
- return self.keyfs._storage.get_connection(
+ return self.keyfs.get_connection(
write=self.write, closing=False)
def iter_relpaths_at(self, typedkeys, at_serial):
- keynames = frozenset(k.name for k in typedkeys)
- seen = set()
- for serial in range(at_serial, -1, -1):
- raw_entry = self.conn.get_raw_changelog_entry(serial)
- changes = loads(raw_entry)[0]
- for relpath, (keyname, back_serial, val) in changes.items():
- if keyname not in keynames:
- continue
- if relpath not in seen:
- seen.add(relpath)
- yield RelpathInfo(
- relpath=relpath, keyname=keyname,
- serial=serial, back_serial=back_serial,
- value=val)
+ return self.conn.iter_relpaths_at(typedkeys, at_serial)
def iter_serial_and_value_backwards(self, relpath, last_serial):
while last_serial >= 0:
- tup = self.conn.get_changes(last_serial).get(relpath)
- if tup is None:
- raise RuntimeError("no transaction entry at %s" %
(last_serial))
- keyname, back_serial, val = tup
+ (last_serial, back_serial, val) = self.conn.get_relpath_at(
+ relpath, last_serial)
yield (last_serial, val)
last_serial = back_serial
- # we could not find any change below at_serial which means
- # the key didn't exist at that point in time
- return
-
def get_last_serial_and_value_at(self, typedkey, at_serial,
raise_on_error=True):
relpath = typedkey.relpath
try:
- (keyname, last_serial) = self.conn.db_read_typedkey(relpath)
+ (last_serial, back_serial, val) =
self.conn.get_relpath_at(relpath, at_serial)
except KeyError:
- if raise_on_error:
- raise
- return None
- serials_and_values = self.iter_serial_and_value_backwards(
- relpath, last_serial)
- try:
- (last_serial, val) = next(serials_and_values)
- while last_serial >= 0:
- if last_serial > at_serial:
- (last_serial, val) = next(serials_and_values)
- continue
- if val is not None or not raise_on_error:
- return (last_serial, val)
- raise KeyError(relpath) # was deleted
- except StopIteration:
- pass
-
- if raise_on_error:
- # we could not find any change below at_serial which means
- # the key didn't exist at that point in time
- raise KeyError(relpath)
+ if not raise_on_error:
+ return None
+ raise
+ if val is None and raise_on_error:
+ raise KeyError(relpath) # was deleted
+ return (last_serial, val)
def get_value_at(self, typedkey, at_serial):
(last_serial, val) = self.get_last_serial_and_value_at(typedkey,
at_serial)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/devpi-server-6.1.0/devpi_server/keyfs_sqlite.py
new/devpi-server-6.2.0/devpi_server/keyfs_sqlite.py
--- old/devpi-server-6.1.0/devpi_server/keyfs_sqlite.py 2021-07-11
17:55:40.000000000 +0200
+++ new/devpi-server-6.2.0/devpi_server/keyfs_sqlite.py 2021-08-12
11:05:00.000000000 +0200
@@ -1,10 +1,14 @@
from devpi_common.types import cached_property
from .config import hookimpl
from .fileutil import dumps, loads
+from .interfaces import IStorageConnection2
+from .keyfs import RelpathInfo
+from .keyfs import get_relpath_at
from .log import threadlog, thread_push_log, thread_pop_log
from .readonly import ReadonlyView
from .readonly import ensure_deeply_readonly, get_mutable_deepcopy
from repoze.lru import LRUCache
+from zope.interface import implementer
import contextlib
import os
import py
@@ -12,6 +16,9 @@
import time
+absent = object()
+
+
class BaseConnection:
def __init__(self, sqlconn, basedir, storage):
self._sqlconn = sqlconn
@@ -20,6 +27,35 @@
self.storage = storage
self._changelog_cache = storage._changelog_cache
+ def _explain(self, query, *args):
+ # for debugging
+ c = self._sqlconn.cursor()
+ r = c.execute("EXPLAIN " + query, *args)
+ return r.fetchall()
+
+ def _explain_query_plan(self, query, *args):
+ # for debugging
+ c = self._sqlconn.cursor()
+ r = c.execute("EXPLAIN QUERY PLAN " + query, *args)
+ return r.fetchall()
+
+ def fetchall(self, query, *args):
+ c = self._sqlconn.cursor()
+ # print(query)
+ # pprint(self._explain(query, *args))
+ # for row in self._explain_query_plan(query, *args):
+ # print(row)
+ r = c.execute(query, *args)
+ return r.fetchall()
+
+ def iterall(self, query, *args):
+ c = self._sqlconn.cursor()
+ # print(query)
+ # pprint(self._explain(query, *args))
+ # for row in self._explain_query_plan(query, *args):
+ # print(row)
+ return c.execute(query, *args)
+
def close(self):
self._sqlconn.close()
@@ -76,7 +112,46 @@
self._changelog_cache.put(serial, changes)
return changes
+ def get_relpath_at(self, relpath, serial):
+ result = self._changelog_cache.get((serial, relpath), absent)
+ if result is absent:
+ changes = self._changelog_cache.get(serial, absent)
+ if changes is not absent and relpath in changes:
+ (keyname, back_serial, value) = changes[relpath]
+ result = (serial, back_serial, value)
+ if result is absent:
+ result = get_relpath_at(self, relpath, serial)
+ self._changelog_cache.put((serial, relpath), result)
+ return result
+
+ def iter_relpaths_at(self, typedkeys, at_serial):
+ keynames = frozenset(k.name for k in typedkeys)
+ keyname_id_values = {"keynameid%i" % i: k for i, k in
enumerate(keynames)}
+ q = """
+ SELECT key, keyname, serial
+ FROM kv
+ WHERE serial=:serial AND keyname IN (:keynames)
+ """
+ q = q.replace(':keynames', ", ".join(':' + x for x in
keyname_id_values))
+ for serial in range(at_serial, -1, -1):
+ rows = self.fetchall(q, dict(
+ serial=serial,
+ **keyname_id_values))
+ if not rows:
+ continue
+ changes = self._changelog_cache.get(serial, absent)
+ if changes is absent:
+ changes = loads(
+ self.get_raw_changelog_entry(serial))[0]
+ for relpath, keyname, serial in rows:
+ (keyname, back_serial, val) = changes[relpath]
+ yield RelpathInfo(
+ relpath=relpath, keyname=keyname,
+ serial=serial, back_serial=back_serial,
+ value=val)
+
+@implementer(IStorageConnection2)
class Connection(BaseConnection):
def io_file_os_path(self, path):
return None
@@ -230,41 +305,73 @@
return contextlib.closing(conn)
return conn
-
-class Storage(BaseStorage):
- Connection = Connection
- db_filename = ".sqlite_db"
-
- def perform_crash_recovery(self):
- pass
+ def _reflect_schema(self):
+ result = {}
+ with self.get_connection(write=False) as conn:
+ c = conn._sqlconn.cursor()
+ rows = c.execute("""
+ SELECT type, name, sql FROM sqlite_master""")
+ for row in rows:
+ result.setdefault(row[0], {})[row[1]] = row[2]
+ return result
def ensure_tables_exist(self):
- if self.sqlpath.exists():
+ schema = self._reflect_schema()
+ missing = dict()
+ for kind, objs in self.expected_schema.items():
+ for name, q in objs.items():
+ if name not in schema.get(kind, set()):
+ missing.setdefault(kind, dict())[name] = q
+ if not missing:
return
with self.get_connection(write=True) as conn:
- threadlog.info("DB: Creating schema")
+ if not schema:
+ threadlog.info("DB: Creating schema")
+ else:
+ threadlog.info("DB: Updating schema")
c = conn._sqlconn.cursor()
- c.execute("""
+ for kind in ('table', 'index'):
+ objs = missing.pop(kind, {})
+ for name in list(objs):
+ q = objs.pop(name)
+ c.execute(q)
+ assert not objs
+ conn.commit()
+ assert not missing
+
+
+class Storage(BaseStorage):
+ Connection = Connection
+ db_filename = ".sqlite_db"
+ expected_schema = dict(
+ index=dict(
+ kv_serial_idx="""
+ CREATE INDEX kv_serial_idx ON kv (serial);
+ """),
+ table=dict(
+ changelog="""
+ CREATE TABLE changelog (
+ serial INTEGER PRIMARY KEY,
+ data BLOB NOT NULL
+ )
+ """,
+ kv="""
CREATE TABLE kv (
key TEXT NOT NULL PRIMARY KEY,
keyname TEXT,
serial INTEGER
)
- """)
- c.execute("""
- CREATE TABLE changelog (
- serial INTEGER PRIMARY KEY,
- data BLOB NOT NULL
- )
- """)
- c.execute("""
+ """,
+ files="""
CREATE TABLE files (
path TEXT PRIMARY KEY,
size INTEGER NOT NULL,
data BLOB NOT NULL
)
- """)
- conn.commit()
+ """))
+
+ def perform_crash_recovery(self):
+ pass
@hookimpl
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/devpi-server-6.1.0/devpi_server/keyfs_sqlite_fs.py
new/devpi-server-6.2.0/devpi_server/keyfs_sqlite_fs.py
--- old/devpi-server-6.1.0/devpi_server/keyfs_sqlite_fs.py 2021-07-11
17:55:40.000000000 +0200
+++ new/devpi-server-6.2.0/devpi_server/keyfs_sqlite_fs.py 2021-08-12
11:05:00.000000000 +0200
@@ -1,5 +1,6 @@
from .config import hookimpl
from .fileutil import BytesForHardlink
+from .interfaces import IStorageConnection2
from .keyfs_sqlite import BaseConnection
from .keyfs_sqlite import BaseStorage
from .log import threadlog, thread_push_log, thread_pop_log
@@ -7,6 +8,7 @@
from .readonly import get_mutable_deepcopy
from .fileutil import get_write_file_ensure_dir, rename, loads
from hashlib import sha256
+from zope.interface import implementer
import errno
import os
import re
@@ -44,6 +46,7 @@
f.write(content)
+@implementer(IStorageConnection2)
class Connection(BaseConnection):
def rollback(self):
BaseConnection.rollback(self)
@@ -130,6 +133,25 @@
class Storage(BaseStorage):
Connection = Connection
db_filename = ".sqlite"
+ expected_schema = dict(
+ index=dict(
+ kv_serial_idx="""
+ CREATE INDEX kv_serial_idx ON kv (serial);
+ """),
+ table=dict(
+ changelog="""
+ CREATE TABLE changelog (
+ serial INTEGER PRIMARY KEY,
+ data BLOB NOT NULL
+ )
+ """,
+ kv="""
+ CREATE TABLE kv (
+ key TEXT NOT NULL PRIMARY KEY,
+ keyname TEXT,
+ serial INTEGER
+ )
+ """))
def perform_crash_recovery(self):
# get last changes and verify all renames took place
@@ -140,27 +162,6 @@
changes, rel_renames = loads(data)
check_pending_renames(str(self.basedir), rel_renames)
- def ensure_tables_exist(self):
- if self.sqlpath.exists():
- return
- with self.get_connection(write=True) as conn:
- threadlog.info("DB: Creating schema")
- c = conn._sqlconn.cursor()
- c.execute("""
- CREATE TABLE kv (
- key TEXT NOT NULL PRIMARY KEY,
- keyname TEXT,
- serial INTEGER
- )
- """)
- c.execute("""
- CREATE TABLE changelog (
- serial INTEGER PRIMARY KEY,
- data BLOB NOT NULL
- )
- """)
- conn.commit()
-
@hookimpl
def devpiserver_storage_backend(settings):
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/devpi-server-6.1.0/devpi_server/log.py
new/devpi-server-6.2.0/devpi_server/log.py
--- old/devpi-server-6.1.0/devpi_server/log.py 2021-07-11 17:55:40.000000000
+0200
+++ new/devpi-server-6.2.0/devpi_server/log.py 2021-08-12 11:05:00.000000000
+0200
@@ -31,7 +31,7 @@
if config_args.logger_cfg.endswith(".json"):
logger_cfg = json.loads(f.read())
else:
- logger_cfg = yaml.safe_load(f.read())
+ logger_cfg = yaml.YAML(typ='safe', pure=True).load(f.read())
logging.config.dictConfig(logger_cfg)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/devpi-server-6.1.0/devpi_server/replica.py
new/devpi-server-6.2.0/devpi_server/replica.py
--- old/devpi-server-6.1.0/devpi_server/replica.py 2021-07-11
17:55:40.000000000 +0200
+++ new/devpi-server-6.2.0/devpi_server/replica.py 2021-08-12
11:05:00.000000000 +0200
@@ -780,7 +780,7 @@
with self.xom.keyfs.transaction(write=False):
mirror_stage = self.xom.model.getstage(username, index)
if mirror_stage and mirror_stage.ixconfig["type"] == "mirror":
- cache_projectnames =
mirror_stage.cache_projectnames.get_inplace()
+ cache_projectnames = mirror_stage.cache_projectnames
if cache is None: # deleted
cache_projectnames.discard(project)
else:
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/devpi-server-6.1.0/devpi_server/views.py
new/devpi-server-6.2.0/devpi_server/views.py
--- old/devpi-server-6.1.0/devpi_server/views.py 2021-07-11
17:55:40.000000000 +0200
+++ new/devpi-server-6.2.0/devpi_server/views.py 2021-08-12
11:05:00.000000000 +0200
@@ -572,8 +572,11 @@
whitelist_info = stage.get_mirror_whitelist_info(project)
embed_form = whitelist_info['has_mirror_base']
blocked_index = whitelist_info['blocked_by_mirror_whitelist']
- response = Response(body=b"".join(self._simple_list_project(
- stage, project, result, embed_form, blocked_index)))
+ body = b"".join(self._simple_list_project(
+ stage, project, result, embed_form, blocked_index))
+ response = Response(
+ content_length=len(body),
+ body=body)
if stage.ixconfig['type'] == 'mirror':
serial = stage.key_projsimplelinks(project).get().get("serial")
if serial > 0:
@@ -581,9 +584,6 @@
return response
def _simple_list_project(self, stage, project, result, embed_form,
blocked_index):
- response = self.request.response
- response.content_type = "text/html ; charset=utf-8"
-
title = "%s: links for %s" % (stage.name, project)
yield ("<!DOCTYPE
html><html><head><title>%s</title></head><body><h1>%s</h1>\n" %
(title, title)).encode("utf-8")
@@ -659,17 +659,20 @@
self.log.info("starting +simple")
stage = self.context.stage
try:
+ # list is called to force iteration over all results in this
+ # try/except block
stage_results = list(stage.list_projects())
except stage.UpstreamError as e:
threadlog.error(e.msg)
abort(self.request, 502, e.msg)
# at this point we are sure we can produce the data without
# depending on remote networks
- return Response(body=b"".join(self._simple_list_all(stage,
stage_results)))
+ body = b"".join(self._simple_list_all(stage, stage_results))
+ return Response(
+ content_length=len(body),
+ body=body)
def _simple_list_all(self, stage, stage_results):
- response = self.request.response
- response.content_type = "text/html ; charset=utf-8"
title = "%s: simple list (including inherited indices)" % (stage.name)
yield ("<!DOCTYPE
html><html><head><title>%s</title></head><body><h1>%s</h1>" %(
title, title)).encode("utf-8")
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/devpi-server-6.1.0/devpi_server.egg-info/PKG-INFO
new/devpi-server-6.2.0/devpi_server.egg-info/PKG-INFO
--- old/devpi-server-6.1.0/devpi_server.egg-info/PKG-INFO 2021-07-11
17:55:42.000000000 +0200
+++ new/devpi-server-6.2.0/devpi_server.egg-info/PKG-INFO 2021-08-12
11:05:02.000000000 +0200
@@ -1,6 +1,6 @@
Metadata-Version: 2.1
Name: devpi-server
-Version: 6.1.0
+Version: 6.2.0
Summary: devpi-server: reliable private and pypi.org caching server
Home-page: https://devpi.net
Maintainer: Holger Krekel, Florian Schulze
@@ -102,6 +102,21 @@
.. towncrier release notes start
+6.2.0 (2021-08-12)
+==================
+
+Bug Fixes
+---------
+
+- Optimized some database access patterns. A new index is added to the
database on first startup. For large databases that can take a while.
+
+- Improved performance of loads from database.
+
+- Optimized memory and cache use for database access.
+
+- Use frozenset for project name cache of mirror indexes. This mitigates
memory fragmentation on some Linux distributions.
+
+
6.1.0 (2021-07-11)
==================
@@ -222,13 +237,4 @@
- Pin to pyramid<2.
-5.5.0 (2020-05-04)
-==================
-
-Features
---------
-
-- Proxy requests from replica to master are now streamed if possible. This
improves reliability of large uploads through replicas and reduces RAM usage on
the replica.
-
-
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/devpi-server-6.1.0/devpi_server.egg-info/SOURCES.txt
new/devpi-server-6.2.0/devpi_server.egg-info/SOURCES.txt
--- old/devpi-server-6.1.0/devpi_server.egg-info/SOURCES.txt 2021-07-11
17:55:42.000000000 +0200
+++ new/devpi-server-6.2.0/devpi_server.egg-info/SOURCES.txt 2021-08-12
11:05:02.000000000 +0200
@@ -13,6 +13,7 @@
devpi_server/auth_basic.py
devpi_server/auth_devpi.py
devpi_server/config.py
+devpi_server/exceptions.py
devpi_server/extpypi.py
devpi_server/filestore.py
devpi_server/fileutil.py
@@ -21,6 +22,7 @@
devpi_server/hookspecs.py
devpi_server/importexport.py
devpi_server/init.py
+devpi_server/interfaces.py
devpi_server/keyfs.py
devpi_server/keyfs_sqlite.py
devpi_server/keyfs_sqlite_fs.py
@@ -61,6 +63,7 @@
test_devpi_server/test_conftest.py
test_devpi_server/test_extpypi.py
test_devpi_server/test_filestore.py
+test_devpi_server/test_fileutil.py
test_devpi_server/test_genconfig.py
test_devpi_server/test_importexport.py
test_devpi_server/test_keyfs.py
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/devpi-server-6.1.0/devpi_server.egg-info/requires.txt
new/devpi-server-6.2.0/devpi_server.egg-info/requires.txt
--- old/devpi-server-6.1.0/devpi_server.egg-info/requires.txt 2021-07-11
17:55:42.000000000 +0200
+++ new/devpi-server-6.2.0/devpi_server.egg-info/requires.txt 2021-08-12
11:05:02.000000000 +0200
@@ -10,6 +10,6 @@
waitress>=1.0.1
repoze.lru>=0.6
passlib[argon2]
-pluggy<1.0,>=0.6.0
+pluggy<2.0,>=0.6.0
ruamel.yaml
strictyaml
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/devpi-server-6.1.0/setup.py
new/devpi-server-6.2.0/setup.py
--- old/devpi-server-6.1.0/setup.py 2021-07-11 17:55:40.000000000 +0200
+++ new/devpi-server-6.2.0/setup.py 2021-08-12 11:05:00.000000000 +0200
@@ -38,7 +38,7 @@
"waitress>=1.0.1",
"repoze.lru>=0.6",
"passlib[argon2]",
- "pluggy>=0.6.0,<1.0",
+ "pluggy>=0.6.0,<2.0",
'ruamel.yaml',
"strictyaml",
]
@@ -56,7 +56,7 @@
'Documentation': 'https://doc.devpi.net',
'Source Code': 'https://github.com/devpi/devpi'
},
- version='6.1.0',
+ version='6.2.0',
maintainer="Holger Krekel, Florian Schulze",
maintainer_email="[email protected]",
packages=[
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/devpi-server-6.1.0/test_devpi_server/conftest.py
new/devpi-server-6.2.0/test_devpi_server/conftest.py
--- old/devpi-server-6.1.0/test_devpi_server/conftest.py 2021-07-11
17:55:40.000000000 +0200
+++ new/devpi-server-6.2.0/test_devpi_server/conftest.py 2021-08-12
11:05:00.000000000 +0200
@@ -155,10 +155,8 @@
return xom
[email protected](autouse=True, scope="session")
-def speed_up_sqlite():
- from devpi_server.keyfs_sqlite import Storage
- old = Storage.ensure_tables_exist
+def _speed_up_sqlite(cls):
+ old = cls.ensure_tables_exist
def make_unsynchronous(self, old=old):
conn = old(self)
@@ -166,7 +164,14 @@
conn._sqlconn.execute("PRAGMA synchronous=OFF")
return
- Storage.ensure_tables_exist = make_unsynchronous
+ cls.ensure_tables_exist = make_unsynchronous
+ return old
+
+
[email protected](autouse=True, scope="session")
+def speed_up_sqlite():
+ from devpi_server.keyfs_sqlite import Storage
+ old = _speed_up_sqlite(Storage)
yield
Storage.ensure_tables_exist = old
@@ -174,15 +179,7 @@
@pytest.fixture(autouse=True, scope="session")
def speed_up_sqlite_fs():
from devpi_server.keyfs_sqlite_fs import Storage
- old = Storage.ensure_tables_exist
-
- def make_unsynchronous(self, old=old):
- conn = old(self)
- with self.get_connection() as conn:
- conn._sqlconn.execute("PRAGMA synchronous=OFF")
- return
-
- Storage.ensure_tables_exist = make_unsynchronous
+ old = _speed_up_sqlite(Storage)
yield
Storage.ensure_tables_exist = old
@@ -203,6 +200,8 @@
if backend is None:
backend = 'devpi_server.keyfs_sqlite_fs'
plugin = locate(backend)
+ if plugin is None:
+ raise RuntimeError("Couldn't find storage backend '%s'" % backend)
result = plugin.devpiserver_storage_backend(settings=None)
result["_test_plugin"] = plugin
return result
@@ -222,6 +221,7 @@
from devpi_server import replica
from devpi_server import view_auth
from devpi_server import views
+ from devpi_server.interfaces import verify_connection_interface
plugins = [
plugin[0] if isinstance(plugin, tuple) else plugin
for plugin in plugins]
@@ -263,6 +263,9 @@
monkeypatch.setattr(extpypi.PyPIStage, "_get_remote_projects",
lambda self: set())
add_pypistage_mocks(monkeypatch, httpget)
+ # verify storage interface
+ with xom.keyfs.get_connection() as conn:
+ verify_connection_interface(conn)
# initialize default indexes
from devpi_server.main import init_default_indexes
if not xom.config.args.master_url:
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/devpi-server-6.1.0/test_devpi_server/test_extpypi.py
new/devpi-server-6.2.0/test_devpi_server/test_extpypi.py
--- old/devpi-server-6.1.0/test_devpi_server/test_extpypi.py 2021-07-11
17:55:40.000000000 +0200
+++ new/devpi-server-6.2.0/test_devpi_server/test_extpypi.py 2021-08-12
11:05:00.000000000 +0200
@@ -665,9 +665,10 @@
cache.set(s)
s.add(4)
assert cache.get() != s
- s2 = cache.get_inplace()
- s2.add(5)
+ cache.add(5)
assert 5 in cache.get()
+ cache.discard(5)
+ assert 5 not in cache.get()
def test_is_expired(self, cache, monkeypatch):
expiry_time = 100
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/devpi-server-6.1.0/test_devpi_server/test_fileutil.py
new/devpi-server-6.2.0/test_devpi_server/test_fileutil.py
--- old/devpi-server-6.1.0/test_devpi_server/test_fileutil.py 1970-01-01
01:00:00.000000000 +0100
+++ new/devpi-server-6.2.0/test_devpi_server/test_fileutil.py 2021-08-12
11:05:00.000000000 +0200
@@ -0,0 +1,69 @@
+from devpi_server.fileutil import BytesIO
+from devpi_server.fileutil import Unserializer
+from devpi_server.fileutil import dumps
+from devpi_server.fileutil import loads
+import pytest
+
+
+# the original function
+def _loads(data):
+ return Unserializer(
+ BytesIO(data),
+ strconfig=(False, False)).load(versioned=False)
+
+
+def test_execnet_opcodes():
+ # we need to make sure execnet doesn't change
+ assert list(Unserializer.num2func.keys()) == [
+ b'@', b'A', b'B', b'C', b'D', b'E', b'F', b'G', b'H', b'I', b'J', b'K',
+ b'L', b'M', b'N', b'O', b'P', b'Q', b'R', b'S', b'T']
+
+
[email protected]('data, expected', [
+ (b'@\x00\x00\x00\x00Q', ()),
+ (b'F\x00\x00\x00\x01@\x00\x00\x00\x01Q', (1,)),
+ (b'F\x00\x00\x00\x01F\x00\x00\x00\x02@\x00\x00\x00\x02Q', (1, 2)),
+ (b'A\x00\x00\x00\x01aQ', b'a'),
+ (b'A\x00\x00\x00\x02abQ', b'ab'),
+ (b'CQ', False),
+ (b'D\x00\x00\x00\x00\x00\x00\x00\x00Q', 0.0),
+ (b'D\x3f\xf0\x00\x00\x00\x00\x00\x00Q', 1.0),
+ (b'E\x00\x00\x00\x00Q', frozenset()),
+ (b'F\x00\x00\x00\x01E\x00\x00\x00\x01Q', frozenset((1,))),
+ (b'F\x00\x00\x00\x01F\x00\x00\x00\x02E\x00\x00\x00\x02Q', frozenset((1,
2))),
+ (b'F\x00\x00\x00\x01Q', int(1)),
+ (b'G\x00\x00\x00\x01Q', int(1)),
+ (b'H\x00\x00\x00\x011Q', int(1)),
+ (b'I\x00\x00\x00\x011Q', int(1)),
+ (b'JQ', {}),
+
(b'JF\x00\x00\x00\x00F\x00\x00\x00\x00PF\x00\x00\x00\x01F\x00\x00\x00\x02PQ',
{0: 0, 1: 2}),
+ (b'K\x00\x00\x00\x00Q', []),
+ (b'K\x00\x00\x00\x01Q', [None]),
+ (b'K\x00\x00\x00\x02Q', [None, None]),
+ (b'K\x00\x00\x00\x01F\x00\x00\x00\x00F\x00\x00\x00\x01PQ', [1]),
+
(b'K\x00\x00\x00\x02F\x00\x00\x00\x00F\x00\x00\x00\x01PF\x00\x00\x00\x01F\x00\x00\x00\x02PQ',
[1, 2]),
+ (b'K\x00\x00\x00\x01F\x00\x00\x00\x00RPQ', [True]),
+ (b'K\x00\x00\x00\x01F\x00\x00\x00\x00CPQ', [False]),
+ (b'LQ', None),
+ (b'M\x00\x00\x00\x01aQ', b'a'),
+ (b'M\x00\x00\x00\x02abQ', b'ab'),
+ (b'N\x00\x00\x00\x01aQ', 'a'),
+ (b'N\x00\x00\x00\x02\xc3\xa4Q', '??'),
+ (b'O\x00\x00\x00\x00Q', set()),
+ (b'F\x00\x00\x00\x01O\x00\x00\x00\x01Q', set((1,))),
+ (b'F\x00\x00\x00\x01F\x00\x00\x00\x02O\x00\x00\x00\x02Q', set((1, 2))),
+ (b'RQ', True),
+ (b'S\x00\x00\x00\x01aQ', 'a'),
+ (b'S\x00\x00\x00\x02\xc3\xa4Q', '??'),
+ (b'T\x3f\xf0\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00Q',
complex(1, 0)),
+ (b'T\x00\x00\x00\x00\x00\x00\x00\x00\x3f\xf0\x00\x00\x00\x00\x00\x00Q',
complex(0, 1)),
+ (b'T\x3f\xf0\x00\x00\x00\x00\x00\x00\x3f\xf0\x00\x00\x00\x00\x00\x00Q',
complex(1, 1))])
+def test_loads(data, expected):
+ result = loads(data)
+ assert result == expected
+ assert type(result) == type(expected)
+ # try round-trip
+ assert loads(dumps(expected)) == expected
+ # compare to original
+ assert result == _loads(data)
+ assert _loads(dumps(expected)) == expected
