Hello community,
here is the log from the commit of package python-devpi-server for
openSUSE:Factory checked in at 2020-04-02 17:45:00
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Comparing /work/SRC/openSUSE:Factory/python-devpi-server (Old)
and /work/SRC/openSUSE:Factory/.python-devpi-server.new.3248 (New)
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "python-devpi-server"
Thu Apr 2 17:45:00 2020 rev:4 rq:790871 version:5.4.1
Changes:
--------
--- /work/SRC/openSUSE:Factory/python-devpi-server/python-devpi-server.changes
2020-01-12 23:26:26.094861635 +0100
+++
/work/SRC/openSUSE:Factory/.python-devpi-server.new.3248/python-devpi-server.changes
2020-04-02 17:45:22.469530734 +0200
@@ -1,0 +2,18 @@
+Thu Apr 2 11:26:56 UTC 2020 - Marketa Calabkova <[email protected]>
+
+- Update to 5.4.1
+ * This is the last feature release with Python 2.7 support!
+ * Import won't abort anymore when a base index was removed.
+ The bases setting will be imported as is.
+ * The ``requires_python`` metadata is now included in version
+ data on mirror indexes.
+ * Downloaded files from mirrors can be included in exports with
+ the ``--include-mirrored-files`` option.
+ * On import files for mirror indexes are now imported when they
+ were included in the dump (see ``--include-mirrored-files``).
+ * Fix ``--no-root-pypi`` option when importing devpi data.
+ * Fix pushing from mirror to an index when the file was removed
+ and ``mirror_use_external_urls`` is active.
+- Drop Python2 anyway because of pyramid dropping Python2
+
+-------------------------------------------------------------------
Old:
----
devpi-server-5.3.1.tar.gz
New:
----
devpi-server-5.4.1.tar.gz
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Other differences:
------------------
++++++ python-devpi-server.spec ++++++
--- /var/tmp/diff_new_pack.sZ0Wnj/_old 2020-04-02 17:45:23.381531836 +0200
+++ /var/tmp/diff_new_pack.sZ0Wnj/_new 2020-04-02 17:45:23.381531836 +0200
@@ -17,8 +17,9 @@
%{?!python_module:%define python_module() python-%{**} python3-%{**}}
+%define skip_python2 1
Name: python-devpi-server
-Version: 5.3.1
+Version: 5.4.1
Release: 0
Summary: Private PyPI caching server
License: MIT
++++++ devpi-server-5.3.1.tar.gz -> devpi-server-5.4.1.tar.gz ++++++
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/devpi-server-5.3.1/CHANGELOG
new/devpi-server-5.4.1/CHANGELOG
--- old/devpi-server-5.3.1/CHANGELOG 2019-12-05 14:39:00.000000000 +0100
+++ new/devpi-server-5.4.1/CHANGELOG 2020-03-26 09:54:54.000000000 +0100
@@ -2,6 +2,50 @@
.. towncrier release notes start
+5.4.1 (2020-03-26)
+==================
+
+Bug Fixes
+---------
+
+- Import won't abort anymore when a base index was removed. The bases setting
will be imported as is.
+
+
+5.4.0 (2020-01-31)
+==================
+
+.. note::
+ This is the last feature release with Python 2.7 support!
+
+ We will only make export related bugfix releases of 5.4.x.
+
+Features
+--------
+
+- The ``requires_python`` metadata is now included in version data on mirror
indexes.
+
+- Downloaded files from mirrors can be included in exports with the
``--include-mirrored-files`` option.
+
+- On import files for mirror indexes are now imported when they were included
in the dump (see ``--include-mirrored-files``).
+
+
+Bug Fixes
+---------
+
+- Fix ``--no-root-pypi`` option when importing devpi data.
+
+- Fix pushing from mirror to an index when the file was removed and
``mirror_use_external_urls`` is active.
+
+
+5.3.1 (2019-12-05)
+==================
+
+Bug Fixes
+---------
+
+- fix #688: on file upload existing metadata is only updated, not replaced.
+
+
5.3.0 (2019-12-03)
==================
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/devpi-server-5.3.1/PKG-INFO
new/devpi-server-5.4.1/PKG-INFO
--- old/devpi-server-5.3.1/PKG-INFO 2019-12-05 14:39:07.000000000 +0100
+++ new/devpi-server-5.4.1/PKG-INFO 2020-03-26 09:54:56.000000000 +0100
@@ -1,6 +1,6 @@
Metadata-Version: 1.2
Name: devpi-server
-Version: 5.3.1
+Version: 5.4.1
Summary: devpi-server: reliable private and pypi.org caching server
Home-page: http://doc.devpi.net
Maintainer: Holger Krekel, Florian Schulze
@@ -79,6 +79,50 @@
.. towncrier release notes start
+ 5.4.1 (2020-03-26)
+ ==================
+
+ Bug Fixes
+ ---------
+
+ - Import won't abort anymore when a base index was removed. The bases
setting will be imported as is.
+
+
+ 5.4.0 (2020-01-31)
+ ==================
+
+ .. note::
+ This is the last feature release with Python 2.7 support!
+
+ We will only make export related bugfix releases of 5.4.x.
+
+ Features
+ --------
+
+ - The ``requires_python`` metadata is now included in version data on
mirror indexes.
+
+ - Downloaded files from mirrors can be included in exports with the
``--include-mirrored-files`` option.
+
+ - On import files for mirror indexes are now imported when they were
included in the dump (see ``--include-mirrored-files``).
+
+
+ Bug Fixes
+ ---------
+
+ - Fix ``--no-root-pypi`` option when importing devpi data.
+
+ - Fix pushing from mirror to an index when the file was removed and
``mirror_use_external_urls`` is active.
+
+
+ 5.3.1 (2019-12-05)
+ ==================
+
+ Bug Fixes
+ ---------
+
+ - fix #688: on file upload existing metadata is only updated, not
replaced.
+
+
5.3.0 (2019-12-03)
==================
@@ -155,103 +199,6 @@
- The timeout when fetching the list of remote projects for a mirror
index is set to a minimum of 30s by default and to 60s when running as replica.
Other fetches of mirrors still use the timeout specified via
``--request-timeout``.
- 5.1.0 (2019-08-05)
- ==================
-
- Features
- --------
-
- - Allow stage customizer plugins to filter projects and versions.
-
- - Replicas will use the multiple changelog endpoint added in
devpi-server 4.9.0 to reduce the number of requests necessary to synchronize
state.
-
-
- 5.0.0 (2019-06-28)
- ==================
-
- Deprecations and Removals
- -------------------------
-
- - fix #518: There are no URLs on PyPI anymore that need to be scraped
or crawled, so the code for that was removed.
-
- - removed support for long deprecated ``acl_upload`` and ``bases``
mirror index option. They were only kept for compatibility with devpi-client <=
2.4.1.
-
- - the ``--start``, ``--stop``, ``--status`` and ``--log`` options are
deprecated. Use ``--gen-config`` to create example configuration files for
various process managers.
-
- - removed long deprecated ``pypi_whitelist`` index option. It was only
kept for compatibility with devpi-client <= 2.4.1.
-
- - deprecated Python 2.7 support. This is the last major version
supporting Python 2.7. For upgrading to Python 3.x you have to export your data
using your current setup with Python 2.7 and import it in a new installation
with Python 3.x.
-
-
- Features
- --------
-
- - fix #249: unknown keys for index configuration now result in an
error instead of being silently ignored.
-
- - fix #625: project registration is now optional. A file upload with
twine or setuptools will automatically register the project.
-
- - fix #636: support ignore_bases argument for project listings.
-
- - support ``:AUTHENTICATED:`` for permissions. This resolves to any
user which is logged in, regardless of username or groups.
-
- - added experimental support for stage customizers to let plugins add
index types with customized behaviour. See ``BaseStageCustomizer`` in
``model.py`` for the API and ``devpiserver_get_stage_customizer_classes`` for
the registration.
-
- - support no_projects argument for index json requests. The list of
projects will not be added to the result.
-
- - when credentials for the user are rejected, the error message now
says so instead of claiming the user could not be found.
-
-
- Other Changes
- -------------
-
- - boolean values can now only be set via the following values:
'false', 'no', 'true', 'yes' and actual booleans in the REST API. Before any
string not matching 'false' and 'no' was converted into boolean true.
-
- - the default logging configuration now outputs to stdout instead of
stderr.
-
- - major releases don't require an export/import cycle anymore except
when explicitly announced. You should always make a backup though! When
upgrading to devpi-server 5.0.0 you can keep the state as is and even downgrade
to the last 4.9.x release if necessary. Don't forget to backup before upgrades!
-
- - the server secret isn't automatically persisted for new
installations. A server restart invalidates login tokens. An existing
installation will still use it's stored secret, but log a warning. Use
``--secretfile`` to explicitly specify a persistent secret file.
-
- - the ``--storage`` option is now required when a storage plugin like
devpi-postgresql is in use. It's recommended to use a configuration file for
devpi-server to have everything in one place (see ``--configfile``).
-
- - for the ``--logger-cfg`` yaml loading we now use ``safe_load`` of
``ruamel.yaml`` instead of ``load`` from ``pyyaml``.
-
-
- 4.9.0 (2019-04-26)
- ==================
-
- Features
- --------
-
- - implement #93: When creating a user, the password hash can be set
directly with ``pwhash``. Upon database initialization allow setting root user
password with ``--root-passwd`` and the password hash with
``--root-passwd-hash`` options. Thanks to Andreas Palsson.
-
- - decouple devpi server version from database version to enable major
releases that do not require export import of data
-
- - support ``--hard-links`` option during import for releases and doc
zips.
-
- - added new endpoint to download multiple changelog entries at once.
This will be used for faster replication in the future.
-
- - add option ``--replica-file-search-path`` to point to existing
files. If a match is found it will be copied locally instead of fetched from
the master. These files could be from a previous replication attempt or
separately copied/restored.
-
- - add ``--hard-links`` support for replicas together with the
``--replica-file-search-path`` option. When a matching file is found it's hard
linked instead of writing a copy.
-
-
- Bug Fixes
- ---------
-
- - fix multiple triggering of mirror project names initialization.
-
- - fix updating time stamp of mirror project name cache when no project
names have changed. This makes subsequent fetches actually use the cache
instead of always fetching the data again from the mirror.
-
- - use timeout when waiting for data from master in replica on mirror
simple pages.
-
-
- Other Changes
- -------------
-
- - slightly improved replica performance by removing unnecessary DB
read and using fewer transactions.
-
-
Keywords: pypi realtime cache server
Platform: UNKNOWN
Classifier: Development Status :: 5 - Production/Stable
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/devpi-server-5.3.1/devpi_server/__init__.py
new/devpi-server-5.4.1/devpi_server/__init__.py
--- old/devpi-server-5.3.1/devpi_server/__init__.py 2019-12-05
14:39:00.000000000 +0100
+++ new/devpi-server-5.4.1/devpi_server/__init__.py 2020-03-26
09:54:54.000000000 +0100
@@ -1 +1 @@
-__version__ = '5.3.1'
+__version__ = '5.4.1'
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/devpi-server-5.3.1/devpi_server/config.py
new/devpi-server-5.4.1/devpi_server/config.py
--- old/devpi-server-5.3.1/devpi_server/config.py 2019-12-05
14:39:00.000000000 +0100
+++ new/devpi-server-5.4.1/devpi_server/config.py 2020-03-26
09:54:54.000000000 +0100
@@ -256,6 +256,10 @@
"This will export all users, indices, release files "
"(except for mirrors), test results and documentation.")
+ parser.addoption(
+ "--include-mirrored-files", action="store_true",
+ help="include downloaded files from mirror indexes in dump.")
+
def add_import_options(parser, pluginmanager, standalone=True):
if not standalone:
@@ -645,6 +649,14 @@
return getattr(self.args, 'offline_mode', False)
@property
+ def replica_cert(self):
+ return getattr(self.args, 'replica_cert', None)
+
+ @property
+ def replica_max_retries(self):
+ return getattr(self.args, 'replica_max_retries', None)
+
+ @property
def requests_only(self):
return getattr(self.args, 'requests_only', False)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/devpi-server-5.3.1/devpi_server/extpypi.py
new/devpi-server-5.4.1/devpi_server/extpypi.py
--- old/devpi-server-5.3.1/devpi_server/extpypi.py 2019-12-05
14:39:00.000000000 +0100
+++ new/devpi-server-5.4.1/devpi_server/extpypi.py 2020-03-26
09:54:54.000000000 +0100
@@ -373,7 +373,7 @@
if self.offline and links is None:
raise self.UpstreamError("offline mode")
if self.offline or not is_expired:
- if project not in self._offline_logging:
+ if self.offline and project not in self._offline_logging:
threadlog.debug(
"using stale links for %r due to offline mode", project)
self._offline_logging.add(project)
@@ -531,6 +531,8 @@
if not verdata:
verdata['name'] = project
verdata['version'] = version
+ if sm.require_python is not None:
+ verdata['requires_python'] = sm.require_python
elinks = verdata.setdefault("+elinks", [])
entrypath = sm._url.path
elinks.append({"rel": "releasefile", "entrypath": entrypath})
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/devpi-server-5.3.1/devpi_server/importexport.py
new/devpi-server-5.4.1/devpi_server/importexport.py
--- old/devpi-server-5.3.1/devpi_server/importexport.py 2019-12-05
14:39:00.000000000 +0100
+++ new/devpi-server-5.4.1/devpi_server/importexport.py 2020-03-26
09:54:54.000000000 +0100
@@ -9,11 +9,13 @@
from devpi_common.validation import normalize_name
from devpi_common.metadata import BasenameMeta
from devpi_common.types import parse_hash_spec
+from devpi_common.url import URL
from devpi_server import __version__ as server_version
from devpi_server.model import is_valid_name
from devpi_server.model import get_stage_customizer_classes
from .config import MyArgumentParser
from .config import add_configfile_option
+from .config import add_export_options
from .config import add_hard_links_option
from .config import add_help_option
from .config import add_import_options
@@ -81,6 +83,7 @@
add_help_option(parser, pluginmanager)
add_configfile_option(parser, pluginmanager)
add_storage_options(parser, pluginmanager)
+ add_export_options(parser, pluginmanager)
add_hard_links_option(parser, pluginmanager)
parser.add_argument("directory")
config = parseoptions(pluginmanager, argv, parser=parser)
@@ -238,10 +241,16 @@
self.indexmeta = exporter.export_indexes[stage.name] = {}
self.indexmeta["indexconfig"] = stage.ixconfig
- def dump(self):
+ def should_dump(self):
if self.stage.ixconfig["type"] == "mirror":
- projects = []
- else:
+ if not self.exporter.config.args.include_mirrored_files:
+ return False
+ return True
+
+ def dump(self):
+ projects = []
+ if self.should_dump():
+ self.stage.offline = True
self.indexmeta["projects"] = {}
self.indexmeta["files"] = []
projects = self.stage.list_projects_perstage()
@@ -263,7 +272,9 @@
self.basedir.ensure(dir=1)
self.dump_releasefiles(linkstore)
self.dump_toxresults(linkstore)
- entry = self.stage.get_doczip_entry(vername, version)
+ entry = None
+ if hasattr(self.stage, 'get_doczip_entry'):
+ entry = self.stage.get_doczip_entry(vername, version)
if entry:
self.dump_docfile(vername, version, entry)
self.exporter.completed("index %r" % self.stage.name)
@@ -271,6 +282,8 @@
def dump_releasefiles(self, linkstore):
for link in linkstore.get_links(rel="releasefile"):
entry = self.exporter.filestore.get_file_entry(link.entrypath)
+ if not entry.last_modified:
+ continue
assert entry.file_exists(), entry.relpath
relpath = self.exporter.copy_file(
entry,
@@ -334,10 +347,8 @@
total_num_projects = 0
total_num_files = 0
for idx_name, idx in self.import_indexes.items():
- if idx['indexconfig']['type'] == 'mirror':
- continue
- num_projects = len(idx['projects'])
- num_files = len(idx['files'])
+ num_projects = len(idx.get('projects', {}))
+ num_files = len(idx.get('files', []))
self.tw.line(
'Index %s has %d projects and %d files'
% (idx_name, num_projects, num_files))
@@ -409,30 +420,42 @@
# first create all users
with self.xom.keyfs.transaction(write=True):
for username, userconfig in self.import_users.items():
+ user = None
if username == "root":
user = self.xom.model.get_user(username)
- else:
+ if user is None:
user = self.xom.model.create_user(username, password="")
user._set(userconfig)
# memorize index inheritance structure
tree = IndexTree()
+ indexes = set(self.import_indexes)
with self.xom.keyfs.transaction(write=False):
stage = self.xom.model.getstage("root/pypi")
if stage is not None:
+ indexes.add("root/pypi")
tree.add("root/pypi")
+ missing_bases = set()
for stagename, import_index in self.import_indexes.items():
bases = import_index["indexconfig"].get("bases")
- tree.add(stagename, bases)
+ if bases is None:
+ tree.add(stagename)
+ else:
+ existing_bases = set(bases).intersection(indexes)
+ missing_bases.update(set(bases) - existing_bases)
+ tree.add(stagename, existing_bases)
+
+ if missing_bases:
+ self.warn(
+ "The following indexes are in bases, but don't exist "
+ "in the import data: %s" % ", ".join(sorted(missing_bases)))
# create stages in inheritance/root-first order
stages = []
with self.xom.keyfs.transaction(write=True):
for stagename in tree.iternames():
- if stagename == "root/pypi":
- stage = self.xom.model.getstage(stagename)
- if stage is not None:
- continue
+ if stagename == "root/pypi" and stagename not in
self.import_indexes:
+ continue
import_index = self.import_indexes[stagename]
indexconfig = dict(import_index["indexconfig"])
if indexconfig['type'] in self.types_to_skip:
@@ -454,22 +477,30 @@
# newer versions don't. To support exports from both we
# have the default None value
bases = indexconfig.pop('bases', None)
- stage = user.create_stage(index, **indexconfig)
+ stage = None
+ if stagename == "root/pypi":
+ stage = self.xom.model.getstage(stagename)
+ if stage is not None:
+ stage.modify(**indexconfig)
+ elif self.xom.config.args.no_root_pypi:
+ continue
+ if stage is None:
+ stage = user.create_stage(index, **indexconfig)
if "bases" in import_index["indexconfig"]:
- indexconfig = stage.ixconfig
- indexconfig["bases"] = bases
- stage.modify(**indexconfig)
+ # we are changing bases directly to allow import with
+ # removed bases without changing the data from the export
+ with stage.user.key.update() as userconfig:
+ indexconfig = userconfig['indexes'][stage.index]
+ indexconfig["bases"] = tuple(bases)
stages.append(stage)
del tree
# create projects and releasefiles for each index
for stage in stages:
- if stage.ixconfig["type"] == "mirror":
- continue
imported_files = set()
import_index = self.import_indexes[stage.name]
- projects = import_index["projects"]
- files = import_index["files"]
+ projects = import_index.get("projects", {})
+ files = import_index.get("files", [])
for project, versions in self.iter_projects_normalized(projects):
with self.xom.keyfs.transaction(write=True):
for version, versiondata in versions.items():
@@ -483,13 +514,16 @@
"version, setting derived %r" %
(name, version))
versiondata["version"] = version
- stage.set_versiondata(versiondata)
+ if hasattr(stage, 'set_versiondata'):
+ stage.set_versiondata(versiondata)
+ else:
+ stage.add_project_name(versiondata["name"])
# import release files
for filedesc in files:
if normalize_name(filedesc["projectname"]) ==
normalize_name(project):
imported_files.add(filedesc["relpath"])
- self.import_filedesc(stage, filedesc)
+ self.import_filedesc(stage, filedesc, versions)
missing = set(x["relpath"] for x in files) - imported_files
if missing:
fatal(
@@ -511,8 +545,7 @@
self.tw.line("wait_for_events: importing finished"
"; latest_serial = %s" % latest_serial)
- def import_filedesc(self, stage, filedesc):
- assert stage.ixconfig["type"] != "mirror"
+ def import_filedesc(self, stage, filedesc, versions):
rel = filedesc["relpath"]
project = filedesc["projectname"]
p = self.import_rootdir.join(rel)
@@ -530,15 +563,33 @@
else:
version = filedesc["version"]
- link = stage.store_releasefile(project, version,
- p.basename, data,
-
last_modified=mapping["last_modified"])
+ if hasattr(stage, 'store_releasefile'):
+ link = stage.store_releasefile(
+ project, version,
+ p.basename, data,
+ last_modified=mapping["last_modified"])
+ entry = link.entry
+ else:
+ link = None
+ url =
URL(mapping['url']).replace(fragment=mapping['hash_spec'])
+ entry = self.xom.filestore.maplink(
+ url, stage.username, stage.index, project)
+ entry.file_set_content(data, mapping["last_modified"])
+ (_, links_with_require_python, serial) =
stage._load_cache_links(project)
+ if links_with_require_python is None:
+ links_with_require_python = []
+ links = [(url.basename, entry.relpath)]
+ requires_python = [versions[version].get('requires_python')]
+ for key, href, require_python in links_with_require_python:
+ links.append((key, href))
+ requires_python.append(require_python)
+ stage._save_cache_links(project, links, requires_python,
serial)
# devpi-server-2.1 exported with md5 checksums
if "md5" in mapping:
assert "hash_spec" not in mapping
mapping["hash_spec"] = "md5=" + mapping["md5"]
hash_algo, hash_value = parse_hash_spec(mapping["hash_spec"])
- digest = hash_algo(link.entry.file_get_content()).hexdigest()
+ digest = hash_algo(entry.file_get_content()).hexdigest()
if digest != hash_value:
fatal("File %s has bad checksum %s, expected %s" % (
p, digest, hash_value))
@@ -557,11 +608,12 @@
link = stage.store_toxresult(link, json.loads(data.decode("utf8")))
else:
fatal("unknown file type: %s" % (type,))
- history_log = filedesc.get('log')
- if history_log is None:
- link.add_log('upload', '<import>', dst=stage.name)
- else:
- link.add_logs(history_log)
+ if link is not None:
+ history_log = filedesc.get('log')
+ if history_log is None:
+ link.add_log('upload', '<import>', dst=stage.name)
+ else:
+ link.add_logs(history_log)
class IndexTree:
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/devpi-server-5.3.1/devpi_server/keyfs.py
new/devpi-server-5.4.1/devpi_server/keyfs.py
--- old/devpi-server-5.3.1/devpi_server/keyfs.py 2019-12-05
14:39:00.000000000 +0100
+++ new/devpi-server-5.4.1/devpi_server/keyfs.py 2020-03-26
09:54:54.000000000 +0100
@@ -3,7 +3,7 @@
basic python types based on parametrizable keys. Multiple
read Transactions can execute concurrently while at most one
write Transaction is ongoing. Each Transaction will see a consistent
-view of key/values refering to the point in time it was started,
+view of key/values referring to the point in time it was started,
independent from any future changes.
"""
from __future__ import unicode_literals
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/devpi-server-5.3.1/devpi_server/main.py
new/devpi-server-5.4.1/devpi_server/main.py
--- old/devpi-server-5.3.1/devpi_server/main.py 2019-12-05 14:39:00.000000000
+0100
+++ new/devpi-server-5.4.1/devpi_server/main.py 2020-03-26 09:54:54.000000000
+0100
@@ -154,7 +154,8 @@
xom = xom_from_config(config)
- init_default_indexes(xom)
+ if args.init:
+ init_default_indexes(xom)
if args.start or args.stop or args.log or args.status:
xprocdir = config.serverdir.join(".xproc")
@@ -358,15 +359,16 @@
def new_http_session(self, component_name, max_retries=None):
session = new_requests_session(agent=(component_name, server_version),
max_retries=max_retries)
- session.cert = self.config.args.replica_cert
+ session.cert = self.config.replica_cert
return session
@cached_property
def _httpsession(self):
- return self.new_http_session("server",
max_retries=self.config.args.replica_max_retries)
+ max_retries = self.config.replica_max_retries
+ return self.new_http_session("server", max_retries=max_retries)
def httpget(self, url, allow_redirects, timeout=None, extra_headers=None):
- if self.config.args.offline_mode:
+ if self.config.offline_mode:
resp = Response()
resp.status_code = 503 # service unavailable
return resp
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/devpi-server-5.3.1/devpi_server/model.py
new/devpi-server-5.4.1/devpi_server/model.py
--- old/devpi-server-5.3.1/devpi_server/model.py 2019-12-05
14:39:00.000000000 +0100
+++ new/devpi-server-5.4.1/devpi_server/model.py 2020-03-26
09:54:54.000000000 +0100
@@ -834,13 +834,12 @@
return newconfig
def modify(self, index=None, **kw):
+ if self.customizer.readonly:
+ raise ReadonlyIndex("index is marked read only")
newconfig = self._modify(**kw)
threadlog.info("modified index %s: %s", self.name, newconfig)
return newconfig
- if self.customizer.readonly:
- raise ReadonlyIndex("index is marked read only")
-
def op_sro(self, opname, **kw):
if "project" in kw:
project = normalize_name(kw["project"])
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/devpi-server-5.3.1/devpi_server/views.py
new/devpi-server-5.4.1/devpi_server/views.py
--- old/devpi-server-5.3.1/devpi_server/views.py 2019-12-05
14:39:00.000000000 +0100
+++ new/devpi-server-5.4.1/devpi_server/views.py 2020-03-26
09:54:54.000000000 +0100
@@ -1397,11 +1397,16 @@
# the file was downloaded before but locally removed, so put
# it back in place without creating a new serial
# we need a direct write connection to use the io_file_* methods
- with xom.keyfs._storage.get_connection(write=True) as conn:
- conn.io_file_set(entry._storepath, content)
+ if tx is not None:
+ tx.conn.io_file_set(entry._storepath, content)
threadlog.debug(
"put missing file back into place: %s", entry._storepath)
- conn.commit_files_without_increasing_serial()
+ else:
+ with xom.keyfs._storage.get_connection(write=True) as conn:
+ conn.io_file_set(entry._storepath, content)
+ threadlog.debug(
+ "put missing file back into place: %s", entry._storepath)
+ conn.commit_files_without_increasing_serial()
def iter_remote_file_replica(xom, entry):
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/devpi-server-5.3.1/devpi_server.egg-info/PKG-INFO
new/devpi-server-5.4.1/devpi_server.egg-info/PKG-INFO
--- old/devpi-server-5.3.1/devpi_server.egg-info/PKG-INFO 2019-12-05
14:39:07.000000000 +0100
+++ new/devpi-server-5.4.1/devpi_server.egg-info/PKG-INFO 2020-03-26
09:54:55.000000000 +0100
@@ -1,6 +1,6 @@
Metadata-Version: 1.2
Name: devpi-server
-Version: 5.3.1
+Version: 5.4.1
Summary: devpi-server: reliable private and pypi.org caching server
Home-page: http://doc.devpi.net
Maintainer: Holger Krekel, Florian Schulze
@@ -79,6 +79,50 @@
.. towncrier release notes start
+ 5.4.1 (2020-03-26)
+ ==================
+
+ Bug Fixes
+ ---------
+
+ - Import won't abort anymore when a base index was removed. The bases
setting will be imported as is.
+
+
+ 5.4.0 (2020-01-31)
+ ==================
+
+ .. note::
+ This is the last feature release with Python 2.7 support!
+
+ We will only make export related bugfix releases of 5.4.x.
+
+ Features
+ --------
+
+ - The ``requires_python`` metadata is now included in version data on
mirror indexes.
+
+ - Downloaded files from mirrors can be included in exports with the
``--include-mirrored-files`` option.
+
+ - On import files for mirror indexes are now imported when they were
included in the dump (see ``--include-mirrored-files``).
+
+
+ Bug Fixes
+ ---------
+
+ - Fix ``--no-root-pypi`` option when importing devpi data.
+
+ - Fix pushing from mirror to an index when the file was removed and
``mirror_use_external_urls`` is active.
+
+
+ 5.3.1 (2019-12-05)
+ ==================
+
+ Bug Fixes
+ ---------
+
+ - fix #688: on file upload existing metadata is only updated, not
replaced.
+
+
5.3.0 (2019-12-03)
==================
@@ -155,103 +199,6 @@
- The timeout when fetching the list of remote projects for a mirror
index is set to a minimum of 30s by default and to 60s when running as replica.
Other fetches of mirrors still use the timeout specified via
``--request-timeout``.
- 5.1.0 (2019-08-05)
- ==================
-
- Features
- --------
-
- - Allow stage customizer plugins to filter projects and versions.
-
- - Replicas will use the multiple changelog endpoint added in
devpi-server 4.9.0 to reduce the number of requests necessary to synchronize
state.
-
-
- 5.0.0 (2019-06-28)
- ==================
-
- Deprecations and Removals
- -------------------------
-
- - fix #518: There are no URLs on PyPI anymore that need to be scraped
or crawled, so the code for that was removed.
-
- - removed support for long deprecated ``acl_upload`` and ``bases``
mirror index option. They were only kept for compatibility with devpi-client <=
2.4.1.
-
- - the ``--start``, ``--stop``, ``--status`` and ``--log`` options are
deprecated. Use ``--gen-config`` to create example configuration files for
various process managers.
-
- - removed long deprecated ``pypi_whitelist`` index option. It was only
kept for compatibility with devpi-client <= 2.4.1.
-
- - deprecated Python 2.7 support. This is the last major version
supporting Python 2.7. For upgrading to Python 3.x you have to export your data
using your current setup with Python 2.7 and import it in a new installation
with Python 3.x.
-
-
- Features
- --------
-
- - fix #249: unknown keys for index configuration now result in an
error instead of being silently ignored.
-
- - fix #625: project registration is now optional. A file upload with
twine or setuptools will automatically register the project.
-
- - fix #636: support ignore_bases argument for project listings.
-
- - support ``:AUTHENTICATED:`` for permissions. This resolves to any
user which is logged in, regardless of username or groups.
-
- - added experimental support for stage customizers to let plugins add
index types with customized behaviour. See ``BaseStageCustomizer`` in
``model.py`` for the API and ``devpiserver_get_stage_customizer_classes`` for
the registration.
-
- - support no_projects argument for index json requests. The list of
projects will not be added to the result.
-
- - when credentials for the user are rejected, the error message now
says so instead of claiming the user could not be found.
-
-
- Other Changes
- -------------
-
- - boolean values can now only be set via the following values:
'false', 'no', 'true', 'yes' and actual booleans in the REST API. Before any
string not matching 'false' and 'no' was converted into boolean true.
-
- - the default logging configuration now outputs to stdout instead of
stderr.
-
- - major releases don't require an export/import cycle anymore except
when explicitly announced. You should always make a backup though! When
upgrading to devpi-server 5.0.0 you can keep the state as is and even downgrade
to the last 4.9.x release if necessary. Don't forget to backup before upgrades!
-
- - the server secret isn't automatically persisted for new
installations. A server restart invalidates login tokens. An existing
installation will still use it's stored secret, but log a warning. Use
``--secretfile`` to explicitly specify a persistent secret file.
-
- - the ``--storage`` option is now required when a storage plugin like
devpi-postgresql is in use. It's recommended to use a configuration file for
devpi-server to have everything in one place (see ``--configfile``).
-
- - for the ``--logger-cfg`` yaml loading we now use ``safe_load`` of
``ruamel.yaml`` instead of ``load`` from ``pyyaml``.
-
-
- 4.9.0 (2019-04-26)
- ==================
-
- Features
- --------
-
- - implement #93: When creating a user, the password hash can be set
directly with ``pwhash``. Upon database initialization allow setting root user
password with ``--root-passwd`` and the password hash with
``--root-passwd-hash`` options. Thanks to Andreas Palsson.
-
- - decouple devpi server version from database version to enable major
releases that do not require export import of data
-
- - support ``--hard-links`` option during import for releases and doc
zips.
-
- - added new endpoint to download multiple changelog entries at once.
This will be used for faster replication in the future.
-
- - add option ``--replica-file-search-path`` to point to existing
files. If a match is found it will be copied locally instead of fetched from
the master. These files could be from a previous replication attempt or
separately copied/restored.
-
- - add ``--hard-links`` support for replicas together with the
``--replica-file-search-path`` option. When a matching file is found it's hard
linked instead of writing a copy.
-
-
- Bug Fixes
- ---------
-
- - fix multiple triggering of mirror project names initialization.
-
- - fix updating time stamp of mirror project name cache when no project
names have changed. This makes subsequent fetches actually use the cache
instead of always fetching the data again from the mirror.
-
- - use timeout when waiting for data from master in replica on mirror
simple pages.
-
-
- Other Changes
- -------------
-
- - slightly improved replica performance by removing unnecessary DB
read and using fewer transactions.
-
-
Keywords: pypi realtime cache server
Platform: UNKNOWN
Classifier: Development Status :: 5 - Production/Stable
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/devpi-server-5.3.1/devpi_server.egg-info/SOURCES.txt
new/devpi-server-5.4.1/devpi_server.egg-info/SOURCES.txt
--- old/devpi-server-5.3.1/devpi_server.egg-info/SOURCES.txt 2019-12-05
14:39:07.000000000 +0100
+++ new/devpi-server-5.4.1/devpi_server.egg-info/SOURCES.txt 2020-03-26
09:54:55.000000000 +0100
@@ -2,6 +2,7 @@
LICENSE
MANIFEST.in
README.rst
+pyproject.toml
setup.cfg
setup.py
tox.ini
@@ -89,11 +90,17 @@
test_devpi_server/importexportdata/badindexname/dataindex.json
test_devpi_server/importexportdata/badusername/dataindex.json
test_devpi_server/importexportdata/basescycle/dataindex.json
+test_devpi_server/importexportdata/deletedbase/dataindex.json
+test_devpi_server/importexportdata/mirrordata/dataindex.json
+test_devpi_server/importexportdata/mirrordata/root/pypi/dddttt/0.1.dev1/dddttt-0.1.dev1.tar.gz
+test_devpi_server/importexportdata/modifiedpypi/dataindex.json
test_devpi_server/importexportdata/normalization/dataindex.json
test_devpi_server/importexportdata/normalization/root/dev/hello.pkg/hello.pkg-1.0.tar.gz
test_devpi_server/importexportdata/normalization_merge/dataindex.json
test_devpi_server/importexportdata/normalization_merge/root/dev/hello-pkg/hello.pkg-1.1.tar.gz
test_devpi_server/importexportdata/normalization_merge/root/dev/hello.pkg/hello.pkg-1.0.tar.gz
+test_devpi_server/importexportdata/norootpypi/dataindex.json
+test_devpi_server/importexportdata/nouser/dataindex.json
test_devpi_server/importexportdata/removedindexplugin/dataindex.json
test_devpi_server/importexportdata/removedindexplugin/user/dev/pkg/pkg-1.0.tar.gz
test_devpi_server/importexportdata/toxresult_upload_default/dataindex.json
\ No newline at end of file
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/devpi-server-5.3.1/pyproject.toml
new/devpi-server-5.4.1/pyproject.toml
--- old/devpi-server-5.3.1/pyproject.toml 1970-01-01 01:00:00.000000000
+0100
+++ new/devpi-server-5.4.1/pyproject.toml 2020-03-26 09:54:54.000000000
+0100
@@ -0,0 +1,26 @@
+[tool.towncrier]
+package = "devpi_server"
+filename = "CHANGELOG"
+directory = "news/"
+title_format = "{version} ({project_date})"
+template = "news/_template.rst"
+
+ [[tool.towncrier.type]]
+ directory = "removal"
+ name = "Deprecations and Removals"
+ showcontent = true
+
+ [[tool.towncrier.type]]
+ directory = "feature"
+ name = "Features"
+ showcontent = true
+
+ [[tool.towncrier.type]]
+ directory = "bugfix"
+ name = "Bug Fixes"
+ showcontent = true
+
+ [[tool.towncrier.type]]
+ directory = "other"
+ name = "Other Changes"
+ showcontent = true
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/devpi-server-5.3.1/setup.py
new/devpi-server-5.4.1/setup.py
--- old/devpi-server-5.3.1/setup.py 2019-12-05 14:39:00.000000000 +0100
+++ new/devpi-server-5.4.1/setup.py 2020-03-26 09:54:54.000000000 +0100
@@ -4,7 +4,7 @@
import re
import io
import sys
-from setuptools import setup, find_packages
+from setuptools import setup
def get_changelog():
@@ -48,10 +48,15 @@
keywords="pypi realtime cache server",
long_description="\n\n".join([README, CHANGELOG]),
url="http://doc.devpi.net";,
- version='5.3.1',
+ version='5.4.1',
maintainer="Holger Krekel, Florian Schulze",
maintainer_email="[email protected]",
- packages=find_packages(),
+ packages=[
+ 'devpi_server',
+ 'devpi_server.cfg',
+ 'devpi_server.vendor',
+ 'pytest_devpi_server',
+ 'test_devpi_server'],
include_package_data=True,
zip_safe=False,
license="MIT",
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/devpi-server-5.3.1/test_devpi_server/conftest.py
new/devpi-server-5.4.1/test_devpi_server/conftest.py
--- old/devpi-server-5.3.1/test_devpi_server/conftest.py 2019-12-05
14:39:00.000000000 +0100
+++ new/devpi-server-5.4.1/test_devpi_server/conftest.py 2020-03-26
09:54:54.000000000 +0100
@@ -96,13 +96,12 @@
return caplog
@pytest.fixture
-def gentmp(request):
- tmpdirhandler = request.config._tmpdirhandler
+def gentmp(request, tmpdir_factory):
cache = []
def gentmp(name=None):
if not cache:
prefix = re.sub(r"[\W]", "_", request.node.name)
- basedir = tmpdirhandler.mktemp(prefix, numbered=True)
+ basedir = tmpdir_factory.mktemp(prefix, numbered=True)
cache.append(basedir)
else:
basedir = cache[0]
@@ -234,10 +233,9 @@
lambda self: set())
add_pypistage_mocks(monkeypatch, httpget)
# initialize default indexes
- from devpi_server.main import set_default_indexes
+ from devpi_server.main import init_default_indexes
if not xom.config.args.master_url:
- with xom.keyfs.transaction(write=True):
- set_default_indexes(xom.model)
+ init_default_indexes(xom)
if request.node.get_closest_marker("with_replica_thread"):
from devpi_server.replica import ReplicaThread
rt = ReplicaThread(xom)
@@ -912,7 +910,7 @@
"The port %s on host %s didn't become accessible" % (port, host))
[email protected]_fixture(scope="module")
[email protected]_fixture(scope="class")
def server_directory():
import tempfile
srvdir = py.path.local(
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/devpi-server-5.3.1/test_devpi_server/importexportdata/deletedbase/dataindex.json
new/devpi-server-5.4.1/test_devpi_server/importexportdata/deletedbase/dataindex.json
---
old/devpi-server-5.3.1/test_devpi_server/importexportdata/deletedbase/dataindex.json
1970-01-01 01:00:00.000000000 +0100
+++
new/devpi-server-5.4.1/test_devpi_server/importexportdata/deletedbase/dataindex.json
2020-03-26 09:54:54.000000000 +0100
@@ -0,0 +1,67 @@
+{
+ "users": {
+ "root": {
+ "username": "root",
+ "pwsalt": "uBLR5DKWtF6ro4H7h3FObQ==",
+ "pwhash":
"667a21361c2d700d6451ecc77c837a595329a74fa5b49c396c6aa6c49600bad6"
+ }
+ },
+ "secret": "ItYAZLP23wgW4/F2uFMNJ+ffjYhIiZg3T3HJlIzVTB8=",
+ "indexes": {
+ "root/dev1": {
+ "files": [],
+ "indexconfig": {
+ "bases": ["root/removed"], "pypi_whitelist": [],
+ "acl_upload": ["root"],
+ "volatile": true, "type": "stage"
+ },
+ "projects": []
+ },
+ "root/dev2": {
+ "files": [],
+ "indexconfig": {
+ "bases": ["root/removed"], "pypi_whitelist": [],
+ "acl_upload": ["root"],
+ "volatile": true, "type": "stage"
+ },
+ "projects": []
+ },
+ "root/dev3": {
+ "files": [],
+ "indexconfig": {
+ "bases": ["root/dev2"], "pypi_whitelist": [],
+ "acl_upload": ["root"],
+ "volatile": true, "type": "stage"
+ },
+ "projects": []
+ },
+ "root/dev4": {
+ "files": [],
+ "indexconfig": {
+ "bases": ["root/removed", "root/pypi"], "pypi_whitelist": [],
+ "acl_upload": ["root"],
+ "volatile": true, "type": "stage"
+ },
+ "projects": []
+ },
+ "root/dev5": {
+ "files": [],
+ "indexconfig": {
+ "bases": ["root/removed", "root/dev2"], "pypi_whitelist": [],
+ "acl_upload": ["root"],
+ "volatile": true, "type": "stage"
+ },
+ "projects": []
+ }
+ },
+ "pythonversion": [
+ 3,
+ 4,
+ 3,
+ "final",
+ 0
+ ],
+ "dumpversion": "2",
+ "devpi_server": "3.0.0b1",
+ "uuid": "3885c05ceae34ed4b7574da2a1bd1e0f"
+}
\ No newline at end of file
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/devpi-server-5.3.1/test_devpi_server/importexportdata/mirrordata/dataindex.json
new/devpi-server-5.4.1/test_devpi_server/importexportdata/mirrordata/dataindex.json
---
old/devpi-server-5.3.1/test_devpi_server/importexportdata/mirrordata/dataindex.json
1970-01-01 01:00:00.000000000 +0100
+++
new/devpi-server-5.4.1/test_devpi_server/importexportdata/mirrordata/dataindex.json
2020-03-26 09:54:54.000000000 +0100
@@ -0,0 +1,53 @@
+{
+ "users": {
+ "root": {
+ "pwhash":
"$argon2id$v=19$m=102400,t=2,p=8$rzUGQKiVsrZ2DkEIIQTA+A$tX1MOZ8fl1T4QrfgEwXj9Q",
+ "username": "root"
+ }
+ },
+ "indexes": {
+ "root/pypi": {
+ "indexconfig": {
+ "type": "mirror",
+ "volatile": false,
+ "title": "PyPI",
+ "mirror_url": "https://pypi.org/simple/";,
+ "mirror_web_url_fmt": "https://pypi.org/project/{name}/";
+ },
+ "projects": {
+ "dddttt": {
+ "0.1.dev1": {
+ "name": "dddttt",
+ "version": "0.1.dev1"
+ }
+ }
+ },
+ "files": [
+ {
+ "version": "0.1.dev1",
+ "entrymapping": {
+ "url":
"https://files.pythonhosted.org/packages/d0/c9/64c9fae84124fec0c08fa1b2db6259fb1ee894d4f41b5d0a79242c7709b7/dddttt-0.1.dev1.tar.gz";,
+ "hash_spec":
"sha256=1007f0cd10aaa290eb84f092e786639bbe930b7f2169f51f755a0f50a6aba489",
+ "project": "dddttt",
+ "version": "0.1.dev1",
+ "last_modified": "Sat, 23 Apr 2016 07:01:31 GMT"
+ },
+ "log": [],
+ "type": "releasefile",
+ "projectname": "dddttt",
+ "relpath": "root/pypi/dddttt/0.1.dev1/dddttt-0.1.dev1.tar.gz"
+ }
+ ]
+ }
+ },
+ "dumpversion": "2",
+ "pythonversion": [
+ 3,
+ 6,
+ 8,
+ "final",
+ 0
+ ],
+ "devpi_server": "5.3.1",
+ "uuid": "0846290854e347f782fd65dd529cde40"
+}
\ No newline at end of file
Binary files
old/devpi-server-5.3.1/test_devpi_server/importexportdata/mirrordata/root/pypi/dddttt/0.1.dev1/dddttt-0.1.dev1.tar.gz
and
new/devpi-server-5.4.1/test_devpi_server/importexportdata/mirrordata/root/pypi/dddttt/0.1.dev1/dddttt-0.1.dev1.tar.gz
differ
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/devpi-server-5.3.1/test_devpi_server/importexportdata/modifiedpypi/dataindex.json
new/devpi-server-5.4.1/test_devpi_server/importexportdata/modifiedpypi/dataindex.json
---
old/devpi-server-5.3.1/test_devpi_server/importexportdata/modifiedpypi/dataindex.json
1970-01-01 01:00:00.000000000 +0100
+++
new/devpi-server-5.4.1/test_devpi_server/importexportdata/modifiedpypi/dataindex.json
2020-03-26 09:54:54.000000000 +0100
@@ -0,0 +1,31 @@
+{
+ "users": {
+ "root": {
+ "pwhash":
"$argon2id$v=19$m=102400,t=2,p=8$rzUGQKiVsrZ2DkEIIQTA+A$tX1MOZ8fl1T4QrfgEwXj9Q",
+ "username": "root"
+ }
+ },
+ "indexes": {
+ "root/pypi": {
+ "indexconfig": {
+ "type": "mirror",
+ "volatile": false,
+ "title": "Modified PyPI",
+ "mirror_url": "https://example.com/simple/";,
+ "mirror_web_url_fmt": "https://example.com/project/{name}/";
+ },
+ "projects": {},
+ "files": []
+ }
+ },
+ "dumpversion": "2",
+ "pythonversion": [
+ 3,
+ 6,
+ 8,
+ "final",
+ 0
+ ],
+ "devpi_server": "5.3.1",
+ "uuid": "0846290854e347f782fd65dd529cde40"
+}
\ No newline at end of file
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/devpi-server-5.3.1/test_devpi_server/importexportdata/norootpypi/dataindex.json
new/devpi-server-5.4.1/test_devpi_server/importexportdata/norootpypi/dataindex.json
---
old/devpi-server-5.3.1/test_devpi_server/importexportdata/norootpypi/dataindex.json
1970-01-01 01:00:00.000000000 +0100
+++
new/devpi-server-5.4.1/test_devpi_server/importexportdata/norootpypi/dataindex.json
2020-03-26 09:54:54.000000000 +0100
@@ -0,0 +1,19 @@
+{
+ "users": {
+ "root": {
+ "pwhash":
"$argon2id$v=19$m=102400,t=2,p=8$0noPYaw1Zqy1traW0nrvvQ$Xle8OjVpx6ZNHsNgAoEhww",
+ "username": "root"
+ }
+ },
+ "indexes": {},
+ "dumpversion": "2",
+ "pythonversion": [
+ 3,
+ 6,
+ 8,
+ "final",
+ 0
+ ],
+ "devpi_server": "5.3.1",
+ "uuid": "b63921e918684b23ae62b9a66095e382"
+}
\ No newline at end of file
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/devpi-server-5.3.1/test_devpi_server/importexportdata/nouser/dataindex.json
new/devpi-server-5.4.1/test_devpi_server/importexportdata/nouser/dataindex.json
---
old/devpi-server-5.3.1/test_devpi_server/importexportdata/nouser/dataindex.json
1970-01-01 01:00:00.000000000 +0100
+++
new/devpi-server-5.4.1/test_devpi_server/importexportdata/nouser/dataindex.json
2020-03-26 09:54:54.000000000 +0100
@@ -0,0 +1,15 @@
+{
+ "users": {
+ },
+ "indexes": {},
+ "dumpversion": "2",
+ "pythonversion": [
+ 3,
+ 6,
+ 8,
+ "final",
+ 0
+ ],
+ "devpi_server": "5.3.1",
+ "uuid": "b63921e918684b23ae62b9a66095e382"
+}
\ No newline at end of file
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/devpi-server-5.3.1/test_devpi_server/test_importexport.py
new/devpi-server-5.4.1/test_devpi_server/test_importexport.py
--- old/devpi-server-5.3.1/test_devpi_server/test_importexport.py
2019-12-05 14:39:00.000000000 +0100
+++ new/devpi-server-5.4.1/test_devpi_server/test_importexport.py
2020-03-26 09:54:54.000000000 +0100
@@ -11,8 +11,10 @@
from devpi_server.importexport import IndexTree
from devpi_server.importexport import do_export, do_import
from devpi_server.main import Fatal
+from devpi_server.readonly import get_mutable_deepcopy
from devpi_common.archive import Archive, zip_dict
from devpi_common.metadata import Version
+from devpi_common.url import URL
import devpi_server
@@ -210,6 +212,7 @@
with pytest.raises(Fatal):
do_import(tmpdir, xom)
+
class TestIndexTree:
def test_basic(self):
tree = IndexTree()
@@ -398,6 +401,21 @@
stage = mapp.xom.model.getstage('root/dev')
assert stage.ixconfig['bases'] == ('root/dev',)
+ def test_deleted_base(self, caplog, impexp):
+ mapp = impexp.import_testdata('deletedbase')
+ with mapp.xom.keyfs.transaction(write=False):
+ assert mapp.xom.model.getstage('root/removed') is None
+ stage = mapp.xom.model.getstage('root/dev1')
+ assert stage.ixconfig['bases'] == ('root/removed',)
+ stage = mapp.xom.model.getstage('root/dev2')
+ assert stage.ixconfig['bases'] == ('root/removed',)
+ stage = mapp.xom.model.getstage('root/dev3')
+ assert stage.ixconfig['bases'] == ('root/dev2',)
+ stage = mapp.xom.model.getstage('root/dev4')
+ assert stage.ixconfig['bases'] == ('root/removed', 'root/pypi')
+ stage = mapp.xom.model.getstage('root/dev5')
+ assert stage.ixconfig['bases'] == ('root/removed', 'root/dev2')
+
def test_bad_username(self, caplog, impexp):
with pytest.raises(SystemExit):
impexp.import_testdata('badusername')
@@ -414,6 +432,94 @@
(record,) = caplog.getrecords('You could also try to edit')
assert 'dataindex.json' in record.message
+ @pytest.mark.parametrize("norootpypi", [False, True])
+ def test_import_no_user(self, caplog, impexp, norootpypi):
+ from devpi_server.main import _pypi_ixconfig_default
+ options = ()
+ if norootpypi:
+ options = ('--no-root-pypi',)
+ mapp = impexp.import_testdata('nouser', options=options)
+ with mapp.xom.keyfs.transaction(write=False):
+ user = mapp.xom.model.get_user("root")
+ assert user is not None
+ stage = mapp.xom.model.getstage("root/pypi")
+ if norootpypi:
+ assert stage is None
+ else:
+ assert stage.ixconfig == _pypi_ixconfig_default
+
+ @pytest.mark.parametrize("norootpypi", [False, True])
+ def test_import_no_root_pypi(self, caplog, impexp, norootpypi):
+ from devpi_server.main import _pypi_ixconfig_default
+ options = ()
+ if norootpypi:
+ options = ('--no-root-pypi',)
+ mapp = impexp.import_testdata('nouser', options=options)
+ with mapp.xom.keyfs.transaction(write=False):
+ user = mapp.xom.model.get_user("root")
+ assert user is not None
+ stage = mapp.xom.model.getstage("root/pypi")
+ if norootpypi:
+ assert stage is None
+ else:
+ assert stage.ixconfig == _pypi_ixconfig_default
+
+ def test_include_mirrordata(self, caplog, makeimpexp, maketestapp,
pypistage):
+ impexp = makeimpexp(options=('--include-mirrored-files',))
+ mapp1 = impexp.mapp1
+ testapp = maketestapp(mapp1.xom)
+ api = mapp1.use('root/pypi')
+ pypistage.mock_simple(
+ "package",
+ '<a href="/package-1.0.zip" />\n'
+ '<a
href="/package-1.1.zip#sha256=a665a45920422f9d417e4867efdc4fb8a04a1f3fff1fa07e998e86f7f7a27ae3"
/>\n'
+ + '<a
href="/package-2.0.zip#sha256=b3a8e0e1f9ab1bfe3a36f231f676f78bb30a519d2b21e6c530c0eee8ebb4a5d0"
data-requires-python=">=3.5" />')
+ pypistage.mock_extfile("/package-1.1.zip", b"123")
+ pypistage.mock_extfile("/package-2.0.zip", b"456")
+ r = testapp.get(api.index + "/+simple/package/")
+ assert r.status_code == 200
+ # fetch some files, so they are included in the dump
+ (_, link1, link2) = sorted(x.attrs['href'] for x in r.html.select('a'))
+ baseurl = URL(r.request.url)
+ r = testapp.get(baseurl.joinpath(link1).url)
+ assert r.body == b"123"
+ r = testapp.get(baseurl.joinpath(link2).url)
+ assert r.body == b"456"
+ impexp.export()
+ mapp2 = impexp.new_import()
+ with mapp2.xom.keyfs.transaction(write=False):
+ stage = mapp2.xom.model.getstage(api.stagename)
+ stage.offline = True
+ projects = stage.list_projects_perstage()
+ assert projects == {'package'}
+ links = sorted(get_mutable_deepcopy(
+ stage.get_simplelinks_perstage("package")))
+ assert links == [
+ ('package-1.1.zip',
'root/pypi/+f/a66/5a45920422f9d/package-1.1.zip', None),
+ ('package-2.0.zip',
'root/pypi/+f/b3a/8e0e1f9ab1bfe/package-2.0.zip', '>=3.5')]
+
+ def test_mirrordata(self, caplog, impexp):
+ mapp = impexp.import_testdata('mirrordata')
+ with mapp.xom.keyfs.transaction(write=False):
+ stage = mapp.xom.model.getstage('root/pypi')
+ stage.offline = True
+ (link,) = stage.get_simplelinks_perstage("dddttt")
+ link = stage.get_link_from_entrypath(link[1])
+ assert link.project == "dddttt"
+ assert link.version == "0.1.dev1"
+ assert link.relpath ==
'root/pypi/+f/100/7f0cd10aaa290/dddttt-0.1.dev1.tar.gz'
+ assert link.entry.hash_spec ==
'sha256=1007f0cd10aaa290eb84f092e786639bbe930b7f2169f51f755a0f50a6aba489'
+
+ def test_modifiedpypi(self, caplog, impexp):
+ mapp = impexp.import_testdata('modifiedpypi')
+ with mapp.xom.keyfs.transaction(write=False):
+ stage = mapp.xom.model.getstage('root/pypi')
+ # test that we actually get the config from the import and not
+ # the default PyPI settings
+ assert stage.ixconfig['title'] == 'Modified PyPI'
+ assert stage.ixconfig['mirror_url'] ==
'https://example.com/simple/'
+ assert stage.ixconfig['mirror_web_url_fmt'] ==
'https://example.com/project/{name}/'
+
def test_normalization(self, caplog, impexp):
mapp = impexp.import_testdata('normalization')
with mapp.xom.keyfs.transaction(write=False):
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/devpi-server-5.3.1/test_devpi_server/test_stage_customizer.py
new/devpi-server-5.4.1/test_devpi_server/test_stage_customizer.py
--- old/devpi-server-5.3.1/test_devpi_server/test_stage_customizer.py
2019-12-05 14:39:00.000000000 +0100
+++ new/devpi-server-5.4.1/test_devpi_server/test_stage_customizer.py
2020-03-26 09:54:54.000000000 +0100
@@ -17,6 +17,7 @@
def test_permissions_for_unknown_index(mapp, xom):
+ from devpi_server.model import ReadonlyIndex
api = mapp.create_and_use()
mapp.upload_file_pypi("hello-1.0.tar.gz", b'content', "hello", "1.0")
(path,) = mapp.get_release_paths("hello")
@@ -28,8 +29,26 @@
stage = xom.model.getstage(api.stagename)
with stage.user.key.update() as userconfig:
userconfig["indexes"][stage.index]['type'] = 'unknown'
+ with xom.keyfs.transaction(write=True):
+ stage = xom.model.getstage(api.stagename)
+ # first check direct stage access
+ with pytest.raises(ReadonlyIndex):
+ stage.modify(**dict(stage.ixconfig, bases=[]))
+ with pytest.raises(ReadonlyIndex):
+ stage.set_versiondata(
+ dict(name="hello", version="1.0", requires_python=">=3.5"))
+ with pytest.raises(ReadonlyIndex):
+ stage.add_project_name("foo")
+ with pytest.raises(ReadonlyIndex):
+ stage.store_releasefile("foo", "2.0", "foo-2.0.zip", b'123')
+ with pytest.raises(ReadonlyIndex):
+ stage.store_doczip("foo", "2.0", b'456')
+ link_store = stage.get_linkstore_perstage("hello", "1.0")
+ (link,) = link_store.get_links()
+ with pytest.raises(ReadonlyIndex):
+ stage.store_toxresult(link, {})
assert mapp.getjson(api.index)['result']['type'] == 'unknown'
- # now check
+ # now check via views, which are protected by permissions most of the time
mapp.modify_index(api.stagename, indexconfig=dict(bases=[]), code=403)
mapp.testapp.xdel(403, path)
mapp.delete_project('hello', code=403)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/devpi-server-5.3.1/test_devpi_server/test_views.py
new/devpi-server-5.4.1/test_devpi_server/test_views.py
--- old/devpi-server-5.3.1/test_devpi_server/test_views.py 2019-12-05
14:39:00.000000000 +0100
+++ new/devpi-server-5.4.1/test_devpi_server/test_views.py 2020-03-26
09:54:54.000000000 +0100
@@ -1049,6 +1049,40 @@
assert r.json["message"] == "error 502 getting
https://pypi.org/simple/hello/hello-1.0.tar.gz";
+def test_push_from_pypi_mirror_switch_to_use_external_urls(httpget, mapp,
pypistage, testapp):
+ pypistage.mock_simple("hello", text='<a href="hello-1.0.tar.gz"/>')
+ pypistage.mock_extfile("/simple/hello/hello-1.0.tar.gz", b"123")
+ mapp.create_and_login_user("foo")
+ mapp.create_index("newindex1", indexconfig=dict(bases=["root/pypi"]))
+ api = mapp.use("root/pypi")
+ assert not pypistage.use_external_url
+ # download the file to the mirror index
+ pkg_url = api.simpleindex + 'hello/'
+ (tag,) = testapp.get(pkg_url).html.select('a')
+ r = testapp.get(URL(pkg_url).joinpath(tag['href']).url)
+ assert r.body == b"123"
+ with mapp.xom.keyfs.transaction(write=True):
+ # switch to external URLs
+ pypistage.modify(mirror_use_external_urls=True)
+ # and remove the file to simulate a cleanup
+ linkstore = pypistage.get_linkstore_perstage("hello", "1.0")
+ (link,) = linkstore.get_links()
+ link.entry.file_delete()
+ assert pypistage.use_external_url
+ # we should now get a redirect when trying to get the file
+ r = testapp.xget(302, URL(pkg_url).joinpath(tag['href']).url)
+ # now we try to push the release to the new index
+ req = dict(name="hello", version="1.0", targetindex="foo/newindex1")
+ r = testapp.push("/root/pypi", json.dumps(req))
+ assert r.status_code == 200
+ assert r.json == {
+ 'result': [
+ [200, 'register', 'hello', '1.0', '->', 'foo/newindex1'],
+ [200, 'store_releasefile',
+ 'foo/newindex1/+f/a66/5a45920422f9d/hello-1.0.tar.gz']],
+ 'type': 'actionlog'}
+
+
def test_upload_docs_for_version_without_release(mapp, testapp, monkeypatch):
mapp.create_and_use()
mapp.upload_file_pypi("pkg1-2.6.tgz", b"123", "pkg1", "2.6")
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/devpi-server-5.3.1/tox.ini
new/devpi-server-5.4.1/tox.ini
--- old/devpi-server-5.3.1/tox.ini 2019-12-05 14:39:00.000000000 +0100
+++ new/devpi-server-5.4.1/tox.ini 2020-03-26 09:54:54.000000000 +0100
@@ -10,6 +10,7 @@
commands=
py.test --instafail --slow {posargs}
deps=
+ py34: colorama<=0.4.1 ; sys_platform == 'win32'
webtest
py27,pypy: mock
pytest>=3
