[gentoo-commits] proj/portage:master commit in: lib/portage/_emirrordist/, lib/portage/tests/update/, lib/portage/dbapi/
commit: 389bb304abf5705b9f0e9e75982a682f193af985
Author: Zac Medico gentoo org>
AuthorDate: Tue Feb 13 04:35:02 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Feb 21 15:27:31 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=389bb304
async_aux_get: Use EbuildMetadataPhase deallocate_config future
For the portdbapi async_aux_get method, there is not a very
good place to store a config pool, so instead use asyncio.Lock
to manage access to the portdbapi doebuild_settings attribute
when using the main event loop in the main thread. For other
threads, clone a config instance since we do not have a
thread-safe config pool. This cloning is expensive, but since
portage internals do not trigger this case, it suffices for now
(an AssertionError ensures that internals do not trigger it).
For the main event loop running in the main thread, performance
with the asyncio.Lock should not be significantly different to
performance prior to commit c95fc64abf96, since check_locale
results are typically cached and before there was only a single
shared doebuild_settings instance with access serialized via
the EbuildMetadataPhase _start method.
Update async_aux_get callers to use asyncio.ensure_future on
the returned coroutine when needed, since it used to return
a future instead of a coroutine, and sometimes a future is
needed for add_done_callback usage.
In the portdbapi async_fetch_map method, fix a broken reference
to "future" which should have been "aux_get_future", an error
discovered while testing this patch.
Bug: https://bugs.gentoo.org/924319
Fixes: c95fc64abf96 ("EbuildPhase: async_check_locale")
Signed-off-by: Zac Medico gentoo.org>
lib/portage/_emirrordist/FetchIterator.py | 10 ++-
lib/portage/dbapi/porttree.py | 129 --
lib/portage/tests/update/test_move_ent.py | 3 +
3 files changed, 97 insertions(+), 45 deletions(-)
diff --git a/lib/portage/_emirrordist/FetchIterator.py
b/lib/portage/_emirrordist/FetchIterator.py
index eaf3e53596..e4fdd092af 100644
--- a/lib/portage/_emirrordist/FetchIterator.py
+++ b/lib/portage/_emirrordist/FetchIterator.py
@@ -1,4 +1,4 @@
-# Copyright 2013-2018 Gentoo Foundation
+# Copyright 2013-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import threading
@@ -14,6 +14,7 @@ from portage.exception import PortageException,
PortageKeyError
from portage.package.ebuild.fetch import DistfileName
from portage.util._async.AsyncTaskFuture import AsyncTaskFuture
from portage.util._async.TaskScheduler import TaskScheduler
+from portage.util.futures import asyncio
from portage.util.futures.iter_completed import iter_gather
from .FetchTask import FetchTask
from _emerge.CompositeTask import CompositeTask
@@ -276,8 +277,11 @@ def _async_fetch_tasks(config, hash_filter, repo_config,
digests_future, cpv, lo
result.set_result(fetch_tasks)
def future_generator():
-yield config.portdb.async_aux_get(
-cpv, ("RESTRICT",), myrepo=repo_config.name, loop=loop
+yield asyncio.ensure_future(
+config.portdb.async_aux_get(
+cpv, ("RESTRICT",), myrepo=repo_config.name, loop=loop
+),
+loop,
)
yield config.portdb.async_fetch_map(cpv, mytree=repo_config.location,
loop=loop)
diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py
index 61d431f917..4eebe1183f 100644
--- a/lib/portage/dbapi/porttree.py
+++ b/lib/portage/dbapi/porttree.py
@@ -1,4 +1,4 @@
-# Copyright 1998-2021 Gentoo Authors
+# Copyright 1998-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
__all__ = ["close_portdbapi_caches", "FetchlistDict", "portagetree",
"portdbapi"]
@@ -41,7 +41,9 @@ from portage.util.futures import asyncio
from portage.util.futures.iter_completed import iter_gather
from _emerge.EbuildMetadataPhase import EbuildMetadataPhase
+import contextlib
import os as _os
+import threading
import traceback
import warnings
import errno
@@ -239,6 +241,7 @@ class portdbapi(dbapi):
# this purpose because doebuild makes many changes to the config
# instance that is passed in.
self.doebuild_settings = config(clone=self.settings)
+self._doebuild_settings_lock = asyncio.Lock()
self.depcachedir = os.path.realpath(self.settings.depcachedir)
if os.environ.get("SANDBOX_ON") == "1":
@@ -356,6 +359,17 @@ class portdbapi(dbapi):
self._better_cache = None
self._broken_ebuilds = set()
+def __getstate__(self):
+state = self.__dict__.copy()
+# These attributes are not picklable, so they are automatically
+# regenerated after unpickling.
+state["_doebuild_settings_lock"] = None
+return state
+
+def __setstate__(self, state):
+self.__dict__.update(state)
+self._doebuild_settings_lock =
[gentoo-commits] proj/portage:master commit in: lib/portage/_emirrordist/
commit: 3c6ea3c726ebcac71912c3f8336af49d3e9abd38
Author: Michał Górny gentoo org>
AuthorDate: Thu May 13 10:21:41 2021 +
Commit: Zac Medico gentoo org>
CommitDate: Mon May 24 05:21:58 2021 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3c6ea3c7
Support selective fetch restriction in emirrordist
Signed-off-by: Michał Górny gentoo.org>
Signed-off-by: Zac Medico gentoo.org>
lib/portage/_emirrordist/FetchIterator.py | 68 ++-
1 file changed, 39 insertions(+), 29 deletions(-)
diff --git a/lib/portage/_emirrordist/FetchIterator.py
b/lib/portage/_emirrordist/FetchIterator.py
index 8ca6a2eac..e23d742f3 100644
--- a/lib/portage/_emirrordist/FetchIterator.py
+++ b/lib/portage/_emirrordist/FetchIterator.py
@@ -171,10 +171,6 @@ def _async_fetch_tasks(config, hash_filter, repo_config,
digests_future, cpv,
result.set_result(fetch_tasks)
return
- if "fetch" in restrict:
- result.set_result(fetch_tasks)
- return
-
try:
uri_map = fetch_map_result.result()
except PortageException as e:
@@ -187,28 +183,42 @@ def _async_fetch_tasks(config, hash_filter, repo_config,
digests_future, cpv,
result.set_result(fetch_tasks)
return
- if "mirror" in restrict:
- skip = False
- if config.restrict_mirror_exemptions is not None:
- new_uri_map = {}
- for filename, uri_tuple in uri_map.items():
- for uri in uri_tuple:
- if uri[:9] == "mirror://":
- i = uri.find("/", 9)
- if i != -1 and
uri[9:i].strip("/") in \
-
config.restrict_mirror_exemptions:
-
new_uri_map[filename] = uri_tuple
- break
- if new_uri_map:
- uri_map = new_uri_map
- else:
- skip = True
- else:
- skip = True
-
- if skip:
- result.set_result(fetch_tasks)
- return
+ new_uri_map = {}
+ restrict_fetch = "fetch" in restrict
+ restrict_mirror = restrict_fetch or "mirror" in restrict
+ for filename, uri_tuple in uri_map.items():
+ new_uris = []
+ for uri in uri_tuple:
+ override_mirror = uri.startswith("mirror+")
+ override_fetch = override_mirror or
uri.startswith("fetch+")
+ if override_fetch:
+ uri = uri.partition("+")[2]
+
+ # skip fetch-restricted files unless overriden
via fetch+
+ # or mirror+
+ if restrict_fetch and not override_fetch:
+ continue
+ # skip mirror-restricted files unless override
via mirror+
+ # or in config_mirror_exemptions
+ if restrict_mirror and not override_mirror:
+ if (config.restrict_mirror_exemptions
is None or
+ not
uri.startswith("mirror://")):
+ continue
+ mirror_name = uri.split('/', 3)[2]
+ if mirror_name not in
config.restrict_mirror_exemptions:
+ continue
+ # if neither fetch or mirror restriction
applies to the URI
+ # or it is exempted from them, readd it (with
fetch+/mirror+
+ # prefix stripped)
+ new_uris.append(uri)
+
+ # if we've gotten any new URIs, then we readd the file
+ if new_uris:
+ new_uri_map[filename] = new_uris
+
+ if not new_uri_map:
+ result.set_result(fetch_tasks)
+ return
# Parse Manifest for this cp if we haven't yet.
try:
@@ -221,7 +231,7 @@ def _async_fetch_tasks(config, hash_filter,
[gentoo-commits] proj/portage:master commit in: lib/portage/_emirrordist/, lib/portage/tests/ebuild/, man/, ...
commit: fd04c5fb1619f86381b5d5e6ff66b20fa3967c43
Author: Zac Medico gentoo org>
AuthorDate: Wed Feb 24 19:56:38 2021 +
Commit: Zac Medico gentoo org>
CommitDate: Sat Feb 27 07:43:23 2021 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=fd04c5fb
emirrordist: add --content-db option required for content-hash layout (bug
756778)
Add a --content-db option which is required for the content-hash
layout because its file listings return content digests instead of
distfile names.
The content db serves to translate content digests to distfiles
names, and distfiles names to content digests. All keys have one or
more prefixes separated by colons. For a digest key, the first
prefix is "digest" and the second prefix is the hash algorithm name.
For a filename key, the prefix is "filename".
The value associated with a digest key is a set of file names. The
value associated with a distfile key is a set of content revisions.
Each content revision is expressed as a dictionary of digests which
is suitable for construction of a DistfileName instance.
A given content digest will translate to multiple distfile names if
multiple associations have been created via the content db add
method. The relationship between a content digest and a distfile
name is similar to the relationship between an inode and a hardlink.
Bug: https://bugs.gentoo.org/756778
Signed-off-by: Zac Medico gentoo.org>
lib/portage/_emirrordist/Config.py | 6 +
lib/portage/_emirrordist/ContentDB.py| 196 +++
lib/portage/_emirrordist/DeletionIterator.py | 25 +++-
lib/portage/_emirrordist/DeletionTask.py | 8 ++
lib/portage/_emirrordist/FetchTask.py| 5 +-
lib/portage/_emirrordist/main.py | 15 +-
lib/portage/package/ebuild/fetch.py | 8 +-
lib/portage/tests/ebuild/test_fetch.py | 148
man/emirrordist.1| 6 +-
9 files changed, 407 insertions(+), 10 deletions(-)
diff --git a/lib/portage/_emirrordist/Config.py
b/lib/portage/_emirrordist/Config.py
index 1c7a27d66..a4b75809f 100644
--- a/lib/portage/_emirrordist/Config.py
+++ b/lib/portage/_emirrordist/Config.py
@@ -10,6 +10,7 @@ import time
from portage import os
from portage.package.ebuild.fetch import MirrorLayoutConfig
from portage.util import grabdict, grablines
+from .ContentDB import ContentDB
class Config:
def __init__(self, options, portdb, event_loop):
@@ -65,6 +66,11 @@ class Config:
self.distfiles_db = self._open_shelve(
options.distfiles_db, 'distfiles')
+ self.content_db = None
+ if getattr(options, 'content_db', None) is not None:
+ self.content_db = ContentDB(self._open_shelve(
+ options.content_db, 'content'))
+
self.deletion_db = None
if getattr(options, 'deletion_db', None) is not None:
self.deletion_db = self._open_shelve(
diff --git a/lib/portage/_emirrordist/ContentDB.py
b/lib/portage/_emirrordist/ContentDB.py
new file mode 100644
index 0..d9ce3cc45
--- /dev/null
+++ b/lib/portage/_emirrordist/ContentDB.py
@@ -0,0 +1,196 @@
+# Copyright 2021 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import logging
+import operator
+import shelve
+import typing
+
+from portage.package.ebuild.fetch import DistfileName
+
+
+class ContentDB:
+ """
+ The content db serves to translate content digests to distfiles
+ names, and distfiles names to content digests. All keys have one or
+ more prefixes separated by colons. For a digest key, the first
+ prefix is "digest" and the second prefix is the hash algorithm name.
+ For a filename key, the prefix is "filename".
+
+ The value associated with a digest key is a set of file names. The
+ value associated with a distfile key is a set of content revisions.
+ Each content revision is expressed as a dictionary of digests which
+ is suitable for construction of a DistfileName instance.
+ """
+
+ def __init__(self, shelve_instance: shelve.Shelf):
+ self._shelve = shelve_instance
+
+ def add(self, filename: DistfileName):
+ """
+ Add file name and digests, creating a new content revision, or
+ incrementing the reference count to an identical content
revision
+ if one exists. If the file name had previous content revisions,
+ then they continue to exist independently of the new one.
+
+ @param filename: file name with digests attribute
+ """
+ distfile_str = str(filename)
+ distfile_key = "filename:{}".format(distfile_str)
+ for k, v in filename.digests.items():
+ if k != "size":
[gentoo-commits] proj/portage:master commit in: lib/portage/_emirrordist/
commit: 0f3070198c56a8bc3b23e3965ab61136d3de76ae
Author: Zac Medico gentoo org>
AuthorDate: Sat Feb 27 03:49:29 2021 +
Commit: Zac Medico gentoo org>
CommitDate: Sat Feb 27 03:54:39 2021 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=0f307019
emirrordist: support minimal object as options for use in unit tests
Signed-off-by: Zac Medico gentoo.org>
lib/portage/_emirrordist/Config.py | 33 +
1 file changed, 17 insertions(+), 16 deletions(-)
diff --git a/lib/portage/_emirrordist/Config.py
b/lib/portage/_emirrordist/Config.py
index 4bee4f45e..1c7a27d66 100644
--- a/lib/portage/_emirrordist/Config.py
+++ b/lib/portage/_emirrordist/Config.py
@@ -1,4 +1,4 @@
-# Copyright 2013-2020 Gentoo Authors
+# Copyright 2013-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import copy
@@ -25,24 +25,24 @@ class Config:
self.start_time = time.time()
self._open_files = []
- self.log_success = self._open_log('success',
options.success_log, 'a')
- self.log_failure = self._open_log('failure',
options.failure_log, 'a')
+ self.log_success = self._open_log('success', getattr(options,
'success_log', None), 'a')
+ self.log_failure = self._open_log('failure', getattr(options,
'failure_log', None), 'a')
self.distfiles = None
- if options.distfiles is not None:
+ if getattr(options, 'distfiles', None) is not None:
self.distfiles = options.distfiles
self.mirrors = copy.copy(portdb.settings.thirdpartymirrors())
- if options.mirror_overrides is not None:
+ if getattr(options, 'mirror_overrides', None) is not None:
self.mirrors.update(grabdict(options.mirror_overrides))
- if options.mirror_skip is not None:
+ if getattr(options, 'mirror_skip', None) is not None:
for x in options.mirror_skip.split(","):
self.mirrors[x] = []
self.whitelist = None
- if options.whitelist_from is not None:
+ if getattr(options, 'whitelist_from', None) is not None:
self.whitelist = set()
for filename in options.whitelist_from:
for line in grablines(filename):
@@ -51,27 +51,27 @@ class Config:
self.whitelist.add(line)
self.restrict_mirror_exemptions = None
- if options.restrict_mirror_exemptions is not None:
+ if getattr(options, 'restrict_mirror_exemptions', None) is not
None:
self.restrict_mirror_exemptions = frozenset(
options.restrict_mirror_exemptions.split(","))
self.recycle_db = None
- if options.recycle_db is not None:
+ if getattr(options, 'recycle_db', None) is not None:
self.recycle_db = self._open_shelve(
options.recycle_db, 'recycle')
self.distfiles_db = None
- if options.distfiles_db is not None:
+ if getattr(options, 'distfiles_db', None) is not None:
self.distfiles_db = self._open_shelve(
options.distfiles_db, 'distfiles')
self.deletion_db = None
- if options.deletion_db is not None:
+ if getattr(options, 'deletion_db', None) is not None:
self.deletion_db = self._open_shelve(
options.deletion_db, 'deletion')
self.layout_conf = MirrorLayoutConfig()
- if options.layout_conf is None:
+ if getattr(options, 'layout_conf', None) is None:
options.layout_conf = os.path.join(self.distfiles,
'layout.conf')
self.layout_conf.read_from_file(options.layout_conf)
@@ -79,7 +79,7 @@ class Config:
def _open_log(self, log_desc, log_path, mode):
- if log_path is None or self.options.dry_run:
+ if log_path is None or getattr(self.options, 'dry_run', False):
log_func = logging.info
line_format = "%s: %%s" % log_desc
add_newline = False
@@ -106,12 +106,13 @@ class Config:
self._log_func(self._line_format % (msg,))
def _open_shelve(self, db_file, db_desc):
- if self.options.dry_run:
+ dry_run = getattr(self.options, 'dry_run', False)
+ if dry_run:
open_flag = "r"
else:
open_flag = "c"
-
[gentoo-commits] proj/portage:master commit in: lib/portage/_emirrordist/
commit: 8b5f9678200964c3cbb2eb06b58ebfa138fbac45 Author: Zac Medico gentoo org> AuthorDate: Fri Jan 22 01:00:16 2021 + Commit: Zac Medico gentoo org> CommitDate: Fri Jan 22 01:11:35 2021 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8b5f9678 emirrordist: prevent distfiles_db _pkg_str pickle problems When storing a cpv as a shelve value, convert _pkg_str to str in order to prevent pickle problems. Bug: https://bugs.gentoo.org/766459 Signed-off-by: Zac Medico gentoo.org> lib/portage/_emirrordist/FetchTask.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/portage/_emirrordist/FetchTask.py b/lib/portage/_emirrordist/FetchTask.py index 41f96b962..997762082 100644 --- a/lib/portage/_emirrordist/FetchTask.py +++ b/lib/portage/_emirrordist/FetchTask.py @@ -44,7 +44,8 @@ class FetchTask(CompositeTask): if self.config.distfiles_db is not None and \ self.distfile not in self.config.distfiles_db: self._previously_added = False - self.config.distfiles_db[self.distfile] = self.cpv + # Convert _pkg_str to str in order to prevent pickle problems. + self.config.distfiles_db[self.distfile] = str(self.cpv) if not self._have_needed_digests(): msg = "incomplete digests: %s" % " ".join(self.digests)
[gentoo-commits] proj/portage:master commit in: lib/portage/_emirrordist/
commit: 1f43925766b641158aa98e08a58385e65d1ac8bb Author: Aaron Bauman gentoo org> AuthorDate: Mon Aug 3 20:20:24 2020 + Commit: Zac Medico gentoo org> CommitDate: Mon Aug 3 21:25:52 2020 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=1f439257 lib/portage/_emirrordist/Config.py: drop unused-import Signed-off-by: Aaron Bauman gentoo.org> Signed-off-by: Zac Medico gentoo.org> lib/portage/_emirrordist/Config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/portage/_emirrordist/Config.py b/lib/portage/_emirrordist/Config.py index 56732089e..4bee4f45e 100644 --- a/lib/portage/_emirrordist/Config.py +++ b/lib/portage/_emirrordist/Config.py @@ -7,7 +7,6 @@ import logging import shelve import time -import portage from portage import os from portage.package.ebuild.fetch import MirrorLayoutConfig from portage.util import grabdict, grablines
[gentoo-commits] proj/portage:master commit in: lib/portage/_emirrordist/
commit: 1db30c8a1dc2215425a2d1036effc749c876c1b4 Author: Aaron Bauman gentoo org> AuthorDate: Mon Aug 3 19:05:54 2020 + Commit: Zac Medico gentoo org> CommitDate: Mon Aug 3 19:22:06 2020 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=1db30c8a lib/portage/_emirrordist/main.py: drop unused-import Signed-off-by: Aaron Bauman gentoo.org> Signed-off-by: Zac Medico gentoo.org> lib/portage/_emirrordist/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/portage/_emirrordist/main.py b/lib/portage/_emirrordist/main.py index 23f6468c5..8d00a05f5 100644 --- a/lib/portage/_emirrordist/main.py +++ b/lib/portage/_emirrordist/main.py @@ -7,7 +7,7 @@ import sys import portage from portage import os -from portage.util import normalize_path, writemsg_level, _recursive_file_list +from portage.util import normalize_path, _recursive_file_list from portage.util._async.run_main_scheduler import run_main_scheduler from portage.util._async.SchedulerInterface import SchedulerInterface from portage.util._eventloop.global_event_loop import global_event_loop
[gentoo-commits] proj/portage:master commit in: lib/portage/_emirrordist/, lib/portage/dep/soname/, lib/portage/repository/, ...
commit: 8aaf817db51f0369095b0f7d93cdbecdfc32c869
Author: Michał Górny gentoo org>
AuthorDate: Thu Jul 16 12:16:27 2020 +
Commit: Michał Górny gentoo org>
CommitDate: Thu Jul 16 18:04:43 2020 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8aaf817d
Eliminate the most of explicit py3 conditionals
Eliminate the most of py2/py3 conditions in the code. Leave a few
where the relevant code is unclear, they will be addressed later.
Reviewed-by: Zac Medico gentoo.org>
Closes: https://github.com/gentoo/portage/pull/574
Signed-off-by: Michał Górny gentoo.org>
bin/check-implicit-pointer-usage.py | 25 +++--
bin/chmod-lite.py | 11 +++---
bin/dohtml.py | 11 +++---
bin/doins.py| 7 ++--
bin/ebuild | 15 +---
bin/filter-bash-environment.py | 11 ++
bin/install.py | 19 +-
bin/pid-ns-init | 10 ++---
bin/xattr-helper.py | 32
lib/_emerge/DependencyArg.py| 10 -
lib/_emerge/JobStatusDisplay.py | 4 +-
lib/_emerge/Package.py | 8
lib/_emerge/PackageVirtualDbapi.py | 3 --
lib/_emerge/SequentialTaskQueue.py | 3 --
lib/_emerge/TaskSequence.py | 3 --
lib/_emerge/UseFlagDisplay.py | 8
lib/_emerge/UserQuery.py| 17 +++--
lib/_emerge/actions.py | 2 -
lib/_emerge/resolver/DbapiProvidesIndex.py | 3 --
lib/_emerge/resolver/output_helpers.py | 16
lib/_emerge/resolver/slot_collision.py | 8
lib/portage/__init__.py | 49 +
lib/portage/_emirrordist/Config.py | 4 --
lib/portage/_selinux.py | 12 --
lib/portage/_sets/base.py | 3 --
lib/portage/dbapi/porttree.py | 7 +---
lib/portage/dep/__init__.py | 18 +
lib/portage/dep/soname/SonameAtom.py| 8
lib/portage/elog/mod_save_summary.py| 2 -
lib/portage/elog/mod_syslog.py | 4 --
lib/portage/exception.py| 45 +--
lib/portage/mail.py | 49 -
lib/portage/manifest.py | 7
lib/portage/output.py | 4 +-
lib/portage/process.py | 16 +---
lib/portage/proxy/objectproxy.py| 6 ---
lib/portage/repository/config.py| 6 ---
lib/portage/tests/unicode/test_string_format.py | 44 +-
lib/portage/util/_ShelveUnicodeWrapper.py | 45 ---
39 files changed, 102 insertions(+), 453 deletions(-)
diff --git a/bin/check-implicit-pointer-usage.py
b/bin/check-implicit-pointer-usage.py
index a49db8107..868e4b3c8 100755
--- a/bin/check-implicit-pointer-usage.py
+++ b/bin/check-implicit-pointer-usage.py
@@ -33,22 +33,10 @@ pointer_pattern = (
+ r"|"
+ r"cast to pointer from integer of different size)")
-if sys.hexversion < 0x300:
-# Use encoded byte strings in python-2.x, since the python ebuilds are
-# known to remove the encodings module when USE=build is enabled (thus
-# disabling unicode decoding/encoding). The portage module has a
-# workaround for this, but currently we don't import that here since we
-# don't want to trigger potential sandbox violations due to stale pyc
-# files for the portage module.
-unicode_quote_open = '\xE2\x80\x98'
-unicode_quote_close = '\xE2\x80\x99'
-def write(msg):
-sys.stdout.write(msg)
-else:
-unicode_quote_open = '\u2018'
-unicode_quote_close = '\u2019'
-def write(msg):
-sys.stdout.buffer.write(msg.encode('utf_8', 'backslashreplace'))
+unicode_quote_open = '\u2018'
+unicode_quote_close = '\u2019'
+def write(msg):
+sys.stdout.buffer.write(msg.encode('utf_8', 'backslashreplace'))
pointer_pattern = re.compile(pointer_pattern)
@@ -57,10 +45,7 @@ last_implicit_linenum = -1
last_implicit_func = ""
while True:
-if sys.hexversion >= 0x300:
-line = sys.stdin.buffer.readline().decode('utf_8', 'replace')
-else:
-line = sys.stdin.readline()
+line = sys.stdin.buffer.readline().decode('utf_8', 'replace')
if not line:
break
# translate unicode open/close quotes to ascii ones
diff --git a/bin/chmod-lite.py b/bin/chmod-lite.py
index 177be7eab..7fe743ed1 100755
--- a/bin/chmod-lite.py
+++ b/bin/chmod-lite.py
@@ -12,12 +12,11 @@ os.chdir(os.environ["__PORTAGE_HELPER_CWD"])
def main(files):
- if
[gentoo-commits] proj/portage:master commit in: lib/portage/_emirrordist/
commit: e5c7b9e292c81c74c6b016aa03ef76214515f0bd
Author: Zac Medico gentoo org>
AuthorDate: Wed Nov 6 08:03:48 2019 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Nov 6 20:05:27 2019 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=e5c7b9e2
emaint: log FileCopier exceptions
Since FileCopier now captures exceptions instead of showing
a traceback, it's necessary to explicitly log them.
Bug: https://bugs.gentoo.org/699400
Signed-off-by: Zac Medico gentoo.org>
lib/portage/_emirrordist/DeletionTask.py | 2 +-
lib/portage/_emirrordist/FetchTask.py| 8
2 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/lib/portage/_emirrordist/DeletionTask.py
b/lib/portage/_emirrordist/DeletionTask.py
index 2fdafd59e..5eb01d840 100644
--- a/lib/portage/_emirrordist/DeletionTask.py
+++ b/lib/portage/_emirrordist/DeletionTask.py
@@ -85,7 +85,7 @@ class DeletionTask(CompositeTask):
else:
logging.error(("%s copy from distfiles "
- "to recycle failed") % (self.distfile,))
+ "to recycle failed: %s") % (self.distfile,
copier.future.exception()))
success = False
if success:
diff --git a/lib/portage/_emirrordist/FetchTask.py
b/lib/portage/_emirrordist/FetchTask.py
index ca573fa49..0441fc677 100644
--- a/lib/portage/_emirrordist/FetchTask.py
+++ b/lib/portage/_emirrordist/FetchTask.py
@@ -372,8 +372,8 @@ class FetchTask(CompositeTask):
current_mirror = self._current_mirror
if copier.returncode != os.EX_OK:
- msg = "%s %s copy failed unexpectedly" % \
- (self.distfile, current_mirror.name)
+ msg = "%s %s copy failed unexpectedly: %s" % \
+ (self.distfile, current_mirror.name,
copier.future.exception())
self.scheduler.output(msg + '\n', background=True,
log_path=self._log_path)
logging.error(msg)
@@ -549,8 +549,8 @@ class FetchTask(CompositeTask):
self._make_layout_links()
else:
# out of space?
- msg = "%s %s copy failed unexpectedly" % \
- (self.distfile, self._fetch_tmp_dir_info)
+ msg = "%s %s copy failed unexpectedly: %s" % \
+ (self.distfile, self._fetch_tmp_dir_info,
copier.future.exception())
self.scheduler.output(msg + '\n', background=True,
log_path=self._log_path)
logging.error(msg)
[gentoo-commits] proj/portage:master commit in: lib/portage/_emirrordist/
commit: 47e0d5b5390873edd0020245307b0febad84a5b5
Author: Zac Medico gentoo org>
AuthorDate: Wed Nov 6 19:56:52 2019 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Nov 6 20:03:42 2019 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=47e0d5b5
emirrordist: _make_layout_links msg UnboundLocalError
Fixes: 0d34d89d5028 ("emirrordist: Implement mirror layout.conf support")
Bug: https://bugs.gentoo.org/699392
Signed-off-by: Zac Medico gentoo.org>
lib/portage/_emirrordist/FetchTask.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/lib/portage/_emirrordist/FetchTask.py
b/lib/portage/_emirrordist/FetchTask.py
index 6f547d397..ca573fa49 100644
--- a/lib/portage/_emirrordist/FetchTask.py
+++ b/lib/portage/_emirrordist/FetchTask.py
@@ -585,6 +585,8 @@ class FetchTask(CompositeTask):
self._success()
self.returncode = os.EX_OK
else:
+ msg = "failed to create distfiles layout {}".format(
+ "symlink" if self.config.options.symlinks else
"hardlink")
self.config.log_failure("%s\t%s\t%s" %
(self.cpv, self.distfile, msg))
self.config.file_failures[self.distfile] = self.cpv
[gentoo-commits] proj/portage:master commit in: lib/portage/_emirrordist/
commit: e6190145c18a275f9e44618deadb9763631d6c19
Author: Zac Medico gentoo org>
AuthorDate: Tue Nov 5 19:01:20 2019 +
Commit: Zac Medico gentoo org>
CommitDate: Tue Nov 5 19:03:21 2019 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=e6190145
emirrordist: _recycle_copier_exit UnboundLocalError (bug 699392)
Bug: https://bugs.gentoo.org/699392
Signed-off-by: Zac Medico gentoo.org>
lib/portage/_emirrordist/DeletionTask.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/portage/_emirrordist/DeletionTask.py
b/lib/portage/_emirrordist/DeletionTask.py
index 4e9c26ca2..2fdafd59e 100644
--- a/lib/portage/_emirrordist/DeletionTask.py
+++ b/lib/portage/_emirrordist/DeletionTask.py
@@ -85,7 +85,7 @@ class DeletionTask(CompositeTask):
else:
logging.error(("%s copy from distfiles "
- "to recycle failed: %s") % (self.distfile, e))
+ "to recycle failed") % (self.distfile,))
success = False
if success:
[gentoo-commits] proj/portage:master commit in: lib/portage/_emirrordist/
commit: af9c77bd037694164755d06034a74456ec45e3ab
Author: Michał Górny gentoo org>
AuthorDate: Mon Oct 21 08:41:54 2019 +
Commit: Michał Górny gentoo org>
CommitDate: Mon Oct 21 13:38:26 2019 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=af9c77bd
emirrordist: Pass path from DeletionIterator to DeletionTask
Since DeletionIterator needs to stat the distfile and therefore find
one working path for it, pass it to DeletionTask instead of recomputing
it there. This also fixes wrongly assuming that first layout will
always be correct.
Bug: https://bugs.gentoo.org/697890
Reviewed-by: Zac Medico gentoo.org>
Signed-off-by: Michał Górny gentoo.org>
lib/portage/_emirrordist/DeletionIterator.py | 2 ++
lib/portage/_emirrordist/DeletionTask.py | 14 +-
2 files changed, 7 insertions(+), 9 deletions(-)
diff --git a/lib/portage/_emirrordist/DeletionIterator.py
b/lib/portage/_emirrordist/DeletionIterator.py
index 5c193911a..3cbff2c3a 100644
--- a/lib/portage/_emirrordist/DeletionIterator.py
+++ b/lib/portage/_emirrordist/DeletionIterator.py
@@ -72,6 +72,7 @@ class DeletionIterator(object):
yield DeletionTask(background=True,
distfile=filename,
+ distfile_path=path,
config=self._config)
else:
@@ -85,6 +86,7 @@ class DeletionIterator(object):
yield
DeletionTask(background=True,
distfile=filename,
+ distfile_path=path,
config=self._config)
if deletion_db is not None:
diff --git a/lib/portage/_emirrordist/DeletionTask.py
b/lib/portage/_emirrordist/DeletionTask.py
index a4bb29419..4e9c26ca2 100644
--- a/lib/portage/_emirrordist/DeletionTask.py
+++ b/lib/portage/_emirrordist/DeletionTask.py
@@ -10,14 +10,9 @@ from _emerge.CompositeTask import CompositeTask
class DeletionTask(CompositeTask):
- __slots__ = ('distfile', 'config')
+ __slots__ = ('distfile', 'distfile_path', 'config')
def _start(self):
-
- distfile_path = os.path.join(
- self.config.options.distfiles,
- self.config.layouts[0].get_path(self.distfile))
-
if self.config.options.recycle_dir is not None:
recycle_path = os.path.join(
self.config.options.recycle_dir, self.distfile)
@@ -29,7 +24,8 @@ class DeletionTask(CompositeTask):
"distfiles to recycle") % self.distfile)
try:
# note: distfile_path can be a symlink
here
-
os.rename(os.path.realpath(distfile_path), recycle_path)
+
os.rename(os.path.realpath(self.distfile_path),
+ recycle_path)
except OSError as e:
if e.errno != errno.EXDEV:
logging.error(("rename %s from
distfiles to "
@@ -40,7 +36,7 @@ class DeletionTask(CompositeTask):
return
self._start_task(
- FileCopier(src_path=distfile_path,
+ FileCopier(src_path=self.distfile_path,
dest_path=recycle_path,
background=False),
self._recycle_copier_exit)
@@ -55,7 +51,7 @@ class DeletionTask(CompositeTask):
logging.debug(("delete '%s' from "
"distfiles") % self.distfile)
try:
- os.unlink(distfile_path)
+ os.unlink(self.distfile_path)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
logging.error("%s unlink failed in
distfiles: %s" %
[gentoo-commits] proj/portage:master commit in: lib/portage/_emirrordist/
commit: dc095fc06d323849a6fe47ae3391bc0356439b65 Author: Michał Górny gentoo org> AuthorDate: Sun Oct 20 10:22:11 2019 + Commit: Michał Górny gentoo org> CommitDate: Mon Oct 21 08:09:31 2019 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=dc095fc0 emirrordist: Clean dangling symlinks up Bug: https://bugs.gentoo.org/697906 Reviewed-by: Zac Medico gentoo.org> Signed-off-by: Michał Górny gentoo.org> lib/portage/_emirrordist/DeletionIterator.py | 11 --- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/portage/_emirrordist/DeletionIterator.py b/lib/portage/_emirrordist/DeletionIterator.py index dab6eaea2..5c193911a 100644 --- a/lib/portage/_emirrordist/DeletionIterator.py +++ b/lib/portage/_emirrordist/DeletionIterator.py @@ -27,11 +27,16 @@ class DeletionIterator(object): # require at least one successful stat() exceptions = [] for layout in reversed(self._config.layouts): + path = os.path.join(distdir, layout.get_path(filename)) try: - st = os.stat( - os.path.join(distdir, layout.get_path(filename))) + st = os.stat(path) except OSError as e: - exceptions.append(e) + # is it a dangling symlink? + try: + if os.path.islink(path): + os.unlink(path) + except OSError as e: + exceptions.append(e) else: if stat.S_ISREG(st.st_mode): break
[gentoo-commits] proj/portage:master commit in: lib/portage/_emirrordist/
commit: b4bb946e65660db34be7ce67f4e1ed00df47fd53
Author: Michał Górny gentoo org>
AuthorDate: Sun Oct 20 09:53:30 2019 +
Commit: Michał Górny gentoo org>
CommitDate: Mon Oct 21 07:39:38 2019 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b4bb946e
emirrordist: Report all stat() errors instead of first one
When DeletionIterator fails, report all stat() errors. Reporting
just the first one results in confusing logs, suggesting that one
of the location did not exist but the other existed and was removed.
Reviewed-by: Zac Medico gentoo.org>
Signed-off-by: Michał Górny gentoo.org>
lib/portage/_emirrordist/DeletionIterator.py | 8
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/lib/portage/_emirrordist/DeletionIterator.py
b/lib/portage/_emirrordist/DeletionIterator.py
index cd773b3c8..dab6eaea2 100644
--- a/lib/portage/_emirrordist/DeletionIterator.py
+++ b/lib/portage/_emirrordist/DeletionIterator.py
@@ -25,20 +25,20 @@ class DeletionIterator(object):
distfiles_set.update(layout.get_filenames(distdir))
for filename in distfiles_set:
# require at least one successful stat()
- first_exception = None
+ exceptions = []
for layout in reversed(self._config.layouts):
try:
st = os.stat(
os.path.join(distdir,
layout.get_path(filename)))
except OSError as e:
- first_exception = e
+ exceptions.append(e)
else:
if stat.S_ISREG(st.st_mode):
break
else:
- if first_exception is not None:
+ if exceptions:
logging.error("stat failed on '%s' in
distfiles: %s\n" %
- (filename, first_exception))
+ (filename, '; '.join(str(x) for
x in exceptions)))
continue
if filename in file_owners:
[gentoo-commits] proj/portage:master commit in: lib/portage/_emirrordist/
commit: 4e9f04a1e11e84a8c513ee334cf2bc1c013d8c11 Author: Michał Górny gentoo org> AuthorDate: Thu Oct 17 12:05:30 2019 + Commit: Michał Górny gentoo org> CommitDate: Thu Oct 17 18:08:58 2019 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=4e9f04a1 emirrordist: Delete potential symlinks for all layouts Unlink for all distfile layouts rather than for all but primary. This wrongly assumed that the primary layout will always be removed as a result of earlier code. However, the rename() call correctly assumes that the primary layout may be an old symlink, and operates on realpath(). Therefore, the real file from a secondary layout may be renamed, leaving dangling symlink. Bug: https://bugs.gentoo.org/697906 Reviewed-by: Zac Medico gentoo.org> Signed-off-by: Michał Górny gentoo.org> lib/portage/_emirrordist/DeletionTask.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/portage/_emirrordist/DeletionTask.py b/lib/portage/_emirrordist/DeletionTask.py index db5ac5ffb..a4bb29419 100644 --- a/lib/portage/_emirrordist/DeletionTask.py +++ b/lib/portage/_emirrordist/DeletionTask.py @@ -102,7 +102,7 @@ class DeletionTask(CompositeTask): def _delete_links(self): success = True - for layout in self.config.layouts[1:]: + for layout in self.config.layouts: distfile_path = os.path.join( self.config.options.distfiles, layout.get_path(self.distfile))
[gentoo-commits] proj/portage:master commit in: lib/portage/_emirrordist/
commit: 87c1f0d53eebfeda28f17400b072d8ffa76732dc
Author: Michał Górny gentoo org>
AuthorDate: Sat Oct 12 11:04:29 2019 +
Commit: Michał Górny gentoo org>
CommitDate: Sun Oct 13 19:38:52 2019 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=87c1f0d5
emirrordist: Refetch only if none of the layouts have the file
Check all layouts for a file before refetching it. This is helpful
when combined with --layout-conf that specifies the new layout
as primary to cover new files but user-oriented layout.conf still uses
old layout.
Reviewed-by: Zac Medico gentoo.org>
Closes: https://github.com/gentoo/portage/pull/466
Signed-off-by: Michał Górny gentoo.org>
lib/portage/_emirrordist/FetchTask.py | 54 ---
1 file changed, 31 insertions(+), 23 deletions(-)
diff --git a/lib/portage/_emirrordist/FetchTask.py
b/lib/portage/_emirrordist/FetchTask.py
index 7b68b7d3e..6f547d397 100644
--- a/lib/portage/_emirrordist/FetchTask.py
+++ b/lib/portage/_emirrordist/FetchTask.py
@@ -61,23 +61,24 @@ class FetchTask(CompositeTask):
self._async_wait()
return
- distfile_path = os.path.join(
- self.config.options.distfiles,
- self.config.layouts[0].get_path(self.distfile))
-
st = None
- size_ok = False
- try:
- st = os.stat(distfile_path)
- except OSError as e:
- if e.errno not in (errno.ENOENT, errno.ESTALE):
- msg = "%s stat failed in %s: %s" % \
- (self.distfile, "distfiles", e)
- self.scheduler.output(msg + '\n',
background=True,
- log_path=self._log_path)
- logging.error(msg)
- else:
- size_ok = st.st_size == self.digests["size"]
+ for layout in self.config.layouts:
+ distfile_path = os.path.join(
+ self.config.options.distfiles,
+ layout.get_path(self.distfile))
+ try:
+ st = os.stat(distfile_path)
+ except OSError as e:
+ if e.errno not in (errno.ENOENT, errno.ESTALE):
+ msg = "%s stat failed in %s: %s" % \
+ (self.distfile, "distfiles", e)
+ self.scheduler.output(msg + '\n',
background=True,
+ log_path=self._log_path)
+ logging.error(msg)
+ else:
+ break
+
+ size_ok = st is not None and st.st_size == self.digests["size"]
if not size_ok:
if self.config.options.dry_run:
@@ -88,13 +89,20 @@ class FetchTask(CompositeTask):
# Do the unlink in order to ensure that the
path is clear,
# even if stat raised ENOENT, since a broken
symlink can
# trigger ENOENT.
- if self._unlink_file(distfile_path,
"distfiles"):
- if st is not None:
- logging.debug(("delete '%s'
with "
- "wrong size from
distfiles") % (self.distfile,))
- else:
- self.config.log_failure("%s\t%s\t%s" %
- (self.cpv, self.distfile,
"unlink failed in distfiles"))
+ unlink_success = True
+ for layout in self.config.layouts:
+ unlink_path = os.path.join(
+ self.config.options.distfiles,
+ layout.get_path(self.distfile))
+ if self._unlink_file(unlink_path,
"distfiles"):
+ if st is not None:
+ logging.debug(("delete
'%s' with "
+ "wrong size
from distfiles") % (self.distfile,))
+ else:
+
self.config.log_failure("%s\t%s\t%s" %
+ (self.cpv,
self.distfile, "unlink failed in distfiles"))
+ unlink_success = False
+ if not
[gentoo-commits] proj/portage:master commit in: lib/portage/_emirrordist/
commit: 8c92c14269dfadc87413ebd952397021b1c4dcc7
Author: Michał Górny gentoo org>
AuthorDate: Fri Oct 11 21:24:12 2019 +
Commit: Michał Górny gentoo org>
CommitDate: Sat Oct 12 06:40:29 2019 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8c92c142
emirrordist: Support specifying custom --layout-conf
Support specifying custom layout.conf file to use. This makes it
possible to start propagating new mirror layout without exposing it
to users.
Reviewed-by: Zac Medico gentoo.org>
Closes: https://github.com/gentoo/portage/pull/465
Signed-off-by: Michał Górny gentoo.org>
lib/portage/_emirrordist/Config.py | 6 --
lib/portage/_emirrordist/main.py | 6 ++
2 files changed, 10 insertions(+), 2 deletions(-)
diff --git a/lib/portage/_emirrordist/Config.py
b/lib/portage/_emirrordist/Config.py
index ace3b0a6a..c1f59f725 100644
--- a/lib/portage/_emirrordist/Config.py
+++ b/lib/portage/_emirrordist/Config.py
@@ -74,8 +74,10 @@ class Config(object):
options.deletion_db, 'deletion')
self.layout_conf = MirrorLayoutConfig()
- self.layout_conf.read_from_file(
- os.path.join(self.distfiles, 'layout.conf'))
+ if options.layout_conf is None:
+ options.layout_conf = os.path.join(self.distfiles,
+ 'layout.conf')
+ self.layout_conf.read_from_file(options.layout_conf)
self.layouts = self.layout_conf.get_all_layouts()
def _open_log(self, log_desc, log_path, mode):
diff --git a/lib/portage/_emirrordist/main.py b/lib/portage/_emirrordist/main.py
index 0ae45ab6f..ce0c2929f 100644
--- a/lib/portage/_emirrordist/main.py
+++ b/lib/portage/_emirrordist/main.py
@@ -193,6 +193,12 @@ common_options = (
"distfiles between layouts",
"action" : "store_true"
},
+ {
+ "longopt" : "--layout-conf",
+ "help" : "specifies layout.conf file to use instead of "
+ "the one present in the distfiles directory",
+ "metavar" : "FILE"
+ },
)
def parse_args(args):
