[gentoo-commits] proj/portage:master commit in: lib/portage/util/, /, lib/portage/dbapi/
commit: 1d3b22621d332331da6e48ae653e983406f44e5f
Author: gcarq protonmail com>
AuthorDate: Fri Sep 1 16:46:23 2023 +
Commit: Mike Gilbert gentoo org>
CommitDate: Fri Sep 1 19:13:27 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=1d3b2262
emerge: ensure paths are UTF-8 encoded in _needs_move()
Bug: https://bugs.gentoo.org/913103
Closes: https://github.com/gentoo/portage/pull/1086
Signed-off-by: Michael Egger protonmail.com>
Signed-off-by: Mike Gilbert gentoo.org>
NEWS | 2 ++
lib/portage/dbapi/vartree.py | 7 +--
lib/portage/util/movefile.py | 5 +++--
3 files changed, 10 insertions(+), 4 deletions(-)
diff --git a/NEWS b/NEWS
index aa7ffd2652..65e75759bf 100644
--- a/NEWS
+++ b/NEWS
@@ -8,6 +8,8 @@ Features:
Bug fixes:
* Prevent gpg from removing /dev/null when unlocking signing key (bug #912808).
+* emerge: ensure paths are UTF-8 encoded in _needs_move() (bug #913103).
+
portage-3.0.51 (2023-08-20)
--
diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index dac3508801..7d1bba712e 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -6273,12 +6273,15 @@ class dblink:
if mydmode is None or not stat.S_ISREG(mydmode) or mymode != mydmode:
return True
+src_bytes = _unicode_encode(mysrc, encoding=_encodings["fs"],
errors="strict")
+dest_bytes = _unicode_encode(mydest, encoding=_encodings["fs"],
errors="strict")
+
if "xattr" in self.settings.features:
excluded_xattrs = self.settings.get("PORTAGE_XATTR_EXCLUDE", "")
-if not _cmpxattr(mysrc, mydest, exclude=excluded_xattrs):
+if not _cmpxattr(src_bytes, dest_bytes, exclude=excluded_xattrs):
return True
-return not filecmp.cmp(mysrc, mydest, shallow=False)
+return not filecmp.cmp(src_bytes, dest_bytes, shallow=False)
def merge(
diff --git a/lib/portage/util/movefile.py b/lib/portage/util/movefile.py
index e2f19ba92b..75100a3acd 100644
--- a/lib/portage/util/movefile.py
+++ b/lib/portage/util/movefile.py
@@ -105,10 +105,11 @@ def _copyxattr(src, dest, exclude=None):
)
-def _cmpxattr(src, dest, exclude=None):
+def _cmpxattr(src: bytes, dest: bytes, exclude=None) -> bool:
"""
Compares extended attributes between |src| and |dest| and returns True
-if they are equal or xattrs are not supported, False otherwise
+if they are equal or xattrs are not supported, False otherwise.
+Assumes all given paths are UTF-8 encoded.
"""
try:
src_attrs = xattr.list(src)
[gentoo-commits] proj/portage:master commit in: lib/portage/util/, lib/portage/dbapi/
commit: 5572b73744121f67c4e55040966bfe91a0e5fb6c
Author: gcarq protonmail com>
AuthorDate: Tue Mar 21 16:19:13 2023 +
Commit: Sam James gentoo org>
CommitDate: Tue May 23 00:22:09 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=5572b737
mergeme: Consider file mode and xattr for comparison
Implements dblink._needs_move(...) to take care of file equality
checks which consist of file mode, extended attributes and
file content.
Signed-off-by: gcarq protonmail.com>
Signed-off-by: Sam James gentoo.org>
lib/portage/dbapi/vartree.py | 32 +---
lib/portage/util/movefile.py | 29 +
2 files changed, 54 insertions(+), 7 deletions(-)
diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 327b72bed..317cf327a 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -3,8 +3,6 @@
__all__ = ["vardbapi", "vartree", "dblink"] + ["write_contents",
"tar_contents"]
-import filecmp
-
import portage
portage.proxy.lazyimport.lazyimport(
@@ -34,7 +32,7 @@ portage.proxy.lazyimport.lazyimport(
"portage.util.env_update:env_update",
"portage.util.install_mask:install_mask_dir,InstallMask,_raise_exc",
"portage.util.listdir:dircache,listdir",
-"portage.util.movefile:movefile",
+"portage.util.movefile:movefile,_cmpxattr",
"portage.util.path:first_existing,iter_parents",
"portage.util.writeable_check:get_ro_checker",
"portage.util._xattr:xattr",
@@ -95,6 +93,7 @@ from ._ContentsCaseSensitivityManager import
ContentsCaseSensitivityManager
import argparse
import errno
+import filecmp
import fnmatch
import functools
import gc
@@ -5803,10 +5802,7 @@ class dblink:
# same way. Unless moveme=0 (blocking directory)
if moveme:
# only replace the existing file if it differs, see #722270
-already_merged = os.path.exists(mydest)
-if already_merged and filecmp.cmp(mysrc, mydest,
shallow=False):
-zing = "==="
-else:
+if self._needs_move(mysrc, mydest, mymode, mydmode):
# Create hardlinks only for source files that already
exist
# as hardlinks (having identical st_dev and st_ino).
hardlink_key = (mystat.st_dev, mystat.st_ino)
@@ -5829,6 +5825,8 @@ class dblink:
return 1
hardlink_candidates.append(mydest)
zing = ">>>"
+else:
+zing = "==="
try:
self._merged_path(mydest, os.lstat(mydest))
@@ -6254,6 +6252,26 @@ class dblink:
finally:
self.unlockdb()
+def _needs_move(self, mysrc, mydest, mymode, mydmode):
+"""
+Checks whether the given file at |mysrc| needs to be moved to |mydest|
or if
+they are identical.
+
+Takes file mode and extended attributes into account.
+Should only be used for regular files.
+"""
+if not os.path.exists(mydest):
+return True
+
+if mymode != mydmode:
+return True
+
+excluded_xattrs = self.settings.get("PORTAGE_XATTR_EXCLUDE", "")
+if not _cmpxattr(mysrc, mydest, exclude=excluded_xattrs):
+return True
+
+return not filecmp.cmp(mysrc, mydest, shallow=False)
+
def merge(
mycat,
diff --git a/lib/portage/util/movefile.py b/lib/portage/util/movefile.py
index d46c56ade..e2f19ba92 100644
--- a/lib/portage/util/movefile.py
+++ b/lib/portage/util/movefile.py
@@ -105,6 +105,35 @@ def _copyxattr(src, dest, exclude=None):
)
+def _cmpxattr(src, dest, exclude=None):
+"""
+Compares extended attributes between |src| and |dest| and returns True
+if they are equal or xattrs are not supported, False otherwise
+"""
+try:
+src_attrs = xattr.list(src)
+dest_attrs = xattr.list(dest)
+except OSError as e:
+if e.errno != OperationNotSupported.errno:
+raise
+return True
+
+if src_attrs:
+if exclude is not None and isinstance(src_attrs[0], bytes):
+exclude = exclude.encode(_encodings["fs"])
+exclude = _get_xattr_excluder(exclude)
+
+src_attrs = {attr for attr in src_attrs if not exclude(attr)}
+dest_attrs = {attr for attr in dest_attrs if not exclude(attr)}
+if src_attrs != dest_attrs:
+return False
+
+for attr in src_attrs:
+if xattr.get(src, attr) != xattr.get(dest, attr):
+return False
+return True
+
+
def movefile(
src,
dest,
[gentoo-commits] proj/portage:master commit in: lib/portage/util/, lib/portage/dbapi/
commit: 6e6d8e7f522fef3c32a7c71298024167c066a3c5
Author: Frédéric Pierret (fepitre) qubes-os
org>
AuthorDate: Thu Aug 20 09:35:23 2020 +
Commit: Zac Medico gentoo org>
CommitDate: Mon Sep 7 22:23:08 2020 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=6e6d8e7f
Use portage proxy settings for fetching BINPKG host
Bug: https://bugs.gentoo.org/740898
See: https://github.com/gentoo/portage/pull/607
Signed-off-by: Frédéric Pierret (fepitre) qubes-os.org>
Signed-off-by: Zac Medico gentoo.org>
lib/portage/dbapi/bintree.py | 9 -
lib/portage/util/_urlopen.py | 10 +++---
2 files changed, 15 insertions(+), 4 deletions(-)
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 620865a79..ee30542a5 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -964,11 +964,18 @@ class binarytree:
download_timestamp + ttl >
time.time():
raise
UseCachedCopyOfRemoteIndex()
+ # Set proxy settings for _urlopen ->
urllib_request
+ proxies = {}
+ for proto in ('http', 'https'):
+ value = self.settings.get(proto +
'_proxy')
+ if value is not None:
+ proxies[proto] = value
+
# Don't use urlopen for https, unless
# PEP 476 is supported (bug #469888).
if parsed_url.scheme not in ('https',) or
_have_pep_476():
try:
- f = _urlopen(url,
if_modified_since=local_timestamp)
+ f = _urlopen(url,
if_modified_since=local_timestamp, proxies=proxies)
if hasattr(f, 'headers') and
f.headers.get('timestamp', ''):
remote_timestamp =
f.headers.get('timestamp')
except IOError as err:
diff --git a/lib/portage/util/_urlopen.py b/lib/portage/util/_urlopen.py
index b46d1554c..b67d02739 100644
--- a/lib/portage/util/_urlopen.py
+++ b/lib/portage/util/_urlopen.py
@@ -26,7 +26,7 @@ def have_pep_476():
return hasattr(__import__('ssl'), '_create_unverified_context')
-def urlopen(url, if_modified_since=None):
+def urlopen(url, if_modified_since=None, proxies=None):
parse_result = urllib_parse.urlparse(url)
if parse_result.scheme not in ("http", "https"):
return _urlopen(url)
@@ -40,8 +40,12 @@ def urlopen(url, if_modified_since=None):
request.add_header('If-Modified-Since',
_timestamp_to_http(if_modified_since))
if parse_result.username is not None:
password_manager.add_password(None, url, parse_result.username,
parse_result.password)
- auth_handler = CompressedResponseProcessor(password_manager)
- opener = urllib_request.build_opener(auth_handler)
+
+ handlers = [CompressedResponseProcessor(password_manager)]
+ if proxies:
+ handlers.append(urllib_request.ProxyHandler(proxies))
+ opener = urllib_request.build_opener(*handlers)
+
hdl = opener.open(request)
if hdl.headers.get('last-modified', ''):
try:
[gentoo-commits] proj/portage:master commit in: lib/portage/util/, lib/portage/dbapi/, lib/portage/xml/, ...
commit: 1aa33a8638610686ee2ca13d7f8c26604f2dd7ec
Author: Manuel Rüger rueg eu>
AuthorDate: Mon Feb 3 08:08:46 2020 +
Commit: Zac Medico gentoo org>
CommitDate: Mon Feb 3 09:05:04 2020 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=1aa33a86
Drop compat code for ancient python versions
Closes: https://github.com/gentoo/portage/pull/503
Signed-off-by: Manuel Rüger rueg.eu>
Signed-off-by: Zac Medico gentoo.org>
lib/portage/_emirrordist/FetchTask.py | 9 -
lib/portage/data.py | 10 --
lib/portage/dbapi/vartree.py | 7 ---
lib/portage/dispatch_conf.py | 9 -
lib/portage/util/__init__.py | 8
lib/portage/util/_desktop_entry.py| 8
lib/portage/xml/metadata.py | 22 +++---
repoman/lib/repoman/_subprocess.py| 18 --
repoman/lib/repoman/gpg.py| 9 -
9 files changed, 7 insertions(+), 93 deletions(-)
diff --git a/lib/portage/_emirrordist/FetchTask.py
b/lib/portage/_emirrordist/FetchTask.py
index 0441fc677..322de79ba 100644
--- a/lib/portage/_emirrordist/FetchTask.py
+++ b/lib/portage/_emirrordist/FetchTask.py
@@ -444,15 +444,6 @@ class FetchTask(CompositeTask):
args = [portage.util.varexpand(x, mydict=variables)
for x in args]
- if sys.hexversion < 0x302 and sys.hexversion >= 0x300
and \
- not os.path.isabs(args[0]):
- # Python 3.1 _execvp throws TypeError for non-absolute
executable
- # path passed as bytes (see
https://bugs.python.org/issue8513).
- fullname = portage.process.find_binary(args[0])
- if fullname is None:
- raise portage.exception.CommandNotFound(args[0])
- args[0] = fullname
-
args = [_unicode_encode(x,
encoding=_encodings['fs'], errors='strict') for x in
args]
diff --git a/lib/portage/data.py b/lib/portage/data.py
index 28d6eb79d..f9d67fc3d 100644
--- a/lib/portage/data.py
+++ b/lib/portage/data.py
@@ -195,16 +195,6 @@ def _get_global(k):
# SIGPIPE problems with nss_ldap.
cmd = ["id", "-G", _portage_username]
- if sys.hexversion < 0x302 and sys.hexversion >=
0x300:
- # Python 3.1 _execvp throws TypeError for
non-absolute executable
- # path passed as bytes (see
https://bugs.python.org/issue8513).
- fullname = portage.process.find_binary(cmd[0])
- if fullname is None:
- globals()[k] = v
- _initialized_globals.add(k)
- return v
- cmd[0] = fullname
-
encoding = portage._encodings['content']
cmd = [portage._unicode_encode(x,
encoding=encoding, errors='strict') for x in
cmd]
diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 050366528..3687b471b 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -661,13 +661,6 @@ class vardbapi(dbapi):
def _aux_cache_init(self):
aux_cache = None
open_kwargs = {}
- if sys.hexversion >= 0x300 and sys.hexversion < 0x302:
- # Buffered io triggers extreme performance issues in
- # Unpickler.load() (problem observed with python-3.0.1).
- # Unfortunately, performance is still poor relative to
- # python-2.x, but buffering makes it much worse (problem
- # appears to be solved in Python >=3.2 at least).
- open_kwargs["buffering"] = 0
try:
with open(_unicode_encode(self._aux_cache_filename,
encoding=_encodings['fs'], errors='strict'),
diff --git a/lib/portage/dispatch_conf.py b/lib/portage/dispatch_conf.py
index eaea59393..2fab19f1a 100644
--- a/lib/portage/dispatch_conf.py
+++ b/lib/portage/dispatch_conf.py
@@ -41,15 +41,6 @@ def diffstatusoutput(cmd, file1, file2):
# raise a UnicodeDecodeError which makes the output inaccessible.
args = shlex_split(cmd % (file1, file2))
- if sys.hexversion < 0x302 and sys.hexversion >= 0x300 and \
- not os.path.isabs(args[0]):
- # Python 3.1 _execvp throws TypeError for non-absolute
executable
- # path passed as bytes (see https://bugs.python.org/issue8513).
- fullname = portage.process.find_binary(args[0])
- if fullname is
