[gentoo-commits] proj/portage:master commit in: lib/portage/package/ebuild/, lib/portage/tests/ebuild/
commit: d9855418352398013ae787bb73f70e935ec109ca
Author: Zac Medico gentoo org>
AuthorDate: Mon Oct 21 09:51:33 2019 +
Commit: Zac Medico gentoo org>
CommitDate: Mon Oct 21 09:55:36 2019 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d9855418
fetch: yield unicode from layout get_filenames methods
Reported-by: Michał Górny gentoo.org>
Signed-off-by: Zac Medico gentoo.org>
lib/portage/package/ebuild/fetch.py| 18 ++
lib/portage/tests/ebuild/test_fetch.py | 26 ++
2 files changed, 40 insertions(+), 4 deletions(-)
diff --git a/lib/portage/package/ebuild/fetch.py
b/lib/portage/package/ebuild/fetch.py
index cedf12b19..ac25e4326 100644
--- a/lib/portage/package/ebuild/fetch.py
+++ b/lib/portage/package/ebuild/fetch.py
@@ -7,6 +7,7 @@ __all__ = ['fetch']
import errno
import functools
+import glob
import io
import itertools
import json
@@ -27,7 +28,6 @@ except ImportError:
import portage
portage.proxy.lazyimport.lazyimport(globals(),
- 'glob:glob',
'portage.package.ebuild.config:check_config_instance,config',
'portage.package.ebuild.doebuild:doebuild_environment,' + \
'_doebuild_spawn',
@@ -272,7 +272,13 @@ class FlatLayout(object):
def get_filenames(self, distdir):
for dirpath, dirnames, filenames in os.walk(distdir,
onerror=_raise_exc):
- return iter(filenames)
+ for filename in filenames:
+ try:
+ yield portage._unicode_decode(filename,
errors='strict')
+ except UnicodeDecodeError:
+ # Ignore it. Distfiles names must have
valid UTF8 encoding.
+ pass
+ return
@staticmethod
def verify_args(args):
@@ -301,8 +307,12 @@ class FilenameHashLayout(object):
c = c // 4
pattern += c * '[0-9a-f]' + '/'
pattern += '*'
- return (x.rsplit('/', 1)[1]
- for x in glob(os.path.join(distdir, pattern)))
+ for x in glob.iglob(os.path.join(distdir, pattern)):
+ try:
+ yield portage._unicode_decode(x,
errors='strict').rsplit('/', 1)[1]
+ except UnicodeDecodeError:
+ # Ignore it. Distfiles names must have valid
UTF8 encoding.
+ pass
@staticmethod
def verify_args(args):
diff --git a/lib/portage/tests/ebuild/test_fetch.py
b/lib/portage/tests/ebuild/test_fetch.py
index 538fb1754..9a8a4a544 100644
--- a/lib/portage/tests/ebuild/test_fetch.py
+++ b/lib/portage/tests/ebuild/test_fetch.py
@@ -448,3 +448,29 @@ class EbuildFetchTestCase(TestCase):
'''
self.assertRaises(ConfigParserError, mlc.read_from_file,
io.StringIO(conf))
+
+ def test_filename_hash_layout_get_filenames(self):
+ layouts = (
+ FlatLayout(),
+ FilenameHashLayout('SHA1', '4'),
+ FilenameHashLayout('SHA1', '8'),
+ FilenameHashLayout('SHA1', '8:16'),
+ FilenameHashLayout('SHA1', '8:16:24'),
+ )
+ filename = 'foo-1.tar.gz'
+
+ for layout in layouts:
+ distdir = tempfile.mkdtemp()
+ try:
+ path = os.path.join(distdir,
layout.get_path(filename))
+ try:
+ os.makedirs(os.path.dirname(path))
+ except OSError:
+ pass
+
+ with open(path, 'wb') as f:
+ pass
+
+ self.assertEqual([filename],
list(layout.get_filenames(distdir)))
+ finally:
+ shutil.rmtree(distdir)
[gentoo-commits] proj/portage:master commit in: lib/portage/package/ebuild/, lib/portage/tests/ebuild/
commit: 6b5889afb1e80bc673ce782e65fc0f49ee7d0908
Author: Zac Medico gentoo org>
AuthorDate: Sun Oct 13 22:13:18 2019 +
Commit: Zac Medico gentoo org>
CommitDate: Mon Oct 14 19:46:16 2019 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=6b5889af
fetch: minimal skiprocheck fixes (bug 220533)
Fix cases here fetch assumes that DISTDIR is writable when it's actually
read-only. This preserves old behavior which allowed users to override
FETCHCOMMAND to fetch files on a remote system, even though DISTDIR is
locally mounted in read-only mode.
Bug: https://bugs.gentoo.org/220533
Fixes: ebbde237d33e ("fetch: atomic downloads (bug 175612)")
Signed-off-by: Zac Medico gentoo.org>
lib/portage/package/ebuild/fetch.py| 38 --
lib/portage/tests/ebuild/test_fetch.py | 22
2 files changed, 45 insertions(+), 15 deletions(-)
diff --git a/lib/portage/package/ebuild/fetch.py
b/lib/portage/package/ebuild/fetch.py
index cd204b755..1d5e07260 100644
--- a/lib/portage/package/ebuild/fetch.py
+++ b/lib/portage/package/ebuild/fetch.py
@@ -529,11 +529,12 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0,
if listonly or ("distlocks" not in features):
use_locks = 0
+ distdir_writable = os.access(mysettings["DISTDIR"], os.W_OK)
fetch_to_ro = 0
if "skiprocheck" in features:
fetch_to_ro = 1
- if not os.access(mysettings["DISTDIR"],os.W_OK) and fetch_to_ro:
+ if not distdir_writable and fetch_to_ro:
if use_locks:
writemsg(colorize("BAD",
_("!!! For fetching to a read-only filesystem, "
@@ -613,8 +614,11 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0,
for myfile, myuri in file_uri_tuples:
if myfile not in filedict:
filedict[myfile]=[]
- mirror_cache = os.path.join(mysettings["DISTDIR"],
- ".mirror-cache.json")
+ if distdir_writable:
+ mirror_cache =
os.path.join(mysettings["DISTDIR"],
+ ".mirror-cache.json")
+ else:
+ mirror_cache = None
for l in locations:
filedict[myfile].append(functools.partial(
get_mirror_url, l, myfile,
mirror_cache))
@@ -790,7 +794,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0,
pruned_digests["size"] = size
myfile_path = os.path.join(mysettings["DISTDIR"], myfile)
- download_path = myfile_path + _download_suffix
+ download_path = myfile_path if fetch_to_ro else myfile_path +
_download_suffix
has_space = True
has_space_superuser = True
file_lock = None
@@ -1058,7 +1062,8 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0,
"File renamed to '%s'\n\n") % \
temp_filename, noiselevel=-1)
else:
-
_movefile(download_path, myfile_path, mysettings=mysettings)
+ if not
fetch_to_ro:
+
_movefile(download_path, myfile_path, mysettings=mysettings)
eout = EOutput()
eout.quiet = \
mysettings.get("PORTAGE_QUIET", None) == "1"
@@ -1177,7 +1182,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0,
del e
fetched = 0
else:
- if mystat.st_size <
fetch_resume_size:
+ if distdir_writable and
mystat.st_size < fetch_resume_size:
writemsg(_(">>>
Deleting distfile with size "
"%d
(smaller than " "PORTAGE_FETCH_RESU"
"ME_MIN_SIZE)\n") % mystat.st_size)
@@ -1315,13 +1320,14 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0,
[gentoo-commits] proj/portage:master commit in: lib/portage/package/ebuild/, lib/portage/tests/ebuild/
commit: 6a539b7c5163899db1d58cf152aeab1b2b4f9be4
Author: Michał Górny gentoo org>
AuthorDate: Thu Oct 3 14:19:54 2019 +
Commit: Michał Górny gentoo org>
CommitDate: Fri Oct 4 21:25:00 2019 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=6a539b7c
fetch: Support GLEP 75 mirror structure
Add a support for the subset of GLEP 75 needed by Gentoo Infra. This
includes fetching and parsing layout.conf, and support for flat layout
and filename-hash layout with cutoffs being multiplies of 4.
Bug: https://bugs.gentoo.org/646898
Closes: https://github.com/gentoo/portage/pull/462
Reviewed-by: Zac Medico gentoo.org>
Signed-off-by: Michał Górny gentoo.org>
lib/portage/package/ebuild/fetch.py| 160 -
lib/portage/tests/ebuild/test_fetch.py | 94 ++-
2 files changed, 250 insertions(+), 4 deletions(-)
diff --git a/lib/portage/package/ebuild/fetch.py
b/lib/portage/package/ebuild/fetch.py
index 227bf45ae..4458796fc 100644
--- a/lib/portage/package/ebuild/fetch.py
+++ b/lib/portage/package/ebuild/fetch.py
@@ -6,13 +6,17 @@ from __future__ import print_function
__all__ = ['fetch']
import errno
+import functools
import io
+import itertools
+import json
import logging
import random
import re
import stat
import sys
import tempfile
+import time
from collections import OrderedDict
@@ -27,12 +31,17 @@ portage.proxy.lazyimport.lazyimport(globals(),
'portage.package.ebuild.doebuild:doebuild_environment,' + \
'_doebuild_spawn',
'portage.package.ebuild.prepare_build_dirs:prepare_build_dirs',
+ 'portage.util:atomic_ofstream',
+ 'portage.util.configparser:SafeConfigParser,read_configs,' +
+ 'NoOptionError,ConfigParserError',
+ 'portage.util._urlopen:urlopen',
)
from portage import os, selinux, shutil, _encodings, \
_movefile, _shell_quote, _unicode_encode
from portage.checksum import (get_valid_checksum_keys, perform_md5, verify_all,
- _filter_unaccelarated_hashes, _hash_filter, _apply_hash_filter)
+ _filter_unaccelarated_hashes, _hash_filter, _apply_hash_filter,
+ checksum_str)
from portage.const import BASH_BINARY, CUSTOM_MIRRORS_FILE, \
GLOBAL_CONFIG_PATH
from portage.data import portage_gid, portage_uid, secpass, userpriv_groups
@@ -253,6 +262,146 @@ _size_suffix_map = {
'Y' : 80,
}
+
+class FlatLayout(object):
+ def get_path(self, filename):
+ return filename
+
+ @staticmethod
+ def verify_args(args):
+ return len(args) == 1
+
+
+class FilenameHashLayout(object):
+ def __init__(self, algo, cutoffs):
+ self.algo = algo
+ self.cutoffs = [int(x) for x in cutoffs.split(':')]
+
+ def get_path(self, filename):
+ fnhash = checksum_str(filename.encode('utf8'), self.algo)
+ ret = ''
+ for c in self.cutoffs:
+ assert c % 4 == 0
+ c = c // 4
+ ret += fnhash[:c] + '/'
+ fnhash = fnhash[c:]
+ return ret + filename
+
+ @staticmethod
+ def verify_args(args):
+ if len(args) != 3:
+ return False
+ if args[1] not in get_valid_checksum_keys():
+ return False
+ # argsidate cutoffs
+ for c in args[2].split(':'):
+ try:
+ c = int(c)
+ except ValueError:
+ break
+ else:
+ if c % 4 != 0:
+ break
+ else:
+ return True
+ return False
+
+
+class MirrorLayoutConfig(object):
+ """
+ Class to read layout.conf from a mirror.
+ """
+
+ def __init__(self):
+ self.structure = ()
+
+ def read_from_file(self, f):
+ cp = SafeConfigParser()
+ read_configs(cp, [f])
+ vals = []
+ for i in itertools.count():
+ try:
+ vals.append(tuple(cp.get('structure', '%d' %
i).split()))
+ except NoOptionError:
+ break
+ self.structure = tuple(vals)
+
+ def serialize(self):
+ return self.structure
+
+ def deserialize(self, data):
+ self.structure = data
+
+ @staticmethod
+ def validate_structure(val):
+ if val[0] == 'flat':
+ return FlatLayout.verify_args(val)
+ if val[0] == 'filename-hash':
+ return FilenameHashLayout.verify_args(val)
+ return False
+
+ def get_best_supported_layout(self):
+ for val in
[gentoo-commits] proj/portage:master commit in: lib/portage/package/ebuild/, lib/portage/tests/ebuild/, lib/_emerge/
commit: ebbde237d33e783c562cc6c70987969ac7228b96
Author: Zac Medico gentoo org>
AuthorDate: Sat Apr 27 21:59:57 2019 +
Commit: Zac Medico gentoo org>
CommitDate: Tue Apr 30 02:28:21 2019 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=ebbde237
fetch: atomic downloads (bug 175612)
Direct FETCHCOMMAND/RESUMECOMMAND output to a temporary file with
a constant .__download__ suffix, and atomically rename the file
to remove the suffix only after the download has completed
successfully (includes digest verification when applicable).
Also add unit tests to cover most fetch cases.
Bug: https://bugs.gentoo.org/175612
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/BinpkgVerifier.py | 4 +-
lib/portage/package/ebuild/fetch.py| 105 ++-
lib/portage/tests/ebuild/test_fetch.py | 230 +
3 files changed, 303 insertions(+), 36 deletions(-)
diff --git a/lib/_emerge/BinpkgVerifier.py b/lib/_emerge/BinpkgVerifier.py
index 7a6d15e80..f98f511a8 100644
--- a/lib/_emerge/BinpkgVerifier.py
+++ b/lib/_emerge/BinpkgVerifier.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2013 Gentoo Foundation
+# Copyright 1999-2019 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import errno
@@ -108,7 +108,7 @@ class BinpkgVerifier(CompositeTask):
def _digest_exception(self, name, value, expected):
head, tail = os.path.split(self._pkg_path)
- temp_filename = _checksum_failure_temp_file(head, tail)
+ temp_filename =
_checksum_failure_temp_file(self.pkg.root_config.settings, head, tail)
self.scheduler.output((
"\n!!! Digest verification failed:\n"
diff --git a/lib/portage/package/ebuild/fetch.py
b/lib/portage/package/ebuild/fetch.py
index bfd97601c..227bf45ae 100644
--- a/lib/portage/package/ebuild/fetch.py
+++ b/lib/portage/package/ebuild/fetch.py
@@ -30,7 +30,7 @@ portage.proxy.lazyimport.lazyimport(globals(),
)
from portage import os, selinux, shutil, _encodings, \
- _shell_quote, _unicode_encode
+ _movefile, _shell_quote, _unicode_encode
from portage.checksum import (get_valid_checksum_keys, perform_md5, verify_all,
_filter_unaccelarated_hashes, _hash_filter, _apply_hash_filter)
from portage.const import BASH_BINARY, CUSTOM_MIRRORS_FILE, \
@@ -46,6 +46,8 @@ from portage.util import apply_recursive_permissions, \
varexpand, writemsg, writemsg_level, writemsg_stdout
from portage.process import spawn
+_download_suffix = '.__download__'
+
_userpriv_spawn_kwargs = (
("uid",portage_uid),
("gid",portage_gid),
@@ -139,7 +141,7 @@ def _userpriv_test_write_file(settings, file_path):
_userpriv_test_write_file_cache[file_path] = rval
return rval
-def _checksum_failure_temp_file(distdir, basename):
+def _checksum_failure_temp_file(settings, distdir, basename):
"""
First try to find a duplicate temp file with the same checksum and
return
that filename if available. Otherwise, use mkstemp to create a new
unique
@@ -149,9 +151,13 @@ def _checksum_failure_temp_file(distdir, basename):
"""
filename = os.path.join(distdir, basename)
+ if basename.endswith(_download_suffix):
+ normal_basename = basename[:-len(_download_suffix)]
+ else:
+ normal_basename = basename
size = os.stat(filename).st_size
checksum = None
- tempfile_re = re.compile(re.escape(basename) +
r'\._checksum_failure_\..*')
+ tempfile_re = re.compile(re.escape(normal_basename) +
r'\._checksum_failure_\..*')
for temp_filename in os.listdir(distdir):
if not tempfile_re.match(temp_filename):
continue
@@ -173,9 +179,9 @@ def _checksum_failure_temp_file(distdir, basename):
return temp_filename
fd, temp_filename = \
- tempfile.mkstemp("", basename + "._checksum_failure_.", distdir)
+ tempfile.mkstemp("", normal_basename + "._checksum_failure_.",
distdir)
os.close(fd)
- os.rename(filename, temp_filename)
+ _movefile(filename, temp_filename, mysettings=settings)
return temp_filename
def _check_digests(filename, digests, show_errors=1):
@@ -602,6 +608,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0,
pruned_digests["size"] = size
myfile_path = os.path.join(mysettings["DISTDIR"], myfile)
+ download_path = myfile_path + _download_suffix
has_space = True
has_space_superuser = True
file_lock = None
@@ -679,12 +686,15 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0,
del e
continue
- if
