commit: e1de82ebe6ef2dbaab7b56bcf2bb6ff75743a000
Author: Michał Górny <mgorny <AT> gentoo <DOT> org>
AuthorDate: Mon Nov 6 15:05:47 2017 +0000
Commit: Michał Górny <mgorny <AT> gentoo <DOT> org>
CommitDate: Tue Nov 7 22:56:07 2017 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=e1de82eb
Make manifest-required-hashes configurable
The set of required hashes specify which hashes must be present for
a distfile not to be refetched. It makes little sense to hardcode this
value, and it is mostly useful for transition periods, so make it
configurable via layout.conf and default to all hashes
in manifest-hashes.
Reviewed-by: Zac Medico <zmedico <AT> gentoo.org>
man/portage.5 | 13 +++++++---
pym/portage/_emirrordist/FetchTask.py | 2 +-
pym/portage/const.py | 2 +-
pym/portage/manifest.py | 26 +++++++++++++-------
pym/portage/package/ebuild/digestgen.py | 4 ++--
pym/portage/repository/config.py | 42 ++++++++++++++++++++++++++-------
pym/portage/tests/ebuild/test_config.py | 1 +
repoman/pym/repoman/repos.py | 14 +++++++----
8 files changed, 74 insertions(+), 30 deletions(-)
diff --git a/man/portage.5 b/man/portage.5
index 7605d7cfa..e724e1f08 100644
--- a/man/portage.5
+++ b/man/portage.5
@@ -1260,9 +1260,16 @@ for every file), "true" (if an entry exists for a file,
enforce it), or "false"
(don't check Manifest files at all).
.TP
.BR manifest\-hashes
-List of hashes to generate/check in Manifest files. Valid hashes depend on the
-current version of portage; see the portage.checksum module for the current
-list.
+List of hashes to generate in new/updated entries Manifest files. Valid hashes
+depend on the current version of portage; see the portage.checksum module for
+the current list. Portage will not rewrite old entries if they satisfy
+manifest\-required\-hashes.
+.TP
+.BR manifest\-required\-hashes
+List of hashes that must be used in all Manifest entries. If the hashes listed
+here are not present in the Manifest, Portage will refetch all distfiles
+and update the respective entries to include them. Must be a subset
+of manifest\-hashes. If not specified, defaults to all manifest\-hashes.
.TP
.BR update\-changelog " = [true|" false "]"
The default setting for repoman's --echangelog option.
diff --git a/pym/portage/_emirrordist/FetchTask.py
b/pym/portage/_emirrordist/FetchTask.py
index 203b8c213..47908cb6b 100644
--- a/pym/portage/_emirrordist/FetchTask.py
+++ b/pym/portage/_emirrordist/FetchTask.py
@@ -20,7 +20,7 @@ from portage.util._async.PipeLogger import PipeLogger
from portage.util._async.PopenProcess import PopenProcess
from _emerge.CompositeTask import CompositeTask
-default_hash_name = portage.const.MANIFEST2_REQUIRED_HASH
+default_hash_name = portage.const.MANIFEST2_HASH_DEFAULT
# Use --no-check-certificate since Manifest digests should provide
# enough security, and certificates can be self-signed or whatnot.
diff --git a/pym/portage/const.py b/pym/portage/const.py
index 0af57d0e2..ec877b841 100644
--- a/pym/portage/const.py
+++ b/pym/portage/const.py
@@ -207,7 +207,7 @@ EAPI = 6
HASHING_BLOCKSIZE = 32768
MANIFEST2_HASH_DEFAULTS = frozenset(["SHA256", "SHA512", "WHIRLPOOL"])
-MANIFEST2_REQUIRED_HASH = "SHA512"
+MANIFEST2_HASH_DEFAULT = "SHA512"
MANIFEST2_IDENTIFIERS = ("AUX", "MISC", "DIST", "EBUILD")
diff --git a/pym/portage/manifest.py b/pym/portage/manifest.py
index 36c82690c..4ec20515e 100644
--- a/pym/portage/manifest.py
+++ b/pym/portage/manifest.py
@@ -26,8 +26,7 @@ from portage import _unicode_encode
from portage.exception import DigestException, FileNotFound, \
InvalidDataType, MissingParameter, PermissionDenied, \
PortageException, PortagePackageException
-from portage.const import (MANIFEST2_HASH_DEFAULTS,
- MANIFEST2_IDENTIFIERS, MANIFEST2_REQUIRED_HASH)
+from portage.const import (MANIFEST2_HASH_DEFAULTS, MANIFEST2_IDENTIFIERS)
from portage.localization import _
_manifest_re = re.compile(
@@ -128,7 +127,7 @@ class Manifest(object):
parsers = (parseManifest2,)
def __init__(self, pkgdir, distdir=None, fetchlist_dict=None,
manifest1_compat=DeprecationWarning, from_scratch=False,
thin=False,
- allow_missing=False, allow_create=True, hashes=None,
+ allow_missing=False, allow_create=True, hashes=None,
required_hashes=None,
find_invalid_path_char=None, strict_misc_digests=True):
""" Create new Manifest instance for package in pkgdir.
Do not parse Manifest file if from_scratch == True (only
for internal use)
@@ -148,15 +147,21 @@ class Manifest(object):
self.pkgdir = _unicode_decode(pkgdir).rstrip(os.sep) + os.sep
self.fhashdict = {}
self.hashes = set()
+ self.required_hashes = set()
if hashes is None:
hashes = MANIFEST2_HASH_DEFAULTS
+ if required_hashes is None:
+ required_hashes = hashes
self.hashes.update(hashes)
self.hashes.difference_update(hashname for hashname in \
list(self.hashes) if hashname not in
get_valid_checksum_keys())
self.hashes.add("size")
- self.hashes.add(MANIFEST2_REQUIRED_HASH)
+
+ self.required_hashes.update(required_hashes)
+ self.required_hashes.intersection_update(self.hashes)
+
for t in MANIFEST2_IDENTIFIERS:
self.fhashdict[t] = {}
if not from_scratch:
@@ -269,9 +274,11 @@ class Manifest(object):
def checkIntegrity(self):
for t in self.fhashdict:
for f in self.fhashdict[t]:
- if MANIFEST2_REQUIRED_HASH not in
self.fhashdict[t][f]:
- raise MissingParameter(_("Missing %s
checksum: %s %s") %
- (MANIFEST2_REQUIRED_HASH, t, f))
+ diff = self.required_hashes.difference(
+ set(self.fhashdict[t][f]))
+ if diff:
+ raise MissingParameter(_("Missing %s
checksum(s): %s %s") %
+ (' '.join(diff), t, f))
def write(self, sign=False, force=False):
""" Write Manifest instance to disk, optionally signing it.
Returns
@@ -422,7 +429,7 @@ class Manifest(object):
self.fhashdict[ftype][fname] = {}
if hashdict != None:
self.fhashdict[ftype][fname].update(hashdict)
- if not MANIFEST2_REQUIRED_HASH in self.fhashdict[ftype][fname]:
+ if
self.required_hashes.difference(set(self.fhashdict[ftype][fname])):
self.updateFileHashes(ftype, fname,
checkExisting=False, ignoreMissing=ignoreMissing)
def removeFile(self, ftype, fname):
@@ -462,6 +469,7 @@ class Manifest(object):
fetchlist_dict=self.fetchlist_dict, from_scratch=True,
thin=self.thin, allow_missing=self.allow_missing,
allow_create=self.allow_create, hashes=self.hashes,
+ required_hashes=self.required_hashes,
find_invalid_path_char=self._find_invalid_path_char,
strict_misc_digests=self.strict_misc_digests)
pn = os.path.basename(self.pkgdir.rstrip(os.path.sep))
@@ -487,7 +495,7 @@ class Manifest(object):
requiredDistfiles = distlist.copy()
required_hash_types = set()
required_hash_types.add("size")
- required_hash_types.add(MANIFEST2_REQUIRED_HASH)
+ required_hash_types.update(self.required_hashes)
for f in distlist:
fname = os.path.join(self.distdir, f)
mystat = None
diff --git a/pym/portage/package/ebuild/digestgen.py
b/pym/portage/package/ebuild/digestgen.py
index 95d02db9b..40c1b7288 100644
--- a/pym/portage/package/ebuild/digestgen.py
+++ b/pym/portage/package/ebuild/digestgen.py
@@ -11,7 +11,6 @@ portage.proxy.lazyimport.lazyimport(globals(),
)
from portage import os
-from portage.const import MANIFEST2_REQUIRED_HASH
from portage.dbapi.porttree import FetchlistDict
from portage.dep import use_reduce
from portage.exception import InvalidDependString, FileNotFound, \
@@ -58,6 +57,7 @@ def digestgen(myarchives=None, mysettings=None,
myportdb=None):
mytree = os.path.realpath(mytree)
mf =
mysettings.repositories.get_repo_for_location(mytree)
+ repo_required_hashes = mf.manifest_required_hashes
mf = mf.load_manifest(mysettings["O"], mysettings["DISTDIR"],
fetchlist_dict=fetchlist_dict)
@@ -72,7 +72,7 @@ def digestgen(myarchives=None, mysettings=None,
myportdb=None):
# exist before and after the transition.
required_hash_types = set()
required_hash_types.add("size")
- required_hash_types.add(MANIFEST2_REQUIRED_HASH)
+ required_hash_types.update(repo_required_hashes)
dist_hashes = mf.fhashdict.get("DIST", {})
# To avoid accidental regeneration of digests with the incorrect
diff --git a/pym/portage/repository/config.py b/pym/portage/repository/config.py
index 3be0e8bda..be31ed3b1 100644
--- a/pym/portage/repository/config.py
+++ b/pym/portage/repository/config.py
@@ -12,8 +12,7 @@ import re
import portage
from portage import eclass_cache, os
from portage.checksum import get_valid_checksum_keys
-from portage.const import (MANIFEST2_REQUIRED_HASH,
- PORTAGE_BASE_PATH, REPO_NAME_LOC, USER_CONFIG_PATH)
+from portage.const import (PORTAGE_BASE_PATH, REPO_NAME_LOC, USER_CONFIG_PATH)
from portage.eapi import
eapi_allows_directories_on_profile_level_and_repository_level
from portage.env.loaders import KeyValuePairFileLoader
from portage.util import (normalize_path, read_corresponding_eapi_file,
shlex_split,
@@ -86,7 +85,7 @@ class RepoConfig(object):
'sync_depth', 'sync_hooks_only_on_change',
'sync_type', 'sync_umask', 'sync_uri', 'sync_user',
'thin_manifest',
'update_changelog', '_eapis_banned', '_eapis_deprecated',
- '_masters_orig', 'module_specific_options',
+ '_masters_orig', 'module_specific_options',
'manifest_required_hashes',
)
def __init__(self, name, repo_opts, local_config=True):
@@ -227,6 +226,7 @@ class RepoConfig(object):
self.create_manifest = True
self.disable_manifest = False
self.manifest_hashes = None
+ self.manifest_required_hashes = None
self.update_changelog = False
self.cache_formats = None
self.portage1_profiles = True
@@ -262,7 +262,7 @@ class RepoConfig(object):
for value in ('allow-missing-manifest',
'allow-provide-virtual', 'cache-formats',
'create-manifest', 'disable-manifest',
'manifest-hashes',
- 'profile-formats',
+ 'manifest-required-hashes', 'profile-formats',
'sign-commit', 'sign-manifest',
'thin-manifest', 'update-changelog'):
setattr(self, value.lower().replace("-", "_"),
layout_data[value])
@@ -337,6 +337,7 @@ class RepoConfig(object):
kwds['allow_missing'] = self.allow_missing_manifest
kwds['allow_create'] = self.create_manifest
kwds['hashes'] = self.manifest_hashes
+ kwds['required_hashes'] = self.manifest_required_hashes
kwds['strict_misc_digests'] = self.strict_misc_digests
if self.disable_manifest:
kwds['from_scratch'] = True
@@ -1046,20 +1047,41 @@ def parse_layout_conf(repo_location, repo_name=None):
data['cache-formats'] = tuple(cache_formats)
manifest_hashes = layout_data.get('manifest-hashes')
+ manifest_required_hashes = layout_data.get('manifest-required-hashes')
+
+ if manifest_required_hashes is not None and manifest_hashes is None:
+ repo_name = _get_repo_name(repo_location, cached=repo_name)
+ warnings.warn((_("Repository named '%(repo_name)s' specifies "
+ "'manifest-required-hashes' setting without
corresponding "
+ "'manifest-hashes'. Portage will default it to match "
+ "the required set but please add the missing entry "
+ "to: %(layout_filename)s") %
+ {"repo_name": repo_name or 'unspecified',
+ "layout_filename":layout_filename}),
+ SyntaxWarning)
+ manifest_hashes = manifest_required_hashes
+
if manifest_hashes is not None:
+ # require all the hashes unless specified otherwise
+ if manifest_required_hashes is None:
+ manifest_required_hashes = manifest_hashes
+
+ manifest_required_hashes =
frozenset(manifest_required_hashes.upper().split())
manifest_hashes = frozenset(manifest_hashes.upper().split())
- if MANIFEST2_REQUIRED_HASH not in manifest_hashes:
+ missing_required_hashes = manifest_required_hashes.difference(
+ manifest_hashes)
+ if missing_required_hashes:
repo_name = _get_repo_name(repo_location,
cached=repo_name)
warnings.warn((_("Repository named '%(repo_name)s' has
a "
"'manifest-hashes' setting that does not
contain "
- "the '%(hash)s' hash which is required by this "
- "portage version. You will have to upgrade
portage "
+ "the '%(hash)s' hashes which are listed in "
+ "'manifest-required-hashes'. Please fix that
file "
"if you want to generate valid manifests for
this "
"repository: %(layout_filename)s") %
{"repo_name": repo_name or 'unspecified',
- "hash":MANIFEST2_REQUIRED_HASH,
+ "hash": ' '.join(missing_required_hashes),
"layout_filename":layout_filename}),
- DeprecationWarning)
+ SyntaxWarning)
unsupported_hashes = manifest_hashes.difference(
get_valid_checksum_keys())
if unsupported_hashes:
@@ -1074,7 +1096,9 @@ def parse_layout_conf(repo_location, repo_name=None):
"hashes":" ".join(sorted(unsupported_hashes)),
"layout_filename":layout_filename}),
DeprecationWarning)
+
data['manifest-hashes'] = manifest_hashes
+ data['manifest-required-hashes'] = manifest_required_hashes
data['update-changelog'] = layout_data.get('update-changelog',
'false').lower() \
== 'true'
diff --git a/pym/portage/tests/ebuild/test_config.py
b/pym/portage/tests/ebuild/test_config.py
index 1dd828538..dcb5ffe0d 100644
--- a/pym/portage/tests/ebuild/test_config.py
+++ b/pym/portage/tests/ebuild/test_config.py
@@ -228,6 +228,7 @@ class ConfigTestCase(TestCase):
"profile-formats = pms",
"thin-manifests = true",
"manifest-hashes = SHA256
SHA512 WHIRLPOOL",
+ "manifest-required-hashes =
SHA512",
"# use implicit masters"
),
}
diff --git a/repoman/pym/repoman/repos.py b/repoman/pym/repoman/repos.py
index 11a6231de..e942a599e 100644
--- a/repoman/pym/repoman/repos.py
+++ b/repoman/pym/repoman/repos.py
@@ -100,18 +100,22 @@ class RepoSettings(object):
sys.exit(1)
manifest_hashes = self.repo_config.manifest_hashes
+ manifest_required_hashes =
self.repo_config.manifest_required_hashes
if manifest_hashes is None:
manifest_hashes = portage.const.MANIFEST2_HASH_DEFAULTS
+ manifest_required_hashes = manifest_hashes
if options.mode in ("commit", "fix", "manifest"):
- if portage.const.MANIFEST2_REQUIRED_HASH not in
manifest_hashes:
+ missing_required_hashes =
manifest_required_hashes.difference(
+ manifest_hashes)
+ if missing_required_hashes:
msg = (
"The 'manifest-hashes' setting in the
'%s' repository's "
- "metadata/layout.conf does not contain
the '%s' hash which "
- "is required by this portage version.
You will have to "
- "upgrade portage if you want to
generate valid manifests for "
+ "metadata/layout.conf does not contain
the '%s' hashes which "
+ "are listed in
'manifest-required-hashes'. Please fix that "
+ "file if you want to generate valid
manifests for "
"this repository.") % (
- self.repo_config.name,
portage.const.MANIFEST2_REQUIRED_HASH)
+ self.repo_config.name, '
'.join(missing_required_hashes))
for line in textwrap.wrap(msg, 70):
logging.error(line)
sys.exit(1)
