[gentoo-commits] proj/portage:master commit in: man/, cnf/, pym/portage/sync/modules/rsync/

2018-04-16 Thread Zac Medico
commit: 7b448e90034de00bc177b3d809aeaf9b94d55ee2
Author: Zac Medico  gentoo  org>
AuthorDate: Fri Apr 13 17:01:45 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Tue Apr 17 02:25:25 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=7b448e90

rsync: default to sync-rsync-verify-jobs = 1 (bug 650696)

Some users have reported that using all processors to verify
manifests results in overloading, therefore default to using
a single processor. On modern hardware, verification of the
gentoo repository completes in less than 20 seconds, so using
multiple processors is not really necessary. Also, gemato-13.0
disables parallel verification due to the unresolved deadlock
issue reported in bug 647964, so this brings the default
portage configuration into alignment with current gemato
behavior.

Bug: https://bugs.gentoo.org/650696
Bug: https://bugs.gentoo.org/647964

 cnf/repos.conf  | 1 +
 man/portage.5   | 8 +---
 pym/portage/sync/modules/rsync/rsync.py | 7 ++-
 3 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/cnf/repos.conf b/cnf/repos.conf
index 5759b8b43..987be6462 100644
--- a/cnf/repos.conf
+++ b/cnf/repos.conf
@@ -6,6 +6,7 @@ location = /usr/portage
 sync-type = rsync
 sync-uri = rsync://rsync.gentoo.org/gentoo-portage
 auto-sync = yes
+sync-rsync-verify-jobs = 1
 sync-rsync-verify-metamanifest = yes
 sync-rsync-verify-max-age = 24
 sync-openpgp-key-path = 
/var/lib/gentoo/gkeys/keyrings/gentoo/release/pubring.gpg

diff --git a/man/portage.5 b/man/portage.5
index 82a50a8f4..5adb07d82 100644
--- a/man/portage.5
+++ b/man/portage.5
@@ -1105,9 +1105,11 @@ Ignore vcs directories that may be present in the 
repository. It is the
 user's responsibility to set sync-rsync-extra-opts to protect vcs
 directories if appropriate.
 .TP
-.B sync\-rsync\-verify\-jobs
-Number of parallel jobs to use when verifying nested Manifests. Defaults
-to the apparent number of processors.
+.B sync\-rsync\-verify\-jobs = 1
+Number of parallel jobs to use when verifying nested Manifests. When
+set to 0, this will use the apparent number of processors if parallel
+verification is supported by the installed version of app-portage/gemato.
+Defaults to 1.
 .TP
 .B sync\-rsync\-verify\-max\-age
 Warn if repository is older than the specified number of days. Disabled

diff --git a/pym/portage/sync/modules/rsync/rsync.py 
b/pym/portage/sync/modules/rsync/rsync.py
index 763f41699..de8327a55 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -107,12 +107,17 @@ class RsyncSync(NewBase):
if self.verify_jobs is not None:
try:
self.verify_jobs = int(self.verify_jobs)
-   if self.verify_jobs <= 0:
+   if self.verify_jobs < 0:
raise ValueError(self.verify_jobs)
except ValueError:
writemsg_level("!!! sync-rsync-verify-jobs not 
a positive integer: %s\n" % (self.verify_jobs,),
level=logging.WARNING, noiselevel=-1)
self.verify_jobs = None
+   else:
+   if self.verify_jobs == 0:
+   # Use the apparent number of processors 
if gemato
+   # supports it.
+   self.verify_jobs = None
# Support overriding max age.
self.max_age = self.repo.module_specific_options.get(
'sync-rsync-verify-max-age', '')



[gentoo-commits] proj/portage:master commit in: man/, cnf/, pym/portage/sync/modules/rsync/

2018-01-25 Thread Michał Górny
commit: eb98d1ac1f255a004e06debfa1611a65fdc493e2
Author: Michał Górny  gentoo  org>
AuthorDate: Wed Jan 24 21:01:06 2018 +
Commit: Michał Górny  gentoo  org>
CommitDate: Thu Jan 25 08:00:07 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=eb98d1ac

rsync: Introduce support for running full-tree gemato verification

Add two new configuration options to rsync repositories:
sync-rsync-verify-metamanifest and sync-rsync-openpgp-key-path.
The first controls whether gemato verification is run for
the repository (defaults to true for ::gentoo, false otherwise),
the second makes it possible to override the key path for custom
repositories.

Reviewed-by: Zac Medico  gentoo.org>

 cnf/repos.conf |  2 ++
 man/portage.5  |  9 +
 pym/portage/sync/modules/rsync/__init__.py |  4 +++-
 pym/portage/sync/modules/rsync/rsync.py| 20 +++-
 4 files changed, 33 insertions(+), 2 deletions(-)

diff --git a/cnf/repos.conf b/cnf/repos.conf
index 062fc0d10..0d2b1f4be 100644
--- a/cnf/repos.conf
+++ b/cnf/repos.conf
@@ -6,6 +6,8 @@ location = /usr/portage
 sync-type = rsync
 sync-uri = rsync://rsync.gentoo.org/gentoo-portage
 auto-sync = yes
+sync-rsync-verify-metamanifest = yes
+sync-rsync-openpgp-key-path = 
/var/lib/gentoo/gkeys/keyrings/gentoo/release/pubring.gpg
 
 # for daily squashfs snapshots
 #sync-type = squashdelta

diff --git a/man/portage.5 b/man/portage.5
index e724e1f08..2d444a86f 100644
--- a/man/portage.5
+++ b/man/portage.5
@@ -1071,10 +1071,19 @@ Extra options to give to rsync on repository 
synchronization. It takes
 precedence over a declaration in [DEFAULT] section, that takes
 precedence over PORTAGE_RSYNC_EXTRA_OPTS.
 .TP
+.B sync\-rsync\-openpgp\-key\-path
+Path to the OpenPGP key(ring) used to verify MetaManifest. Used only
+if \fBsync\-rsync\-verify\-metamanifest\fR is enabled. If unset,
+the user's keyring is used.
+.TP
 .B sync-rsync-vcs-ignore = true|false
 Ignore vcs directories that may be present in the repository. It is the
 user's responsibility to set sync-rsync-extra-opts to protect vcs
 directories if appropriate.
+.TP
+.B sync\-rsync\-verify\-metamanifest = true|false
+Require the repository to contain a signed MetaManifest and verify
+it using \fBapp\-portage/gemato\fR. Defaults to false.
 
 .RE
 

diff --git a/pym/portage/sync/modules/rsync/__init__.py 
b/pym/portage/sync/modules/rsync/__init__.py
index c2fdc4188..df9a1995a 100644
--- a/pym/portage/sync/modules/rsync/__init__.py
+++ b/pym/portage/sync/modules/rsync/__init__.py
@@ -1,4 +1,4 @@
-# Copyright 2014 Gentoo Foundation
+# Copyright 2014-2018 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 doc = """Rsync plug-in module for portage.
@@ -27,7 +27,9 @@ module_spec = {
'validate_config': CheckSyncConfig,
'module_specific_options': (
'sync-rsync-extra-opts',
+   'sync-rsync-openpgp-key-path',
'sync-rsync-vcs-ignore',
+   'sync-rsync-verify-metamanifest',
),
}
}

diff --git a/pym/portage/sync/modules/rsync/rsync.py 
b/pym/portage/sync/modules/rsync/rsync.py
index c80641ba3..47f0e1ea3 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2015 Gentoo Foundation
+# Copyright 1999-2018 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 import sys
@@ -82,6 +82,16 @@ class RsyncSync(NewBase):
self.extra_rsync_opts.extend(portage.util.shlex_split(

self.repo.module_specific_options['sync-rsync-extra-opts']))
 
+   # Process GLEP74 verification options.
+   # Default verification to 'on' for ::gentoo, 'off' otherwise.
+   self.verify_metamanifest = (
+   self.repo.module_specific_options.get(
+   'sync-rsync-verify-metamanifest', 
False))
+   # Default to gentoo-keys keyring.
+   self.openpgp_key_path = (
+   self.repo.module_specific_options.get(
+   'sync-rsync-openpgp-key-path', None))
+
# Real local timestamp file.
self.servertimestampfile = os.path.join(
self.repo.location, "metadata", "timestamp.chk")
@@ -259,6 +269,14 @@ class RsyncSync(NewBase):
exitcode = EXCEEDED_MAX_RETRIES
break
self._process_exitcode(exitcode, dosyncuri, out, maxretries)
+
+   # if synced successfully, verify now
+   if exitcode == 0 and