repoman slows down ~linearly with amount of profiles being scanned.
In case of amd64 we have 28 stable profiles.
To speed up processing and fit into time budged of various CIs we can
split the work across different processes that handle different profiles.
Example benchmark on ::haskell overlay:
$ ./repoman full --include-arches=amd64
~65 minutes
$ ./repoman full --include-profiles=default/linux/amd64/17.0
~4 minutes
This allows for a crude sharding of work across processes and allows for
cheap tree-wide scans for early failures.
Bug: https://bugs.gentoo.org/700456
Signed-off-by: Sergei Trofimovich <sly...@gentoo.org>
---
repoman/lib/repoman/actions.py | 4 ++++
repoman/lib/repoman/argparser.py | 7 +++++++
repoman/lib/repoman/modules/scan/depend/__init__.py | 3 ++-
repoman/lib/repoman/modules/scan/depend/profile.py | 9 +++++++--
repoman/lib/repoman/scanner.py | 5 +++++
repoman/man/repoman.1 | 4 ++++
6 files changed, 29 insertions(+), 3 deletions(-)
diff --git a/repoman/lib/repoman/actions.py b/repoman/lib/repoman/actions.py
index 1c9989a72..92d4d4e94 100644
--- a/repoman/lib/repoman/actions.py
+++ b/repoman/lib/repoman/actions.py
@@ -412,6 +412,10 @@ the whole commit message to abort.
report_options.append(
"--include-arches=\"%s\"" %
" ".join(sorted(self.scanner.include_arches)))
+ if self.scanner.include_profiles is not None:
+ report_options.append(
+ "--include-profiles=\"%s\"" %
+ " ".join(sorted(self.scanner.include_profiles)))
if portage_version is None:
sys.stderr.write("Failed to insert portage version in
message!\n")
diff --git a/repoman/lib/repoman/argparser.py b/repoman/lib/repoman/argparser.py
index fa0e6ff90..670a0e91d 100644
--- a/repoman/lib/repoman/argparser.py
+++ b/repoman/lib/repoman/argparser.py
@@ -164,6 +164,13 @@ def parse_args(argv, repoman_default_opts):
'A space separated list of arches used to '
'filter the selection of profiles for dependency
checks'))
+ parser.add_argument(
+ '--include-profiles',
+ dest='include_profiles', metavar='PROFILES', action='append',
+ help=(
+ 'A space separated list of profiles used to '
+ 'define the selection of profiles for dependency
checks'))
+
parser.add_argument(
'-d', '--include-dev', dest='include_dev', action='store_true',
default=False,
diff --git a/repoman/lib/repoman/modules/scan/depend/__init__.py
b/repoman/lib/repoman/modules/scan/depend/__init__.py
index c3cc0ddeb..9068760bb 100644
--- a/repoman/lib/repoman/modules/scan/depend/__init__.py
+++ b/repoman/lib/repoman/modules/scan/depend/__init__.py
@@ -19,7 +19,8 @@ module_spec = {
'func_desc': {
},
'mod_kwargs': ['qatracker', 'portdb', 'profiles',
'options',
- 'repo_metadata', 'repo_settings',
'include_arches', 'caches',
+ 'repo_metadata', 'repo_settings',
'include_arches',
+ 'include_profiles', 'caches',
'repoman_incrementals', 'env', 'have',
'dev_keywords'
],
'func_kwargs': {
diff --git a/repoman/lib/repoman/modules/scan/depend/profile.py
b/repoman/lib/repoman/modules/scan/depend/profile.py
index d980f4eca..0b1d74483 100644
--- a/repoman/lib/repoman/modules/scan/depend/profile.py
+++ b/repoman/lib/repoman/modules/scan/depend/profile.py
@@ -33,6 +33,7 @@ class ProfileDependsChecks(ScanBase):
@param options: cli options
@param repo_settings: repository settings instance
@param include_arches: set
+ @param include_profiles: set
@param caches: dictionary of our caches
@param repoman_incrementals: tuple
@param env: the environment
@@ -46,6 +47,7 @@ class ProfileDependsChecks(ScanBase):
self.options = kwargs.get('options')
self.repo_settings = kwargs.get('repo_settings')
self.include_arches = kwargs.get('include_arches')
+ self.include_profiles = kwargs.get('include_profiles')
self.caches = kwargs.get('caches')
self.repoman_incrementals = kwargs.get('repoman_incrementals')
self.env = kwargs.get('env')
@@ -81,8 +83,11 @@ class ProfileDependsChecks(ScanBase):
if arch not in self.include_arches:
continue
- relevant_profiles.extend(
- (keyword, groups, prof) for prof in
self.profiles[arch])
+ for prof in self.profiles[arch]:
+ if self.include_profiles is not None:
+ if prof not in self.include_profiles:
+ continue
+ relevant_profiles.append((keyword, groups,
prof))
relevant_profiles.sort(key=sort_key)
diff --git a/repoman/lib/repoman/scanner.py b/repoman/lib/repoman/scanner.py
index 1b3242a51..06234b0ad 100644
--- a/repoman/lib/repoman/scanner.py
+++ b/repoman/lib/repoman/scanner.py
@@ -164,6 +164,10 @@ class Scanner(object):
if self.options.include_arches:
self.include_arches = set()
self.include_arches.update(*[x.split() for x in
self.options.include_arches])
+ self.include_profiles = None
+ if self.options.include_profiles:
+ self.include_profiles = set()
+ self.include_profiles.update(*[x.split() for x in
self.options.include_profiles])
# Disable the "self.modules['Ebuild'].notadded" check when not
in commit mode and
# running `svn status` in every package dir will be too
expensive.
@@ -190,6 +194,7 @@ class Scanner(object):
"repo_metadata": self.repo_metadata,
"profiles": profiles,
"include_arches": self.include_arches,
+ "include_profiles": self.include_profiles,
"caches": self.caches,
"repoman_incrementals": self.repoman_incrementals,
"env": self.env,
diff --git a/repoman/man/repoman.1 b/repoman/man/repoman.1
index 766146f57..f33fb6098 100644
--- a/repoman/man/repoman.1
+++ b/repoman/man/repoman.1
@@ -120,6 +120,10 @@ Ignore masked packages (not allowed with commit mode)
A space separated list of arches used to filter the selection of
profiles for dependency checks.
.TP
+.BR "\-\-include\-profiles " PROFILES
+A space separated list of profiles used to
+define the selection of profiles for dependency checks.
+.TP
\fB\-d\fR, \fB\-\-include\-dev\fR
Include dev profiles in dependency checks.
.TP
--
2.24.0
