On 11/18/19 11:41 PM, Sergei Trofimovich wrote: > On Mon, 18 Nov 2019 16:45:58 -0800 > Zac Medico <zmed...@gentoo.org> wrote: > >> On 11/18/19 4:21 PM, Sergei Trofimovich wrote: >>> repoman slows down ~linearly with amount of profiles being scanned. >>> In case of amd64 we have 28 stable profiles. >>> >>> To speed up processing and fit into time budged of various CIs we can >>> split the work across different processes that handle different profiles. >>> >>> Example benchmark on ::haskell overlay: >>> $ ./repoman full --include-arches=amd64 >>> ~65 minutes >>> $ ./repoman full --include-profiles=default/linux/amd64/17.0 >>> ~4 minutes >>> This allows for a crude sharding of work across processes and allows for >>> cheap tree-wide scans for early failures. >>> >>> Bug: https://bugs.gentoo.org/700456 >>> Signed-off-by: Sergei Trofimovich <sly...@gentoo.org> >>> --- >>> repoman/lib/repoman/actions.py | 4 ++++ >>> repoman/lib/repoman/argparser.py | 7 +++++++ >>> repoman/lib/repoman/modules/scan/depend/__init__.py | 3 ++- >>> repoman/lib/repoman/modules/scan/depend/profile.py | 9 +++++++-- >>> repoman/lib/repoman/scanner.py | 5 +++++ >>> repoman/man/repoman.1 | 4 ++++ >>> 6 files changed, 29 insertions(+), 3 deletions(-) >>> >>> diff --git a/repoman/lib/repoman/actions.py b/repoman/lib/repoman/actions.py >>> index 1c9989a72..92d4d4e94 100644 >>> --- a/repoman/lib/repoman/actions.py >>> +++ b/repoman/lib/repoman/actions.py >>> @@ -412,6 +412,10 @@ the whole commit message to abort. >>> report_options.append( >>> "--include-arches=\"%s\"" % >>> " ".join(sorted(self.scanner.include_arches))) >>> + if self.scanner.include_profiles is not None: >>> + report_options.append( >>> + "--include-profiles=\"%s\"" % >>> + " ".join(sorted(self.scanner.include_profiles))) >>> >>> if portage_version is None: >>> sys.stderr.write("Failed to insert portage version in >>> message!\n") >>> diff --git a/repoman/lib/repoman/argparser.py >>> b/repoman/lib/repoman/argparser.py >>> index fa0e6ff90..670a0e91d 100644 >>> --- a/repoman/lib/repoman/argparser.py >>> +++ b/repoman/lib/repoman/argparser.py >>> @@ -164,6 +164,13 @@ def parse_args(argv, repoman_default_opts): >>> 'A space separated list of arches used to ' >>> 'filter the selection of profiles for dependency >>> checks')) >>> >>> + parser.add_argument( >>> + '--include-profiles', >>> + dest='include_profiles', metavar='PROFILES', action='append', >>> + help=( >>> + 'A space separated list of profiles used to ' >>> + 'define the selection of profiles for dependency >>> checks')) >>> + >>> parser.add_argument( >>> '-d', '--include-dev', dest='include_dev', action='store_true', >>> default=False, >>> diff --git a/repoman/lib/repoman/modules/scan/depend/__init__.py >>> b/repoman/lib/repoman/modules/scan/depend/__init__.py >>> index c3cc0ddeb..9068760bb 100644 >>> --- a/repoman/lib/repoman/modules/scan/depend/__init__.py >>> +++ b/repoman/lib/repoman/modules/scan/depend/__init__.py >>> @@ -19,7 +19,8 @@ module_spec = { >>> 'func_desc': { >>> }, >>> 'mod_kwargs': ['qatracker', 'portdb', 'profiles', >>> 'options', >>> - 'repo_metadata', 'repo_settings', >>> 'include_arches', 'caches', >>> + 'repo_metadata', 'repo_settings', >>> 'include_arches', >>> + 'include_profiles', 'caches', >>> 'repoman_incrementals', 'env', 'have', >>> 'dev_keywords' >>> ], >>> 'func_kwargs': { >>> diff --git a/repoman/lib/repoman/modules/scan/depend/profile.py >>> b/repoman/lib/repoman/modules/scan/depend/profile.py >>> index d980f4eca..0b1d74483 100644 >>> --- a/repoman/lib/repoman/modules/scan/depend/profile.py >>> +++ b/repoman/lib/repoman/modules/scan/depend/profile.py >>> @@ -33,6 +33,7 @@ class ProfileDependsChecks(ScanBase): >>> @param options: cli options >>> @param repo_settings: repository settings instance >>> @param include_arches: set >>> + @param include_profiles: set >>> @param caches: dictionary of our caches >>> @param repoman_incrementals: tuple >>> @param env: the environment >>> @@ -46,6 +47,7 @@ class ProfileDependsChecks(ScanBase): >>> self.options = kwargs.get('options') >>> self.repo_settings = kwargs.get('repo_settings') >>> self.include_arches = kwargs.get('include_arches') >>> + self.include_profiles = kwargs.get('include_profiles') >>> self.caches = kwargs.get('caches') >>> self.repoman_incrementals = kwargs.get('repoman_incrementals') >>> self.env = kwargs.get('env') >>> @@ -81,8 +83,11 @@ class ProfileDependsChecks(ScanBase): >>> if arch not in self.include_arches: >>> continue >>> >>> - relevant_profiles.extend( >>> - (keyword, groups, prof) for prof in >>> self.profiles[arch]) >>> + for prof in self.profiles[arch]: >>> + if self.include_profiles is not None: >>> + if prof not in self.include_profiles: >>> + continue >>> + relevant_profiles.append((keyword, groups, >>> prof)) >>> >>> relevant_profiles.sort(key=sort_key) >>> >>> diff --git a/repoman/lib/repoman/scanner.py b/repoman/lib/repoman/scanner.py >>> index 1b3242a51..06234b0ad 100644 >>> --- a/repoman/lib/repoman/scanner.py >>> +++ b/repoman/lib/repoman/scanner.py >>> @@ -164,6 +164,10 @@ class Scanner(object): >>> if self.options.include_arches: >>> self.include_arches = set() >>> self.include_arches.update(*[x.split() for x in >>> self.options.include_arches]) >>> + self.include_profiles = None >>> + if self.options.include_profiles: >>> + self.include_profiles = set() >>> + self.include_profiles.update(*[x.split() for x in >>> self.options.include_profiles]) >>> >>> # Disable the "self.modules['Ebuild'].notadded" check when not >>> in commit mode and >>> # running `svn status` in every package dir will be too >>> expensive. >>> @@ -190,6 +194,7 @@ class Scanner(object): >>> "repo_metadata": self.repo_metadata, >>> "profiles": profiles, >>> "include_arches": self.include_arches, >>> + "include_profiles": self.include_profiles, >>> "caches": self.caches, >>> "repoman_incrementals": self.repoman_incrementals, >>> "env": self.env, >>> diff --git a/repoman/man/repoman.1 b/repoman/man/repoman.1 >>> index 766146f57..f33fb6098 100644 >>> --- a/repoman/man/repoman.1 >>> +++ b/repoman/man/repoman.1 >>> @@ -120,6 +120,10 @@ Ignore masked packages (not allowed with commit mode) >>> A space separated list of arches used to filter the selection of >>> profiles for dependency checks. >>> .TP >>> +.BR "\-\-include\-profiles " PROFILES >>> +A space separated list of profiles used to >>> +define the selection of profiles for dependency checks. >>> +.TP >>> \fB\-d\fR, \fB\-\-include\-dev\fR >>> Include dev profiles in dependency checks. >>> .TP >>> >> >> Looks good. Please merge. >> -- >> Thanks, >> Zac >> > > Thank you! > > Does not look like I have permissions to do it:
Ok, merged with prof.sub_path fix: https://gitweb.gentoo.org/proj/portage.git/commit/?id=e9bb1e9681685f4e4d7174f51751356fd3f67d0c -- Thanks, Zac
signature.asc
Description: OpenPGP digital signature
