On 11/18/19 4:21 PM, Sergei Trofimovich wrote: > repoman slows down ~linearly with amount of profiles being scanned. > In case of amd64 we have 28 stable profiles. > > To speed up processing and fit into time budged of various CIs we can > split the work across different processes that handle different profiles. > > Example benchmark on ::haskell overlay: > $ ./repoman full --include-arches=amd64 > ~65 minutes > $ ./repoman full --include-profiles=default/linux/amd64/17.0 > ~4 minutes > This allows for a crude sharding of work across processes and allows for > cheap tree-wide scans for early failures. > > Bug: https://bugs.gentoo.org/700456 > Signed-off-by: Sergei Trofimovich <sly...@gentoo.org> > --- > repoman/lib/repoman/actions.py | 4 ++++ > repoman/lib/repoman/argparser.py | 7 +++++++ > repoman/lib/repoman/modules/scan/depend/__init__.py | 3 ++- > repoman/lib/repoman/modules/scan/depend/profile.py | 9 +++++++-- > repoman/lib/repoman/scanner.py | 5 +++++ > repoman/man/repoman.1 | 4 ++++ > 6 files changed, 29 insertions(+), 3 deletions(-) > > diff --git a/repoman/lib/repoman/actions.py b/repoman/lib/repoman/actions.py > index 1c9989a72..92d4d4e94 100644 > --- a/repoman/lib/repoman/actions.py > +++ b/repoman/lib/repoman/actions.py > @@ -412,6 +412,10 @@ the whole commit message to abort. > report_options.append( > "--include-arches=\"%s\"" % > " ".join(sorted(self.scanner.include_arches))) > + if self.scanner.include_profiles is not None: > + report_options.append( > + "--include-profiles=\"%s\"" % > + " ".join(sorted(self.scanner.include_profiles))) > > if portage_version is None: > sys.stderr.write("Failed to insert portage version in > message!\n") > diff --git a/repoman/lib/repoman/argparser.py > b/repoman/lib/repoman/argparser.py > index fa0e6ff90..670a0e91d 100644 > --- a/repoman/lib/repoman/argparser.py > +++ b/repoman/lib/repoman/argparser.py > @@ -164,6 +164,13 @@ def parse_args(argv, repoman_default_opts): > 'A space separated list of arches used to ' > 'filter the selection of profiles for dependency > checks')) > > + parser.add_argument( > + '--include-profiles', > + dest='include_profiles', metavar='PROFILES', action='append', > + help=( > + 'A space separated list of profiles used to ' > + 'define the selection of profiles for dependency > checks')) > + > parser.add_argument( > '-d', '--include-dev', dest='include_dev', action='store_true', > default=False, > diff --git a/repoman/lib/repoman/modules/scan/depend/__init__.py > b/repoman/lib/repoman/modules/scan/depend/__init__.py > index c3cc0ddeb..9068760bb 100644 > --- a/repoman/lib/repoman/modules/scan/depend/__init__.py > +++ b/repoman/lib/repoman/modules/scan/depend/__init__.py > @@ -19,7 +19,8 @@ module_spec = { > 'func_desc': { > }, > 'mod_kwargs': ['qatracker', 'portdb', 'profiles', > 'options', > - 'repo_metadata', 'repo_settings', > 'include_arches', 'caches', > + 'repo_metadata', 'repo_settings', > 'include_arches', > + 'include_profiles', 'caches', > 'repoman_incrementals', 'env', 'have', > 'dev_keywords' > ], > 'func_kwargs': { > diff --git a/repoman/lib/repoman/modules/scan/depend/profile.py > b/repoman/lib/repoman/modules/scan/depend/profile.py > index d980f4eca..0b1d74483 100644 > --- a/repoman/lib/repoman/modules/scan/depend/profile.py > +++ b/repoman/lib/repoman/modules/scan/depend/profile.py > @@ -33,6 +33,7 @@ class ProfileDependsChecks(ScanBase): > @param options: cli options > @param repo_settings: repository settings instance > @param include_arches: set > + @param include_profiles: set > @param caches: dictionary of our caches > @param repoman_incrementals: tuple > @param env: the environment > @@ -46,6 +47,7 @@ class ProfileDependsChecks(ScanBase): > self.options = kwargs.get('options') > self.repo_settings = kwargs.get('repo_settings') > self.include_arches = kwargs.get('include_arches') > + self.include_profiles = kwargs.get('include_profiles') > self.caches = kwargs.get('caches') > self.repoman_incrementals = kwargs.get('repoman_incrementals') > self.env = kwargs.get('env') > @@ -81,8 +83,11 @@ class ProfileDependsChecks(ScanBase): > if arch not in self.include_arches: > continue > > - relevant_profiles.extend( > - (keyword, groups, prof) for prof in > self.profiles[arch]) > + for prof in self.profiles[arch]: > + if self.include_profiles is not None: > + if prof not in self.include_profiles: > + continue > + relevant_profiles.append((keyword, groups, > prof)) > > relevant_profiles.sort(key=sort_key) > > diff --git a/repoman/lib/repoman/scanner.py b/repoman/lib/repoman/scanner.py > index 1b3242a51..06234b0ad 100644 > --- a/repoman/lib/repoman/scanner.py > +++ b/repoman/lib/repoman/scanner.py > @@ -164,6 +164,10 @@ class Scanner(object): > if self.options.include_arches: > self.include_arches = set() > self.include_arches.update(*[x.split() for x in > self.options.include_arches]) > + self.include_profiles = None > + if self.options.include_profiles: > + self.include_profiles = set() > + self.include_profiles.update(*[x.split() for x in > self.options.include_profiles]) > > # Disable the "self.modules['Ebuild'].notadded" check when not > in commit mode and > # running `svn status` in every package dir will be too > expensive. > @@ -190,6 +194,7 @@ class Scanner(object): > "repo_metadata": self.repo_metadata, > "profiles": profiles, > "include_arches": self.include_arches, > + "include_profiles": self.include_profiles, > "caches": self.caches, > "repoman_incrementals": self.repoman_incrementals, > "env": self.env, > diff --git a/repoman/man/repoman.1 b/repoman/man/repoman.1 > index 766146f57..f33fb6098 100644 > --- a/repoman/man/repoman.1 > +++ b/repoman/man/repoman.1 > @@ -120,6 +120,10 @@ Ignore masked packages (not allowed with commit mode) > A space separated list of arches used to filter the selection of > profiles for dependency checks. > .TP > +.BR "\-\-include\-profiles " PROFILES > +A space separated list of profiles used to > +define the selection of profiles for dependency checks. > +.TP > \fB\-d\fR, \fB\-\-include\-dev\fR > Include dev profiles in dependency checks. > .TP >
Looks good. Please merge. -- Thanks, Zac
signature.asc
Description: OpenPGP digital signature
