On Mon, 18 Nov 2019 16:45:58 -0800 Zac Medico <zmed...@gentoo.org> wrote:
> On 11/18/19 4:21 PM, Sergei Trofimovich wrote: > > repoman slows down ~linearly with amount of profiles being scanned. > > In case of amd64 we have 28 stable profiles. > > > > To speed up processing and fit into time budged of various CIs we can > > split the work across different processes that handle different profiles. > > > > Example benchmark on ::haskell overlay: > > $ ./repoman full --include-arches=amd64 > > ~65 minutes > > $ ./repoman full --include-profiles=default/linux/amd64/17.0 > > ~4 minutes > > This allows for a crude sharding of work across processes and allows for > > cheap tree-wide scans for early failures. > > > > Bug: https://bugs.gentoo.org/700456 > > Signed-off-by: Sergei Trofimovich <sly...@gentoo.org> > > --- > > repoman/lib/repoman/actions.py | 4 ++++ > > repoman/lib/repoman/argparser.py | 7 +++++++ > > repoman/lib/repoman/modules/scan/depend/__init__.py | 3 ++- > > repoman/lib/repoman/modules/scan/depend/profile.py | 9 +++++++-- > > repoman/lib/repoman/scanner.py | 5 +++++ > > repoman/man/repoman.1 | 4 ++++ > > 6 files changed, 29 insertions(+), 3 deletions(-) > > > > diff --git a/repoman/lib/repoman/actions.py b/repoman/lib/repoman/actions.py > > index 1c9989a72..92d4d4e94 100644 > > --- a/repoman/lib/repoman/actions.py > > +++ b/repoman/lib/repoman/actions.py > > @@ -412,6 +412,10 @@ the whole commit message to abort. > > report_options.append( > > "--include-arches=\"%s\"" % > > " ".join(sorted(self.scanner.include_arches))) > > + if self.scanner.include_profiles is not None: > > + report_options.append( > > + "--include-profiles=\"%s\"" % > > + " ".join(sorted(self.scanner.include_profiles))) > > > > if portage_version is None: > > sys.stderr.write("Failed to insert portage version in > > message!\n") > > diff --git a/repoman/lib/repoman/argparser.py > > b/repoman/lib/repoman/argparser.py > > index fa0e6ff90..670a0e91d 100644 > > --- a/repoman/lib/repoman/argparser.py > > +++ b/repoman/lib/repoman/argparser.py > > @@ -164,6 +164,13 @@ def parse_args(argv, repoman_default_opts): > > 'A space separated list of arches used to ' > > 'filter the selection of profiles for dependency > > checks')) > > > > + parser.add_argument( > > + '--include-profiles', > > + dest='include_profiles', metavar='PROFILES', action='append', > > + help=( > > + 'A space separated list of profiles used to ' > > + 'define the selection of profiles for dependency > > checks')) > > + > > parser.add_argument( > > '-d', '--include-dev', dest='include_dev', action='store_true', > > default=False, > > diff --git a/repoman/lib/repoman/modules/scan/depend/__init__.py > > b/repoman/lib/repoman/modules/scan/depend/__init__.py > > index c3cc0ddeb..9068760bb 100644 > > --- a/repoman/lib/repoman/modules/scan/depend/__init__.py > > +++ b/repoman/lib/repoman/modules/scan/depend/__init__.py > > @@ -19,7 +19,8 @@ module_spec = { > > 'func_desc': { > > }, > > 'mod_kwargs': ['qatracker', 'portdb', 'profiles', > > 'options', > > - 'repo_metadata', 'repo_settings', > > 'include_arches', 'caches', > > + 'repo_metadata', 'repo_settings', > > 'include_arches', > > + 'include_profiles', 'caches', > > 'repoman_incrementals', 'env', 'have', > > 'dev_keywords' > > ], > > 'func_kwargs': { > > diff --git a/repoman/lib/repoman/modules/scan/depend/profile.py > > b/repoman/lib/repoman/modules/scan/depend/profile.py > > index d980f4eca..0b1d74483 100644 > > --- a/repoman/lib/repoman/modules/scan/depend/profile.py > > +++ b/repoman/lib/repoman/modules/scan/depend/profile.py > > @@ -33,6 +33,7 @@ class ProfileDependsChecks(ScanBase): > > @param options: cli options > > @param repo_settings: repository settings instance > > @param include_arches: set > > + @param include_profiles: set > > @param caches: dictionary of our caches > > @param repoman_incrementals: tuple > > @param env: the environment > > @@ -46,6 +47,7 @@ class ProfileDependsChecks(ScanBase): > > self.options = kwargs.get('options') > > self.repo_settings = kwargs.get('repo_settings') > > self.include_arches = kwargs.get('include_arches') > > + self.include_profiles = kwargs.get('include_profiles') > > self.caches = kwargs.get('caches') > > self.repoman_incrementals = kwargs.get('repoman_incrementals') > > self.env = kwargs.get('env') > > @@ -81,8 +83,11 @@ class ProfileDependsChecks(ScanBase): > > if arch not in self.include_arches: > > continue > > > > - relevant_profiles.extend( > > - (keyword, groups, prof) for prof in > > self.profiles[arch]) > > + for prof in self.profiles[arch]: > > + if self.include_profiles is not None: > > + if prof not in self.include_profiles: > > + continue > > + relevant_profiles.append((keyword, groups, > > prof)) > > > > relevant_profiles.sort(key=sort_key) > > > > diff --git a/repoman/lib/repoman/scanner.py b/repoman/lib/repoman/scanner.py > > index 1b3242a51..06234b0ad 100644 > > --- a/repoman/lib/repoman/scanner.py > > +++ b/repoman/lib/repoman/scanner.py > > @@ -164,6 +164,10 @@ class Scanner(object): > > if self.options.include_arches: > > self.include_arches = set() > > self.include_arches.update(*[x.split() for x in > > self.options.include_arches]) > > + self.include_profiles = None > > + if self.options.include_profiles: > > + self.include_profiles = set() > > + self.include_profiles.update(*[x.split() for x in > > self.options.include_profiles]) > > > > # Disable the "self.modules['Ebuild'].notadded" check when not > > in commit mode and > > # running `svn status` in every package dir will be too > > expensive. > > @@ -190,6 +194,7 @@ class Scanner(object): > > "repo_metadata": self.repo_metadata, > > "profiles": profiles, > > "include_arches": self.include_arches, > > + "include_profiles": self.include_profiles, > > "caches": self.caches, > > "repoman_incrementals": self.repoman_incrementals, > > "env": self.env, > > diff --git a/repoman/man/repoman.1 b/repoman/man/repoman.1 > > index 766146f57..f33fb6098 100644 > > --- a/repoman/man/repoman.1 > > +++ b/repoman/man/repoman.1 > > @@ -120,6 +120,10 @@ Ignore masked packages (not allowed with commit mode) > > A space separated list of arches used to filter the selection of > > profiles for dependency checks. > > .TP > > +.BR "\-\-include\-profiles " PROFILES > > +A space separated list of profiles used to > > +define the selection of profiles for dependency checks. > > +.TP > > \fB\-d\fR, \fB\-\-include\-dev\fR > > Include dev profiles in dependency checks. > > .TP > > > > Looks good. Please merge. > -- > Thanks, > Zac > Thank you! Does not look like I have permissions to do it: """ $ cat .git/config [core] repositoryformatversion = 0 filemode = true bare = false logallrefupdates = true [remote "origin"] url = git+ssh://g...@git.gentoo.org/proj/portage.git fetch = +refs/heads/*:refs/remotes/origin/* [branch "master"] remote = origin merge = refs/heads/master $ LANG=C git push origin master FATAL -- ACCESS DENIED Repo proj/portage User sly...@gentoo.org Stage Before git was called Operation Repo write FATAL: W any proj/portage sly...@gentoo.org DENIED by fallthru (or you mis-spelled the reponame) fatal: Could not read from remote repository. Please make sure you have the correct access rights and the repository exists. """ -- Sergei
pgpBExd7IPfiz.pgp
Description: Цифровая подпись OpenPGP
