commit: d22ba91d4adc177551be8b4be95a6fc1f061fc2e Author: Zac Medico <zmedico <AT> gentoo <DOT> org> AuthorDate: Sat Nov 1 15:04:46 2014 +0000 Commit: Zac Medico <zmedico <AT> gentoo <DOT> org> CommitDate: Sun Dec 7 23:10:48 2014 +0000 URL: http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=d22ba91d
Add egencache --update-pkg-desc-index action. This adds an egencache --update-pkg-desc-index action which generates a plain-text index of package names, versions, and descriptions. The index can then be used to optimize emerge --search / --searchdesc actions. X-Gentoo-Bug: 525718 X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=525718 --- bin/egencache | 38 ++++++++++++++++++-- man/egencache.1 | 4 +++ man/portage.5 | 12 +++++++ pym/portage/cache/index/__init__.py | 2 ++ pym/portage/cache/index/pkg_desc_index.py | 59 +++++++++++++++++++++++++++++++ 5 files changed, 113 insertions(+), 2 deletions(-) diff --git a/bin/egencache b/bin/egencache index e366058..f97432f 100755 --- a/bin/egencache +++ b/bin/egencache @@ -48,6 +48,7 @@ portage._internal_caller = True from portage import os, _encodings, _unicode_encode, _unicode_decode from _emerge.MetadataRegen import MetadataRegen from portage.cache.cache_errors import CacheError, StatCollision +from portage.cache.index.pkg_desc_index import pkg_desc_index_line_format from portage.const import TIMESTAMP_FORMAT from portage.manifest import guessManifestFileType from portage.package.ebuild._parallel_manifest.ManifestScheduler import ManifestScheduler @@ -57,7 +58,7 @@ from portage.util._async.run_main_scheduler import run_main_scheduler from portage.util._eventloop.global_event_loop import global_event_loop from portage import cpv_getkey from portage.dep import Atom, isjustname -from portage.versions import pkgsplit, vercmp +from portage.versions import pkgsplit, vercmp, _pkg_str try: from xml.etree import ElementTree @@ -91,6 +92,9 @@ def parse_args(args): actions.add_argument("--update-changelogs", action="store_true", help="update the ChangeLog files from SCM logs") + actions.add_argument("--update-pkg-desc-index", + action="store_true", + help="update package description index") actions.add_argument("--update-manifests", action="store_true", help="update manifests") @@ -451,6 +455,29 @@ class GenCache(object): if hasattr(trg_cache, '_prune_empty_dirs'): trg_cache._prune_empty_dirs() +class GenPkgDescIndex(object): + def __init__(self, portdb, output_file): + self.returncode = os.EX_OK + self._portdb = portdb + self._output_file = output_file + + def run(self): + + portage.util.ensure_dirs(os.path.dirname(self._output_file)) + f = portage.util.atomic_ofstream(self._output_file, + encoding=_encodings["repo.content"]) + + portdb = self._portdb + for cp in portdb.cp_all(): + pkgs = portdb.cp_list(cp) + if not pkgs: + continue + desc, = portdb.aux_get(pkgs[-1], ["DESCRIPTION"]) + + f.write(pkg_desc_index_line_format(cp, pkgs, desc)) + + f.close() + class GenUseLocalDesc(object): def __init__(self, portdb, output=None, preserve_comments=False): @@ -893,7 +920,8 @@ def egencache_main(args): local_config=False, env=env) if not (options.update or options.update_use_local_desc or - options.update_changelogs or options.update_manifests): + options.update_changelogs or options.update_manifests or + options.update_pkg_desc_index): parser.error('No action specified') return 1 @@ -1057,6 +1085,12 @@ def egencache_main(args): else: ret.append(scheduler.returncode) + if options.update_pkg_desc_index: + gen_index = GenPkgDescIndex(portdb, os.path.join( + repo_config.location, "metadata", "pkg_desc_index")) + gen_index.run() + ret.append(gen_index.returncode) + if options.update_use_local_desc: gen_desc = GenUseLocalDesc(portdb, output=options.uld_output, diff --git a/man/egencache.1 b/man/egencache.1 index f71feb3..3a3197f 100644 --- a/man/egencache.1 +++ b/man/egencache.1 @@ -19,6 +19,10 @@ for the details on package atom syntax. .BR "\-\-update\-changelogs" Update the ChangeLog files from SCM logs (supported only in git repos). .TP +.BR "\-\-update\-pkg\-desc\-index" +Update the package description index which is located at +\fImetadata/pkg_desc_index\fR in the repository. +.TP .BR "\-\-update\-use\-local\-desc" Update the \fIprofiles/use.local.desc\fR file from metadata.xml. .TP diff --git a/man/portage.5 b/man/portage.5 index 4a02c64..2fa699c 100644 --- a/man/portage.5 +++ b/man/portage.5 @@ -76,6 +76,7 @@ user\-defined package sets .BR /usr/portage/metadata/ .nf layout.conf +pkg_desc_index .fi .TP .BR /usr/portage/profiles/ @@ -1165,6 +1166,17 @@ cache\-formats = md5-dict pms profile\-formats = portage-2 .fi .RE +.TP +.BR pkg_desc_index +This is an index of package names, versions, and descriptions which +may be generated by \fBegencache\fR(1) in order to optimize +\fBemerge\fR(1) search actions. + +.I Example: +.nf +sys-apps/sed 4.2 4.2.1 4.2.1-r1 4.2.2: Super-useful stream editor +sys-apps/usleep 0.1: A wrapper for usleep +.fi .RE .TP .BR /usr/portage/profiles/ diff --git a/pym/portage/cache/index/__init__.py b/pym/portage/cache/index/__init__.py new file mode 100644 index 0000000..7cd880e --- /dev/null +++ b/pym/portage/cache/index/__init__.py @@ -0,0 +1,2 @@ +# Copyright 2014 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 diff --git a/pym/portage/cache/index/pkg_desc_index.py b/pym/portage/cache/index/pkg_desc_index.py new file mode 100644 index 0000000..a2e45da --- /dev/null +++ b/pym/portage/cache/index/pkg_desc_index.py @@ -0,0 +1,59 @@ +# Copyright 2014 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from __future__ import unicode_literals + +import collections +import sys + +from portage.versions import _pkg_str + +if sys.hexversion >= 0x3000000: + _unicode = str +else: + _unicode = unicode + +pkg_desc_index_node = collections.namedtuple("pkg_desc_index_node", + ["cp", "cpv_list", "desc"]) + +class pkg_node(_unicode): + """ + A minimal package node class. For performance reasons, inputs + are not validated. + """ + + def __init__(self, cp, version, repo=None): + self.__dict__['cp'] = cp + self.__dict__['repo'] = repo + self.__dict__['version'] = version + + def __new__(cls, cp, version, repo=None): + return _unicode.__new__(cls, cp + "-" + version) + + def __setattr__(self, name, value): + raise AttributeError("pkg_node instances are immutable", + self.__class__, name, value) + +def pkg_desc_index_line_format(cp, pkgs, desc): + return "%s %s: %s\n" % (cp, + " ".join(_pkg_str(cpv).version + for cpv in pkgs), desc) + +def pkg_desc_index_line_read(line, repo=None): + + try: + pkgs, desc = line.split(":", 1) + except ValueError: + return None + desc = desc.strip() + + try: + cp, pkgs = pkgs.split(" ", 1) + except ValueError: + return None + + cp_list = [] + for ver in pkgs.split(): + cp_list.append(pkg_node(cp, ver, repo)) + + return pkg_desc_index_node(cp, tuple(cp_list), desc)
