commit: 55c8c8bc7a781e3f71ce92922eea64ad4cafce3c Author: Zac Medico <zmedico <AT> gentoo <DOT> org> AuthorDate: Wed Nov 5 05:07:26 2014 +0000 Commit: Zac Medico <zmedico <AT> gentoo <DOT> org> CommitDate: Sun Dec 7 23:10:48 2014 +0000 URL: http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=55c8c8bc
Add emerge --search-index option. The new emerge --search-index option, which is enabled by default, causes pkg_desc_index to be used for search optimization. The search index needs to be regenerated by egencache after changes are made to a repository (see the --update-pkg-desc-index action). For users that would like to modify ebuilds in a repository without running egencache afterwards, emerge --search-index=n can be used to get non-indexed search. Alternatively, the user could simply remove the stale index file, in order to disable the search index for a particular repository. In order to conserve memory, indices are read as streams, and MultiIterGroupBy is used to group results from IndexedPortdb and IndexedVardb. Stream-oriented search also makes it possible to display search results incrementally (for bug #412471). X-Gentoo-Bug: 525718 X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=525718 --- man/emerge.1 | 8 ++++++++ pym/_emerge/actions.py | 3 ++- pym/_emerge/depgraph.py | 2 +- pym/_emerge/main.py | 5 +++++ pym/_emerge/search.py | 31 ++++++++++++++++++++++++++----- 5 files changed, 42 insertions(+), 7 deletions(-) diff --git a/man/emerge.1 b/man/emerge.1 index be46a7d..b9ce79c 100644 --- a/man/emerge.1 +++ b/man/emerge.1 @@ -804,6 +804,14 @@ If ebuilds using EAPIs which \fIdo not\fR support \fBHDEPEND\fR are built in the same \fBemerge\fR run as those using EAPIs which \fIdo\fR support \fBHDEPEND\fR, this option affects only the former. .TP +.BR "\-\-search\-index < y | n >" +Enable or disable indexed search for search actions. This option is +enabled by default. The search index needs to be regenerated by +\fBegencache\fR(1) after changes are made to a repository (see the +\fB\-\-update\-pkg\-desc\-index\fR action). This setting can be added +to \fBEMERGE_DEFAULT_OPTS\fR (see \fBmake.conf\fR(5)) and later +overridden via the command line. +.TP .BR "\-\-select [ y | n ] (\-w short option)" Add specified packages to the world set (inverse of \fB\-\-oneshot\fR). This is useful if you want to diff --git a/pym/_emerge/actions.py b/pym/_emerge/actions.py index 434fc47..c7246a9 100644 --- a/pym/_emerge/actions.py +++ b/pym/_emerge/actions.py @@ -1901,7 +1901,8 @@ def action_search(root_config, myopts, myfiles, spinner): searchinstance = search(root_config, spinner, "--searchdesc" in myopts, "--quiet" not in myopts, "--usepkg" in myopts, - "--usepkgonly" in myopts) + "--usepkgonly" in myopts, + search_index = myopts.get("--search-index", "y") != "n") for mysearch in myfiles: try: searchinstance.execute(mysearch) diff --git a/pym/_emerge/depgraph.py b/pym/_emerge/depgraph.py index 3455b6b..df1e266 100644 --- a/pym/_emerge/depgraph.py +++ b/pym/_emerge/depgraph.py @@ -8730,7 +8730,7 @@ def ambiguous_package_name(arg, atoms, root_config, spinner, myopts): s = search(root_config, spinner, "--searchdesc" in myopts, "--quiet" not in myopts, "--usepkg" in myopts, - "--usepkgonly" in myopts) + "--usepkgonly" in myopts, search_index = False) null_cp = portage.dep_getkey(insert_category_into_atom( arg, "null")) cat, atom_pn = portage.catsplit(null_cp) diff --git a/pym/_emerge/main.py b/pym/_emerge/main.py index 1506207..66d4891 100644 --- a/pym/_emerge/main.py +++ b/pym/_emerge/main.py @@ -617,6 +617,11 @@ def parse_opts(tmpcmdline, silent=False): "choices" :("True", "rdeps") }, + "--search-index": { + "help": "Enable or disable indexed search (enabled by default)", + "choices": y_or_n + }, + "--select": { "shortopt" : "-w", "help" : "add specified packages to the world set " + \ diff --git a/pym/_emerge/search.py b/pym/_emerge/search.py index e51d206..4b1833a 100644 --- a/pym/_emerge/search.py +++ b/pym/_emerge/search.py @@ -5,9 +5,12 @@ import re import portage from portage import os from portage.dbapi.porttree import _parse_uri_map +from portage.dbapi.IndexedPortdb import IndexedPortdb +from portage.dbapi.IndexedVardb import IndexedVardb from portage.localization import localized_size from portage.output import bold, bold as white, darkgreen, green, red from portage.util import writemsg_stdout +from portage.util.iterators.MultiIterGroupBy import MultiIterGroupBy from _emerge.Package import Package @@ -23,7 +26,7 @@ class search(object): # public interface # def __init__(self, root_config, spinner, searchdesc, - verbose, usepkg, usepkgonly): + verbose, usepkg, usepkgonly, search_index=True): """Searches the available and installed packages for the supplied search key. The list of available and installed packages is created at object instantiation. This makes successive searches faster.""" @@ -45,6 +48,10 @@ class search(object): bindb = root_config.trees["bintree"].dbapi vardb = root_config.trees["vartree"].dbapi + if search_index: + portdb = IndexedPortdb(portdb) + vardb = IndexedVardb(vardb) + if not usepkgonly and portdb._have_root_eclass_dir: self._dbs.append(portdb) @@ -60,10 +67,16 @@ class search(object): self.spinner.update() def _cp_all(self): - cp_all = set() + iterators = [] for db in self._dbs: - cp_all.update(db.cp_all()) - return list(sorted(cp_all)) + i = db.cp_all() + try: + i = iter(i) + except TypeError: + pass + iterators.append(i) + for group in MultiIterGroupBy(iterators): + yield group[0] def _aux_get(self, *args, **kwargs): for db in self._dbs: @@ -154,9 +167,17 @@ class search(object): result = cpv else: db_keys = list(db._aux_cache_keys) + matches = db.match(atom) + try: + db.match_unordered + except AttributeError: + pass + else: + db._cpv_sort_ascending(matches) + # break out of this loop with highest visible # match, checked in descending order - for cpv in reversed(db.match(atom)): + for cpv in reversed(matches): if portage.cpv_getkey(cpv) != cp: continue metadata = zip(db_keys,
