This is another attempt to prepare the way for including metascan, this time based on a new restriction framework (which has absolutely nothing to do with the one in savior/bcportage) included in the attached and completely undocumented patch (once you get the concept it should be self-explanatory however). It basically consists of three class hierarchies: - dataclasses responsibe for delivering the data to match against - filterclasses implementing the match algorithms - boolean connectors to combine multiple restrictions There is a fourth type called prefilter that can adjust the match algorithm of filter classes (e.g. a Split() class to break multi-value metadata fields up for the string matchers).
The filterclasses and connectors implement the same interface for
performing the matches, so the most basic application would be:
for cpv in cpvlist:
if restriction.filter.PartialMatch(data.Metadata(key, db),
value).match(cpv):
matches.append(cpv)
For a more complex application see the patched emerge or metascan2
(at d.g.o/~genone/scripts).
A word of warning: I've just written all this code in the past few hours,
it is absolutely unoptimized, incomplete and certainly has bugs (none known
though).
That also means I'm not interested in comments about the code or algorithm
yet, as this needs a lot of work and so far I've only done the work to get
it working, mainly sending this mail so people can play with it a bit.
Marius
--
Public Key at http://www.genone.de/info/gpg-key.pub
In the beginning, there was nothing. And God said, 'Let there be
Light.' And there was still nothing, but you could see a bit better.
diff -ruN vanilla/bin/emerge portage/bin/emerge
--- vanilla/bin/emerge 2006-02-16 09:38:57.000000000 +0100
+++ portage/bin/emerge 2006-02-18 12:19:52.000000000 +0100
@@ -189,7 +189,8 @@
"--tree",
"--update", "--upgradeonly",
"--usepkg", "--usepkgonly",
-"--verbose", "--version"
+"--verbose", "--version",
+"--newsearch", "--regex"
]
shortmapping={
@@ -693,7 +694,86 @@
if x in myparams:
myparams.remove(x)
+class dumb_formatter(object):
+ def __init__(self):
+ pass
+
+ def generateReport(self, result):
+ rval = "\n"
+ for pkg in result.keys():
+ rval += str(result[pkg])+"\n\n"
+ return rval
+
# search functionality
+class restriction_search(object):
+ DATA_NAME = 1
+ DATA_DESC = 2
+
+ FILTER_REGEX = 1
+ FILTER_PARTIAL = 2
+ FILTER_EXACT = 3
+
+ def __init__(self, formatter):
+ global myopts
+ if "--searchdesc" in myopts:
+ self.key = self.DATA_DESC
+ else:
+ self.key = self.DATA_NAME
+ if "--regex" in myopts:
+ self.type = self.FILTER_REGEX
+ else:
+ self.type = self.FILTER_PARTIAL
+ self.formatter = formatter
+
+ def execute(self, searchkey):
+ from restrictions import data, filter
+
+ search_category = False
+ if searchkey.startswith("@"):
+ search_category = True
+ searchkey = searchkey[1:]
+
+ if self.key == self.DATA_DESC:
+ primarydata = data.Metadata("DESCRIPTION", portage.portdb)
+ elif self.key == self.DATA_NAME:
+ primarydata = data.PackageName(with_category=search_category)
+ if self.type == self.FILTER_REGEX:
+ primaryfilter = filter.RegexMatch(primarydata, searchkey)
+ presearch_re = re.compile(searchkey)
+ elif self.type == self.FILTER_PARTIAL:
+ primaryfilter = filter.PartialMatch(primarydata, searchkey, ignoreCase=True)
+ presearch_re = re.compile(".*"+re.escape(searchkey)+".*")
+
+ self.result = {}
+ for package in portage.portdb.cp_all():
+ update_spinner()
+ # This is required to get similar performance as the old code, if we
+ # don't then we have to do at least one xmatch() call per package
+ # slowing things down extremely
+ if self.key == self.DATA_NAME and not presearch_re.match(package):
+ continue
+ masked = False
+ cpv = portage.portdb.xmatch("bestmatch-visible",package)
+ if len(cpv) == 0:
+ cpv = portage.best(portage.portdb.xmatch("match-all",package))
+ masked = True
+ if len(cpv) == 0:
+ print package
+ if primaryfilter.match(cpv):
+ update_spinner()
+ self.result[package] = {}
+ self.result[package]["masked"] = masked
+ self.result[package]["cpv_tree"] = cpv
+ self.result[package]["cpv_installed"] = portage.best(portage.db["/"]["vartree"].dbapi.match(package))
+ metadata_result = portage.portdb.aux_get(cpv, ["DESCRIPTION", "LICENSE", "HOMEPAGE"])
+ self.result[package]["desc"] = metadata_result[0]
+ self.result[package]["license"] = metadata_result[1]
+ self.result[package]["homepage"] = metadata_result[2]
+ self.result[package]["size"] = format_size(portage.portdb.getsize(cpv))
+
+ def output(self):
+ print self.formatter.generateReport(self.result)
+
class search:
#
@@ -3059,7 +3139,10 @@
if not myfiles:
print "emerge: no search terms provided."
else:
- searchinstance = search()
+ if "--newsearch" in myopts:
+ searchinstance = restriction_search(dumb_formatter())
+ else:
+ searchinstance = search()
for mysearch in myfiles:
try:
searchinstance.execute(mysearch)
--- vanilla/pym/portage.py 2006-02-16 09:38:58.000000000 +0100
+++ portage/pym/portage.py 2006-02-18 11:42:49.000000000 +0100
@@ -285,7 +285,7 @@
if ftype is None:
ftype=[]
- if not filesonly and not recursive:
+ if not (filesonly or dirsonly or recursive):
return list
if recursive:
@@ -5017,7 +5017,7 @@
filesdict=self.getfetchsizes(mypkg,useflags=useflags,debug=debug)
if filesdict==None:
return "[empty/missing/bad digest]"
- mysize=0
+ mysum=0
for myfile in filesdict.keys():
mysum+=filesdict[myfile]
return mysum
diff -ruN vanilla/pym/restrictions/__init__.py portage/pym/restrictions/__init__.py
--- vanilla/pym/restrictions/__init__.py 1970-01-01 01:00:00.000000000 +0100
+++ portage/pym/restrictions/__init__.py 2006-02-18 07:43:31.000000000 +0100
@@ -0,0 +1 @@
+import base, boolean, data, filter, prefilter
diff -ruN vanilla/pym/restrictions/base.py portage/pym/restrictions/base.py
--- vanilla/pym/restrictions/base.py 1970-01-01 01:00:00.000000000 +0100
+++ portage/pym/restrictions/base.py 2006-02-18 06:08:41.000000000 +0100
@@ -0,0 +1,29 @@
+class RestrictionData(object):
+ def __init__(self):
+ raise NotImplementedError()
+
+ def read(self, cpv):
+ raise NotImplementedError()
+
+class RestrictionFilter(object):
+ def __init__(self, data, search_key):
+ raise NotImplementedError()
+
+ def match(self, cpv):
+ raise NotImplementedError()
+
+class RestrictionBoolean(object):
+ def __init__(self, restrictions):
+ raise NotImplementedError()
+
+ def match(self, cpv):
+ raise NotImplementedError()
+
+class BaseRestriction(object):
+ def __init__(self, restriction, invert=False):
+ self.restriction = restriction
+ self.invert = invert
+
+ def match(self, cpv):
+ return self.invert ^ self.restriction.match(cpv)
+
diff -ruN vanilla/pym/restrictions/boolean.py portage/pym/restrictions/boolean.py
--- vanilla/pym/restrictions/boolean.py 1970-01-01 01:00:00.000000000 +0100
+++ portage/pym/restrictions/boolean.py 2006-02-18 05:28:42.000000000 +0100
@@ -0,0 +1,29 @@
+from base import *
+
+class And(RestrictionBoolean):
+ def __init__(self, restrictions):
+ self.restrictions = restrictions
+
+ def match(self, cpv):
+ for r in self.restrictions:
+ if not r.match(cpv):
+ return False
+ return True
+
+class Or(RestrictionBoolean):
+ def __init__(self, restrictions):
+ self.restrictions = restrictions
+
+ def match(self, cpv):
+ for r in self.restrictions:
+ if r.match(cpv):
+ return True
+ return False
+
+class Xor(RestrictionBoolean):
+ def __init__(self, restrictions):
+ self.restrictions = restrictions
+
+ def match(self, cpv):
+ result = [r.match(cpv) for r in self.restrictions]
+ return (result.count(True) == 1)
diff -ruN vanilla/pym/restrictions/data.py portage/pym/restrictions/data.py
--- vanilla/pym/restrictions/data.py 1970-01-01 01:00:00.000000000 +0100
+++ portage/pym/restrictions/data.py 2006-02-18 09:26:51.000000000 +0100
@@ -0,0 +1,89 @@
+from base import *
+from portage_versions import catpkgsplit
+import sets
+import portage
+
+class String(RestrictionData):
+ pass
+
+class Boolean(RestrictionData):
+ pass
+
+class PackageName(String):
+ def __init__(self, with_category=True, with_name=True, with_version=False, with_revision=False):
+ self.category = with_category
+ self.name = with_name
+ self.version = with_version
+ self.revision = with_revision
+
+ def _append(self, oldval, sep, element):
+ if len(oldval) > 0:
+ return oldval+sep+element
+ else:
+ return element
+
+ def read(self, cpv):
+ mysplit=catpkgsplit(cpv)
+ rval = ""
+ mylist = []
+ if self.category:
+ rval = self._append(rval, "", mysplit[0])
+ if self.name:
+ rval = self._append(rval, "/", mysplit[1])
+ if self.version:
+ rval = self._append(rval, "-", mysplit[2])
+ if self.revision:
+ rval = self._append(rval, "-", mysplit[2])
+ return rval
+
+class Metadata(String):
+ def __init__(self, key, db, use_cache=True, update_cache=True):
+ self.key = key
+ self.use_cache = use_cache
+ self.update_cache = update_cache
+ self.db = db
+
+ data_cache={}
+ def read(self, cpv):
+ db = self.db
+ if self.use_cache:
+ try:
+ rval = self.data_cache[db][cpv][self.key]
+ return rval
+ except KeyError:
+ pass
+
+ readkeys = portage.auxdbkeys
+ values = self.db.aux_get(cpv, readkeys)
+ rval = values[readkeys.index(self.key)]
+ if self.update_cache:
+ if not self.data_cache.has_key(db):
+ self.data_cache[db] = {}
+ if not self.data_cache[db].has_key(cpv):
+ self.data_cache[db][cpv] = {}
+ self.data_cache[db][cpv][self.key] = rval
+
+ return rval
+
+ def getKey(self):
+ return self.key
+
+class PackageMasked(Boolean):
+ def __init__(self, settings):
+ self.pmask = settings.pmaskdict
+ self.punmask = settings.punmaskdict
+
+ def read(self, cpv):
+ cp = portage.dep_getkey(cpv)
+ masked = False
+ if cp in self.pmask.keys():
+ for atom in self.pmask[cp]:
+ if len(portage.match_from_list(atom, [cpv])) > 0:
+ masked = True
+ break
+ if masked and cp in self.punmask.keys():
+ for atom in self.punmask[cp]:
+ if len(portage.match_from_list(atom, [cpv])) > 0:
+ masked = False
+ break
+ return masked
diff -ruN vanilla/pym/restrictions/filter.py portage/pym/restrictions/filter.py
--- vanilla/pym/restrictions/filter.py 1970-01-01 01:00:00.000000000 +0100
+++ portage/pym/restrictions/filter.py 2006-02-18 09:29:39.000000000 +0100
@@ -0,0 +1,90 @@
+from base import *
+from prefilter import *
+from data import Boolean, String
+import re
+
+class Boolean(RestrictionFilter):
+ def __init__(self, dataclass, value):
+ if not isinstance(dataclass, data.Boolean):
+ raise TypeError()
+ self.dataclass = dataclass
+ self.value = value
+
+ def match(self, cpv):
+ return (self.dataclass.read(cpv) == self.value)
+
+class Text(RestrictionFilter):
+ def init(self, dataclass, prefilters):
+ if not isinstance(dataclass, String):
+ raise TypeError()
+ self.dataclass = dataclass
+ if isinstance(prefilters, list):
+ self.prefilters = prefilters
+ else:
+ self.prefilters = []
+
+ def _match(self, element):
+ raise NotImplementedError()
+
+ def _prefilter(self, dataclass):
+ rval = dataclass
+ for f in self.prefilters:
+ rval = f.filter(rval)
+ return rval
+
+ def match(self, cpv):
+ dataclass = self.dataclass.read(cpv)
+ result = self._prefilter(dataclass)
+
+ if isinstance(result, list):
+ for e in result:
+ if self._match(e):
+ return True
+ else:
+ return self._match(result)
+
+ def addPreFilter(self, prefilter):
+ if not isinstance(prefilter, RestrictionFilter):
+ raise TypeError()
+ self.prefilters.append(prefilter)
+
+ def clearPreFilters(self):
+ self.prefilters = []
+
+class RegexMatch(Text):
+ def __init__(self, dataclass, regex, prefilters=None):
+ self.regex = re.compile(regex)
+ self.init(dataclass, prefilters)
+
+ def _match(self, element):
+ return self.regex.match(element)
+
+class PartialMatch(Text):
+ def __init__(self, dataclass, string, ignoreCase=False, prefilters=None):
+ if ignoreCase:
+ self.string = string.upper()
+ else:
+ self.string = string
+ self.ignoreCase = ignoreCase
+ self.init(dataclass, prefilters)
+
+ def _match(self, element):
+ if self.ignoreCase:
+ element = element.upper()
+ return (element.find(self.string) >= 0)
+
+class ExactMatch(Text):
+ def __init__(self, dataclass, string, ignoreCase=False, prefilters=None):
+ if ignoreCase:
+ self.string = string.upper()
+ else:
+ self.string = string
+ self.ignoreCase = ignoreCase
+ self.init(dataclass, prefilters)
+
+ def _match(self, element):
+ if self.ignoreCase:
+ rval = (element.upper() == self.string)
+ else:
+ rval = (element == self.string)
+ return rval
diff -ruN vanilla/pym/restrictions/prefilter.py portage/pym/restrictions/prefilter.py
--- vanilla/pym/restrictions/prefilter.py 1970-01-01 01:00:00.000000000 +0100
+++ portage/pym/restrictions/prefilter.py 2006-02-18 08:33:15.000000000 +0100
@@ -0,0 +1,41 @@
+from portage import isvalidatom, dep_getkey
+
+class PreFilter(object):
+ def __init__(self):
+ raise NotImplementedError()
+
+ def _filter(self, data):
+ raise NotImplementedError()
+
+ def filter(self, data):
+ if isinstance(data, list):
+ rval = []
+ for e in data:
+ result = self._filter(e)
+ if isinstance(result, list):
+ rval.extend(result)
+ else:
+ rval.append(result)
+ else:
+ rval = self._filter(data)
+ return rval
+
+class Split(PreFilter):
+ def __init__(self):
+ pass
+
+ def _filter(self, data):
+ if not isinstance(data, str):
+ return TypeError()
+ return data.split()
+
+class AtomStrip(PreFilter):
+ def __init__(self):
+ pass
+
+ def _filter(self, data):
+ if isvalidatom(data):
+ rval = dep_getkey(data)
+ else:
+ rval = data
+ return rval
signature.asc
Description: PGP signature
