Log message for revision 117468: apply index refactored Changed: U Zope/branches/andig-compositeindex/src/Products/PluginIndexes/CompositeIndex/CompositeIndex.py U Zope/branches/andig-compositeindex/src/Products/PluginIndexes/CompositeIndex/util.py
-=- Modified: Zope/branches/andig-compositeindex/src/Products/PluginIndexes/CompositeIndex/CompositeIndex.py =================================================================== --- Zope/branches/andig-compositeindex/src/Products/PluginIndexes/CompositeIndex/CompositeIndex.py 2010-10-11 19:59:49 UTC (rev 117467) +++ Zope/branches/andig-compositeindex/src/Products/PluginIndexes/CompositeIndex/CompositeIndex.py 2010-10-11 22:11:58 UTC (rev 117468) @@ -21,6 +21,7 @@ from App.special_dtml import DTMLFile from BTrees.IIBTree import IIBTree, IITreeSet, IISet, union, intersection, difference +from BTrees.IIBTree import multiunion from BTrees.OOBTree import OOBTree from BTrees.IOBTree import IOBTree from BTrees.Length import Length @@ -36,7 +37,7 @@ from Products.PluginIndexes.common.util import parseIndexRequest from Products.PluginIndexes.common import safe_callable -from util import PermuteKeywordList +from util import PermuteKeywordList, powerset QUERY_OPTIONS = { 'FieldIndex' : ["query","range"] , 'KeywordIndex' : ["query","operator","range"] } @@ -197,89 +198,44 @@ operator = self.useOperator - rank=[] + + tmp = [] for c, rec in record.keys: - # experimental code for specifing the operator - if operator == self.useOperator: - operator = rec.get('operator',operator) - - if not operator in self.operators : - raise RuntimeError,"operator not valid: %s" % escape(operator) - - res = self._apply_component_index(rec,c) - - if res is None: - continue - - res, dummy = res - - rank.append((len(res),res)) + res, dummy = self._apply_component_index(rec,c) + tmp.append(res) + if len(tmp) > 2: + setlist = sorted(tmp, key=len) + else: + setlist = tmp - # sort from short to long sets - rank.sort() + ks = None + for s in setlist: + ks = intersection(ks, s) + if not ks: + break - k = None - - for l,res in rank: + tmp = [] + for k in ks: + ds = self._index.get(k, None) + if ds is None: + continue + elif isinstance(ds, int): + ds = IISet((ds,)) + tmp.append(ds) - k = intersection(k, res) - - if not k: - break + r = multiunion(tmp) - # if any operator of composite indexes is set to "and" - # switch to intersecton mode - - if operator == 'or': - res = None - set_func = union + if isinstance(r, int): + r = IISet((r, )) + if r is None: + return IISet(), (self.id,) else: - res = resultset - set_func = intersection - - - - rank=[] - if set_func == intersection: - for key in k: - - s=self._index.get(key, IISet()) - if isinstance(s, int): - rank.append((1,key)) - else: - rank.append((len(s),key)) - - # sort from short to long sets - rank.sort() + return r, (self.id,) - else: - # dummy length - if k: - rank = enumerate(k) - # collect docIds - for l,key in rank: - - s=self._index.get(key, None) - if s is None: - s = IISet(()) - elif isinstance(s, int): - s = IISet((s,)) - res = set_func(res, s) - if not res and set_func is intersection: - break - - if isinstance(res, int): res = IISet((res,)) - - if res is None: - res = IISet(),(self.id,) - - return res, (self.id,) - - - def _apply_component_index(self, record, cid): + def _apply_component_index(self, record, cid,resultset = None): """ Apply the component index to query parameters given in the record arg. """ if record.keys==None: return None @@ -288,7 +244,10 @@ r = None opr = None - + operator = record.get('operator',self.useOperator) + if not operator in self.operators : + raise RuntimeError,"operator not valid: %s" % escape(operator) + # Range parameter range_parm = record.get('range',None) if range_parm: @@ -304,29 +263,51 @@ opr = record.usage.lower().split(':') opr, opr_args=opr[0], opr[1:] + if opr=="range": # range search if 'min' in opr_args: lo = min(record.keys) else: lo = None if 'max' in opr_args: hi = max(record.keys) else: hi = None if hi: - setlist = index.items(lo,hi) + setlist = index.values(lo,hi) else: - setlist = index.items(lo) + setlist = index.values(lo) - for k, s in setlist: + tmp=[] + for s in setlist: if isinstance(s, tuple): s = IISet((s,)) - r = union(r, set) + tmp.append(s) + + r = multiunion(tmp) + else: # not a range search - for key in record.keys: - s=index.get(key, None) + tmp = [] + if operator == 'or': + for key in record.keys: + s=index.get(key, None) + if s is None: + continue + elif isinstance(s, int): + s = IISet((s,)) + tmp.append(s) + + r = multiunion(tmp) + + else: + r = None + if len(record.keys) > 1: + key = tuple(sorted(record.keys)) + else: + key = record.keys[0] + s=index.get(key, None) if s is None: - s = IISet(()) + s = IISet() elif isinstance(s, int): s = IISet((s,)) - r = union(r, s) + r = s if isinstance(r, int): r=IISet((r,)) @@ -361,7 +342,7 @@ # unhashed keywords newUKeywords = self._get_permuted_keywords(obj) - + # hashed keywords newKeywords = map(lambda x: hash(x),newUKeywords) @@ -565,6 +546,9 @@ datum.append(newKeywords) else: datum.extend(unique.keys()) + + datum.sort() + datum.extend(powerset(datum,start=2)) return datum else: raise KeyError Modified: Zope/branches/andig-compositeindex/src/Products/PluginIndexes/CompositeIndex/util.py =================================================================== --- Zope/branches/andig-compositeindex/src/Products/PluginIndexes/CompositeIndex/util.py 2010-10-11 19:59:49 UTC (rev 117467) +++ Zope/branches/andig-compositeindex/src/Products/PluginIndexes/CompositeIndex/util.py 2010-10-11 22:11:58 UTC (rev 117468) @@ -11,6 +11,7 @@ # ############################################################################## +from itertools import chain, combinations _marker = [] @@ -95,3 +96,6 @@ +def powerset(iterable,start=0): + s = list(iterable) + return chain.from_iterable(combinations(s, r) for r in range(start,len(s)+1)) _______________________________________________ Zope-Checkins maillist - Zope-Checkins@zope.org https://mail.zope.org/mailman/listinfo/zope-checkins