Log message for revision 117157: Added date range index improvements from ``experimental.catalogqueryplan``.
Changed: U Zope/trunk/doc/CHANGES.rst U Zope/trunk/src/Products/PluginIndexes/DateRangeIndex/DateRangeIndex.py -=- Modified: Zope/trunk/doc/CHANGES.rst =================================================================== --- Zope/trunk/doc/CHANGES.rst 2010-10-02 09:42:05 UTC (rev 117156) +++ Zope/trunk/doc/CHANGES.rst 2010-10-02 09:42:50 UTC (rev 117157) @@ -26,6 +26,8 @@ Features Added ++++++++++++++ +- Added date range index improvements from ``experimental.catalogqueryplan``. + - Changed policy on handling exceptions during ZCML parsing in ``Products``. We no longer catch any exceptions in non-debug mode. Modified: Zope/trunk/src/Products/PluginIndexes/DateRangeIndex/DateRangeIndex.py =================================================================== --- Zope/trunk/src/Products/PluginIndexes/DateRangeIndex/DateRangeIndex.py 2010-10-02 09:42:05 UTC (rev 117156) +++ Zope/trunk/src/Products/PluginIndexes/DateRangeIndex/DateRangeIndex.py 2010-10-02 09:42:50 UTC (rev 117157) @@ -20,10 +20,15 @@ from AccessControl.Permissions import manage_zcatalog_indexes from AccessControl.Permissions import view from AccessControl.SecurityInfo import ClassSecurityInfo +from Acquisition import aq_base +from Acquisition import aq_get +from Acquisition import aq_inner +from Acquisition import aq_parent from App.Common import package_home from App.special_dtml import DTMLFile from BTrees.IIBTree import IISet from BTrees.IIBTree import IITreeSet +from BTrees.IIBTree import difference from BTrees.IIBTree import intersection from BTrees.IIBTree import multiunion from BTrees.IOBTree import IOBTree @@ -40,6 +45,12 @@ MAX32 = int(2**31 - 1) +class RequestCache(dict): + + def __str__(self): + return "<RequestCache %s items>" % len(self) + + class DateRangeIndex(UnIndex): """Index for date ranges, such as the "effective-expiration" range in CMF. @@ -240,6 +251,13 @@ return tuple( result ) + def _cache_key(self, catalog): + cid = catalog.getId() + counter = getattr(aq_base(catalog), 'getCounter', None) + if counter is not None: + return '%s_%s' % (cid, counter()) + return cid + def _apply_index(self, request, resultset=None): """ Apply the index to query parameters given in 'request', which @@ -253,33 +271,71 @@ second object is a tuple containing the names of all data fields used. """ - record = parseIndexRequest(request, self.id, self.query_options) + iid = self.id + record = parseIndexRequest(request, iid, self.query_options) if record.keys is None: return None - term = self._convertDateTime( record.keys[0] ) + term = self._convertDateTime(record.keys[0]) - # - # Aggregate sets for each bucket separately, to avoid - # large-small union penalties. - # - until_only = multiunion( self._until_only.values( term ) ) - since_only = multiunion( self._since_only.values( None, term ) ) - until = multiunion( self._until.values( term ) ) + REQUEST = aq_get(self, 'REQUEST', None) + if REQUEST is not None: + catalog = aq_parent(aq_parent(aq_inner(self))) + if catalog is not None: + key = self._cache_key(catalog) + cache = REQUEST.get(key, None) + tid = term / 10 + if resultset is None: + cachekey = '_daterangeindex_%s_%s' % (iid, tid) + else: + cachekey = '_daterangeindex_inverse_%s_%s' % (iid, tid) + if cache is None: + cache = REQUEST[key] = RequestCache() + else: + cached = cache.get(cachekey, None) + if cached is not None: + if resultset is None: + return (cached, + (self._since_field, self._until_field)) + else: + return (difference(resultset, cached), + (self._since_field, self._until_field)) - # Total result is bound by resultset - until = intersection(resultset, until) - since = multiunion(self._since.values(None, term)) - bounded = intersection(until, since) + if resultset is None: + # Aggregate sets for each bucket separately, to avoid + # large-small union penalties. + until_only = multiunion(self._until_only.values(term)) + since_only = multiunion(self._since_only.values(None, term)) + until = multiunion(self._until.values(term)) - # Merge from smallest to largest. - result = multiunion([bounded, until_only, since_only, self._always]) + # Total result is bound by resultset + if REQUEST is None: + until = intersection(resultset, until) - return result, ( self._since_field, self._until_field ) + since = multiunion(self._since.values(None, term)) + bounded = intersection(until, since) - # - # Helper functions. - # + # Merge from smallest to largest. + result = multiunion([bounded, until_only, since_only, + self._always]) + if REQUEST is not None and catalog is not None: + cache[cachekey] = result + + return (result, (self._since_field, self._until_field)) + else: + # Compute the inverse and subtract from res + until_only = multiunion(self._until_only.values(None, term)) + since_only = multiunion(self._since_only.values(term)) + until = multiunion(self._until.values(None, term)) + since = multiunion(self._since.values(term)) + + result = multiunion([until_only, since_only, until,since]) + if REQUEST is not None and catalog is not None: + cache[cachekey] = result + + return (difference(resultset, result), + (self._since_field, self._until_field)) + def _insertForwardIndexEntry( self, since, until, documentId ): """ Insert 'documentId' into the appropriate set based on _______________________________________________ Zope-Checkins maillist - Zope-Checkins@zope.org https://mail.zope.org/mailman/listinfo/zope-checkins