dabo Commit
Revision 7110
Date: 2012-03-12 13:14:57 -0700 (Mon, 12 Mar 2012)
Author: Paul
Trac: http://trac.dabodev.com/changeset/7110
Changed:
U trunk/dabo/biz/dBizobj.py
Log:
I believe I've successfully worked around the ongoing performance problems
associated with biz.saveAll() and biz.cancelAll() in the common case of
calling these on a biz with no Parent but a tree of children below. The
most common example is when the user clicks the 'save' button in a dForm.
The problem was that saveAll() and cancelAll() had to iterate every single
record, and every single record in all child bizobjs, to find changes. My
change only makes the entire iteration necessary in an exceptional case
if at all.
We now maintain a set() of keys of visited rows on the top-parent bizobj,
and during saveAll() and cancelAll() we first save/cancel changes to the
records we know we've visited since the last requery(), save(), or cancel().
Then, we call isAnyChanged() which will be lightning fast if there are no
more changes, to determine if we need to do the old behavior of iterating all
rows to catch the outliers.
Testing on my app yields huge performance gains (<1 second down from ~20) when
modifying one field in one parent record, and clicking the save button.
Diff:
Modified: trunk/dabo/biz/dBizobj.py
===================================================================
--- trunk/dabo/biz/dBizobj.py 2012-03-11 16:57:41 UTC (rev 7109)
+++ trunk/dabo/biz/dBizobj.py 2012-03-12 20:14:57 UTC (rev 7110)
@@ -30,6 +30,7 @@
def __init__(self, conn=None, properties=None, *args, **kwargs):
""" User code should override beforeInit() and/or afterInit()
instead."""
self.__att_try_setFieldVal = False
+ self._visitedKeys = set()
# Collection of cursor objects. MUST be defined first.
self.__cursors = {}
# PK of the currently-selected cursor
@@ -37,7 +38,6 @@
# Description of the data represented by this bizobj
self._dataStructure = None
self._dataSource = self._dataSourceName = ""
-
# Dictionary holding any default values to apply when a new
record is created. This is
# now the DefaultValues property (used to be self.defaultValues
attribute)
self._defaultValues = {}
@@ -261,7 +261,7 @@
self._CurrentCursor.first()
self.requeryAllChildren(_doRequery=self.RequeryChildrenOnNavigate)
- self.afterPointerMove()
+ self._afterPointerMove()
self.afterFirst()
@@ -281,7 +281,7 @@
self._CurrentCursor.prior()
self.requeryAllChildren(_doRequery=self.RequeryChildrenOnNavigate)
- self.afterPointerMove()
+ self._afterPointerMove()
self.afterPrior()
@@ -301,7 +301,7 @@
self._CurrentCursor.next()
self.requeryAllChildren(_doRequery=self.RequeryChildrenOnNavigate)
- self.afterPointerMove()
+ self._afterPointerMove()
self.afterNext()
@@ -321,7 +321,7 @@
self._CurrentCursor.last()
self.requeryAllChildren(_doRequery=self.RequeryChildrenOnNavigate)
- self.afterPointerMove()
+ self._afterPointerMove()
self.afterLast()
@@ -430,19 +430,33 @@
raise dException.BusinessRuleViolation(errMsg)
startTransaction = startTransaction and self.beginTransaction()
+
+ # First save the rows we know we've visited:
try:
- self.scan(self.save, startTransaction=False,
+ self.scanKeys(self.save, self._visitedKeys,
startTransaction=False,
saveTheChildren=saveTheChildren,
scanRequeryChildren=False)
- if startTransaction:
- self.commitTransaction()
- except dException.ConnectionLostException:
- pass
except (dException.DBQueryException, dException.dException):
- # Something failed; reset things.
if startTransaction:
self.rollbackTransaction()
raise
+ # Finally, scan all rows only if there are still potentially
unsaved rows.
+ # The isAnyChanged() call will be expensive if there are
changes buried
+ # in some out-of-context child cursor, but that should be rare.
In the
+ # common case, all the changes would have already been made in
the above
+ # block, and isAnyChanged() will return False very quickly in
that case.
+ if self.isAnyChanged():
+ try:
+ self.scan(self.save, startTransaction=False,
+
saveTheChildren=saveTheChildren, scanRequeryChildren=False)
+ except (dException.DBQueryException,
dException.dException):
+ if startTransaction:
+ self.rollbackTransaction()
+ raise
+
+ self.commitTransaction()
+ self._visitedKeys.clear()
+ self._addVisitedKey()
self.afterSaveAll()
@@ -524,8 +538,21 @@
Cancel all changes made in all rows, including by default all
children
and all new, unmodified records.
"""
- self.scanChangedRows(self.cancel, allCursors=False,
includeNewUnchanged=True,
- cancelTheChildren=cancelTheChildren,
ignoreNoRecords=ignoreNoRecords, reverse=True)
+ # First cancel the rows we know we've visited:
+ self.scanKeys(self.cancel, self._visitedKeys,
+ cancelTheChildren=cancelTheChildren,
+ ignoreNoRecords=ignoreNoRecords,
scanRequeryChildren=False)
+ # Finally, scan all rows only if there are still potentially
changed rows.
+ # The isAnyChanged() call will be expensive if there are
changes buried
+ # in some out-of-context child cursor, but that should be rare.
In the
+ # common case, all the cancellations would have already
happened in the
+ # above block, and isAnyChanged() will return False very
quickly.
+ if self.isAnyChanged():
+ self.scanChangedRows(self.cancel, allCursors=False,
+ includeNewUnchanged=True,
cancelTheChildren=cancelTheChildren,
+ ignoreNoRecords=ignoreNoRecords,
reverse=True)
+ self._visitedKeys.clear()
+ self._addVisitedKey()
def cancel(self, ignoreNoRecords=None, cancelTheChildren=True):
@@ -624,7 +651,7 @@
self.requeryAllChildren()
if not inLoop:
- self.afterPointerMove()
+ self._afterPointerMove()
self.afterChange()
self.afterDelete()
except dException.DBQueryException:
@@ -651,7 +678,7 @@
if startTransaction:
self.commitTransaction()
- self.afterPointerMove()
+ self._afterPointerMove()
self.afterChange()
self.afterDelete()
except dException.DBQueryException:
@@ -860,6 +887,40 @@
return ret
+ def scanKeys(self, func, keys, *args, **kwargs):
+ """
+ Iterate over the specified keys (defined in KeyField) and apply
+ the passed function to each.
+
+ If a passed key doesn't exist, it is ignored.
+
+ Set self.exitScan to True to exit the scan on the next
iteration.
+ """
+ # Flag that the function can set to prematurely exit the scan
+ self.exitScan = False
+ keys = set(keys)
+ requeryChildren = kwargs.pop("scanRequeryChildren",
self.ScanRequeryChildren)
+ currentStatus = self.__getCurrentStatus()
+ ret = None
+
+ try:
+ for key in keys:
+ if self.locate(key, self.KeyField):
+ ret = func(*args, **kwargs)
+ if self.exitScan:
+ break
+ except Exception, e:
+ if self._logScanException(e):
+ dabo.log.error(_("Error in scanKeys of %s: %s")
% (self.Name, ustr(e)))
+ if self.ScanRestorePosition:
+ self.__setCurrentStatus(currentStatus)
+ raise
+
+ if self.ScanRestorePosition:
+ self.__setCurrentStatus(currentStatus)
+ return ret
+
+
def scanChangedRows(self, func, allCursors=False,
includeNewUnchanged=False,
*args, **kwargs):
"""
@@ -1048,7 +1109,7 @@
if child.NewRecordOnNewParent:
child.new()
- self.afterPointerMove()
+ self._afterPointerMove()
self.afterNew()
@@ -1115,6 +1176,7 @@
uiException = dException.NoRecordsException
except dException.dException:
raise
+ self._visitedKeys.clear()
if self.RestorePositionOnRequery:
self._positionUsingPK(currPK,
updateChildren=False)
if hash(self.DataStructure) != oldDataStructure:
@@ -1125,6 +1187,7 @@
except dException.NoRecordsException:
pass
self.afterRequery()
+ self._addVisitedKey()
if uiException:
raise uiException
@@ -1489,7 +1552,7 @@
if ret:
if movePointer and runRequery:
self.requeryAllChildren()
- self.afterPointerMove()
+ self._afterPointerMove()
return ret
@@ -1524,7 +1587,7 @@
if ret != -1:
if runRequery:
self.requeryAllChildren()
- self.afterPointerMove()
+ self._afterPointerMove()
return ret
@@ -2383,6 +2446,21 @@
########## END - SQL Builder interface section ##############
+ def _afterPointerMove(self):
+ self._addVisitedKey()
+ self.afterPointerMove()
+
+
+ def _addVisitedKey(self):
+ """
+ The _visitedKeys set is used for optimization of cancelAll()
+ and saveAll(), and only applies to bizobjs with no parent.
+ """
+ if not self.Parent:
+ self._visitedKeys.add(self.getPK())
+ #print self, len(self._visitedKeys)
+
+
def _makeHookMethod(name, action, mainDoc=None, additionalDoc=None):
mode = name[:5]
if mode == "befor":
@@ -2889,7 +2967,7 @@
if errMsg:
raise dException.BusinessRuleViolation(errMsg)
self._moveToRowNum(rownum, self.RequeryChildrenOnNavigate)
- self.afterPointerMove()
+ self._afterPointerMove()
self.afterSetRowNumber()
_______________________________________________
Post Messages to: [email protected]
Subscription Maintenance: http://leafe.com/mailman/listinfo/dabo-dev
Searchable Archives: http://leafe.com/archives/search/dabo-dev
This message:
http://leafe.com/archives/byMID/[email protected]
