Preparations for better index management. - open_index() now can place the index on a temporary location (ie: a tmpfs when low on diskspace or at ENOSPC)
- don't set index_updated flag when on a temporary location - exit the process when hitting an error in _flush(). This is normally caused by running into ENOSPC during a session, and is unrecoverable. dbus will respawn the process, the new spawn has a chance at cleanup, moving things to tmpfs. Signed-off-by: Martin Langhoff <mar...@laptop.org> --- This is v2 of the patch, making sure that later sessions rebuild the on-disk index. src/carquinyol/indexstore.py | 57 +++++++++++++++++++++++++++++++++--------- 1 file changed, 45 insertions(+), 12 deletions(-) diff --git a/src/carquinyol/indexstore.py b/src/carquinyol/indexstore.py index dc721e7..0bbd93f 100644 --- a/src/carquinyol/indexstore.py +++ b/src/carquinyol/indexstore.py @@ -235,12 +235,32 @@ class IndexStore(object): self._database = None self._flush_timeout = None self._pending_writes = 0 - self._index_updated_path = os.path.join( - layoutmanager.get_instance().get_root_path(), 'index_updated') - - def open_index(self): - index_path = layoutmanager.get_instance().get_index_path() - self._database = WritableDatabase(index_path, xapian.DB_CREATE_OR_OPEN) + root_path=layoutmanager.get_instance().get_root_path() + self._index_updated_path = os.path.join(root_path, + 'index_updated') + self._std_index_path = layoutmanager.get_instance().get_index_path() + self._index_path = self._std_index_path + + def open_index(self, temp_path=False): + # callers to open_index must be able to + # handle an exception -- usually caused by + # IO errors such as ENOSPC and retry putting + # the index on a temp_path + if temp_path: + try: + # mark the on-disk index stale + self._set_index_updated(False) + except: + pass + self._index_path = temp_path + else: + self._index_path = self._std_index_path + try: + self._database = WritableDatabase(self._index_path, + xapian.DB_CREATE_OR_OPEN) + except Exception as e: + logging.error('Exception opening database') + raise def close_index(self): """Close index database if it is open.""" @@ -248,14 +268,18 @@ class IndexStore(object): return self._flush(True) - self._database = None + try: + # does Xapian write in its destructors? + self._database = None + except Exception as e: + logging.error('Exception tearing down database') + raise def remove_index(self): - index_path = layoutmanager.get_instance().get_index_path() - if not os.path.exists(index_path): + if not os.path.exists(self._index_path): return - for f in os.listdir(index_path): - os.remove(os.path.join(index_path, f)) + for f in os.listdir(self._index_path): + os.remove(os.path.join(self._index_path, f)) def contains(self, uid): postings = self._database.postlist(_PREFIX_FULL_VALUE + \ @@ -347,6 +371,9 @@ class IndexStore(object): index_updated = property(get_index_updated) def _set_index_updated(self, index_updated): + if self._std_index_path != self._index_path: + # operating from tmpfs + return True if index_updated != self.index_updated: if index_updated: index_updated_file = open(self._index_updated_path, 'w') @@ -374,7 +401,13 @@ class IndexStore(object): self._pending_writes += 1 if force or self._pending_writes > _FLUSH_THRESHOLD: - self._database.flush() + try: + self._database.flush() + except Exception, e: + logging.exception(e) + logging.error("Exception during database.flush()") + # bail out to trigger a reindex + sys.exit(1) self._pending_writes = 0 self._set_index_updated(True) else: -- 1.7.10.4 _______________________________________________ Sugar-devel mailing list Sugar-devel@lists.sugarlabs.org http://lists.sugarlabs.org/listinfo/sugar-devel