On Fri, May 23, 2008 at 10:19 AM, Tomeu Vizoso <[EMAIL PROTECTED]> wrote:
> On Fri, May 23, 2008 at 6:27 AM, Martin Langhoff
> <[EMAIL PROTECTED]> wrote:
>> On Fri, May 23, 2008 at 3:25 PM, Martin Langhoff
>> <[EMAIL PROTECTED]> wrote:
>>> I'll diagnose a bit more and post back.
>>
>> I'm baffled. I initially suspected PEBKAC but it keeps happening. With
>> the patch the DS gets renamed, and journal never starts. If the
>> datastore code gets to complain somewhere I haven't been able to find
>> it.
>>
>> If I back the patch out, and rename the datastore back to "datastore",
>> I get a new, empty datastore on reboot. (Data files are there,
>> though).
>>
>> Nothing in the patch looks suspicious so I am a bit lost. Might still
>> be a problem with something silly. I am testing this on a 703 image
>> where I moved /usr/lib/python2.5/site-packages/olpc/datastore aside,
>> and replaced it with a symlink to my git checkout. The git checkout
>> was identical to the files from the 703 image, and I applied your
>> patch.
>>
>> I've now merged in your latest commit, but it's just a version
>> identifier change, so bah.
>
> Ouch, sorry, I should have been more explicit in that this patch
> introduces a dependency on cjson. I'm going to send next another patch
> that falls back to simplejson (slower) if cjson is not available.
Attached, I'm not sure this is the right approach for json.
When we care about performance and not about interchanging complex
data with other platforms, may be better to do our text-based format
for storing the metadata.
So we should perhaps drop cjson in this patch, and move to a custom
format when the DS starts reading the metadata for those files.
If you have any other problem with the DS:
- check the shell.log file in ~/.sugar/default/logs
- check datastore.log
- start the DS manually by 'datastore-service -m'
Regards,
Tomeu
From 0b2511fe91e0b87896f875c9d7e81ac63d33e896 Mon Sep 17 00:00:00 2001
From: Tomeu Vizoso <[EMAIL PROTECTED]>
Date: Fri, 23 May 2008 10:28:37 +0200
Subject: [PATCH] Maintain a metadata copy outside the index.
---
src/olpc/datastore/backingstore.py | 73 ++++++++++++++++++++++++++++++++++--
src/olpc/datastore/datastore.py | 22 +----------
2 files changed, 71 insertions(+), 24 deletions(-)
diff --git a/src/olpc/datastore/backingstore.py b/src/olpc/datastore/backingstore.py
index fc3c05f..1c5c9f9 100644
--- a/src/olpc/datastore/backingstore.py
+++ b/src/olpc/datastore/backingstore.py
@@ -29,6 +29,13 @@ import dbus
import xapian
import gobject
+try:
+ import cjson
+ has_cjson = True
+except ImportError:
+ import simplejson
+ has_cjson = False
+
from olpc.datastore.xapianindex import IndexManager
from olpc.datastore import bin_copy
from olpc.datastore import utils
@@ -215,7 +222,11 @@ class FileBackingStore(BackingStore):
instead of a method parameter because this is less invasive for Update 1.
"""
self.current_user_id = None
-
+
+ # source for an idle callback that exports to the file system the
+ # metadata from the index
+ self._export_metadata_source = None
+
# Informational
def descriptor(self):
"""return a dict with atleast the following keys
@@ -327,7 +338,29 @@ class FileBackingStore(BackingStore):
im.connect(index_name)
self.indexmanager = im
-
+
+ # Check that all entries have their metadata in the file system.
+ if not os.path.exists(os.path.join(self.base, '.metadata.exported')):
+ uids_to_export = []
+ uids = self.indexmanager.get_all_ids()
+
+ t = time.time()
+ for uid in uids:
+ if not os.path.exists(os.path.join(self.base, uid + '.metadata')):
+ uids_to_export.append(uid)
+
+ if uids_to_export:
+ self._export_metadata_source = gobject.idle_add(
+ self._export_metadata, uids_to_export)
+ else:
+ open(os.path.join(self.base, '.metadata.exported'), 'w').close()
+
+ def _export_metadata(self, uids_to_export):
+ uid = uids_to_export.pop()
+ props = self.indexmanager.get(uid).properties
+ self._store_metadata(uid, props)
+ return len(uids_to_export) > 0
+
def bind_to(self, datastore):
## signal from datastore that we are being bound to it
self.datastore = datastore
@@ -502,6 +535,29 @@ class FileBackingStore(BackingStore):
return c.hexdigest()
# File Management API
+ def _encode_json(self, metadata, file_path):
+ if has_cjson:
+ f = open(file_path, 'w')
+ f.write(cjson.encode(metadata))
+ f.close()
+ else:
+ simplejson.dump(metadata, open(file_path, 'w'))
+
+ def _store_metadata(self, uid, props):
+ t = time.time()
+ path = os.path.join(self.base, uid + '.metadata')
+ props = props.copy()
+ for property_name in model.defaultModel.get_external_properties():
+ if property_name in props:
+ del props[property_name]
+ self._encode_json(props, path)
+ logging.debug('exported metadata: %r s.' % (time.time() - t))
+
+ def _delete_metadata(self, uid, props):
+ path = os.path.join(self.base, uid + '.metadata')
+ if os.path.exists(path):
+ os.unlink(path)
+
def _create_completion(self, uid, props, completion, exc=None, path=None):
if exc:
completion(exc)
@@ -517,6 +573,7 @@ class FileBackingStore(BackingStore):
if completion is None:
raise RuntimeError("Completion must be valid for async create")
uid = self.indexmanager.index(props)
+ self._store_metadata(uid, props)
props['uid'] = uid
if filelike:
if isinstance(filelike, basestring):
@@ -531,6 +588,7 @@ class FileBackingStore(BackingStore):
def create(self, props, filelike, can_move=False):
if filelike:
uid = self.indexmanager.index(props)
+ self._store_metadata(uid, props)
props['uid'] = uid
if isinstance(filelike, basestring):
# lets treat it as a filename
@@ -540,7 +598,9 @@ class FileBackingStore(BackingStore):
self.indexmanager.index(props, path)
return uid
else:
- return self.indexmanager.index(props)
+ uid = self.indexmanager.index(props)
+ self._store_metadata(uid, props)
+ return uid
def get(self, uid, env=None, allowMissing=False, includeFile=False):
content = self.indexmanager.get(uid)
@@ -575,6 +635,7 @@ class FileBackingStore(BackingStore):
raise RuntimeError("Completion must be valid for async update")
props['uid'] = uid
+ self._store_metadata(uid, props)
if filelike:
uid = self.indexmanager.index(props, filelike)
props['uid'] = uid
@@ -590,6 +651,7 @@ class FileBackingStore(BackingStore):
def update(self, uid, props, filelike=None, can_move=False):
props['uid'] = uid
+ self._store_metadata(uid, props)
if filelike:
if isinstance(filelike, basestring):
# lets treat it as a filename
@@ -610,6 +672,7 @@ class FileBackingStore(BackingStore):
def delete(self, uid, allowMissing=True):
self._delete_external_properties(uid)
+ self._delete_metadata(uid, props)
self.indexmanager.delete(uid)
path = self._translatePath(uid)
@@ -617,7 +680,7 @@ class FileBackingStore(BackingStore):
os.unlink(path)
else:
if not allowMissing:
- raise KeyError("object for uid:%s missing" % uid)
+ raise KeyError("object for uid:%s missing" % uid)
def get_uniquevaluesfor(self, propertyname):
return self.indexmanager.get_uniquevaluesfor(propertyname)
@@ -651,6 +714,8 @@ class FileBackingStore(BackingStore):
return self.indexmanager.get_all_ids()
def stop(self):
+ if self._export_metadata_source is not None:
+ gobject.source_remove(self._export_metadata_source)
self.indexmanager.stop()
def complete_indexing(self):
diff --git a/src/olpc/datastore/datastore.py b/src/olpc/datastore/datastore.py
index 67ddca9..a15d5cf 100644
--- a/src/olpc/datastore/datastore.py
+++ b/src/olpc/datastore/datastore.py
@@ -128,28 +128,10 @@ class DataStore(dbus.service.Object):
### Backup support
def pause(self, mountpoints=None):
- """pause the datastore, during this time it will not process
- requests. this allows the underlying stores to be backup up via
- traditional mechanisms
- """
- if mountpoints:
- mps = [self.mountpoints[mp] for mp in mountpoints]
- else:
- mps = self.mountpoints.values()
-
- for mp in mps:
- mp.stop()
+ """ Deprecated. """
def unpause(self, mountpoints=None):
- """resume the operation of a set of paused mountpoints"""
- if mountpoints:
- mps = [self.mountpoints[mp] for mp in mountpoints]
- else:
- mps = self.mountpoints.values()
-
- for mp in mps:
- mp.initialize_and_load()
-
+ """ Deprecated. """
### End Backups
def connect_backingstore(self, uri, **kwargs):
--
1.5.2.5
_______________________________________________
Devel mailing list
[email protected]
http://lists.laptop.org/listinfo/devel
