Author: dmeyer
Date: Thu Jun 28 19:31:58 2007
New Revision: 2745
Modified:
trunk/WIP/beacon2/src/db.py
trunk/WIP/beacon2/src/server/crawl.py
trunk/WIP/beacon2/src/server/hwmon/client.py
trunk/WIP/beacon2/src/server/parser.py
trunk/WIP/beacon2/src/server/server.py
Log:
do database changes when we need them
Modified: trunk/WIP/beacon2/src/db.py
==============================================================================
--- trunk/WIP/beacon2/src/db.py (original)
+++ trunk/WIP/beacon2/src/db.py Thu Jun 28 19:31:58 2007
@@ -232,6 +232,10 @@
pos = -1
+ # FIXME: ACTIVE WAITING:
+ while self.read_lock:
+ yield kaa.notifier.YieldContinue
+
for pos, (f, fullname, overlay, stat_res) in enumerate(listing[0]):
isdir = stat.S_ISDIR(stat_res[stat.ST_MODE])
if pos == len(items):
@@ -250,8 +254,7 @@
# no client == server == write access
# delete from database by adding it to the internal changes
# list. It will be deleted right before the next commit.
- self.changes.append(('delete', i, {}, None))
- # delete
+ self.delete_object(i)
if pos < len(items) and f == items[pos]._beacon_name:
# same file
continue
@@ -267,13 +270,9 @@
if not self.client:
# no client == server == write access
for i in items[pos+1-len(items):]:
- self.changes.append(('delete', i, {}, None))
+ self.delete_object(i)
items = items[:pos+1-len(items)]
- if self.changes:
- # need commit because some items were deleted from the db
- self.commit()
-
# no need to sort the items again, they are already sorted based
# on name, let us keep it that way. And name is unique in a directory.
# items.sort(lambda x,y: cmp(x.url, y.url))
@@ -474,37 +473,7 @@
t1 = time.time()
# set internal variables
changes = self.changes
- changed_id = []
self.changes = []
- callbacks = []
-
- # NOTE: Database will be locked now
-
- # walk through the list of changes
- for function, arg1, kwargs, callback in changes:
- if function == 'delete':
- # delete items and all subitems from the db. The delete
function
- # will return all ids deleted, callbacks are not allowed, so
- # we can just continue
- changed_id.extend(self._delete(arg1))
- continue
- if function == 'update':
- try:
- self._db.update_object(arg1, **kwargs)
- changed_id.append(arg1)
- except Exception, e:
- log.error('%s not in the db: %s: %s' % (arg1, e, kwargs))
- continue
- if function == 'add':
- # arg1 is the type, kwargs should contain parent and name, the
- # result is the return of a query, so it has (type, id)
- result = self._db.add_object(arg1, **kwargs)
- changed_id.append((result['type'], result['id']))
- if callback:
- callbacks.append((callback, result))
- continue
- # programming error, this should never happen
- log.error('unknown change <%s>' % function)
# db commit
t2 = time.time()
@@ -516,59 +485,37 @@
# some time debugging
log.info('db.commit %d items; %.5fs (kaa.db commit %.5f / %.2f%%)' % \
(len(changes), t3-t1, t3-t2, (t3-t2)/(t3-t1)*100.0))
- # now call all callbacks
- for callback, result in callbacks:
- callback(result)
# fire db changed signal
- self.signals['changed'].emit(changed_id)
+ self.signals['changed'].emit(changes)
- def get_object(self, name, parent):
+ def add_object(self, type, metadata=None, **kwargs):
"""
- Get the object with the given type, name and parent. This function will
- look at the pending commits and also in the database.
+ Add an object to the db.
"""
- for func, type, kwargs, callback in self.changes:
- if func == 'add' and 'name' in kwargs and kwargs['name'] == name \
- and 'parent' in kwargs and kwargs['parent'] == parent:
- self.commit()
- break
- result = self._db.query(name=name, parent=parent)
- if result:
- return result[0]
- return None
+ if self.read_lock:
+ raise IOError('database is locked')
-
- def add_object(self, type, metadata=None, beacon_immediately=False,
- callback=None, **kwargs):
- """
- Add an object to the db. If the keyword 'beacon_immediately' is set,
- the object will be added now and the db will be locked until the next
- commit. To avoid locking, do not se the keyword, but this means that a
- requery on the object won't find it before the next commit.
- """
if metadata:
for key in self._db._object_types[type][1].keys():
if metadata.has_key(key) and metadata[key] != None and \
not key in kwargs:
kwargs[key] = metadata[key]
- if beacon_immediately:
- self.commit()
- return self._db.add_object(type, **kwargs)
- self.changes.append(('add', type, kwargs, callback))
+ result = self._db.add_object(type, **kwargs)
+ self.changes.append((result['type'], result['id']))
if len(self.changes) > MAX_BUFFER_CHANGES:
self.commit()
+ return result
- def update_object(self, (type, id), metadata=None,
- beacon_immediately=False, **kwargs):
+ def update_object(self, (type, id), metadata=None, **kwargs):
"""
- Update an object to the db. If the keyword 'beacon_immediately' is set,
- the object will be updated now and the db will be locked until the next
- commit. To avoid locking, do not se the keyword, but this means that a
- requery on the object will return the old values.
+ Update an object to the db.
"""
+ if self.read_lock:
+ raise IOError('database is locked')
+
if metadata:
for key in self._db._object_types[type][1].keys():
if metadata.has_key(key) and metadata[key] != None and \
@@ -579,12 +526,16 @@
del kwargs['media']
if isinstance(kwargs.get('media'), str):
raise SystemError
- self.changes.append(('update', (type, id), kwargs, None))
- if len(self.changes) > MAX_BUFFER_CHANGES or beacon_immediately:
+ self._db.update_object((type, id), **kwargs)
+ self.changes.append((type, id))
+ if len(self.changes) > MAX_BUFFER_CHANGES:
self.commit()
def update_object_type(self, (type, id), new_type):
+ if self.read_lock:
+ raise IOError('database is locked')
+
old_entry = self._db.query(type=type, id=id)
if not old_entry:
# already changed by something
@@ -616,9 +567,12 @@
return metadata
- def delete_object(self, item_or_type_id_list, beacon_immediately=False):
- self.changes.append(('delete', item_or_type_id_list, {}, None))
- if len(self.changes) > MAX_BUFFER_CHANGES or beacon_immediately:
+ def delete_object(self, item_or_type_id_list):
+ if self.read_lock:
+ raise IOError('database is locked')
+ for id in self._delete(item_or_type_id_list):
+ self.changes.append(id)
+ if len(self.changes) > MAX_BUFFER_CHANGES:
self.commit()
Modified: trunk/WIP/beacon2/src/server/crawl.py
==============================================================================
--- trunk/WIP/beacon2/src/server/crawl.py (original)
+++ trunk/WIP/beacon2/src/server/crawl.py Thu Jun 28 19:31:58 2007
@@ -226,6 +226,11 @@
log.info('inotify: move to hidden file, delete')
self._inotify_event(INotify.DELETE, name)
yield True
+
+ # FIXME: ACTIVE WAITING:
+ while self.db.read_lock:
+ yield kaa.notifier.YieldContinue
+
if move._beacon_id:
# New item already in the db, delete it first
log.info('inotify delete: %s', item)
@@ -308,11 +313,12 @@
yield True
# The file does not exist, we need to delete it in the database
- if self.db.get_object(item._beacon_data['name'],
- item._beacon_parent._beacon_id):
- # Still in the db, delete it
- log.info('inotify: delete %s', item)
- self.db.delete_object(item, beacon_immediately=True)
+
+ # FIXME: ACTIVE WAITING:
+ while self.db.read_lock:
+ yield kaa.notifier.YieldContinue
+ log.info('inotify: delete %s', item)
+ self.db.delete_object(item)
# remove directory and all subdirs from the inotify. The directory
# is gone, so all subdirs are invalid, too.
@@ -567,6 +573,10 @@
# scans can remove it if it differs.
data['image_from_items'] = True
+ # FIXME: ACTIVE WAITING:
+ while self.db.read_lock:
+ yield kaa.notifier.YieldContinue
+
# update directory in database
self.db.update_object(directory._beacon_id, **data)
directory._beacon_data.update(data)
Modified: trunk/WIP/beacon2/src/server/hwmon/client.py
==============================================================================
--- trunk/WIP/beacon2/src/server/hwmon/client.py (original)
+++ trunk/WIP/beacon2/src/server/hwmon/client.py Thu Jun 28 19:31:58 2007
@@ -137,8 +137,13 @@
self._device_add(dev)
+ @kaa.notifier.yield_execution(lock=True)
def _device_scanned(self, metadata, dev):
+ # FIXME: ACTIVE WAITING:
+ while self.db.read_lock:
+ yield kaa.notifier.YieldContinue
+
# FIXME: check if the device is still valid
# FIXME: handle failed dvd detection
id = dev.get('beacon.id')
@@ -147,15 +152,13 @@
# pass rom drive
type = metadata['mime'][6:]
log.info('detect %s as %s' % (id, type))
- mid = self.db.add_object("media", name=id, content=type,
- beacon_immediately=True)['id']
+ mid = self.db.add_object("media", name=id, content=type)['id']
# FIXME: better label
vid = self.db.add_object("video",
name="",
parent=('media', mid),
title=unicode(get_title(metadata['label'])),
- media = mid,
- beacon_immediately=True)['id']
+ media = mid)['id']
self.db.commit(force=True)
for track in metadata.tracks:
self.db.add_object('track_%s' % type, name='%02d' %
track.trackno,
@@ -167,16 +170,14 @@
elif dev.get('volume.disc.has_audio') and metadata:
# Audio CD
log.info('detect %s as audio cd' % id)
- mid = self.db.add_object("media", name=id, content='cdda',
- beacon_immediately=True)['id']
+ mid = self.db.add_object("media", name=id, content='cdda')['id']
# FIXME: better label
aid = self.db.add_object("audio",
name='',
title = metadata.get('title'),
artist = metadata.get('artist'),
parent=('media', mid),
- media = mid,
- beacon_immediately=True)['id']
+ media = mid)['id']
self.db.commit(force=True)
for track in metadata.tracks:
self.db.add_object('track_cdda', name=str(track.trackno),
@@ -190,15 +191,13 @@
else:
log.info('detect %s as normal filesystem' % id)
- mid = self.db.add_object("media", name=id, content='file',
- beacon_immediately=True)['id']
+ mid = self.db.add_object("media", name=id, content='file')['id']
mtime = 0 # FIXME: wrong for /
if dev.get('block.device'):
mtime = os.stat(dev.get('block.device'))[stat.ST_MTIME]
dir = self.db.add_object("dir",
name="",
parent=('media', mid),
- media=mid, mtime=mtime,
- beacon_immediately=True)
+ media=mid, mtime=mtime)
self.db.commit(force=True)
self._device_add(dev)
Modified: trunk/WIP/beacon2/src/server/parser.py
==============================================================================
--- trunk/WIP/beacon2/src/server/parser.py (original)
+++ trunk/WIP/beacon2/src/server/parser.py Thu Jun 28 19:31:58 2007
@@ -125,6 +125,10 @@
@kaa.notifier.yield_execution()
def _parse(db, item, mtime, store):
+ # FIXME: ACTIVE WAITING:
+ while db.read_lock:
+ yield kaa.notifier.YieldContinue
+
parent = item._beacon_parent
if not parent._beacon_id:
# There is a parent without id, update the parent now. We know that the
@@ -139,15 +143,6 @@
# This should never happen
raise AttributeError('parent for %s has no dbid' % item)
- if not item._beacon_id:
- # New file, maybe already added? Do a small check to be sure we don't
- # add the same item to the db again.
- data = db.get_object(item._beacon_data['name'], parent._beacon_id)
- if data:
- item._beacon_database_update(data)
- if item._beacon_data.get('mtime') == mtime:
- yield 0
-
t1 = time.time()
# FIXME: add force parameter from config file:
@@ -177,8 +172,7 @@
# The item changed its type. Adjust the db
data = db.update_object_type(item._beacon_id, type)
if not data:
- log.warning('item to change not in the db anymore, try to find it')
- data = db.get_object(item._beacon_data['name'], parent._beacon_id)
+ log.error('item to change not in the db anymore')
log.info('change item %s to %s' % (item._beacon_id, type))
item._beacon_database_update(data)
@@ -284,12 +278,12 @@
# Create. Maybe the object is already in the db. This could happen
# because of bad timing but should not matter. Only one entry will be
# there after the next update
- db.add_object(type, name=item._beacon_data['name'],
- parent=parent._beacon_id,
- overlay=item._beacon_overlay,
- callback=item._beacon_database_update,
- media=item._beacon_media._beacon_id[1],
- **attributes)
+ r = db.add_object(type, name=item._beacon_data['name'],
+ parent=parent._beacon_id,
+ overlay=item._beacon_overlay,
+ media=item._beacon_media._beacon_id[1],
+ **attributes)
+ item._beacon_database_update(r)
if hasattr(metadata, 'tracks'):
# The item has tracks, e.g. a dvd image on hd. Sync with the database
Modified: trunk/WIP/beacon2/src/server/server.py
==============================================================================
--- trunk/WIP/beacon2/src/server/server.py (original)
+++ trunk/WIP/beacon2/src/server/server.py Thu Jun 28 19:31:58 2007
@@ -331,10 +331,14 @@
@kaa.rpc.expose('item.update')
+ @kaa.notifier.yield_execution()
def update(self, items):
"""
Update items from the client.
"""
+ # FIXME: ACTIVE WAITING:
+ while self._db.read_lock:
+ yield kaa.notifier.YieldContinue
for dbid, attributes in items:
self._db.update_object(dbid, **attributes)
self._db.commit()
-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take
control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/
_______________________________________________
Freevo-cvslog mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/freevo-cvslog
