Author: dmeyer
Date: Thu Jun 28 17:27:29 2007
New Revision: 2738
Modified:
trunk/WIP/beacon2/src/server/crawl.py
trunk/WIP/beacon2/src/server/monitor.py
trunk/WIP/beacon2/src/server/parser.py
trunk/WIP/beacon2/src/server/server.py
Log:
make it possible that parse() is async
Modified: trunk/WIP/beacon2/src/server/crawl.py
==============================================================================
--- trunk/WIP/beacon2/src/server/crawl.py (original)
+++ trunk/WIP/beacon2/src/server/crawl.py Thu Jun 28 17:27:29 2007
@@ -443,8 +443,9 @@
yield []
# parse directory
- if parse(self.db, directory, check_image=self._startup):
- yield kaa.notifier.YieldContinue
+ async = parse(self.db, directory, check_image=self._startup)
+ if isinstance(async, kaa.notifier.InProgress):
+ yield async
# check if it is still a directory
if not directory._beacon_isdir:
@@ -460,8 +461,9 @@
self.monitoring.add(directory.filename, use_inotify=False)
dirname = os.path.realpath(directory.filename)
directory = self.db.query(filename=dirname)
- if parse(self.db, directory, check_image=self._startup):
- yield kaa.notifier.YieldContinue
+ async = parse(self.db, directory, check_image=self._startup)
+ if isinstance(async, kaa.notifier.InProgress):
+ yield async
# add to monitor list using inotify
self.monitoring.add(directory.filename)
@@ -476,7 +478,11 @@
subdirs.append(child)
continue
# check file
- counter += parse(self.db, child, check_image=self._startup) * 20
+ async = parse(self.db, child, check_image=self._startup) * 20
+ if isinstance(async, kaa.notifier.InProgress):
+ yield async
+ async = async()
+ counter += async
while counter >= 20:
counter -= 20
yield kaa.notifier.YieldContinue
Modified: trunk/WIP/beacon2/src/server/monitor.py
==============================================================================
--- trunk/WIP/beacon2/src/server/monitor.py (original)
+++ trunk/WIP/beacon2/src/server/monitor.py Thu Jun 28 17:27:29 2007
@@ -234,8 +234,9 @@
for pos, item in enumerate(changed):
self.notify_client('progress', pos+1, len(changed), item.url)
- parser.parse(self._db, item)
- yield YieldContinue
+ async = parser.parse(self._db, item)
+ if isinstance(async, kaa.notifier.InProgress):
+ yield async
if not self._running:
break
Modified: trunk/WIP/beacon2/src/server/parser.py
==============================================================================
--- trunk/WIP/beacon2/src/server/parser.py (original)
+++ trunk/WIP/beacon2/src/server/parser.py Thu Jun 28 17:27:29 2007
@@ -41,6 +41,7 @@
# kaa imports
from kaa.strutils import str_to_unicode
import kaa.metadata
+import kaa.notifier
import kaa.imlib2
# kaa.beacon imports
@@ -97,24 +98,10 @@
log.warning('no mtime, skip %s' % item)
return 0
- parent = item._beacon_parent
- if not parent:
+ if not item._beacon_parent:
log.warning('no parent, skip %s' % item)
return 0
- if not parent._beacon_id:
- # There is a parent without id, update the parent now. We know that the
- # parent should be in the db, so commit and it should work
- db.commit()
- if not parent._beacon_id:
- # Still not in the database? Well, this should never happen but
does
- # when we use some strange softlinks around the filesystem. So in
- # that case we need to scan the parent, too.
- parse(db, parent, True)
- if not parent._beacon_id:
- # This should never happen
- raise AttributeError('parent for %s has no dbid' % item)
-
if item._beacon_data.get('mtime') == mtime:
# The item already is in the database and the mtime is unchanged.
# This menas we don't need to scan again, but we check if the
@@ -132,6 +119,26 @@
else:
return 0
+ return _parse(db, item, mtime, store)
+
+
[EMAIL PROTECTED]()
+def _parse(db, item, mtime, store):
+
+ parent = item._beacon_parent
+ if not parent._beacon_id:
+ # There is a parent without id, update the parent now. We know that the
+ # parent should be in the db, so commit and it should work
+ db.commit()
+ if not parent._beacon_id:
+ # Still not in the database? Well, this should never happen but
does
+ # when we use some strange softlinks around the filesystem. So in
+ # that case we need to scan the parent, too.
+ parse(db, parent, True)
+ if not parent._beacon_id:
+ # This should never happen
+ raise AttributeError('parent for %s has no dbid' % item)
+
if not item._beacon_id:
# New file, maybe already added? Do a small check to be sure we don't
# add the same item to the db again.
@@ -139,7 +146,7 @@
if data:
item._beacon_database_update(data)
if item._beacon_data.get('mtime') == mtime:
- return 0
+ yield 0
t1 = time.time()
@@ -290,14 +297,14 @@
db.commit()
if not metadata.get('type'):
log.error('%s metadata has no type', item)
- return produced_load
+ yield produced_load
# delete all known tracks before adding new
for track in db.query(parent=item):
db.delete_object(track)
if not 'track_%s' % metadata.get('type').lower() in \
db.object_types().keys():
log.error('track_%s not in database keys',
metadata.get('type').lower())
- return produced_load
+ yield produced_load
type = 'track_%s' % metadata.get('type').lower()
for track in metadata.tracks:
db.add_object(type, name=str(track.trackno),
@@ -310,4 +317,4 @@
db.commit()
log.info('scan %s (%0.3f)' % (item, time.time() - t1))
- return produced_load
+ yield produced_load
Modified: trunk/WIP/beacon2/src/server/server.py
==============================================================================
--- trunk/WIP/beacon2/src/server/server.py (original)
+++ trunk/WIP/beacon2/src/server/server.py Thu Jun 28 17:27:29 2007
@@ -255,6 +255,7 @@
@kaa.rpc.expose('monitor.directory')
+ @kaa.notifier.yield_execution()
def monitor_dir(self, directory):
"""
Monitor a directory in the background. One directories with a monitor
@@ -263,7 +264,7 @@
self._db.commit()
if not os.path.isdir(directory):
log.warning("monitor_dir: %s is not a directory." % directory)
- return False
+ yield False
# TODO: check if directory is already being monitored.
directory = os.path.realpath(directory)
@@ -274,7 +275,9 @@
break
items.append(i)
while items:
- parser.parse(self._db, items.pop(), store=True)
+ async = parser.parse(self._db, items.pop(), store=True)
+ if isinstance(async, kaa.notifier.InProgress):
+ yield async
self._db.commit()
log.info('monitor %s on %s', directory, data._beacon_media)
data._beacon_media.crawler.append(data)
@@ -330,6 +333,7 @@
@kaa.rpc.expose('item.request')
+ @kaa.notifier.yield_execution()
def request(self, filename):
"""
Request item data.
@@ -342,9 +346,11 @@
break
items.append(i)
while items:
- parser.parse(self._db, items.pop(), store=True)
+ async = parser.parse(self._db, items.pop(), store=True)
+ if isinstance(async, kaa.notifier.InProgress):
+ yield async
self._db.commit()
- return data._beacon_data
+ yield data._beacon_data
@kaa.rpc.expose('beacon.shutdown')
-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take
control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/
_______________________________________________
Freevo-cvslog mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/freevo-cvslog
