Author: dmeyer
Date: Sat Sep 15 17:18:32 2007
New Revision: 2813
Log:
Move the feed code to kaa.beacon and integrate it into the
server. This is still experimental. One bug showed up when
a client locked the db and quit before unlocking it. This
should be fixed now but something is still not right.
Added:
trunk/beacon/src/server/channel/
trunk/beacon/src/server/channel/__init__.py
- copied, changed from r2798, /trunk/WIP/netsearch/src/feed/__init__.py
trunk/beacon/src/server/channel/channel.py
- copied, changed from r2809, /trunk/WIP/netsearch/src/feed/channel.py
trunk/beacon/src/server/channel/feedparser.py
- copied unchanged from r2794,
/trunk/WIP/netsearch/src/feed/lib/feedparser.py
trunk/beacon/src/server/channel/manager.py
- copied, changed from r2809, /trunk/WIP/netsearch/src/feed/manager.py
trunk/beacon/src/server/channel/rss.py
- copied, changed from r2809, /trunk/WIP/netsearch/src/feed/plugins/rss.py
trunk/beacon/test/feeds.py
Removed:
trunk/WIP/netsearch/src/feed/__init__.py
trunk/WIP/netsearch/src/feed/channel.py
trunk/WIP/netsearch/src/feed/lib/feedparser.py
trunk/WIP/netsearch/src/feed/manager.py
trunk/WIP/netsearch/src/feed/plugins/rss.py
Modified:
trunk/beacon/src/__init__.py
trunk/beacon/src/client.py
trunk/beacon/src/server/db.py
trunk/beacon/src/server/server.py
Modified: trunk/beacon/src/__init__.py
==============================================================================
--- trunk/beacon/src/__init__.py (original)
+++ trunk/beacon/src/__init__.py Sat Sep 15 17:18:32 2007
@@ -194,3 +194,39 @@
while not _client.is_connected():
kaa.notifier.step()
return _client.delete_media(id)
+
+
+class Feed(dict):
+ def update(self):
+ return _client.feed_func('channels.update', self.get('id'))
+
+ def remove(self):
+ return _client.feed_func('channels.remove', self.get('id'))
+
[EMAIL PROTECTED]()
+def list_feeds():
+ """
+ List all feeds
+ """
+ if not _client:
+ connect()
+
+ async = _client.feed_func('channels.list')
+ yield async
+ feeds = []
+ for f in async.get_result():
+ feeds.append(Feed(f))
+ yield feeds
+
+
[EMAIL PROTECTED]()
+def add_feed(url, destdir, download=True, num=0, keep=True):
+ """
+ Add a feed
+ """
+ if not _client:
+ connect()
+ async = _client.feed_func('channels.add', url, destdir, download, num,
keep)
+ yield async
+ yield Feed(async.get_result())
+
Modified: trunk/beacon/src/client.py
==============================================================================
--- trunk/beacon/src/client.py (original)
+++ trunk/beacon/src/client.py Sat Sep 15 17:18:32 2007
@@ -188,6 +188,14 @@
self.rpc('db.media.delete', id)
+ def feed_func(self, func, *args, **kwargs):
+ """
+ Call feed manager function in the server
+ """
+ if self.status != DISCONNECTED:
+ return self.rpc(func, *args, **kwargs)
+
+
# -------------------------------------------------------------------------
# Server connect / disconnect / reconnect handling
# -------------------------------------------------------------------------
Copied: trunk/beacon/src/server/channel/__init__.py (from r2798,
/trunk/WIP/netsearch/src/feed/__init__.py)
==============================================================================
--- /trunk/WIP/netsearch/src/feed/__init__.py (original)
+++ trunk/beacon/src/server/channel/__init__.py Sat Sep 15 17:18:32 2007
@@ -19,11 +19,40 @@
#
# ##################################################################
+import kaa.rpc
import manager
-import plugins
+import channel
+import rss
-add_channel = manager.add_channel
-list_channels = manager.list_channels
-remove_channel = manager.remove_channel
-update = manager.update
[EMAIL PROTECTED]('channels.update')
+def update(id=None):
+ if id == None:
+ return manager.update()
+ for c in manager.list_channels():
+ if id == c.id:
+ return c.update()
+ return False
+
[EMAIL PROTECTED]('channels.list')
+def list_channels():
+ channels = []
+ for c in manager.list_channels():
+ channels.append(c.get_config())
+ return channels
+
[EMAIL PROTECTED]('channels.add')
+def add_channel(url, destdir, download=True, num=0, keep=True):
+ return manager.add_channel(url, destdir, download, num, keep).get_config()
+
[EMAIL PROTECTED]('channels.remove')
+def remove_channel(id):
+ for c in manager.list_channels():
+ if id == c.id:
+ manager.remove_channel(c)
+ return True
+ return False
+
+def set_database(database):
+ channel.Channel._db = database
+ manager.init()
Copied: trunk/beacon/src/server/channel/channel.py (from r2809,
/trunk/WIP/netsearch/src/feed/channel.py)
==============================================================================
--- /trunk/WIP/netsearch/src/feed/channel.py (original)
+++ trunk/beacon/src/server/channel/channel.py Sat Sep 15 17:18:32 2007
@@ -10,7 +10,6 @@
# kaa imports
import kaa.notifier
import kaa.notifier.url
-import kaa.beacon
from kaa.strutils import str_to_unicode, unicode_to_str
# get manager module
@@ -28,26 +27,29 @@
if not os.path.isdir(IMAGEDIR):
os.makedirs(IMAGEDIR)
-class Channel(object):
+class Entry(dict):
- class Entry(dict):
+ def __getattr__(self, attr):
+ if attr == 'basename' and not 'basename' in self.keys():
+ basename = os.path.basename(self['url'])
+ if self.url.endswith('/'):
+ ext = os.path.splitext(self['url'])[1]
+ basename = self['title'].replace('/', '') + ext
+ self['basename'] = unicode_to_str(basename)
+ return self.get(attr)
+
+ def fetch(self, filename):
+ log.info('%s -> %s' % (self.url, filename))
+ tmpname = os.path.join(os.path.dirname(filename),
+ '.' + os.path.basename(filename))
+ return kaa.notifier.url.fetch(self.url, filename, tmpname)
- def __getattr__(self, attr):
- if attr == 'basename' and not 'basename' in self.keys():
- basename = os.path.basename(self['url'])
- if self.url.endswith('/'):
- ext = os.path.splitext(self['url'])[1]
- basename = self['title'].replace('/', '') + ext
- self['basename'] = unicode_to_str(basename)
- return self.get(attr)
-
- def fetch(self, filename):
- log.info('%s -> %s' % (self.url, filename))
- tmpname = os.path.join(os.path.dirname(filename),
- '.' + os.path.basename(filename))
- return kaa.notifier.url.fetch(self.url, filename, tmpname)
+class Channel(object):
+ _db = None
+ NEXT_ID = 0
+
def __init__(self, url, destdir):
self.url = url
self.dirname = destdir
@@ -58,7 +60,9 @@
self._keep = True
if not os.path.isdir(destdir):
os.makedirs(destdir)
-
+ self.id = Channel.NEXT_ID
+ Channel.NEXT_ID += 1
+
def configure(self, download=True, num=0, keep=True):
"""
@@ -73,6 +77,17 @@
manager.save()
+ def get_config(self):
+ """
+ Get channel configuration.
+ """
+ return dict(id = self.id,
+ url = self.url,
+ directory = self.dirname,
+ download = self._download,
+ num = self._num,
+ keep = self._keep)
+
def _readxml(self, node):
"""
Read XML node with channel configuration and cache.
@@ -106,7 +121,7 @@
d.appendChild(doc.createTextNode(self.dirname))
node.appendChild(d)
for url, fname in self._entries:
- e = node.createElement('entry')
+ e = doc.createElement('entry')
e.setAttribute('url', url)
if fname:
e.setAttribute('filename', str_to_unicode(fname))
@@ -137,9 +152,15 @@
sys.stdout.write("%s\r" % s.get_progressbar())
sys.stdout.flush()
+ log.info('update channel %s', self.url)
+
# get directory information
- beacondir = kaa.beacon.get(self.dirname)
- allurls = [ f.url for f in beacondir.list() ]
+ beacondir = self._db.query(filename=self.dirname)
+ listing = beacondir.list()
+ if isinstance(listing, kaa.notifier.InProgress):
+ yield listing
+ listing = listing.get_result()
+ allurls = [ f.url for f in listing ]
num = self._num
allfiles = [ e[1] for e in self._entries ]
@@ -152,6 +173,7 @@
yield entry
continue
+ log.info('process %s', entry.url)
filename = None
if not self._download and entry.url in allurls:
@@ -159,7 +181,11 @@
pass
elif not self._download:
# add to beacon
- i = kaa.beacon.add_item(parent=beacondir, **entry)
+ while self._db.read_lock.is_locked():
+ yield self._db.read_lock.yield_unlock()
+ # use url as name
+ entry['name'] = unicode_to_str(entry.url)
+ i = self._db.add_object(parent=beacondir, **entry)
else:
# download
filename = os.path.join(self.dirname, entry.basename)
@@ -181,21 +207,21 @@
continue
if os.path.isfile(filename):
- item = kaa.beacon.get(filename)
+ item = self._db.query(filename=filename)
if not item.scanned():
- # BEACON_FIXME
- item._beacon_request()
- while not item.scanned():
- yield kaa.notifier.YieldContinue
+ async = item.scan()
+ if isinstance(async, kaa.notifier.InProgress):
+ yield async
for key, value in entry.items():
if not key in ('type', 'url', 'basename'):
item[key] = value
-
+
self._entries.append((entry['url'], filename))
num -= 1
if num == 0:
break
+ log.info('*** finished with %s ***', self.url)
manager.save()
# delete old files or remove old entries from beacon
@@ -210,3 +236,26 @@
elif os.path.isfile(filename):
# delete file on disc
os.unlink(filename)
+
+
+ @kaa.notifier.yield_execution()
+ def remove(self):
+ """
+ Remove entries from this channel.
+ """
+ log.info('remove %s', self.url)
+ if self._keep or self._download:
+ # only delete links in the filesystem
+ return
+
+ # get directory information
+ beacondir = self._db.query(filename=self.dirname)
+ allurls = [ e[0] for e in self._entries ]
+ listing = beacondir.list()
+ if isinstance(listing, kaa.notifier.InProgress):
+ yield listing
+ listing = listing.get_result()
+ for entry in listing:
+ if entry.url in allurls:
+ log.info('delete %s', entry.url)
+ entry.delete()
Copied: trunk/beacon/src/server/channel/manager.py (from r2809,
/trunk/WIP/netsearch/src/feed/manager.py)
==============================================================================
--- /trunk/WIP/netsearch/src/feed/manager.py (original)
+++ trunk/beacon/src/server/channel/manager.py Sat Sep 15 17:18:32 2007
@@ -5,13 +5,14 @@
import kaa.notifier
from kaa.strutils import unicode_to_str
+# fallback RSS parser
+import rss
+
# get logging object
log = logging.getLogger('beacon.channel')
CACHE = os.path.expanduser("~/.beacon/channels.xml")
-_initialized = False
-
# list of all channel objects
_channels = []
@@ -32,29 +33,26 @@
for regexp, generator in _generators:
if regexp.match(url):
return generator(url, destdir)
- raise RuntimeError
+ return rss.Channel(url, destdir)
def add_channel(url, destdir, download=True, num=0, keep=True):
"""
Add a new channel.
"""
- if not _initialized:
- _init()
for c in _channels:
if c.dirname == destdir and c.url == url:
raise RuntimeError('channel already exists')
channel = _get_channel(url, destdir)
_channels.append(channel)
channel.configure(download, num, keep)
+ return channel
def list_channels():
"""
Return a list of all channels.
"""
- if not _initialized:
- _init()
return _channels
@@ -63,6 +61,7 @@
Remove a channel.
"""
_channels.remove(channel)
+ channel.remove()
save()
@@ -70,8 +69,6 @@
"""
Save all channel information
"""
- if not _initialized:
- _init()
doc = minidom.getDOMImplementation().createDocument(None, "channels", None)
top = doc.documentElement
for c in _channels:
@@ -83,7 +80,7 @@
f.close()
-def _init():
+def init():
"""
Load cached channels from disc.
"""
@@ -99,8 +96,6 @@
_channels.append(channel)
return
- global _initialized
- _initialized = True
if not os.path.isfile(CACHE):
return
@@ -130,11 +125,14 @@
"""
global _updating
if _updating:
+ log.error('update already in progress')
yield False
- if not _initialized:
- _init()
+ log.info('start channel update')
_updating = True
for channel in _channels:
- yield channel.update(verbose=verbose)
+ x = channel.update(verbose=verbose)
+ yield x
+ log.info('XXXXXXXXX')
+ x()
_updating = False
yield True
Copied: trunk/beacon/src/server/channel/rss.py (from r2809,
/trunk/WIP/netsearch/src/feed/plugins/rss.py)
==============================================================================
--- /trunk/WIP/netsearch/src/feed/plugins/rss.py (original)
+++ trunk/beacon/src/server/channel/rss.py Sat Sep 15 17:18:32 2007
@@ -1,18 +1,22 @@
import re
import time
import logging
+import urllib2
import kaa.notifier
-from kaa.netsearch.feed.lib import feedparser
-from kaa.netsearch.feed.channel import Channel
-from kaa.netsearch.feed.manager import register
+import feedparser as _feedparser
+import channel
# get logging object
log = logging.getLogger('beacon.feed')
isotime = '%a, %d %b %Y %H:%M:%S'
-class RSS(Channel):
[EMAIL PROTECTED]()
+def feedparser(url):
+ return _feedparser.parse(urllib2.urlopen(url))
+
+class Channel(channel.Channel):
def __iter__(self):
# get feed in a thread
@@ -85,7 +89,5 @@
# getting a good basename for urls ending with /
# based on type.
# create entry
- entry = Channel.Entry(**metadata)
+ entry = channel.Entry(**metadata)
yield entry
-
-register(re.compile('^(http|https|file)://.*'), RSS)
Modified: trunk/beacon/src/server/db.py
==============================================================================
--- trunk/beacon/src/server/db.py (original)
+++ trunk/beacon/src/server/db.py Sat Sep 15 17:18:32 2007
@@ -56,29 +56,34 @@
Read lock for the database.
"""
def __init__(self):
- self._clients = 0
+ self._clients = []
self.signals = kaa.notifier.Signals('lock', 'unlock')
- def lock(self):
+
+ def lock(self, client):
"""
Lock the database for reading.
"""
log.debug('lock++')
- self._clients += 1
- if self._clients == 1:
+ self._clients.append(client)
+ if len(self._clients) == 1:
self.signals['lock'].emit()
log.debug('locked')
- def unlock(self):
+ def unlock(self, client, all=True):
"""
Unlock the database. If more than one lock was made
this will only decrease the lock variable but not
unlock the database.
"""
log.debug('lock--')
- self._clients -= 1
- if self._clients == 0:
+ if client in self._clients:
+ self._clients.remove(client)
+ if all:
+ # remove all locks from client
+ return self.unlock(client, all)
+ if len(self._clients) == 0:
self.signals['unlock'].emit()
log.debug('unlocked')
Modified: trunk/beacon/src/server/server.py
==============================================================================
--- trunk/beacon/src/server/server.py (original)
+++ trunk/beacon/src/server/server.py Sat Sep 15 17:18:32 2007
@@ -50,6 +50,7 @@
from monitor import Monitor
from crawl import Crawler
from config import config
+import channel
# get logging object
log = logging.getLogger('beacon.server')
@@ -162,7 +163,9 @@
for dir in config.monitors:
self.monitor_dir(os.path.expandvars(os.path.expanduser(dir)))
-
+ channel.set_database(self._db)
+ self.ipc.connect(channel)
+
# -------------------------------------------------------------
# client handling
@@ -191,6 +194,8 @@
for m in client_info[2]:
m.stop()
self._clients.remove(client_info)
+ self._db.read_lock.unlock(client_info[1], True)
+
def has_clients(self):
"""
@@ -258,21 +263,20 @@
self._db.delete_media(id)
- @kaa.rpc.expose('db.lock')
- def db_lock(self):
+ @kaa.rpc.expose('db.lock', add_client=True)
+ def db_lock(self, client_id):
"""
Lock the database so clients can read
- FIXME: if a client locks and dies before unlock the server is dead
"""
- self._db.read_lock.lock()
+ self._db.read_lock.lock(client_id)
- @kaa.rpc.expose('db.unlock')
- def db_unlock(self):
+ @kaa.rpc.expose('db.unlock', add_client=True)
+ def db_unlock(self, client_id):
"""
Unlock the database again
"""
- self._db.read_lock.unlock()
+ self._db.read_lock.unlock(client_id)
@kaa.rpc.expose('monitor.directory')
Added: trunk/beacon/test/feeds.py
==============================================================================
--- (empty file)
+++ trunk/beacon/test/feeds.py Sat Sep 15 17:18:32 2007
@@ -0,0 +1,59 @@
+import sys
+import kaa.notifier
+import kaa.beacon
+
+kaa.beacon.connect()
+
+if len(sys.argv) > 1:
+ if sys.argv[1] in ('--update', '-u'):
+ if len(sys.argv) > 2:
+ def update(feeds, id):
+ for f in feeds:
+ if f.get('id') == id:
+ f.update().connect(sys.exit)
+ kaa.beacon.list_feeds().connect(update, int(sys.argv[2]))
+ else:
+ kaa.beacon.update_feeds().connect(sys.exit)
+
+ elif sys.argv[1] in ('--list', '-l'):
+ def show(feeds):
+ for f in feeds:
+ print f
+ sys.exit(0)
+ kaa.beacon.list_feeds().connect(show)
+
+ elif sys.argv[1] in ('--add', '-a') and len(sys.argv) > 3:
+ url = sys.argv[2]
+ destdir = sys.argv[3]
+ if len(sys.argv) > 4:
+ if sys.argv[4].lower() in ('true', 'yes'):
+ download = True
+ elif sys.argv[4].lower() in ('false', 'no'):
+ download = False
+ num = int(sys.argv[5])
+ if sys.argv[6].lower() in ('true', 'yes'):
+ keep = True
+ elif sys.argv[6].lower() in ('false', 'no'):
+ keep = False
+ kaa.beacon.add_feed(url, destdir, download, num,
keep).connect(sys.exit)
+ else:
+ kaa.beacon.add_feed(url, destdir).connect(sys.exit)
+
+ elif sys.argv[1] in ('--remove', '-r') and len(sys.argv) > 2:
+ def remove(feeds, id):
+ for f in feeds:
+ if f.get('id') == id:
+ f.remove().connect(sys.exit)
+ kaa.beacon.list_feeds().connect(remove, int(sys.argv[2]))
+
+ else:
+ print 'help'
+ sys.exit(0)
+
+ kaa.notifier.loop()
+ sys.exit(0)
+
+print 'help'
+# Add example
+# python test/feeds.py -a http://www.radiobremen.de/podcast/bestof/ \
+# /local/podcast False 2 False
-------------------------------------------------------------------------
This SF.net email is sponsored by: Microsoft
Defy all challenges. Microsoft(R) Visual Studio 2005.
http://clk.atdmt.com/MRT/go/vse0120000070mrt/direct/01/
_______________________________________________
Freevo-cvslog mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/freevo-cvslog
