Author: dmeyer
Date: Sat Feb 18 12:48:50 2006
New Revision: 1191
Removed:
trunk/WIP/vfs/test/dump.py
trunk/WIP/vfs/test/mediadb.py
trunk/WIP/vfs/test/scan.py
trunk/WIP/vfs/test/server.py
trunk/WIP/vfs/test/testdb.py
trunk/WIP/vfs/test/testdb2.py
Modified:
trunk/WIP/vfs/src/__init__.py
trunk/WIP/vfs/src/db.py
trunk/WIP/vfs/src/monitor.py
trunk/WIP/vfs/src/parser.py
trunk/WIP/vfs/src/query.py
trunk/WIP/vfs/src/server.py
trunk/WIP/vfs/test/cache.py
trunk/WIP/vfs/test/client.py
Log:
autostart server if needed
Modified: trunk/WIP/vfs/src/__init__.py
==============================================================================
--- trunk/WIP/vfs/src/__init__.py (original)
+++ trunk/WIP/vfs/src/__init__.py Sat Feb 18 12:48:50 2006
@@ -4,9 +4,6 @@
# -----------------------------------------------------------------------------
# $Id$
#
-# TODO: handle all the FIXME and TODO comments inside this file and
-# add docs for functions, variables and how to use this file
-#
# -----------------------------------------------------------------------------
# kaa-vfs - A virtual filesystem with metadata
# Copyright (C) 2005 Dirk Meyer
@@ -14,7 +11,7 @@
# First Edition: Dirk Meyer <[EMAIL PROTECTED]>
# Maintainer: Dirk Meyer <[EMAIL PROTECTED]>
#
-# Please see the file doc/CREDITS for a complete list of authors.
+# Please see the file AUTHORS for a complete list of authors.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -32,26 +29,73 @@
#
# -----------------------------------------------------------------------------
-_db = None
-
-def _error(*args, **kwargs):
- raise RuntimeError('not connected to database')
-
-query = listdir = file = commit = _error
+__all__ = [ 'connect' ]
-def connect(filename):
- global _db
- if _db:
- raise RuntimeError('already connected')
- _db = MediaDB(filename)
-
- global query
- global listdir
- global file
- global commit
-
- query = _db.do_query
- listdir = _db.listdir
- file = _db.file
- commit = _db.commit
+import os
+import popen2
+import time
+import socket
+import logging
+
+from kaa.notifier import step, Timer
+
+from client import Client
+
+# get logging object
+log = logging.getLogger('vfs')
+
+def connect(vfsdb, logfile=None, loglevel=logging.INFO):
+ """
+ Connect to the vfs database dir given by 'vfsdb'. A server will be started
+ if no server is running. The new server will print debug output to the
+ given logfile. If a server is already running, logfile has no effect. If
+ a loglevel is given and the server will be started, it will use the given
+ loglevel. If no logfile is given, the server will log to vfsdb/log.
+ The server can be used by different clients in different applications if
+ the are started by the same user. It will shutdown if no client is
connected
+ for over 5 seconds.
+ """
+ try:
+ # try to connect to an already running server
+ return Client(vfsdb)
+ except socket.error:
+ pass
+
+ # check logfile
+ if not logfile:
+ logfile = os.path.join(vfsdb, 'log')
+ # start server
+ server = os.path.join(os.path.dirname(__file__), 'server.py')
+ server_fd = popen2.popen3(['python', '-OO', server, logfile,
str(loglevel)])
+
+ # wait for server to start
+ # use a small timer to make sure step() comes back
+ stop = time.time() + 2
+ t = Timer(lambda x: True, 1)
+ t.start(0.01)
+ while time.time() < stop:
+ step()
+ try:
+ c = Client(vfsdb)
+ # client ready, close fd to server
+ for fd in server_fd:
+ fd.close()
+ # stop temp timer
+ t.stop()
+ return c
+ except socket.error:
+ pass
+
+ # no server found, print debug
+ for fd in server_fd:
+ try:
+ for msg in fd.readlines():
+ log.error(msg[:-1])
+ except IOError:
+ pass
+ fd.close()
+ # stop temp timer
+ t.stop()
+ # raise error
+ raise OSError('Unable to start vfs server')
Modified: trunk/WIP/vfs/src/db.py
==============================================================================
--- trunk/WIP/vfs/src/db.py (original)
+++ trunk/WIP/vfs/src/db.py Sat Feb 18 12:48:50 2006
@@ -274,7 +274,7 @@
if not self.changes:
return
- print 'COMMIT'
+ log.info('COMMIT')
t1 = time.time()
changes = self.changes
for function, arg1, args, kwargs in self.changes:
Modified: trunk/WIP/vfs/src/monitor.py
==============================================================================
--- trunk/WIP/vfs/src/monitor.py (original)
+++ trunk/WIP/vfs/src/monitor.py Sat Feb 18 12:48:50 2006
@@ -38,7 +38,7 @@
# kaa imports
from kaa.base.weakref import weakref
-from kaa.notifier import OneShotTimer, WeakTimer, Timer, execute_in_timer
+from kaa.notifier import WeakTimer, Timer, execute_in_timer, Callback
# kaa.vfs imports
import parser
@@ -130,27 +130,36 @@
i = items.pop(0)
# FIXME: check parents
if i._vfs_changed():
- changed.append(weakref(i))
+ changed.append(i)
return True
def _update(self, changed, first_call):
if changed:
- c = parser.Checker(weakref(self), self._db, changed, first_call)
- self._checker = weakref(c)
+ cb = Callback(self.checked, first_call)
+ c = parser.Checker(self.callback, self._db, changed, cb)
+ self._checker = c
if not first_call and len(changed) > 10:
self.callback('changed')
elif first_call:
self.callback('checked')
else:
self.callback('changed')
-
- def send_update(self, changed):
- changed = [ (x.url, x._vfs_data) for x in changed ]
- changed.sort(lambda x,y: cmp(x[0], y[0]))
- self.callback('updated', changed)
-
+
+ def checked(self, first_call):
+ self._checker = None
+ self.callback('changed')
+ if first_call:
+ self.callback('checked')
+
+
+ def stop(self):
+ if self._checker:
+ self._checker.stop()
+ self._checker = None
+
+
# if self._query.has_key('device'):
# log.info('unable to update device query, just send notification
here')
# # device query, can't update it
@@ -191,8 +200,6 @@
# continue
# to_check.append(weakref(i))
-# print 'mtime query took %s, %s items to check' % (time.time()-t1,
len(to_check))
-
# if to_check:
# # FIXME: a constantly growing file like a recording will result
in
# # a huge db activity on both client and server because checker
calls
Modified: trunk/WIP/vfs/src/parser.py
==============================================================================
--- trunk/WIP/vfs/src/parser.py (original)
+++ trunk/WIP/vfs/src/parser.py Sat Feb 18 12:48:50 2006
@@ -49,7 +49,7 @@
log = logging.getLogger('vfs')
def parse(db, item, store=False):
- print 'check', item.url
+ log.info('check %s', item.url)
mtime = item._vfs_mtime()
if not mtime:
log.info('oops, no mtime %s' % item)
@@ -112,20 +112,25 @@
class Checker(object):
- def __init__(self, monitor, db, items, notify_checked):
- self.monitor = monitor
+ def __init__(self, notify, db, items, callback):
+ self.notify = notify
self.db = db
self.items = items
+ self.callback = callback
+
self.max = len(items)
self.pos = 0
+
self.updated = []
- self.notify_checked = notify_checked
+ self.stopped = False
self.check()
@execute_in_timer(Timer, 0.01)
def check(self):
-
+ if self.stopped:
+ return False
+
if self.items:
self.pos += 1
item = self.items[0]
@@ -134,30 +139,34 @@
self.notify('progress', self.pos, self.max, item.url)
parse(self.db, item)
if item._vfs_id:
- self.monitor.send_update([item])
+ self.notify('updated', [ (item.url, item._vfs_data) ])
else:
self.updated.append(item)
+
if not self.items:
self.db.commit()
- self.notify('changed')
- if self.notify_checked:
- self.notify('checked')
+ self.stop()
+ self.callback()
+
updated = []
while self.updated and self.updated[0] and self.updated[0]._vfs_id:
updated.append(self.updated.pop(0))
if updated:
- self.monitor.send_update(updated)
-
+ updated = [ (x.url, x._vfs_data) for x in updated ]
+ updated.sort(lambda x,y: cmp(x[0], y[0]))
+ self.notify('updated', updated)
+
if not self.items:
return False
return True
- def notify(self, *args, **kwargs):
- if self.monitor:
- self.monitor.callback(*args, **kwargs)
+ def stop(self):
+ self.items = []
+ self.stopped = True
+
-# def __del__(self):
-# print 'del parser'
+ def __del__(self):
+ log.info('del parser')
Modified: trunk/WIP/vfs/src/query.py
==============================================================================
--- trunk/WIP/vfs/src/query.py (original)
+++ trunk/WIP/vfs/src/query.py Sat Feb 18 12:48:50 2006
@@ -91,7 +91,6 @@
"""
Checked message from server.
"""
-# print 'UPDATE'
url, data = items.pop(0)
for r in self.result:
if r.url == url:
@@ -119,7 +118,9 @@
"""
Memory debug
"""
- print 'del', repr(self)
+ if self._monitor:
+ self.monitor(False)
+ log.debug('del %s' % repr(self))
def __iter__(self):
Modified: trunk/WIP/vfs/src/server.py
==============================================================================
--- trunk/WIP/vfs/src/server.py (original)
+++ trunk/WIP/vfs/src/server.py Sat Feb 18 12:48:50 2006
@@ -149,6 +149,7 @@
log.debug('remove monitor')
for m in monitors:
if m.id == request_id:
+ m.stop()
monitors.remove(m)
return None
log.error('unable to find monitor %s:%s', client_id, request_id)
@@ -224,6 +225,7 @@
# internal list of server
_server = {}
+_num_client = 0
def connect(dbdir):
"""
@@ -232,8 +234,11 @@
"""
log.info('connect to %s' % dbdir)
+ global _num_client
+ _num_client += 1
+
# TODO: delete databases not used anymore
-
+
if not dbdir in _server:
log.info('create server object')
server = Server(dbdir)
@@ -243,13 +248,70 @@
def _client_closed(client):
+ global _num_client
for server in _server.values():
for client_info in server._clients:
if ipc.get_ipc_from_proxy(client_info[1]) == client:
log.warning('disconnect client')
+ for m in client_info[3]:
+ m.stop()
server._clients.remove(client_info)
+ _num_client -= 1
+
+
+if __name__ == "__main__":
+
+ # python imports
+ import gc
+ import sys
+
+ # kaa imports
+ from kaa.notifier import Timer, execute_in_timer, loop
+
+ @execute_in_timer(Timer, 1)
+ def garbage_collect():
+ g = gc.collect()
+ if g:
+ log.info('gc: deleted %s objects' % g)
+ if gc.garbage:
+ log.warning('gc: found %s garbage objects' % len(gc.garbage))
+ for g in gc.garbage:
+ log.warning(g)
+ return True
+
+
+ shutdown_timer = 5
+
+ @execute_in_timer(Timer, 1)
+ def autoshutdown():
+ global shutdown_timer
+ if _num_client > 0:
+ shutdown_timer = 5
+ return True
+ shutdown_timer -= 1
+ if shutdown_timer == 0:
+ log.info('shutdown vfs server')
+ sys.exit(0)
+ return True
+
+ # setup logger
+ f = logging.Formatter('%(asctime)s %(levelname)-8s [%(name)6s] '+\
+ '%(filename)s %(lineno)s: '+\
+ '%(message)s')
+ handler = logging.FileHandler(sys.argv[1])
+ handler.setFormatter(f)
+ logging.getLogger().addHandler(handler)
+
+ # set log level
+ logging.getLogger().setLevel(int(sys.argv[2]))
+
+ log.info('start vfs server')
-# connect to the ipc code
-_ipc = ipc.IPCServer('vfs')
-_ipc.register_object(connect, 'vfs')
-_ipc.signals["client_closed"].connect(_client_closed)
+ # connect to the ipc code
+ _ipc = ipc.IPCServer('vfs')
+ _ipc.register_object(connect, 'vfs')
+ _ipc.signals["client_closed"].connect(_client_closed)
+
+ garbage_collect()
+ autoshutdown()
+ loop()
Modified: trunk/WIP/vfs/test/cache.py
==============================================================================
--- trunk/WIP/vfs/test/cache.py (original)
+++ trunk/WIP/vfs/test/cache.py Sat Feb 18 12:48:50 2006
@@ -1,8 +1,13 @@
import sys
import kaa
-import kaa.vfs.client
+import kaa.vfs
+import logging
-c = kaa.vfs.client.Client('vfsdb')
+# full parameter set for connect
+# c = kaa.vfs.connect('vfsdb', 'logfile', logging.INFO)
+
+# simple connect
+c = kaa.vfs.connect('vfsdb')
checked = []
to_check = []
Modified: trunk/WIP/vfs/test/client.py
==============================================================================
--- trunk/WIP/vfs/test/client.py (original)
+++ trunk/WIP/vfs/test/client.py Sat Feb 18 12:48:50 2006
@@ -1,4 +1,4 @@
-import kaa.vfs.client
+import kaa.vfs
from kaa.notifier import Timer, OneShotTimer
import sys
import kaa
@@ -8,82 +8,22 @@
import logging
-VERBOSE = True
+def msg(*args):
+ print '>>>>>>>>>', args
+
+c = kaa.vfs.connect('vfsdb')
+video = c.get(sys.argv[1])
+print video
+x = video.listdir()
+
+x.signals['changed'].connect(msg, 'changed')
+x.signals['progress'].connect(msg, 'progress')
+x.signals['up-to-date'].connect(msg, 'up-to-date')
+
+x.monitor()
+for f in x:
+ print repr(f)
-def foo():
- print 'delete all'
- global q
- q = None
-
-def progress(pos, max, last):
- if VERBOSE:
- print 'progress (%s of %s) -- %s' % (pos, max, last)
-
-def update(client, query):
- print 'update'
- result = query.get()
- if not VERBOSE:
- return
- if isinstance(result, list):
- for item in q.get():
- print item
- else:
- print 'Disc', result
- result = result.item()
- if isinstance(result, list):
- for f in result:
- print ' ', f
- else:
- print result
- if result and result.filename:
- # it is a disc, scan dir (hope it's mounted)
- print 'files on disc (needs to be mounted manually):'
- for f in client.query(dirname=result.filename).get():
- print ' ', f
- print
-
-def show_artists_list():
- for artist in c.query(attr='artist', type='audio').get():
- print artist
- for album in c.query(attr='album', artist=artist, type='audio').get():
- print ' ', album
-
-c = kaa.vfs.client.Client('vfsdb')
-c.add_mountpoint('/dev/cdrom', '/mnt/cdrom')
-c.add_mountpoint('/dev/dvd', '/mnt/dvd')
-
-if len(sys.argv) < 2 or sys.argv[1].find('=') <= 0:
- print 'usage: client query'
- print 'e.g. client directory=/home/dmeyer/video'
- print ' client file=/path/to/file.mp3'
- print ' client device=/mnt/cdrom'
- print ' client attr=album type=audio'
- sys.exit(0)
-
-if sys.argv[1][:sys.argv[1].find('=')] == 'file':
- t1 = time.time()
- q = c.query(files=(sys.argv[1][sys.argv[1].find('=')+1:],))
-else:
- query = {}
- for attr in sys.argv[1:]:
- query[attr[:attr.find('=')]]=attr[attr.find('=')+1:]
- t1 = time.time()
- q = c.query(**query)
-q.signals['changed'].connect(update, c, weakref(q))
-q.signals['progress'].connect(progress)
-t2 = time.time()
-
-result = q.get()
-if VERBOSE:
- if isinstance(result, list):
- for item in q.get():
- print item
- else:
- print 'Disc', result
-
-print 'q took %s' % (t2 - t1)
-
-#OneShotTimer(show_artists_list).start(1)
-#Timer(foo).start(5)
+# OneShotTimer(x.monitor, False).start(1)
print 'loop'
kaa.main()
-------------------------------------------------------
This SF.net email is sponsored by: Splunk Inc. Do you grep through log files
for problems? Stop! Download the new AJAX search engine that makes
searching your log files as easy as surfing the web. DOWNLOAD SPLUNK!
http://sel.as-us.falkag.net/sel?cmd=lnk&kid=103432&bid=230486&dat=121642
_______________________________________________
Freevo-cvslog mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/freevo-cvslog
