Update of /cvsroot/freevo/freevo/src/mediadb
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv28228
Modified Files:
__init__.py audio_parser.py cache.py db.py listing.py
parser.py video_parser.py
Log Message:
more improvements, still not ready
Index: parser.py
===================================================================
RCS file: /cvsroot/freevo/freevo/src/mediadb/parser.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** parser.py 4 Apr 2005 18:28:15 -0000 1.1
--- parser.py 5 Apr 2005 18:48:18 -0000 1.2
***************
*** 15,24 ****
_parser = []
! def _init():
"""
Init the parser module
"""
! d = os.path.dirname(__file__)
! for f in os.listdir(d):
if f.endswith('_parser.py'):
exec('import %s' % f[:-3])
--- 15,23 ----
_parser = []
! def init():
"""
Init the parser module
"""
! for f in os.listdir(os.path.dirname(__file__)):
if f.endswith('_parser.py'):
exec('import %s' % f[:-3])
***************
*** 26,30 ****
! def _simplify(object):
"""
mmpython has huge objects to cache, we don't need them.
--- 25,29 ----
! def simplify(object):
"""
mmpython has huge objects to cache, we don't need them.
***************
*** 47,51 ****
ret[k] = []
for o in getattr(object, k):
! ret[k].append(_simplify(o))
if hasattr(object, 'tracks') and object.tracks:
--- 46,50 ----
ret[k] = []
for o in getattr(object, k):
! ret[k].append(simplify(o))
if hasattr(object, 'tracks') and object.tracks:
***************
*** 53,57 ****
ret['tracks'] = []
for o in object.tracks:
! track = _simplify(o)
if not track.has_key('audio'):
track['audio'] = []
--- 52,56 ----
ret['tracks'] = []
for o in object.tracks:
! track = simplify(o)
if not track.has_key('audio'):
track['audio'] = []
***************
*** 66,102 ****
return ret
! def _parse_fxd_node(node):
! """
! Parse a fxd node.
! """
! children = []
! for c in node.children:
! children.append(_parse_fxd_node(c))
! return (node.name, node.attrs, children, node.textof(), node.first_cdata,
! node.following_cdata)
!
!
! def _parse_fxd(filename):
! """
! Parse a fxd file.
! """
! data = util.fxdparser.FXDtree(filename, False)
! if data.tree.name != 'freevo':
! return {}
! is_skin_fxd = False
! for node in data.tree.children:
! if node.name == 'skin':
! is_skin_fxd = True
! break
! tree = []
! for node in data.tree.children:
! tree.append(_parse_fxd_node(node))
! return is_skin_fxd, tree
!
!
! # regexp for filenames used in _getname
_FILENAME_REGEXP = re.compile("^(.*?)_(.)(.*)$")
! def _getname(file):
"""
make a nicer display name from file
--- 65,72 ----
return ret
! # regexp for filenames used in getname
_FILENAME_REGEXP = re.compile("^(.*?)_(.)(.*)$")
! def getname(file):
"""
make a nicer display name from file
***************
*** 139,152 ****
Add additional informations to filename, object.
"""
- if not _parser:
- _init()
mminfo = None
if not object['ext'] in [ 'xml', 'fxd' ]:
mminfo = mmpython.parse(filename)
! title = _getname(filename)
object['title:filename'] = title
if mminfo:
# store mmpython data as pickle for faster loading
! object['mminfo'] = cPickle.dumps(_simplify(mminfo),
pickle.HIGHEST_PROTOCOL)
if mminfo.title:
--- 109,120 ----
Add additional informations to filename, object.
"""
mminfo = None
if not object['ext'] in [ 'xml', 'fxd' ]:
mminfo = mmpython.parse(filename)
! title = getname(filename)
object['title:filename'] = title
if mminfo:
# store mmpython data as pickle for faster loading
! object['mminfo'] = cPickle.dumps(simplify(mminfo),
pickle.HIGHEST_PROTOCOL)
if mminfo.title:
***************
*** 160,168 ****
object['title'] = title
- if filename.endswith('.fxd'):
- # store fxd tree as pickle for faster loading
- object['fxd'] = cPickle.dumps(_parse_fxd(filename),
- pickle.HIGHEST_PROTOCOL)
-
if os.path.isdir(filename):
object['isdir'] = True
--- 128,131 ----
***************
*** 203,213 ****
! def cache():
"""
Function for the 'cache' helper.
"""
- if not _parser:
- _init()
for p in _parser:
! p.cache()
--- 166,174 ----
! def cache(listing):
"""
Function for the 'cache' helper.
"""
for p in _parser:
! p.cache(listing)
Index: video_parser.py
===================================================================
RCS file: /cvsroot/freevo/freevo/src/mediadb/video_parser.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** video_parser.py 4 Apr 2005 18:28:16 -0000 1.1
--- video_parser.py 5 Apr 2005 18:48:18 -0000 1.2
***************
*** 11,15 ****
! def cache():
"""
Function for the 'cache' helper.
--- 11,15 ----
! def cache(listing):
"""
Function for the 'cache' helper.
Index: cache.py
===================================================================
RCS file: /cvsroot/freevo/freevo/src/mediadb/cache.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** cache.py 4 Apr 2005 18:28:14 -0000 1.1
--- cache.py 5 Apr 2005 18:48:18 -0000 1.2
***************
*** 3,8 ****
import util.fileops as fileops
import notifier
! from listing import Listing
class ProgressBox:
--- 3,10 ----
import util.fileops as fileops
import notifier
+ import time
! import config
! import mediadb
class ProgressBox:
***************
*** 21,35 ****
! def cache_directories(directories, rebuild=False):
"""
! cache all directories with mmpython
"""
! if rebuild:
! print 'deleting cache files..................................',
! sys.__stdout__.flush()
! mediainfo.del_cache()
! print 'done'
!
! print 'checking mmpython cache files.........................',
sys.__stdout__.flush()
listings = []
--- 23,31 ----
! def cache_directories(directories):
"""
! cache all directories
"""
! print 'checking database files...............................',
sys.__stdout__.flush()
listings = []
***************
*** 52,71 ****
!
! # MAIN
!
! import config
! import parser
! import db
!
! def get_direcories(dirlist, msg):
"""
! cache a list of directories recursive
"""
! if not dirlist:
! return
! all_dirs = []
! all_listing = []
! # create a list of all subdirs
for dir in dirlist:
progress = '%3d%%' % (dirlist.index(dir) * 100 / len(dirlist))
--- 48,57 ----
! def get_directory_listings(dirlist, msg):
"""
! Get a list of Listings recursive of all given directories.
"""
! subdirs = []
! listings = []
for dir in dirlist:
progress = '%3d%%' % (dirlist.index(dir) * 100 / len(dirlist))
***************
*** 73,101 ****
sys.__stdout__.flush()
for dirname in fileops.get_subdirs_recursively(dir):
! if not dirname in all_dirs:
! all_dirs.append(dirname)
! all_listing.append(Listing(dirname))
! if not dir in all_dirs:
! all_dirs.append(dir)
! all_listing.append(Listing(dir))
! return all_dirs, all_listing
notifier.init( notifier.GENERIC )
! all_dirs = []
msg = 'scanning directory structure..........................'
print msg,
sys.__stdout__.flush()
! config.AUDIO_ITEMS = [ ( 'foo', '/local/mp3/Artists/Dixie Chicks' ) ]
! config.VIDEO_ITEMS = [ ( 'foo', '/local/video/movie' ) ]
! config.IMAGE_ITEMS = [ ( 'foo', '/local/images/fotos/misc' ) ]
for d in config.VIDEO_ITEMS + config.AUDIO_ITEMS + config.IMAGE_ITEMS:
if os.path.isdir(d[1]) and d[1] != '/':
! all_dirs.append(d[1])
! all_dirs, all_listing = get_direcories(all_dirs, msg)
print '\r%s done' % msg
! cache_directories(all_listing)
! parser.cache()
! db.save()
--- 59,96 ----
sys.__stdout__.flush()
for dirname in fileops.get_subdirs_recursively(dir):
! if not dirname in subdirs:
! subdirs.append(dirname)
! listings.append(mediadb.Listing(dirname))
! if not dir in subdirs:
! subdirs.append(dir)
! listings.append(mediadb.Listing(dir))
! return listings
+
+ # init the notifier
notifier.init( notifier.GENERIC )
!
! start = time.clock()
msg = 'scanning directory structure..........................'
print msg,
sys.__stdout__.flush()
! # config.AUDIO_ITEMS = [ ( 'foo', '/local/mp3/Artists/Dixie Chicks' ) ]
! # config.VIDEO_ITEMS = [ ( 'foo', '/local/video/movie' ) ]
! # config.IMAGE_ITEMS = [ ( 'foo', '/local/images/fotos/misc' ) ]
+ directories = []
for d in config.VIDEO_ITEMS + config.AUDIO_ITEMS + config.IMAGE_ITEMS:
if os.path.isdir(d[1]) and d[1] != '/':
! directories.append(d[1])
! listings = get_directory_listings(directories, msg)
print '\r%s done' % msg
! cache_directories(listings)
! l = mediadb.FileListing(directories)
! if l.num_changes:
! l.update()
! mediadb.cache(l)
! mediadb.save()
! print
! print 'caching complete after %s seconds' % (time.clock() - start)
Index: db.py
===================================================================
RCS file: /cvsroot/freevo/freevo/src/mediadb/db.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** db.py 4 Apr 2005 18:28:15 -0000 1.2
--- db.py 5 Apr 2005 18:48:18 -0000 1.3
***************
*** 51,55 ****
log = logging.getLogger('mediadb')
! VERSION = 2.4
class CacheList:
--- 51,56 ----
log = logging.getLogger('mediadb')
! # internal format version
! VERSION = 0.1
class CacheList:
***************
*** 353,357 ****
return 'mediadb.db.Cache for %s' % self.dirname
!
class FileCache:
"""
--- 354,358 ----
return 'mediadb.db.Cache for %s' % self.dirname
!
class FileCache:
"""
Index: audio_parser.py
===================================================================
RCS file: /cvsroot/freevo/freevo/src/mediadb/audio_parser.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** audio_parser.py 4 Apr 2005 18:28:14 -0000 1.1
--- audio_parser.py 5 Apr 2005 18:48:17 -0000 1.2
***************
*** 1,2 ****
--- 1,34 ----
+ # -*- coding: iso-8859-1 -*-
+ #
-----------------------------------------------------------------------------
+ # audio_parser.py - parser for audio metadata
+ #
-----------------------------------------------------------------------------
+ # $Id$
+ #
+ #
-----------------------------------------------------------------------------
+ # Freevo - A Home Theater PC framework
+ # Copyright (C) 2002-2004 Krister Lagerstrom, Dirk Meyer, et al.
+ #
+ # First Edition: Dirk Meyer <[EMAIL PROTECTED]>
+ # Maintainer: Dirk Meyer <[EMAIL PROTECTED]>
+ #
+ # Please see the file freevo/Docs/CREDITS for a complete list of authors.
+ #
+ # This program is free software; you can redistribute it and/or modify
+ # it under the terms of the GNU General Public License as published by
+ # the Free Software Foundation; either version 2 of the License, or
+ # (at your option) any later version.
+ #
+ # This program is distributed in the hope that it will be useful, but
+ # WITHOUT ANY WARRANTY; without even the implied warranty of MER-
+ # CHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
+ # Public License for more details.
+ #
+ # You should have received a copy of the GNU General Public License along
+ # with this program; if not, write to the Free Software Foundation, Inc.,
+ # 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ #
+ #
-----------------------------------------------------------------------------
+
+ # python imports
import sys
import os
***************
*** 4,17 ****
import md5
! import config
! import util
!
from listing import Listing, FileListing
! #
# Interface
#
! def cache():
"""
Function for the 'cache' helper.
--- 36,47 ----
import md5
! # mediadb imports
from listing import Listing, FileListing
! #
# Interface
#
! def cache(listing):
"""
Function for the 'cache' helper.
***************
*** 19,25 ****
print 'creating audio metadata...............................',
sys.__stdout__.flush()
! for dir in config.AUDIO_ITEMS:
! if os.path.isdir(dir[1]):
! AudioParser(dir[1], rescan=True)
print 'done'
--- 49,54 ----
print 'creating audio metadata...............................',
sys.__stdout__.flush()
! for item in listing:
! check_info(item)
print 'done'
***************
*** 32,181 ****
! #
# Internal helper functions and classes
#
! _VARIOUS = u'__various__'
! class AudioParser:
! def __init__(self, dirname, force=False, rescan=False):
! self.artist = ''
! self.album = ''
! self.year = ''
! self.length = 0
! self.changed = False
! self.force = force
! cachefile = vfs.getoverlay(os.path.join(dirname, '..',
'freevo.cache'))
! subdirs = util.getdirnames(dirname, softlinks=False)
! filelist = None
! parent = FileListing( [ dirname ] )
! if parent.num_changes:
! parent.update()
! dirinfo = parent.get_by_name(os.path.basename(dirname))
!
! if not rescan:
! for subdir in subdirs:
! d = AudioParser(subdir, rescan)
! if d.changed:
! break
! else:
! # no changes in all subdirs, looks good
! if os.path.isfile(cachefile) and \
! os.stat(dirname)[stat.ST_MTIME] <=
os.stat(cachefile)[stat.ST_MTIME]:
! # and no changes in here. Do not parse everything again
! if force:
! # forces? We need to load our current values
! for type in ('artist', 'album', 'year', 'length'):
! if info.has_key(type):
! setattr(self, type, info[type])
! return
! if not filelist:
! filelist = util.match_files(dirname, config.AUDIO_SUFFIX)
!
! if not filelist and not subdirs:
! # no files in here? We are done
! return
!
! # ok, something changed here, too bad :-(
! self.changed = True
! self.force = False
! # scan all subdirs
! for subdir in subdirs:
! d = AudioParser(subdir, force=True, rescan=rescan)
! for type in ('artist', 'album', 'year'):
! setattr(self, type, self.strcmp(getattr(self, type),
getattr(d, type)))
! self.length += d.length
! # cache dir first
! listing = Listing(dirname)
! if listing.num_changes:
! listing.update()
!
! use_tracks = True
! for data in listing.match_suffix(config.AUDIO_SUFFIX):
! try:
! for type in ('artist', 'album'):
! setattr(self, type, self.strcmp(getattr(self, type),
data[type]))
! self.year = self.strcmp(self.year, data['date'])
! if data['length']:
! self.length += int(data['length'])
! use_tracks = use_tracks and data['trackno']
! except OSError:
! pass
! if use_tracks and (self.album or self.artist):
! dirinfo.store_with_mtime('audio_advanced_sort', True)
! if not self.length:
! return
! for type in ('artist', 'album', 'year', 'length'):
! if getattr(self, type):
! dirinfo.store_with_mtime(type, getattr(self, type))
- modtime = os.stat(dirname)[stat.ST_MTIME]
- if not dirinfo['coverscan'] or dirinfo['coverscan'] != modtime:
- dirinfo.store('coverscan', modtime)
- self.extract_image(dirname)
!
! def strcmp(self, s1, s2):
! s1 = Unicode(s1)
! s2 = Unicode(s2)
!
! if not s1 or not s2:
! return s1 or s2
! if s1 == _VARIOUS or s2 == _VARIOUS:
! return _VARIOUS
! if s1.replace(u' ', u'').lower() == s2.replace(u' ', u'').lower():
! return s1
! return _VARIOUS
- def get_md5(self, obj):
- m = md5.new()
- if isinstance(obj,file): # file
- for line in obj.readlines():
- m.update(line)
- return m.digest()
- else: # str
- m.update(obj)
- return m.digest()
- def extract_image(self, path):
- for i in util.match_files(path, ['mp3']):
- try:
- id3 = eyeD3.Mp3AudioFile( i )
- except:
- continue
- myname = vfs.getoverlay(os.path.join(path, 'cover.jpg'))
- if id3.tag:
- images = id3.tag.getImages();
- for img in images:
- if vfs.isfile(myname) and
(self.get_md5(vfs.open(myname,'rb')) == \
- self.get_md5(img.imageData)):
- # Image already there and has identical md5, skip
- pass
- elif not vfs.isfile(myname):
- f = vfs.open(myname, "wb")
- f.write(img.imageData)
- f.flush()
- f.close()
- else:
- # image exists, but sums are different, write a
unique cover
- iname =
os.path.splitext(os.path.basename(i))[0]+'.jpg'
- myname = vfs.getoverlay(os.path.join(path, iname))
- f = vfs.open(myname, "wb")
- f.write(img.imageData)
- f.flush()
- f.close()
--- 61,205 ----
! #
# Internal helper functions and classes
#
! VARIOUS = u'__various__'
! def check_info(dirinfo):
! """
! Parse additional information recursive over the given listing. The
function
! will try to find 'length', 'artist', 'album' and 'year'.
! """
! listing = Listing(dirinfo.filename)
! if listing.num_changes:
! listing.update()
! # check if an update is needed
! for item in listing.get_dir():
! if os.path.islink(item.filename):
! # ignore softlinks
! continue
! if check_info(item):
! # yes, something changed
! break
! else:
! # no changes in all subdirs, looks good
! if dirinfo['length'] != None:
! # Since 'length' is not None and the info is stored with mtime,
! # no changes in here. Do not parse everything again
! return False
! # get a new listing again
! listing = Listing(dirinfo.filename)
! # variables to scan
! length = 0
! artist = ''
! album = ''
! year = ''
! # scan all subdirs
! for item in listing.get_dir():
! if os.path.islink(item.filename):
! # ignore softlinks
! continue
! check_info(item)
! for type in ('artist', 'album', 'year'):
! exec('%s = strcmp(dirinfo[type], %s)' % (type, type))
! length += item['length']
! use_tracks = True
! for item in listing:
! try:
! for type in ('artist', 'album'):
! exec('%s = strcmp(%s, item[type])' % (type, type))
! year = strcmp(year, item['date'])
! if item['length']:
! length += int(item['length'])
! use_tracks = use_tracks and item['trackno']
! except OSError:
! pass
! if use_tracks and (dirinfo['album'] or dirinfo['artist']):
! dirinfo.store_with_mtime('audio_advanced_sort', True)
! dirinfo.store_with_mtime('length', length)
! if not length:
! return True
! for type in ('artist', 'album', 'year'):
! if eval(type):
! dirinfo.store_with_mtime(type, eval(type))
! if not dirinfo['coverscan']:
! dirinfo.store_with_mtime('coverscan', True)
! extract_image(listing)
! return True
! def strcmp(s1, s2):
! """
! Compare the given strings. If one is empty, return the other. If both are
! equal, return the string. If they are different, return VARIOUS.
! """
! s1 = Unicode(s1)
! s2 = Unicode(s2)
! if not s1 or not s2:
! return s1 or s2
! if s1 == VARIOUS or s2 == VARIOUS:
! return VARIOUS
+ if s1.replace(u' ', u'').lower() == s2.replace(u' ', u'').lower():
+ return s1
+ return VARIOUS
+ def get_md5(obj):
+ """
+ Get md5 checksum of the given object (string or file)
+ """
+ m = md5.new()
+ if isinstance(obj,file):
+ for line in obj.readlines():
+ m.update(line)
+ return m.digest()
+ else:
+ m.update(obj)
+ return m.digest()
+ def extract_image(listing):
+ """
+ Extract image from mp3 files in listing.
+ """
+ for item in listing.match_suffix(['mp3']):
+ try:
+ id3 = eyeD3.Mp3AudioFile( item.filename )
+ except:
+ continue
+ myname = vfs.getoverlay(os.path.join(path, 'cover.jpg'))
+ if id3.tag:
+ images = id3.tag.getImages();
+ for img in images:
+ if vfs.isfile(myname) and (get_md5(vfs.open(myname,'rb')) == \
+ get_md5(img.imageData)):
+ # Image already there and has identical md5, skip
+ pass
+ elif not vfs.isfile(myname):
+ f = vfs.open(myname, "wb")
+ f.write(img.imageData)
+ f.flush()
+ f.close()
+ else:
+ # image exists, but sums are different, write a unique
+ # cover
+ iname = os.path.splitext(os.path.basename(i))[0]+'.jpg'
+ myname = vfs.getoverlay(os.path.join(path, iname))
+ f = vfs.open(myname, "wb")
+ f.write(img.imageData)
+ f.flush()
+ f.close()
Index: listing.py
===================================================================
RCS file: /cvsroot/freevo/freevo/src/mediadb/listing.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** listing.py 4 Apr 2005 18:28:15 -0000 1.2
--- listing.py 5 Apr 2005 18:48:18 -0000 1.3
***************
*** 121,125 ****
self.visible.append(v)
return ret
!
class FileListing(Listing):
--- 121,128 ----
self.visible.append(v)
return ret
!
! def __iter__(self):
! return self.visible.__iter__()
!
class FileListing(Listing):
Index: __init__.py
===================================================================
RCS file: /cvsroot/freevo/freevo/src/mediadb/__init__.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** __init__.py 4 Apr 2005 18:28:04 -0000 1.2
--- __init__.py 5 Apr 2005 18:48:17 -0000 1.3
***************
*** 51,58 ****
--- 51,61 ----
from item import ItemInfo
from listing import Listing, FileListing
+ from parser import cache, init
# get logging object
log = logging.getLogger('mediadb')
+ # init parsing module
+ init()
def disc_info(media, force=False):
***************
*** 74,83 ****
#
! # FIXME: the following functions will be removed in the future
# also missing the attribute mmdata used by videoitem.py
#
def get(filename):
! # used by directory.py, item.py, videoitem.py and extendedmeta.py
log.warning('get simple info %s', filename)
listing = FileListing( [ filename ] )
--- 77,86 ----
#
! # FIXME: the following function will be removed in the future
# also missing the attribute mmdata used by videoitem.py
#
def get(filename):
! # used by directory.py, item.py
log.warning('get simple info %s', filename)
listing = FileListing( [ filename ] )
***************
*** 85,101 ****
listing.update()
return listing.get_by_name(os.path.basename(filename))
-
-
- def del_cache():
- # used by cache.py
- log.error('del_cache not defined anymore')
-
-
- def set(filename, key, value):
- # used by extendedmeta.py
- log.error('set not defined anymore')
-
-
- def cache_dir(dirname, callback=None):
- # used by cache.py, extendedmeta.py
- log.error('cache_dir not defined anymore')
--- 88,89 ----
-------------------------------------------------------
SF email is sponsored by - The IT Product Guide
Read honest & candid reviews on hundreds of IT Products from real users.
Discover which products truly live up to the hype. Start reading now.
http://ads.osdn.com/?ad_id=6595&alloc_id=14396&op=click
_______________________________________________
Freevo-cvslog mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/freevo-cvslog
