Author: dmeyer
Date: Sat Feb 24 21:59:55 2007
New Revision: 2520
Added:
trunk/epg/test/config_vdr.cxml
- copied unchanged from r2517, /trunk/epg/src/sources/config_vdr.cxml
trunk/epg/test/vdr.py
- copied unchanged from r2517, /trunk/epg/src/sources/vdr.py
Removed:
trunk/epg/src/sources/config_vdr.cxml
trunk/epg/src/sources/vdr.py
Modified:
trunk/epg/src/client.py
trunk/epg/src/sources/epgdata.py
trunk/epg/src/sources/xmltv.py
trunk/epg/src/sources/zap2it.py
Log:
Make the update function in the source parser use yield_execution.
This has the advantage that the client calling the update gets the
return when the update is finished and not before. Possible exceptions
are also passed to the client so it can deal with it.
The xmltv and epgdata parser are tested, the zaptoit parser was changed
without testing, I have no access to the service. The vdr parser was
moved to test, it was way to old, not even using the config code. This
parser needs to be fixed to get back in.
Modified: trunk/epg/src/client.py
==============================================================================
--- trunk/epg/src/client.py (original)
+++ trunk/epg/src/client.py Sat Feb 24 21:59:55 2007
@@ -243,4 +243,4 @@
"""
if self.status == DISCONNECTED:
return False
- self.server.rpc('guide.update', *args, **kwargs)
+ return self.server.rpc('guide.update', *args, **kwargs)
Modified: trunk/epg/src/sources/epgdata.py
==============================================================================
--- trunk/epg/src/sources/epgdata.py (original)
+++ trunk/epg/src/sources/epgdata.py Sat Feb 24 21:59:55 2007
@@ -36,8 +36,8 @@
# kaa imports
from kaa import xml, TEMP
-from kaa.config import Var, Group
-from kaa.notifier import Timer, Thread
+import kaa.notifier
+import kaa.xml
from config_epgdata import config
@@ -190,7 +190,8 @@
pass
-def _update_parse_xml_thread(epg, pin, days):
[EMAIL PROTECTED]('epg')
+def _parse_xml(epg):
"""
Thread to parse the xml file. It will also call the grabber if needed.
"""
@@ -217,7 +218,7 @@
# create download adresse for meta data
addresse = 'http://www.epgdata.com/index.php'
addresse+= '?action=sendInclude&iLang=de&iOEM=xml&iCountry=de'
- addresse+= '&pin=%s' %pin
+ addresse+= '&pin=%s' % config.pin
addresse+= '&dataType=xml'
@@ -232,7 +233,6 @@
%(tmpfile, addresse, logfile, logfile))
if not os.path.exists(tmpfile) or exit:
log.error('Cannot get file from epgdata.com, see %s' %logfile)
- epg.guide_changed()
return
# and unzip the zip file
log.info('Unzipping data for meta data')
@@ -240,7 +240,6 @@
%(tempdir, tmpfile, logfile, logfile))
if exit:
log.error('Cannot unzip the downloaded file, see %s' %logfile)
- epg.guide_changed()
return
# list of channel info xml files
@@ -252,7 +251,7 @@
# parse this files
for xmlfile in chfiles:
try:
- doc = xml.Document(xmlfile, 'channel')
+ doc = kaa.xml.Document(xmlfile, 'channel')
except:
log.warning('error while parsing %s' %xmlfile)
continue
@@ -264,7 +263,7 @@
try:
# the genre file
xmlfile = os.path.join(tempdir, 'genre.xml')
- doc = xml.Document(xmlfile, 'genre')
+ doc = kaa.xml.Document(xmlfile, 'genre')
except:
log.warning('error while parsing %s' %xmlfile)
else:
@@ -274,7 +273,7 @@
try:
# the category file
xmlfile = os.path.join(tempdir, 'category.xml')
- doc = xml.Document(xmlfile, 'category')
+ doc = kaa.xml.Document(xmlfile, 'category')
except:
log.warning('error while parsing %s' %xmlfile)
else:
@@ -286,11 +285,11 @@
# create download adresse for programm files
addresse = 'http://www.epgdata.com/index.php'
addresse+= '?action=sendPackage&iLang=de&iOEM=xml&iCountry=de'
- addresse+= '&pin=%s' %pin
+ addresse+= '&pin=%s' % config.pin
addresse+= '&dayOffset=%s&dataType=xml'
# get the file for each day
- for i in range(0, days):
+ for i in range(0, int(config.days)):
# remove old file if needed
try:
os.remove(tmpfile)
@@ -302,7 +301,6 @@
%(tmpfile, addresse %i, logfile, logfile))
if not os.path.exists(tmpfile) or exit:
log.error('Cannot get file from epgdata.com, see %s' %logfile)
- epg.guide_changed()
return
# and unzip the zip file
log.info('Unzipping data for day %s' %(i+1))
@@ -310,7 +308,6 @@
%(tempdir, tmpfile, logfile, logfile))
if exit:
log.error('Cannot unzip the downloaded file, see %s' %logfile)
- epg.guide_changed()
return
@@ -322,7 +319,7 @@
# parse the progam xml files
for xmlfile in progfiles:
try:
- doc = xml.Document(xmlfile, 'pack')
+ doc = kaa.xml.Document(xmlfile, 'pack')
except:
log.warning('error while parsing %s' %xmlfile)
continue
@@ -337,7 +334,7 @@
# put the informations in the UpdateInfo object.
info = UpdateInfo()
info.epg = epg
- info.pin = pin
+ info.pin = config.pin
info.channel_id_to_db_id = {}
info.meta_id_to_meta_name = {}
info.docs =docs
@@ -346,54 +343,43 @@
info.total = nodes
info.progress_step = info.total / 100
-
- # start parser in main loop again, thread is done
- timer = Timer(_update_process_step, info)
- timer.start(0)
+ return info
-def _update_process_step(info):
[EMAIL PROTECTED]()
+def update(epg):
"""
- Step in main loop for the parsing of the epgdata xml files.
- This function will be called in a Timer until everything is parsed.
+ Interface to source_epgdata.
"""
+ if not config.pin:
+ log.error('PIN for epgdata.com is missing in tvserver.conf')
+ yield False
+
+ # _parse_xml is forced to be executed in a thread. This means that
+ # it always returns an InProgress object that needs to be yielded.
+ # When yield returns we need to call the InProgress object to get
+ # the result. If the result is None, the thread run into an error.
+ info = _parse_xml(epg)
+ yield info
+ info = info()
+ if not info:
+ yield False
+
t0 = time.time()
- while info.node:
- if info.node.name == "data":
- # parse!
- parse_data(info)
- info.node = info.node.get_next()
- if time.time() - t0 > 0.1:
- # time to return to the main loop
- break
- if not info.node:
- # check if there are more files to parse
- if len(info.docs)>0:
+ while info.node or len(info.docs) > 0:
+ while info.node:
+ if info.node.name == "data":
+ # parse!
+ parse_data(info)
+ info.node = info.node.get_next()
+ if time.time() - t0 > 0.1:
+ # time to return to the main loop
+ yield kaa.notifier.YieldContinue
+ t0 = time.time()
+ if len(info.docs) > 0:
# take the next one
info.doc = info.docs.pop(0)
# and start with its first node
info.node = info.doc.first
- else:
- # no more files to parse, mission completed!
- info.epg.guide_changed()
- log.info('epg grabbing finished!')
- return False
-
- return True
-
-
-def update(epg):
- """
- Interface to source_epgdata. This function will start the update process.
- """
- try:
- pin = config.pin
- except KeyError:
- log.exception('PIN for epgdata.com is missing in tvserver.conf')
- epg.guide_changed()
- return False
-
- thread = Thread(_update_parse_xml_thread, epg,
- str(config.pin), int(config.days))
- thread.start()
- return True
+ epg.guide_changed()
+ yield True
Modified: trunk/epg/src/sources/xmltv.py
==============================================================================
--- trunk/epg/src/sources/xmltv.py (original)
+++ trunk/epg/src/sources/xmltv.py Sat Feb 24 21:59:55 2007
@@ -37,9 +37,9 @@
import logging
# kaa imports
-from kaa import xml, TEMP
-from kaa.config import Var, Group
-from kaa.notifier import Timer, Thread
+from kaa import TEMP
+import kaa.notifier
+import kaa.xml
from config_xmltv import config
@@ -191,7 +191,8 @@
pass
-def _update_parse_xml_thread(epg):
[EMAIL PROTECTED]('epg')
+def _parse_xml(epg):
"""
Thread to parse the xml file. It will also call the grabber if needed.
"""
@@ -208,7 +209,6 @@
(config.grabber, xmltv_file, config.days, log_file,
log_file))
if not os.path.exists(xmltv_file) or ec:
log.error('grabber failed, see %s', log_file)
- epg.guide_changed()
return
if config.sort:
@@ -219,7 +219,6 @@
os.unlink(xmltv_file + '.tmp')
if not os.path.exists(xmltv_file):
log.error('sorting failed, see %s', log_file)
- epg.guide_changed()
return
else:
log.info('not configured to use tv_sort, skipping')
@@ -229,10 +228,9 @@
# Now we have a xmltv file and need to parse it
log.info('parse xml file')
try:
- doc = xml.Document(xmltv_file, 'tv')
+ doc = kaa.xml.Document(xmltv_file, 'tv')
except:
log.exception('error parsing xmltv file')
- epg.guide_changed()
return
channel_id_to_db_id = {}
@@ -250,16 +248,27 @@
info.epg = epg
info.progress_step = info.total / 100
- # start parser in main loop again, thread is done
- timer = Timer(_update_process_step, info)
- timer.start(0)
+ return info
-def _update_process_step(info):
[EMAIL PROTECTED]()
+def update(epg):
"""
- Step in main loop for the parsing of the xmltv file. This function
- will be called in a Timer until everything is parsed.
+ Interface to source_xmltv.
"""
+ if not config.data_file and not config.grabber:
+ log.error('XMLTV gabber not configured.')
+ yield False
+ # _parse_xml is forced to be executed in a thread. This means that
+ # it always returns an InProgress object that needs to be yielded.
+ # When yield returns we need to call the InProgress object to get
+ # the result. If the result is None, the thread run into an error.
+ info = _parse_xml(epg)
+ yield info
+ info = info()
+ if not info:
+ yield False
+
t0 = time.time()
while info.node:
if info.node.name == "channel":
@@ -270,24 +279,8 @@
info.node = info.node.get_next()
if time.time() - t0 > 0.1:
# time to return to the main loop
- break
-
- if not info.node:
- info.epg.guide_changed()
- return False
-
- return True
+ yield kaa.notifier.YieldContinue
+ t0 = time.time()
-
-def update(epg):
- """
- Interface to source_xmltv. This function will start the update
- process.
- """
- if not config.data_file and not config.grabber:
- log.error('XMLTV gabber not configured.')
- epg.guide_changed()
- return False
- thread = Thread(_update_parse_xml_thread, epg)
- thread.start()
- return True
+ info.epg.guide_changed()
+ yield True
Modified: trunk/epg/src/sources/zap2it.py
==============================================================================
--- trunk/epg/src/sources/zap2it.py (original)
+++ trunk/epg/src/sources/zap2it.py Sat Feb 24 21:59:55 2007
@@ -41,6 +41,7 @@
# kaa imports
import kaa
+import kaa.notifier
from kaa.strutils import str_to_unicode
from config_zap2it import config
@@ -233,7 +234,8 @@
class UpdateInfo:
pass
-def _update_parse_xml_thread(epg, username, passwd, start, stop):
[EMAIL PROTECTED]('epg')
+def _parse_xml(epg, username, passwd, start, stop):
filename = request(username, passwd, ZAP2IT_HOST, ZAP2IT_URI, start, stop)
if not filename:
return
@@ -250,7 +252,8 @@
info = UpdateInfo()
info.doc = doc
- info.roots = [roots["stations"], roots["lineup"], roots["programs"],
roots["schedules"]]
+ info.roots = [roots["stations"], roots["lineup"], roots["programs"], \
+ roots["schedules"]]
info.node_names = ["station", "map", "program", "schedule"]
info.node = None
info.total = nprograms
@@ -260,34 +263,8 @@
info.progress_step = info.total / 100
info.t0 = time.time()
- timer = kaa.notifier.Timer(_update_process_step, info)
- timer.start(0)
-
-
-def _update_process_step(info):
- t0=time.time()
- if not info.node and info.roots:
- info.node = info.roots.pop(0).children
- info.cur_node_name = info.node_names.pop(0)
-
- while info.node:
- if info.node.name == info.cur_node_name:
- globals()["parse_%s" % info.cur_node_name](info.node, info)
-
- info.node = info.node.get_next()
- if time.time() - t0 > 0.1:
- break
-
- if not info.node and not info.roots:
- os.unlink(info.doc.name)
- info.doc.freeDoc()
- info.epg.guide_changed()
- log.info('Processed %d programs in %.02f seconds' %
(info.epg._num_programs, time.time() - info.t0))
- return False
-
- return True
-
[EMAIL PROTECTED]()
def update(epg, start = None, stop = None):
if not start:
# If start isn't specified, choose current time (rounded down to the
@@ -297,8 +274,34 @@
# If stop isn't specified, use 24 hours after start.
stop = start + (24 * 60 * 60)
- thread = kaa.notifier.Thread(_update_parse_xml_thread, epg,
- str(config.username), str(config.password),
- start, stop)
- thread.start()
- return True
+ # _parse_xml is forced to be executed in a thread. This means that
+ # it always returns an InProgress object that needs to be yielded.
+ # When yield returns we need to call the InProgress object to get
+ # the result. If the result is None, the thread run into an error.
+ info = _parse_xml(epg, str(config.username), str(config.password), start,
stop)
+ yield info
+ info = info()
+ if not info:
+ yield False
+
+ t0 = time.time()
+ while info.node or info.roots:
+ if not info.node:
+ info.node = info.roots.pop(0).children
+ info.cur_node_name = info.node_names.pop(0)
+
+ if info.node.name == info.cur_node_name:
+ globals()["parse_%s" % info.cur_node_name](info.node, info)
+
+ info.node = info.node.get_next()
+ if time.time() - t0 > 0.1:
+ # time to return to the main loop
+ yield kaa.notifier.YieldContinue
+ t0 = time.time()
+
+ os.unlink(info.doc.name)
+ info.doc.freeDoc()
+ info.epg.guide_changed()
+ log.info('Processed %d programs in %.02f seconds', info.epg._num_programs,
+ time.time() - info.t0)
+ yield True
-------------------------------------------------------------------------
Take Surveys. Earn Cash. Influence the Future of IT
Join SourceForge.net's Techsay panel and you'll get the chance to share your
opinions on IT & business topics through brief surveys-and earn cash
http://www.techsay.com/default.php?page=join.php&p=sourceforge&CID=DEVDEV
_______________________________________________
Freevo-cvslog mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/freevo-cvslog
