This callback is called when urlgrab request fails.
If grab is wrapped in a mirror group, only the mirror
group issues the callback.
---
urlgrabber/grabber.py | 25 ++++++++++++++++++++++++-
urlgrabber/mirror.py | 9 +++++++--
2 files changed, 31 insertions(+), 3 deletions(-)
diff --git a/urlgrabber/grabber.py b/urlgrabber/grabber.py
index 5614bcd..e4a7eba 100644
--- a/urlgrabber/grabber.py
+++ b/urlgrabber/grabber.py
@@ -339,6 +339,15 @@ RETRY RELATED ARGUMENTS
but it cannot (without severe trickiness) prevent the exception
from being raised.
+ failfunc = None
+
+ The callback that gets called when urlgrab request fails.
+ If defined, urlgrab() calls it instead of raising URLGrabError.
+ Callback syntax is identical to failure_callback.
+
+ Contrary to failure_callback, it's called only once. It's primary
+ purpose is to use urlgrab() without a try/except block.
+
interrupt_callback = None
This callback is called if KeyboardInterrupt is received at any
@@ -838,6 +847,7 @@ class URLGrabberOptions:
self.retry = None
self.retrycodes = [-1,2,4,5,6,7]
self.checkfunc = None
+ self.failfunc = _do_raise
self.copy_local = 0
self.close_connection = 0
self.range = None
@@ -891,6 +901,15 @@ class URLGrabberOptions:
s = s + indent + '}'
return s
+def _do_raise(obj):
+ raise obj.exception
+
+def _run_callback(cb, obj):
+ if callable(cb):
+ return cb(obj)
+ cb, arg, karg = cb
+ return cb(obj, *arg, **karg)
+
class URLGrabber(object):
"""Provides easy opening of URLs with a variety of options.
@@ -1023,7 +1042,11 @@ class URLGrabber(object):
fo.close()
return filename
- return self._retry(opts, retryfunc, url, filename)
+ try:
+ return self._retry(opts, retryfunc, url, filename)
+ except URLGrabError, e:
+ opts.exception = e
+ return _run_callback(opts.failfunc, opts)
def urlread(self, url, limit=None, **kwargs):
"""read the url into a string, up to 'limit' bytes
diff --git a/urlgrabber/mirror.py b/urlgrabber/mirror.py
index 8731aed..575ef7b 100644
--- a/urlgrabber/mirror.py
+++ b/urlgrabber/mirror.py
@@ -91,6 +91,7 @@ import random
import thread # needed for locking to make this threadsafe
from grabber import URLGrabError, CallbackObject, DEBUG, _to_utf8
+from grabber import _run_callback, _do_raise
def _(st):
return st
@@ -254,7 +255,7 @@ class MirrorGroup:
# if these values are found in **kwargs passed to one of the urlXXX
# methods, they will be stripped before getting passed on to the
# grabber
- options = ['default_action', 'failure_callback']
+ options = ['default_action', 'failure_callback', 'failfunc']
def _process_kwargs(self, kwargs):
self.failure_callback = kwargs.get('failure_callback')
@@ -406,7 +407,11 @@ class MirrorGroup:
kw = dict(kwargs)
kw['filename'] = filename
func = 'urlgrab'
- return self._mirror_try(func, url, kw)
+ try:
+ return self._mirror_try(func, url, kw)
+ except URLGrabError, e:
+ obj = CallbackObject(url=url, filename=filename, exception=e,
**kwargs)
+ return _run_callback(kwargs.get('failfunc', _do_raise), obj)
def urlopen(self, url, **kwargs):
kw = dict(kwargs)
--
1.7.4.4
_______________________________________________
Yum-devel mailing list
Yum-devel@lists.baseurl.org
http://lists.baseurl.org/mailman/listinfo/yum-devel
