This callback is called when urlgrab request fails.
If grab is wrapped in a mirror group, only the mirror
group issues the callback.
---
urlgrabber/grabber.py | 20 +++++++++++++++++++-
urlgrabber/mirror.py | 11 ++++++++++-
2 files changed, 29 insertions(+), 2 deletions(-)
diff --git a/urlgrabber/grabber.py b/urlgrabber/grabber.py
index f0f4c6b..dc6a342 100644
--- a/urlgrabber/grabber.py
+++ b/urlgrabber/grabber.py
@@ -339,6 +339,15 @@ RETRY RELATED ARGUMENTS
but it cannot (without severe trickiness) prevent the exception
from being raised.
+ failfunc = None
+
+ The callback that gets called when urlgrab request fails.
+ If defined, urlgrab() calls it instead of raising URLGrabError.
+ Callback syntax is identical to failure_callback.
+
+ Contrary to failure_callback, it's called only once. It's primary
+ purpose is to use urlgrab() without a try/except block.
+
interrupt_callback = None
This callback is called if KeyboardInterrupt is received at any
@@ -1031,7 +1040,16 @@ class URLGrabber(object):
fo.close()
return filename
- return self._retry(opts, retryfunc, url, filename)
+ try: return self._retry(opts, retryfunc, url, filename)
+ except URLGrabError, e:
+ if hasattr(opts, 'mirror_group'):
+ raise e # MG tries other mirror
+
+ # urlgrab failed
+ if hasattr(opts, 'failfunc'):
+ opts.exception = e
+ return _callback(opts.failfunc, opts)
+ raise e
def urlread(self, url, limit=None, **kwargs):
"""read the url into a string, up to 'limit' bytes
diff --git a/urlgrabber/mirror.py b/urlgrabber/mirror.py
index 8731aed..222a519 100644
--- a/urlgrabber/mirror.py
+++ b/urlgrabber/mirror.py
@@ -91,6 +91,7 @@ import random
import thread # needed for locking to make this threadsafe
from grabber import URLGrabError, CallbackObject, DEBUG, _to_utf8
+from grabber import _callback
def _(st):
return st
@@ -391,6 +392,7 @@ class MirrorGroup:
grabber = mirrorchoice.get('grabber') or self.grabber
func_ref = getattr(grabber, func)
if DEBUG: DEBUG.info('MIRROR: trying %s -> %s', url, fullurl)
+ kwargs['mirror_group'] = self, gr, mirrorchoice
try:
return func_ref( *(fullurl,), **kwargs )
except URLGrabError, e:
@@ -406,7 +408,14 @@ class MirrorGroup:
kw = dict(kwargs)
kw['filename'] = filename
func = 'urlgrab'
- return self._mirror_try(func, url, kw)
+ try: return self._mirror_try(func, url, kw)
+ except URLGrabError, e:
+ opts = CallbackObject(url=url, exception=e, **kw)
+
+ # urlgrab failed
+ if hasattr(opts, 'failfunc'):
+ return _callback(opts.failfunc, opts)
+ raise e
def urlopen(self, url, **kwargs):
kw = dict(kwargs)
--
1.7.4.4
_______________________________________________
Yum-devel mailing list
Yum-devel@lists.baseurl.org
http://lists.baseurl.org/mailman/listinfo/yum-devel
