Hi,
Please do not send copies to my personal address, I am reading this list
(believe it or not!).
On Jul 22 2015, [email protected] wrote:
>> Would you mind trying out the attached patch (while setting tcp-timeout
>> back to the smaller value)? This should cause S3QL to ignore 404 when
>> deleting objects if and only if there have been prior attempts to delete
>> the object.
>
> I'm also interested about this patch.
> Can you send me the patch please ?
That's odd, it seems my email isn't available on the mailing list
archive. Here it is again.
Best,
-Nikolaus
--
GPG encrypted emails preferred. Key id: 0xD113FCAC3C4E599F
Fingerprint: ED31 791B 2C5C 1613 AF38 8B8A D113 FCAC 3C4E 599F
»Time flies like an arrow, fruit flies like a Banana.«
--
You received this message because you are subscribed to the Google Groups
"s3ql" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
For more options, visit https://groups.google.com/d/optout.
diff --git a/src/s3ql/backends/common.py b/src/s3ql/backends/common.py
--- a/src/s3ql/backends/common.py
+++ b/src/s3ql/backends/common.py
@@ -81,14 +81,19 @@
'''Wrap *method* for retrying on some exceptions
If *method* raises an exception for which the instance's
- `is_temp_failure(exc)` method is true, the *method* is called again
- at increasing intervals. If this persists for more than `RETRY_TIMEOUT`
- seconds, the most-recently caught exception is re-raised.
+ `is_temp_failure(exc)` method is true, the *method* is called again at
+ increasing intervals. If this persists for more than `RETRY_TIMEOUT`
+ seconds, the most-recently caught exception is re-raised. If the
+ method defines a keyword parameter *is_retry*, then this parameter
+ will be set to True whenever the function is retried.
'''
if inspect.isgeneratorfunction(method):
raise TypeError('Wrapping a generator function is pointless')
+ sig = inspect.signature(method)
+ has_is_retry = 'is_retry' in sig.parameters
+
@wraps(method)
def wrapped(*a, **kw):
self = a[0]
@@ -96,6 +101,8 @@
waited = 0
retries = 0
while True:
+ if has_is_retry:
+ kw['is_retry'] = (retries > 0)
try:
return method(*a, **kw)
except Exception as exc:
diff --git a/src/s3ql/backends/s3c.py b/src/s3ql/backends/s3c.py
--- a/src/s3ql/backends/s3c.py
+++ b/src/s3ql/backends/s3c.py
@@ -194,13 +194,15 @@
@retry
@copy_ancestor_docstring
- def delete(self, key, force=False):
+ def delete(self, key, force=False, is_retry=False):
log.debug('started with %s', key)
try:
resp = self._do_request('DELETE', '/%s%s' % (self.prefix, key))
self._assert_empty_response(resp)
except NoSuchKeyError:
- if force:
+ # Server may have deleted the object even though we did not
+ # receive the response.
+ if force or is_retry:
pass
else:
raise NoSuchObject(key)
diff --git a/src/s3ql/backends/swift.py b/src/s3ql/backends/swift.py
--- a/src/s3ql/backends/swift.py
+++ b/src/s3ql/backends/swift.py
@@ -379,7 +379,7 @@
@retry
@copy_ancestor_docstring
- def delete(self, key, force=False):
+ def delete(self, key, force=False, is_retry=False):
if key.endswith(TEMP_SUFFIX):
raise ValueError('Keys must not end with %s' % TEMP_SUFFIX)
log.debug('started with %s', key)
@@ -387,7 +387,9 @@
resp = self._do_request('DELETE', '/%s%s' % (self.prefix, key))
self._assert_empty_response(resp)
except HTTPError as exc:
- if exc.status == 404 and not force:
+ # Server may have deleted the object even though we did not
+ # receive the response.
+ if exc.status == 404 and not (force or is_retry):
raise NoSuchObject(key)
elif exc.status != 404:
raise
diff --git a/tests/t1_retry.py b/tests/t1_retry.py
--- a/tests/t1_retry.py
+++ b/tests/t1_retry.py
@@ -50,6 +50,14 @@
yield i
+ @retry
+ def test_is_retry(self, is_retry=False):
+ assert is_retry == (self.count != 0)
+ if self.count == self.succeed_on:
+ return True
+ self.count += 1
+ raise TemporaryProblem()
+
def test_retry():
inst = NthAttempt(3)
@@ -59,6 +67,10 @@
inst = NthAttempt(3)
assert list(inst.list_stuff(10)) == list(range(10))
+def test_is_retry():
+ inst = NthAttempt(3)
+ assert inst.test_is_retry()
+
def test_logging():
inst = NthAttempt(6)
with catch_logmsg(r'^Encountered %s \(%s\), retrying ',
