Good day,
I am using Wasabi as backend and every now and then there is a seemingly
"temporary" backend error, which causes the filesystem to be unmounted.
Is this expected behaviour or should s3ql simply wait until the backend
becomes available again?
I have seen two distinct errors so far:
TemporarilyUnavailable: Resources for this operation are temporarily
unavailable. Please try again later.
TemporarilyUnavailable: Storage service is temporarily not available.
Please try again later (name: UnexpectedError | detail: ConnWaitError)
Logs attached for context.
Best,
Bas
--
You received this message because you are subscribed to the Google Groups
"s3ql" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
To view this discussion on the web visit
https://groups.google.com/d/msgid/s3ql/49c6e84c-4602-4334-8ba6-b519a0a8aa9en%40googlegroups.com.
2021-11-08 13:30:34.230 1134:Trio worker thread 13 s3ql.metadata.cycle_metadata: Backing up old metadata...
2021-11-09 06:31:01.396 1134:Thread-2 s3ql.mount.exchook: Unhandled exception in thread, terminating
Traceback (most recent call last):
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/mount.py", line 58, in run_with_except_hook
run_old(*args, **kw)
File "/usr/lib/python3.8/threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/block_cache.py", line 319, in _upload_loop
self._do_upload(*tmp)
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/block_cache.py", line 384, in _do_upload
obj_size = backend.perform_write(do_write, 's3ql_data_%d'
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/common.py", line 108, in wrapped
return method(*a, **kw)
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/common.py", line 279, in perform_write
return fn(fh)
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/comprenc.py", line 554, in __exit__
self.close()
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/comprenc.py", line 548, in close
self.fh.close()
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/common.py", line 108, in wrapped
return method(*a, **kw)
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/s3c.py", line 912, in close
resp = self.backend._do_request('PUT', '/%s%s' % (self.backend.prefix, self.key),
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/s3c.py", line 531, in _do_request
self._parse_error_response(resp)
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/s3c.py", line 565, in _parse_error_response
raise get_S3Error(tree.findtext('Code'), tree.findtext('Message'), resp.headers)
s3ql.backends.s3c.S3Error: TemporarilyUnavailable: Resources for this operation are temporarily unavailable. Please try again later.
2021-11-09 06:31:01.409 1134:Thread-1 s3ql.mount.exchook: Unhandled exception in thread, terminating
Traceback (most recent call last):
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/mount.py", line 58, in run_with_except_hook
run_old(*args, **kw)
File "/usr/lib/python3.8/threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/block_cache.py", line 319, in _upload_loop
self._do_upload(*tmp)
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/block_cache.py", line 384, in _do_upload
obj_size = backend.perform_write(do_write, 's3ql_data_%d'
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/common.py", line 108, in wrapped
return method(*a, **kw)
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/common.py", line 279, in perform_write
return fn(fh)
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/comprenc.py", line 554, in __exit__
self.close()
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/comprenc.py", line 548, in close
self.fh.close()
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/common.py", line 108, in wrapped
return method(*a, **kw)
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/s3c.py", line 912, in close
resp = self.backend._do_request('PUT', '/%s%s' % (self.backend.prefix, self.key),
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/s3c.py", line 531, in _do_request
self._parse_error_response(resp)
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/s3c.py", line 565, in _parse_error_response
raise get_S3Error(tree.findtext('Code'), tree.findtext('Message'), resp.headers)
s3ql.backends.s3c.S3Error: TemporarilyUnavailable: Resources for this operation are temporarily unavailable. Please try again later.
2021-11-09 06:31:01.445 1134:MainThread s3ql.mount.unmount: Unmounting file system...
2021-11-09 06:31:01.446 1134:Thread-2 root.excepthook: Uncaught top-level exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/mount.py", line 58, in run_with_except_hook
run_old(*args, **kw)
File "/usr/lib/python3.8/threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/block_cache.py", line 319, in _upload_loop
self._do_upload(*tmp)
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/block_cache.py", line 384, in _do_upload
obj_size = backend.perform_write(do_write, 's3ql_data_%d'
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/common.py", line 108, in wrapped
return method(*a, **kw)
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/common.py", line 279, in perform_write
return fn(fh)
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/comprenc.py", line 554, in __exit__
self.close()
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/comprenc.py", line 548, in close
self.fh.close()
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/common.py", line 108, in wrapped
return method(*a, **kw)
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/s3c.py", line 912, in close
resp = self.backend._do_request('PUT', '/%s%s' % (self.backend.prefix, self.key),
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/s3c.py", line 531, in _do_request
self._parse_error_response(resp)
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/s3c.py", line 565, in _parse_error_response
raise get_S3Error(tree.findtext('Code'), tree.findtext('Message'), resp.headers)
s3ql.backends.s3c.S3Error: TemporarilyUnavailable: Resources for this operation are temporarily unavailable. Please try again later.
...
2021-12-09 06:59:20.521 14040:Trio worker thread 447 s3ql.metadata.cycle_metadata: Backing up old metadata...
2021-12-10 02:46:43.608 14040:Thread-2 s3ql.mount.exchook: Unhandled exception in thread, terminating
Traceback (most recent call last):
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/mount.py", line 58, in run_with_except_hook
run_old(*args, **kw)
File "/usr/lib/python3.8/threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/block_cache.py", line 319, in _upload_loop
self._do_upload(*tmp)
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/block_cache.py", line 384, in _do_upload
obj_size = backend.perform_write(do_write, 's3ql_data_%d'
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/common.py", line 108, in wrapped
return method(*a, **kw)
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/common.py", line 279, in perform_write
return fn(fh)
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/comprenc.py", line 554, in __exit__
self.close()
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/comprenc.py", line 548, in close
self.fh.close()
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/common.py", line 108, in wrapped
return method(*a, **kw)
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/s3c.py", line 912, in close
resp = self.backend._do_request('PUT', '/%s%s' % (self.backend.prefix, self.key),
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/s3c.py", line 531, in _do_request
self._parse_error_response(resp)
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/s3c.py", line 565, in _parse_error_response
raise get_S3Error(tree.findtext('Code'), tree.findtext('Message'), resp.headers)
s3ql.backends.s3c.S3Error: TemporarilyUnavailable: Storage service is temporarily not available. Please try again later (name: UnexpectedError | detail: ConnWaitError)
2021-12-10 02:46:43.688 14040:Thread-2 root.excepthook: Uncaught top-level exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/mount.py", line 58, in run_with_except_hook
run_old(*args, **kw)
File "/usr/lib/python3.8/threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/block_cache.py", line 319, in _upload_loop
self._do_upload(*tmp)
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/block_cache.py", line 384, in _do_upload
obj_size = backend.perform_write(do_write, 's3ql_data_%d'
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/common.py", line 108, in wrapped
return method(*a, **kw)
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/common.py", line 279, in perform_write
return fn(fh)
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/comprenc.py", line 554, in __exit__
self.close()
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/comprenc.py", line 548, in close
self.fh.close()
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/common.py", line 108, in wrapped
return method(*a, **kw)
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/s3c.py", line 912, in close
resp = self.backend._do_request('PUT', '/%s%s' % (self.backend.prefix, self.key),
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/s3c.py", line 531, in _do_request
self._parse_error_response(resp)
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/s3c.py", line 565, in _parse_error_response
raise get_S3Error(tree.findtext('Code'), tree.findtext('Message'), resp.headers)
s3ql.backends.s3c.S3Error: TemporarilyUnavailable: Storage service is temporarily not available. Please try again later (name: UnexpectedError | detail: ConnWaitError)
2021-12-10 02:46:43.691 14040:MainThread s3ql.mount.unmount: Unmounting file system...
...
2021-12-16 10:53:26.836 453913:Trio worker thread 45 s3ql.metadata.cycle_metadata: Backing up old metadata...
2021-12-17 01:37:49.624 453913:Thread-2 s3ql.mount.exchook: Unhandled exception in thread, terminating
Traceback (most recent call last):
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/mount.py", line 58, in run_with_except_hook
run_old(*args, **kw)
File "/usr/lib/python3.8/threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/block_cache.py", line 319, in _upload_loop
self._do_upload(*tmp)
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/block_cache.py", line 384, in _do_upload
obj_size = backend.perform_write(do_write, 's3ql_data_%d'
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/common.py", line 108, in wrapped
return method(*a, **kw)
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/common.py", line 279, in perform_write
return fn(fh)
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/comprenc.py", line 554, in __exit__
self.close()
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/comprenc.py", line 548, in close
self.fh.close()
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/common.py", line 108, in wrapped
return method(*a, **kw)
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/s3c.py", line 912, in close
resp = self.backend._do_request('PUT', '/%s%s' % (self.backend.prefix, self.key),
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/s3c.py", line 531, in _do_request
self._parse_error_response(resp)
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/s3c.py", line 565, in _parse_error_response
raise get_S3Error(tree.findtext('Code'), tree.findtext('Message'), resp.headers)
s3ql.backends.s3c.S3Error: TemporarilyUnavailable: Storage service is temporarily not available. Please try again later (name: UnexpectedError | detail: ConnWaitError)
2021-12-17 01:37:49.721 453913:Thread-2 root.excepthook: Uncaught top-level exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/mount.py", line 58, in run_with_except_hook
run_old(*args, **kw)
File "/usr/lib/python3.8/threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/block_cache.py", line 319, in _upload_loop
self._do_upload(*tmp)
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/block_cache.py", line 384, in _do_upload
obj_size = backend.perform_write(do_write, 's3ql_data_%d'
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/common.py", line 108, in wrapped
return method(*a, **kw)
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/common.py", line 279, in perform_write
return fn(fh)
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/comprenc.py", line 554, in __exit__
self.close()
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/comprenc.py", line 548, in close
self.fh.close()
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/common.py", line 108, in wrapped
return method(*a, **kw)
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/s3c.py", line 912, in close
resp = self.backend._do_request('PUT', '/%s%s' % (self.backend.prefix, self.key),
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/s3c.py", line 531, in _do_request
self._parse_error_response(resp)
File "/usr/local/lib/python3.8/dist-packages/s3ql-3.7.3-py3.8-linux-x86_64.egg/s3ql/backends/s3c.py", line 565, in _parse_error_response
raise get_S3Error(tree.findtext('Code'), tree.findtext('Message'), resp.headers)
s3ql.backends.s3c.S3Error: TemporarilyUnavailable: Storage service is temporarily not available. Please try again later (name: UnexpectedError | detail: ConnWaitError)
2021-12-17 01:37:49.722 453913:MainThread s3ql.mount.unmount: Unmounting file system...
