Hi,
I installed the s3ql that's bundled with Ubuntu 16.04, but I see now
that this is a fairly old release - v2.15. I've been copying just over
100GB of data for a couple of days to an Amazon S3 backed s3ql
filsystem (yes, my uploads are slow), and now I'd like to finish the
process and unmount. s3ql.umount has been working for something like 6
hours now, and the filesystem is still mounted.
I don't think s3ql is hanging, tcpdump shows quite a bit of traffic to
Amazon's servers, there's a steady upload rate. Is this cached data
being uploaded after the "cp" process has finished? The relevant
~/.s3ql/...-cache folder (I only have one) contains 34GB and over 6000
files, does this matter? It's not shrinking much. Is there a better
way to monitor what's going on than looking at the size of the cache
folder?
Would upgrading change this, or is this to be expected?
I'm attaching the relevant parts of mount.log, everything since the
last mount. mount.s3ql_crit.log is empty. I forced a python stacktrace
with the setfattr command to be on the safe side. I also tried to
enable debug logging with "s3qlctrl log ~/storage/ debug", but that
command has been hanging for the past hour or so, and I haven't seen
any effects of running it.
Thanks.
Regards, Ketil
--
You received this message because you are subscribed to the Google Groups
"s3ql" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
For more options, visit https://groups.google.com/d/optout.
2017-02-03 19:30:32.770 5387:MainThread s3ql.mount.determine_threads: Using 8 upload threads.
2017-02-03 19:30:32.771 5387:MainThread s3ql.mount.main: Autodetected 65480 file descriptors available for cache entries
2017-02-03 19:30:33.935 5387:MainThread s3ql.mount.get_metadata: Using cached metadata.
2017-02-03 19:30:33.938 5387:MainThread s3ql.mount.main: Setting cache size to 47271 MB
2017-02-03 19:30:33.939 5387:MainThread s3ql.mount.main: Mounting filesystem...
2017-02-03 19:30:33.955 5396:MainThread s3ql.daemonize.detach_process_context: Daemonizing, new PID is 5399
2017-02-04 19:30:34.103 5399:Metadata-Upload-Thread s3ql.mount.run: Dumping metadata...
2017-02-04 19:30:34.104 5399:Metadata-Upload-Thread s3ql.metadata.dump_metadata: ..objects..
2017-02-04 19:30:34.125 5399:Metadata-Upload-Thread s3ql.metadata.dump_metadata: ..blocks..
2017-02-04 19:30:34.149 5399:Metadata-Upload-Thread s3ql.metadata.dump_metadata: ..inodes..
2017-02-04 19:30:34.199 5399:Metadata-Upload-Thread s3ql.metadata.dump_metadata: ..inode_blocks..
2017-02-04 19:30:34.218 5399:Metadata-Upload-Thread s3ql.metadata.dump_metadata: ..symlink_targets..
2017-02-04 19:30:34.219 5399:Metadata-Upload-Thread s3ql.metadata.dump_metadata: ..names..
2017-02-04 19:30:34.234 5399:Metadata-Upload-Thread s3ql.metadata.dump_metadata: ..contents..
2017-02-04 19:30:34.256 5399:Metadata-Upload-Thread s3ql.metadata.dump_metadata: ..ext_attributes..
2017-02-04 19:30:36.888 5399:Metadata-Upload-Thread s3ql.metadata.upload_metadata: Compressing and uploading metadata...
2017-02-04 19:30:58.359 5399:Metadata-Upload-Thread s3ql.metadata.upload_metadata: Wrote 1.47 MiB of compressed metadata.
2017-02-04 19:30:58.360 5399:Metadata-Upload-Thread s3ql.metadata.upload_metadata: Cycling metadata backups...
2017-02-04 19:30:58.361 5399:Metadata-Upload-Thread s3ql.metadata.cycle_metadata: Backing up old metadata...
2017-02-05 19:31:06.642 5399:Metadata-Upload-Thread s3ql.mount.run: Dumping metadata...
2017-02-05 19:31:06.646 5399:Metadata-Upload-Thread s3ql.metadata.dump_metadata: ..objects..
2017-02-05 19:31:06.716 5399:Metadata-Upload-Thread s3ql.metadata.dump_metadata: ..blocks..
2017-02-05 19:31:06.801 5399:Metadata-Upload-Thread s3ql.metadata.dump_metadata: ..inodes..
2017-02-05 19:31:06.910 5399:Metadata-Upload-Thread s3ql.metadata.dump_metadata: ..inode_blocks..
2017-02-05 19:31:06.961 5399:Metadata-Upload-Thread s3ql.metadata.dump_metadata: ..symlink_targets..
2017-02-05 19:31:06.962 5399:Metadata-Upload-Thread s3ql.metadata.dump_metadata: ..names..
2017-02-05 19:31:06.983 5399:Metadata-Upload-Thread s3ql.metadata.dump_metadata: ..contents..
2017-02-05 19:31:07.017 5399:Metadata-Upload-Thread s3ql.metadata.dump_metadata: ..ext_attributes..
2017-02-05 19:31:11.641 5399:Metadata-Upload-Thread s3ql.metadata.upload_metadata: Compressing and uploading metadata...
2017-02-05 19:31:49.681 5399:Metadata-Upload-Thread s3ql.metadata.upload_metadata: Wrote 2.11 MiB of compressed metadata.
2017-02-05 19:31:49.682 5399:Metadata-Upload-Thread s3ql.metadata.upload_metadata: Cycling metadata backups...
2017-02-05 19:31:49.682 5399:Metadata-Upload-Thread s3ql.metadata.cycle_metadata: Backing up old metadata...
2017-02-05 23:33:34.164 5399:Dummy-22 llfuse.stacktrace:
# ThreadID: 140535868147456
pyapi.py:522, in stacktrace
for filename, lineno, name, line in traceback.extract_stack(frame):
# ThreadID: 140535851362048
fs.py:243, in setxattr
self.cache.clear()
block_cache.py:928, in clear
self.expire() # Releases global lock
block_cache.py:824, in expire
self.upload(el) # Releases global lock
block_cache.py:554, in upload
self._queue_upload((el, obj_id))
block_cache.py:591, in _queue_upload
if self.to_upload.put(obj, timeout=5):
block_cache.py:74, in put
if not self.cv.wait(timeout):
threading.py:297, in wait
gotit = waiter.acquire(True, timeout)
# ThreadID: 140537451157248
threading.py:882, in _bootstrap
self._bootstrap_inner()
threading.py:914, in _bootstrap_inner
self.run()
mount.py:64, in run_with_except_hook
run_old(*args, **kw)
threading.py:862, in run
self._target(*self._args, **self._kwargs)
block_cache.py:404, in _upload_loop
self._do_upload(*tmp)
block_cache.py:431, in _do_upload
% obj_id).get_obj_size()
common.py:107, in wrapped
return method(*a, **kw)
common.py:337, in perform_write
return fn(fh)
comprenc.py:551, in __exit__
self.close()
comprenc.py:545, in close
self.fh.close()
comprenc.py:710, in close
self.fh.close()
common.py:107, in wrapped
return method(*a, **kw)
s3c.py:910, in close
headers=self.headers, body=self.fh)
s3c.py:440, in _do_request
query_string=query_string, body=body)
s3c.py:683, in _send_request
copyfileobj(body, self.conn, BUFSIZE)
shutil.py:76, in copyfileobj
fdst.write(buf)
__init__.py:653, in write
eval_coroutine(self.co_write(buf), self.timeout)
__init__.py:1396, in eval_coroutine
if not next(crt).poll(timeout=timeout):
__init__.py:115, in poll
return bool(poll.poll(timeout*1000)) # convert to ms
# ThreadID: 140537476335360
threading.py:882, in _bootstrap
self._bootstrap_inner()
threading.py:914, in _bootstrap_inner
self.run()
mount.py:64, in run_with_except_hook
run_old(*args, **kw)
threading.py:862, in run
self._target(*self._args, **self._kwargs)
block_cache.py:404, in _upload_loop
self._do_upload(*tmp)
block_cache.py:431, in _do_upload
% obj_id).get_obj_size()
common.py:107, in wrapped
return method(*a, **kw)
common.py:337, in perform_write
return fn(fh)
comprenc.py:551, in __exit__
self.close()
comprenc.py:545, in close
self.fh.close()
comprenc.py:710, in close
self.fh.close()
common.py:107, in wrapped
return method(*a, **kw)
s3c.py:910, in close
headers=self.headers, body=self.fh)
s3c.py:440, in _do_request
query_string=query_string, body=body)
s3c.py:683, in _send_request
copyfileobj(body, self.conn, BUFSIZE)
shutil.py:76, in copyfileobj
fdst.write(buf)
__init__.py:653, in write
eval_coroutine(self.co_write(buf), self.timeout)
__init__.py:1396, in eval_coroutine
if not next(crt).poll(timeout=timeout):
__init__.py:115, in poll
return bool(poll.poll(timeout*1000)) # convert to ms
# ThreadID: 140536941889280
threading.py:882, in _bootstrap
self._bootstrap_inner()
threading.py:914, in _bootstrap_inner
self.run()
mount.py:64, in run_with_except_hook
run_old(*args, **kw)
threading.py:862, in run
self._target(*self._args, **self._kwargs)
block_cache.py:681, in _removal_loop
tmp = self.to_remove.get(block=len(ids)==0)
queue.py:164, in get
self.not_empty.wait()
threading.py:293, in wait
waiter.acquire()
# ThreadID: 140536379840256
threading.py:882, in _bootstrap
self._bootstrap_inner()
threading.py:914, in _bootstrap_inner
self.run()
mount.py:64, in run_with_except_hook
run_old(*args, **kw)
threading.py:862, in run
self._target(*self._args, **self._kwargs)
queue.py:164, in get
self.not_empty.wait()
threading.py:293, in wait
waiter.acquire()
# ThreadID: 140537484728064
threading.py:882, in _bootstrap
self._bootstrap_inner()
threading.py:914, in _bootstrap_inner
self.run()
mount.py:64, in run_with_except_hook
run_old(*args, **kw)
threading.py:862, in run
self._target(*self._args, **self._kwargs)
block_cache.py:404, in _upload_loop
self._do_upload(*tmp)
block_cache.py:431, in _do_upload
% obj_id).get_obj_size()
common.py:107, in wrapped
return method(*a, **kw)
common.py:337, in perform_write
return fn(fh)
block_cache.py:416, in do_write
fh.write(buf)
comprenc.py:530, in write
buf = self.compr.compress(data)
# ThreadID: 140537615447808
mount.s3ql:9, in <module>
load_entry_point('s3ql==2.15', 'console_scripts', 'mount.s3ql')()
mount.py:214, in main
llfuse.main(options.single)
# ThreadID: 140536933496576
threading.py:882, in _bootstrap
self._bootstrap_inner()
threading.py:914, in _bootstrap_inner
self.run()
mount.py:64, in run_with_except_hook
run_old(*args, **kw)
threading.py:862, in run
self._target(*self._args, **self._kwargs)
block_cache.py:681, in _removal_loop
tmp = self.to_remove.get(block=len(ids)==0)
queue.py:164, in get
self.not_empty.wait()
threading.py:293, in wait
waiter.acquire()
# ThreadID: 140536413411072
threading.py:882, in _bootstrap
self._bootstrap_inner()
threading.py:914, in _bootstrap_inner
self.run()
mount.py:64, in run_with_except_hook
run_old(*args, **kw)
threading.py:862, in run
self._target(*self._args, **self._kwargs)
block_cache.py:681, in _removal_loop
tmp = self.to_remove.get(block=len(ids)==0)
queue.py:164, in get
self.not_empty.wait()
threading.py:293, in wait
waiter.acquire()
# ThreadID: 140537459549952
threading.py:882, in _bootstrap
self._bootstrap_inner()
threading.py:914, in _bootstrap_inner
self.run()
mount.py:64, in run_with_except_hook
run_old(*args, **kw)
threading.py:862, in run
self._target(*self._args, **self._kwargs)
block_cache.py:404, in _upload_loop
self._do_upload(*tmp)
block_cache.py:431, in _do_upload
% obj_id).get_obj_size()
common.py:107, in wrapped
return method(*a, **kw)
common.py:337, in perform_write
return fn(fh)
comprenc.py:551, in __exit__
self.close()
comprenc.py:545, in close
self.fh.close()
comprenc.py:710, in close
self.fh.close()
common.py:107, in wrapped
return method(*a, **kw)
s3c.py:910, in close
headers=self.headers, body=self.fh)
s3c.py:440, in _do_request
query_string=query_string, body=body)
s3c.py:683, in _send_request
copyfileobj(body, self.conn, BUFSIZE)
shutil.py:76, in copyfileobj
fdst.write(buf)
__init__.py:653, in write
eval_coroutine(self.co_write(buf), self.timeout)
__init__.py:1396, in eval_coroutine
if not next(crt).poll(timeout=timeout):
__init__.py:115, in poll
return bool(poll.poll(timeout*1000)) # convert to ms
# ThreadID: 140536916711168
threading.py:882, in _bootstrap
self._bootstrap_inner()
threading.py:914, in _bootstrap_inner
self.run()
mount.py:64, in run_with_except_hook
run_old(*args, **kw)
threading.py:862, in run
self._target(*self._args, **self._kwargs)
block_cache.py:681, in _removal_loop
tmp = self.to_remove.get(block=len(ids)==0)
queue.py:164, in get
self.not_empty.wait()
threading.py:293, in wait
waiter.acquire()
# ThreadID: 140536396625664
threading.py:882, in _bootstrap
self._bootstrap_inner()
threading.py:914, in _bootstrap_inner
self.run()
mount.py:64, in run_with_except_hook
run_old(*args, **kw)
mount.py:624, in run
self.event.wait(self.interval)
threading.py:549, in wait
signaled = self._cond.wait(timeout)
threading.py:297, in wait
gotit = waiter.acquire(True, timeout)
# ThreadID: 140537493120768
threading.py:882, in _bootstrap
self._bootstrap_inner()
threading.py:914, in _bootstrap_inner
self.run()
mount.py:64, in run_with_except_hook
run_old(*args, **kw)
threading.py:862, in run
self._target(*self._args, **self._kwargs)
block_cache.py:404, in _upload_loop
self._do_upload(*tmp)
block_cache.py:431, in _do_upload
% obj_id).get_obj_size()
common.py:107, in wrapped
return method(*a, **kw)
common.py:337, in perform_write
return fn(fh)
comprenc.py:551, in __exit__
self.close()
comprenc.py:545, in close
self.fh.close()
comprenc.py:710, in close
self.fh.close()
common.py:107, in wrapped
return method(*a, **kw)
s3c.py:910, in close
headers=self.headers, body=self.fh)
s3c.py:440, in _do_request
query_string=query_string, body=body)
s3c.py:683, in _send_request
copyfileobj(body, self.conn, BUFSIZE)
shutil.py:76, in copyfileobj
fdst.write(buf)
__init__.py:653, in write
eval_coroutine(self.co_write(buf), self.timeout)
__init__.py:1396, in eval_coroutine
if not next(crt).poll(timeout=timeout):
__init__.py:115, in poll
return bool(poll.poll(timeout*1000)) # convert to ms
# ThreadID: 140536405018368
threading.py:882, in _bootstrap
self._bootstrap_inner()
threading.py:914, in _bootstrap_inner
self.run()
mount.py:64, in run_with_except_hook
run_old(*args, **kw)
threading.py:862, in run
self._target(*self._args, **self._kwargs)
block_cache.py:681, in _removal_loop
tmp = self.to_remove.get(block=len(ids)==0)
queue.py:164, in get
self.not_empty.wait()
threading.py:293, in wait
waiter.acquire()
# ThreadID: 140537467942656
threading.py:882, in _bootstrap
self._bootstrap_inner()
threading.py:914, in _bootstrap_inner
self.run()
mount.py:64, in run_with_except_hook
run_old(*args, **kw)
threading.py:862, in run
self._target(*self._args, **self._kwargs)
block_cache.py:404, in _upload_loop
self._do_upload(*tmp)
block_cache.py:431, in _do_upload
% obj_id).get_obj_size()
common.py:107, in wrapped
return method(*a, **kw)
common.py:337, in perform_write
return fn(fh)
comprenc.py:551, in __exit__
self.close()
comprenc.py:545, in close
self.fh.close()
comprenc.py:710, in close
self.fh.close()
common.py:107, in wrapped
return method(*a, **kw)
s3c.py:910, in close
headers=self.headers, body=self.fh)
s3c.py:440, in _do_request
query_string=query_string, body=body)
s3c.py:683, in _send_request
copyfileobj(body, self.conn, BUFSIZE)
shutil.py:76, in copyfileobj
fdst.write(buf)
__init__.py:653, in write
eval_coroutine(self.co_write(buf), self.timeout)
__init__.py:1396, in eval_coroutine
if not next(crt).poll(timeout=timeout):
__init__.py:115, in poll
return bool(poll.poll(timeout*1000)) # convert to ms
# ThreadID: 140536950281984
threading.py:882, in _bootstrap
self._bootstrap_inner()
threading.py:914, in _bootstrap_inner
self.run()
mount.py:64, in run_with_except_hook
run_old(*args, **kw)
threading.py:862, in run
self._target(*self._args, **self._kwargs)
block_cache.py:681, in _removal_loop
tmp = self.to_remove.get(block=len(ids)==0)
queue.py:164, in get
self.not_empty.wait()
threading.py:293, in wait
waiter.acquire()
# ThreadID: 140536421803776
threading.py:882, in _bootstrap
self._bootstrap_inner()
threading.py:914, in _bootstrap_inner
self.run()
mount.py:64, in run_with_except_hook
run_old(*args, **kw)
threading.py:862, in run
self._target(*self._args, **self._kwargs)
block_cache.py:681, in _removal_loop
tmp = self.to_remove.get(block=len(ids)==0)
queue.py:164, in get
self.not_empty.wait()
threading.py:293, in wait
waiter.acquire()
# ThreadID: 140536430196480
threading.py:882, in _bootstrap
self._bootstrap_inner()
threading.py:914, in _bootstrap_inner
self.run()
mount.py:64, in run_with_except_hook
run_old(*args, **kw)
threading.py:862, in run
self._target(*self._args, **self._kwargs)
block_cache.py:681, in _removal_loop
tmp = self.to_remove.get(block=len(ids)==0)
queue.py:164, in get
self.not_empty.wait()
threading.py:293, in wait
waiter.acquire()
# ThreadID: 140536958674688
threading.py:882, in _bootstrap
self._bootstrap_inner()
threading.py:914, in _bootstrap_inner
self.run()
mount.py:64, in run_with_except_hook
run_old(*args, **kw)
threading.py:862, in run
self._target(*self._args, **self._kwargs)
block_cache.py:681, in _removal_loop
tmp = self.to_remove.get(block=len(ids)==0)
queue.py:164, in get
self.not_empty.wait()
threading.py:293, in wait
waiter.acquire()
# ThreadID: 140536388232960
threading.py:882, in _bootstrap
self._bootstrap_inner()
threading.py:914, in _bootstrap_inner
self.run()
mount.py:64, in run_with_except_hook
run_old(*args, **kw)
mount.py:754, in run
self.block_cache.upload(el)
block_cache.py:554, in upload
self._queue_upload((el, obj_id))
block_cache.py:591, in _queue_upload
if self.to_upload.put(obj, timeout=5):
block_cache.py:74, in put
if not self.cv.wait(timeout):
threading.py:297, in wait
gotit = waiter.acquire(True, timeout)
# ThreadID: 140537501513472
threading.py:882, in _bootstrap
self._bootstrap_inner()
threading.py:914, in _bootstrap_inner
self.run()
mount.py:64, in run_with_except_hook
run_old(*args, **kw)
threading.py:862, in run
self._target(*self._args, **self._kwargs)
block_cache.py:404, in _upload_loop
self._do_upload(*tmp)
block_cache.py:431, in _do_upload
% obj_id).get_obj_size()
common.py:107, in wrapped
return method(*a, **kw)
common.py:337, in perform_write
return fn(fh)
comprenc.py:551, in __exit__
self.close()
comprenc.py:545, in close
self.fh.close()
comprenc.py:710, in close
self.fh.close()
common.py:107, in wrapped
return method(*a, **kw)
s3c.py:910, in close
headers=self.headers, body=self.fh)
s3c.py:440, in _do_request
query_string=query_string, body=body)
s3c.py:683, in _send_request
copyfileobj(body, self.conn, BUFSIZE)
shutil.py:76, in copyfileobj
fdst.write(buf)
__init__.py:653, in write
eval_coroutine(self.co_write(buf), self.timeout)
__init__.py:1396, in eval_coroutine
if not next(crt).poll(timeout=timeout):
__init__.py:115, in poll
return bool(poll.poll(timeout*1000)) # convert to ms
# ThreadID: 140536967067392
threading.py:882, in _bootstrap
self._bootstrap_inner()
threading.py:914, in _bootstrap_inner
self.run()
mount.py:64, in run_with_except_hook
run_old(*args, **kw)
threading.py:862, in run
self._target(*self._args, **self._kwargs)
block_cache.py:404, in _upload_loop
self._do_upload(*tmp)
block_cache.py:431, in _do_upload
% obj_id).get_obj_size()
common.py:107, in wrapped
return method(*a, **kw)
common.py:337, in perform_write
return fn(fh)
comprenc.py:551, in __exit__
self.close()
comprenc.py:545, in close
self.fh.close()
comprenc.py:710, in close
self.fh.close()
common.py:107, in wrapped
return method(*a, **kw)
s3c.py:910, in close
headers=self.headers, body=self.fh)
s3c.py:440, in _do_request
query_string=query_string, body=body)
s3c.py:683, in _send_request
copyfileobj(body, self.conn, BUFSIZE)
shutil.py:76, in copyfileobj
fdst.write(buf)
__init__.py:653, in write
eval_coroutine(self.co_write(buf), self.timeout)
__init__.py:1396, in eval_coroutine
if not next(crt).poll(timeout=timeout):
__init__.py:115, in poll
return bool(poll.poll(timeout*1000)) # convert to ms
# ThreadID: 140536925103872
threading.py:882, in _bootstrap
self._bootstrap_inner()
threading.py:914, in _bootstrap_inner
self.run()
mount.py:64, in run_with_except_hook
run_old(*args, **kw)
threading.py:862, in run
self._target(*self._args, **self._kwargs)
block_cache.py:681, in _removal_loop
tmp = self.to_remove.get(block=len(ids)==0)
queue.py:164, in get
self.not_empty.wait()
threading.py:293, in wait
waiter.acquire()
