[gentoo-portage-dev] [PATCH 2/2] emaint: log FileCopier exceptions

2019-11-06 Thread Zac Medico
Since FileCopier now captures exceptions instead of showing
a traceback, it's necessary to explicitly log them.

Bug: https://bugs.gentoo.org/699400
Signed-off-by: Zac Medico 
---
 lib/portage/_emirrordist/DeletionTask.py | 2 +-
 lib/portage/_emirrordist/FetchTask.py| 8 
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/lib/portage/_emirrordist/DeletionTask.py 
b/lib/portage/_emirrordist/DeletionTask.py
index 2fdafd59e..5eb01d840 100644
--- a/lib/portage/_emirrordist/DeletionTask.py
+++ b/lib/portage/_emirrordist/DeletionTask.py
@@ -85,7 +85,7 @@ class DeletionTask(CompositeTask):
 
else:
logging.error(("%s copy from distfiles "
-   "to recycle failed") % (self.distfile,))
+   "to recycle failed: %s") % (self.distfile, 
copier.future.exception()))
success = False
 
if success:
diff --git a/lib/portage/_emirrordist/FetchTask.py 
b/lib/portage/_emirrordist/FetchTask.py
index 6f547d397..61812ab59 100644
--- a/lib/portage/_emirrordist/FetchTask.py
+++ b/lib/portage/_emirrordist/FetchTask.py
@@ -372,8 +372,8 @@ class FetchTask(CompositeTask):
 
current_mirror = self._current_mirror
if copier.returncode != os.EX_OK:
-   msg = "%s %s copy failed unexpectedly" % \
-   (self.distfile, current_mirror.name)
+   msg = "%s %s copy failed unexpectedly: %s" % \
+   (self.distfile, current_mirror.name, 
copier.future.exception())
self.scheduler.output(msg + '\n', background=True,
log_path=self._log_path)
logging.error(msg)
@@ -549,8 +549,8 @@ class FetchTask(CompositeTask):
self._make_layout_links()
else:
# out of space?
-   msg = "%s %s copy failed unexpectedly" % \
-   (self.distfile, self._fetch_tmp_dir_info)
+   msg = "%s %s copy failed unexpectedly: %s" % \
+   (self.distfile, self._fetch_tmp_dir_info, 
copier.future.exception())
self.scheduler.output(msg + '\n', background=True,
log_path=self._log_path)
logging.error(msg)
-- 
2.21.0




[gentoo-portage-dev] [PATCH 0/2] emaint: clean up FileCopier exception logging

2019-11-06 Thread Zac Medico
Use ForkExecutor to capture exceptions instead of showing
a full traceback. FileCopier callers will now be responsible
for displaying relevant exception messages.

Bug: https://bugs.gentoo.org/699400

Zac Medico (2):
  FileCopier: capture exceptions
  emaint: log FileCopier exceptions

 lib/portage/_emirrordist/DeletionTask.py   |  2 +-
 lib/portage/_emirrordist/FetchTask.py  |  8 ++--
 lib/portage/tests/util/test_file_copier.py | 44 ++
 lib/portage/util/_async/FileCopier.py  | 16 
 4 files changed, 58 insertions(+), 12 deletions(-)
 create mode 100644 lib/portage/tests/util/test_file_copier.py

-- 
2.21.0




[gentoo-portage-dev] [PATCH 1/2] FileCopier: capture exceptions

2019-11-06 Thread Zac Medico
Use ForkExecutor to capture exceptions instead of showing
a full traceback. FileCopier callers will now be responsible
for displaying relevant exception messages.

Bug: https://bugs.gentoo.org/699400
Signed-off-by: Zac Medico 
---
 lib/portage/tests/util/test_file_copier.py | 44 ++
 lib/portage/util/_async/FileCopier.py  | 16 
 2 files changed, 53 insertions(+), 7 deletions(-)
 create mode 100644 lib/portage/tests/util/test_file_copier.py

diff --git a/lib/portage/tests/util/test_file_copier.py 
b/lib/portage/tests/util/test_file_copier.py
new file mode 100644
index 0..01dfba494
--- /dev/null
+++ b/lib/portage/tests/util/test_file_copier.py
@@ -0,0 +1,44 @@
+# Copyright 2019 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import errno
+import os
+import shutil
+import tempfile
+
+from portage.tests import TestCase
+from portage.util._async.FileCopier import FileCopier
+from portage.util._eventloop.global_event_loop import global_event_loop
+
+
+class FileCopierTestCase(TestCase):
+
+   def testFileCopier(self):
+   loop = global_event_loop()
+   tempdir = tempfile.mkdtemp()
+   try:
+
+   # regular successful copy
+   src_path = os.path.join(tempdir, 'src')
+   dest_path = os.path.join(tempdir, 'dest')
+   content = b'foo'
+   with open(src_path, 'wb') as f:
+   f.write(content)
+   copier = FileCopier(src_path=src_path, 
dest_path=dest_path, scheduler=loop)
+   copier.start()
+   loop.run_until_complete(copier.async_wait())
+   self.assertEqual(copier.returncode, 0)
+   copier.future.result()
+   with open(dest_path, 'rb') as f:
+   self.assertEqual(f.read(), content)
+
+   # failure due to nonexistent src_path
+   src_path = os.path.join(tempdir, 'does-not-exist')
+   copier = FileCopier(src_path=src_path, 
dest_path=dest_path, scheduler=loop)
+   copier.start()
+   loop.run_until_complete(copier.async_wait())
+   self.assertEqual(copier.returncode, 1)
+   self.assertEqual(copier.future.exception().errno, 
errno.ENOENT)
+   self.assertEqual(copier.future.exception().filename, 
src_path.encode('utf8'))
+   finally:
+   shutil.rmtree(tempdir)
diff --git a/lib/portage/util/_async/FileCopier.py 
b/lib/portage/util/_async/FileCopier.py
index 27e5ab4c0..3a0be4b63 100644
--- a/lib/portage/util/_async/FileCopier.py
+++ b/lib/portage/util/_async/FileCopier.py
@@ -1,17 +1,19 @@
-# Copyright 2013 Gentoo Foundation
+# Copyright 2013-2019 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
-from portage import os
 from portage import shutil
-from portage.util._async.ForkProcess import ForkProcess
+from portage.util.futures import asyncio
+from portage.util.futures.executor.fork import ForkExecutor
+from portage.util._async.AsyncTaskFuture import AsyncTaskFuture
 
-class FileCopier(ForkProcess):
+class FileCopier(AsyncTaskFuture):
"""
Asynchronously copy a file.
"""
 
__slots__ = ('src_path', 'dest_path')
 
-   def _run(self):
-   shutil.copy(self.src_path, self.dest_path)
-   return os.EX_OK
+   def _start(self):
+   self.future = 
asyncio.ensure_future(self.scheduler.run_in_executor(ForkExecutor(loop=self.scheduler),
+   shutil.copy, self.src_path, self.dest_path))
+   super(FileCopier, self)._start()
-- 
2.21.0