Improve error handling in gcsio.py
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/4d6da9cf Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/4d6da9cf Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/4d6da9cf Branch: refs/heads/python-sdk Commit: 4d6da9cf19373a4f5a6c9513f44e12341c985a97 Parents: 1d53e28 Author: Charles Chen <c...@google.com> Authored: Mon Aug 15 15:12:00 2016 -0700 Committer: Dan Halperin <dhalp...@google.com> Committed: Mon Aug 15 15:26:05 2016 -0700 ---------------------------------------------------------------------- sdks/python/apache_beam/io/gcsio.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4d6da9cf/sdks/python/apache_beam/io/gcsio.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/io/gcsio.py b/sdks/python/apache_beam/io/gcsio.py index 7bb532c..4c733d9 100644 --- a/sdks/python/apache_beam/io/gcsio.py +++ b/sdks/python/apache_beam/io/gcsio.py @@ -29,6 +29,7 @@ import os import re import StringIO import threading +import traceback from apitools.base.py.exceptions import HttpError import apitools.base.py.transfer as transfer @@ -591,7 +592,8 @@ class GcsBufferedWriter(object): self.client.objects.Insert(self.insert_request, upload=self.upload) except Exception as e: # pylint: disable=broad-except logging.error( - 'Error in _start_upload while inserting file %s: %s', self.path, e) + 'Error in _start_upload while inserting file %s: %s', self.path, + traceback.format_exc()) self.upload_thread.last_error = e finally: self.child_conn.close() @@ -623,6 +625,9 @@ class GcsBufferedWriter(object): self.closed = True self.conn.close() self.upload_thread.join() + # Check for exception since the last _flush_write_buffer() call. + if self.upload_thread.last_error: + raise self.upload_thread.last_error # pylint: disable=raising-bad-type def __enter__(self): return self