Repository: beam Updated Branches: refs/heads/master 07d9bd5f8 -> 3b4e0fb16
[BEAM-1441] Fix size check for windows and improve error message Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/6ceae061 Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/6ceae061 Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/6ceae061 Branch: refs/heads/master Commit: 6ceae061e1e3c8dadd575f2c9f4e12dea391ae66 Parents: 07d9bd5 Author: Sourabh Bajaj <[email protected]> Authored: Thu Mar 30 17:17:01 2017 -0700 Committer: Ahmet Altay <[email protected]> Committed: Thu Mar 30 17:47:58 2017 -0700 ---------------------------------------------------------------------- sdks/python/apache_beam/io/filesystem.py | 8 +++++--- sdks/python/apache_beam/io/gcp/gcsfilesystem_test.py | 12 ++++++++---- sdks/python/apache_beam/io/localfilesystem_test.py | 12 ++++++++---- 3 files changed, 21 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/beam/blob/6ceae061/sdks/python/apache_beam/io/filesystem.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/io/filesystem.py b/sdks/python/apache_beam/io/filesystem.py index 14493c0..b0d2f48 100644 --- a/sdks/python/apache_beam/io/filesystem.py +++ b/sdks/python/apache_beam/io/filesystem.py @@ -269,8 +269,9 @@ class FileMetadata(object): """ def __init__(self, path, size_in_bytes): assert isinstance(path, basestring) and path, "Path should be a string" - assert isinstance(size_in_bytes, int) and size_in_bytes >= 0, \ - "Size of bytes should be greater than equal to zero" + assert isinstance(size_in_bytes, (int, long)) and size_in_bytes >= 0, \ + "Invalid value for size_in_bytes should %s (of type %s)" % ( + size_in_bytes, type(size_in_bytes)) self.path = path self.size_in_bytes = size_in_bytes @@ -312,7 +313,8 @@ class BeamIOError(IOError): may have failed anywhere so the user should use match to determine the current state of the system. """ - super(BeamIOError, self).__init__(msg) + message = "%s with exceptions %s" % (msg, exception_details) + super(BeamIOError, self).__init__(message) self.exception_details = exception_details http://git-wip-us.apache.org/repos/asf/beam/blob/6ceae061/sdks/python/apache_beam/io/gcp/gcsfilesystem_test.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/io/gcp/gcsfilesystem_test.py b/sdks/python/apache_beam/io/gcp/gcsfilesystem_test.py index 3fe5cce..73a3893 100644 --- a/sdks/python/apache_beam/io/gcp/gcsfilesystem_test.py +++ b/sdks/python/apache_beam/io/gcp/gcsfilesystem_test.py @@ -68,7 +68,8 @@ class GCSFileSystemTest(unittest.TestCase): file_system = gcsfilesystem.GCSFileSystem() with self.assertRaises(BeamIOError) as error: file_system.match(['gs://bucket/']) - self.assertEqual(error.exception.message, 'Match operation failed') + self.assertTrue( + error.exception.message.startswith('Match operation failed')) self.assertEqual(error.exception.exception_details, expected_results) gcsio_mock.size_of_files_in_glob.assert_called_once_with('gs://bucket/*') @@ -148,7 +149,8 @@ class GCSFileSystemTest(unittest.TestCase): file_system = gcsfilesystem.GCSFileSystem() with self.assertRaises(BeamIOError) as error: file_system.copy(sources, destinations) - self.assertEqual(error.exception.message, 'Copy operation failed') + self.assertTrue( + error.exception.message.startswith('Copy operation failed')) self.assertEqual(error.exception.exception_details, expected_results) gcsio_mock.copy.assert_called_once_with( @@ -240,7 +242,8 @@ class GCSFileSystemTest(unittest.TestCase): file_system = gcsfilesystem.GCSFileSystem() with self.assertRaises(BeamIOError) as error: file_system.rename(sources, destinations) - self.assertEqual(error.exception.message, 'Rename operation failed') + self.assertTrue( + error.exception.message.startswith('Rename operation failed')) self.assertEqual(error.exception.exception_details, expected_results) gcsio_mock.copy_batch.assert_called_once_with([ @@ -288,6 +291,7 @@ class GCSFileSystemTest(unittest.TestCase): file_system = gcsfilesystem.GCSFileSystem() with self.assertRaises(BeamIOError) as error: file_system.delete(files) - self.assertEqual(error.exception.message, 'Delete operation failed') + self.assertTrue( + error.exception.message.startswith('Delete operation failed')) self.assertEqual(error.exception.exception_details, expected_results) gcsio_mock.delete_batch.assert_called() http://git-wip-us.apache.org/repos/asf/beam/blob/6ceae061/sdks/python/apache_beam/io/localfilesystem_test.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/io/localfilesystem_test.py b/sdks/python/apache_beam/io/localfilesystem_test.py index 00059ef..5eacbc2 100644 --- a/sdks/python/apache_beam/io/localfilesystem_test.py +++ b/sdks/python/apache_beam/io/localfilesystem_test.py @@ -74,7 +74,8 @@ class LocalFileSystemTest(unittest.TestCase): # Match files with None so that it throws an exception with self.assertRaises(BeamIOError) as error: self.fs.match([None]) - self.assertEqual(error.exception.message, 'Match operation failed') + self.assertTrue( + error.exception.message.startswith('Match operation failed')) self.assertEqual(error.exception.exception_details.keys(), [None]) def test_match_directory(self): @@ -108,7 +109,8 @@ class LocalFileSystemTest(unittest.TestCase): path2 = os.path.join(self.tmpdir, 'f2') with self.assertRaises(BeamIOError) as error: self.fs.copy([path1], [path2]) - self.assertEqual(error.exception.message, 'Copy operation failed') + self.assertTrue( + error.exception.message.startswith('Copy operation failed')) self.assertEqual(error.exception.exception_details.keys(), [(path1, path2)]) def test_copy_directory(self): @@ -140,7 +142,8 @@ class LocalFileSystemTest(unittest.TestCase): path2 = os.path.join(self.tmpdir, 'f2') with self.assertRaises(BeamIOError) as error: self.fs.rename([path1], [path2]) - self.assertEqual(error.exception.message, 'Rename operation failed') + self.assertTrue( + error.exception.message.startswith('Rename operation failed')) self.assertEqual(error.exception.exception_details.keys(), [(path1, path2)]) def test_rename_directory(self): @@ -181,5 +184,6 @@ class LocalFileSystemTest(unittest.TestCase): path1 = os.path.join(self.tmpdir, 'f1') with self.assertRaises(BeamIOError) as error: self.fs.delete([path1]) - self.assertEqual(error.exception.message, 'Delete operation failed') + self.assertTrue( + error.exception.message.startswith('Delete operation failed')) self.assertEqual(error.exception.exception_details.keys(), [path1])
