This is an automated email from the ASF dual-hosted git repository. pabloem pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push: new 2798d62 [BEAM-7510] Fixing fileio tests checking JSON serialization (flaky) (#8795) 2798d62 is described below commit 2798d626507c7f5c8699096b3a188200ac64a299 Author: Pablo <pabl...@users.noreply.github.com> AuthorDate: Wed Jun 12 11:11:49 2019 -0700 [BEAM-7510] Fixing fileio tests checking JSON serialization (flaky) (#8795) * Fixing test * Unskip test --- sdks/python/apache_beam/io/fileio_test.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/sdks/python/apache_beam/io/fileio_test.py b/sdks/python/apache_beam/io/fileio_test.py index 98d6033..f87aa2f 100644 --- a/sdks/python/apache_beam/io/fileio_test.py +++ b/sdks/python/apache_beam/io/fileio_test.py @@ -296,10 +296,11 @@ class WriteFilesTest(_TestCaseWithTempDirCleanUp): result = (p | fileio.MatchFiles(FileSystems.join(dir, '*')) | fileio.ReadMatches() - | beam.FlatMap(lambda f: f.read_utf8().strip().split('\n'))) + | beam.FlatMap(lambda f: f.read_utf8().strip().split('\n')) + | beam.Map(json.loads)) assert_that(result, - equal_to([json.dumps(row) for row in self.SIMPLE_COLLECTION])) + equal_to([row for row in self.SIMPLE_COLLECTION])) def test_write_to_different_file_types_some_spilling(self): @@ -321,7 +322,8 @@ class WriteFilesTest(_TestCaseWithTempDirCleanUp): cncf_res = (p | fileio.MatchFiles(FileSystems.join(dir, 'cncf*')) | fileio.ReadMatches() - | beam.FlatMap(lambda f: f.read_utf8().strip().split('\n'))) + | beam.FlatMap(lambda f: f.read_utf8().strip().split('\n')) + | beam.Map(json.loads)) apache_res = (p | "MatchApache" >> fileio.MatchFiles( @@ -331,7 +333,7 @@ class WriteFilesTest(_TestCaseWithTempDirCleanUp): lambda rf: csv.reader(_get_file_reader(rf)))) assert_that(cncf_res, - equal_to([json.dumps(row) + equal_to([row for row in self.SIMPLE_COLLECTION if row['foundation'] == 'cncf']), label='verifyCNCF') @@ -379,7 +381,6 @@ class WriteFilesTest(_TestCaseWithTempDirCleanUp): found_files = [m.path for m in leftovers[0].metadata_list] self.assertListEqual(found_files, [f2]) - @unittest.skip('BEAM-7510 - test is flaky') def test_write_to_different_file_types(self): dir = self._new_tempdir() @@ -399,7 +400,8 @@ class WriteFilesTest(_TestCaseWithTempDirCleanUp): cncf_res = (p | fileio.MatchFiles(FileSystems.join(dir, 'cncf*')) | fileio.ReadMatches() - | beam.FlatMap(lambda f: f.read_utf8().strip().split('\n'))) + | beam.FlatMap(lambda f: f.read_utf8().strip().split('\n')) + | beam.Map(json.loads)) apache_res = (p | "MatchApache" >> fileio.MatchFiles( @@ -409,7 +411,7 @@ class WriteFilesTest(_TestCaseWithTempDirCleanUp): lambda rf: csv.reader(_get_file_reader(rf)))) assert_that(cncf_res, - equal_to([json.dumps(row) + equal_to([row for row in self.SIMPLE_COLLECTION if row['foundation'] == 'cncf']), label='verifyCNCF')