Repository: beam Updated Branches: refs/heads/master 6ea4b3a42 -> 168d6a18f
Reduce test sizes to improve unit test speeds Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/0f563435 Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/0f563435 Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/0f563435 Branch: refs/heads/master Commit: 0f563435aa2416e1597a52fdc0c30935f5ffc7b8 Parents: 6ea4b3a Author: Sourabh Bajaj <[email protected]> Authored: Wed Feb 1 16:36:05 2017 -0800 Committer: Ahmet Altay <[email protected]> Committed: Wed Feb 1 17:38:29 2017 -0800 ---------------------------------------------------------------------- .../python/apache_beam/examples/complete/estimate_pi.py | 5 ++++- .../apache_beam/examples/complete/estimate_pi_test.py | 2 +- sdks/python/apache_beam/io/avroio_test.py | 4 ++-- sdks/python/apache_beam/io/bigquery_test.py | 3 ++- sdks/python/apache_beam/io/source_test_utils_test.py | 3 +-- sdks/python/apache_beam/io/textio_test.py | 12 ++++++------ sdks/python/apache_beam/transforms/combiners_test.py | 5 ++--- 7 files changed, 18 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/beam/blob/0f563435/sdks/python/apache_beam/examples/complete/estimate_pi.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/examples/complete/estimate_pi.py b/sdks/python/apache_beam/examples/complete/estimate_pi.py index 11081a6..3e57f71 100644 --- a/sdks/python/apache_beam/examples/complete/estimate_pi.py +++ b/sdks/python/apache_beam/examples/complete/estimate_pi.py @@ -90,11 +90,14 @@ class JsonCoder(object): class EstimatePiTransform(beam.PTransform): """Runs 10M trials, and combine the results to estimate pi.""" + def __init__(self, tries_per_work_item=100000): + self.tries_per_work_item = tries_per_work_item def expand(self, pcoll): # A hundred work items of a hundred thousand tries each. return (pcoll - | 'Initialize' >> beam.Create([100000] * 100).with_output_types(int) + | 'Initialize' >> beam.Create( + [self.tries_per_work_item] * 100).with_output_types(int) | 'Run trials' >> beam.Map(run_trials) | 'Sum' >> beam.CombineGlobally(combine_results).without_defaults()) http://git-wip-us.apache.org/repos/asf/beam/blob/0f563435/sdks/python/apache_beam/examples/complete/estimate_pi_test.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/examples/complete/estimate_pi_test.py b/sdks/python/apache_beam/examples/complete/estimate_pi_test.py index 10010cb..ddd5f76 100644 --- a/sdks/python/apache_beam/examples/complete/estimate_pi_test.py +++ b/sdks/python/apache_beam/examples/complete/estimate_pi_test.py @@ -39,7 +39,7 @@ class EstimatePiTest(unittest.TestCase): def test_basics(self): p = TestPipeline() - result = p | 'Estimate' >> estimate_pi.EstimatePiTransform() + result = p | 'Estimate' >> estimate_pi.EstimatePiTransform(5000) # Note: Probabilistically speaking this test can fail with a probability # that is very small (VERY) given that we run at least 10 million trials. http://git-wip-us.apache.org/repos/asf/beam/blob/0f563435/sdks/python/apache_beam/io/avroio_test.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/io/avroio_test.py b/sdks/python/apache_beam/io/avroio_test.py index d2fb1d1..76e723f 100644 --- a/sdks/python/apache_beam/io/avroio_test.py +++ b/sdks/python/apache_beam/io/avroio_test.py @@ -284,8 +284,8 @@ class TestAvro(unittest.TestCase): # work rebalancing test that completes within an acceptable amount of time. old_sync_interval = avro.datafile.SYNC_INTERVAL try: - avro.datafile.SYNC_INTERVAL = 5 - file_name = self._write_data(count=20) + avro.datafile.SYNC_INTERVAL = 2 + file_name = self._write_data(count=5) source = AvroSource(file_name) splits = [split for split in source.split(desired_bundle_size=float('inf'))] http://git-wip-us.apache.org/repos/asf/beam/blob/0f563435/sdks/python/apache_beam/io/bigquery_test.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/io/bigquery_test.py b/sdks/python/apache_beam/io/bigquery_test.py index 14eb035..cdca884 100644 --- a/sdks/python/apache_beam/io/bigquery_test.py +++ b/sdks/python/apache_beam/io/bigquery_test.py @@ -486,7 +486,8 @@ class TestBigQueryReader(unittest.TestCase): self.assertEqual(actual_rows, table_rows) self.assertEqual(schema, reader.schema) - def test_read_from_table_and_job_complete_retry(self): + @mock.patch('time.sleep', return_value=None) + def test_read_from_table_and_job_complete_retry(self, patched_time_sleep): client = mock.Mock() client.jobs.Insert.return_value = bigquery.Job( jobReference=bigquery.JobReference( http://git-wip-us.apache.org/repos/asf/beam/blob/0f563435/sdks/python/apache_beam/io/source_test_utils_test.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/io/source_test_utils_test.py b/sdks/python/apache_beam/io/source_test_utils_test.py index 2a207b6..f6f9ec3 100644 --- a/sdks/python/apache_beam/io/source_test_utils_test.py +++ b/sdks/python/apache_beam/io/source_test_utils_test.py @@ -113,9 +113,8 @@ class SourceTestUtilsTest(unittest.TestCase): self.assertTrue(stats.non_trivial_fractions) def test_split_at_fraction_exhaustive(self): - data = self._create_data(20) + data = self._create_data(10) source = self._create_source(data) - source_test_utils.assertSplitAtFractionExhaustive(source) if __name__ == '__main__': http://git-wip-us.apache.org/repos/asf/beam/blob/0f563435/sdks/python/apache_beam/io/textio_test.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/io/textio_test.py b/sdks/python/apache_beam/io/textio_test.py index 07c6d9c..a7133ed 100644 --- a/sdks/python/apache_beam/io/textio_test.py +++ b/sdks/python/apache_beam/io/textio_test.py @@ -258,8 +258,8 @@ class TextSourceTest(unittest.TestCase): (splits[0].source, splits[0].start_position, splits[0].stop_position)) def test_dynamic_work_rebalancing(self): - file_name, expected_data = write_data(15) - assert len(expected_data) == 15 + file_name, expected_data = write_data(5) + assert len(expected_data) == 5 source = TextSource(file_name, 0, CompressionTypes.UNCOMPRESSED, True, coders.StrUtf8Coder()) splits = [split for split in source.split(desired_bundle_size=100000)] @@ -279,8 +279,8 @@ class TextSourceTest(unittest.TestCase): perform_multi_threaded_test=False) def test_dynamic_work_rebalancing_mixed_eol(self): - file_name, expected_data = write_data(15, eol=EOL.MIXED) - assert len(expected_data) == 15 + file_name, expected_data = write_data(5, eol=EOL.MIXED) + assert len(expected_data) == 5 source = TextSource(file_name, 0, CompressionTypes.UNCOMPRESSED, True, coders.StrUtf8Coder()) splits = [split for split in source.split(desired_bundle_size=100000)] @@ -374,7 +374,7 @@ class TextSourceTest(unittest.TestCase): pipeline.run() def test_read_gzip_large(self): - _, lines = write_data(10000) + _, lines = write_data(1000) file_name = tempfile.NamedTemporaryFile( delete=False, prefix=tempfile.template).name with gzip.GzipFile(file_name, 'wb') as f: @@ -389,7 +389,7 @@ class TextSourceTest(unittest.TestCase): pipeline.run() def test_read_gzip_large_after_splitting(self): - _, lines = write_data(10000) + _, lines = write_data(1000) file_name = tempfile.NamedTemporaryFile( delete=False, prefix=tempfile.template).name with gzip.GzipFile(file_name, 'wb') as f: http://git-wip-us.apache.org/repos/asf/beam/blob/0f563435/sdks/python/apache_beam/transforms/combiners_test.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/transforms/combiners_test.py b/sdks/python/apache_beam/transforms/combiners_test.py index ba8ae82..6367dc8 100644 --- a/sdks/python/apache_beam/transforms/combiners_test.py +++ b/sdks/python/apache_beam/transforms/combiners_test.py @@ -219,14 +219,13 @@ class CombineTest(unittest.TestCase): pipeline.run() def test_global_sample(self): - def is_good_sample(actual): assert len(actual) == 1 assert sorted(actual[0]) in [[1, 1, 2], [1, 2, 2]], actual with TestPipeline() as pipeline: pcoll = pipeline | 'start' >> Create([1, 1, 2, 2]) - for ix in xrange(30): + for ix in xrange(9): assert_that( pcoll | combine.Sample.FixedSizeGlobally('sample-%d' % ix, 3), is_good_sample, @@ -235,7 +234,7 @@ class CombineTest(unittest.TestCase): def test_per_key_sample(self): pipeline = TestPipeline() pcoll = pipeline | 'start-perkey' >> Create( - sum(([(i, 1), (i, 1), (i, 2), (i, 2)] for i in xrange(300)), [])) + sum(([(i, 1), (i, 1), (i, 2), (i, 2)] for i in xrange(9)), [])) result = pcoll | 'sample' >> combine.Sample.FixedSizePerKey(3) def matcher():
