Repository: beam Updated Branches: refs/heads/master 570d0e2a1 -> 3711c0caf
Update integration tests in post commits Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/ffe8355b Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/ffe8355b Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/ffe8355b Branch: refs/heads/master Commit: ffe8355b212e2d7f98d2dd152bbf7bc37448ad58 Parents: 570d0e2 Author: Ahmet Altay <[email protected]> Authored: Fri Mar 31 15:11:43 2017 -0700 Committer: Ahmet Altay <[email protected]> Committed: Mon Apr 3 15:14:28 2017 -0700 ---------------------------------------------------------------------- .../cookbook/bigquery_tornadoes_it_test.py | 3 +++ .../apache_beam/examples/wordcount_it_test.py | 6 +++++- sdks/python/run_postcommit.sh | 20 +++++++++----------- 3 files changed, 17 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/beam/blob/ffe8355b/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py b/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py index 306a09e..709e3d5 100644 --- a/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py +++ b/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py @@ -32,6 +32,9 @@ from apache_beam.tests.pipeline_verifiers import PipelineStateMatcher class BigqueryTornadoesIT(unittest.TestCase): + # Enable nose tests running in parallel + _multiprocess_can_split_ = True + # The default checksum is a SHA-1 hash generated from sorted rows reading # from expected Bigquery table. DEFAULT_CHECKSUM = '83789a7c1bca7959dcf23d3bc37e9204e594330f' http://git-wip-us.apache.org/repos/asf/beam/blob/ffe8355b/sdks/python/apache_beam/examples/wordcount_it_test.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/examples/wordcount_it_test.py b/sdks/python/apache_beam/examples/wordcount_it_test.py index 1c700b6..54e54e8 100644 --- a/sdks/python/apache_beam/examples/wordcount_it_test.py +++ b/sdks/python/apache_beam/examples/wordcount_it_test.py @@ -18,6 +18,7 @@ """End-to-end test for the wordcount example.""" import logging +import time import unittest from hamcrest.core.core.allof import all_of @@ -31,6 +32,9 @@ from apache_beam.tests.pipeline_verifiers import FileChecksumMatcher class WordCountIT(unittest.TestCase): + # Enable nose tests running in parallel + _multiprocess_can_split_ = True + # The default checksum is a SHA-1 hash generated from a sorted list of # lines read from expected output. DEFAULT_CHECKSUM = '33535a832b7db6d78389759577d4ff495980b9c0' @@ -41,7 +45,7 @@ class WordCountIT(unittest.TestCase): # Set extra options to the pipeline for test purpose output = '/'.join([test_pipeline.get_option('output'), - test_pipeline.get_option('job_name'), + str(int(time.time())), 'results']) arg_sleep_secs = test_pipeline.get_option('sleep_secs') sleep_secs = int(arg_sleep_secs) if arg_sleep_secs is not None else None http://git-wip-us.apache.org/repos/asf/beam/blob/ffe8355b/sdks/python/run_postcommit.sh ---------------------------------------------------------------------- diff --git a/sdks/python/run_postcommit.sh b/sdks/python/run_postcommit.sh index 50338e2..dd3182a 100755 --- a/sdks/python/run_postcommit.sh +++ b/sdks/python/run_postcommit.sh @@ -56,12 +56,9 @@ python -m apache_beam.examples.wordcount --output /tmp/py-wordcount-direct # Run tests on the service. -# Where to store wordcount output. +# Where to store integration test outputs. GCS_LOCATION=gs://temp-storage-for-end-to-end-tests -# Job name needs to be unique -JOBNAME_E2E_WC=py-wordcount-`date +%s` - PROJECT=apache-beam-testing # Create a tarball @@ -88,18 +85,19 @@ python setup.py nosetests \ --requirements_file=postcommit_requirements.txt \ --num_workers=1" -# Run wordcount on the Google Cloud Dataflow service -# and validate job that finishes successfully. -echo ">>> RUNNING TEST DATAFLOW RUNNER py-wordcount" +# Run integration tests on the Google Cloud Dataflow service +# and validate that jobs finish successfully. +echo ">>> RUNNING TEST DATAFLOW RUNNER it tests" python setup.py nosetests \ -a IT \ + --processes=4 \ + --process-timeout=600 \ --test-pipeline-options=" \ --runner=TestDataflowRunner \ --project=$PROJECT \ - --staging_location=$GCS_LOCATION/staging-wordcount \ - --temp_location=$GCS_LOCATION/temp-wordcount \ - --output=$GCS_LOCATION/py-wordcount-cloud/output \ + --staging_location=$GCS_LOCATION/staging-it \ + --temp_location=$GCS_LOCATION/temp-it \ + --output=$GCS_LOCATION/py-it-cloud/output \ --sdk_location=$SDK_LOCATION \ - --job_name=$JOBNAME_E2E_WC \ --num_workers=1 \ --sleep_secs=20"
