Repository: beam Updated Branches: refs/heads/master dee8e93e7 -> afe8da8bc
[BEAM-2999] Split ValidatesRunner test from Python Postcommit Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/4f00899e Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/4f00899e Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/4f00899e Branch: refs/heads/master Commit: 4f00899e595f542344a196c9a3a472b1cf00d9b3 Parents: dee8e93 Author: Mark Liu <[email protected]> Authored: Wed Sep 27 18:18:02 2017 -0700 Committer: Ahmet Altay <[email protected]> Committed: Wed Oct 4 16:25:56 2017 -0700 ---------------------------------------------------------------------- ...ommit_Python_ValidatesRunner_Dataflow.groovy | 54 +++++++++++++++ sdks/python/run_postcommit.sh | 20 ------ sdks/python/run_validatesrunner.sh | 71 ++++++++++++++++++++ 3 files changed, 125 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/beam/blob/4f00899e/.test-infra/jenkins/job_beam_PostCommit_Python_ValidatesRunner_Dataflow.groovy ---------------------------------------------------------------------- diff --git a/.test-infra/jenkins/job_beam_PostCommit_Python_ValidatesRunner_Dataflow.groovy b/.test-infra/jenkins/job_beam_PostCommit_Python_ValidatesRunner_Dataflow.groovy new file mode 100644 index 0000000..06bbfb7 --- /dev/null +++ b/.test-infra/jenkins/job_beam_PostCommit_Python_ValidatesRunner_Dataflow.groovy @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import common_job_properties + +// This job runs the suite of Python ValidatesRunner tests against the +// Dataflow runner. +job('beam_PostCommit_Python_ValidatesRunner_Dataflow') { + description('Runs Python ValidatesRunner suite on the Dataflow runner.') + + // Set common parameters. + common_job_properties.setTopLevelMainJobProperties(delegate) + + // Sets that this is a PostCommit job. + common_job_properties.setPostCommit(delegate, '0 3-22/6 * * *') + + // Allows triggering this build against pull requests. + common_job_properties.enablePhraseTriggeringFromPullRequest( + delegate, + 'Google Cloud Dataflow Runner Python ValidatesRunner Tests', + 'Run Python Dataflow ValidatesRunner') + + // Allow the test to only run on particular nodes + // TODO(BEAM-1817): Remove once the tests can run on all nodes + parameters { + nodeParam('TEST_HOST') { + description('select test host as either beam1, 2 or 3') + defaultNodes(['beam3']) + allowedNodes(['beam1', 'beam2', 'beam3']) + trigger('multiSelectionDisallowed') + eligibility('IgnoreOfflineNodeEligibility') + } + } + + // Execute shell command to test Python SDK. + steps { + shell('bash sdks/python/run_validatesrunner.sh') + } +} http://git-wip-us.apache.org/repos/asf/beam/blob/4f00899e/sdks/python/run_postcommit.sh ---------------------------------------------------------------------- diff --git a/sdks/python/run_postcommit.sh b/sdks/python/run_postcommit.sh index ddc3dc7..5e1c6b2 100755 --- a/sdks/python/run_postcommit.sh +++ b/sdks/python/run_postcommit.sh @@ -66,26 +66,6 @@ python setup.py sdist SDK_LOCATION=$(find dist/apache-beam-*.tar.gz) -# Install test dependencies for ValidatesRunner tests. -echo "pyhamcrest" > postcommit_requirements.txt -echo "mock" >> postcommit_requirements.txt - -# Run ValidatesRunner tests on Google Cloud Dataflow service -echo ">>> RUNNING DATAFLOW RUNNER VALIDATESRUNNER TESTS" -python setup.py nosetests \ - --attr ValidatesRunner \ - --nocapture \ - --processes=4 \ - --process-timeout=900 \ - --test-pipeline-options=" \ - --runner=TestDataflowRunner \ - --project=$PROJECT \ - --staging_location=$GCS_LOCATION/staging-validatesrunner-test \ - --temp_location=$GCS_LOCATION/temp-validatesrunner-test \ - --sdk_location=$SDK_LOCATION \ - --requirements_file=postcommit_requirements.txt \ - --num_workers=1" - # Run integration tests on the Google Cloud Dataflow service # and validate that jobs finish successfully. echo ">>> RUNNING TEST DATAFLOW RUNNER it tests" http://git-wip-us.apache.org/repos/asf/beam/blob/4f00899e/sdks/python/run_validatesrunner.sh ---------------------------------------------------------------------- diff --git a/sdks/python/run_validatesrunner.sh b/sdks/python/run_validatesrunner.sh new file mode 100755 index 0000000..7d20a75 --- /dev/null +++ b/sdks/python/run_validatesrunner.sh @@ -0,0 +1,71 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This script will be run by Jenkins as a post commit test. In order to run +# locally make the following changes: +# +# LOCAL_PATH -> Path of tox and virtualenv if you have them already installed. +# GCS_LOCATION -> Temporary location to use for service tests. +# PROJECT -> Project name to use for service jobs. +# +# Execute from the root of the repository: sdks/python/run_postcommit.sh + +set -e +set -v + +# pip install --user installation location. +LOCAL_PATH=$HOME/.local/bin/ + +# INFRA does not install virtualenv +pip install virtualenv --user + +# Virtualenv for the rest of the script to run setup & e2e tests +${LOCAL_PATH}/virtualenv sdks/python +. sdks/python/bin/activate +cd sdks/python +pip install -e .[gcp,test] + +# Where to store integration test outputs. +GCS_LOCATION=gs://temp-storage-for-end-to-end-tests + +PROJECT=apache-beam-testing + +# Create a tarball +python setup.py sdist + +SDK_LOCATION=$(find dist/apache-beam-*.tar.gz) + +# Install test dependencies for ValidatesRunner tests. +echo "pyhamcrest" > postcommit_requirements.txt +echo "mock" >> postcommit_requirements.txt + +# Run ValidatesRunner tests on Google Cloud Dataflow service +echo ">>> RUNNING DATAFLOW RUNNER VALIDATESRUNNER TESTS" +python setup.py nosetests \ + --attr ValidatesRunner \ + --nocapture \ + --processes=4 \ + --process-timeout=900 \ + --test-pipeline-options=" \ + --runner=TestDataflowRunner \ + --project=$PROJECT \ + --staging_location=$GCS_LOCATION/staging-validatesrunner-test \ + --temp_location=$GCS_LOCATION/temp-validatesrunner-test \ + --sdk_location=$SDK_LOCATION \ + --requirements_file=postcommit_requirements.txt \ + --num_workers=1"
