This is an automated email from the ASF dual-hosted git repository.
pabloem pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new d41df7b [BEAM-4300] Fix ValidatesRunner tests in Python: run with
same mechanism as ITs (#5372)
d41df7b is described below
commit d41df7b17f49569d37a7cdcc6184222208daba55
Author: Pablo <pabl...@users.noreply.github.com>
AuthorDate: Wed May 16 17:07:36 2018 -0700
[BEAM-4300] Fix ValidatesRunner tests in Python: run with same mechanism as
ITs (#5372)
* ValidatesRunner and ITs run with same mechanism in Python. Also receive
test type and GCP arguments.
---
...stCommit_Python_ValidatesRunner_Dataflow.groovy | 12 +++-
.../runners/dataflow/internal/dependency.py | 2 +-
sdks/python/build.gradle | 13 ++++-
sdks/python/run_postcommit.sh | 21 +++++--
sdks/python/run_validatesrunner.sh | 64 ----------------------
5 files changed, 37 insertions(+), 75 deletions(-)
diff --git
a/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Dataflow.groovy
b/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Dataflow.groovy
index 392a935..4461106 100644
--- a/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Dataflow.groovy
+++ b/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Dataflow.groovy
@@ -20,9 +20,11 @@ import common_job_properties
// This job runs the suite of Python ValidatesRunner tests against the
// Dataflow runner.
-job('beam_PostCommit_Python_ValidatesRunner_Dataflow') {
+job('beam_PostCommit_Py_VR_Dataflow') {
description('Runs Python ValidatesRunner suite on the Dataflow runner.')
+ previousNames('beam_PostCommit_Python_ValidatesRunner_Dataflow')
+
// Set common parameters.
common_job_properties.setTopLevelMainJobProperties(delegate)
@@ -35,8 +37,12 @@ job('beam_PostCommit_Python_ValidatesRunner_Dataflow') {
'Google Cloud Dataflow Runner Python ValidatesRunner Tests',
'Run Python Dataflow ValidatesRunner')
- // Execute shell command to test Python SDK.
+ // Execute gradle task to test Python SDK.
steps {
- shell('cd ' + common_job_properties.checkoutDir + ' && bash
sdks/python/run_validatesrunner.sh')
+ gradle {
+ rootBuildScriptDir(common_job_properties.checkoutDir)
+ tasks(':beam-sdks-python:validatesRunnerTests')
+ common_job_properties.setGradleSwitches(delegate)
+ }
}
}
diff --git a/sdks/python/apache_beam/runners/dataflow/internal/dependency.py
b/sdks/python/apache_beam/runners/dataflow/internal/dependency.py
index 289e078..2307371 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/dependency.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/dependency.py
@@ -256,7 +256,7 @@ def _populate_requirements_cache(requirements_file,
cache_dir):
# the requirements file and will not download package dependencies.
cmd_args = [
_get_python_executable(), '-m', 'pip', 'download', '--dest', cache_dir,
- '-r', requirements_file,
+ '-r', requirements_file, '--exists-action', 'i',
# Download from PyPI source distributions.
'--no-binary', ':all:']
logging.info('Executing command: %s', cmd_args)
diff --git a/sdks/python/build.gradle b/sdks/python/build.gradle
index df5a098..aa7d736 100644
--- a/sdks/python/build.gradle
+++ b/sdks/python/build.gradle
@@ -187,11 +187,20 @@ task localWordCount(dependsOn: 'installGcpTest') {
}
}
+task postCommitVRTests(dependsOn: 'installGcpTest') {
+ doLast {
+ exec {
+ executable 'sh'
+ args '-c', ". ${envdir}/bin/activate && ./run_postcommit.sh IT"
+ }
+ }
+}
+
task validatesRunnerTests(dependsOn: 'installGcpTest') {
doLast {
exec {
executable 'sh'
- args '-c', ". ${envdir}/bin/activate && ./run_postcommit.sh"
+ args '-c', ". ${envdir}/bin/activate && ./run_postcommit.sh
ValidatesRunner"
}
}
}
@@ -209,5 +218,5 @@ task postCommit() {
dependsOn "preCommit"
dependsOn "localWordCount"
dependsOn "hdfsIntegrationTest"
- dependsOn "validatesRunnerTests"
+ dependsOn "postCommitVRTests"
}
diff --git a/sdks/python/run_postcommit.sh b/sdks/python/run_postcommit.sh
index 582c432..d26a1c9 100755
--- a/sdks/python/run_postcommit.sh
+++ b/sdks/python/run_postcommit.sh
@@ -19,11 +19,17 @@
# This script will be run by Jenkins as a post commit test. In order to run
# locally make the following changes:
#
-# LOCAL_PATH -> Path of tox and virtualenv if you have them already
installed.
# GCS_LOCATION -> Temporary location to use for service tests.
# PROJECT -> Project name to use for service jobs.
#
-# Execute from the root of the repository: sdks/python/run_postcommit.sh
+
+if [ -z "$1" ]; then
+ printf "Usage: \n$> ./run_postcommit.sh <test_type> [gcp_location]
[gcp_project]"
+ printf "\n\ttest_type: ValidatesRunner or IT"
+ printf "\n\tgcp_location: A gs:// path to stage artifacts and output results"
+ printf "\n\tgcp_project: A GCP project to run Dataflow pipelines\n"
+ exit 1
+fi
set -e
set -v
@@ -31,20 +37,24 @@ set -v
# Run tests on the service.
# Where to store integration test outputs.
-GCS_LOCATION=gs://temp-storage-for-end-to-end-tests
+GCS_LOCATION=${2:-gs://temp-storage-for-end-to-end-tests}
-PROJECT=apache-beam-testing
+PROJECT=${3:-apache-beam-testing}
# Create a tarball
python setup.py sdist
SDK_LOCATION=$(find dist/apache-beam-*.tar.gz)
+# Install test dependencies for ValidatesRunner tests.
+echo "pyhamcrest" > postcommit_requirements.txt
+echo "mock" >> postcommit_requirements.txt
+
# Run integration tests on the Google Cloud Dataflow service
# and validate that jobs finish successfully.
echo ">>> RUNNING TEST DATAFLOW RUNNER it tests"
python setup.py nosetests \
- --attr IT \
+ --attr $1 \
--nocapture \
--processes=4 \
--process-timeout=1800 \
@@ -55,5 +65,6 @@ python setup.py nosetests \
--temp_location=$GCS_LOCATION/temp-it \
--output=$GCS_LOCATION/py-it-cloud/output \
--sdk_location=$SDK_LOCATION \
+ --requirements_file=postcommit_requirements.txt \
--num_workers=1 \
--sleep_secs=20"
diff --git a/sdks/python/run_validatesrunner.sh
b/sdks/python/run_validatesrunner.sh
deleted file mode 100755
index 6b2b80d..0000000
--- a/sdks/python/run_validatesrunner.sh
+++ /dev/null
@@ -1,64 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# This script will be run by Jenkins as a post commit test. In order to run
-# locally make the following changes:
-#
-# GCS_LOCATION -> Temporary location to use for service tests.
-# PROJECT -> Project name to use for service jobs.
-#
-# Execute from the root of the repository: sdks/python/run_postcommit.sh
-
-set -e
-set -v
-
-# Virtualenv for the rest of the script to run setup & e2e tests
-/usr/bin/virtualenv sdks/python
-. sdks/python/bin/activate
-cd sdks/python
-pip install -e .[gcp,test]
-
-# Where to store integration test outputs.
-GCS_LOCATION=gs://temp-storage-for-end-to-end-tests
-
-PROJECT=apache-beam-testing
-
-# Create a tarball
-python setup.py sdist
-
-SDK_LOCATION=$(find dist/apache-beam-*.tar.gz)
-
-# Install test dependencies for ValidatesRunner tests.
-echo "pyhamcrest" > postcommit_requirements.txt
-echo "mock" >> postcommit_requirements.txt
-
-# Run ValidatesRunner tests on Google Cloud Dataflow service
-echo ">>> RUNNING DATAFLOW RUNNER VALIDATESRUNNER TESTS"
-python setup.py nosetests \
- --attr ValidatesRunner \
- --nocapture \
- --processes=4 \
- --process-timeout=900 \
- --test-pipeline-options=" \
- --runner=TestDataflowRunner \
- --project=$PROJECT \
- --staging_location=$GCS_LOCATION/staging-validatesrunner-test \
- --temp_location=$GCS_LOCATION/temp-validatesrunner-test \
- --sdk_location=$SDK_LOCATION \
- --requirements_file=postcommit_requirements.txt \
- --num_workers=1"
--
To stop receiving notification emails like this one, please contact
pabl...@apache.org.
