This is an automated email from the ASF dual-hosted git repository. markliu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push: new 43be51f [BEAM-6908] Support Python3 performance benchmarks - part 2 new c97e391 Merge pull request #8324 from markflyhigh/py3-perf 43be51f is described below commit 43be51f1d64ffe3ae856570ac68348ab99ea5946 Author: Mark Liu <mark...@google.com> AuthorDate: Tue Apr 16 14:28:38 2019 -0700 [BEAM-6908] Support Python3 performance benchmarks - part 2 --- .../org/apache/beam/gradle/BeamModulePlugin.groovy | 57 ++++++++++++++++++++++ .../apache_beam/io/gcp/gcsio_integration_test.py | 4 +- sdks/python/build.gradle | 31 +----------- sdks/python/scripts/run_integration_test.sh | 25 +++++----- sdks/python/test-suites/dataflow/py3/build.gradle | 1 + 5 files changed, 75 insertions(+), 43 deletions(-) diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index 6614954..4b6c75d 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -235,6 +235,24 @@ class BeamModulePlugin implements Plugin<Project> { String itModule = System.getProperty('itModule') } + // Reads and contains all necessary performance test parameters + class PythonPerformanceTestConfiguration { + // Fully qualified name of the test to run. + String tests = System.getProperty('tests') + + // Attribute tag that can filter the test set. + String attribute = System.getProperty('attr') + + // Extra test options pass to nose. + String[] extraTestOptions = ["--nocapture"] + + // Name of Cloud KMS encryption key to use in some tests. + String kmsKeyName = System.getProperty('kmsKeyName') + + // Pipeline options to be used for pipeline invocation. + String pipelineOptions = System.getProperty('pipelineOptions', '') + } + // A class defining the set of configurable properties accepted by containerImageName. class ContainerImageNameConfiguration { String root = null // Sets the docker repository root (optional). @@ -1733,6 +1751,45 @@ class BeamModulePlugin implements Plugin<Project> { outputs.files project.fileTree(dir: "${pythonRootDir}/target/.tox/${tox_env}/log/") } } + + // Run single or a set of integration tests with provided test options and pipeline options. + project.ext.enablePythonPerformanceTest = { + + // Use the implicit it parameter of the closure to handle zero argument or one argument map calls. + // See: http://groovy-lang.org/closures.html#implicit-it + def config = it ? it as PythonPerformanceTestConfiguration : new PythonPerformanceTestConfiguration() + + project.task('integrationTest') { + dependsOn 'installGcpTest' + dependsOn 'sdist' + + doLast { + def argMap = [:] + + // Build test options that configures test environment and framework + def testOptions = [] + if (config.tests) + testOptions += "--tests=$config.tests" + if (config.attribute) + testOptions += "--attr=$config.attribute" + testOptions.addAll(config.extraTestOptions) + argMap["test_opts"] = testOptions + + // Build pipeline options that configures pipeline job + if (config.pipelineOptions) + argMap["pipeline_opts"] = config.pipelineOptions + if (config.kmsKeyName) + argMap["kms_key_name"] = config.kmsKeyName + + def cmdArgs = project.mapToArgString(argMap) + def runScriptsDir = "${pythonRootDir}/scripts" + project.exec { + executable 'sh' + args '-c', ". ${project.ext.envdir}/bin/activate && ${runScriptsDir}/run_integration_test.sh ${cmdArgs}" + } + } + } + } } } } diff --git a/sdks/python/apache_beam/io/gcp/gcsio_integration_test.py b/sdks/python/apache_beam/io/gcp/gcsio_integration_test.py index 18a367d..8312f07 100644 --- a/sdks/python/apache_beam/io/gcp/gcsio_integration_test.py +++ b/sdks/python/apache_beam/io/gcp/gcsio_integration_test.py @@ -32,8 +32,8 @@ permissions for the key specified in --kms_key_name. To run these tests manually: ./gradlew beam-sdks-python:integrationTest \ - -Ptests=apache_beam.io.gcp.gcsio_integration_test:GcsIOIntegrationTest \ - -PkmsKeyName=KMS_KEY_NAME + -Dtests=apache_beam.io.gcp.gcsio_integration_test:GcsIOIntegrationTest \ + -DkmsKeyName=KMS_KEY_NAME """ from __future__ import absolute_import diff --git a/sdks/python/build.gradle b/sdks/python/build.gradle index 3c9cd93..495a724 100644 --- a/sdks/python/build.gradle +++ b/sdks/python/build.gradle @@ -20,6 +20,7 @@ import org.apache.tools.ant.taskdefs.condition.Os plugins { id 'org.apache.beam.module' } applyPythonNature() +enablePythonPerformanceTest() /*************************************************************************************************/ @@ -197,36 +198,6 @@ def portableWordCountTask(name, streaming) { } } -// Run single or a set of integration tests with provided test options and pipeline options. -task integrationTest(dependsOn: ['installGcpTest', 'sdist']) { - doLast { - def argMap = [:] - - // Build test options that configures test environment and framework - def testOptions = basicTestOpts - if (project.hasProperty('testOptions')) - // Customize test options by adding more nose flags to -Ptest_opt in commandline - testOptions = project.property('testOptions') - if (project.hasProperty('attr')) - testOptions += ["--attr=${project.property('attr')}"] - if (project.hasProperty('tests')) - testOptions += ["--tests=${project.property('tests')}"] - argMap["test_opts"] = testOptions - - // Build pipeline options that configures pipeline job - if (project.hasProperty('pipelineOptions')) - argMap["pipeline_opts"] = project.property('pipelineOptions') - if (project.hasProperty('kmsKeyName')) - argMap["kms_key_name"] = project.property('kmsKeyName') - - def cmdArgs = project.mapToArgString(argMap) - exec { - executable 'sh' - args '-c', ". ${project.ext.envdir}/bin/activate && ./scripts/run_integration_test.sh $cmdArgs" - } - } -} - // Run PostCommit integration tests on default runner (TestDataflowRunner) task postCommitIT(dependsOn: ['installGcpTest', 'sdist']) { dependsOn ":beam-runners-google-cloud-dataflow-java-fn-api-worker:shadowJar" diff --git a/sdks/python/scripts/run_integration_test.sh b/sdks/python/scripts/run_integration_test.sh index af9b965..ba72a14 100755 --- a/sdks/python/scripts/run_integration_test.sh +++ b/sdks/python/scripts/run_integration_test.sh @@ -153,20 +153,23 @@ set -o errexit ########################################################################### -# Build pipeline options if not provided in --pipeline_opts from commandline -if [[ -z $PIPELINE_OPTS ]]; then +# Check that the script is running in a known directory. +if [[ $PWD != *sdks/python* ]]; then + echo 'Unable to locate Apache Beam Python SDK root directory' + exit 1 +fi - # Check that the script is running in a known directory. - if [[ $PWD != *sdks/python* ]]; then - echo 'Unable to locate Apache Beam Python SDK root directory' - exit 1 - fi +# Go to the Apache Beam Python SDK root +if [[ $PWD != *sdks/python ]]; then + cd $(pwd | sed 's/sdks\/python.*/sdks\/python/') +fi - # Go to the Apache Beam Python SDK root - if [[ $PWD != *sdks/python ]]; then - cd $(pwd | sed 's/sdks\/python.*/sdks\/python/') - fi + +########################################################################### +# Build pipeline options if not provided in --pipeline_opts from commandline + +if [[ -z $PIPELINE_OPTS ]]; then # Create a tarball if not exists if [[ $(find ${SDK_LOCATION} 2> /dev/null) ]]; then diff --git a/sdks/python/test-suites/dataflow/py3/build.gradle b/sdks/python/test-suites/dataflow/py3/build.gradle index 66cc985..7d31cb3 100644 --- a/sdks/python/test-suites/dataflow/py3/build.gradle +++ b/sdks/python/test-suites/dataflow/py3/build.gradle @@ -18,6 +18,7 @@ apply plugin: org.apache.beam.gradle.BeamModulePlugin applyPythonNature() +enablePythonPerformanceTest() // Required to setup a Python 3 virtualenv. pythonVersion = '3.5'