This is an automated email from the ASF dual-hosted git repository.
altay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 943dc60 Fix beam_PerformanceTests_Python (#6142)
943dc60 is described below
commit 943dc6062713fabb8d3eafe4508d97db758978f6
Author: Mark Liu <[email protected]>
AuthorDate: Mon Aug 6 10:14:16 2018 -0700
Fix beam_PerformanceTests_Python (#6142)
* [BEAM-3230] Fix beam_PerformanceTests_Python
---
.test-infra/jenkins/CommonJobProperties.groovy | 32 ++++++++++++++--------
.../jenkins/job_PerformanceTests_Python.groovy | 12 ++++----
sdks/python/setup.py | 8 +++++-
3 files changed, 34 insertions(+), 18 deletions(-)
diff --git a/.test-infra/jenkins/CommonJobProperties.groovy
b/.test-infra/jenkins/CommonJobProperties.groovy
index d267f87..2a5d56b 100644
--- a/.test-infra/jenkins/CommonJobProperties.groovy
+++ b/.test-infra/jenkins/CommonJobProperties.groovy
@@ -315,31 +315,41 @@ class CommonJobProperties {
// Adds the standard performance test job steps.
static def buildPerformanceTest(def context, def argMap) {
def pkbArgs = genPerformanceArgs(argMap)
+
+ // Absolute path of project root and virtualenv path of Beam and Perfkit.
+ def beam_root = makePathAbsolute(checkoutDir)
+ def perfkit_root = makePathAbsolute("PerfKitBenchmarker")
+ def beam_env = makePathAbsolute("env/.beam_env")
+ def perfkit_env = makePathAbsolute("env/.perfkit_env")
+
context.steps {
// Clean up environment.
- shell('rm -rf PerfKitBenchmarker')
- shell('rm -rf .beam_env')
- shell('rm -rf .perfkit_env')
+ shell("rm -rf ${perfkit_root}")
+ shell("rm -rf ${beam_env}")
+ shell("rm -rf ${perfkit_env}")
// create new VirtualEnv, inherit already existing packages
- shell('virtualenv .beam_env --system-site-packages')
- shell('virtualenv .perfkit_env --system-site-packages')
+ shell("virtualenv ${beam_env} --system-site-packages")
+ shell("virtualenv ${perfkit_env} --system-site-packages")
// update setuptools and pip
- shell('.beam_env/bin/pip install --upgrade setuptools pip')
- shell('.perfkit_env/bin/pip install --upgrade setuptools pip')
+ shell("${beam_env}/bin/pip install --upgrade setuptools pip
grpcio-tools==1.3.5")
+ shell("${perfkit_env}/bin/pip install --upgrade setuptools pip")
// Clone appropriate perfkit branch
- shell('git clone
https://github.com/GoogleCloudPlatform/PerfKitBenchmarker.git')
+ shell("git clone
https://github.com/GoogleCloudPlatform/PerfKitBenchmarker.git ${perfkit_root}")
// Install job requirements for Python SDK.
- shell('.beam_env/bin/pip install -e ' +
CommonJobProperties.checkoutDir + '/sdks/python/[gcp,test]')
+ shell("${beam_env}/bin/pip install -e
${beam_root}/sdks/python/[gcp,test]")
+
+ // Build PythonSDK tar ball.
+ shell("(cd ${beam_root}/sdks/python && ${beam_env}/bin/python setup.py
sdist --dist-dir=target)")
// Install Perfkit benchmark requirements.
- shell('.perfkit_env/bin/pip install -r
PerfKitBenchmarker/requirements.txt')
+ shell("${perfkit_env}/bin/pip install -r
${perfkit_root}/requirements.txt")
// Launch performance test.
- shell(".perfkit_env/bin/python PerfKitBenchmarker/pkb.py $pkbArgs")
+ shell("${perfkit_env}/bin/python ${perfkit_root}/pkb.py ${pkbArgs}")
}
}
diff --git a/.test-infra/jenkins/job_PerformanceTests_Python.groovy
b/.test-infra/jenkins/job_PerformanceTests_Python.groovy
index e9591fb..63b9cc9 100644
--- a/.test-infra/jenkins/job_PerformanceTests_Python.groovy
+++ b/.test-infra/jenkins/job_PerformanceTests_Python.groovy
@@ -47,13 +47,13 @@ job('beam_PerformanceTests_Python'){
def pipelineArgsJoined = pipelineArgList.join(',')
def argMap = [
- beam_sdk : 'python',
- benchmarks: 'beam_integration_benchmark',
- beam_it_args: pipelineArgsJoined
+ beam_sdk : 'python',
+ benchmarks : 'beam_integration_benchmark',
+ bigquery_table : 'beam_performance.wordcount_py_pkb_results',
+ beam_it_class :
'apache_beam.examples.wordcount_it_test:WordCountIT.test_wordcount_it',
+ beam_prebuilt : 'true', // skip beam prebuild
+ beam_it_args : pipelineArgsJoined,
]
commonJobProperties.buildPerformanceTest(delegate, argMap)
-
- // [BEAM-3809] Python performance tests are failing.
- disabled()
}
diff --git a/sdks/python/setup.py b/sdks/python/setup.py
index 2cdd374..ce7591e 100644
--- a/sdks/python/setup.py
+++ b/sdks/python/setup.py
@@ -38,7 +38,13 @@ from setuptools.command.test import test
def get_version():
global_names = {}
- exec(open(os.path.normpath('./apache_beam/version.py')).read(),
global_names) # pylint: disable=exec-used
+ exec( # pylint: disable=exec-used
+ open(os.path.join(
+ os.path.dirname(os.path.abspath(__file__)),
+ 'apache_beam/version.py')
+ ).read(),
+ global_names
+ )
return global_names['__version__']
