[
https://issues.apache.org/jira/browse/BEAM-4021?focusedWorklogId=88289&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-88289
]
ASF GitHub Bot logged work on BEAM-4021:
----------------------------------------
Author: ASF GitHub Bot
Created on: 06/Apr/18 00:19
Start Date: 06/Apr/18 00:19
Worklog Time Spent: 10m
Work Description: aaltay closed pull request #5032: [BEAM-4021] Work
around tox bug.
URL: https://github.com/apache/beam/pull/5032
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/sdks/python/build.gradle b/sdks/python/build.gradle
index be17644e8d3..6c0f7f8e45a 100644
--- a/sdks/python/build.gradle
+++ b/sdks/python/build.gradle
@@ -26,6 +26,7 @@ task test {}
check.dependsOn test
def envdir = "${project.buildDir}/gradleenv"
+def tox_opts = "-c tox.ini --recreate"
task setupVirtualenv {
doLast {
@@ -37,7 +38,10 @@ task setupVirtualenv {
args '-c', ". ${envdir}/bin/activate && pip install --upgrade tox"
}
}
- outputs.files("${envdir}/bin/tox")
+ // Gradle will delete outputs whenever it thinks they are stale. Putting a
+ // specific binary here could make gradle delete it while pip will believe
+ // the package is fully installed.
+ outputs.dirs(envdir)
}
task sdist(dependsOn: 'setupVirtualenv') {
@@ -77,7 +81,7 @@ task lintPy27(dependsOn: 'setupVirtualenv') {
doLast {
exec {
executable 'sh'
- args '-c', ". ${envdir}/bin/activate && tox -e py27-lint -c tox.ini"
+ args '-c', ". ${envdir}/bin/activate && tox ${tox_opts} -e py27-lint"
}
}
}
@@ -87,7 +91,7 @@ task lintPy3(dependsOn: 'setupVirtualenv') {
doLast {
exec {
executable 'sh'
- args '-c', ". ${envdir}/bin/activate && tox -e py3-lint -c tox.ini"
+ args '-c', ". ${envdir}/bin/activate && tox ${tox_opts} -e py3-lint"
}
}
}
@@ -97,7 +101,7 @@ task testGcp(dependsOn: 'setupVirtualenv') {
doLast {
exec {
executable 'sh'
- args '-c', ". ${envdir}/bin/activate && tox -e py27-gcp -c tox.ini"
+ args '-c', ". ${envdir}/bin/activate && tox ${tox_opts} -e py27-gcp"
}
}
}
@@ -107,7 +111,7 @@ task testPython(dependsOn: 'setupVirtualenv') {
doLast {
exec {
executable 'sh'
- args '-c', ". ${envdir}/bin/activate && tox -e py27 -c tox.ini"
+ args '-c', ". ${envdir}/bin/activate && tox ${tox_opts} -e py27"
}
}
}
@@ -117,7 +121,7 @@ task testCython(dependsOn: 'setupVirtualenv') {
doLast {
exec {
executable 'sh'
- args '-c', ". ${envdir}/bin/activate && tox -e py27-cython2 -c tox.ini"
+ args '-c', ". ${envdir}/bin/activate && tox ${tox_opts} -e py27-cython"
}
}
}
@@ -131,7 +135,7 @@ task docs(dependsOn: 'setupVirtualenv') {
doLast {
exec {
executable 'sh'
- args '-c', ". ${envdir}/bin/activate && tox -e docs -c tox.ini"
+ args '-c', ". ${envdir}/bin/activate && tox ${tox_opts} -e docs"
}
}
}
@@ -141,7 +145,7 @@ task cover(dependsOn: 'setupVirtualenv') {
doLast {
exec {
executable 'sh'
- args '-c', ". ${envdir}/bin/activate && tox -e cover -c tox.ini"
+ args '-c', ". ${envdir}/bin/activate && tox ${tox_opts} -e cover"
}
}
}
diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini
index add9bbfb2cc..ff88ac42fa8 100644
--- a/sdks/python/tox.ini
+++ b/sdks/python/tox.ini
@@ -17,7 +17,7 @@
[tox]
# new environments will be excluded by default unless explicitly added to
envlist.
-envlist = py27,py27-{gcp,cython2,lint},py3-lint,docs
+envlist = py27,py27-{gcp,cython,lint},py3-lint,docs
toxworkdir = {toxinidir}/target/.tox
[pycodestyle]
@@ -33,6 +33,16 @@ extras = test
whitelist_externals =
find
time
+deps =
+ grpcio-tools==1.3.5
+ cython: cython==0.26.1
+
+# These 2 magic command overrides are required for Jenkins builds.
+# Otherwise we get "OSError: [Errno 2] No such file or directory" errors.
+# Source:
+# https://github.com/tox-dev/tox/issues/123#issuecomment-284714629
+install_command = {envbindir}/python {envbindir}/pip install {opts} {packages}
+list_dependencies_command = {envbindir}/python {envbindir}/pip freeze
[testenv:py27]
commands =
@@ -43,15 +53,12 @@ commands =
python setup.py test
{toxinidir}/run_tox_cleanup.sh
-# This environment will fail in Jenkins if named "py27-cython".
-[testenv:py27-cython2]
+[testenv:py27-cython]
# cython tests are only expected to work in linux (2.x and 3.x)
# If we want to add other platforms in the future, it should be:
# `platform = linux2|darwin|...`
# See https://docs.python.org/2/library/sys.html#sys.platform for platform
codes
platform = linux2
-deps =
- cython==0.26.1
commands =
python --version
pip --version
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
-------------------
Worklog Id: (was: 88289)
Time Spent: 1h 40m (was: 1.5h)
> "No such file or directory" in beam_PreCommit_Python_GradleBuild
> ----------------------------------------------------------------
>
> Key: BEAM-4021
> URL: https://issues.apache.org/jira/browse/BEAM-4021
> Project: Beam
> Issue Type: Sub-task
> Components: testing
> Reporter: Udi Meiri
> Assignee: Udi Meiri
> Priority: Major
> Time Spent: 1h 40m
> Remaining Estimate: 0h
>
> Seems to only happen in this working directory:
> {{/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_GradleBuild}}
> but not this:
> {{/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_GradleBuild@2}}
> {{ERROR: invocation failed (errno 2), args:
> ['/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_GradleBuild/src/sdks/python/target/.tox/py27-cython2/bin/pip',
> 'install', 'cython==0.26.1'], cwd:
> /home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_GradleBuild/src/sdks/python}}
> {{ Traceback (most recent call last):}}
> {{ File
> "/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_GradleBuild/src/sdks/python/build/gradleenv/bin/tox",
> line 11, in <module>}}
> {{ sys.exit(run_main())}}
> {{ File
> "/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_GradleBuild/src/sdks/python/build/gradleenv/local/lib/python2.7/site-packages/tox/session.py",
> line 40, in run_main}}
> {{ main(args)}}
> {{ File
> "/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_GradleBuild/src/sdks/python/build/gradleenv/local/lib/python2.7/site-packages/tox/session.py",
> line 46, in main}}
> {{ retcode = Session(config).runcommand()}}
> {{ File
> "/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_GradleBuild/src/sdks/python/build/gradleenv/local/lib/python2.7/site-packages/tox/session.py",
> line 415, in runcommand}}
> {{ return self.subcommand_test()}}
> {{ File
> "/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_GradleBuild/src/sdks/python/build/gradleenv/local/lib/python2.7/site-packages/tox/session.py",
> line 599, in subcommand_test}}
> {{ if self.setupenv(venv):}}
> {{ File
> "/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_GradleBuild/src/sdks/python/build/gradleenv/local/lib/python2.7/site-packages/tox/session.py",
> line 491, in setupenv}}
> {{ status = venv.update(action=action)}}
> {{ File
> "/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_GradleBuild/src/sdks/python/build/gradleenv/local/lib/python2.7/site-packages/tox/venv.py",
> line 171, in update}}
> {{ self.hook.tox_testenv_install_deps(action=action, venv=self)}}
> {{ File
> "/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_GradleBuild/src/sdks/python/build/gradleenv/local/lib/python2.7/site-packages/pluggy/__init__.py",
> line 617, in __call__}}
> {{ return self._hookexec(self, self._nonwrappers + self._wrappers, kwargs)}}
> {{ File
> "/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_GradleBuild/src/sdks/python/build/gradleenv/local/lib/python2.7/site-packages/pluggy/__init__.py",
> line 222, in _hookexec}}
> {{ return self._inner_hookexec(hook, methods, kwargs)}}
> {{ File
> "/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_GradleBuild/src/sdks/python/build/gradleenv/local/lib/python2.7/site-packages/pluggy/__init__.py",
> line 216, in <lambda>}}
> {{ firstresult=hook.spec_opts.get('firstresult'),}}
> {{ File
> "/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_GradleBuild/src/sdks/python/build/gradleenv/local/lib/python2.7/site-packages/pluggy/callers.py",
> line 201, in _multicall}}
> {{ return outcome.get_result()}}
> {{ File
> "/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_GradleBuild/src/sdks/python/build/gradleenv/local/lib/python2.7/site-packages/pluggy/callers.py",
> line 77, in get_result}}
> {{ _reraise(*ex) # noqa}}
> {{ File
> "/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_GradleBuild/src/sdks/python/build/gradleenv/local/lib/python2.7/site-packages/pluggy/callers.py",
> line 180, in _multicall}}
> {{ res = hook_impl.function(*args)}}
> {{ File
> "/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_GradleBuild/src/sdks/python/build/gradleenv/local/lib/python2.7/site-packages/tox/venv.py",
> line 452, in tox_testenv_install_deps}}
> {{ venv._install(deps, action=action)}}
> {{ File
> "/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_GradleBuild/src/sdks/python/build/gradleenv/local/lib/python2.7/site-packages/tox/venv.py",
> line 331, in _install}}
> {{ action=action)}}
> {{ File
> "/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_GradleBuild/src/sdks/python/build/gradleenv/local/lib/python2.7/site-packages/tox/venv.py",
> line 303, in run_install_command}}
> {{ action=action, redirect=self.session.report.verbosity < 2)}}
> {{ File
> "/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_GradleBuild/src/sdks/python/build/gradleenv/local/lib/python2.7/site-packages/tox/venv.py",
> line 409, in _pcall}}
> {{ redirect=redirect, ignore_ret=ignore_ret)}}
> {{ File
> "/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_GradleBuild/src/sdks/python/build/gradleenv/local/lib/python2.7/site-packages/tox/session.py",
> line 150, in popen}}
> {{ stdout=stdout, stderr=subprocess.STDOUT)}}
> {{ File
> "/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_GradleBuild/src/sdks/python/build/gradleenv/local/lib/python2.7/site-packages/tox/session.py",
> line 243, in _popen}}
> {{ stdout=stdout, stderr=stderr, env=env)}}
> {{ File "/usr/lib/python2.7/subprocess.py", line 710, in __init__}}
> {{ errread, errwrite)}}
> {{ File "/usr/lib/python2.7/subprocess.py", line 1327, in _execute_child}}
> {{ raise child_exception}}
> {{ OSError: [Errno 2] No such file or directory}}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
