See
<https://builds.apache.org/job/beam_PostCommit_Python_ValidatesRunner_Dataflow/1358/display/redirect>
------------------------------------------
[...truncated 1.02 KB...]
> git checkout -f 9e3e9c4d0a0dc1574c8956c7f8379b37ba262cb2
Commit message: "This closes #5028"
> git rev-list --no-walk 9e3e9c4d0a0dc1574c8956c7f8379b37ba262cb2 # timeout=10
Cleaning workspace
> git rev-parse --verify HEAD # timeout=10
Resetting working tree
> git reset --hard # timeout=10
> git clean -fdx # timeout=10
[EnvInject] - Executing scripts and injecting environment variables after the
SCM step.
[EnvInject] - Injecting as environment variables the properties content
SPARK_LOCAL_IP=127.0.0.1
[EnvInject] - Variables injected successfully.
[beam_PostCommit_Python_ValidatesRunner_Dataflow] $ /bin/bash -xe
/tmp/jenkins4661799742538718044.sh
+ cd src
+ bash sdks/python/run_validatesrunner.sh
# pip install --user installation location.
LOCAL_PATH=$HOME/.local/bin/
# INFRA does not install virtualenv
pip install virtualenv --user
Requirement already satisfied: virtualenv in
/home/jenkins/.local/lib/python2.7/site-packages (15.2.0)
cheetah 2.4.4 requires Markdown>=2.0.1, which is not installed.
apache-beam 2.5.0.dev0 requires hdfs<3.0.0,>=2.1.0, which is not installed.
apache-beam 2.5.0.dev0 requires pytz>=2018.3, which is not installed.
apache-beam 2.5.0.dev0 has requirement grpcio<2,>=1.8, but you'll have grpcio
1.4.0 which is incompatible.
# Virtualenv for the rest of the script to run setup & e2e tests
${LOCAL_PATH}/virtualenv sdks/python
New python executable in
<https://builds.apache.org/job/beam_PostCommit_Python_ValidatesRunner_Dataflow/ws/src/sdks/python/bin/python>
Installing setuptools, pip, wheel...done.
. sdks/python/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_Python_ValidatesRunner_Dataflow/ws/src/sdks/python";>
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="$PS1"
if [ "x" != x ] ; then
PS1="$PS1"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) $PS1"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
cd sdks/python
pip install -e .[gcp,test]
Obtaining
file://<https://builds.apache.org/job/beam_PostCommit_Python_ValidatesRunner_Dataflow/ws/src/sdks/python>
Collecting avro<2.0.0,>=1.8.1 (from apache-beam==2.5.0.dev0)
<https://builds.apache.org/job/beam_PostCommit_Python_ValidatesRunner_Dataflow/ws/src/sdks/python/local/lib/python2.7/site-packages/pip/_vendor/urllib3/util/ssl_.py>:339:
SNIMissingWarning: An HTTPS request has been made, but the SNI (Subject Name
Indication) extension to TLS is not available on this platform. This may cause
the server to present an incorrect TLS certificate, which can cause validation
failures. You can upgrade to a newer version of Python to solve this. For more
information, see
https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings
SNIMissingWarning
<https://builds.apache.org/job/beam_PostCommit_Python_ValidatesRunner_Dataflow/ws/src/sdks/python/local/lib/python2.7/site-packages/pip/_vendor/urllib3/util/ssl_.py>:137:
InsecurePlatformWarning: A true SSLContext object is not available. This
prevents urllib3 from configuring SSL appropriately and may cause certain SSL
connections to fail. You can upgrade to a newer version of Python to solve
this. For more information, see
https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings
InsecurePlatformWarning
Collecting crcmod<2.0,>=1.7 (from apache-beam==2.5.0.dev0)
Collecting dill==0.2.6 (from apache-beam==2.5.0.dev0)
Collecting grpcio<2,>=1.8 (from apache-beam==2.5.0.dev0)
Using cached grpcio-1.11.0-cp27-cp27mu-manylinux1_x86_64.whl
Collecting hdfs<3.0.0,>=2.1.0 (from apache-beam==2.5.0.dev0)
Collecting httplib2<0.10,>=0.8 (from apache-beam==2.5.0.dev0)
Collecting mock<3.0.0,>=1.0.1 (from apache-beam==2.5.0.dev0)
Using cached mock-2.0.0-py2.py3-none-any.whl
Collecting oauth2client<5,>=2.0.1 (from apache-beam==2.5.0.dev0)
Using cached oauth2client-4.1.2-py2.py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1 (from apache-beam==2.5.0.dev0)
Using cached protobuf-3.5.2.post1-cp27-cp27mu-manylinux1_x86_64.whl
Collecting pytz>=2018.3 (from apache-beam==2.5.0.dev0)
Using cached pytz-2018.4-py2.py3-none-any.whl
Collecting pyyaml<4.0.0,>=3.12 (from apache-beam==2.5.0.dev0)
Collecting pyvcf<0.7.0,>=0.6.8 (from apache-beam==2.5.0.dev0)
Collecting six<1.12,>=1.9 (from apache-beam==2.5.0.dev0)
Using cached six-1.11.0-py2.py3-none-any.whl
Collecting typing<3.7.0,>=3.6.0 (from apache-beam==2.5.0.dev0)
Using cached typing-3.6.4-py2-none-any.whl
Collecting futures<4.0.0,>=3.1.1 (from apache-beam==2.5.0.dev0)
Using cached futures-3.2.0-py2-none-any.whl
Collecting google-apitools<=0.5.20,>=0.5.18 (from apache-beam==2.5.0.dev0)
Using cached google_apitools-0.5.20-py2-none-any.whl
Collecting proto-google-cloud-datastore-v1<=0.90.4,>=0.90.0 (from
apache-beam==2.5.0.dev0)
Collecting googledatastore==7.0.1 (from apache-beam==2.5.0.dev0)
Collecting google-cloud-pubsub==0.26.0 (from apache-beam==2.5.0.dev0)
Using cached google_cloud_pubsub-0.26.0-py2.py3-none-any.whl
Collecting proto-google-cloud-pubsub-v1==0.15.4 (from apache-beam==2.5.0.dev0)
Collecting google-cloud-bigquery==0.25.0 (from apache-beam==2.5.0.dev0)
Using cached google_cloud_bigquery-0.25.0-py2.py3-none-any.whl
Collecting nose>=1.3.7 (from apache-beam==2.5.0.dev0)
Using cached nose-1.3.7-py2-none-any.whl
Collecting pyhamcrest<2.0,>=1.9 (from apache-beam==2.5.0.dev0)
Using cached PyHamcrest-1.9.0-py2.py3-none-any.whl
Collecting enum34>=1.0.4 (from grpcio<2,>=1.8->apache-beam==2.5.0.dev0)
Using cached enum34-1.1.6-py2-none-any.whl
Collecting requests>=2.7.0 (from hdfs<3.0.0,>=2.1.0->apache-beam==2.5.0.dev0)
Using cached requests-2.18.4-py2.py3-none-any.whl
Collecting docopt (from hdfs<3.0.0,>=2.1.0->apache-beam==2.5.0.dev0)
Collecting funcsigs>=1; python_version < "3.3" (from
mock<3.0.0,>=1.0.1->apache-beam==2.5.0.dev0)
Using cached funcsigs-1.0.2-py2.py3-none-any.whl
Collecting pbr>=0.11 (from mock<3.0.0,>=1.0.1->apache-beam==2.5.0.dev0)
Using cached pbr-4.0.2-py2.py3-none-any.whl
Collecting rsa>=3.1.4 (from oauth2client<5,>=2.0.1->apache-beam==2.5.0.dev0)
Using cached rsa-3.4.2-py2.py3-none-any.whl
Collecting pyasn1-modules>=0.0.5 (from
oauth2client<5,>=2.0.1->apache-beam==2.5.0.dev0)
Using cached pyasn1_modules-0.2.1-py2.py3-none-any.whl
Collecting pyasn1>=0.1.7 (from oauth2client<5,>=2.0.1->apache-beam==2.5.0.dev0)
Using cached pyasn1-0.4.2-py2.py3-none-any.whl
Requirement already satisfied: setuptools in ./lib/python2.7/site-packages
(from protobuf<4,>=3.5.0.post1->apache-beam==2.5.0.dev0) (39.0.1)
Collecting fasteners>=0.14 (from
google-apitools<=0.5.20,>=0.5.18->apache-beam==2.5.0.dev0)
Using cached fasteners-0.14.1-py2.py3-none-any.whl
Collecting googleapis-common-protos<2.0dev,>=1.5.2 (from
proto-google-cloud-datastore-v1<=0.90.4,>=0.90.0->apache-beam==2.5.0.dev0)
Collecting google-cloud-core<0.26dev,>=0.25.0 (from
google-cloud-pubsub==0.26.0->apache-beam==2.5.0.dev0)
Using cached google_cloud_core-0.25.0-py2.py3-none-any.whl
Collecting gapic-google-cloud-pubsub-v1<0.16dev,>=0.15.0 (from
google-cloud-pubsub==0.26.0->apache-beam==2.5.0.dev0)
Collecting urllib3<1.23,>=1.21.1 (from
requests>=2.7.0->hdfs<3.0.0,>=2.1.0->apache-beam==2.5.0.dev0)
Using cached urllib3-1.22-py2.py3-none-any.whl
Collecting idna<2.7,>=2.5 (from
requests>=2.7.0->hdfs<3.0.0,>=2.1.0->apache-beam==2.5.0.dev0)
Using cached idna-2.6-py2.py3-none-any.whl
Collecting chardet<3.1.0,>=3.0.2 (from
requests>=2.7.0->hdfs<3.0.0,>=2.1.0->apache-beam==2.5.0.dev0)
Using cached chardet-3.0.4-py2.py3-none-any.whl
Collecting certifi>=2017.4.17 (from
requests>=2.7.0->hdfs<3.0.0,>=2.1.0->apache-beam==2.5.0.dev0)
Using cached certifi-2018.1.18-py2.py3-none-any.whl
Collecting monotonic>=0.1 (from
fasteners>=0.14->google-apitools<=0.5.20,>=0.5.18->apache-beam==2.5.0.dev0)
Using cached monotonic-1.4-py2.py3-none-any.whl
Collecting google-auth<2.0.0dev,>=0.4.0 (from
google-cloud-core<0.26dev,>=0.25.0->google-cloud-pubsub==0.26.0->apache-beam==2.5.0.dev0)
Using cached google_auth-1.4.1-py2.py3-none-any.whl
Collecting google-auth-httplib2 (from
google-cloud-core<0.26dev,>=0.25.0->google-cloud-pubsub==0.26.0->apache-beam==2.5.0.dev0)
Using cached google_auth_httplib2-0.0.3-py2.py3-none-any.whl
Collecting google-gax<0.16dev,>=0.15.7 (from
gapic-google-cloud-pubsub-v1<0.16dev,>=0.15.0->google-cloud-pubsub==0.26.0->apache-beam==2.5.0.dev0)
Using cached google_gax-0.15.16-py2.py3-none-any.whl
Collecting grpc-google-iam-v1<0.12dev,>=0.11.1 (from
gapic-google-cloud-pubsub-v1<0.16dev,>=0.15.0->google-cloud-pubsub==0.26.0->apache-beam==2.5.0.dev0)
Collecting cachetools>=2.0.0 (from
google-auth<2.0.0dev,>=0.4.0->google-cloud-core<0.26dev,>=0.25.0->google-cloud-pubsub==0.26.0->apache-beam==2.5.0.dev0)
Using cached cachetools-2.0.1-py2.py3-none-any.whl
Collecting future<0.17dev,>=0.16.0 (from
google-gax<0.16dev,>=0.15.7->gapic-google-cloud-pubsub-v1<0.16dev,>=0.15.0->google-cloud-pubsub==0.26.0->apache-beam==2.5.0.dev0)
Collecting ply==3.8 (from
google-gax<0.16dev,>=0.15.7->gapic-google-cloud-pubsub-v1<0.16dev,>=0.15.0->google-cloud-pubsub==0.26.0->apache-beam==2.5.0.dev0)
gapic-google-cloud-pubsub-v1 0.15.4 has requirement
oauth2client<4.0dev,>=2.0.0, but you'll have oauth2client 4.1.2 which is
incompatible.
googledatastore 7.0.1 has requirement oauth2client<4.0.0,>=2.0.1, but you'll
have oauth2client 4.1.2 which is incompatible.
proto-google-cloud-pubsub-v1 0.15.4 has requirement
oauth2client<4.0dev,>=2.0.0, but you'll have oauth2client 4.1.2 which is
incompatible.
proto-google-cloud-datastore-v1 0.90.4 has requirement
oauth2client<4.0dev,>=2.0.0, but you'll have oauth2client 4.1.2 which is
incompatible.
Installing collected packages: avro, crcmod, dill, futures, six, protobuf,
enum34, grpcio, urllib3, idna, chardet, certifi, requests, docopt, hdfs,
httplib2, funcsigs, pbr, mock, pyasn1, rsa, pyasn1-modules, oauth2client, pytz,
pyyaml, pyvcf, typing, monotonic, fasteners, google-apitools,
googleapis-common-protos, proto-google-cloud-datastore-v1, googledatastore,
cachetools, google-auth, google-auth-httplib2, google-cloud-core, future, ply,
google-gax, grpc-google-iam-v1, proto-google-cloud-pubsub-v1,
gapic-google-cloud-pubsub-v1, google-cloud-pubsub, google-cloud-bigquery, nose,
pyhamcrest, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-1.8.2 cachetools-2.0.1
certifi-2018.1.18 chardet-3.0.4 crcmod-1.7 dill-0.2.6 docopt-0.6.2 enum34-1.1.6
fasteners-0.14.1 funcsigs-1.0.2 future-0.16.0 futures-3.2.0
gapic-google-cloud-pubsub-v1-0.15.4 google-apitools-0.5.20 google-auth-1.4.1
google-auth-httplib2-0.0.3 google-cloud-bigquery-0.25.0
google-cloud-core-0.25.0 google-cloud-pubsub-0.26.0 google-gax-0.15.16
googleapis-common-protos-1.5.3 googledatastore-7.0.1 grpc-google-iam-v1-0.11.4
grpcio-1.11.0 hdfs-2.1.0 httplib2-0.9.2 idna-2.6 mock-2.0.0 monotonic-1.4
nose-1.3.7 oauth2client-4.1.2 pbr-4.0.2 ply-3.8
proto-google-cloud-datastore-v1-0.90.4 proto-google-cloud-pubsub-v1-0.15.4
protobuf-3.5.2.post1 pyasn1-0.4.2 pyasn1-modules-0.2.1 pyhamcrest-1.9.0
pytz-2018.4 pyvcf-0.6.8 pyyaml-3.12 requests-2.18.4 rsa-3.4.2 six-1.11.0
typing-3.6.4 urllib3-1.22
# Where to store integration test outputs.
GCS_LOCATION=gs://temp-storage-for-end-to-end-tests
PROJECT=apache-beam-testing
# Create a tarball
python setup.py sdist
<https://builds.apache.org/job/beam_PostCommit_Python_ValidatesRunner_Dataflow/ws/src/sdks/python/local/lib/python2.7/site-packages/setuptools/dist.py>:397:
UserWarning: Normalizing '2.5.0.dev' to '2.5.0.dev0'
normalized_version,
running sdist
<https://builds.apache.org/job/beam_PostCommit_Python_ValidatesRunner_Dataflow/ws/src/sdks/python/gen_protos.py>:51:
UserWarning: Installing grpcio-tools is recommended for development.
warnings.warn('Installing grpcio-tools is recommended for development.')
WARNING:root:Installing grpcio-tools into
<https://builds.apache.org/job/beam_PostCommit_Python_ValidatesRunner_Dataflow/ws/src/sdks/python/.eggs/grpcio-wheels>
Process Process-1:
Traceback (most recent call last):
File "/usr/lib/python2.7/multiprocessing/process.py", line 258, in _bootstrap
self.run()
File "/usr/lib/python2.7/multiprocessing/process.py", line 114, in run
self._target(*self._args, **self._kwargs)
File
"<https://builds.apache.org/job/beam_PostCommit_Python_ValidatesRunner_Dataflow/ws/src/sdks/python/gen_protos.py";,>
line 149, in _install_grpcio_tools_and_generate_proto_files
shutil.rmtree(build_path)
File "/usr/lib/python2.7/shutil.py", line 239, in rmtree
onerror(os.listdir, path, sys.exc_info())
File "/usr/lib/python2.7/shutil.py", line 237, in rmtree
names = os.listdir(path)
OSError: [Errno 2] No such file or directory:
'<https://builds.apache.org/job/beam_PostCommit_Python_ValidatesRunner_Dataflow/ws/src/sdks/python/.eggs/grpcio-wheels-build'>
Traceback (most recent call last):
File "setup.py", line 235, in <module>
'test': generate_protos_first(test),
File
"<https://builds.apache.org/job/beam_PostCommit_Python_ValidatesRunner_Dataflow/ws/src/sdks/python/local/lib/python2.7/site-packages/setuptools/__init__.py";,>
line 129, in setup
return distutils.core.setup(**attrs)
File "/usr/lib/python2.7/distutils/core.py", line 151, in setup
dist.run_commands()
File "/usr/lib/python2.7/distutils/dist.py", line 953, in run_commands
self.run_command(cmd)
File "/usr/lib/python2.7/distutils/dist.py", line 972, in run_command
cmd_obj.run()
File "setup.py", line 141, in run
gen_protos.generate_proto_files()
File
"<https://builds.apache.org/job/beam_PostCommit_Python_ValidatesRunner_Dataflow/ws/src/sdks/python/gen_protos.py";,>
line 97, in generate_proto_files
raise ValueError("Proto generation failed (see log for details).")
ValueError: Proto generation failed (see log for details).
Build step 'Execute shell' marked build as failure
