See
<https://ci-beam.apache.org/job/beam_PostCommit_PortableJar_Flink/4681/display/redirect?page=changes>
Changes:
[noreply] Minor: Add 2.38.0 section to CHANGES.md (#16804)
------------------------------------------
[...truncated 789.64 KB...]
# Verify docker command exists
command -v docker
docker -v
/usr/bin/docker
Docker version 18.09.4, build d14af54
# Verify container has already been built
echo "Checking for Docker image ${PYTHON_CONTAINER_IMAGE}"
Checking for Docker image apache/beam_python3.6_sdk:2.38.0.dev
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.6_sdk:2.38.0.dev
# Set up Python environment
python$PYTHON_VERSION -m venv $ENV_DIR
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
# reset old environment variables
if [ -n "${_OLD_VIRTUAL_PATH:-}" ] ; then
PATH="${_OLD_VIRTUAL_PATH:-}"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if [ -n "${_OLD_VIRTUAL_PYTHONHOME:-}" ] ; then
PYTHONHOME="${_OLD_VIRTUAL_PYTHONHOME:-}"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then
hash -r
fi
if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then
PS1="${_OLD_VIRTUAL_PS1:-}"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "$1" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="/home/jenkins/jenkins-slave/workspace/beam_PostCommit_PortableJar_Flink/src/build/gradleenv/2022703440"
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
# this will fail if PYTHONHOME is set to the empty string (which is bad anyway)
# could use `if (set -u; : $PYTHONHOME) ;` in bash
if [ -n "${PYTHONHOME:-}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="${PYTHONHOME:-}"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1:-}"
if [ "x(2022703440) " != x ] ; then
PS1="(2022703440) ${PS1:-}"
else
if [ "`basename \"$VIRTUAL_ENV\"`" = "__" ] ; then
# special case for Aspen magic directories
# see http://www.zetadev.com/software/aspen/
PS1="[`basename \`dirname \"$VIRTUAL_ENV\"\``] $PS1"
else
PS1="(`basename \"$VIRTUAL_ENV\"`)$PS1"
fi
fi
export PS1
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then
hash -r
fi
pip install --retries 10 --upgrade pip setuptools wheel
Collecting pip
Using cached
https://files.pythonhosted.org/packages/a4/6d/6463d49a933f547439d6b5b98b46af8742cc03ae83543e4d7688c2420f8b/pip-21.3.1-py3-none-any.whl
Collecting setuptools
Using cached
https://files.pythonhosted.org/packages/b0/3a/88b210db68e56854d0bcf4b38e165e03be377e13907746f825790f3df5bf/setuptools-59.6.0-py3-none-any.whl
Collecting wheel
Using cached
https://files.pythonhosted.org/packages/27/d6/003e593296a85fd6ed616ed962795b2f87709c3eee2bca4f6d0fe55c6d00/wheel-0.37.1-py2.py3-none-any.whl
Installing collected packages: pip, setuptools, wheel
Found existing installation: pip 18.1
Uninstalling pip-18.1:
Successfully uninstalled pip-18.1
Found existing installation: setuptools 40.6.2
Uninstalling setuptools-40.6.2:
Successfully uninstalled setuptools-40.6.2
Successfully installed pip-21.3.1 setuptools-59.6.0 wheel-0.37.1
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining
file:///home/jenkins/jenkins-slave/workspace/beam_PostCommit_PortableJar_Flink/src/sdks/python
Preparing metadata (setup.py): started
Preparing metadata (setup.py): finished with status 'done'
Collecting crcmod<2.0,>=1.7
Using cached crcmod-1.7-cp36-cp36m-linux_x86_64.whl
Collecting orjson<4.0
Using cached orjson-3.6.1-cp36-cp36m-manylinux_2_24_x86_64.whl (233 kB)
Collecting dill<0.3.2,>=0.3.1.1
Using cached dill-0.3.1.1-py3-none-any.whl
Collecting cloudpickle<3,>=2.0.0
Using cached cloudpickle-2.0.0-py3-none-any.whl (25 kB)
Collecting fastavro<2,>=0.23.6
Using cached
fastavro-1.4.7-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.3
MB)
Collecting grpcio<2,>=1.29.0
Using cached
grpcio-1.43.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (4.1 MB)
Collecting hdfs<3.0.0,>=2.1.0
Using cached hdfs-2.6.0-py3-none-any.whl (33 kB)
Collecting httplib2<0.20.0,>=0.8
Using cached httplib2-0.19.1-py3-none-any.whl (95 kB)
Collecting numpy<1.22.0,>=1.14.3
Using cached numpy-1.19.5-cp36-cp36m-manylinux2010_x86_64.whl (14.8 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached
pymongo-3.12.3-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (506
kB)
Collecting oauth2client<5,>=2.0.1
Using cached oauth2client-4.1.3-py2.py3-none-any.whl (98 kB)
Collecting protobuf<4,>=3.12.2
Using cached
protobuf-3.19.4-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.1
MB)
Collecting proto-plus<2,>=1.7.1
Using cached proto_plus-1.20.0-py3-none-any.whl (45 kB)
Collecting pyarrow<7.0.0,>=0.15.1
Using cached
pyarrow-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (25.6
MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.2-py2.py3-none-any.whl (21 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.2-py2.py3-none-any.whl (247 kB)
Collecting pytz>=2018.3
Using cached pytz-2021.3-py2.py3-none-any.whl (503 kB)
Collecting requests<3.0.0,>=2.24.0
Using cached requests-2.27.1-py2.py3-none-any.whl (63 kB)
Collecting typing-extensions>=3.7.0
Using cached typing_extensions-4.0.1-py3-none-any.whl (22 kB)
Collecting dataclasses
Using cached dataclasses-0.8-py3-none-any.whl (19 kB)
Collecting six>=1.5.2
Using cached six-1.16.0-py2.py3-none-any.whl (11 kB)
Collecting docopt
Using cached docopt-0.6.2-py2.py3-none-any.whl
Collecting pyparsing<3,>=2.4.2
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.8-py3-none-any.whl (39 kB)
Collecting charset-normalizer~=2.0.0
Using cached charset_normalizer-2.0.11-py3-none-any.whl (39 kB)
Collecting urllib3<1.27,>=1.21.1
Using cached urllib3-1.26.8-py2.py3-none-any.whl (138 kB)
Collecting idna<4,>=2.5
Using cached idna-3.3-py3-none-any.whl (61 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2021.10.8-py2.py3-none-any.whl (149 kB)
Installing collected packages: urllib3, pyparsing, pyasn1, idna,
charset-normalizer, certifi, six, rsa, requests, pyasn1-modules, protobuf,
numpy, httplib2, docopt, typing-extensions, pytz, python-dateutil, pymongo,
pydot, pyarrow, proto-plus, orjson, oauth2client, hdfs, grpcio, fastavro, dill,
dataclasses, crcmod, cloudpickle, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam-2.38.0.dev0 certifi-2021.10.8
charset-normalizer-2.0.11 cloudpickle-2.0.0 crcmod-1.7 dataclasses-0.8
dill-0.3.1.1 docopt-0.6.2 fastavro-1.4.7 grpcio-1.43.0 hdfs-2.6.0
httplib2-0.19.1 idna-3.3 numpy-1.19.5 oauth2client-4.1.3 orjson-3.6.1
proto-plus-1.20.0 protobuf-3.19.4 pyarrow-6.0.1 pyasn1-0.4.8
pyasn1-modules-0.2.8 pydot-1.4.2 pymongo-3.12.3 pyparsing-2.4.7
python-dateutil-2.8.2 pytz-2021.3 requests-2.27.1 rsa-4.8 six-1.16.0
typing-extensions-4.0.1 urllib3-1.26.8
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR="test-pipeline-${RUNNER}-$(date +%Y%m%d-%H%M%S).jar"
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_options "docker_container_image=$PYTHON_CONTAINER_IMAGE" \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python
3.6 interpreter.
FATAL: command execution failed
java.io.IOException: Backing channel 'apache-beam-jenkins-3' is disconnected.
at
hudson.remoting.RemoteInvocationHandler.channelOrFail(RemoteInvocationHandler.java:216)
at
hudson.remoting.RemoteInvocationHandler.invoke(RemoteInvocationHandler.java:286)
at com.sun.proxy.$Proxy133.isAlive(Unknown Source)
at hudson.Launcher$RemoteLauncher$ProcImpl.isAlive(Launcher.java:1211)
at hudson.Launcher$RemoteLauncher$ProcImpl.join(Launcher.java:1203)
at hudson.Launcher$ProcStarter.join(Launcher.java:523)
at hudson.plugins.gradle.Gradle.perform(Gradle.java:317)
at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
at
hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:803)
at hudson.model.Build$BuildExecution.build(Build.java:197)
at hudson.model.Build$BuildExecution.doRun(Build.java:163)
at
hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:513)
at hudson.model.Run.execute(Run.java:1906)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
Caused by: java.io.IOException: Pipe closed after 0 cycles
at
org.apache.sshd.common.channel.ChannelPipedInputStream.read(ChannelPipedInputStream.java:118)
at
org.apache.sshd.common.channel.ChannelPipedInputStream.read(ChannelPipedInputStream.java:101)
at
hudson.remoting.FlightRecorderInputStream.read(FlightRecorderInputStream.java:93)
at
hudson.remoting.ChunkedInputStream.readHeader(ChunkedInputStream.java:74)
at
hudson.remoting.ChunkedInputStream.readUntilBreak(ChunkedInputStream.java:104)
at
hudson.remoting.ChunkedCommandTransport.readBlock(ChunkedCommandTransport.java:39)
at
hudson.remoting.AbstractSynchronousByteArrayCommandTransport.read(AbstractSynchronousByteArrayCommandTransport.java:34)
at
hudson.remoting.SynchronousCommandTransport$ReaderThread.run(SynchronousCommandTransport.java:61)
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
ERROR: apache-beam-jenkins-3 is offline; cannot locate jdk_1.8_latest
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
