See
<https://ci-beam.apache.org/job/beam_PostCommit_PortableJar_Flink/6980/display/redirect>
Changes:
------------------------------------------
[...truncated 610.47 KB...]
if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then
hash -r
fi
if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then
PS1="${_OLD_VIRTUAL_PS1:-}"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1:-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="/home/jenkins/jenkins-slave/workspace/beam_PostCommit_PortableJar_Flink/src/build/gradleenv/2022703442"
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
# this will fail if PYTHONHOME is set to the empty string (which is bad anyway)
# could use `if (set -u; : $PYTHONHOME) ;` in bash
if [ -n "${PYTHONHOME:-}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="${PYTHONHOME:-}"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1:-}"
if [ "x(2022703442) " != x ] ; then
PS1="(2022703442) ${PS1:-}"
else
if [ "`basename \"$VIRTUAL_ENV\"`" = "__" ] ; then
# special case for Aspen magic directories
# see https://aspen.io/
PS1="[`basename \`dirname \"$VIRTUAL_ENV\"\``] $PS1"
else
PS1="(`basename \"$VIRTUAL_ENV\"`)$PS1"
fi
fi
export PS1
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then
hash -r
fi
pip install --retries 10 --upgrade pip setuptools wheel
Collecting pip
Using cached pip-23.2.1-py3-none-any.whl (2.1 MB)
Collecting setuptools
Using cached setuptools-68.2.1-py3-none-any.whl (807 kB)
Collecting wheel
Using cached wheel-0.41.2-py3-none-any.whl (64 kB)
Installing collected packages: pip, setuptools, wheel
Attempting uninstall: pip
Found existing installation: pip 20.0.2
Uninstalling pip-20.0.2:
Successfully uninstalled pip-20.0.2
Attempting uninstall: setuptools
Found existing installation: setuptools 44.0.0
Uninstalling setuptools-44.0.0:
Successfully uninstalled setuptools-44.0.0
Successfully installed pip-23.2.1 setuptools-68.2.1 wheel-0.41.2
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining
file:///home/jenkins/jenkins-slave/workspace/beam_PostCommit_PortableJar_Flink/src/sdks/python
Preparing metadata (setup.py): started
Preparing metadata (setup.py): finished with status 'done'
Collecting crcmod<2.0,>=1.7 (from apache-beam==2.51.0.dev0)
Using cached crcmod-1.7-cp38-cp38-linux_x86_64.whl
Collecting orjson<3.9.3 (from apache-beam==2.51.0.dev0)
Obtaining dependency information for orjson<3.9.3 from
https://files.pythonhosted.org/packages/9b/f4/b258592db7523f3601cc6c70ddc9bcf711cab1d1cef20ae9f03e77f149b2/orjson-3.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata
Using cached
orjson-3.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata
(49 kB)
Collecting dill<0.3.2,>=0.3.1.1 (from apache-beam==2.51.0.dev0)
Using cached dill-0.3.1.1-py3-none-any.whl
Collecting cloudpickle~=2.2.1 (from apache-beam==2.51.0.dev0)
Using cached cloudpickle-2.2.1-py3-none-any.whl (25 kB)
Collecting fastavro<2,>=0.23.6 (from apache-beam==2.51.0.dev0)
Obtaining dependency information for fastavro<2,>=0.23.6 from
https://files.pythonhosted.org/packages/a9/2a/af06dcb43345f87b7265211ab95f694a37015c4ae13464778dca5fd017af/fastavro-1.8.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata
Using cached
fastavro-1.8.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata
(5.5 kB)
Collecting fasteners<1.0,>=0.3 (from apache-beam==2.51.0.dev0)
Using cached fasteners-0.18-py3-none-any.whl (18 kB)
Collecting grpcio!=1.48.0,<2,>=1.33.1 (from apache-beam==2.51.0.dev0)
Obtaining dependency information for grpcio!=1.48.0,<2,>=1.33.1 from
https://files.pythonhosted.org/packages/29/f7/32768339888afbd04ce81121d00cc171a84cff3cb9be135e7c3b9c5836b9/grpcio-1.58.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata
Using cached
grpcio-1.58.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata
(4.0 kB)
Collecting hdfs<3.0.0,>=2.1.0 (from apache-beam==2.51.0.dev0)
Using cached hdfs-2.7.2-py3-none-any.whl
Collecting httplib2<0.23.0,>=0.8 (from apache-beam==2.51.0.dev0)
Using cached httplib2-0.22.0-py3-none-any.whl (96 kB)
Collecting numpy<1.25.0,>=1.14.3 (from apache-beam==2.51.0.dev0)
Obtaining dependency information for numpy<1.25.0,>=1.14.3 from
https://files.pythonhosted.org/packages/98/5d/5738903efe0ecb73e51eb44feafba32bdba2081263d40c5043568ff60faf/numpy-1.24.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata
Using cached
numpy-1.24.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata
(5.6 kB)
Collecting objsize<0.7.0,>=0.6.1 (from apache-beam==2.51.0.dev0)
Using cached objsize-0.6.1-py3-none-any.whl (9.3 kB)
Collecting packaging>=22.0 (from apache-beam==2.51.0.dev0)
Using cached packaging-23.1-py3-none-any.whl (48 kB)
Collecting pymongo<5.0.0,>=3.8.0 (from apache-beam==2.51.0.dev0)
Obtaining dependency information for pymongo<5.0.0,>=3.8.0 from
https://files.pythonhosted.org/packages/bf/b7/63cebc24f4690b85d0c91fe11a32ad14e791ff499f4054cde8f1a4ea08dc/pymongo-4.5.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata
Using cached
pymongo-4.5.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata
(22 kB)
Collecting proto-plus<2,>=1.7.1 (from apache-beam==2.51.0.dev0)
Obtaining dependency information for proto-plus<2,>=1.7.1 from
https://files.pythonhosted.org/packages/36/5b/e02636d221917d6fa2a61289b3f16002eb4c93d51c0191ac8e896d527182/proto_plus-1.22.3-py3-none-any.whl.metadata
Using cached proto_plus-1.22.3-py3-none-any.whl.metadata (2.2 kB)
Collecting protobuf<4.24.0,>=3.20.3 (from apache-beam==2.51.0.dev0)
Obtaining dependency information for protobuf<4.24.0,>=3.20.3 from
https://files.pythonhosted.org/packages/01/cb/445b3e465abdb8042a41957dc8f60c54620dc7540dbcf9b458a921531ca2/protobuf-4.23.4-cp37-abi3-manylinux2014_x86_64.whl.metadata
Using cached protobuf-4.23.4-cp37-abi3-manylinux2014_x86_64.whl.metadata (540
bytes)
Collecting pydot<2,>=1.2.0 (from apache-beam==2.51.0.dev0)
Using cached pydot-1.4.2-py2.py3-none-any.whl (21 kB)
Collecting python-dateutil<3,>=2.8.0 (from apache-beam==2.51.0.dev0)
Using cached python_dateutil-2.8.2-py2.py3-none-any.whl (247 kB)
Collecting pytz>=2018.3 (from apache-beam==2.51.0.dev0)
Obtaining dependency information for pytz>=2018.3 from
https://files.pythonhosted.org/packages/32/4d/aaf7eff5deb402fd9a24a1449a8119f00d74ae9c2efa79f8ef9994261fc2/pytz-2023.3.post1-py2.py3-none-any.whl.metadata
Using cached pytz-2023.3.post1-py2.py3-none-any.whl.metadata (22 kB)
Collecting regex>=2020.6.8 (from apache-beam==2.51.0.dev0)
Obtaining dependency information for regex>=2020.6.8 from
https://files.pythonhosted.org/packages/1f/5c/374ac3fa3c7ed9a967ad273a5e841897ef6b10aa6aad938ff10717a3e2a3/regex-2023.8.8-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata
Using cached
regex-2023.8.8-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata
(40 kB)
Collecting requests<3.0.0,>=2.24.0 (from apache-beam==2.51.0.dev0)
Obtaining dependency information for requests<3.0.0,>=2.24.0 from
https://files.pythonhosted.org/packages/70/8e/0e2d847013cb52cd35b38c009bb167a1a26b2ce6cd6965bf26b47bc0bf44/requests-2.31.0-py3-none-any.whl.metadata
Using cached requests-2.31.0-py3-none-any.whl.metadata (4.6 kB)
Collecting typing-extensions>=3.7.0 (from apache-beam==2.51.0.dev0)
Obtaining dependency information for typing-extensions>=3.7.0 from
https://files.pythonhosted.org/packages/ec/6b/63cc3df74987c36fe26157ee12e09e8f9db4de771e0f3404263117e75b95/typing_extensions-4.7.1-py3-none-any.whl.metadata
Using cached typing_extensions-4.7.1-py3-none-any.whl.metadata (3.1 kB)
Collecting zstandard<1,>=0.18.0 (from apache-beam==2.51.0.dev0)
Using cached
zstandard-0.21.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.7
MB)
Collecting pyarrow<12.0.0,>=3.0.0 (from apache-beam==2.51.0.dev0)
Using cached
pyarrow-11.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (35.0
MB)
Collecting docopt (from hdfs<3.0.0,>=2.1.0->apache-beam==2.51.0.dev0)
Using cached docopt-0.6.2-py2.py3-none-any.whl
Collecting six>=1.9.0 (from hdfs<3.0.0,>=2.1.0->apache-beam==2.51.0.dev0)
Using cached six-1.16.0-py2.py3-none-any.whl (11 kB)
Collecting pyparsing!=3.0.0,!=3.0.1,!=3.0.2,!=3.0.3,<4,>=2.4.2 (from
httplib2<0.23.0,>=0.8->apache-beam==2.51.0.dev0)
Obtaining dependency information for
pyparsing!=3.0.0,!=3.0.1,!=3.0.2,!=3.0.3,<4,>=2.4.2 from
https://files.pythonhosted.org/packages/39/92/8486ede85fcc088f1b3dba4ce92dd29d126fd96b0008ea213167940a2475/pyparsing-3.1.1-py3-none-any.whl.metadata
Using cached pyparsing-3.1.1-py3-none-any.whl.metadata (5.1 kB)
Collecting dnspython<3.0.0,>=1.16.0 (from
pymongo<5.0.0,>=3.8.0->apache-beam==2.51.0.dev0)
Obtaining dependency information for dnspython<3.0.0,>=1.16.0 from
https://files.pythonhosted.org/packages/f6/b4/0a9bee52c50f226a3cbfb54263d02bb421c7f2adc136520729c2c689c1e5/dnspython-2.4.2-py3-none-any.whl.metadata
Using cached dnspython-2.4.2-py3-none-any.whl.metadata (4.9 kB)
Collecting charset-normalizer<4,>=2 (from
requests<3.0.0,>=2.24.0->apache-beam==2.51.0.dev0)
Obtaining dependency information for charset-normalizer<4,>=2 from
https://files.pythonhosted.org/packages/cb/e7/5e43745003bf1f90668c7be23fc5952b3a2b9c2558f16749411c18039b36/charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata
Using cached
charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata
(31 kB)
Collecting idna<4,>=2.5 (from requests<3.0.0,>=2.24.0->apache-beam==2.51.0.dev0)
Using cached idna-3.4-py3-none-any.whl (61 kB)
Collecting urllib3<3,>=1.21.1 (from
requests<3.0.0,>=2.24.0->apache-beam==2.51.0.dev0)
Obtaining dependency information for urllib3<3,>=1.21.1 from
https://files.pythonhosted.org/packages/9b/81/62fd61001fa4b9d0df6e31d47ff49cfa9de4af03adecf339c7bc30656b37/urllib3-2.0.4-py3-none-any.whl.metadata
Using cached urllib3-2.0.4-py3-none-any.whl.metadata (6.6 kB)
Collecting certifi>=2017.4.17 (from
requests<3.0.0,>=2.24.0->apache-beam==2.51.0.dev0)
Obtaining dependency information for certifi>=2017.4.17 from
https://files.pythonhosted.org/packages/4c/dd/2234eab22353ffc7d94e8d13177aaa050113286e93e7b40eae01fbf7c3d9/certifi-2023.7.22-py3-none-any.whl.metadata
Using cached certifi-2023.7.22-py3-none-any.whl.metadata (2.2 kB)
Using cached
fastavro-1.8.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.8 MB)
Using cached
grpcio-1.58.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (5.3 MB)
Using cached
numpy-1.24.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (17.3 MB)
Using cached
orjson-3.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (138 kB)
Using cached proto_plus-1.22.3-py3-none-any.whl (48 kB)
Using cached protobuf-4.23.4-cp37-abi3-manylinux2014_x86_64.whl (304 kB)
Using cached
pymongo-4.5.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (687 kB)
Using cached pytz-2023.3.post1-py2.py3-none-any.whl (502 kB)
Using cached
regex-2023.8.8-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (774 kB)
Using cached requests-2.31.0-py3-none-any.whl (62 kB)
Using cached typing_extensions-4.7.1-py3-none-any.whl (33 kB)
Using cached certifi-2023.7.22-py3-none-any.whl (158 kB)
Using cached
charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
(199 kB)
Using cached dnspython-2.4.2-py3-none-any.whl (300 kB)
Using cached pyparsing-3.1.1-py3-none-any.whl (103 kB)
Using cached urllib3-2.0.4-py3-none-any.whl (123 kB)
Installing collected packages: pytz, docopt, crcmod, zstandard, urllib3,
typing-extensions, six, regex, pyparsing, protobuf, packaging, orjson, objsize,
numpy, idna, grpcio, fasteners, fastavro, dnspython, dill, cloudpickle,
charset-normalizer, certifi, requests, python-dateutil, pymongo, pydot,
pyarrow, proto-plus, httplib2, hdfs, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam-2.51.0.dev0 certifi-2023.7.22
charset-normalizer-3.2.0 cloudpickle-2.2.1 crcmod-1.7 dill-0.3.1.1
dnspython-2.4.2 docopt-0.6.2 fastavro-1.8.3 fasteners-0.18 grpcio-1.58.0
hdfs-2.7.2 httplib2-0.22.0 idna-3.4 numpy-1.24.4 objsize-0.6.1 orjson-3.9.2
packaging-23.1 proto-plus-1.22.3 protobuf-4.23.4 pyarrow-11.0.0 pydot-1.4.2
pymongo-4.5.0 pyparsing-3.1.1 python-dateutil-2.8.2 pytz-2023.3.post1
regex-2023.8.8 requests-2.31.0 six-1.16.0 typing-extensions-4.7.1 urllib3-2.0.4
zstandard-0.21.0
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR="test-pipeline-${RUNNER}-$(date +%Y%m%d-%H%M%S).jar"
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_options "docker_container_image=$PYTHON_CONTAINER_IMAGE" \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
FATAL: command execution failed
java.io.IOException: Backing channel 'apache-beam-jenkins-6' is disconnected.
at
hudson.remoting.RemoteInvocationHandler.channelOrFail(RemoteInvocationHandler.java:215)
at
hudson.remoting.RemoteInvocationHandler.invoke(RemoteInvocationHandler.java:285)
at com.sun.proxy.$Proxy141.isAlive(Unknown Source)
at hudson.Launcher$RemoteLauncher$ProcImpl.isAlive(Launcher.java:1215)
at hudson.Launcher$RemoteLauncher$ProcImpl.join(Launcher.java:1207)
at hudson.Launcher$ProcStarter.join(Launcher.java:524)
at hudson.plugins.gradle.Gradle.perform(Gradle.java:321)
at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
at
hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:814)
at hudson.model.Build$BuildExecution.build(Build.java:199)
at hudson.model.Build$BuildExecution.doRun(Build.java:164)
at
hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:522)
at hudson.model.Run.execute(Run.java:1896)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:44)
at hudson.model.ResourceController.execute(ResourceController.java:101)
at hudson.model.Executor.run(Executor.java:442)
Caused by: java.io.IOException: Pipe closed after 0 cycles
at
org.apache.sshd.common.channel.ChannelPipedInputStream.read(ChannelPipedInputStream.java:126)
at
org.apache.sshd.common.channel.ChannelPipedInputStream.read(ChannelPipedInputStream.java:105)
at
hudson.remoting.FlightRecorderInputStream.read(FlightRecorderInputStream.java:94)
at
hudson.remoting.ChunkedInputStream.readHeader(ChunkedInputStream.java:75)
at
hudson.remoting.ChunkedInputStream.readUntilBreak(ChunkedInputStream.java:105)
at
hudson.remoting.ChunkedCommandTransport.readBlock(ChunkedCommandTransport.java:39)
at
hudson.remoting.AbstractSynchronousByteArrayCommandTransport.read(AbstractSynchronousByteArrayCommandTransport.java:34)
at
hudson.remoting.SynchronousCommandTransport$ReaderThread.run(SynchronousCommandTransport.java:61)
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
ERROR: apache-beam-jenkins-6 is offline; cannot locate jdk_1.8_latest
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
