See
<https://ci-beam.apache.org/job/beam_PostCommit_Python37/4866/display/redirect?page=changes>
Changes:
[noreply] Minor: Add 2.38.0 section to CHANGES.md (#16804)
------------------------------------------
[...truncated 21.75 MB...]
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b' }'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b' },
{'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'
"urn": "beam:metric:pardo_execution_time:process_bundle_msecs:v1",'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'
"type": "beam:metrics:sum_int64:v1",'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'
"payload": "AA==",'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'
"labels": {'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'
"PTRANSFORM": "ref_AppliedPTransform_assert_that-Unkey_19"'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b' }'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b' },
{'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'
"urn": "beam:metric:ptransform_execution_time:total_msecs:v1",'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'
"type": "beam:metrics:sum_int64:v1",'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'
"payload": "AA==",'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'
"labels": {'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'
"PTRANSFORM": "ref_AppliedPTransform_assert_that-Match_20"'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b' }'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b' },
{'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'
"urn": "beam:metric:pardo_execution_time:process_bundle_msecs:v1",'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'
"type": "beam:metrics:sum_int64:v1",'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'
"payload": "AA==",'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'
"labels": {'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'
"PTRANSFORM":
"ref_AppliedPTransform_assert_that-Group-CoGroupByKeyImpl-MapTuple-collect_values-_17"'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b' }'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b' },
{'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'
"urn": "beam:metric:sampled_byte_size:v1",'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'
"type": "beam:metrics:distribution_int64:v1",'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'
"payload": "AbsLuwu7Cw==",'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'
"labels": {'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'
"PCOLLECTION": "ref_PCollection_PCollection_11"'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b' }'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b' },
{'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'
"urn": "beam:metric:sampled_byte_size:v1",'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'
"type": "beam:metrics:distribution_int64:v1",'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'
"payload": "ATU1NQ==",'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'
"labels": {'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'
"PCOLLECTION": "ref_PCollection_PCollection_12"'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b' }'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b' },
{'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'
"urn": "beam:metric:sampled_byte_size:v1",'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'
"type": "beam:metrics:distribution_int64:v1",'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'
"payload": "AYAMgAyADA==",'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'
"labels": {'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'
"PCOLLECTION": "ref_PCollection_PCollection_10"'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b' }'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b' },
{'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'
"urn": "beam:metric:sampled_byte_size:v1",'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'
"type": "beam:metrics:distribution_int64:v1",'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'
"payload": "AS8vLw==",'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'
"labels": {'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'
"PCOLLECTION": "ref_PCollection_PCollection_13"'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b' }'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b' },
{'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'
"urn": "beam:metric:sampled_byte_size:v1",'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'
"type": "beam:metrics:distribution_int64:v1",'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'
"payload": "AQ4ODg==",'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'
"labels": {'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'
"PCOLLECTION": "ref_PCollection_PCollection_14"'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b' }'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b' }]'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b' }'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'}'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'Feb 10,
2022 6:15:11 PM org.apache.flink.runtime.io.disk.FileChannelManagerImpl
lambda$getFileCloser$0'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'INFO:
FileChannelManager removed spill file directory
/tmp/flink-netty-shuffle-574a4f79-69a4-494e-a3b7-4b3488b8853b'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'Feb 10,
2022 6:15:11 PM org.apache.flink.runtime.taskexecutor.KvStateService shutdown'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'INFO:
Shutting down the kvState service and its components.'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'Feb 10,
2022 6:15:11 PM org.apache.flink.runtime.taskexecutor.DefaultJobLeaderService
stop'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'INFO:
Stop job leader service.'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'Feb 10,
2022 6:15:11 PM org.apache.flink.runtime.filecache.FileCache shutdown'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'INFO:
removed file cache directory
/tmp/flink-dist-cache-aa6ae465-be5c-43fd-ad77-b3d09aec0251'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'Feb 10,
2022 6:15:11 PM org.apache.flink.runtime.taskexecutor.TaskExecutor
handleOnStopException'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'INFO:
Stopped TaskExecutor akka://flink/user/rpc/taskmanager_0.'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'Feb 10,
2022 6:15:11 PM org.apache.flink.runtime.rpc.akka.AkkaRpcService stopService'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'INFO:
Stopping Akka RPC service.'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'Feb 10,
2022 6:15:11 PM org.apache.flink.runtime.rpc.akka.AkkaRpcService stopService'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'INFO:
Stopping Akka RPC service.'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'Feb 10,
2022 6:15:11 PM org.apache.flink.runtime.rpc.akka.AkkaRpcService
lambda$stopService$7'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'INFO:
Stopped Akka RPC service.'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'Feb 10,
2022 6:15:11 PM org.apache.flink.runtime.blob.AbstractBlobCache close'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'INFO:
Shutting down BLOB cache'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'Feb 10,
2022 6:15:11 PM org.apache.flink.runtime.blob.AbstractBlobCache close'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'INFO:
Shutting down BLOB cache'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'Feb 10,
2022 6:15:11 PM org.apache.flink.runtime.blob.BlobServer close'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'INFO:
Stopped BLOB server at 0.0.0.0:41813'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'Feb 10,
2022 6:15:11 PM org.apache.flink.runtime.rpc.akka.AkkaRpcService
lambda$stopService$7'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'INFO:
Stopped Akka RPC service.'
INFO apache_beam.runners.portability.portable_runner:portable_runner.py:576
Job state changed to DONE
PASSED [ 71%]
apache_beam/io/external/xlang_jdbcio_it_test.py::CrossLanguageJdbcIOTest::test_xlang_jdbc_write_0_postgres
-------------------------------- live log call ---------------------------------
INFO testcontainers.core.container:container.py:46 Pulling image
postgres:12.3
INFO testcontainers.core.container:container.py:57 Container started:
9994e1d643
INFO testcontainers.core.waiting_utils:waiting_utils.py:39 Waiting to be
ready...
INFO apache_beam.utils.subprocess_server:subprocess_server.py:238 Using
pre-built snapshot at
/home/jenkins/jenkins-slave/workspace/beam_PostCommit_Python37/src/sdks/java/extensions/schemaio-expansion-service/build/libs/beam-sdks-java-extensions-schemaio-expansion-service-2.38.0-SNAPSHOT.jar
INFO root:external.py:740 Starting a JAR-based expansion service from JAR
/home/jenkins/jenkins-slave/workspace/beam_PostCommit_Python37/src/sdks/java/extensions/schemaio-expansion-service/build/libs/beam-sdks-java-extensions-schemaio-expansion-service-2.38.0-SNAPSHOT.jar
and with classpath: ['org.postgresql:postgresql:42.2.16']
INFO apache_beam.utils.subprocess_server:subprocess_server.py:267 Using
cached job server jar from
https://repo.maven.apache.org/maven2/org/postgresql/postgresql/42.2.16/postgresql-42.2.16.jar
INFO apache_beam.utils.subprocess_server:subprocess_server.py:116 Starting
service with ['java' '-jar'
'/home/jenkins/.apache_beam/cache/jars/composite-jars/edadbea71eb63f4417ad467b9784c4043f78c478ffc7012584905a0ca9164738.jar'
'50609'
'--filesToStage=/home/jenkins/jenkins-slave/workspace/beam_PostCommit_Python37/src/sdks/java/extensions/schemaio-expansion-service/build/libs/beam-sdks-java-extensions-schemaio-expansion-service-2.38.0-SNAPSHOT.jar,https://repo.maven.apache.org/maven2/org/postgresql/postgresql/42.2.16/postgresql-42.2.16.jar']
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125
b'Starting expansion service at localhost:50609'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'Feb 10,
2022 6:15:16 PM org.apache.beam.sdk.expansion.service.ExpansionService
loadRegisteredTransforms'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125 b'INFO:
Registering external transforms:
[beam:transform:org.apache.beam:schemaio_jdbc_read:v1,
beam:transform:org.apache.beam:schemaio_jdbc_write:v1,
beam:transform:org.apache.beam:schemaio_avro_read:v1,
beam:transform:org.apache.beam:schemaio_avro_write:v1,
beam:external:java:generate_sequence:v1]'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125
b'\tbeam:transform:org.apache.beam:schemaio_jdbc_read:v1:
org.apache.beam.sdk.expansion.service.ExpansionService$ExternalTransformRegistrarLoader$1@52aa2946'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125
b'\tbeam:transform:org.apache.beam:schemaio_jdbc_write:v1:
org.apache.beam.sdk.expansion.service.ExpansionService$ExternalTransformRegistrarLoader$1@4de5031f'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125
b'\tbeam:transform:org.apache.beam:schemaio_avro_read:v1:
org.apache.beam.sdk.expansion.service.ExpansionService$ExternalTransformRegistrarLoader$1@67e2d983'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125
b'\tbeam:transform:org.apache.beam:schemaio_avro_write:v1:
org.apache.beam.sdk.expansion.service.ExpansionService$ExternalTransformRegistrarLoader$1@5d47c63f'
INFO apache_beam.utils.subprocess_server:subprocess_server.py:125
b'\tbeam:external:java:generate_sequence:v1:
org.apache.beam.sdk.expansion.service.ExpansionService$ExternalTransformRegistrarLoader$1@5ea434c8'
> Task :sdks:python:container:py37:docker
Collecting pip-licenses<3.0.0
Downloading pip_licenses-2.3.0-py3-none-any.whl (14 kB)
Requirement already satisfied: pyyaml in /usr/local/lib/python3.7/site-packages
(6.0)
Requirement already satisfied: tenacity in
/usr/local/lib/python3.7/site-packages (5.1.5)
Collecting PTable
Downloading PTable-0.9.2.tar.gz (31 kB)
Requirement already satisfied: six>=1.9.0 in
/usr/local/lib/python3.7/site-packages (from tenacity) (1.16.0)
Building wheels for collected packages: PTable
Building wheel for PTable (setup.py): started
Building wheel for PTable (setup.py): finished with status 'done'
Created wheel for PTable: filename=PTable-0.9.2-py3-none-any.whl size=22925
sha256=257f39ec0a8b5bc649c68f48fd9f746fc79f1f929027b0db2e8f2b0b3e416a04
Stored in directory:
/root/.cache/pip/wheels/33/df/2f/674985b3f8a2de3f96357d1eadef5110f74fa91b3785e52a54
Successfully built PTable
Installing collected packages: PTable, pip-licenses
Successfully installed PTable-0.9.2 pip-licenses-2.3.0
[91mWARNING: Running pip as the 'root' user can result in broken permissions
and conflicting behaviour with the system package manager. It is recommended to
use a virtual environment instead: https://pip.pypa.io/warnings/venv
[0m[91mWARNING: You are using pip version 21.2.4; however, version 22.0.3 is
available.
You should consider upgrading via the '/usr/local/bin/python -m pip install
--upgrade pip' command.
[0mSkip pulling license for bs4
[91mINFO:root:Successfully pulled licenses for 134 dependencies
[0mRemoving intermediate container c5b933c47c03
---> 32540f064afa
Step 28/31 : FROM beam
---> 35669f2dc86f
Step 29/31 : ARG pull_licenses
---> Running in 00b6a0248d8e
Removing intermediate container 00b6a0248d8e
---> dde2d0db187a
Step 30/31 : COPY --from=third_party_licenses
/opt/apache/beam/third_party_licenses /opt/apache/beam/third_party_licenses
---> 56f507b73b69
Step 31/31 : RUN if [ "$pull_licenses" != "true" ] ; then rm -rf
/opt/apache/beam/third_party_licenses ; fi
---> Running in 5305ad0a35db
Removing intermediate container 5305ad0a35db
---> 2d8cf9b65375
Successfully built 2d8cf9b65375
Successfully tagged apache/beam_python3.7_sdk:2.38.0.dev
> Task :sdks:python:test-suites:direct:py37:mongodbioIT
INFO:__main__:Writing 100000 documents to mongodb finished in 35.377 seconds
INFO:root:Missing pipeline option (runner). Executing pipeline using the
default runner: DirectRunner.
INFO:__main__:================================================================================
INFO:__main__:Reading from mongodb
beam_mongodbio_it_db:integration_test_1644516874
INFO:__main__:reader params : {'projection': ['number']}
INFO:__main__:expected results: {'number_sum': 4999950000, 'docs_count': 100000}
/home/jenkins/jenkins-slave/workspace/beam_PostCommit_Python37/src/sdks/python/apache_beam/io/mongodbio_it_test.py:153:
FutureWarning: ReadFromMongoDB is experimental.
| 'Map' >> beam.Map(lambda doc: doc['number']))
WARNING:root:Make sure that locally built Python SDK docker image has Python
3.7 interpreter.
INFO:root:Default Python SDK image for environment is
apache/beam_python3.7_sdk:2.38.0.dev
INFO:apache_beam.runners.portability.fn_api_runner.translations:====================
<function annotate_downstream_side_inputs at 0x7f6afc04f3b0>
====================
INFO:apache_beam.runners.portability.fn_api_runner.translations:====================
<function fix_side_input_pcoll_coders at 0x7f6afc04f4d0> ====================
INFO:apache_beam.runners.portability.fn_api_runner.translations:====================
<function pack_combiners at 0x7f6afc04f9e0> ====================
INFO:apache_beam.runners.portability.fn_api_runner.translations:====================
<function lift_combiners at 0x7f6afc04fa70> ====================
INFO:apache_beam.runners.portability.fn_api_runner.translations:====================
<function expand_sdf at 0x7f6afc04fc20> ====================
INFO:apache_beam.runners.portability.fn_api_runner.translations:====================
<function expand_gbk at 0x7f6afc04fcb0> ====================
INFO:apache_beam.runners.portability.fn_api_runner.translations:====================
<function sink_flattens at 0x7f6afc04fdd0> ====================
INFO:apache_beam.runners.portability.fn_api_runner.translations:====================
<function greedily_fuse at 0x7f6afc04fe60> ====================
INFO:apache_beam.runners.portability.fn_api_runner.translations:====================
<function read_to_impulse at 0x7f6afc04fef0> ====================
INFO:apache_beam.runners.portability.fn_api_runner.translations:====================
<function impulse_to_input at 0x7f6afc04ff80> ====================
INFO:apache_beam.runners.portability.fn_api_runner.translations:====================
<function sort_stages at 0x7f6afc050200> ====================
INFO:apache_beam.runners.portability.fn_api_runner.translations:====================
<function setup_timer_mapping at 0x7f6afc050170> ====================
INFO:apache_beam.runners.portability.fn_api_runner.translations:====================
<function populate_data_channel_coders at 0x7f6afc050290> ====================
INFO:apache_beam.runners.worker.statecache:Creating state cache with size 100
INFO:apache_beam.runners.portability.fn_api_runner.worker_handlers:Created
Worker handler
<apache_beam.runners.portability.fn_api_runner.worker_handlers.EmbeddedWorkerHandler
object at 0x7f6af7747990> for environment
ref_Environment_default_environment_1 (beam:env:embedded_python:v1, b'')
INFO:apache_beam.runners.portability.fn_api_runner.fn_runner:Running
((((ref_AppliedPTransform_ReadFromMongoDB-Read-Impulse_4)+(ref_AppliedPTransform_ReadFromMongoDB-Read-Map-lambda-at-iobase-py-898-_5))+(ReadFromMongoDB/Read/SDFBoundedSourceReader/ParDo(SDFBoundedSourceDoFn)/PairWithRestriction))+(ReadFromMongoDB/Read/SDFBoundedSourceReader/ParDo(SDFBoundedSourceDoFn)/SplitAndSizeRestriction))+(ref_PCollection_PCollection_2_split/Write)
INFO:apache_beam.runners.portability.fn_api_runner.fn_runner:Running
((((((((ref_PCollection_PCollection_2_split/Read)+(ReadFromMongoDB/Read/SDFBoundedSourceReader/ParDo(SDFBoundedSourceDoFn)/Process))+(ref_AppliedPTransform_Map_8))+(ref_AppliedPTransform_Combine-KeyWithVoid_10))+(ref_AppliedPTransform_Count-CombineGlobally-CountCombineFn-KeyWithVoid_24))+(Combine/CombinePerKey/Precombine))+(Combine/CombinePerKey/Group/Write))+(Count/CombineGlobally(CountCombineFn)/CombinePerKey/Precombine))+(Count/CombineGlobally(CountCombineFn)/CombinePerKey/Group/Write)
INFO:apache_beam.runners.portability.fn_api_runner.fn_runner:Running
((((Combine/CombinePerKey/Group/Read)+(Combine/CombinePerKey/Merge))+(Combine/CombinePerKey/ExtractOutputs))+(ref_AppliedPTransform_Combine-UnKey_15))+(ref_PCollection_PCollection_8/Write)
INFO:apache_beam.runners.portability.fn_api_runner.fn_runner:Running
((((ref_AppliedPTransform_Combine-DoOnce-Impulse_17)+(ref_AppliedPTransform_Combine-DoOnce-FlatMap-lambda-at-core-py-3228-_18))+(ref_AppliedPTransform_Combine-DoOnce-Map-decode-_20))+(ref_AppliedPTransform_Combine-InjectDefault_21))+(Flatten/Write/0)
INFO:apache_beam.runners.portability.fn_api_runner.fn_runner:Running
((((Count/CombineGlobally(CountCombineFn)/CombinePerKey/Group/Read)+(Count/CombineGlobally(CountCombineFn)/CombinePerKey/Merge))+(Count/CombineGlobally(CountCombineFn)/CombinePerKey/ExtractOutputs))+(ref_AppliedPTransform_Count-CombineGlobally-CountCombineFn-UnKey_29))+(ref_PCollection_PCollection_16/Write)
INFO:apache_beam.runners.portability.fn_api_runner.fn_runner:Running
(((((ref_AppliedPTransform_Count-CombineGlobally-CountCombineFn-DoOnce-Impulse_31)+(ref_AppliedPTransform_Count-CombineGlobally-CountCombineFn-DoOnce-FlatMap-lambda-at-core-py-3228-_32))+(ref_AppliedPTransform_Count-CombineGlobally-CountCombineFn-DoOnce-Map-decode-_34))+(ref_AppliedPTransform_Count-CombineGlobally-CountCombineFn-InjectDefault_35))+(Flatten/Transcode/1))+(Flatten/Write/1)
INFO:apache_beam.runners.portability.fn_api_runner.fn_runner:Running
(((((Flatten/Read)+(ref_AppliedPTransform_assert_that-WindowInto-WindowIntoFn-_43))+(ref_AppliedPTransform_assert_that-ToVoidKey_44))+(ref_AppliedPTransform_assert_that-Group-CoGroupByKeyImpl-Tag-1-_48))+(assert_that/Group/CoGroupByKeyImpl/Flatten/Transcode/1))+(assert_that/Group/CoGroupByKeyImpl/Flatten/Write/1)
INFO:apache_beam.runners.portability.fn_api_runner.fn_runner:Running
(((((ref_AppliedPTransform_assert_that-Create-Impulse_39)+(ref_AppliedPTransform_assert_that-Create-FlatMap-lambda-at-core-py-3228-_40))+(ref_AppliedPTransform_assert_that-Create-Map-decode-_42))+(ref_AppliedPTransform_assert_that-Group-CoGroupByKeyImpl-Tag-0-_47))+(assert_that/Group/CoGroupByKeyImpl/Flatten/Transcode/0))+(assert_that/Group/CoGroupByKeyImpl/Flatten/Write/0)
INFO:apache_beam.runners.portability.fn_api_runner.fn_runner:Running
(assert_that/Group/CoGroupByKeyImpl/Flatten/Read)+(assert_that/Group/CoGroupByKeyImpl/GroupByKey/Write)
INFO:apache_beam.runners.portability.fn_api_runner.fn_runner:Running
((((assert_that/Group/CoGroupByKeyImpl/GroupByKey/Read)+(ref_AppliedPTransform_assert_that-Group-CoGroupByKeyImpl-MapTuple-collect_values-_51))+(ref_AppliedPTransform_assert_that-Group-RestoreTags_52))+(ref_AppliedPTransform_assert_that-Unkey_53))+(ref_AppliedPTransform_assert_that-Match_54)
INFO:__main__:Reading documents from mongodb finished in 6.453 seconds
INFO:root:Missing pipeline option (runner). Executing pipeline using the
default runner: DirectRunner.
INFO:__main__:================================================================================
INFO:__main__:Reading from mongodb
beam_mongodbio_it_db:integration_test_1644516874
INFO:__main__:reader params : {'filter': {'number_mod_3': 0}, 'projection':
['number']}
INFO:__main__:expected results: {'number_sum': 1666683333, 'docs_count': 33334}
WARNING:root:Make sure that locally built Python SDK docker image has Python
3.7 interpreter.
INFO:root:Default Python SDK image for environment is
apache/beam_python3.7_sdk:2.38.0.dev
INFO:apache_beam.runners.portability.fn_api_runner.translations:====================
<function annotate_downstream_side_inputs at 0x7f6afc04f3b0>
====================
INFO:apache_beam.runners.portability.fn_api_runner.translations:====================
<function fix_side_input_pcoll_coders at 0x7f6afc04f4d0> ====================
INFO:apache_beam.runners.portability.fn_api_runner.translations:====================
<function pack_combiners at 0x7f6afc04f9e0> ====================
INFO:apache_beam.runners.portability.fn_api_runner.translations:====================
<function lift_combiners at 0x7f6afc04fa70> ====================
INFO:apache_beam.runners.portability.fn_api_runner.translations:====================
<function expand_sdf at 0x7f6afc04fc20> ====================
INFO:apache_beam.runners.portability.fn_api_runner.translations:====================
<function expand_gbk at 0x7f6afc04fcb0> ====================
INFO:apache_beam.runners.portability.fn_api_runner.translations:====================
<function sink_flattens at 0x7f6afc04fdd0> ====================
INFO:apache_beam.runners.portability.fn_api_runner.translations:====================
<function greedily_fuse at 0x7f6afc04fe60> ====================
INFO:apache_beam.runners.portability.fn_api_runner.translations:====================
<function read_to_impulse at 0x7f6afc04fef0> ====================
INFO:apache_beam.runners.portability.fn_api_runner.translations:====================
<function impulse_to_input at 0x7f6afc04ff80> ====================
INFO:apache_beam.runners.portability.fn_api_runner.translations:====================
<function sort_stages at 0x7f6afc050200> ====================
INFO:apache_beam.runners.portability.fn_api_runner.translations:====================
<function setup_timer_mapping at 0x7f6afc050170> ====================
INFO:apache_beam.runners.portability.fn_api_runner.translations:====================
<function populate_data_channel_coders at 0x7f6afc050290> ====================
INFO:apache_beam.runners.worker.statecache:Creating state cache with size 100
INFO:apache_beam.runners.portability.fn_api_runner.worker_handlers:Created
Worker handler
<apache_beam.runners.portability.fn_api_runner.worker_handlers.EmbeddedWorkerHandler
object at 0x7f6af6443090> for environment
ref_Environment_default_environment_1 (beam:env:embedded_python:v1, b'')
INFO:apache_beam.runners.portability.fn_api_runner.fn_runner:Running
((((ref_AppliedPTransform_ReadFromMongoDB-Read-Impulse_4)+(ref_AppliedPTransform_ReadFromMongoDB-Read-Map-lambda-at-iobase-py-898-_5))+(ReadFromMongoDB/Read/SDFBoundedSourceReader/ParDo(SDFBoundedSourceDoFn)/PairWithRestriction))+(ReadFromMongoDB/Read/SDFBoundedSourceReader/ParDo(SDFBoundedSourceDoFn)/SplitAndSizeRestriction))+(ref_PCollection_PCollection_2_split/Write)
INFO:apache_beam.runners.portability.fn_api_runner.fn_runner:Running
((((((((ref_PCollection_PCollection_2_split/Read)+(ReadFromMongoDB/Read/SDFBoundedSourceReader/ParDo(SDFBoundedSourceDoFn)/Process))+(ref_AppliedPTransform_Map_8))+(ref_AppliedPTransform_Combine-KeyWithVoid_10))+(ref_AppliedPTransform_Count-CombineGlobally-CountCombineFn-KeyWithVoid_24))+(Combine/CombinePerKey/Precombine))+(Combine/CombinePerKey/Group/Write))+(Count/CombineGlobally(CountCombineFn)/CombinePerKey/Precombine))+(Count/CombineGlobally(CountCombineFn)/CombinePerKey/Group/Write)
INFO:apache_beam.runners.portability.fn_api_runner.fn_runner:Running
((((Count/CombineGlobally(CountCombineFn)/CombinePerKey/Group/Read)+(Count/CombineGlobally(CountCombineFn)/CombinePerKey/Merge))+(Count/CombineGlobally(CountCombineFn)/CombinePerKey/ExtractOutputs))+(ref_AppliedPTransform_Count-CombineGlobally-CountCombineFn-UnKey_29))+(ref_PCollection_PCollection_16/Write)
INFO:apache_beam.runners.portability.fn_api_runner.fn_runner:Running
(((((ref_AppliedPTransform_Count-CombineGlobally-CountCombineFn-DoOnce-Impulse_31)+(ref_AppliedPTransform_Count-CombineGlobally-CountCombineFn-DoOnce-FlatMap-lambda-at-core-py-3228-_32))+(ref_AppliedPTransform_Count-CombineGlobally-CountCombineFn-DoOnce-Map-decode-_34))+(ref_AppliedPTransform_Count-CombineGlobally-CountCombineFn-InjectDefault_35))+(Flatten/Transcode/1))+(Flatten/Write/1)
INFO:apache_beam.runners.portability.fn_api_runner.fn_runner:Running
((((Combine/CombinePerKey/Group/Read)+(Combine/CombinePerKey/Merge))+(Combine/CombinePerKey/ExtractOutputs))+(ref_AppliedPTransform_Combine-UnKey_15))+(ref_PCollection_PCollection_8/Write)
INFO:apache_beam.runners.portability.fn_api_runner.fn_runner:Running
((((ref_AppliedPTransform_Combine-DoOnce-Impulse_17)+(ref_AppliedPTransform_Combine-DoOnce-FlatMap-lambda-at-core-py-3228-_18))+(ref_AppliedPTransform_Combine-DoOnce-Map-decode-_20))+(ref_AppliedPTransform_Combine-InjectDefault_21))+(Flatten/Write/0)
INFO:apache_beam.runners.portability.fn_api_runner.fn_runner:Running
(((((Flatten/Read)+(ref_AppliedPTransform_assert_that-WindowInto-WindowIntoFn-_43))+(ref_AppliedPTransform_assert_that-ToVoidKey_44))+(ref_AppliedPTransform_assert_that-Group-CoGroupByKeyImpl-Tag-1-_48))+(assert_that/Group/CoGroupByKeyImpl/Flatten/Transcode/1))+(assert_that/Group/CoGroupByKeyImpl/Flatten/Write/1)
INFO:apache_beam.runners.portability.fn_api_runner.fn_runner:Running
(((((ref_AppliedPTransform_assert_that-Create-Impulse_39)+(ref_AppliedPTransform_assert_that-Create-FlatMap-lambda-at-core-py-3228-_40))+(ref_AppliedPTransform_assert_that-Create-Map-decode-_42))+(ref_AppliedPTransform_assert_that-Group-CoGroupByKeyImpl-Tag-0-_47))+(assert_that/Group/CoGroupByKeyImpl/Flatten/Transcode/0))+(assert_that/Group/CoGroupByKeyImpl/Flatten/Write/0)
INFO:apache_beam.runners.portability.fn_api_runner.fn_runner:Running
(assert_that/Group/CoGroupByKeyImpl/Flatten/Read)+(assert_that/Group/CoGroupByKeyImpl/GroupByKey/Write)
INFO:apache_beam.runners.portability.fn_api_runner.fn_runner:Running
((((assert_that/Group/CoGroupByKeyImpl/GroupByKey/Read)+(ref_AppliedPTransform_assert_that-Group-CoGroupByKeyImpl-MapTuple-collect_values-_51))+(ref_AppliedPTransform_assert_that-Group-RestoreTags_52))+(ref_AppliedPTransform_assert_that-Unkey_53))+(ref_AppliedPTransform_assert_that-Match_54)
INFO:__main__:Reading documents from mongodb finished in 2.961 seconds
INFO:root:Missing pipeline option (runner). Executing pipeline using the
default runner: DirectRunner.
INFO:__main__:================================================================================
INFO:__main__:Reading from mongodb
beam_mongodbio_it_db:integration_test_1644516874
INFO:__main__:reader params : {'projection': ['number'], 'bucket_auto': True}
INFO:__main__:expected results: {'number_sum': 4999950000, 'docs_count': 100000}
WARNING:root:Make sure that locally built Python SDK docker image has Python
3.7 interpreter.
INFO:root:Default Python SDK image for environment is
apache/beam_python3.7_sdk:2.38.0.dev
FATAL: command execution failed
java.io.IOException: Backing channel 'apache-beam-jenkins-13' is disconnected.
at
hudson.remoting.RemoteInvocationHandler.channelOrFail(RemoteInvocationHandler.java:216)
at
hudson.remoting.RemoteInvocationHandler.invoke(RemoteInvocationHandler.java:286)
at com.sun.proxy.$Proxy133.isAlive(Unknown Source)
at hudson.Launcher$RemoteLauncher$ProcImpl.isAlive(Launcher.java:1211)
at hudson.Launcher$RemoteLauncher$ProcImpl.join(Launcher.java:1203)
at hudson.Launcher$ProcStarter.join(Launcher.java:523)
at hudson.plugins.gradle.Gradle.perform(Gradle.java:317)
at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
at
hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:803)
at hudson.model.Build$BuildExecution.build(Build.java:197)
at hudson.model.Build$BuildExecution.doRun(Build.java:163)
at
hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:513)
at hudson.model.Run.execute(Run.java:1906)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
Caused by: java.io.IOException: Pipe closed after 0 cycles
at
org.apache.sshd.common.channel.ChannelPipedInputStream.read(ChannelPipedInputStream.java:118)
at
org.apache.sshd.common.channel.ChannelPipedInputStream.read(ChannelPipedInputStream.java:101)
at
hudson.remoting.FlightRecorderInputStream.read(FlightRecorderInputStream.java:93)
at
hudson.remoting.ChunkedInputStream.readHeader(ChunkedInputStream.java:74)
at
hudson.remoting.ChunkedInputStream.readUntilBreak(ChunkedInputStream.java:104)
at
hudson.remoting.ChunkedCommandTransport.readBlock(ChunkedCommandTransport.java:39)
at
hudson.remoting.AbstractSynchronousByteArrayCommandTransport.read(AbstractSynchronousByteArrayCommandTransport.java:34)
at
hudson.remoting.SynchronousCommandTransport$ReaderThread.run(SynchronousCommandTransport.java:61)
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
ERROR: apache-beam-jenkins-13 is offline; cannot locate jdk_1.8_latest
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
