[
https://issues.apache.org/jira/browse/BEAM-12603?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Tyson Hamilton updated BEAM-12603:
----------------------------------
Description:
The
`apache_beam.runners.portability.fn_api_runner.fn_runner_test.FnApiRunnerTestWithGrpcAndMultiWorkers`
tests are flaky and causing precommit failures that seem similar.
`test_pardo_windowed_side_inputs` :
[https://ci-beam.apache.org/job/beam_PreCommit_Python_Cron/4417/console]
{code:java}
23:32:35 Exception in thread read_grpc_client_inputs:
23:32:35 Traceback (most recent call last):
23:32:35 File "/usr/lib/python3.8/threading.py", line 932, in _bootstrap_inner
23:32:35 self.run()
23:32:35 File "/usr/lib/python3.8/threading.py", line 870, in run
23:32:35 self._target(*self._args, **self._kwargs)
23:32:35 File
"/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py38/build/srcs/sdks/python/apache_beam/runners/worker/data_plane.py",
line 587, in <lambda>
23:32:35 target=lambda: self._read_inputs(elements_iterator),
23:32:35 File
"/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py38/build/srcs/sdks/python/apache_beam/runners/worker/data_plane.py",
line 570, in _read_inputs
23:32:35 for elements in elements_iterator:
23:32:35 File
"/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py38/build/srcs/sdks/python/target/.tox-py38-cython/py38-cython/lib/python3.8/site-packages/grpc/_channel.py",
line 426, in __next__
23:32:35 return self._next()
23:32:35 File
"/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py38/build/srcs/sdks/python/target/.tox-py38-cython/py38-cython/lib/python3.8/site-packages/grpc/_channel.py",
line 826, in _next
23:32:35 raise self
23:32:35 grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of
RPC that terminated with:
23:32:35 status = StatusCode.UNAVAILABLE
23:32:35 details = "Broken pipe"
23:32:35 debug_error_string =
"{"created":"@1626071403.252458842","description":"Error received from peer
ipv4:127.0.0.1:37459","file":"src/core/lib/surface/call.cc","file_line":1066,"grpc_message":"Broken
pipe","grpc_status":14}"
23:32:35 >
23:32:35 Traceback (most recent call last):
23:32:35 File
"/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py38/build/srcs/sdks/python/apache_beam/runners/worker/data_plane.py",
line 459, in input_elements
23:32:35 element = received.get(timeout=1)
23:32:35 File "/usr/lib/python3.8/queue.py", line 178, in get
23:32:35 raise Empty
23:32:35 _queue.Empty
{code}
`test_pack_combiners` :
[https://ci-beam.apache.org/job/beam_PreCommit_Python_Cron/4415/consoleFull]
{code:java}
11:41:14 Exception ignored in: <object repr() failed>
11:41:14 Traceback (most recent call last):
11:41:14 File
"/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py36/build/srcs/sdks/python/target/.tox-py36-cython/py36-cython/lib/python3.6/site-packages/grpc/_channel.py",
line 444, in __del__
11:41:14 with self._state.condition:
11:41:14 AttributeError: '_MultiThreadedRendezvous' object has no attribute
'_state'
11:41:14 Traceback (most recent call last):
11:41:14 File "apache_beam/runners/common.py", line 1223, in
apache_beam.runners.common.DoFnRunner.process
11:41:14 return self.do_fn_invoker.invoke_process(windowed_value)
11:41:14 File "apache_beam/runners/common.py", line 752, in
apache_beam.runners.common.PerWindowInvoker.invoke_process
11:41:14 self._invoke_process_per_window(
11:41:14 File "apache_beam/runners/common.py", line 816, in
apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
11:41:14 [si[global_window] for si in self.side_inputs]))
11:41:14 File
"/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py36/build/srcs/sdks/python/apache_beam/runners/worker/bundle_processor.py",
line 427, in __getitem__
11:41:14 self._cache[target_window] =
self._side_input_data.view_fn(raw_view)
11:41:14 File
"/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py36/build/srcs/sdks/python/apache_beam/runners/worker/bundle_processor.py",
line 353, in __iter__
11:41:14 self._state_handler.blocking_get(self._state_key,
self._coder_impl))
11:41:14 File
"/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py36/build/srcs/sdks/python/apache_beam/runners/worker/sdk_worker.py",
line 1184, in blocking_get
11:41:14 self._partially_cached_iterable(state_key, coder))
11:41:14 File
"/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py36/build/srcs/sdks/python/apache_beam/runners/worker/sdk_worker.py",
line 1290, in _partially_cached_iterable
11:41:14 data, continuation_token = self._underlying.get_raw(state_key,
None)
11:41:14 File
"/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py36/build/srcs/sdks/python/apache_beam/runners/worker/sdk_worker.py",
line 1057, in get_raw
11:41:14 continuation_token=continuation_token)))
11:41:14 File
"/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py36/build/srcs/sdks/python/apache_beam/runners/worker/sdk_worker.py",
line 1095, in _blocking_request
11:41:14 raise t(v).with_traceback(tb)
11:41:14 TypeError: __init__() missing 3 required positional arguments: 'call',
'response_deserializer', and 'deadline'
{code}
was:
The
`apache_beam.runners.portability.fn_api_runner.fn_runner_test.FnApiRunnerTestWithGrpcAndMultiWorkers`
tests are flaky and causing precommit failures that seem similar.
`test_pardo_windowed_side_inputs` :
https://ci-beam.apache.org/job/beam_PreCommit_Python_Cron/4417/console
{code:java}
23:24:27 Traceback (most recent call last):
23:24:27 File "/usr/lib/python3.6/threading.py", line 916, in _bootstrap_inner
23:24:27 self.run()
23:24:27 File "/usr/lib/python3.6/threading.py", line 864, in run
23:24:27 self._target(*self._args, **self._kwargs)
23:24:27 File
"/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py36/build/srcs/sdks/python/apache_beam/runners/portability/local_job_service.py",
line 274, in _run_job
23:24:27 self._pipeline_proto)
23:24:27 File
"/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py36/build/srcs/sdks/python/apache_beam/runners/portability/fn_api_runner/fn_runner.py",
line 202, in run_via_runner_api
23:24:27 self._check_requirements(pipeline_proto)
23:24:27 File
"/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py36/build/srcs/sdks/python/apache_beam/runners/portability/fn_api_runner/fn_runner.py",
line 301, in _check_requirements
23:24:27 'Unable to run pipeline with requirement: %s' % requirement)
23:24:27 ValueError: Unable to run pipeline with requirement:
unsupported_requirement
23:24:27
23:24:27 .... [ 42%]
23:24:29
...sssssssss...........ssssssss.sssssssssssssssssss................s.... [ 44%]
23:24:32 ...s.........s.
23:24:32 > Task :sdks:python:test-suites:dataflow:py37:preCommitIT_batch_V2
23:24:32
INFO:apache_beam.runners.dataflow.dataflow_runner:2021-07-12T06:24:29.370Z:
JOB_MESSAGE_BASIC: Your project already contains 100 Dataflow-created metric
descriptors, so new user metrics of the form custom.googleapis.com/* will not
be created. However, all user metrics are also available in the metric
dataflow.googleapis.com/job/user_counter. If you rely on the custom metrics,
you can delete old / unused metric descriptors. See
https://developers.google.com/apis-explorer/#p/monitoring/v3/monitoring.projects.metricDescriptors.list
and
https://developers.google.com/apis-explorer/#p/monitoring/v3/monitoring.projects.metricDescriptors.delete
23:24:52
23:24:52 > Task :sdks:python:test-suites:tox:py37:testPy37Cython
23:24:52 .....s.............................Exception in thread Thread-943:
23:24:52 Traceback (most recent call last):
23:24:52 File "/usr/lib/python3.7/threading.py", line 917, in _bootstrap_inner
23:24:52 self.run()
23:24:52 File "/usr/lib/python3.7/threading.py", line 865, in run
23:24:52 self._target(*self._args, **self._kwargs)
23:24:52 File
"/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py37/build/srcs/sdks/python/apache_beam/runners/portability/local_job_service.py",
line 274, in _run_job
23:24:52 self._pipeline_proto)
23:24:52 File
"/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py37/build/srcs/sdks/python/apache_beam/runners/portability/fn_api_runner/fn_runner.py",
line 202, in run_via_runner_api
23:24:52 self._check_requirements(pipeline_proto)
23:24:52 File
"/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py37/build/srcs/sdks/python/apache_beam/runners/portability/fn_api_runner/fn_runner.py",
line 301, in _check_requirements
23:24:52 'Unable to run pipeline with requirement: %s' % requirement)
23:24:52 ValueError: Unable to run pipeline with requirement:
unsupported_requirement
23:24:52
23:24:52 ........ [ 41%]
23:24:52
.........sssssssssssssssssssssssssssssssssssssssssssssssssssssssssss...s [ 42%]
23:24:52
sssss.ssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssss [ 44%]
23:24:52 ssssssssssss..............Exception in thread Thread-947:
23:24:52 Traceback (most recent call last):
23:24:52 File "apache_beam/runners/common.py", line 1223, in
apache_beam.runners.common.DoFnRunner.process
23:24:52 return self.do_fn_invoker.invoke_process(windowed_value)
23:24:52 File "apache_beam/runners/common.py", line 752, in
apache_beam.runners.common.PerWindowInvoker.invoke_process
23:24:52 self._invoke_process_per_window(
23:24:52 File "apache_beam/runners/common.py", line 877, in
apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
23:24:52 self.process_method(*args_for_process),
23:24:52 File
"/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py37/build/srcs/sdks/python/apache_beam/transforms/core.py",
line 1568, in <lambda>
23:24:52 wrapper = lambda x, *args, **kwargs: [fn(x, *args, **kwargs)]
23:24:52 File
"/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py37/build/srcs/sdks/python/apache_beam/testing/util.py",
line 190, in _equal
23:24:52 raise BeamAssertException(msg)
23:24:52 apache_beam.testing.util.BeamAssertException: Failed assert: ['a'] ==
['a', 'b'], unexpected elements ['b']
23:24:52
23:24:52 During handling of the above exception, another exception occurred:
23:24:52
23:24:52 Traceback (most recent call last):
23:24:52 File "/usr/lib/python3.7/threading.py", line 917, in _bootstrap_inner
23:24:52 self.run()
23:24:52 File "/usr/lib/python3.7/threading.py", line 865, in run
23:24:52 self._target(*self._args, **self._kwargs)
23:24:52 File
"/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py37/build/srcs/sdks/python/apache_beam/runners/portability/local_job_service.py",
line 274, in _run_job
23:24:52 self._pipeline_proto)
23:24:52 File
"/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py37/build/srcs/sdks/python/apache_beam/runners/portability/fn_api_runner/fn_runner.py",
line 206, in run_via_runner_api
23:24:52 return self.run_stages(stage_context, stages)
23:24:52 File
"/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py37/build/srcs/sdks/python/apache_beam/runners/portability/fn_api_runner/fn_runner.py",
line 385, in run_stages
23:24:52 runner_execution_context, bundle_context_manager)
23:24:52 File
"/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py37/build/srcs/sdks/python/apache_beam/runners/portability/fn_api_runner/fn_runner.py",
line 653, in _run_stage
23:24:52 bundle_manager))
23:24:52 File
"/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py37/build/srcs/sdks/python/apache_beam/runners/portability/fn_api_runner/fn_runner.py",
line 770, in _run_bundle
23:24:52 data_input, data_output, input_timers, expected_timer_output)
23:24:52 File
"/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py37/build/srcs/sdks/python/apache_beam/runners/portability/fn_api_runner/fn_runner.py",
line 1080, in process_bundle
23:24:52 result_future =
self._worker_handler.control_conn.push(process_bundle_req)
23:24:52 File
"/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py37/build/srcs/sdks/python/apache_beam/runners/portability/fn_api_runner/worker_handlers.py",
line 378, in push
23:24:52 response = self.worker.do_instruction(request)
23:24:52 File
"/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py37/build/srcs/sdks/python/apache_beam/runners/worker/sdk_worker.py",
line 602, in do_instruction
23:24:52 getattr(request, request_type), request.instruction_id)
23:24:52 File
"/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py37/build/srcs/sdks/python/apache_beam/runners/worker/sdk_worker.py",
line 639, in process_bundle
23:24:52 bundle_processor.process_bundle(instruction_id))
23:24:52 File
"/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py37/build/srcs/sdks/python/apache_beam/runners/worker/bundle_processor.py",
line 997, in process_bundle
23:24:52 element.data)
23:24:52 File
"/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py37/build/srcs/sdks/python/apache_beam/runners/worker/bundle_processor.py",
line 222, in process_encoded
23:24:52 self.output(decoded_value)
23:24:52 File "apache_beam/runners/worker/operations.py", line 351, in
apache_beam.runners.worker.operations.Operation.output
23:24:52 def output(self, windowed_value, output_index=0):
23:24:52 File "apache_beam/runners/worker/operations.py", line 353, in
apache_beam.runners.worker.operations.Operation.output
23:24:52 cython.cast(Receiver,
self.receivers[output_index]).receive(windowed_value)
23:24:52 File "apache_beam/runners/worker/operations.py", line 215, in
apache_beam.runners.worker.operations.SingletonConsumerSet.receive
23:24:52 self.consumer.process(windowed_value)
23:24:52 File "apache_beam/runners/worker/operations.py", line 712, in
apache_beam.runners.worker.operations.DoOperation.process
23:24:52 with self.scoped_process_state:
23:24:52 File "apache_beam/runners/worker/operations.py", line 713, in
apache_beam.runners.worker.operations.DoOperation.process
23:24:52 delayed_application = self.dofn_runner.process(o)
23:24:52 File "apache_beam/runners/common.py", line 1225, in
apache_beam.runners.common.DoFnRunner.process
23:24:52 self._reraise_augmented(exn)
23:24:52 File "apache_beam/runners/common.py", line 1290, in
apache_beam.runners.common.DoFnRunner._reraise_augmented
23:24:52 raise
23:24:52 File "apache_beam/runners/common.py", line 1223, in
apache_beam.runners.common.DoFnRunner.process
23:24:52 return self.do_fn_invoker.invoke_process(windowed_value)
23:24:52 File "apache_beam/runners/common.py", line 571, in
apache_beam.runners.common.SimpleInvoker.invoke_process
23:24:52 self.output_processor.process_outputs(
23:24:52 File "apache_beam/runners/common.py", line 1386, in
apache_beam.runners.common._OutputProcessor.process_outputs
23:24:52 self.main_receivers.receive(windowed_value)
23:24:52 File "apache_beam/runners/worker/operations.py", line 215, in
apache_beam.runners.worker.operations.SingletonConsumerSet.receive
23:24:52 self.consumer.process(windowed_value)
23:24:52 File "apache_beam/runners/worker/operations.py", line 712, in
apache_beam.runners.worker.operations.DoOperation.process
23:24:52 with self.scoped_process_state:
23:24:52 File "apache_beam/runners/worker/operations.py", line 713, in
apache_beam.runners.worker.operations.DoOperation.process
23:24:52 delayed_application = self.dofn_runner.process(o)
23:24:52 File "apache_beam/runners/common.py", line 1225, in
apache_beam.runners.common.DoFnRunner.process
23:24:52 self._reraise_augmented(exn)
23:24:52 File "apache_beam/runners/common.py", line 1290, in
apache_beam.runners.common.DoFnRunner._reraise_augmented
23:24:52 raise
23:24:52 File "apache_beam/runners/common.py", line 1223, in
apache_beam.runners.common.DoFnRunner.process
23:24:52 return self.do_fn_invoker.invoke_process(windowed_value)
23:24:52 File "apache_beam/runners/common.py", line 571, in
apache_beam.runners.common.SimpleInvoker.invoke_process
23:24:52 self.output_processor.process_outputs(
23:24:52 File "apache_beam/runners/common.py", line 1386, in
apache_beam.runners.common._OutputProcessor.process_outputs
23:24:52 self.main_receivers.receive(windowed_value)
23:24:52 File "apache_beam/runners/worker/operations.py", line 215, in
apache_beam.runners.worker.operations.SingletonConsumerSet.receive
23:24:52 self.consumer.process(windowed_value)
23:24:52 File "apache_beam/runners/worker/operations.py", line 712, in
apache_beam.runners.worker.operations.DoOperation.process
23:24:52 with self.scoped_process_state:
23:24:52 File "apache_beam/runners/worker/operations.py", line 713, in
apache_beam.runners.worker.operations.DoOperation.process
23:24:52 delayed_application = self.dofn_runner.process(o)
23:24:52 File "apache_beam/runners/common.py", line 1225, in
apache_beam.runners.common.DoFnRunner.process
23:24:52 self._reraise_augmented(exn)
23:24:52 File "apache_beam/runners/common.py", line 1290, in
apache_beam.runners.common.DoFnRunner._reraise_augmented
23:24:52 raise
23:24:52 File "apache_beam/runners/common.py", line 1223, in
apache_beam.runners.common.DoFnRunner.process
23:24:52 return self.do_fn_invoker.invoke_process(windowed_value)
23:24:52 File "apache_beam/runners/common.py", line 571, in
apache_beam.runners.common.SimpleInvoker.invoke_process
23:24:52 self.output_processor.process_outputs(
23:24:52 File "apache_beam/runners/common.py", line 1386, in
apache_beam.runners.common._OutputProcessor.process_outputs
23:24:52 self.main_receivers.receive(windowed_value)
23:24:52 File "apache_beam/runners/worker/operations.py", line 215, in
apache_beam.runners.worker.operations.SingletonConsumerSet.receive
23:24:52 self.consumer.process(windowed_value)
23:24:52 File "apache_beam/runners/worker/operations.py", line 712, in
apache_beam.runners.worker.operations.DoOperation.process
23:24:52 with self.scoped_process_state:
23:24:52 File "apache_beam/runners/worker/operations.py", line 713, in
apache_beam.runners.worker.operations.DoOperation.process
23:24:52 delayed_application = self.dofn_runner.process(o)
23:24:52 File "apache_beam/runners/common.py", line 1225, in
apache_beam.runners.common.DoFnRunner.process
23:24:52 self._reraise_augmented(exn)
23:24:52 File "apache_beam/runners/common.py", line 1306, in
apache_beam.runners.common.DoFnRunner._reraise_augmented
23:24:52 raise new_exn.with_traceback(tb)
23:24:52 File "apache_beam/runners/common.py", line 1223, in
apache_beam.runners.common.DoFnRunner.process
23:24:52 return self.do_fn_invoker.invoke_process(windowed_value)
23:24:52 File "apache_beam/runners/common.py", line 752, in
apache_beam.runners.common.PerWindowInvoker.invoke_process
23:24:52 self._invoke_process_per_window(
23:24:52 File "apache_beam/runners/common.py", line 877, in
apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
23:24:52 self.process_method(*args_for_process),
23:24:52 File
"/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py37/build/srcs/sdks/python/apache_beam/transforms/core.py",
line 1568, in <lambda>
23:24:52 wrapper = lambda x, *args, **kwargs: [fn(x, *args, **kwargs)]
23:24:52 File
"/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py37/build/srcs/sdks/python/apache_beam/testing/util.py",
line 190, in _equal
23:24:52 raise BeamAssertException(msg)
23:24:52 apache_beam.testing.util.BeamAssertException: Failed assert: ['a'] ==
['a', 'b'], unexpected elements ['b'] [while running 'assert_that/Match']
23:24:52 .
{code}
`test_pack_combiners` :
https://ci-beam.apache.org/job/beam_PreCommit_Python_Cron/4415/consoleFull
{code:java}
11:20:01 Traceback (most recent call last):
11:20:01 File "/usr/lib/python3.6/threading.py", line 916, in _bootstrap_inner
11:20:01 self.run()
11:20:01 File "/usr/lib/python3.6/threading.py", line 864, in run
11:20:01 self._target(*self._args, **self._kwargs)
11:20:01 File
"/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py36/build/srcs/sdks/python/apache_beam/runners/portability/local_job_service.py",
line 274, in _run_job
11:20:01 self._pipeline_proto)
11:20:01 File
"/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py36/build/srcs/sdks/python/apache_beam/runners/portability/fn_api_runner/fn_runner.py",
line 202, in run_via_runner_api
11:20:01 self._check_requirements(pipeline_proto)
11:20:01 File
"/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py36/build/srcs/sdks/python/apache_beam/runners/portability/fn_api_runner/fn_runner.py",
line 301, in _check_requirements
11:20:01 'Unable to run pipeline with requirement: %s' % requirement)
11:20:01 ValueError: Unable to run pipeline with requirement:
unsupported_requirement
11:20:01 .
11:20:08 ...............................ssss..... [ 48%]
11:20:09
...sssss.........s....sss.ssssssssssssssssssssssssssssssssssssssssssssss [ 50%]
11:20:13
sssssssssssssssss.ssssssssssssssss...................................... [ 51%]
11:20:18
..............sss.sssssss..................................s............ [ 53%]
11:20:25 .........................................
11:20:25 > Task :sdks:python:test-suites:tox:py37:testPy37Cloud
11:20:25 ...............ssss.......... [ 48%]
11:20:26 .........................sss.sssssss.................................
11:20:26 > Task :sdks:python:test-suites:tox:py38:testPy38CloudCoverage
11:20:26 ................................................. [ 22%]
11:20:26
.................s..s................................................... [ 24%]
11:20:26
...sssss................................................................ [ 25%]
11:20:26
..................................s..................................... [ 27%]
11:20:26
.....................................................sssssss............ [ 28%]
11:20:26
......................ss......................................ss........ [ 30%]
11:20:26
......................................................................s. [ 31%]
11:20:27
.....s................................sssssssss...ssssss................ [ 33%]
11:20:37
.............s...sss.....................................sssssss........ [ 34%]
11:20:41
....................................s...........................s...s... [ 36%]
11:20:51
.................s............sss......................................s [ 37%]
11:20:54 .sss........................s
11:20:54 > Task :sdks:python:test-suites:tox:py36:testPy36Cloud
11:20:54 .............s................. [ 54%]
11:20:56 .........ss...s..............................ss.....s.......s.....
11:20:56 > Task :sdks:python:test-suites:tox:py37:testPy37Cloud
11:20:56 ... [ 50%]
11:20:56
..s.........................................s........................... [ 51%]
11:20:56 .................ss.ss........Exception in thread Thread-1007:
11:20:56 Traceback (most recent call last):
11:20:56 File
"/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py37/build/srcs/sdks/python/apache_beam/runners/common.py",
line 1223, in process
11:20:56 return self.do_fn_invoker.invoke_process(windowed_value)
11:20:56 File
"/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py37/build/srcs/sdks/python/apache_beam/runners/common.py",
line 572, in invoke_process
11:20:56 windowed_value, self.process_method(windowed_value.value))
11:20:56 File
"/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py37/build/srcs/sdks/python/apache_beam/transforms/core.py",
line 1570, in <lambda>
11:20:56 wrapper = lambda x: [fn(x)]
11:20:56 File
"/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py37/build/srcs/sdks/python/apache_beam/runners/portability/fn_api_runner/fn_runner_test.py",
line 820, in raise_error
11:20:56 raise RuntimeError('x')
11:20:56 RuntimeError: x
{code}
> Flaky:
> apache_beam.runners.portability.fn_api_runner.fn_runner_test.FnApiRunnerTestWithGrpcAndMultiWorkers
> ----------------------------------------------------------------------------------------------------------
>
> Key: BEAM-12603
> URL: https://issues.apache.org/jira/browse/BEAM-12603
> Project: Beam
> Issue Type: Bug
> Components: sdk-py-core, test-failures
> Reporter: Tyson Hamilton
> Priority: P1
>
> The
> `apache_beam.runners.portability.fn_api_runner.fn_runner_test.FnApiRunnerTestWithGrpcAndMultiWorkers`
> tests are flaky and causing precommit failures that seem similar.
> `test_pardo_windowed_side_inputs` :
> [https://ci-beam.apache.org/job/beam_PreCommit_Python_Cron/4417/console]
> {code:java}
> 23:32:35 Exception in thread read_grpc_client_inputs:
> 23:32:35 Traceback (most recent call last):
> 23:32:35 File "/usr/lib/python3.8/threading.py", line 932, in
> _bootstrap_inner
> 23:32:35 self.run()
> 23:32:35 File "/usr/lib/python3.8/threading.py", line 870, in run
> 23:32:35 self._target(*self._args, **self._kwargs)
> 23:32:35 File
> "/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py38/build/srcs/sdks/python/apache_beam/runners/worker/data_plane.py",
> line 587, in <lambda>
> 23:32:35 target=lambda: self._read_inputs(elements_iterator),
> 23:32:35 File
> "/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py38/build/srcs/sdks/python/apache_beam/runners/worker/data_plane.py",
> line 570, in _read_inputs
> 23:32:35 for elements in elements_iterator:
> 23:32:35 File
> "/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py38/build/srcs/sdks/python/target/.tox-py38-cython/py38-cython/lib/python3.8/site-packages/grpc/_channel.py",
> line 426, in __next__
> 23:32:35 return self._next()
> 23:32:35 File
> "/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py38/build/srcs/sdks/python/target/.tox-py38-cython/py38-cython/lib/python3.8/site-packages/grpc/_channel.py",
> line 826, in _next
> 23:32:35 raise self
> 23:32:35 grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of
> RPC that terminated with:
> 23:32:35 status = StatusCode.UNAVAILABLE
> 23:32:35 details = "Broken pipe"
> 23:32:35 debug_error_string =
> "{"created":"@1626071403.252458842","description":"Error received from peer
> ipv4:127.0.0.1:37459","file":"src/core/lib/surface/call.cc","file_line":1066,"grpc_message":"Broken
> pipe","grpc_status":14}"
> 23:32:35 >
> 23:32:35 Traceback (most recent call last):
> 23:32:35 File
> "/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py38/build/srcs/sdks/python/apache_beam/runners/worker/data_plane.py",
> line 459, in input_elements
> 23:32:35 element = received.get(timeout=1)
> 23:32:35 File "/usr/lib/python3.8/queue.py", line 178, in get
> 23:32:35 raise Empty
> 23:32:35 _queue.Empty
> {code}
>
> `test_pack_combiners` :
> [https://ci-beam.apache.org/job/beam_PreCommit_Python_Cron/4415/consoleFull]
> {code:java}
> 11:41:14 Exception ignored in: <object repr() failed>
> 11:41:14 Traceback (most recent call last):
> 11:41:14 File
> "/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py36/build/srcs/sdks/python/target/.tox-py36-cython/py36-cython/lib/python3.6/site-packages/grpc/_channel.py",
> line 444, in __del__
> 11:41:14 with self._state.condition:
> 11:41:14 AttributeError: '_MultiThreadedRendezvous' object has no attribute
> '_state'
> 11:41:14 Traceback (most recent call last):
> 11:41:14 File "apache_beam/runners/common.py", line 1223, in
> apache_beam.runners.common.DoFnRunner.process
> 11:41:14 return self.do_fn_invoker.invoke_process(windowed_value)
> 11:41:14 File "apache_beam/runners/common.py", line 752, in
> apache_beam.runners.common.PerWindowInvoker.invoke_process
> 11:41:14 self._invoke_process_per_window(
> 11:41:14 File "apache_beam/runners/common.py", line 816, in
> apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
> 11:41:14 [si[global_window] for si in self.side_inputs]))
> 11:41:14 File
> "/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py36/build/srcs/sdks/python/apache_beam/runners/worker/bundle_processor.py",
> line 427, in __getitem__
> 11:41:14 self._cache[target_window] =
> self._side_input_data.view_fn(raw_view)
> 11:41:14 File
> "/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py36/build/srcs/sdks/python/apache_beam/runners/worker/bundle_processor.py",
> line 353, in __iter__
> 11:41:14 self._state_handler.blocking_get(self._state_key,
> self._coder_impl))
> 11:41:14 File
> "/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py36/build/srcs/sdks/python/apache_beam/runners/worker/sdk_worker.py",
> line 1184, in blocking_get
> 11:41:14 self._partially_cached_iterable(state_key, coder))
> 11:41:14 File
> "/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py36/build/srcs/sdks/python/apache_beam/runners/worker/sdk_worker.py",
> line 1290, in _partially_cached_iterable
> 11:41:14 data, continuation_token = self._underlying.get_raw(state_key,
> None)
> 11:41:14 File
> "/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py36/build/srcs/sdks/python/apache_beam/runners/worker/sdk_worker.py",
> line 1057, in get_raw
> 11:41:14 continuation_token=continuation_token)))
> 11:41:14 File
> "/home/jenkins/jenkins-slave/workspace/beam_PreCommit_Python_Cron/src/sdks/python/test-suites/tox/py36/build/srcs/sdks/python/apache_beam/runners/worker/sdk_worker.py",
> line 1095, in _blocking_request
> 11:41:14 raise t(v).with_traceback(tb)
> 11:41:14 TypeError: __init__() missing 3 required positional arguments:
> 'call', 'response_deserializer', and 'deadline'
> {code}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
