tvalentyn commented on issue #30867:
URL: https://github.com/apache/beam/issues/30867#issuecomment-2048463960
follow up investigation: ssh into the VM:
```
gcloud compute ssh --zone "us-central1-b"
"beamapp-valentyn-04101932-04101232-3i3x-harness-3sqg" --project
"apache-beam-testing"
```
```
curl localhost:8081/sdk_status
```
is unresponsive.
However, we can ssh into the docker container and use pystack:
```
valentyn@...harness-3sqg ~ $ docker exec --privileged -it 1142b2c7eb3c
/bin/bash
pip install pystack
ps -A
root@beamapp-valentyn-04101932-04101232-3i3x-harness-3sqg:/# ps -A
PID TTY TIME CMD
1 ? 00:00:00 boot
29 ? 00:03:02 python
30 ? 00:02:46 python
114 pts/0 00:00:00 bash
125 pts/0 00:00:00 ps
pystack remote 29
```
Which reveals:
```
docker exec --privileged -it 1142b2c7eb3c /bin/bash
root@beamapp-valentyn-04101932-04101232-3i3x-harness-3sqg:/# pystack remote
29
Traceback for thread 100 (python) [Has the GIL] (most recent call last):
(Python) File "/usr/local/lib/python3.8/threading.py", line 890, in
_bootstrap
self._bootstrap_inner()
(Python) File "/usr/local/lib/python3.8/threading.py", line 932, in
_bootstrap_inner
self.run()
(Python) File "/usr/local/lib/python3.8/threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
(Python) File "/usr/local/lib/python3.8/concurrent/futures/thread.py",
line 80, in _worker
work_item.run()
(Python) File "/usr/local/lib/python3.8/concurrent/futures/thread.py",
line 57, in run
result = self.fn(*self.args, **self.kwargs)
(Python) File
"/usr/local/lib/python3.8/site-packages/google/cloud/bigtable/batcher.py", line
385, in _flush_rows
response = self.table.mutate_rows(rows_to_flush)
(Python) File
"/usr/local/lib/python3.8/site-packages/google/cloud/bigtable/table.py", line
724, in mutate_rows
self.name,
(Python) File
"/usr/local/lib/python3.8/site-packages/google/cloud/bigtable/table.py", line
160, in name
table_client = self._instance._client.table_data_client
(Python) File
"/usr/local/lib/python3.8/site-packages/google/cloud/bigtable/client.py", line
332, in table_data_client
transport = self._create_gapic_client_channel(
(Python) File
"/usr/local/lib/python3.8/site-packages/google/cloud/bigtable/client.py", line
285, in _create_gapic_client_channel
channel = grpc_transport.create_channel(
(Python) File
"/usr/local/lib/python3.8/site-packages/google/cloud/bigtable_v2/services/bigtable/transports/grpc.py",
line 217, in create_channel
return grpc_helpers.create_channel(
(Python) File
"/usr/local/lib/python3.8/site-packages/google/api_core/grpc_helpers.py", line
386, in create_channel
return grpc.secure_channel(
(Python) File "/usr/local/lib/python3.8/site-packages/grpc/__init__.py",
line 2119, in secure_channel
return _channel.Channel(
(Python) File "/usr/local/lib/python3.8/site-packages/grpc/_channel.py",
line 2046, in __init__
self._channel = cygrpc.Channel(
Traceback for thread 99 (python) [] (most recent call last):
(Python) File "/usr/local/lib/python3.8/threading.py", line 890, in
_bootstrap
self._bootstrap_inner()
(Python) File "/usr/local/lib/python3.8/threading.py", line 932, in
_bootstrap_inner
self.run()
(Python) File "/usr/local/lib/python3.8/threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
(Python) File "/usr/local/lib/python3.8/concurrent/futures/thread.py",
line 80, in _worker
work_item.run()
(Python) File "/usr/local/lib/python3.8/concurrent/futures/thread.py",
line 57, in run
result = self.fn(*self.args, **self.kwargs)
(Python) File
"/usr/local/lib/python3.8/site-packages/google/cloud/bigtable/batcher.py", line
385, in _flush_rows
response = self.table.mutate_rows(rows_to_flush)
(Python) File
"/usr/local/lib/python3.8/site-packages/google/cloud/bigtable/table.py", line
724, in mutate_rows
self.name,
(Python) File
"/usr/local/lib/python3.8/site-packages/google/cloud/bigtable/table.py", line
160, in name
table_client = self._instance._client.table_data_client
(Python) File
"/usr/local/lib/python3.8/site-packages/google/cloud/bigtable/client.py", line
332, in table_data_client
transport = self._create_gapic_client_channel(
(Python) File
"/usr/local/lib/python3.8/site-packages/google/cloud/bigtable/client.py", line
285, in _create_gapic_client_channel
channel = grpc_transport.create_channel(
(Python) File
"/usr/local/lib/python3.8/site-packages/google/cloud/bigtable_v2/services/bigtable/transports/grpc.py",
line 217, in create_channel
return grpc_helpers.create_channel(
(Python) File
"/usr/local/lib/python3.8/site-packages/google/api_core/grpc_helpers.py", line
386, in create_channel
return grpc.secure_channel(
(Python) File "/usr/local/lib/python3.8/site-packages/grpc/__init__.py",
line 2119, in secure_channel
return _channel.Channel(
(Python) File "/usr/local/lib/python3.8/site-packages/grpc/_channel.py",
line 2046, in __init__
self._channel = cygrpc.Channel(
(Python) File "/usr/local/lib/python3.8/pkgutil.py", line 638, in
get_data
return loader.get_data(resource_name)
(Python) File "<frozen importlib._bootstrap_external>", line 1033, in
get_data
Traceback for thread 98 (python) [] (most recent call last):
(Python) File "/usr/local/lib/python3.8/threading.py", line 890, in
_bootstrap
self._bootstrap_inner()
(Python) File "/usr/local/lib/python3.8/threading.py", line 932, in
_bootstrap_inner
self.run()
(Python) File "/usr/local/lib/python3.8/threading.py", line 1252, in run
self.finished.wait(self.interval)
(Python) File "/usr/local/lib/python3.8/threading.py", line 558, in wait
signaled = self._cond.wait(timeout)
(Python) File "/usr/local/lib/python3.8/threading.py", line 306, in wait
gotit = waiter.acquire(True, timeout)
Traceback for thread 97 (python) [] (most recent call last):
(Python) File "/usr/local/lib/python3.8/threading.py", line 890, in
_bootstrap
self._bootstrap_inner()
(Python) File "/usr/local/lib/python3.8/threading.py", line 932, in
_bootstrap_inner
self.run()
(Python) File "/usr/local/lib/python3.8/threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
Traceback for thread 95 (python) [] (most recent call last):
(Python) File "/usr/local/lib/python3.8/threading.py", line 890, in
_bootstrap
self._bootstrap_inner()
(Python) File "/usr/local/lib/python3.8/threading.py", line 932, in
_bootstrap_inner
self.run()
(Python) File "/usr/local/lib/python3.8/threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
(Python) File
"/usr/local/lib/python3.8/site-packages/apache_beam/runners/worker/data_plane.py",
line 686, in <lambda>
target=lambda: self._read_inputs(elements_iterator),
(Python) File
"/usr/local/lib/python3.8/site-packages/apache_beam/runners/worker/data_plane.py",
line 669, in _read_inputs
for elements in elements_iterator:
(Python) File "/usr/local/lib/python3.8/site-packages/grpc/_channel.py",
line 542, in __next__
return self._next()
(Python) File "/usr/local/lib/python3.8/site-packages/grpc/_channel.py",
line 959, in _next
_common.wait(self._state.condition.wait, _response_ready)
(Python) File "/usr/local/lib/python3.8/site-packages/grpc/_common.py",
line 156, in wait
_wait_once(wait_fn, MAXIMUM_WAIT_TIMEOUT, spin_cb)
(Python) File "/usr/local/lib/python3.8/site-packages/grpc/_common.py",
line 116, in _wait_once
wait_fn(timeout=timeout)
(Python) File "/usr/local/lib/python3.8/threading.py", line 306, in wait
gotit = waiter.acquire(True, timeout)
Traceback for thread 94 (python) [] (most recent call last):
(Python) File "/usr/local/lib/python3.8/threading.py", line 890, in
_bootstrap
self._bootstrap_inner()
(Python) File "/usr/local/lib/python3.8/threading.py", line 932, in
_bootstrap_inner
self.run()
(Python) File "/usr/local/lib/python3.8/threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
(Python) File "/usr/local/lib/python3.8/site-packages/grpc/_channel.py",
line 275, in consume_request_iterator
request = next(request_iterator)
(Python) File
"/usr/local/lib/python3.8/site-packages/apache_beam/runners/worker/data_plane.py",
line 608, in _write_outputs
streams = [self._to_send.get()]
(Python) File "/usr/local/lib/python3.8/queue.py", line 170, in get
self.not_empty.wait()
(Python) File "/usr/local/lib/python3.8/threading.py", line 302, in wait
waiter.acquire()
Traceback for thread 93 (python) [] (most recent call last):
(Python) File "/usr/local/lib/python3.8/threading.py", line 890, in
_bootstrap
self._bootstrap_inner()
(Python) File "/usr/local/lib/python3.8/threading.py", line 932, in
_bootstrap_inner
self.run()
(Python) File "/usr/local/lib/python3.8/threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
(Python) File "/usr/local/lib/python3.8/site-packages/grpc/_channel.py",
line 1728, in channel_spin
event = state.channel.next_call_event()
Traceback for thread 92 (python) [] (most recent call last):
(Python) File "/usr/local/lib/python3.8/threading.py", line 890, in
_bootstrap
self._bootstrap_inner()
(Python) File "/usr/local/lib/python3.8/threading.py", line 932, in
_bootstrap_inner
self.run()
(Python) File "/usr/local/lib/python3.8/threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
(Python) File
"/usr/local/lib/python3.8/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 1060, in pull_responses
for response in responses:
(Python) File "/usr/local/lib/python3.8/site-packages/grpc/_channel.py",
line 542, in __next__
return self._next()
(Python) File "/usr/local/lib/python3.8/site-packages/grpc/_channel.py",
line 959, in _next
_common.wait(self._state.condition.wait, _response_ready)
(Python) File "/usr/local/lib/python3.8/site-packages/grpc/_common.py",
line 156, in wait
_wait_once(wait_fn, MAXIMUM_WAIT_TIMEOUT, spin_cb)
(Python) File "/usr/local/lib/python3.8/site-packages/grpc/_common.py",
line 116, in _wait_once
wait_fn(timeout=timeout)
(Python) File "/usr/local/lib/python3.8/threading.py", line 306, in wait
gotit = waiter.acquire(True, timeout)
Traceback for thread 91 (python) [] (most recent call last):
(Python) File "/usr/local/lib/python3.8/threading.py", line 890, in
_bootstrap
self._bootstrap_inner()
(Python) File "/usr/local/lib/python3.8/threading.py", line 932, in
_bootstrap_inner
self.run()
(Python) File "/usr/local/lib/python3.8/threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
(Python) File "/usr/local/lib/python3.8/site-packages/grpc/_channel.py",
line 275, in consume_request_iterator
request = next(request_iterator)
(Python) File
"/usr/local/lib/python3.8/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 1050, in request_iter
request = self._requests.get()
(Python) File "/usr/local/lib/python3.8/queue.py", line 170, in get
self.not_empty.wait()
(Python) File "/usr/local/lib/python3.8/threading.py", line 302, in wait
waiter.acquire()
Traceback for thread 90 (python) [] (most recent call last):
(Python) File "/usr/local/lib/python3.8/threading.py", line 890, in
_bootstrap
self._bootstrap_inner()
(Python) File "/usr/local/lib/python3.8/threading.py", line 932, in
_bootstrap_inner
self.run()
(Python) File "/usr/local/lib/python3.8/threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
(Python) File "/usr/local/lib/python3.8/site-packages/grpc/_channel.py",
line 1728, in channel_spin
event = state.channel.next_call_event()
Traceback for thread 89 (python) [] (most recent call last):
(Python) File "/usr/local/lib/python3.8/threading.py", line 890, in
_bootstrap
self._bootstrap_inner()
(Python) File "/usr/local/lib/python3.8/threading.py", line 932, in
_bootstrap_inner
self.run()
(Python) File
"/usr/local/lib/python3.8/site-packages/apache_beam/utils/thread_pool_executor.py",
line 53, in run
self._work_item.run()
(Python) File
"/usr/local/lib/python3.8/site-packages/apache_beam/utils/thread_pool_executor.py",
line 37, in run
self._future.set_result(self._fn(*self._fn_args, **self._fn_kwargs))
(Python) File
"/usr/local/lib/python3.8/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 385, in task
self._execute(
(Python) File
"/usr/local/lib/python3.8/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 311, in _execute
response = task()
(Python) File
"/usr/local/lib/python3.8/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 386, in <lambda>
lambda: self.create_worker().do_instruction(request), request)
(Python) File
"/usr/local/lib/python3.8/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 650, in do_instruction
return getattr(self, request_type)(
(Python) File
"/usr/local/lib/python3.8/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 688, in process_bundle
bundle_processor.process_bundle(instruction_id))
(Python) File
"/usr/local/lib/python3.8/site-packages/apache_beam/runners/worker/bundle_processor.py",
line 1113, in process_bundle
input_op_by_transform_id[element.transform_id].process_encoded(
(Python) File
"/usr/local/lib/python3.8/site-packages/apache_beam/runners/worker/bundle_processor.py",
line 237, in process_encoded
self.output(decoded_value)
(Python) File
"/usr/local/lib/python3.8/site-packages/apache_beam/io/gcp/bigtableio.py", line
156, in process
self.batcher.mutate(row)
(Python) File
"/usr/local/lib/python3.8/site-packages/google/cloud/bigtable/batcher.py", line
266, in mutate
self._flush_async()
(Python) File
"/usr/local/lib/python3.8/site-packages/google/cloud/bigtable/batcher.py", line
340, in _flush_async
future = self._executor.submit(self._flush_rows, rows_to_flush)
(Python) File "/usr/local/lib/python3.8/concurrent/futures/thread.py",
line 188, in submit
self._adjust_thread_count()
(Python) File "/usr/local/lib/python3.8/concurrent/futures/thread.py",
line 213, in _adjust_thread_count
t.start()
(Python) File "/usr/local/lib/python3.8/threading.py", line 857, in start
self._started.wait()
(Python) File "/usr/local/lib/python3.8/threading.py", line 558, in wait
signaled = self._cond.wait(timeout)
(Python) File "/usr/local/lib/python3.8/threading.py", line 302, in wait
waiter.acquire()
Traceback for thread 88 (python) [] (most recent call last):
(Python) File "/usr/local/lib/python3.8/threading.py", line 890, in
_bootstrap
self._bootstrap_inner()
(Python) File "/usr/local/lib/python3.8/threading.py", line 932, in
_bootstrap_inner
self.run()
(Python) File "/usr/local/lib/python3.8/threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
(Python) File "/usr/local/lib/python3.8/site-packages/grpc/_channel.py",
line 275, in consume_request_iterator
request = next(request_iterator)
(Python) File
"/usr/local/lib/python3.8/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 258, in get_responses
response = self._responses.get()
(Python) File "/usr/local/lib/python3.8/queue.py", line 170, in get
self.not_empty.wait()
(Python) File "/usr/local/lib/python3.8/threading.py", line 302, in wait
waiter.acquire()
Traceback for thread 87 (python) [] (most recent call last):
(Python) File "/usr/local/lib/python3.8/threading.py", line 890, in
_bootstrap
self._bootstrap_inner()
(Python) File "/usr/local/lib/python3.8/threading.py", line 932, in
_bootstrap_inner
self.run()
(Python) File "/usr/local/lib/python3.8/threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
(Python) File "/usr/local/lib/python3.8/site-packages/grpc/_channel.py",
line 1728, in channel_spin
event = state.channel.next_call_event()
Traceback for thread 83 (python) [] (most recent call last):
(Python) File "/usr/local/lib/python3.8/threading.py", line 890, in
_bootstrap
self._bootstrap_inner()
(Python) File "/usr/local/lib/python3.8/threading.py", line 932, in
_bootstrap_inner
self.run()
(Python) File "/usr/local/lib/python3.8/threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
(Python) File "/usr/local/lib/python3.8/site-packages/grpc/_channel.py",
line 275, in consume_request_iterator
request = next(request_iterator)
(Python) File
"/usr/local/lib/python3.8/site-packages/apache_beam/runners/worker/worker_status.py",
line 190, in _get_responses
response = self._responses.get()
(Python) File "/usr/local/lib/python3.8/queue.py", line 170, in get
self.not_empty.wait()
(Python) File "/usr/local/lib/python3.8/threading.py", line 302, in wait
waiter.acquire()
Traceback for thread 82 (python) [] (most recent call last):
(Python) File "/usr/local/lib/python3.8/threading.py", line 890, in
_bootstrap
self._bootstrap_inner()
(Python) File "/usr/local/lib/python3.8/threading.py", line 932, in
_bootstrap_inner
self.run()
(Python) File "/usr/local/lib/python3.8/threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
(Python) File "/usr/local/lib/python3.8/site-packages/grpc/_channel.py",
line 1728, in channel_spin
event = state.channel.next_call_event()
Traceback for thread 81 (python) [] (most recent call last):
(Python) File "/usr/local/lib/python3.8/threading.py", line 890, in
_bootstrap
self._bootstrap_inner()
(Python) File "/usr/local/lib/python3.8/threading.py", line 932, in
_bootstrap_inner
self.run()
(Python) File "/usr/local/lib/python3.8/threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
(Python) File
"/usr/local/lib/python3.8/site-packages/apache_beam/runners/worker/worker_status.py",
line 182, in <lambda>
target=lambda: self._log_lull_in_bundle_processor(
(Python) File
"/usr/local/lib/python3.8/site-packages/apache_beam/runners/worker/worker_status.py",
line 232, in _log_lull_in_bundle_processor
time.sleep(2 * 60)
Traceback for thread 80 (python) [] (most recent call last):
(Python) File "/usr/local/lib/python3.8/threading.py", line 890, in
_bootstrap
self._bootstrap_inner()
(Python) File "/usr/local/lib/python3.8/threading.py", line 932, in
_bootstrap_inner
self.run()
(Python) File "/usr/local/lib/python3.8/threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
(Python) File
"/usr/local/lib/python3.8/site-packages/apache_beam/runners/worker/worker_status.py",
line 177, in <lambda>
target=lambda: self._serve(), name='fn_api_status_handler')
(Python) File
"/usr/local/lib/python3.8/site-packages/apache_beam/runners/worker/worker_status.py",
line 198, in _serve
for request in self._status_stub.WorkerStatus(self._get_responses()):
(Python) File "/usr/local/lib/python3.8/site-packages/grpc/_channel.py",
line 542, in __next__
return self._next()
(Python) File "/usr/local/lib/python3.8/site-packages/grpc/_channel.py",
line 959, in _next
_common.wait(self._state.condition.wait, _response_ready)
(Python) File "/usr/local/lib/python3.8/site-packages/grpc/_common.py",
line 156, in wait
_wait_once(wait_fn, MAXIMUM_WAIT_TIMEOUT, spin_cb)
(Python) File "/usr/local/lib/python3.8/site-packages/grpc/_common.py",
line 116, in _wait_once
wait_fn(timeout=timeout)
(Python) File "/usr/local/lib/python3.8/threading.py", line 306, in wait
gotit = waiter.acquire(True, timeout)
Traceback for thread 72 (python) [] (most recent call last):
(Python) File "/usr/local/lib/python3.8/threading.py", line 890, in
_bootstrap
self._bootstrap_inner()
(Python) File "/usr/local/lib/python3.8/threading.py", line 932, in
_bootstrap_inner
self.run()
(Python) File
"/usr/local/lib/python3.8/site-packages/apache_beam/runners/worker/data_plane.py",
line 224, in run
while not self._finished.wait(next_call - time.time()):
(Python) File "/usr/local/lib/python3.8/threading.py", line 558, in wait
signaled = self._cond.wait(timeout)
(Python) File "/usr/local/lib/python3.8/threading.py", line 306, in wait
gotit = waiter.acquire(True, timeout)
Traceback for thread 65 (python) [] (most recent call last):
(Python) File "/usr/local/lib/python3.8/threading.py", line 890, in
_bootstrap
self._bootstrap_inner()
(Python) File "/usr/local/lib/python3.8/threading.py", line 932, in
_bootstrap_inner
self.run()
(Python) File "/usr/local/lib/python3.8/threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
(Python) File "/usr/local/lib/python3.8/site-packages/grpc/_channel.py",
line 275, in consume_request_iterator
request = next(request_iterator)
(Python) File
"/usr/local/lib/python3.8/site-packages/apache_beam/runners/worker/log_handler.py",
line 182, in _write_log_entries
log_entries = [self._log_entry_queue.get()]
(Python) File "/usr/local/lib/python3.8/queue.py", line 170, in get
self.not_empty.wait()
(Python) File "/usr/local/lib/python3.8/threading.py", line 302, in wait
waiter.acquire()
Traceback for thread 63 (python) [] (most recent call last):
(Python) File "/usr/local/lib/python3.8/threading.py", line 890, in
_bootstrap
self._bootstrap_inner()
(Python) File "/usr/local/lib/python3.8/threading.py", line 932, in
_bootstrap_inner
self.run()
(Python) File "/usr/local/lib/python3.8/threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
(Python) File "/usr/local/lib/python3.8/site-packages/grpc/_channel.py",
line 1728, in channel_spin
event = state.channel.next_call_event()
Traceback for thread 60 (python) [] (most recent call last):
(Python) File "/usr/local/lib/python3.8/threading.py", line 890, in
_bootstrap
self._bootstrap_inner()
(Python) File "/usr/local/lib/python3.8/threading.py", line 932, in
_bootstrap_inner
self.run()
(Python) File "/usr/local/lib/python3.8/threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
(Python) File
"/usr/local/lib/python3.8/site-packages/apache_beam/runners/worker/log_handler.py",
line 99, in <lambda>
target=lambda: self._read_log_control_messages(),
(Python) File
"/usr/local/lib/python3.8/site-packages/apache_beam/runners/worker/log_handler.py",
line 216, in _read_log_control_messages
for _ in log_control_iterator:
(Python) File "/usr/local/lib/python3.8/site-packages/grpc/_channel.py",
line 542, in __next__
return self._next()
(Python) File "/usr/local/lib/python3.8/site-packages/grpc/_channel.py",
line 959, in _next
_common.wait(self._state.condition.wait, _response_ready)
(Python) File "/usr/local/lib/python3.8/site-packages/grpc/_common.py",
line 156, in wait
_wait_once(wait_fn, MAXIMUM_WAIT_TIMEOUT, spin_cb)
(Python) File "/usr/local/lib/python3.8/site-packages/grpc/_common.py",
line 116, in _wait_once
wait_fn(timeout=timeout)
(Python) File "/usr/local/lib/python3.8/threading.py", line 306, in wait
gotit = waiter.acquire(True, timeout)
Traceback for thread 29 (python) [] (most recent call last):
(Python) File "/usr/local/lib/python3.8/runpy.py", line 194, in
_run_module_as_main
return _run_code(code, main_globals, None,
(Python) File "/usr/local/lib/python3.8/runpy.py", line 87, in _run_code
exec(code, run_globals)
(Python) File
"/usr/local/lib/python3.8/site-packages/apache_beam/runners/worker/sdk_worker_main.py",
line 367, in <module>
main(sys.argv)
(Python) File
"/usr/local/lib/python3.8/site-packages/apache_beam/runners/worker/sdk_worker_main.py",
line 212, in main
sdk_harness.run()
(Python) File
"/usr/local/lib/python3.8/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 266, in run
for work_request in self._control_stub.Control(get_responses()):
(Python) File "/usr/local/lib/python3.8/site-packages/grpc/_channel.py",
line 542, in __next__
return self._next()
(Python) File "/usr/local/lib/python3.8/site-packages/grpc/_channel.py",
line 959, in _next
_common.wait(self._state.condition.wait, _response_ready)
(Python) File "/usr/local/lib/python3.8/site-packages/grpc/_common.py",
line 156, in wait
_wait_once(wait_fn, MAXIMUM_WAIT_TIMEOUT, spin_cb)
(Python) File "/usr/local/lib/python3.8/site-packages/grpc/_common.py",
line 116, in _wait_once
wait_fn(timeout=timeout)
(Python) File "/usr/local/lib/python3.8/threading.py", line 306, in wait
gotit = waiter.acquire(True, timeout)
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscr...@beam.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
