[
https://issues.apache.org/jira/browse/BEAM-6258?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17543078#comment-17543078
]
Luke Cwik commented on BEAM-6258:
---------------------------------
The underlying issue was fixed in gRPC c-core and a min version update to
1.33.1 will ensure that this no longer happens since it will contain
https://github.com/grpc/grpc/commit/6e1655447ab2146a643114687d7916249bfdf018
which is the fix for https://github.com/grpc/grpc-java/issues/5188
> Data channel failing after some time for 1G data input
> ------------------------------------------------------
>
> Key: BEAM-6258
> URL: https://issues.apache.org/jira/browse/BEAM-6258
> Project: Beam
> Issue Type: Bug
> Components: sdk-py-harness
> Reporter: Ankur Goenka
> Priority: P3
> Attachments: d44b7eda9e4c_java_server_logs.logs.gz,
> d44b7eda9e4c_python_client_logs.log.bz2
>
> Time Spent: 2h 10m
> Remaining Estimate: 0h
>
> Data channel and logging channel are failing after some time with 1GB input
> data for chicago taxi.
>
> E1218 02:44:02.837680206 72 chttp2_transport.cc:1148] Received a GOAWAY with
> error code ENHANCE_YOUR_CALM and debug data equal to "too_many_pings"
> Exception in thread read_grpc_client_inputs:
> Traceback (most recent call last):
> File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner
> self.run()
> File "/usr/local/lib/python2.7/threading.py", line 754, in run
> self.__target(*self.__args, **self.__kwargs)
> File
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py",
> line 273, in <lambda>
> target=lambda: self._read_inputs(elements_iterator),
> File
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py",
> line 260, in _read_inputs
> for elements in elements_iterator:
> File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 347, in
> next
> return self._next()
> File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 338, in
> _next
> raise self
> _Rendezvous: <_Rendezvous of RPC that terminated with
> (StatusCode.RESOURCE_EXHAUSTED, GOAWAY received)>
> Traceback (most recent call last):
> File
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
> line 145, in _execute
> response = task()
> File
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
> line 180, in <lambda>
> self._execute(lambda: worker.do_instruction(work), work)
> File
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
> line 253, in do_instruction
> request.instruction_id)
> File
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
> line 269, in process_bundle
> bundle_processor.process_bundle(instruction_id)
> File
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py",
> line 481, in process_bundle
> instruction_id, expected_targets):
> File
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py",
> line 209, in input_elements
> raise_(t, v, tb)
> File
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py",
> line 260, in _read_inputs
> for elements in elements_iterator:
> File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 347, in
> next
> return self._next()
> File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 338, in
> _next
> raise self
> _Rendezvous: <_Rendezvous of RPC that terminated with
> (StatusCode.RESOURCE_EXHAUSTED, GOAWAY received)>
> Traceback (most recent call last):
> File
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
> line 145, in _execute
> response = task()
> File
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
> line 180, in <lambda>
> self._execute(lambda: worker.do_instruction(work), work)
> File
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
> line 253, in do_instruction
> request.instruction_id)
> File
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
> line 269, in process_bundle
> bundle_processor.process_bundle(instruction_id)
> File
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py",
> line 481, in process_bundle
> instruction_id, expected_targets):
> File
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py",
> line 209, in input_elements
> raise_(t, v, tb)
> File
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py",
> line 260, in _read_inputs
> for elements in elements_iterator:
> File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 347, in
> next
> return self._next()
> File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 338, in
> _next
> raise self
> _Rendezvous: <_Rendezvous of RPC that terminated with
> (StatusCode.RESOURCE_EXHAUSTED, GOAWAY received)>
> Traceback (most recent call last):
> File
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
> line 145, in _execute
> response = task()
> File
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
> line 180, in <lambda>
> self._execute(lambda: worker.do_instruction(work), work)
> File
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
> line 253, in do_instruction
> request.instruction_id)
> File
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
> line 269, in process_bundle
> bundle_processor.process_bundle(instruction_id)
> File
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py",
> line 481, in process_bundle
> instruction_id, expected_targets):
> File
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py",
> line 209, in input_elements
> raise_(t, v, tb)
> File
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py",
> line 260, in _read_inputs
> for elements in elements_iterator:
> File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 347, in
> next
> return self._next()
> File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 338, in
> _next
> raise self
> _Rendezvous: <_Rendezvous of RPC that terminated with
> (StatusCode.RESOURCE_EXHAUSTED, GOAWAY received)>
> Traceback (most recent call last):
> File
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
> line 145, in _execute
> response = task()
> File
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
> line 180, in <lambda>
> self._execute(lambda: worker.do_instruction(work), work)
> File
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
> line 253, in do_instruction
> request.instruction_id)
> File
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
> line 269, in process_bundle
> bundle_processor.process_bundle(instruction_id)
> File
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py",
> line 481, in process_bundle
> instruction_id, expected_targets):
> File
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py",
> line 209, in input_elements
> raise_(t, v, tb)
> File
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py",
> line 260, in _read_inputs
> for elements in elements_iterator:
> File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 347, in
> next
> return self._next()
> File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 338, in
> _next
> raise self
> _Rendezvous: <_Rendezvous of RPC that terminated with
> (StatusCode.RESOURCE_EXHAUSTED, GOAWAY received)>
--
This message was sent by Atlassian Jira
(v8.20.7#820007)
