Ankur Goenka created BEAM-6258:
----------------------------------
Summary: Data channel failing after some time for 1G data input
Key: BEAM-6258
URL: https://issues.apache.org/jira/browse/BEAM-6258
Project: Beam
Issue Type: Bug
Components: sdk-py-harness
Reporter: Ankur Goenka
Assignee: Ankur Goenka
Data channel and logging channel are failing after some time with 1GB input
data for chicago taxi.
E1218 02:44:02.837680206 72 chttp2_transport.cc:1148] Received a GOAWAY with
error code ENHANCE_YOUR_CALM and debug data equal to "too_many_pings"
Exception in thread read_grpc_client_inputs:
Traceback (most recent call last):
File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner
self.run()
File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs)
File
"/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py",
line 273, in <lambda>
target=lambda: self._read_inputs(elements_iterator),
File
"/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py",
line 260, in _read_inputs
for elements in elements_iterator:
File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 347, in
next
return self._next()
File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 338, in
_next
raise self
_Rendezvous: <_Rendezvous of RPC that terminated with
(StatusCode.RESOURCE_EXHAUSTED, GOAWAY received)>
Traceback (most recent call last):
File
"/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 145, in _execute
response = task()
File
"/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 180, in <lambda>
self._execute(lambda: worker.do_instruction(work), work)
File
"/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 253, in do_instruction
request.instruction_id)
File
"/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 269, in process_bundle
bundle_processor.process_bundle(instruction_id)
File
"/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py",
line 481, in process_bundle
instruction_id, expected_targets):
File
"/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py",
line 209, in input_elements
raise_(t, v, tb)
File
"/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py",
line 260, in _read_inputs
for elements in elements_iterator:
File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 347, in
next
return self._next()
File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 338, in
_next
raise self
_Rendezvous: <_Rendezvous of RPC that terminated with
(StatusCode.RESOURCE_EXHAUSTED, GOAWAY received)>
Traceback (most recent call last):
File
"/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 145, in _execute
response = task()
File
"/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 180, in <lambda>
self._execute(lambda: worker.do_instruction(work), work)
File
"/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 253, in do_instruction
request.instruction_id)
File
"/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 269, in process_bundle
bundle_processor.process_bundle(instruction_id)
File
"/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py",
line 481, in process_bundle
instruction_id, expected_targets):
File
"/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py",
line 209, in input_elements
raise_(t, v, tb)
File
"/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py",
line 260, in _read_inputs
for elements in elements_iterator:
File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 347, in
next
return self._next()
File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 338, in
_next
raise self
_Rendezvous: <_Rendezvous of RPC that terminated with
(StatusCode.RESOURCE_EXHAUSTED, GOAWAY received)>
Traceback (most recent call last):
File
"/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 145, in _execute
response = task()
File
"/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 180, in <lambda>
self._execute(lambda: worker.do_instruction(work), work)
File
"/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 253, in do_instruction
request.instruction_id)
File
"/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 269, in process_bundle
bundle_processor.process_bundle(instruction_id)
File
"/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py",
line 481, in process_bundle
instruction_id, expected_targets):
File
"/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py",
line 209, in input_elements
raise_(t, v, tb)
File
"/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py",
line 260, in _read_inputs
for elements in elements_iterator:
File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 347, in
next
return self._next()
File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 338, in
_next
raise self
_Rendezvous: <_Rendezvous of RPC that terminated with
(StatusCode.RESOURCE_EXHAUSTED, GOAWAY received)>
Traceback (most recent call last):
File
"/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 145, in _execute
response = task()
File
"/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 180, in <lambda>
self._execute(lambda: worker.do_instruction(work), work)
File
"/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 253, in do_instruction
request.instruction_id)
File
"/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 269, in process_bundle
bundle_processor.process_bundle(instruction_id)
File
"/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py",
line 481, in process_bundle
instruction_id, expected_targets):
File
"/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py",
line 209, in input_elements
raise_(t, v, tb)
File
"/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py",
line 260, in _read_inputs
for elements in elements_iterator:
File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 347, in
next
return self._next()
File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 338, in
_next
raise self
_Rendezvous: <_Rendezvous of RPC that terminated with
(StatusCode.RESOURCE_EXHAUSTED, GOAWAY received)>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
