[
https://issues.apache.org/jira/browse/BEAM-7476?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Dmytro Sadovnychyi updated BEAM-7476:
-------------------------------------
Description:
We are getting lots of Broken pipe errors and it's only a matter of luck for
write to succeed. It's been happening for months.
Partial stack trace:
File
"/usr/local/lib/python2.7/dist-packages/apache_beam/io/gcp/datastore/v1/helper.py",
line 225, in commit
response = datastore.commit(request)
File "/usr/local/lib/python2.7/dist-packages/googledatastore/connection.py",
line 140, in commit
datastore_pb2.CommitResponse)
File "/usr/local/lib/python2.7/dist-packages/googledatastore/connection.py",
line 199, in _call_method
method='POST', body=payload, headers=headers)
File "/usr/local/lib/python2.7/dist-packages/oauth2client/transport.py", line
169, in new_request
redirections, connection_type)
File "/usr/local/lib/python2.7/dist-packages/httplib2/__init__.py", line
1609, in request
(response, content) = self._request(conn, authority, uri, request_uri,
method, body, headers, redirections, cachekey)
File "/usr/local/lib/python2.7/dist-packages/httplib2/__init__.py", line
1351, in _request
(response, content) = self._conn_request(conn, request_uri, method, body,
headers)
File "/usr/local/lib/python2.7/dist-packages/httplib2/__init__.py", line
1273, in _conn_request
conn.request(method, request_uri, body, headers)
File "/usr/lib/python2.7/httplib.py", line 1042, in request
self._send_request(method, url, body, headers)
File "/usr/lib/python2.7/httplib.py", line 1082, in _send_request
self.endheaders(body)
File "/usr/lib/python2.7/httplib.py", line 1038, in endheaders
self._send_output(message_body)
File "/usr/lib/python2.7/httplib.py", line 882, in _send_output
self.send(msg)
File "/usr/lib/python2.7/httplib.py", line 858, in send
self.sock.sendall(data)
File "/usr/lib/python2.7/ssl.py", line 753, in sendall
v = self.send(data[count:])
File "/usr/lib/python2.7/ssl.py", line 719, in send
v = self._sslobj.write(data)
RuntimeError: error: [Errno 32] Broken pipe [while running 'Groups to
datastore/Write Mutation to Datastore']
Workaround: https://github.com/apache/beam/pull/8346
was:
We are getting lots of Broken pipe errors and it's only a matter of luck for
write to succeed. It's been happening for months.
Partial stack trace:
{{File
"/usr/local/lib/python2.7/dist-packages/apache_beam/io/gcp/datastore/v1/helper.py",
line 225, in commit
response = datastore.commit(request)
File "/usr/local/lib/python2.7/dist-packages/googledatastore/connection.py",
line 140, in commit
datastore_pb2.CommitResponse)
File "/usr/local/lib/python2.7/dist-packages/googledatastore/connection.py",
line 199, in _call_method
method='POST', body=payload, headers=headers)
File "/usr/local/lib/python2.7/dist-packages/oauth2client/transport.py", line
169, in new_request
redirections, connection_type)
File "/usr/local/lib/python2.7/dist-packages/httplib2/__init__.py", line
1609, in request
(response, content) = self._request(conn, authority, uri, request_uri,
method, body, headers, redirections, cachekey)
File "/usr/local/lib/python2.7/dist-packages/httplib2/__init__.py", line
1351, in _request
(response, content) = self._conn_request(conn, request_uri, method, body,
headers)
File "/usr/local/lib/python2.7/dist-packages/httplib2/__init__.py", line
1273, in _conn_request
conn.request(method, request_uri, body, headers)
File "/usr/lib/python2.7/httplib.py", line 1042, in request
self._send_request(method, url, body, headers)
File "/usr/lib/python2.7/httplib.py", line 1082, in _send_request
self.endheaders(body)
File "/usr/lib/python2.7/httplib.py", line 1038, in endheaders
self._send_output(message_body)
File "/usr/lib/python2.7/httplib.py", line 882, in _send_output
self.send(msg)
File "/usr/lib/python2.7/httplib.py", line 858, in send
self.sock.sendall(data)
File "/usr/lib/python2.7/ssl.py", line 753, in sendall
v = self.send(data[count:])
File "/usr/lib/python2.7/ssl.py", line 719, in send
v = self._sslobj.write(data)
RuntimeError: error: [Errno 32] Broken pipe [while running 'Groups to
datastore/Write Mutation to Datastore']}}
Workaround: https://github.com/apache/beam/pull/8346
> Datastore write failures with "[Errno 32] Broken pipe"
> ------------------------------------------------------
>
> Key: BEAM-7476
> URL: https://issues.apache.org/jira/browse/BEAM-7476
> Project: Beam
> Issue Type: Bug
> Components: io-python-gcp
> Affects Versions: 2.11.0
> Environment: dataflow python 2.7
> Reporter: Dmytro Sadovnychyi
> Priority: Minor
> Labels: pull-request-available
>
> We are getting lots of Broken pipe errors and it's only a matter of luck for
> write to succeed. It's been happening for months.
> Partial stack trace:
> File
> "/usr/local/lib/python2.7/dist-packages/apache_beam/io/gcp/datastore/v1/helper.py",
> line 225, in commit
> response = datastore.commit(request)
> File
> "/usr/local/lib/python2.7/dist-packages/googledatastore/connection.py", line
> 140, in commit
> datastore_pb2.CommitResponse)
> File
> "/usr/local/lib/python2.7/dist-packages/googledatastore/connection.py", line
> 199, in _call_method
> method='POST', body=payload, headers=headers)
> File "/usr/local/lib/python2.7/dist-packages/oauth2client/transport.py",
> line 169, in new_request
> redirections, connection_type)
> File "/usr/local/lib/python2.7/dist-packages/httplib2/__init__.py", line
> 1609, in request
> (response, content) = self._request(conn, authority, uri, request_uri,
> method, body, headers, redirections, cachekey)
> File "/usr/local/lib/python2.7/dist-packages/httplib2/__init__.py", line
> 1351, in _request
> (response, content) = self._conn_request(conn, request_uri, method, body,
> headers)
> File "/usr/local/lib/python2.7/dist-packages/httplib2/__init__.py", line
> 1273, in _conn_request
> conn.request(method, request_uri, body, headers)
> File "/usr/lib/python2.7/httplib.py", line 1042, in request
> self._send_request(method, url, body, headers)
> File "/usr/lib/python2.7/httplib.py", line 1082, in _send_request
> self.endheaders(body)
> File "/usr/lib/python2.7/httplib.py", line 1038, in endheaders
> self._send_output(message_body)
> File "/usr/lib/python2.7/httplib.py", line 882, in _send_output
> self.send(msg)
> File "/usr/lib/python2.7/httplib.py", line 858, in send
> self.sock.sendall(data)
> File "/usr/lib/python2.7/ssl.py", line 753, in sendall
> v = self.send(data[count:])
> File "/usr/lib/python2.7/ssl.py", line 719, in send
> v = self._sslobj.write(data)
> RuntimeError: error: [Errno 32] Broken pipe [while running 'Groups to
> datastore/Write Mutation to Datastore']
> Workaround: https://github.com/apache/beam/pull/8346
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
