Greg Padgett has uploaded a new change for review. Change subject: WIP lib: add socket timeout on broker I/O ......................................................................
WIP lib: add socket timeout on broker I/O This prevents vdsm from hanging when the NFS storage I/O hangs. It needs additional consideration before merging, as there may be better places to time out. Change-Id: I889fd3972449e0c6f0dbe831e5810e7962cbded7 Bug-Url: https://bugzilla.redhat.com/?????? Signed-off-by: Greg Padgett <[email protected]> --- M ovirt_hosted_engine_ha/env/constants.py.in M ovirt_hosted_engine_ha/lib/brokerlink.py M ovirt_hosted_engine_ha/lib/util.py 3 files changed, 10 insertions(+), 1 deletion(-) git pull ssh://gerrit.ovirt.org:29418/ovirt-hosted-engine-ha refs/changes/80/26580/1 diff --git a/ovirt_hosted_engine_ha/env/constants.py.in b/ovirt_hosted_engine_ha/env/constants.py.in index 3fefd40..730ec25 100644 --- a/ovirt_hosted_engine_ha/env/constants.py.in +++ b/ovirt_hosted_engine_ha/env/constants.py.in @@ -31,6 +31,7 @@ # See http://www.gnu.org/software/automake/manual/html_node/Scripts.html BROKER_SOCKET_FILE = '@ENGINE_HA_RUNDIR@/broker.socket' +BROKER_SOCKET_TIMEOUT_SECS = 5 ENGINE_SETUP_CONF_FILE = '/etc/ovirt-hosted-engine/hosted-engine.conf' VM_CONF_FILE = '/etc/ovirt-hosted-engine/vm.conf' diff --git a/ovirt_hosted_engine_ha/lib/brokerlink.py b/ovirt_hosted_engine_ha/lib/brokerlink.py index d97b46c..b52ed40 100644 --- a/ovirt_hosted_engine_ha/lib/brokerlink.py +++ b/ovirt_hosted_engine_ha/lib/brokerlink.py @@ -57,12 +57,13 @@ self._socket.close() self._socket = None raise + self._socket.settimeout(constants.BROKER_SOCKET_TIMEOUT_SECS) attempt = 0 while True: try: self._socket.connect(constants.BROKER_SOCKET_FILE) - except socket.error as e: + except (socket.error, socket.timeout) as e: if attempt < retries: self._log.info("Failed to connect to broker: %s", str(e)) self._log.info("Retrying broker connection...") @@ -244,5 +245,10 @@ self._log.error("Connection closed") self.disconnect() raise + except (socket.error, socket.timeout) as e: + self._log.error("Error communicating with broker: %s", str(e)) + self.disconnect() + raise + self._log.debug("Full response: %s", response) return response diff --git a/ovirt_hosted_engine_ha/lib/util.py b/ovirt_hosted_engine_ha/lib/util.py index 08ec771..a4719fc 100644 --- a/ovirt_hosted_engine_ha/lib/util.py +++ b/ovirt_hosted_engine_ha/lib/util.py @@ -80,6 +80,8 @@ """ try: sock.sendall(data + "\n") + except socket.timeout: + raise except IOError as e: log.debug("Connection closed while writing to socket: %s", str(e)) raise DisconnectionError("Connection closed") -- To view, visit http://gerrit.ovirt.org/26580 To unsubscribe, visit http://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I889fd3972449e0c6f0dbe831e5810e7962cbded7 Gerrit-PatchSet: 1 Gerrit-Project: ovirt-hosted-engine-ha Gerrit-Branch: master Gerrit-Owner: Greg Padgett <[email protected]> _______________________________________________ Engine-patches mailing list [email protected] http://lists.ovirt.org/mailman/listinfo/engine-patches
