TS-3087: Fix flow control stall on low water marks.
Project: http://git-wip-us.apache.org/repos/asf/trafficserver/repo Commit: http://git-wip-us.apache.org/repos/asf/trafficserver/commit/4a143e58 Tree: http://git-wip-us.apache.org/repos/asf/trafficserver/tree/4a143e58 Diff: http://git-wip-us.apache.org/repos/asf/trafficserver/diff/4a143e58 Branch: refs/heads/master Commit: 4a143e584a000d0d591ef323e0789ae80de34b1a Parents: 4b48b2c Author: Alan M. Carroll <[email protected]> Authored: Fri Sep 19 08:28:57 2014 -0500 Committer: Alan M. Carroll <[email protected]> Committed: Fri Sep 19 08:28:57 2014 -0500 ---------------------------------------------------------------------- iocore/net/I_NetVConnection.h | 22 +++++++++++++++++++++- iocore/net/P_UnixNetVConnection.h | 2 ++ iocore/net/UnixNetVConnection.cc | 12 ++++++++++++ proxy/http/HttpTunnel.cc | 9 +++++++++ 4 files changed, 44 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/trafficserver/blob/4a143e58/iocore/net/I_NetVConnection.h ---------------------------------------------------------------------- diff --git a/iocore/net/I_NetVConnection.h b/iocore/net/I_NetVConnection.h index cf97c64..0927144 100644 --- a/iocore/net/I_NetVConnection.h +++ b/iocore/net/I_NetVConnection.h @@ -434,6 +434,17 @@ public: /** @return current inactivity_timeout value in nanosecs */ virtual ink_hrtime get_inactivity_timeout() = 0; + /** Force an @a event if a write operation empties the write buffer. + + This event will be sent to the VIO, the same place as other IO events. + Use an @a event value of 0 to cancel the trap. + + The event is sent only the next time the write buffer is emptied, not + every future time. The event is sent only if otherwise no event would + be generated. + */ + virtual void trapWriteBufferEmpty(int event = VC_EVENT_WRITE_READY); + /** Returns local sockaddr storage. */ sockaddr const* get_local_addr(); @@ -535,6 +546,8 @@ protected: bool is_internal_request; /// Set if this connection is transparent. bool is_transparent; + /// Set if the next write IO that empties the write buffer should generate an event. + int write_buffer_empty_event; }; inline @@ -545,10 +558,17 @@ NetVConnection::NetVConnection(): got_local_addr(0), got_remote_addr(0), is_internal_request(false), - is_transparent(false) + is_transparent(false), + write_buffer_empty_event(0) { ink_zero(local_addr); ink_zero(remote_addr); } +inline void +NetVConnection::trapWriteBufferEmpty(int event) +{ + write_buffer_empty_event = event; +} + #endif http://git-wip-us.apache.org/repos/asf/trafficserver/blob/4a143e58/iocore/net/P_UnixNetVConnection.h ---------------------------------------------------------------------- diff --git a/iocore/net/P_UnixNetVConnection.h b/iocore/net/P_UnixNetVConnection.h index 7fb597d..4690556 100644 --- a/iocore/net/P_UnixNetVConnection.h +++ b/iocore/net/P_UnixNetVConnection.h @@ -249,6 +249,8 @@ public: virtual void set_remote_addr(); virtual int set_tcp_init_cwnd(int init_cwnd); virtual void apply_options(); + + friend void write_to_net_io(NetHandler*, UnixNetVConnection*, EThread*); }; extern ClassAllocator<UnixNetVConnection> netVCAllocator; http://git-wip-us.apache.org/repos/asf/trafficserver/blob/4a143e58/iocore/net/UnixNetVConnection.cc ---------------------------------------------------------------------- diff --git a/iocore/net/UnixNetVConnection.cc b/iocore/net/UnixNetVConnection.cc index cf1d9c8..ec8605c 100644 --- a/iocore/net/UnixNetVConnection.cc +++ b/iocore/net/UnixNetVConnection.cc @@ -474,6 +474,8 @@ write_to_net_io(NetHandler *nh, UnixNetVConnection *vc, EThread *thread) write_signal_error(nh, vc, (int)-r); return; } else { + int wbe_event = vc->write_buffer_empty_event; // save so we can clear if needed. + NET_SUM_DYN_STAT(net_write_bytes_stat, r); // Remove data from the buffer and signal continuation. @@ -482,12 +484,22 @@ write_to_net_io(NetHandler *nh, UnixNetVConnection *vc, EThread *thread) ink_assert(buf.reader()->read_avail() >= 0); s->vio.ndone += r; + // If the empty write buffer trap is set, clear it. + if (!(buf.reader()->is_read_avail_more_than(0))) + vc->write_buffer_empty_event = 0; + net_activity(vc, thread); // If there are no more bytes to write, signal write complete, ink_assert(ntodo >= 0); if (s->vio.ntodo() <= 0) { write_signal_done(VC_EVENT_WRITE_COMPLETE, nh, vc); return; + } else if (signalled && (wbe_event != vc->write_buffer_empty_event)) { + // @a signalled means we won't send an event, and the event values differing means we + // had a write buffer trap and cleared it, so we need to send it now. + Debug("amc", "empty write buffer trap [%d]", wbe_event); + if (write_signal_and_update(wbe_event, vc) != EVENT_CONT) + return; } else if (!signalled) { if (write_signal_and_update(VC_EVENT_WRITE_READY, vc) != EVENT_CONT) { return; http://git-wip-us.apache.org/repos/asf/trafficserver/blob/4a143e58/proxy/http/HttpTunnel.cc ---------------------------------------------------------------------- diff --git a/proxy/http/HttpTunnel.cc b/proxy/http/HttpTunnel.cc index ef0da2c..d71f6c8 100644 --- a/proxy/http/HttpTunnel.cc +++ b/proxy/http/HttpTunnel.cc @@ -1241,6 +1241,15 @@ HttpTunnel::consumer_reenable(HttpTunnelConsumer* c) srcp->read_vio->reenable(); // Kick source producer to get flow ... well, flowing. this->producer_handler(VC_EVENT_READ_READY, srcp); + } else { + // We can stall for small thresholds on network sinks because this event happens + // before the actual socket write. So we trap for the buffer becoming empty to + // make sure we get an event to unthrottle after the write. + if (HT_HTTP_CLIENT == c->vc_type) { + NetVConnection* netvc = dynamic_cast<NetVConnection*>(c->write_vio->vc_server); + if (netvc) // really, this should always be true. + netvc->trapWriteBufferEmpty(); + } } } p->read_vio->reenable();
