This is an automated email from the ASF dual-hosted git repository. nickva pushed a commit to branch changes-heartbeat-interval in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit ac4ac5bfa0f08f0e4d1bb09f1abb14becc2284fb Author: Nick Vatamaniuc <[email protected]> AuthorDate: Tue Oct 13 01:25:14 2026 -0400 Fix heartbeat timing for continuous change feeds Previously, coordinator signaled a timeout after every batch of updates, not only when its heartbeat timer fired, and the HTTP layer turned those into an empty lines (or hearbeat event for eventsource). Instead, we should remember when we sent the last chunk (row, or first newline, or another heartbeat) and send a heartbeat only if nothing else was sent. The interval is the same value the coordinator uses. --- src/chttpd/src/chttpd_db.erl | 67 ++++++++++-------- src/chttpd/test/eunit/chttpd_changes_test.erl | 98 +++++++++++++++++++++++++++ 2 files changed, 137 insertions(+), 28 deletions(-) diff --git a/src/chttpd/src/chttpd_db.erl b/src/chttpd/src/chttpd_db.erl index 9bafed01c..627f9e838 100644 --- a/src/chttpd/src/chttpd_db.erl +++ b/src/chttpd/src/chttpd_db.erl @@ -67,7 +67,9 @@ chunks_sent = 0, buffer = [], bufsize = 0, - threshold + threshold, + heartbeat_dt, + heartbeat_ts }). -define(IS_ALL_DOCS(T), @@ -149,10 +151,13 @@ handle_changes_req1(#httpd{} = Req, Db) -> end); Feed when Feed =:= "continuous"; Feed =:= "longpoll"; Feed =:= "eventsource" -> couch_stats:increment_counter([couchdb, httpd, clients_requesting_changes]), + {HDt, _} = couch_changes:get_changes_timeout(ChangesArgs, fun changes_callback/2), Acc0 = #cacc{ feed = list_to_atom(Feed), mochi = Req, - threshold = Max + threshold = Max, + heartbeat_dt = HDt, + heartbeat_ts = now_msec() }, try fabric:changes(Db, fun changes_callback/2, Acc0, ChangesArgs) @@ -206,10 +211,7 @@ changes_callback({change, {ChangeProp} = Change}, #cacc{feed = eventsource} = Ac Len = iolist_size(Chunk), maybe_flush_changes_feed(Acc, Chunk, Len); changes_callback(timeout, #cacc{feed = eventsource} = Acc) -> - #cacc{mochi = Resp, chunks_sent = ChunksSet} = Acc, - Chunk = "event: heartbeat\ndata: \n\n", - {ok, Resp1} = chttpd:send_delayed_chunk(Resp, Chunk), - {ok, Acc#cacc{mochi = Resp1, chunks_sent = ChunksSet + 1}}; + maybe_heartbeat(Acc, "event: heartbeat\ndata: \n\n"); changes_callback({stop, _EndSeq, _Pending}, #cacc{feed = eventsource} = Acc) -> #cacc{mochi = Resp, buffer = Buf} = Acc, {ok, Resp1} = chttpd:send_delayed_chunk(Resp, Buf), @@ -246,28 +248,16 @@ changes_callback({stop, EndSeq, Pending}, Acc) -> ], {ok, Resp1} = chttpd:close_delayed_json_object(Resp, Buf, Terminator, Max), chttpd:end_delayed_json_response(Resp1); +changes_callback(waiting_for_updates, #cacc{buffer = [], chunks_sent = 0} = Acc) -> + % didn't send anything yet, send response headers + send_changes_chunk(Acc, ~"\n"); changes_callback(waiting_for_updates, #cacc{buffer = []} = Acc) -> - #cacc{mochi = Resp, chunks_sent = ChunksSent} = Acc, - case ChunksSent > 0 of - true -> - {ok, Acc}; - false -> - {ok, Resp1} = chttpd:send_delayed_chunk(Resp, <<"\n">>), - {ok, Acc#cacc{mochi = Resp1, chunks_sent = 1}} - end; + {ok, Acc}; changes_callback(waiting_for_updates, Acc) -> - #cacc{buffer = Buf, mochi = Resp, chunks_sent = ChunksSent} = Acc, - {ok, Resp1} = chttpd:send_delayed_chunk(Resp, Buf), - {ok, Acc#cacc{ - buffer = [], - bufsize = 0, - mochi = Resp1, - chunks_sent = ChunksSent + 1 - }}; + #cacc{buffer = Buf} = Acc, + send_changes_chunk(Acc#cacc{buffer = [], bufsize = 0}, Buf); changes_callback(timeout, Acc) -> - #cacc{mochi = Resp, chunks_sent = ChunksSent} = Acc, - {ok, Resp1} = chttpd:send_delayed_chunk(Resp, "\n"), - {ok, Acc#cacc{mochi = Resp1, chunks_sent = ChunksSent + 1}}; + maybe_heartbeat(Acc, "\n"); changes_callback({error, Reason}, #cacc{mochi = #httpd{}} = Acc) -> #cacc{mochi = Req} = Acc, chttpd:send_error(Req, Reason); @@ -280,9 +270,8 @@ changes_callback({error, Reason}, Acc) -> maybe_flush_changes_feed(#cacc{bufsize = Size, threshold = Max} = Acc, Data, Len) when Size > 0 andalso (Size + Len) > Max -> - #cacc{buffer = Buffer, mochi = Resp} = Acc, - {ok, R1} = chttpd:send_delayed_chunk(Resp, Buffer), - {ok, Acc#cacc{prepend = ",\r\n", buffer = Data, bufsize = Len, mochi = R1}}; + #cacc{buffer = Buffer} = Acc, + send_changes_chunk(Acc#cacc{prepend = ",\r\n", buffer = Data, bufsize = Len}, Buffer); maybe_flush_changes_feed(Acc0, Data, Len) -> #cacc{buffer = Buf, bufsize = Size, chunks_sent = ChunksSent} = Acc0, Acc = Acc0#cacc{ @@ -293,6 +282,25 @@ maybe_flush_changes_feed(Acc0, Data, Len) -> }, {ok, Acc}. +send_changes_chunk(#cacc{} = Acc, Data) -> + #cacc{mochi = Resp, chunks_sent = ChunksSent} = Acc, + {ok, Resp1} = chttpd:send_delayed_chunk(Resp, Data), + Acc1 = Acc#cacc{ + mochi = Resp1, + chunks_sent = ChunksSent + 1, + heartbeat_ts = now_msec() + }, + {ok, Acc1}. + +% Send a heartbeat if nothing else was sent for a whole interval +maybe_heartbeat(#cacc{heartbeat_dt = undefined} = Acc, Data) -> + send_changes_chunk(Acc, Data); +maybe_heartbeat(#cacc{heartbeat_dt = DtMSec, heartbeat_ts = Ts} = Acc, Data) -> + case now_msec() - Ts >= DtMSec of + true -> send_changes_chunk(Acc, Data); + false -> {ok, Acc} + end. + handle_compact_req(#httpd{method = 'POST'} = Req, Db) -> chttpd:validate_ctype(Req, "application/json"), case Req#httpd.path_parts of @@ -2749,6 +2757,9 @@ bulk_get_rev_error(all) -> bulk_get_rev_error([{Pos, RevId} = Rev]) when is_integer(Pos), is_binary(RevId) -> couch_doc:rev_to_str(Rev). +now_msec() -> + erlang:monotonic_time(millisecond). + -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). diff --git a/src/chttpd/test/eunit/chttpd_changes_test.erl b/src/chttpd/test/eunit/chttpd_changes_test.erl index 8c70c3288..0da5bb7dd 100644 --- a/src/chttpd/test/eunit/chttpd_changes_test.erl +++ b/src/chttpd/test/eunit/chttpd_changes_test.erl @@ -93,6 +93,18 @@ changes_test_() -> % so we'll test all the changes return and that the sequences % are increasing. % +heartbeat_test_() -> + { + foreach, + fun setup_basic/0, + fun teardown_basic/1, + [ + ?TDEF_FE(t_continuous_heartbeat_idle), + ?TDEF_FE(t_continuous_heartbeat), + ?TDEF_FE(t_eventsource_heartbeat) + ] + }. + changes_q8_test_() -> { setup, @@ -298,6 +310,54 @@ t_continuous_limit_over_total({_, DbUrl}) -> DtMSec = erlang:monotonic_time(millisecond) - T0, ?assert(DtMSec > 500). +t_continuous_heartbeat_idle({_, DbUrl}) -> + % Heartbeats sent every interval if nothing else happens + Params = "?feed=continuous&heartbeat=10", + Body = with_changes_dur(100, fun() -> changes_raw(DbUrl, Params) end), + ?assert(heartbeats(Body) >= 3), + {Seq, Pending, Rows} = res(Body, Params), + ?assertEqual(8, Seq), + ?assertEqual(0, Pending), + ?assertEqual(3, length(Rows)). + +t_continuous_heartbeat({_, DbUrl}) -> + % Coordinator could wake up on every update, but heartbeats should still be + % sent only once per interval. + Params = "?feed=continuous&filter=_selector&heartbeat=100", + Selector = #{<<"selector">> => #{<<"_id">> => <<"nomatch">>}}, + Self = self(), + Writer = spawn_link(fun() -> Self ! {self(), write_docs_for(DbUrl, 300)} end), + Body = with_changes_dur(300, fun() -> changes_raw_post(DbUrl, Params, Selector) end), + Written = + receive + {Writer, N} -> N + after 5000 -> error(writer_timeout) + end, + ?assert(Written >= 3), + Heartbeats = heartbeats(Body), + ?assert(Heartbeats >= 1), + ?assert(Heartbeats =< 6), + {_, _, Rows} = res(Body, Params), + ?assertEqual([], Rows). + +t_eventsource_heartbeat({_, DbUrl}) -> + % Like for continuous heartbeat test but for eventsource + Params = "?feed=eventsource&filter=_selector&heartbeat=100", + Selector = #{<<"selector">> => #{<<"_id">> => <<"nomatch">>}}, + Self = self(), + Writer = spawn_link(fun() -> Self ! {self(), write_docs_for(DbUrl, 300)} end), + Body = with_changes_dur(300, fun() -> changes_raw_post(DbUrl, Params, Selector) end), + Written = + receive + {Writer, N} -> N + after 5000 -> error(writer_timeout) + end, + ?assert(Written >= 3), + Heartbeats = length(binary:matches(Body, <<"event: heartbeat">>)), + ?assert(Heartbeats >= 1), + ?assert(Heartbeats =< 6), + ?assertEqual(0, length(binary:matches(Body, <<"data: {">>))). + t_limit_one({_, DbUrl}) -> Params = "?limit=1", ?assertEqual( @@ -1208,6 +1268,44 @@ seq(<<_/binary>> = Seq) -> seq(null) -> null. +with_changes_dur(Msec, Fun) -> + config:set("fabric", "changes_duration", integer_to_list(Msec), false), + try + Fun() + after + config:delete("fabric", "changes_duration", false) + end. + +changes_raw(DbUrl, Params) -> + {Code, Res} = reqraw(get, DbUrl ++ "/_changes" ++ Params), + ?assertEqual(200, Code), + Res. + +changes_raw_post(DbUrl, Params, #{} = Body) -> + {Code, Res} = reqraw(post, DbUrl ++ "/_changes" ++ Params, Body), + ?assertEqual(200, Code), + Res. + +% heartbeats are empty lines. Request ends with a newline so we subtract one. +heartbeats(Body) -> + Lines = binary:split(Body, ~"\n", [global]), + length([L || L <- Lines, L =:= <<>>]) - 1. + +% Create docs for the given number of msec. Return how many were created. +write_docs_for(DbUrl, MSec) -> + AbsMSec = erlang:monotonic_time(millisecond) + MSec, + write_docs_until(DbUrl, AbsMSec, 0). + +write_docs_until(DbUrl, AbsMSec, N) -> + case erlang:monotonic_time(millisecond) < AbsMSec of + true -> + Url = DbUrl ++ "/writer" ++ integer_to_list(N), + {201, #{~"ok" := true}} = req(put, Url, #{}), + write_docs_until(DbUrl, AbsMSec, N + 1); + false -> + N + end. + called_times(ReqFun, DbUrl) -> meck:reset(couch_db), ReqFun(),
