This is an automated email from the ASF dual-hosted git repository.
nickva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
The following commit(s) were added to refs/heads/main by this push:
new d3884102b Fix heartbeat timing for continuous change feeds
d3884102b is described below
commit d3884102b5a836f022d7d6a730ab29305d6e098a
Author: Nick Vatamaniuc <[email protected]>
AuthorDate: Tue Oct 13 01:25:14 2026 -0400
Fix heartbeat timing for continuous change feeds
Previously, coordinator signaled a timeout after every batch of updates, not
only when its heartbeat timer fired, and the HTTP layer turned those into an
empty lines (or hearbeat event for eventsource).
Instead, we should remember when we sent the last chunk (row, or first
newline,
or another heartbeat) and send a heartbeat only if nothing else was sent.
The
interval is the same value the coordinator uses.
---
src/chttpd/src/chttpd_db.erl | 67 ++++++++++--------
src/chttpd/test/eunit/chttpd_changes_test.erl | 98 +++++++++++++++++++++++++++
2 files changed, 137 insertions(+), 28 deletions(-)
diff --git a/src/chttpd/src/chttpd_db.erl b/src/chttpd/src/chttpd_db.erl
index 9bafed01c..627f9e838 100644
--- a/src/chttpd/src/chttpd_db.erl
+++ b/src/chttpd/src/chttpd_db.erl
@@ -67,7 +67,9 @@
chunks_sent = 0,
buffer = [],
bufsize = 0,
- threshold
+ threshold,
+ heartbeat_dt,
+ heartbeat_ts
}).
-define(IS_ALL_DOCS(T),
@@ -149,10 +151,13 @@ handle_changes_req1(#httpd{} = Req, Db) ->
end);
Feed when Feed =:= "continuous"; Feed =:= "longpoll"; Feed =:=
"eventsource" ->
couch_stats:increment_counter([couchdb, httpd,
clients_requesting_changes]),
+ {HDt, _} = couch_changes:get_changes_timeout(ChangesArgs, fun
changes_callback/2),
Acc0 = #cacc{
feed = list_to_atom(Feed),
mochi = Req,
- threshold = Max
+ threshold = Max,
+ heartbeat_dt = HDt,
+ heartbeat_ts = now_msec()
},
try
fabric:changes(Db, fun changes_callback/2, Acc0, ChangesArgs)
@@ -206,10 +211,7 @@ changes_callback({change, {ChangeProp} = Change},
#cacc{feed = eventsource} = Ac
Len = iolist_size(Chunk),
maybe_flush_changes_feed(Acc, Chunk, Len);
changes_callback(timeout, #cacc{feed = eventsource} = Acc) ->
- #cacc{mochi = Resp, chunks_sent = ChunksSet} = Acc,
- Chunk = "event: heartbeat\ndata: \n\n",
- {ok, Resp1} = chttpd:send_delayed_chunk(Resp, Chunk),
- {ok, Acc#cacc{mochi = Resp1, chunks_sent = ChunksSet + 1}};
+ maybe_heartbeat(Acc, "event: heartbeat\ndata: \n\n");
changes_callback({stop, _EndSeq, _Pending}, #cacc{feed = eventsource} = Acc) ->
#cacc{mochi = Resp, buffer = Buf} = Acc,
{ok, Resp1} = chttpd:send_delayed_chunk(Resp, Buf),
@@ -246,28 +248,16 @@ changes_callback({stop, EndSeq, Pending}, Acc) ->
],
{ok, Resp1} = chttpd:close_delayed_json_object(Resp, Buf, Terminator, Max),
chttpd:end_delayed_json_response(Resp1);
+changes_callback(waiting_for_updates, #cacc{buffer = [], chunks_sent = 0} =
Acc) ->
+ % didn't send anything yet, send response headers
+ send_changes_chunk(Acc, ~"\n");
changes_callback(waiting_for_updates, #cacc{buffer = []} = Acc) ->
- #cacc{mochi = Resp, chunks_sent = ChunksSent} = Acc,
- case ChunksSent > 0 of
- true ->
- {ok, Acc};
- false ->
- {ok, Resp1} = chttpd:send_delayed_chunk(Resp, <<"\n">>),
- {ok, Acc#cacc{mochi = Resp1, chunks_sent = 1}}
- end;
+ {ok, Acc};
changes_callback(waiting_for_updates, Acc) ->
- #cacc{buffer = Buf, mochi = Resp, chunks_sent = ChunksSent} = Acc,
- {ok, Resp1} = chttpd:send_delayed_chunk(Resp, Buf),
- {ok, Acc#cacc{
- buffer = [],
- bufsize = 0,
- mochi = Resp1,
- chunks_sent = ChunksSent + 1
- }};
+ #cacc{buffer = Buf} = Acc,
+ send_changes_chunk(Acc#cacc{buffer = [], bufsize = 0}, Buf);
changes_callback(timeout, Acc) ->
- #cacc{mochi = Resp, chunks_sent = ChunksSent} = Acc,
- {ok, Resp1} = chttpd:send_delayed_chunk(Resp, "\n"),
- {ok, Acc#cacc{mochi = Resp1, chunks_sent = ChunksSent + 1}};
+ maybe_heartbeat(Acc, "\n");
changes_callback({error, Reason}, #cacc{mochi = #httpd{}} = Acc) ->
#cacc{mochi = Req} = Acc,
chttpd:send_error(Req, Reason);
@@ -280,9 +270,8 @@ changes_callback({error, Reason}, Acc) ->
maybe_flush_changes_feed(#cacc{bufsize = Size, threshold = Max} = Acc, Data,
Len) when
Size > 0 andalso (Size + Len) > Max
->
- #cacc{buffer = Buffer, mochi = Resp} = Acc,
- {ok, R1} = chttpd:send_delayed_chunk(Resp, Buffer),
- {ok, Acc#cacc{prepend = ",\r\n", buffer = Data, bufsize = Len, mochi =
R1}};
+ #cacc{buffer = Buffer} = Acc,
+ send_changes_chunk(Acc#cacc{prepend = ",\r\n", buffer = Data, bufsize =
Len}, Buffer);
maybe_flush_changes_feed(Acc0, Data, Len) ->
#cacc{buffer = Buf, bufsize = Size, chunks_sent = ChunksSent} = Acc0,
Acc = Acc0#cacc{
@@ -293,6 +282,25 @@ maybe_flush_changes_feed(Acc0, Data, Len) ->
},
{ok, Acc}.
+send_changes_chunk(#cacc{} = Acc, Data) ->
+ #cacc{mochi = Resp, chunks_sent = ChunksSent} = Acc,
+ {ok, Resp1} = chttpd:send_delayed_chunk(Resp, Data),
+ Acc1 = Acc#cacc{
+ mochi = Resp1,
+ chunks_sent = ChunksSent + 1,
+ heartbeat_ts = now_msec()
+ },
+ {ok, Acc1}.
+
+% Send a heartbeat if nothing else was sent for a whole interval
+maybe_heartbeat(#cacc{heartbeat_dt = undefined} = Acc, Data) ->
+ send_changes_chunk(Acc, Data);
+maybe_heartbeat(#cacc{heartbeat_dt = DtMSec, heartbeat_ts = Ts} = Acc, Data) ->
+ case now_msec() - Ts >= DtMSec of
+ true -> send_changes_chunk(Acc, Data);
+ false -> {ok, Acc}
+ end.
+
handle_compact_req(#httpd{method = 'POST'} = Req, Db) ->
chttpd:validate_ctype(Req, "application/json"),
case Req#httpd.path_parts of
@@ -2749,6 +2757,9 @@ bulk_get_rev_error(all) ->
bulk_get_rev_error([{Pos, RevId} = Rev]) when is_integer(Pos),
is_binary(RevId) ->
couch_doc:rev_to_str(Rev).
+now_msec() ->
+ erlang:monotonic_time(millisecond).
+
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
diff --git a/src/chttpd/test/eunit/chttpd_changes_test.erl
b/src/chttpd/test/eunit/chttpd_changes_test.erl
index 8c70c3288..0da5bb7dd 100644
--- a/src/chttpd/test/eunit/chttpd_changes_test.erl
+++ b/src/chttpd/test/eunit/chttpd_changes_test.erl
@@ -93,6 +93,18 @@ changes_test_() ->
% so we'll test all the changes return and that the sequences
% are increasing.
%
+heartbeat_test_() ->
+ {
+ foreach,
+ fun setup_basic/0,
+ fun teardown_basic/1,
+ [
+ ?TDEF_FE(t_continuous_heartbeat_idle),
+ ?TDEF_FE(t_continuous_heartbeat),
+ ?TDEF_FE(t_eventsource_heartbeat)
+ ]
+ }.
+
changes_q8_test_() ->
{
setup,
@@ -298,6 +310,54 @@ t_continuous_limit_over_total({_, DbUrl}) ->
DtMSec = erlang:monotonic_time(millisecond) - T0,
?assert(DtMSec > 500).
+t_continuous_heartbeat_idle({_, DbUrl}) ->
+ % Heartbeats sent every interval if nothing else happens
+ Params = "?feed=continuous&heartbeat=10",
+ Body = with_changes_dur(100, fun() -> changes_raw(DbUrl, Params) end),
+ ?assert(heartbeats(Body) >= 3),
+ {Seq, Pending, Rows} = res(Body, Params),
+ ?assertEqual(8, Seq),
+ ?assertEqual(0, Pending),
+ ?assertEqual(3, length(Rows)).
+
+t_continuous_heartbeat({_, DbUrl}) ->
+ % Coordinator could wake up on every update, but heartbeats should still be
+ % sent only once per interval.
+ Params = "?feed=continuous&filter=_selector&heartbeat=100",
+ Selector = #{<<"selector">> => #{<<"_id">> => <<"nomatch">>}},
+ Self = self(),
+ Writer = spawn_link(fun() -> Self ! {self(), write_docs_for(DbUrl, 300)}
end),
+ Body = with_changes_dur(300, fun() -> changes_raw_post(DbUrl, Params,
Selector) end),
+ Written =
+ receive
+ {Writer, N} -> N
+ after 5000 -> error(writer_timeout)
+ end,
+ ?assert(Written >= 3),
+ Heartbeats = heartbeats(Body),
+ ?assert(Heartbeats >= 1),
+ ?assert(Heartbeats =< 6),
+ {_, _, Rows} = res(Body, Params),
+ ?assertEqual([], Rows).
+
+t_eventsource_heartbeat({_, DbUrl}) ->
+ % Like for continuous heartbeat test but for eventsource
+ Params = "?feed=eventsource&filter=_selector&heartbeat=100",
+ Selector = #{<<"selector">> => #{<<"_id">> => <<"nomatch">>}},
+ Self = self(),
+ Writer = spawn_link(fun() -> Self ! {self(), write_docs_for(DbUrl, 300)}
end),
+ Body = with_changes_dur(300, fun() -> changes_raw_post(DbUrl, Params,
Selector) end),
+ Written =
+ receive
+ {Writer, N} -> N
+ after 5000 -> error(writer_timeout)
+ end,
+ ?assert(Written >= 3),
+ Heartbeats = length(binary:matches(Body, <<"event: heartbeat">>)),
+ ?assert(Heartbeats >= 1),
+ ?assert(Heartbeats =< 6),
+ ?assertEqual(0, length(binary:matches(Body, <<"data: {">>))).
+
t_limit_one({_, DbUrl}) ->
Params = "?limit=1",
?assertEqual(
@@ -1208,6 +1268,44 @@ seq(<<_/binary>> = Seq) ->
seq(null) ->
null.
+with_changes_dur(Msec, Fun) ->
+ config:set("fabric", "changes_duration", integer_to_list(Msec), false),
+ try
+ Fun()
+ after
+ config:delete("fabric", "changes_duration", false)
+ end.
+
+changes_raw(DbUrl, Params) ->
+ {Code, Res} = reqraw(get, DbUrl ++ "/_changes" ++ Params),
+ ?assertEqual(200, Code),
+ Res.
+
+changes_raw_post(DbUrl, Params, #{} = Body) ->
+ {Code, Res} = reqraw(post, DbUrl ++ "/_changes" ++ Params, Body),
+ ?assertEqual(200, Code),
+ Res.
+
+% heartbeats are empty lines. Request ends with a newline so we subtract one.
+heartbeats(Body) ->
+ Lines = binary:split(Body, ~"\n", [global]),
+ length([L || L <- Lines, L =:= <<>>]) - 1.
+
+% Create docs for the given number of msec. Return how many were created.
+write_docs_for(DbUrl, MSec) ->
+ AbsMSec = erlang:monotonic_time(millisecond) + MSec,
+ write_docs_until(DbUrl, AbsMSec, 0).
+
+write_docs_until(DbUrl, AbsMSec, N) ->
+ case erlang:monotonic_time(millisecond) < AbsMSec of
+ true ->
+ Url = DbUrl ++ "/writer" ++ integer_to_list(N),
+ {201, #{~"ok" := true}} = req(put, Url, #{}),
+ write_docs_until(DbUrl, AbsMSec, N + 1);
+ false ->
+ N
+ end.
+
called_times(ReqFun, DbUrl) ->
meck:reset(couch_db),
ReqFun(),