This is an automated email from the ASF dual-hosted git repository. nickva pushed a commit to branch changes-heartbeat-seq in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 8eb818b4f35aaf842ca30452a229ab6aaf7ea946 Author: Nick Vatamaniuc <[email protected]> AuthorDate: Wed Oct 14 01:52:26 2026 -0400 heartbeat_seq _changes parameter With a selective filter a client may go a long while without getting a matching row with a sequence. Without getting a sequence a client cannot checkpoint. If the feed crashes or times out, a client may have to rewind all the way to the last received row. The worst case of this is when due to timeouts it becomes impossible to make any progress at all. To fix this, implement heartbeats as json object which have a sequence field. With the new `heartbeat_seq=true` parameter continuous, normal and longpoll feeds will send `{"seq": ..., "id": null, "changes": []}` heartbeat objects as a separate line or an element of `results` array. With `eventsource` feeds the heartbeat event gets an `id` field set to the sequence, so browsers can resume from it with `Last-Event-ID`. The seq heartbeat object is sent at most once per heartbeat interval if the sequence changed since the last one sent. A heartbeat value now also sets the keepalive period of a normal feed while it is being read. To make the new feature work the coordinator timeout callback now takes a sequence parameter (`{timeout, Seq}`) so that required updating other change feed users (chttpd_auth_cache, global_changes, fabric_bench, etc). But this shouldn't affect inter-node compatibility as we haven't changed the `#changes_args{}` record passed to workers. Note: this just adds the new option to the _changes endpoint. There will be a separate PR to enchance the replicator to use this feature to checkpoint replication jobs. Fixes #6120 --- src/chttpd/src/chttpd_auth_cache.erl | 2 +- src/chttpd/src/chttpd_db.erl | 110 +++++++-- src/chttpd/test/eunit/chttpd_changes_test.erl | 262 +++++++++++++++++++++ .../test/eunit/couch_replicator_test_helper.erl | 2 +- src/docs/src/api/database/changes.rst | 47 ++++ src/docs/src/replication/replicator.rst | 1 - src/fabric/src/fabric_bench.erl | 2 +- src/fabric/src/fabric_view_changes.erl | 23 +- src/fabric/test/eunit/fabric_changes_test.erl | 40 +++- src/global_changes/src/global_changes_httpd.erl | 4 +- .../test/eunit/mem3_reshard_changes_feed_test.erl | 2 +- test/elixir/test/changes_async_test.exs | 65 +++++ test/elixir/test/config/suite.elixir | 1 + 13 files changed, 528 insertions(+), 33 deletions(-) diff --git a/src/chttpd/src/chttpd_auth_cache.erl b/src/chttpd/src/chttpd_auth_cache.erl index c3eb0a50f..a7865cdd8 100644 --- a/src/chttpd/src/chttpd_auth_cache.erl +++ b/src/chttpd/src/chttpd_auth_cache.erl @@ -179,7 +179,7 @@ changes_callback({change, {Change}}, _) -> ets_lru:remove(?CACHE, UserName) end, {ok, couch_util:get_value(seq, Change)}; -changes_callback(timeout, Acc) -> +changes_callback({timeout, _Seq}, Acc) -> {ok, Acc}; changes_callback({error, _}, EndSeq) -> exit({seq, EndSeq}). diff --git a/src/chttpd/src/chttpd_db.erl b/src/chttpd/src/chttpd_db.erl index 627f9e838..2b8452b3f 100644 --- a/src/chttpd/src/chttpd_db.erl +++ b/src/chttpd/src/chttpd_db.erl @@ -69,7 +69,9 @@ bufsize = 0, threshold, heartbeat_dt, - heartbeat_ts + heartbeat_ts, + heartbeat_seq = false, + heartbeat_seq_sent }). -define(IS_ALL_DOCS(T), @@ -126,11 +128,13 @@ handle_changes_req(#httpd{path_parts = [_, <<"_changes">>]} = Req, _Db) -> send_method_not_allowed(Req, "GET,POST,HEAD"). handle_changes_req1(#httpd{} = Req, Db) -> - #changes_args{filter = Raw, style = Style} = Args0 = parse_changes_query(Req), + {Args0, HeartbeatSeq} = parse_changes_query(Req), + #changes_args{filter = Raw, style = Style} = Args0, ChangesArgs = Args0#changes_args{ filter_fun = couch_changes:configure_filter(Raw, Style, Req, Db), db_open_options = [{user_ctx, couch_db:get_user_ctx(Db)}] }, + {HDt, _} = couch_changes:get_changes_timeout(ChangesArgs, fun changes_callback/2), Max = chttpd:chunked_response_buffer_size(), case ChangesArgs#changes_args.feed of "normal" -> @@ -145,19 +149,22 @@ handle_changes_req1(#httpd{} = Req, Db) -> feed = normal, etag = Etag, mochi = Req, - threshold = Max + threshold = Max, + heartbeat_dt = HDt, + heartbeat_seq = HeartbeatSeq, + heartbeat_seq_sent = {now_msec(), undefined} }, fabric:changes(Db, fun changes_callback/2, Acc0, ChangesArgs) end); Feed when Feed =:= "continuous"; Feed =:= "longpoll"; Feed =:= "eventsource" -> couch_stats:increment_counter([couchdb, httpd, clients_requesting_changes]), - {HDt, _} = couch_changes:get_changes_timeout(ChangesArgs, fun changes_callback/2), Acc0 = #cacc{ feed = list_to_atom(Feed), mochi = Req, threshold = Max, heartbeat_dt = HDt, - heartbeat_ts = now_msec() + heartbeat_ts = now_msec(), + heartbeat_seq = HeartbeatSeq }, try fabric:changes(Db, fun changes_callback/2, Acc0, ChangesArgs) @@ -178,6 +185,14 @@ changes_callback({change, Change}, #cacc{feed = continuous} = Acc) -> Data = [?JSON_ENCODE(Change) | "\n"], Len = iolist_size(Data), maybe_flush_changes_feed(Acc, Data, Len); +changes_callback({timeout, Seq}, #cacc{feed = continuous, heartbeat_seq = true} = Acc) -> + case heartbeat_seq_due(Seq, Acc) of + true -> + Data = [?JSON_ENCODE(heartbeat_json(Seq)) | "\n"], + send_heartbeat_seq(Seq, Acc, Data); + false -> + maybe_heartbeat(Acc, "\n") + end; changes_callback({stop, EndSeq, Pending}, #cacc{feed = continuous} = Acc) -> #cacc{mochi = Resp, buffer = Buf} = Acc, Row = @@ -210,7 +225,15 @@ changes_callback({change, {ChangeProp} = Change}, #cacc{feed = eventsource} = Ac ], Len = iolist_size(Chunk), maybe_flush_changes_feed(Acc, Chunk, Len); -changes_callback(timeout, #cacc{feed = eventsource} = Acc) -> +changes_callback({timeout, Seq}, #cacc{feed = eventsource, heartbeat_seq = true} = Acc) -> + case heartbeat_seq_due(Seq, Acc) of + true -> + Chunk = ["event: heartbeat\ndata: \nid: ", ?JSON_ENCODE(Seq), "\n\n"], + send_heartbeat_seq(Seq, Acc, Chunk); + false -> + maybe_heartbeat(Acc, "event: heartbeat\ndata: \n\n") + end; +changes_callback({timeout, _Seq}, #cacc{feed = eventsource} = Acc) -> maybe_heartbeat(Acc, "event: heartbeat\ndata: \n\n"); changes_callback({stop, _EndSeq, _Pending}, #cacc{feed = eventsource} = Acc) -> #cacc{mochi = Resp, buffer = Buf} = Acc, @@ -256,7 +279,15 @@ changes_callback(waiting_for_updates, #cacc{buffer = []} = Acc) -> changes_callback(waiting_for_updates, Acc) -> #cacc{buffer = Buf} = Acc, send_changes_chunk(Acc#cacc{buffer = [], bufsize = 0}, Buf); -changes_callback(timeout, Acc) -> +changes_callback({timeout, Seq}, #cacc{heartbeat_seq = true} = Acc) -> + case heartbeat_seq_due(Seq, Acc) of + true -> + Data = [Acc#cacc.prepend, ?JSON_ENCODE(heartbeat_json(Seq))], + send_heartbeat_seq(Seq, Acc#cacc{prepend = ",\r\n"}, Data); + false -> + maybe_heartbeat(Acc, "\n") + end; +changes_callback({timeout, _Seq}, Acc) -> maybe_heartbeat(Acc, "\n"); changes_callback({error, Reason}, #cacc{mochi = #httpd{}} = Acc) -> #cacc{mochi = Req} = Acc, @@ -293,7 +324,7 @@ send_changes_chunk(#cacc{} = Acc, Data) -> {ok, Acc1}. % Send a heartbeat if nothing else was sent for a whole interval -maybe_heartbeat(#cacc{heartbeat_dt = undefined} = Acc, Data) -> +maybe_heartbeat(#cacc{feed = normal} = Acc, Data) -> send_changes_chunk(Acc, Data); maybe_heartbeat(#cacc{heartbeat_dt = DtMSec, heartbeat_ts = Ts} = Acc, Data) -> case now_msec() - Ts >= DtMSec of @@ -301,6 +332,28 @@ maybe_heartbeat(#cacc{heartbeat_dt = DtMSec, heartbeat_ts = Ts} = Acc, Data) -> false -> {ok, Acc} end. +heartbeat_json(Seq) -> + {[ + {~"seq", Seq}, + {~"id", null}, + {~"changes", []} + ]}. + +% Heartbeats are sent _after_ any pending rows +send_heartbeat_seq(Seq, #cacc{buffer = Buf} = Acc, Data) -> + Acc1 = Acc#cacc{ + buffer = [], + bufsize = 0, + heartbeat_seq_sent = {now_msec(), Seq} + }, + send_changes_chunk(Acc1, [Buf, Data]). + +% Send heartbeat seq if it's due and different than the last one +heartbeat_seq_due(_Seq, #cacc{heartbeat_seq_sent = undefined}) -> + true; +heartbeat_seq_due(Seq, #cacc{heartbeat_dt = Dt, heartbeat_seq_sent = {T0, OldSeq}}) -> + Seq /= OldSeq andalso (now_msec() - T0) >= Dt. + handle_compact_req(#httpd{method = 'POST'} = Req, Db) -> chttpd:validate_ctype(Req, "application/json"), case Req#httpd.path_parts of @@ -2242,8 +2295,11 @@ parse_doc_query({Key, Value}, Args) -> Args end. +% Return {#changes_args{}, HeartbeatSeq}. HeartbeatSeq is not sent to the workers +% so we keep it in the coordinator only (it's formatting hint for heartbeats) parse_changes_query(Req) -> erlang:erase(changes_seq_interval), + QArgs = chttpd:qs(Req), ChangesArgs = lists:foldl( fun({Key, Value}, Args) -> case {string:to_lower(Key), Value} of @@ -2339,22 +2395,36 @@ parse_changes_query(Req) -> end end, #changes_args{}, - chttpd:qs(Req) + QArgs ), %% if it's an EventSource request with a Last-event-ID header %% that should override the `since` query string, since it's %% probably the browser reconnecting. - case ChangesArgs#changes_args.feed of - "eventsource" -> - case couch_httpd:header_value(Req, "last-event-id") of - undefined -> - ChangesArgs; - Value -> - ChangesArgs#changes_args{since = Value} - end; - _ -> - ChangesArgs - end. + ChangesArgs1 = + case ChangesArgs#changes_args.feed of + "eventsource" -> + case couch_httpd:header_value(Req, "last-event-id") of + undefined -> + ChangesArgs; + Value -> + ChangesArgs#changes_args{since = Value} + end; + _ -> + ChangesArgs + end, + {ChangesArgs1, parse_heartbeat_seq(QArgs)}. + +parse_heartbeat_seq(QArgs) -> + lists:foldl( + fun({K, V}, Acc) -> + case string:to_lower(K) of + "heartbeat_seq" -> V =:= "true"; + _ -> Acc + end + end, + false, + QArgs + ). extract_header_rev(Req, ExplicitRev) when is_binary(ExplicitRev) or is_list(ExplicitRev) -> extract_header_rev(Req, couch_doc:parse_rev(ExplicitRev)); diff --git a/src/chttpd/test/eunit/chttpd_changes_test.erl b/src/chttpd/test/eunit/chttpd_changes_test.erl index 0da5bb7dd..356767823 100644 --- a/src/chttpd/test/eunit/chttpd_changes_test.erl +++ b/src/chttpd/test/eunit/chttpd_changes_test.erl @@ -66,6 +66,11 @@ changes_test_() -> ?TDEF(t_continuous_limit_under_total), ?TDEF(t_continuous_limit_exact_total), ?TDEF(t_continuous_limit_over_total), + ?TDEF(t_heartbeat_seq_normal), + ?TDEF(t_heartbeat_seq_longpoll), + ?TDEF(t_eventsource_heartbeat_plain), + ?TDEF(t_continuous_heartbeat_seq), + ?TDEF(t_eventsource_heartbeat_seq), ?TDEF(t_limit_one), ?TDEF(t_since_now), ?TDEF(t_continuous_since_now), @@ -105,6 +110,22 @@ heartbeat_test_() -> ] }. +% These tests write docs to the db so each one gets a fresh db in a foreach +% setup to avoid messing with other tests. +changes_heartbeat_seq_test_() -> + { + foreach, + fun setup_basic/0, + fun teardown_basic/1, + [ + ?TDEF_FE(t_continuous_heartbeat_seq_no_heartbeat), + ?TDEF_FE(t_continuous_heartbeat_seq_deduped), + ?TDEF_FE(t_normal_heartbeat_seq_filtered), + ?TDEF_FE(t_longpoll_heartbeat_seq_filtered), + ?TDEF_FE(t_heartbeat_seq_not_true) + ] + }. + changes_q8_test_() -> { setup, @@ -113,6 +134,7 @@ changes_q8_test_() -> with([ ?TDEF(t_basic_q8), ?TDEF(t_continuous_q8), + ?TDEF(t_continuous_heartbeat_seq_q8), ?TDEF(t_limit_zero), ?TDEF(t_limit_one_q8), ?TDEF(t_since_now), @@ -358,6 +380,174 @@ t_eventsource_heartbeat({_, DbUrl}) -> ?assert(Heartbeats =< 6), ?assertEqual(0, length(binary:matches(Body, <<"data: {">>))). +t_continuous_heartbeat_seq({_, DbUrl}) -> + Params = "?feed=continuous&heartbeat=10&heartbeat_seq=true", + Lines = with_changes_dur(100, fun() -> changes_lines(DbUrl, Params) end), + {LastSeq, Rows, Heartbeats} = parse_heartbeats(Lines), + ?assertMatch(#{~"last_seq" := _, ~"pending" := 0}, LastSeq), + ?assertEqual( + [ + {6, {?DOC1, ~"2-c"}, ?LEAFREV}, + {7, {?DOC3, ~"2-b"}, ?DELETED}, + {8, {?DDOC2, ~"2-c"}, ?LEAFREV} + ], + lists:map(fun parse_row/1, Rows) + ), + ?assert(length(Heartbeats) >= 1), + #{~"last_seq" := EndSeq} = LastSeq, + lists:foreach( + fun(Heartbeat) -> + ?assertEqual([~"changes", ~"id", ~"seq"], lists:sort(maps:keys(Heartbeat))), + ?assertMatch(#{~"id" := null, ~"changes" := []}, Heartbeat) + end, + Heartbeats + ), + ?assertMatch(#{~"seq" := EndSeq}, lists:last(Heartbeats)), + ?assert(unique_seqs(Heartbeats)), + % Heartbeats never ahead of rows and seqs emitted in order + Seqs = [seq(S) || #{~"seq" := S} <- lists:droplast(Lines)], + ?assertEqual(lists:sort(Seqs), Seqs), + ?assertEqual(8, seq(EndSeq)). + +t_continuous_heartbeat_seq_q8({_, DbUrl}) -> + Params = "?feed=continuous&heartbeat=10&heartbeat_seq=true", + Lines = with_changes_dur(100, fun() -> changes_lines(DbUrl, Params) end), + {LastSeq, Rows, Heartbeats} = parse_heartbeats(Lines), + #{~"last_seq" := EndSeq} = LastSeq, + {_, Revs, _} = lists:unzip3(lists:map(fun parse_row/1, Rows)), + ?assertEqual( + [ + {?DDOC2, ~"2-c"}, + {?DOC1, ~"2-c"}, + {?DOC3, ~"2-b"} + ], + lists:sort(Revs) + ), + ?assert(length(Heartbeats) >= 1), + ?assertMatch(#{~"seq" := EndSeq}, lists:last(Heartbeats)), + ?assert(unique_seqs(Heartbeats)), + Seqs = [seq(S) || #{~"seq" := S} <- lists:droplast(Lines)], + ?assertEqual(lists:sort(Seqs), Seqs), + ?assertEqual(8, length(fabric_view_changes:decode_seq(EndSeq))). + +t_continuous_heartbeat_seq_deduped({_, DbUrl}) -> + Params = "?feed=continuous&filter=_selector&heartbeat=100&heartbeat_seq=true", + Body = #{~"selector" => #{~"_id" => ~"nomatch"}}, + Self = self(), + Writer = spawn_link(fun() -> Self ! {self(), write_docs_for(DbUrl, 300)} end), + Lines = with_changes_dur(300, fun() -> changes_lines_post(DbUrl, Params, Body) end), + Written = + receive + {Writer, N} -> N + after 5000 -> error(writer_timeout) + end, + ?assert(Written >= 3), + {LastSeq, Rows, Heartbeats} = parse_heartbeats(Lines), + ?assertMatch(#{~"last_seq" := _}, LastSeq), + ?assertEqual([], Rows), + % Feed will be open for 300msec + at least one more interval. So we should + % get a bunch of hearbeats. + ?assert(length(Heartbeats) >= 1), + ?assert(length(Heartbeats) =< 6), + ?assert(unique_seqs(Heartbeats)), + Seqs = [seq(S) || #{~"seq" := S} <- Heartbeats], + ?assertEqual(lists:sort(Seqs), Seqs). + +t_continuous_heartbeat_seq_no_heartbeat({_, DbUrl}) -> + % Seq heartbeat sent as soon as feed is caught up and then no more, since + % the timeoutu itnerval passes without any updates and feed ends there. + Params = "?feed=continuous&timeout=300&heartbeat_seq=true", + T0 = erlang:monotonic_time(millisecond), + Lines = changes_lines(DbUrl, Params), + Dt = erlang:monotonic_time(millisecond) - T0, + ?assert(Dt >= 250), + {#{~"last_seq" := EndSeq}, Rows, Heartbeats} = parse_heartbeats(Lines), + ?assertEqual(3, length(Rows)), + ?assertMatch([#{~"seq" := EndSeq}], Heartbeats). + +t_normal_heartbeat_seq_filtered({_, DbUrl}) -> + add_slow_filter(DbUrl, 30), + Params = "?filter=sloth/f&heartbeat=10&heartbeat_seq=true", + {Rows, Heartbeats, EndSeq} = results_with_heartbeats(DbUrl, Params), + ?assertEqual([{7, {?DOC3, ~"2-b"}, ?DELETED}], lists:map(fun parse_row/1, Rows)), + ?assert(length(Heartbeats) >= 1), + ?assertEqual(9, seq(EndSeq)), + % Seq heartbeats are not counted for the limit + {Rows1, _, _} = results_with_heartbeats(DbUrl, Params ++ "&limit=1"), + ?assertEqual(1, length(Rows1)), + % Without a heartbeat period, should get none for a small db + {_, [], _} = results_with_heartbeats(DbUrl, "?filter=sloth/f&heartbeat_seq=true"). + +t_longpoll_heartbeat_seq_filtered({_, DbUrl}) -> + add_slow_filter(DbUrl, 30), + Params = "?feed=longpoll&filter=sloth/f&heartbeat=10&heartbeat_seq=true", + {Rows, Heartbeats, EndSeq} = results_with_heartbeats(DbUrl, Params), + ?assertEqual([{7, {?DOC3, ~"2-b"}, ?DELETED}], lists:map(fun parse_row/1, Rows)), + ?assert(length(Heartbeats) >= 1), + ?assertEqual(9, seq(EndSeq)). + +t_heartbeat_seq_not_true({_, DbUrl}) -> + % Anything besides true is false (like other boolean params) + add_slow_filter(DbUrl, 30), + Plain = results_with_heartbeats(DbUrl, "?filter=sloth/f&heartbeat=10"), + ?assertMatch({[_], [], _}, Plain), + lists:foreach( + fun(Value) -> + Params = "?filter=sloth/f&heartbeat=10&heartbeat_seq=" ++ Value, + ?assertEqual(Plain, results_with_heartbeats(DbUrl, Params)) + end, + ["false", "0", "1000", "potato", ""] + ). + +t_eventsource_heartbeat_plain({_, DbUrl}) -> + Params = "?feed=eventsource&heartbeat=10", + {Events, Heartbeats} = with_changes_dur(100, fun() -> eventsource(DbUrl, Params) end), + ?assertEqual( + [ + {6, {?DOC1, ~"2-c"}, ?LEAFREV}, + {7, {?DOC3, ~"2-b"}, ?DELETED}, + {8, {?DDOC2, ~"2-c"}, ?LEAFREV} + ], + [parse_row(Data) || #{data := Data} <- Events] + ), + ?assert(length(Heartbeats) >= 1), + ?assert(lists:all(fun(#{} = H) -> not maps:is_key(id, H) end, Heartbeats)). + +t_eventsource_heartbeat_seq({_, DbUrl}) -> + Params = "?feed=eventsource&heartbeat=10&heartbeat_seq=true", + {Events, Heartbeats} = with_changes_dur(100, fun() -> eventsource(DbUrl, Params) end), + ?assertEqual( + [ + {6, {?DOC1, ~"2-c"}, ?LEAFREV}, + {7, {?DOC3, ~"2-b"}, ?DELETED}, + {8, {?DDOC2, ~"2-c"}, ?LEAFREV} + ], + [parse_row(Data) || #{data := Data} <- Events] + ), + % Heartbeats have a seq as their event id like the change events do. Those + % can then be used as Last-Event-ID when they reconnect. If there are no + % changes to the sequence then eventsource heartbeats have no id + {WithId, WithoutId} = lists:partition(fun(H) -> maps:is_key(id, H) end, Heartbeats), + ?assert(length(WithId) >= 1), + ?assert(length(WithoutId) >= 1), + lists:foreach( + fun(#{id := Id}) -> + ?assertMatch(<<"8-", _/binary>>, json(Id)) + end, + WithId + ), + #{id := LastEventId} = lists:last(Events), + ?assertMatch(#{id := LastEventId}, lists:last(WithId)). + +t_heartbeat_seq_normal({_, DbUrl}) -> + % Db is small and with default times we don't get any heartbeats + ?assertEqual(changes(DbUrl), changes(DbUrl, "?heartbeat_seq=true")), + ?assertEqual(changes(DbUrl), changes(DbUrl, "?heartbeat_seq=false")). + +t_heartbeat_seq_longpoll({_, DbUrl}) -> + Params = "?feed=longpoll&timeout=10&heartbeat_seq=true", + ?assertEqual(changes(DbUrl, "?feed=longpoll&timeout=10"), changes(DbUrl, Params)). + t_limit_one({_, DbUrl}) -> Params = "?limit=1", ?assertEqual( @@ -1268,6 +1458,78 @@ seq(<<_/binary>> = Seq) -> seq(null) -> null. +changes_lines(DbUrl, Params) -> + {Code, Res} = reqraw(get, DbUrl ++ "/_changes" ++ Params), + ?assertEqual(200, Code), + jsonl(Res). + +changes_lines_post(DbUrl, Params, #{} = Body) -> + {Code, Res} = reqraw(post, DbUrl ++ "/_changes" ++ Params, Body), + ?assertEqual(200, Code), + jsonl(Res). + +unique_seqs(Heartbeats) -> + Seqs = [S || #{~"seq" := S} <- Heartbeats], + case length(lists:usort(Seqs)) =:= length(Seqs) of + true -> true; + false -> {false, Seqs} + end. + +parse_heartbeats(Lines) -> + LastSeq = lists:last(Lines), + Rows = lists:droplast(Lines), + IsHB = fun(#{~"id" := Id}) -> Id =:= null end, + {Heartbeats, Changes} = lists:partition(IsHB, Rows), + {LastSeq, Changes, Heartbeats}. + +% Parse an eventsource response into change events and heartbeat events. Each +% event is a map of its fields, with data parsed as JSON. +eventsource(DbUrl, Params) -> + {Code, Res} = reqraw(get, DbUrl ++ "/_changes" ++ Params), + ?assertEqual(200, Code), + Chunks = binary:split(Res, ~"\n\n", [global]), + Events = [parse_event(C) || C <- Chunks, C =/= <<>>], + lists:partition(fun(#{event := E}) -> E =/= ~"heartbeat" end, Events). + +parse_event(Chunk) -> + Lines = binary:split(Chunk, ~"\n", [global]), + Fields = [binary:split(L, ~": ") || L <- Lines, L =/= ~""], + Event = maps:from_list([{binary_to_atom(K), V} || [K, V] <- Fields]), + % HB events have an empty data field + ParseFun = fun + (~"") -> ~""; + (Data) -> json(Data) + end, + Event1 = maps:update_with(data, ParseFun, Event), + maps:merge(#{event => ~"message"}, Event1). + +% Slow filter which allows doc3 only through +add_slow_filter(DbUrl, MSec) -> + T = integer_to_binary(MSec), + Oof = <<"var t = Date.now(); while (Date.now()-t < ", T/binary, ") {};">>, + Flt = <<"function(doc, req) {", Oof/binary, "return doc._id == 'doc3';}">>, + DDoc = #{~"filters" => #{~"f" => Flt}}, + {201, #{~"ok" := true}} = req(put, DbUrl ++ "/_design/sloth", DDoc), + ok. + +% Get normal or longpoll result and split out rows, heartbeats and endseq +results_with_heartbeats(DbUrl, Params) -> + {200, Res} = req(get, DbUrl ++ "/_changes" ++ Params), + #{~"results" := Results, ~"last_seq" := EndSeq} = Res, + Seqs = [seq(S) || #{~"seq" := S} <- Results], + ?assertEqual(lists:sort(Seqs), Seqs), + ?assert(lists:all(fun(S) -> S =< seq(EndSeq) end, Seqs)), + IsHB = fun(#{~"id" := Id}) -> Id =:= null end, + {Heartbeats, Rows} = lists:partition(IsHB, Results), + lists:foreach( + fun(HB) -> + ?assertEqual([~"changes", ~"id", ~"seq"], lists:sort(maps:keys(HB))), + ?assertMatch(#{~"changes" := []}, HB) + end, + Heartbeats + ), + {Rows, Heartbeats, EndSeq}. + with_changes_dur(Msec, Fun) -> config:set("fabric", "changes_duration", integer_to_list(Msec), false), try diff --git a/src/couch_replicator/test/eunit/couch_replicator_test_helper.erl b/src/couch_replicator/test/eunit/couch_replicator_test_helper.erl index 39f803cf5..bb9a780da 100644 --- a/src/couch_replicator/test/eunit/couch_replicator_test_helper.erl +++ b/src/couch_replicator/test/eunit/couch_replicator_test_helper.erl @@ -66,7 +66,7 @@ changes_callback({change, {Change}}, Acc) -> Revs = proplists:get_value(changes, Change), IdRevs = [{Id, couch_doc:parse_rev(R)} || {[{<<"rev">>, R}]} <- Revs], {ok, IdRevs ++ Acc}; -changes_callback(timeout, Acc) -> +changes_callback({timeout, _Seq}, Acc) -> {ok, Acc}; changes_callback({stop, _EndSeq, _Pending}, Acc) -> {ok, Acc}. diff --git a/src/docs/src/api/database/changes.rst b/src/docs/src/api/database/changes.rst index 7d4c3fb30..17f267e41 100644 --- a/src/docs/src/api/database/changes.rst +++ b/src/docs/src/api/database/changes.rst @@ -76,6 +76,17 @@ :ref:`eventsource <changes/eventsource>` feeds. Overrides any timeout to keep the feed alive indefinitely. Default is ``60000``. May be ``true`` to use default value. + :query boolean heartbeat_seq: Send heartbeats with the current + update sequence. It' ssent at most once + per ``heartbeat`` interval, or per ``timeout`` when no + heartbeat is set. On :ref:`continuous + <changes/continuous>`, :ref:`normal <changes/normal>` and + :ref:`longpoll <changes/longpoll>` feeds, a heartbeat is a row + of the form ``{"seq": "...", "id": null, "changes": []}``. It is sent as a + line of its own or as an element of ``results``. On :ref:`eventsource + <changes/eventsource>` feeds it's a ``heartbeat`` event with ``id`` + set to the sequence. These types of heartbeats can help a client + checkpoint a restrictive filtered feed. Default is ``false``. :query boolean include_docs: Include the associated document with each result. If there are conflicts, only the winning revision is returned. Default is ``false``. When used with ``all_docs`` style and a filter, @@ -376,6 +387,12 @@ By default all changes are immediately returned within the JSON body:: "last_seq":"5-g1AAAAIreJyVkEsKwjAURZ-toI5cgq5A0sQ0OrI70XyppcaRY92J7kR3ojupaSPUUgotgRd4yTlwbw4A0zRUMLdnpaMkwmyF3Ily9xBwEIuiKLI05KOTW0wkV4rruP29UyGWbordzwKVxWBNOGMKZhertDlarbr5pOT3DV4gudUC9-MPJX9tpEAYx4TQASns2E24ucuJ7rXJSL1BbEgf3vTwpmedCZkYa7Pulck7Xt7x_usFU2aIHOD4eEfVTVA5KMGUkqhNZV-8_o5i", "pending": 0} +Reading a large changes feed with a selective filter can take a long time, and +produce few results. With ``heartbeat_seq=true`` the ``results`` array then +might contain heartbeat rows that look like ``{"seq": "...", "id": null, +"changes": []}``. A client incrementally parsing the respose and then +checkpoint those sequences before the response is complete. + ``results`` is the list of changes in sequential order. New and changed documents only differ in the value of the rev; deleted documents include the ``"deleted": true`` attribute. (In the ``style=all_docs mode``, deleted applies @@ -449,6 +466,11 @@ if no change arrives, and hold the connection indefinitely" while ``timeout`` means "Hold this connection open for ``x`` ms, and if no change arrives in that time, close the socket." ``heartbeat`` overrides ``timeout``. +With ``heartbeat_seq=true`` the ``results`` array may have rows of the form +``{"seq": "...", "id": null, "changes": []}``. They periodiclaly indicate how +far the changes feed has made progress even if the changes feed runs a very +selective filter. + .. _changes/continuous: Continuous @@ -478,6 +500,24 @@ results. If `limit` has been specified the feed will end with a `{ last_seq }` object. +By default, a heartbeat is an empty line. With a very restrictive filter, a +client may go a long time without receiving back any matching rows. With +``heartbeat_seq=true`` a heartbeat will return a current update sequence which +a client then may use to checkpoint the progress made so far. + +.. code-block:: http + + GET /db/_changes?feed=continuous&filter=mailbox/new_mail&heartbeat=10000&heartbeat_seq=true HTTP/1.1 + Accept: application/json + Host: localhost:5984 + +.. code-block:: javascript + + {"seq":"5-g1AAAAIreJyVkEsKwjAURZ...","id":null,"changes":[]} + +The ``seq`` value can be used as a ``since`` parameter when resuming the feed. When ``heartbeat`` is not +set. + .. code-block:: http GET /somedatabase/_changes?feed=continuous HTTP/1.1 @@ -543,6 +583,13 @@ This can be monitored by the client application to restart the EventSource connection if needed (i.e. if the TCP connection gets stuck in a half-open state). +With ``heartbeat_seq=true`` a heartbeat event also has an ``id`` field set to +the current update sequence, just like the change events do, when the sequence +changed since the last one and at most once per heartbeat interval. Browsers +send it back in the ``Last-Event-ID`` header when they reconnect, so a filtered +feed resumes from its most recent heartbeat rather than from the last change +event which passed the filter. + .. note:: EventSource connections are subject to cross-origin resource sharing restrictions. You might need to configure :ref:`CORS support diff --git a/src/docs/src/replication/replicator.rst b/src/docs/src/replication/replicator.rst index e3ae9e2ce..ad4a80ae9 100644 --- a/src/docs/src/replication/replicator.rst +++ b/src/docs/src/replication/replicator.rst @@ -832,7 +832,6 @@ your source or target as follows; ... } - Replicate Winning Revisions Only ================================ diff --git a/src/fabric/src/fabric_bench.erl b/src/fabric/src/fabric_bench.erl index e130340a3..30167e626 100644 --- a/src/fabric/src/fabric_bench.erl +++ b/src/fabric/src/fabric_bench.erl @@ -196,7 +196,7 @@ changes_cb(start, Acc) -> changes_cb({change, Row}, Acc) -> _ = jiffy:encode(Row), {ok, Acc + 1}; -changes_cb(timeout, Acc) -> +changes_cb({timeout, _Seq}, Acc) -> {ok, Acc}; changes_cb({stop, _, _}, Acc) -> {ok, Acc}. diff --git a/src/fabric/src/fabric_view_changes.erl b/src/fabric/src/fabric_view_changes.erl index e9c60a4d5..f97475b8a 100644 --- a/src/fabric/src/fabric_view_changes.erl +++ b/src/fabric/src/fabric_view_changes.erl @@ -39,6 +39,9 @@ % compatibilty reasons and to make replication self-healing. -define(BADSEQ, badseq). +% Heartbeat frequency for normal feeds +-define(NORMAL_TIMEOUT, 5000). + go(DbName, Feed, Options, Callback, Acc0) when Feed == "continuous" orelse Feed == "longpoll" orelse Feed == "eventsource" @@ -90,7 +93,7 @@ go(DbName, "normal", Options, Callback, Acc0) -> Callback, Since, Acc, - 5000 + normal_timeout(Args) ), #collector{counters = Seqs, user_acc = AccOut, offset = Offset} = Collector, Callback({stop, pack_seqs(Seqs), pending_count(Offset)}, AccOut); @@ -98,6 +101,11 @@ go(DbName, "normal", Options, Callback, Acc0) -> Callback(Error, Acc0) end. +normal_timeout(#changes_args{heartbeat = HB}) when is_integer(HB) -> + min(?NORMAL_TIMEOUT, HB); +normal_timeout(#changes_args{}) -> + ?NORMAL_TIMEOUT. + keep_sending_changes(DbName, Args, Callback, Seqs, AccIn, Timeout, UpListen, T0) -> #changes_args{limit = Limit, feed = Feed, heartbeat = Heartbeat, dir = Dir} = Args, {ok, Collector} = send_changes(DbName, Args, Callback, Seqs, AccIn, Timeout), @@ -120,7 +128,10 @@ keep_sending_changes(DbName, Args, Callback, Seqs, AccIn, Timeout, UpListen, T0) NewEpoch -> Callback({stop, LastSeq, PendingCount}, AccOut0); true -> - {ok, AccOut} = Callback(waiting_for_updates, AccOut0), + {ok, AccOut1} = Callback(waiting_for_updates, AccOut0), + % Heartbeat the sequence at end of pass and emit it before waiting + % for updates. THis way we cover the updates just processed. + {ok, AccOut} = Callback({timeout, LastSeq}, AccOut1), WaitForUpdate = wait_db_updated(UpListen), AccumulatedTime = timer:now_diff(os:timestamp(), T0) div 1000, Max = @@ -138,10 +149,9 @@ keep_sending_changes(DbName, Args, Callback, Seqs, AccIn, Timeout, UpListen, T0) {_, true, timeout} -> Callback({stop, LastSeq, PendingCount}, AccOut); _ -> - {ok, AccTimeout} = Callback(timeout, AccOut), Args1 = Args#changes_args{limit = Limit2}, keep_sending_changes( - DbName, Args1, Callback, LastSeq, AccTimeout, Timeout, UpListen, T0 + DbName, Args1, Callback, LastSeq, AccOut, Timeout, UpListen, T0 ) end end. @@ -253,7 +263,10 @@ receive_results(Workers, State, Timeout, Callback) -> ) of {timeout, NewState0} -> - {ok, AccOut} = Callback(timeout, NewState0#collector.user_acc), + % Send the seq with the heartbeat so callbacks can checkpoint if + % they want to. + Seq = pack_seqs(NewState0#collector.counters), + {ok, AccOut} = Callback({timeout, Seq}, NewState0#collector.user_acc), NewState = NewState0#collector{user_acc = AccOut}, receive_results(Workers, NewState, Timeout, Callback); {_, NewState} -> diff --git a/src/fabric/test/eunit/fabric_changes_test.erl b/src/fabric/test/eunit/fabric_changes_test.erl index 9b850696d..b23ea6704 100644 --- a/src/fabric/test/eunit/fabric_changes_test.erl +++ b/src/fabric/test/eunit/fabric_changes_test.erl @@ -32,7 +32,8 @@ fabric_changes_test_() -> ?TDEF(t_main_only_include_docs), ?TDEF(t_all_docs_include_docs), ?TDEF(t_main_only_include_docs_conflicts), - ?TDEF(t_all_docs_include_docs_conflicts) + ?TDEF(t_all_docs_include_docs_conflicts), + ?TDEF(t_continuous_timeout_seq) ]) }. @@ -515,9 +516,26 @@ t_custom_all_docs_include_docs_conflicts({_, DbName, Rev}) -> Doc ). +t_continuous_timeout_seq({_, DbName}) -> + config:set("fabric", "changes_duration", "200", false), + Args = #changes_args{feed = "continuous", heartbeat = 20}, + {ok, Events, EndSeq, 0} = changes_events(DbName, Args), + Changes = [C || {change, C} <- Events], + ?assertMatch([#{id := ~"a"}], Changes), + Timeouts = [Seq || {timeout, Seq} <- Events], + ?assert(length(Timeouts) >= 1), + ?assert(lists:all(fun(Seq) -> Seq =:= EndSeq end, Timeouts)), + % The feed ends with the one change we have + [#{seq := ChangeSeq}] = Changes, + ?assertEqual(seq_num(ChangeSeq), seq_num(EndSeq)), + % First we get the change row, then the heartbeat + ?assertMatch([{change, _}, {timeout, EndSeq} | _], Events), + config:delete("fabric", "changes_duration", false). + %%%%%%%%%%%%%%%%%%%% Utility Functions %%%%%%%%%%%%%%%%%%%% setup() -> Ctx = test_util:start_couch([fabric]), + config:delete("fabric", "changes_duration", false), DbName = ?tempdb(), ok = fabric:create_db(DbName, [{q, 1}, {n, 1}]), Docs = [ @@ -544,6 +562,7 @@ teardown({Ctx, DbName, _}) -> teardown({Ctx, DbName}); teardown({Ctx, DbName}) -> ok = fabric:delete_db(DbName, [?ADMIN_CTX]), + config:delete("fabric", "changes_duration", false), test_util:stop_couch(Ctx). changes_callback(start, Acc) -> @@ -556,3 +575,22 @@ changes_callback({stop, EndSeq, Pending}, Acc) -> changes(DbName, #changes_args{} = Args) -> fabric_util:isolate(fun() -> fabric:changes(DbName, fun changes_callback/2, [], Args) end). + +% Collect all the callback events, in order, including timeouts +events_callback(start, Acc) -> + {ok, Acc}; +events_callback({change, {Change}}, Acc) -> + {ok, [{change, maps:from_list(Change)} | Acc]}; +events_callback({timeout, Seq}, Acc) -> + {ok, [{timeout, Seq} | Acc]}; +events_callback(waiting_for_updates, Acc) -> + {ok, Acc}; +events_callback({stop, EndSeq, Pending}, Acc) -> + {ok, lists:reverse(Acc), EndSeq, Pending}. + +changes_events(DbName, #changes_args{} = Args) -> + fabric_util:isolate(fun() -> fabric:changes(DbName, fun events_callback/2, [], Args) end). + +seq_num(<<_/binary>> = Seq) -> + [NumStr, _] = binary:split(Seq, ~"-"), + binary_to_integer(NumStr). diff --git a/src/global_changes/src/global_changes_httpd.erl b/src/global_changes/src/global_changes_httpd.erl index a71403e7b..d94183a7e 100644 --- a/src/global_changes/src/global_changes_httpd.erl +++ b/src/global_changes/src/global_changes_httpd.erl @@ -150,7 +150,7 @@ changes_callback({change, {ChangeProp} = Change}, #acc{resp = Resp, feed = "even ], {ok, Resp1} = chttpd:send_delayed_chunk(Resp, Chunk), maybe_finish(Acc#acc{resp = Resp1}); -changes_callback(timeout, #acc{feed = "eventsource"} = Acc) -> +changes_callback({timeout, _Seq}, #acc{feed = "eventsource"} = Acc) -> #acc{resp = Resp} = Acc, Chunk = "event: heartbeat\ndata: \n\n", {ok, Resp1} = chttpd:send_delayed_chunk(Resp, Chunk), @@ -204,7 +204,7 @@ changes_callback({stop, EndSeq, _Pending}, Acc) -> ["\n],\n\"last_seq\":", ?JSON_ENCODE(EndSeq), "}\n"] ), chttpd:end_delayed_json_response(Resp1); -changes_callback(timeout, Acc) -> +changes_callback({timeout, _Seq}, Acc) -> {ok, maybe_send_heartbeat(Acc)}; changes_callback({error, Reason}, #acc{resp = Req = #httpd{}}) -> chttpd:send_error(Req, Reason); diff --git a/src/mem3/test/eunit/mem3_reshard_changes_feed_test.erl b/src/mem3/test/eunit/mem3_reshard_changes_feed_test.erl index 799c0721f..45ed90f3c 100644 --- a/src/mem3/test/eunit/mem3_reshard_changes_feed_test.erl +++ b/src/mem3/test/eunit/mem3_reshard_changes_feed_test.erl @@ -195,7 +195,7 @@ continuous_feed_should_work_during_split(#{db1 := Db}) -> {"in_process", _} -> {ok, Acc} end; - (timeout, Acc) -> + ({timeout, _Seq}, Acc) -> {ok, Acc}; ({change, {Change}}, Acc) -> CM = maps:from_list(Change), diff --git a/test/elixir/test/changes_async_test.exs b/test/elixir/test/changes_async_test.exs index 97a4b2ebb..9deb6f328 100644 --- a/test/elixir/test/changes_async_test.exs +++ b/test/elixir/test/changes_async_test.exs @@ -204,6 +204,48 @@ defmodule ChangesAsyncTest do assert beats == 3 end + @tag :with_db + test "continuous heartbeat with seq", context do + db_name = context[:db_name] + create_doc(db_name, sample_doc_foo()) + + req_id = + Rawresp.get( + "/#{db_name}/_changes?feed=continuous&heartbeat=10&heartbeat_seq=true", + stream_to: self() + ) + + :ok = wait_for_headers(req_id.id, 200) + + rows = wait_for_rows(req_id.id, [], 2) + assert length(rows) == 2 + {changes, heartbeats} = Enum.split_with(rows, fn row -> row["id"] != nil end) + assert [change] = changes + assert change["id"] == "foo" + assert String.starts_with?(change["seq"], "1-") + assert [heartbeat] = heartbeats + assert heartbeat == %{"seq" => heartbeat["seq"], "id" => nil, "changes" => []} + assert String.starts_with?(heartbeat["seq"], "1-") + + # A new change bumps the sequence. We should get a row then a heartbeat + # with a seq. + create_doc_bar(db_name, "bar") + seq2? = fn row -> row["id"] == nil and String.starts_with?(row["seq"], "2-") end + + rows = + wait_until(req_id.id, &parse_changes_line/1, [], fn rows -> + Enum.any?(rows, seq2?) + end) + + {changes, heartbeats} = Enum.split_with(rows, fn row -> row["id"] != nil end) + assert [change] = changes + assert change["id"] == "bar" + assert String.starts_with?(change["seq"], "2-") + assert [heartbeat] = heartbeats + assert heartbeat["changes"] == [] + assert String.starts_with?(heartbeat["seq"], "2-") + end + @tag :with_db test "longpoll filtered changes", context do db_name = context[:db_name] @@ -351,6 +393,29 @@ defmodule ChangesAsyncTest do end end + defp wait_for_rows(id, rows, expected) do + if length(rows) < expected do + case next_chunk(id) do + :timeout -> rows + chunk -> wait_for_rows(id, rows ++ parse_changes_line(chunk), expected) + end + else + rows + end + end + + # Get items from feed until `done?` or the feed is quiet + defp wait_until(id, parser, items, done?) do + if done?.(items) do + items + else + case next_chunk(id) do + :timeout -> items + chunk -> wait_until(id, parser, items ++ parser.(chunk), done?) + end + end + end + defp wait_for_headers(id, status, timeout \\ 1000) do receive do %Couch.AsyncHeaders{id: ^id, status_code: ^status} -> diff --git a/test/elixir/test/config/suite.elixir b/test/elixir/test/config/suite.elixir index 37ed47e15..135be31e4 100644 --- a/test/elixir/test/config/suite.elixir +++ b/test/elixir/test/config/suite.elixir @@ -106,6 +106,7 @@ "continuous changes", "continuous filtered changes", "continuous filtered changes with doc ids", + "continuous heartbeat with seq", "eventsource changes", "eventsource changes with limit under", "eventsource changes with limit exact",
