This is an automated email from the ASF dual-hosted git repository.

nickva pushed a commit to branch replicator-heartbeat-seq
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit cc1d408601546df12b07ef025633e6420ef11c23
Author: Nick Vatamaniuc <[email protected]>
AuthorDate: Wed Oct 21 14:38:43 2026 -0400

    Replicator support for heartbeat_seq change feed parameter
    
    This is a follow-up to heartbeat_seq PR #6165 where implementing 
heartbeat_seq
    handling in the replicator. Take advantage of the fact that unknown 
parameters
    are ignored by endpoints, so we always pass `heartbeat_seq=true`, and if we 
get
    any hearbeat rows we'll use them for checkpointing, and if we don't (an old
    source endpoint) we'll function as before.
    
    Issue: #6120
---
 .../src/couch_replicator_api_wrap.erl              | 42 ++++++++++++++---
 .../src/couch_replicator_changes_reader.erl        | 54 +++++++++++++++++++---
 src/docs/src/replication/protocol.rst              |  6 +++
 3 files changed, 90 insertions(+), 12 deletions(-)

diff --git a/src/couch_replicator/src/couch_replicator_api_wrap.erl 
b/src/couch_replicator/src/couch_replicator_api_wrap.erl
index c1ab912d4..57e15d2c4 100644
--- a/src/couch_replicator/src/couch_replicator_api_wrap.erl
+++ b/src/couch_replicator/src/couch_replicator_api_wrap.erl
@@ -547,12 +547,7 @@ changes_since(
     Timeout = max(1000, InactiveTimeout div 3),
     EncodedSeq = couch_replicator_utils:seq_encode(StartSeq),
     BaseQArgs =
-        case get_value(continuous, Options, false) of
-            false ->
-                [{"feed", "normal"}];
-            true ->
-                [{"feed", "continuous"}]
-        end ++
+        changes_feed_qargs(Options) ++
             [
                 {"style", atom_to_list(Style)},
                 {"since", EncodedSeq},
@@ -669,6 +664,17 @@ maybe_add_changes_filter_q_args(BaseQS, Options) ->
             ]
     end.
 
+changes_feed_qargs(Options) ->
+    Feed =
+        case get_value(continuous, Options, false) of
+            false -> "normal";
+            true -> "continuous"
+        end,
+    % If the source supports heartbeat_seq=true we'll get and use {seq: ...,
+    % id: null, changes: []} heartbeat rows for checkpointing. If the source
+    % doesn't support it, it will ignore the parameter.
+    [{"feed", Feed}, {"heartbeat_seq", "true"}].
+
 parse_changes_feed(Options, UserFun, DataStreamFun) ->
     case get_value(continuous, Options, false) of
         true ->
@@ -961,6 +967,9 @@ json_to_doc_info({Props}) ->
     case get_value(<<"changes">>, Props) of
         undefined ->
             {last_seq, get_value(<<"last_seq">>, Props)};
+        [] ->
+            % Got a heartbeat row from heartbeat_seq=true param
+            {heartbeat_seq, get_value(~"seq", Props)};
         Changes ->
             RevsInfo0 = lists:map(
                 fun({Change}) ->
@@ -1131,4 +1140,25 @@ normalize_http_db_test() ->
     ?assertEqual(Expected, normalize_db(HttpDb)),
     ?assertEqual(<<"local">>, normalize_db(<<"local">>)).
 
+json_to_doc_info_test() ->
+    Seq = ~"42-abc",
+    ?assertEqual({last_seq, Seq}, json_to_doc_info({[{~"last_seq", Seq}]})),
+    ?assertEqual({last_seq, undefined}, json_to_doc_info({[{~"pending", 0}]})),
+    HB = {[{~"seq", Seq}, {~"id", null}, {~"changes", []}]},
+    ?assertEqual({heartbeat_seq, Seq}, json_to_doc_info(HB)),
+    Row =
+        {[
+            {~"seq", Seq},
+            {~"id", ~"doc1"},
+            {~"changes", [{[{~"rev", ~"1-abc"}]}]}
+        ]},
+    ?assertEqual(
+        #doc_info{
+            id = ~"doc1",
+            high_seq = Seq,
+            revs = [#rev_info{rev = {1, ~"abc"}}]
+        },
+        json_to_doc_info(Row)
+    ).
+
 -endif.
diff --git a/src/couch_replicator/src/couch_replicator_changes_reader.erl 
b/src/couch_replicator/src/couch_replicator_changes_reader.erl
index bb6733608..7ee121ced 100644
--- a/src/couch_replicator/src/couch_replicator_changes_reader.erl
+++ b/src/couch_replicator/src/couch_replicator_changes_reader.erl
@@ -132,18 +132,31 @@ process_change({last_seq, LS}, {_Parent, _, ChangesQueue, 
true = _Continuous}) -
             undefined -> get(last_seq);
             _ -> LS
         end,
-    OldSeq = get(last_seq),
-    if
-        Seq == OldSeq -> ok;
-        true -> ok = couch_work_queue:queue(ChangesQueue, {last_seq, Seq})
-    end,
-    put(last_seq, Seq),
+    queue_last_seq(ChangesQueue, Seq),
     throw(recurse);
 process_change({last_seq, _}, _) ->
     % This clause is unreachable today, but let's plan ahead for the future
     % where we checkpoint against last_seq instead of the sequence of the last
     % change.  The two can differ substantially in the case of a restrictive
     % filter.
+    ok;
+process_change({heartbeat_seq, Seq}, {_Parent, _, ChangesQueue, _Continuous}) 
when
+    is_binary(Seq); is_integer(Seq)
+->
+    queue_last_seq(ChangesQueue, Seq);
+process_change({heartbeat_seq, _}, _) ->
+    % We're being defensive here since we're in the replicator, and have to
+    % handle other, possibly non-ASF CouchDB implementations, or strange
+    % proxies. Here, we'd catch heartbeat objects without a seq or seq set to
+    % null, for example and we ignore them instead of crashing everything.
+    ok.
+
+queue_last_seq(ChangesQueue, Seq) ->
+    case get(last_seq) of
+        Seq -> ok;
+        _ -> ok = couch_work_queue:queue(ChangesQueue, {last_seq, Seq})
+    end,
+    put(last_seq, Seq),
     ok.
 
 is_doc_id_too_long(IdLength) ->
@@ -154,3 +167,32 @@ is_doc_id_too_long(IdLength) ->
             ConfigMax = list_to_integer(ConfigMaxStr),
             ConfigMax > 0 andalso IdLength > ConfigMax
     end.
+
+-ifdef(TEST).
+
+-include_lib("eunit/include/eunit.hrl").
+
+heartbeat_seq_test() ->
+    {ok, Q} = couch_work_queue:new([{max_size, 100}, {max_items, 100}]),
+    Ctx = {self(), #httpdb{}, Q, true},
+    put(last_seq, ~"1-a"),
+    % new sequence => last_seq
+    ?assertEqual(ok, process_change({heartbeat_seq, ~"2-b"}, Ctx)),
+    ?assertEqual(~"2-b", get(last_seq)),
+    ?assertEqual({ok, [{last_seq, ~"2-b"}]}, couch_work_queue:dequeue(Q)),
+    % dedup works
+    ?assertEqual(ok, process_change({heartbeat_seq, ~"2-b"}, Ctx)),
+    ?assertEqual(0, couch_work_queue:item_count(Q)),
+    % new seq is queued
+    ?assertEqual(ok, process_change({heartbeat_seq, ~"4-d"}, Ctx)),
+    ?assertEqual({ok, [{last_seq, ~"4-d"}]}, couch_work_queue:dequeue(Q)),
+    ?assertEqual(~"4-d", get(last_seq)),
+    % old replicators had integer sequences, we're maximally backwards
+    % compatible so test them too
+    ?assertEqual(ok, process_change({heartbeat_seq, 5}, Ctx)),
+    ?assertEqual({ok, [{last_seq, 5}]}, couch_work_queue:dequeue(Q)),
+    ?assertEqual(5, get(last_seq)),
+    erase(last_seq),
+    couch_work_queue:close(Q).
+
+-endif.
diff --git a/src/docs/src/replication/protocol.rst 
b/src/docs/src/replication/protocol.rst
index f361bc334..902af38bc 100644
--- a/src/docs/src/replication/protocol.rst
+++ b/src/docs/src/replication/protocol.rst
@@ -766,6 +766,12 @@ request. This request MUST be made with the following 
query parameters:
   period in *milliseconds*. The RECOMMENDED value by default is ``10000``
   (10 seconds).
 
+- The ``heartbeat_seq=true`` parameter MAY be used so that heartbeats will be
+  emitted as rows of the form ``{"seq": ..., "id": null, "changes": []}``.
+  Replicator SHOULD use its sequence to record a Checkpoint. This advances
+  Checkpoints when a filter rejects the Changes for a long time. A Source which
+  does not support the parameter SHOULD ignore it.
+
 - If a startup Checkpoint was found during the Replication Logs comparison,
   the ``since`` query parameter MUST be passed with this value.
   In case of Full Replication it MAY be ``0`` (number zero) or

Reply via email to