This is an automated email from the ASF dual-hosted git repository.

nickva pushed a commit to branch fix-pull-purge-checkpoints
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit 23af37f903a65633c379eea5e55698468c176db8
Author: Nick Vatamaniuc <[email protected]>
AuthorDate: Sat Nov 7 18:40:04 2026 -0500

    Fix stuck mem3 purge checkpoints
    
    Observed a few rare cases of purges being stuck in production
    
    We had the wrong field name in read repair purge checkpoint filter. Before 
we
    looked up `target_node` in the purge checkpoint docs but it should really be
    `target` (see mem3_rep:purge_cp_body/3). This way every node started at 
purge
    seq 0.
    
    `pull_purges/5` had the target/source swapped. The checkpoint lives on the 
target
    so we should always write it from the target's perspective, meaning the 
target
    is the source and we're the target.
    
    There was an off-by-one error in `have_all_purge_checkpoints/1`
    
    Shards db checkpoints have an undefined range (since they are effectively 
local
    dbs replicated in special ring topology) and they replicated by name. As 
such
    they never mathed the `{Node, [0, ?RING_END]}` copies so
    have_all_purge_checkpoints/1 was always false for them.
    
    Fortunately this is easy to fix, and just deleting the checkpoints and 
letting
    the next internal replicaton cycle start over from the oldest purge seq 
works.
---
 src/couch/src/couch_bt_engine_compactor.erl        |  5 +-
 .../test/eunit/couch_bt_engine_compactor_tests.erl | 33 ++++++++++-
 src/fabric/src/fabric_rpc.erl                      | 12 +++-
 src/fabric/test/eunit/fabric_rpc_purge_tests.erl   | 51 ++++++++++++++++-
 src/mem3/src/mem3_rep.erl                          | 27 +++++++--
 src/mem3/test/eunit/mem3_rep_test.erl              | 65 +++++++++++++++++++++-
 6 files changed, 179 insertions(+), 14 deletions(-)

diff --git a/src/couch/src/couch_bt_engine_compactor.erl 
b/src/couch/src/couch_bt_engine_compactor.erl
index 36c149217..2a8de05cf 100644
--- a/src/couch/src/couch_bt_engine_compactor.erl
+++ b/src/couch/src/couch_bt_engine_compactor.erl
@@ -176,9 +176,12 @@ copy_purge_info(#comp_st{} = CompSt) ->
         % that case we'd end up with a "hole" between 500 and 1001 -- a new
         % minimum purge sequence of 500, but the oldest checkpoint is would be
         % 1001.
+        %
+        % Keep infos with PSeq > MinPurgeSeq. To get all of them, including the
+        % oldest need to use Oldest-1.
         case mem3_rep:have_all_purge_checkpoints(Db) of
             true -> couch_db:get_minimum_purge_seq(Db);
-            false -> couch_db:get_oldest_purge_seq(Db)
+            false -> max(0, couch_db:get_oldest_purge_seq(Db) - 1)
         end
     end),
     OldPSTree = OldSt#st.purge_seq_tree,
diff --git a/src/couch/test/eunit/couch_bt_engine_compactor_tests.erl 
b/src/couch/test/eunit/couch_bt_engine_compactor_tests.erl
index 63df30fb0..69667a7e4 100644
--- a/src/couch/test/eunit/couch_bt_engine_compactor_tests.erl
+++ b/src/couch/test/eunit/couch_bt_engine_compactor_tests.erl
@@ -41,7 +41,8 @@ basic_compaction_test_() ->
             fun teardown/1,
             [
                 fun compaction_resume/1,
-                fun is_compacting_works/1
+                fun is_compacting_works/1,
+                fun preserve_purges_without_all_checkpoints/1
             ]
         }
     }.
@@ -81,6 +82,36 @@ is_compacting_works(DbName) ->
         meck:unload(couch_emsort)
     end).
 
+preserve_purges_without_all_checkpoints(DbName) ->
+    ?_test(begin
+        purge_docs(DbName),
+        ?assertEqual({1, 3, 3}, purge_info_stats(DbName)),
+        meck:new(mem3_rep, [passthrough]),
+        meck:expect(mem3_rep, have_all_purge_checkpoints, 1, false),
+        compact_db(DbName),
+        ?assert(meck:called(mem3_rep, have_all_purge_checkpoints, 1)),
+        ?assertEqual({1, 3, 3}, purge_info_stats(DbName)),
+        meck:unload(mem3_rep)
+    end).
+
+purge_docs(DbName) ->
+    couch_util:with_db(DbName, fun(Db) ->
+        PurgeFun = fun(DocId) ->
+            {ok, #doc{revs = {Pos, [Rev | _]}}} = couch_db:open_doc(Db, DocId, 
[]),
+            {couch_uuids:random(), DocId, [{Pos, Rev}]}
+        end,
+        PInfos = lists:map(PurgeFun, [~"doc1", ~"doc2", ~"doc3"]),
+        {ok, _} = couch_db:purge_docs(Db, PInfos)
+    end).
+
+% Return {OldestPurgeSeq, PurgeSeq, PurgeInfosCount}
+purge_info_stats(DbName) ->
+    couch_util:with_db(DbName, fun(Db) ->
+        FoldFun = fun(_, Acc) -> {ok, Acc + 1} end,
+        {ok, Count} = couch_db:fold_purge_infos(Db, FoldFun, 0),
+        {couch_db:get_oldest_purge_seq(Db), couch_db:get_purge_seq(Db), Count}
+    end).
+
 check_db_validity(DbName) ->
     couch_util:with_db(DbName, fun(Db) ->
         ?assertEqual({ok, 3}, couch_db:get_doc_count(Db)),
diff --git a/src/fabric/src/fabric_rpc.erl b/src/fabric/src/fabric_rpc.erl
index a05d59f96..1a2bc4928 100644
--- a/src/fabric/src/fabric_rpc.erl
+++ b/src/fabric/src/fabric_rpc.erl
@@ -582,14 +582,18 @@ read_repair_filter(Db, Docs, NodeRevs) ->
     NodeSeqs = get_node_seqs(Db, Nodes),
 
     DbPSeq = couch_db:get_purge_seq(Db),
+    OldestPSeq = couch_db:get_oldest_purge_seq(Db),
     Lag = config:get_integer("couchdb", "read_repair_lag", 100),
 
     % Filter out read-repair updates from any node that is
     % so out of date that it would force us to scan a large
-    % number of purge infos
+    % number of purge infos. And drop nodes behind our oldest purge info.
+    % Compaction might already removed infos they haven't seen, so we can't 
tell if
+    % their revisions were purged, and fold_purge_infos/4 would
+    % fail with an invalid_start_purge_seq error.
     NodeFiltFun = fun({Node, _Revs}) ->
         {Node, NodeSeq} = lists:keyfind(Node, 1, NodeSeqs),
-        NodeSeq >= DbPSeq - Lag
+        NodeSeq >= DbPSeq - Lag andalso NodeSeq >= OldestPSeq - 1
     end,
     RecentNodeRevs = lists:filter(NodeFiltFun, NonLocalNodeRevs),
 
@@ -625,7 +629,9 @@ get_node_seqs(Db, Nodes) ->
     FoldFun = fun(#doc{id = Id, body = {Props}}, Acc) ->
         case Id of
             <<?LOCAL_DOC_PREFIX, "purge-mem3-", _/binary>> ->
-                TgtNode = couch_util:get_value(<<"target_node">>, Props),
+                % Should match mem3_rep:purge_cp_body/3. There "target"
+                % is the shard we replicate our purges TO.
+                TgtNode = couch_util:get_value(~"target", Props),
                 PurgeSeq = couch_util:get_value(<<"purge_seq">>, Props),
                 case lists:keyfind(TgtNode, 1, Acc) of
                     {_, OldSeq} ->
diff --git a/src/fabric/test/eunit/fabric_rpc_purge_tests.erl 
b/src/fabric/test/eunit/fabric_rpc_purge_tests.erl
index fa6123dfc..3d2eb35b2 100644
--- a/src/fabric/test/eunit/fabric_rpc_purge_tests.erl
+++ b/src/fabric/test/eunit/fabric_rpc_purge_tests.erl
@@ -60,6 +60,19 @@ main_test_() ->
                     ?TDEF_FE(t_no_filter_different_node),
                     ?TDEF_FE(t_no_filter_after_repl)
                 ]
+            },
+            {
+                foreach,
+                fun setup_trimmed_purge/0,
+                fun teardown_trimmed_purge/1,
+                [
+                    ?TDEF_FE(t_filter),
+                    ?TDEF_FE(t_filter_unknown_node),
+                    ?TDEF_FE(t_filter_local_node),
+                    ?TDEF_FE(t_no_filter_old_node),
+                    ?TDEF_FE(t_no_filter_different_node),
+                    ?TDEF_FE(t_no_filter_after_repl)
+                ]
             }
         ]
     }.
@@ -104,6 +117,21 @@ setup_multi_purge() ->
 teardown_multi_purge(Ctx) ->
     teardown_single_purge(Ctx).
 
+% Like the multi case above, but with the compactor trimmed oldest purge infos
+% 1 to 3. After that the oldest purge seq is 4. Nodes with checkpoints before 3
+% are too far behind and their read repair updates should be filtered out.
+setup_trimmed_purge() ->
+    {DbName, _, _, _} = Ctx = setup_multi_purge(),
+    couch_util:with_db(DbName, fun(Db) ->
+        ok = couch_db:set_purge_infos_limit(Db, 2)
+    end),
+    compact(DbName),
+    ?assertEqual({4, 5}, purge_seqs(DbName)),
+    Ctx.
+
+teardown_trimmed_purge(Ctx) ->
+    teardown_single_purge(Ctx).
+
 t_no_purge_no_filter(DbName) ->
     DocId = <<"0003">>,
 
@@ -239,18 +267,37 @@ create_purge_checkpoint(DbName, PurgeSeq, TgtNode) when 
is_binary(TgtNode) ->
     Resp = couch_util:with_db(DbName, fun(Db) ->
         SrcUUID = couch_db:get_uuid(Db),
         TgtUUID = couch_uuids:random(),
+        % See mem3_rep:purge_cp_body/3
         CPDoc = #doc{
             id = mem3_rep:make_purge_id(SrcUUID, TgtUUID),
             body =
                 {[
-                    {<<"target_node">>, TgtNode},
-                    {<<"purge_seq">>, PurgeSeq}
+                    {~"type", ~"internal_replication"},
+                    {~"updated_on", os:system_time(second)},
+                    {~"purge_seq", PurgeSeq},
+                    {~"source", atom_to_binary(node())},
+                    {~"target", TgtNode},
+                    {~"range", [0, 16#ffffffff]}
                 ]}
         },
         couch_db:update_docs(Db, [CPDoc], [])
     end),
     ?assertMatch({ok, [_]}, Resp).
 
+compact(DbName) ->
+    {ok, _} = couch_util:with_db(DbName, fun couch_db:start_compact/1),
+    test_util:wait(fun() ->
+        case couch_util:with_db(DbName, fun couch_db:get_compactor_pid/1) of
+            Pid when is_pid(Pid) -> wait;
+            _ -> ok
+        end
+    end).
+
+purge_seqs(DbName) ->
+    couch_util:with_db(DbName, fun(Db) ->
+        {couch_db:get_oldest_purge_seq(Db), couch_db:get_purge_seq(Db)}
+    end).
+
 rpc_update_doc(DbName, Doc) ->
     {Pos, [Rev | _]} = Doc#doc.revs,
     RROpt = {read_repair, [{tgt_node(), [{Pos, Rev}]}]},
diff --git a/src/mem3/src/mem3_rep.erl b/src/mem3/src/mem3_rep.erl
index 659adf275..d622a0f56 100644
--- a/src/mem3/src/mem3_rep.erl
+++ b/src/mem3/src/mem3_rep.erl
@@ -226,12 +226,21 @@ have_all_purge_checkpoints(Db) ->
 have_all_purge_checkpoints(true, Db, [_ | _] = Shards) ->
     ShardName = couch_db:name(Db),
     UUID = couch_db:get_uuid(Db),
+    IsShardsDb = ShardName == mem3_sync:shards_db(),
     FoldFun = fun(#doc{id = Id, body = {Props}}, Acc) ->
         case Id of
             <<?PURGE_PREFIX, UUID:?UUID_SIZE/binary, "-", 
_:?UUID_SIZE/binary>> ->
                 case verify_checkpoint_shard(Shards, Props) of
                     true ->
-                        Range = couch_util:get_value(<<"range">>, Props),
+                        Range =
+                            case IsShardsDb of
+                                % Shards db doesn't have a bona-fide range but
+                                % we pretend it's a real db but givig it the
+                                % full range to prevent it from being
+                                % undefined.
+                                true -> [0, ?RING_END];
+                                false -> couch_util:get_value(<<"range">>, 
Props)
+                            end,
                         TBin = couch_util:get_value(<<"target">>, Props),
                         TNode = binary_to_existing_atom(TBin, latin1),
                         {ok, sets:add_element({TNode, Range}, Acc)};
@@ -244,7 +253,7 @@ have_all_purge_checkpoints(true, Db, [_ | _] = Shards) ->
     end,
     CheckpointSet = fold_purge_checkpoints(Db, FoldFun, couch_util:new_set()),
     CopySet =
-        case ShardName == mem3_sync:shards_db() of
+        case IsShardsDb of
             true ->
                 % We're dealing with the shards db itself. By convention
                 % mem3:shards/1 returns a single #shard{} record with node =
@@ -543,7 +552,10 @@ pull_purges(Db, Count, #shard{} = SrcShard, #tgt{} = Tgt0, 
HashFun) ->
             end,
             Infos1 = lists:filter(BelongsFun, Infos),
             {ok, _} = couch_db:purge_docs(Db, Infos1, [?REPLICATED_CHANGES]),
-            Body = purge_cp_body(SrcShard, TgtShard, ThroughSeq),
+            % A checkpoint form the target but it's the same one the target
+            % uses when it pushes to us. So we write it from the target's point
+            % of view: it's the source and we are the target.
+            Body = purge_cp_body(TgtShard, SrcShard, ThroughSeq),
             mem3_rpc:save_purge_checkpoint(TgtNode, TgtDbName, RemoteId, Body)
     end,
     Tgt#tgt{remaining = max(0, Remaining)}.
@@ -1443,9 +1455,12 @@ t_have_all_shards_db(_) ->
     ],
     meck:expect(mem3, shards, 1, Shards),
 
-    Src1 = #shard{name = Dbs, node = node(), range = Range},
-    Tgt1 = #shard{name = Dbs, node = 'n2', range = Range},
-    Tgt2 = #shard{name = Dbs, node = 'n3', range = Range},
+    % Same asin go/3, mem3_sync replicates by name, so its source and target
+    % shards don't have a range
+    Src1 = add_range(#shard{name = Dbs, node = node()}),
+    Tgt1 = add_range(#shard{name = Dbs, node = 'n2'}),
+    Tgt2 = add_range(#shard{name = Dbs, node = 'n3'}),
+    ?assertEqual(undefined, Src1#shard.range),
 
     % We're the only node: don't expect any other checkpoints
     meck:expect(mem3_sync, find_next_node, 0, node()),
diff --git a/src/mem3/test/eunit/mem3_rep_test.erl 
b/src/mem3/test/eunit/mem3_rep_test.erl
index 321230ce4..ec06d90ec 100644
--- a/src/mem3/test/eunit/mem3_rep_test.erl
+++ b/src/mem3/test/eunit/mem3_rep_test.erl
@@ -74,7 +74,8 @@ mem3_reshard_db_test_() ->
                     ?TDEF_FE(replicate_with_partitions, ?TIMEOUT),
                     ?TDEF_FE(replicate_to_and_from_local, ?TIMEOUT),
                     ?TDEF_FE(replicate_with_purges, ?TIMEOUT),
-                    ?TDEF_FE(clean_purge_checkpoints, ?TIMEOUT)
+                    ?TDEF_FE(clean_purge_checkpoints, ?TIMEOUT),
+                    ?TDEF_FE(pull_purges_checkpoints, ?TIMEOUT)
                 ]
             }
         }
@@ -287,6 +288,62 @@ clean_purge_checkpoints(#{allsrc := AllSrc, alltgt := 
AllTgt}) ->
     ?assertNot(is_map_key(Id3, LocalDocs)),
     ?assertNot(is_map_key(Id4, LocalDocs)).
 
+pull_purges_checkpoints(#{allsrc := AllSrc, alltgt := AllTgt}) ->
+    % Purge on the target, source has to pull purges both targets hten
+    add_test_docs(AllTgt, #{docs => 10, purge => [0, 9]}),
+    [Src] = lists:sort(mem3:local_shards(AllSrc)),
+    [Tgt1, Tgt2] = lists:sort(mem3:local_shards(AllTgt)),
+    #shard{dbname = SrcDbName, range = SrcRange} = Src,
+    #shard{dbname = TgtDbName, range = R1} = Tgt1,
+    #shard{range = R2} = Tgt2,
+
+    % Fix up shard map as if source and target shards are copies of the same
+    % db. Then on each target the expected purge checkpoint is the same as the
+    % source
+    meck:expect(mem3, shards, fun(DbName) ->
+        case lists:member(DbName, [SrcDbName, TgtDbName]) of
+            true -> [Src, Tgt1, Tgt2];
+            false -> meck:passthrough([DbName])
+        end
+    end),
+    TgtNames = [Tgt1#shard.name, Tgt2#shard.name],
+    lists:foreach(
+        fun(TgtName) ->
+            % Targets should have some purges for source to pull
+            ?assert(purge_seq(TgtName) > 0),
+            ?assertNot(mem3_rep:have_all_purge_checkpoints(TgtName))
+        end,
+        TgtNames
+    ),
+
+    TMap = #{R1 => Tgt1, R2 => Tgt2},
+    Opts = [{batch_size, 1000}, {batch_count, all}],
+    ?assertMatch({ok, 0}, mem3_rep:go(Src, TMap, Opts)),
+    ?assertEqual(#{}, get_all_docs(AllSrc)),
+
+    % Source checkpoints pulls on the target. Those should look the same as if
+    % the target pushed to the source.
+    SrcUuid = uuid(Src#shard.name),
+    Node = atom_to_binary(Src#shard.node),
+    lists:foreach(
+        fun(TgtName) ->
+            TgtUuid = uuid(TgtName),
+            DocId = <<"_local/purge-mem3-", TgtUuid/binary, "-", 
SrcUuid/binary>>,
+            #{DocId := Body} = local_docs(TgtName),
+            ?assertMatch(
+                #{
+                    ~"source" := Node,
+                    ~"target" := Node,
+                    ~"range" := SrcRange,
+                    ~"purge_seq" := PSeq
+                } when PSeq > 0,
+                Body
+            ),
+            ?assert(mem3_rep:have_all_purge_checkpoints(TgtName))
+        end,
+        TgtNames
+    ).
+
 replicate_to_and_from_local(#{localdb := LocalDb, allsrc := ClusteredDb}) ->
     % We'll just tests that we can pull purges from the target
     add_test_docs(ClusteredDb, #{docs => 6, purge => [0, 2]}),
@@ -496,6 +553,12 @@ atts(Size) when is_integer(Size), Size >= 1 ->
         ])
     ].
 
+uuid(DbName) ->
+    couch_util:with_db(DbName, fun couch_db:get_uuid/1).
+
+purge_seq(DbName) ->
+    couch_util:with_db(DbName, fun couch_db:get_purge_seq/1).
+
 local_docs(DbName) ->
     {ok, Db} = couch_db:open(DbName, [?ADMIN_CTX]),
     FoldFun = fun(#doc{id = DocId, body = Body}, Acc) ->

Reply via email to