This is an automated email from the ASF dual-hosted git repository. nickva pushed a commit to branch fix-mrview-non-leaf-purges in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 98c983153556bbaf09e408a8fa21980df2d30e4e Author: Nick Vatamaniuc <[email protected]> AuthorDate: Tue Sep 29 17:52:34 2026 -0400 Fix purging non-leaf revisions in map-reduce views Previously, when we purged a non-leaf revision (either a parent revision or a completely unknown one) we incorrectly removed the still possibly active live index entry from that document. To fix it, check for doc's existence before purging. A doc exists if it's either unchanged (a bogus or non-leaf was purged) or a leaf was purged. If a leaf was purged we would have relabeled the leafs with new update sequences we would be possibly updating this doc's entry in the index via the normal changes feed driven update process. After adding tests also clean up the tests to use the TDEF_FE macro and sigils for binaries. Fix: https://github.com/apache/couchdb/issues/6139 --- src/couch_mrview/src/couch_mrview_updater.erl | 11 +- .../test/eunit/couch_mrview_purge_docs_tests.erl | 1025 +++++++++++--------- 2 files changed, 583 insertions(+), 453 deletions(-) diff --git a/src/couch_mrview/src/couch_mrview_updater.erl b/src/couch_mrview/src/couch_mrview_updater.erl index d642ceb5c..79d9fdc25 100644 --- a/src/couch_mrview/src/couch_mrview_updater.erl +++ b/src/couch_mrview/src/couch_mrview_updater.erl @@ -69,14 +69,21 @@ start_update(Partial, State, NumChanges, NumChangesDone) -> {ok, InitState}. -purge(_Db, PurgeSeq, PurgedIdRevs, State) -> +purge(Db, PurgeSeq, PurgedIdRevs, State) -> #mrst{ id_btree = IdBtree, views = Views, partitioned = Partitioned } = State, - Ids = [Id || {Id, _Revs} <- PurgedIdRevs], + % We cannot remove rows if there are still FDIs present in the db. That + % means the FDI was either not changed since the index saw it (non-leaf or + % a bogus revision was purged) or it changed after that when we relabel the + % leafs with update sequences, and then it will be in the regular changes + % feed for the index. + Ids0 = [Id || {Id, _Revs} <- PurgedIdRevs], + FDIs = couch_db:get_full_doc_infos(Db, Ids0), + Ids = [Id || {Id, not_found} <- lists:zip(Ids0, FDIs)], {ok, Lookups, IdBtree2} = couch_btree:query_modify(IdBtree, Ids, [], Ids), MakeDictFun = fun diff --git a/src/couch_mrview/test/eunit/couch_mrview_purge_docs_tests.erl b/src/couch_mrview/test/eunit/couch_mrview_purge_docs_tests.erl index c4ac9ed13..bda1ae5cc 100644 --- a/src/couch_mrview/test/eunit/couch_mrview_purge_docs_tests.erl +++ b/src/couch_mrview/test/eunit/couch_mrview_purge_docs_tests.erl @@ -41,490 +41,613 @@ view_purge_test_() -> fun setup/0, fun teardown/1, [ - fun test_purge_single/1, - fun test_purge_partial/1, - fun test_purge_complete/1, - fun test_purge_nochange/1, - fun test_purge_index_reset/1, - fun test_purge_compact_size_check/1, - fun test_purge_single_for_docid_with_list/1, - fun test_purge_complete_for_docid_with_list/1, - fun test_purge_compact_for_stale_purge_cp_without_client/1, - fun test_purge_compact_for_stale_purge_cp_with_client/1 + ?TDEF_FE(test_purge_single), + ?TDEF_FE(test_purge_partial), + ?TDEF_FE(test_purge_complete), + ?TDEF_FE(test_purge_nochange), + ?TDEF_FE(test_purge_nochange_non_leaf_rev), + ?TDEF_FE(test_purge_nochange_unknown_rev), + ?TDEF_FE(test_purge_partial_indexed_winner), + ?TDEF_FE(test_purge_recreate_indexed), + ?TDEF_FE(test_purge_index_reset), + ?TDEF_FE(test_purge_compact_size_check), + ?TDEF_FE(test_purge_single_for_docid_with_list), + ?TDEF_FE(test_purge_complete_for_docid_with_list), + ?TDEF_FE(test_purge_compact_for_stale_purge_cp_without_client), + ?TDEF_FE(test_purge_compact_for_stale_purge_cp_with_client) ] } } }. test_purge_single(Db) -> - ?_test(begin - Result = run_query(Db, []), - Expect = - {ok, [ - {meta, [{total, 5}, {offset, 0}]}, - {row, [{id, <<"1">>}, {key, 1}, {value, 1}]}, - {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, - {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, - {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, - {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} - ]}, - ?assertEqual(Expect, Result), - - FDI = couch_db:get_full_doc_info(Db, <<"1">>), - Rev = get_rev(FDI), - {ok, [{ok, _PRevs}]} = couch_db:purge_docs( - Db, - [{<<"UUID1">>, <<"1">>, [Rev]}] - ), - {ok, Db2} = couch_db:reopen(Db), - - Result2 = run_query(Db2, []), - Expect2 = - {ok, [ - {meta, [{total, 4}, {offset, 0}]}, - {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, - {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, - {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, - {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} - ]}, - ?assertEqual(Expect2, Result2) - end). + Result = run_query(Db, []), + Expect = + {ok, [ + {meta, [{total, 5}, {offset, 0}]}, + {row, [{id, ~"1"}, {key, 1}, {value, 1}]}, + {row, [{id, ~"2"}, {key, 2}, {value, 2}]}, + {row, [{id, ~"3"}, {key, 3}, {value, 3}]}, + {row, [{id, ~"4"}, {key, 4}, {value, 4}]}, + {row, [{id, ~"5"}, {key, 5}, {value, 5}]} + ]}, + ?assertEqual(Expect, Result), + + FDI = couch_db:get_full_doc_info(Db, ~"1"), + Rev = get_rev(FDI), + {ok, [{ok, _PRevs}]} = couch_db:purge_docs( + Db, + [{~"UUID1", ~"1", [Rev]}] + ), + {ok, Db2} = couch_db:reopen(Db), + + Result2 = run_query(Db2, []), + Expect2 = + {ok, [ + {meta, [{total, 4}, {offset, 0}]}, + {row, [{id, ~"2"}, {key, 2}, {value, 2}]}, + {row, [{id, ~"3"}, {key, 3}, {value, 3}]}, + {row, [{id, ~"4"}, {key, 4}, {value, 4}]}, + {row, [{id, ~"5"}, {key, 5}, {value, 5}]} + ]}, + ?assertEqual(Expect2, Result2). test_purge_single_for_docid_with_list(Db) -> - ?_test(begin - Result = run_query(Db, []), - Expect = - {ok, [ - {meta, [{total, 5}, {offset, 0}]}, - {row, [{id, <<"1">>}, {key, 1}, {value, 1}]}, - {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, - {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, - {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, - {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} - ]}, - ?assertEqual(Expect, Result), - - FDI = couch_db:get_full_doc_info(Db, <<"1">>), - Rev = get_rev(FDI), - {ok, [{ok, _PRevs}]} = couch_db:purge_docs( - Db, - [{<<"UUID1">>, "1", [Rev]}] - ), - {ok, Db2} = couch_db:reopen(Db), - - Result2 = run_query(Db2, []), - Expect2 = - {ok, [ - {meta, [{total, 4}, {offset, 0}]}, - {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, - {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, - {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, - {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} - ]}, - ?assertEqual(Expect2, Result2) - end). + Result = run_query(Db, []), + Expect = + {ok, [ + {meta, [{total, 5}, {offset, 0}]}, + {row, [{id, ~"1"}, {key, 1}, {value, 1}]}, + {row, [{id, ~"2"}, {key, 2}, {value, 2}]}, + {row, [{id, ~"3"}, {key, 3}, {value, 3}]}, + {row, [{id, ~"4"}, {key, 4}, {value, 4}]}, + {row, [{id, ~"5"}, {key, 5}, {value, 5}]} + ]}, + ?assertEqual(Expect, Result), + + FDI = couch_db:get_full_doc_info(Db, ~"1"), + Rev = get_rev(FDI), + {ok, [{ok, _PRevs}]} = couch_db:purge_docs( + Db, + [{~"UUID1", "1", [Rev]}] + ), + {ok, Db2} = couch_db:reopen(Db), + + Result2 = run_query(Db2, []), + Expect2 = + {ok, [ + {meta, [{total, 4}, {offset, 0}]}, + {row, [{id, ~"2"}, {key, 2}, {value, 2}]}, + {row, [{id, ~"3"}, {key, 3}, {value, 3}]}, + {row, [{id, ~"4"}, {key, 4}, {value, 4}]}, + {row, [{id, ~"5"}, {key, 5}, {value, 5}]} + ]}, + ?assertEqual(Expect2, Result2). test_purge_partial(Db) -> - ?_test(begin - Result = run_query(Db, []), - Expect = - {ok, [ - {meta, [{total, 5}, {offset, 0}]}, - {row, [{id, <<"1">>}, {key, 1}, {value, 1}]}, - {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, - {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, - {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, - {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} - ]}, - ?assertEqual(Expect, Result), - - FDI1 = couch_db:get_full_doc_info(Db, <<"1">>), - Rev1 = get_rev(FDI1), - Update = - {[ - {'_id', <<"1">>}, - {'_rev', couch_doc:rev_to_str({1, [couch_hash:md5_hash(<<"1.2">>)]})}, - {'val', 1.2} - ]}, - {ok, [_Rev2]} = save_docs(Db, [Update], [?REPLICATED_CHANGES]), - - PurgeInfos = [{<<"UUID1">>, <<"1">>, [Rev1]}], - - {ok, _} = couch_db:purge_docs(Db, PurgeInfos), - {ok, Db2} = couch_db:reopen(Db), - - Result2 = run_query(Db2, []), - Expect2 = - {ok, [ - {meta, [{total, 5}, {offset, 0}]}, - {row, [{id, <<"1">>}, {key, 1.2}, {value, 1.2}]}, - {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, - {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, - {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, - {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} - ]}, - ?assertEqual(Expect2, Result2) - end). + Result = run_query(Db, []), + Expect = + {ok, [ + {meta, [{total, 5}, {offset, 0}]}, + {row, [{id, ~"1"}, {key, 1}, {value, 1}]}, + {row, [{id, ~"2"}, {key, 2}, {value, 2}]}, + {row, [{id, ~"3"}, {key, 3}, {value, 3}]}, + {row, [{id, ~"4"}, {key, 4}, {value, 4}]}, + {row, [{id, ~"5"}, {key, 5}, {value, 5}]} + ]}, + ?assertEqual(Expect, Result), + + FDI1 = couch_db:get_full_doc_info(Db, ~"1"), + Rev1 = get_rev(FDI1), + Update = + {[ + {'_id', ~"1"}, + {'_rev', couch_doc:rev_to_str({1, [couch_hash:md5_hash(~"1.2")]})}, + {'val', 1.2} + ]}, + {ok, [_Rev2]} = save_docs(Db, [Update], [?REPLICATED_CHANGES]), + + PurgeInfos = [{~"UUID1", ~"1", [Rev1]}], + + {ok, _} = couch_db:purge_docs(Db, PurgeInfos), + {ok, Db2} = couch_db:reopen(Db), + + Result2 = run_query(Db2, []), + Expect2 = + {ok, [ + {meta, [{total, 5}, {offset, 0}]}, + {row, [{id, ~"1"}, {key, 1.2}, {value, 1.2}]}, + {row, [{id, ~"2"}, {key, 2}, {value, 2}]}, + {row, [{id, ~"3"}, {key, 3}, {value, 3}]}, + {row, [{id, ~"4"}, {key, 4}, {value, 4}]}, + {row, [{id, ~"5"}, {key, 5}, {value, 5}]} + ]}, + ?assertEqual(Expect2, Result2). test_purge_complete(Db) -> - ?_test(begin - Result = run_query(Db, []), - Expect = - {ok, [ - {meta, [{total, 5}, {offset, 0}]}, - {row, [{id, <<"1">>}, {key, 1}, {value, 1}]}, - {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, - {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, - {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, - {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} - ]}, - ?assertEqual(Expect, Result), - - FDI1 = couch_db:get_full_doc_info(Db, <<"1">>), - Rev1 = get_rev(FDI1), - FDI2 = couch_db:get_full_doc_info(Db, <<"2">>), - Rev2 = get_rev(FDI2), - FDI5 = couch_db:get_full_doc_info(Db, <<"5">>), - Rev5 = get_rev(FDI5), - - PurgeInfos = [ - {<<"UUID1">>, <<"1">>, [Rev1]}, - {<<"UUID2">>, <<"2">>, [Rev2]}, - {<<"UUID5">>, <<"5">>, [Rev5]} - ], - {ok, _} = couch_db:purge_docs(Db, PurgeInfos), - {ok, Db2} = couch_db:reopen(Db), - - Result2 = run_query(Db2, []), - Expect2 = - {ok, [ - {meta, [{total, 2}, {offset, 0}]}, - {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, - {row, [{id, <<"4">>}, {key, 4}, {value, 4}]} - ]}, - ?assertEqual(Expect2, Result2) - end). + Result = run_query(Db, []), + Expect = + {ok, [ + {meta, [{total, 5}, {offset, 0}]}, + {row, [{id, ~"1"}, {key, 1}, {value, 1}]}, + {row, [{id, ~"2"}, {key, 2}, {value, 2}]}, + {row, [{id, ~"3"}, {key, 3}, {value, 3}]}, + {row, [{id, ~"4"}, {key, 4}, {value, 4}]}, + {row, [{id, ~"5"}, {key, 5}, {value, 5}]} + ]}, + ?assertEqual(Expect, Result), + + FDI1 = couch_db:get_full_doc_info(Db, ~"1"), + Rev1 = get_rev(FDI1), + FDI2 = couch_db:get_full_doc_info(Db, ~"2"), + Rev2 = get_rev(FDI2), + FDI5 = couch_db:get_full_doc_info(Db, ~"5"), + Rev5 = get_rev(FDI5), + + PurgeInfos = [ + {~"UUID1", ~"1", [Rev1]}, + {~"UUID2", ~"2", [Rev2]}, + {~"UUID5", ~"5", [Rev5]} + ], + {ok, _} = couch_db:purge_docs(Db, PurgeInfos), + {ok, Db2} = couch_db:reopen(Db), + + Result2 = run_query(Db2, []), + Expect2 = + {ok, [ + {meta, [{total, 2}, {offset, 0}]}, + {row, [{id, ~"3"}, {key, 3}, {value, 3}]}, + {row, [{id, ~"4"}, {key, 4}, {value, 4}]} + ]}, + ?assertEqual(Expect2, Result2). test_purge_complete_for_docid_with_list(Db) -> - ?_test(begin - Result = run_query(Db, []), - Expect = - {ok, [ - {meta, [{total, 5}, {offset, 0}]}, - {row, [{id, <<"1">>}, {key, 1}, {value, 1}]}, - {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, - {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, - {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, - {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} - ]}, - ?assertEqual(Expect, Result), - - FDI1 = couch_db:get_full_doc_info(Db, <<"1">>), - Rev1 = get_rev(FDI1), - FDI2 = couch_db:get_full_doc_info(Db, <<"2">>), - Rev2 = get_rev(FDI2), - FDI5 = couch_db:get_full_doc_info(Db, <<"5">>), - Rev5 = get_rev(FDI5), - - PurgeInfos = [ - {<<"UUID1">>, "1", [Rev1]}, - {<<"UUID2">>, "2", [Rev2]}, - {<<"UUID5">>, "5", [Rev5]} - ], - {ok, _} = couch_db:purge_docs(Db, PurgeInfos), - {ok, Db2} = couch_db:reopen(Db), - - Result2 = run_query(Db2, []), - Expect2 = - {ok, [ - {meta, [{total, 2}, {offset, 0}]}, - {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, - {row, [{id, <<"4">>}, {key, 4}, {value, 4}]} - ]}, - ?assertEqual(Expect2, Result2) - end). + Result = run_query(Db, []), + Expect = + {ok, [ + {meta, [{total, 5}, {offset, 0}]}, + {row, [{id, ~"1"}, {key, 1}, {value, 1}]}, + {row, [{id, ~"2"}, {key, 2}, {value, 2}]}, + {row, [{id, ~"3"}, {key, 3}, {value, 3}]}, + {row, [{id, ~"4"}, {key, 4}, {value, 4}]}, + {row, [{id, ~"5"}, {key, 5}, {value, 5}]} + ]}, + ?assertEqual(Expect, Result), + + FDI1 = couch_db:get_full_doc_info(Db, ~"1"), + Rev1 = get_rev(FDI1), + FDI2 = couch_db:get_full_doc_info(Db, ~"2"), + Rev2 = get_rev(FDI2), + FDI5 = couch_db:get_full_doc_info(Db, ~"5"), + Rev5 = get_rev(FDI5), + + PurgeInfos = [ + {~"UUID1", "1", [Rev1]}, + {~"UUID2", "2", [Rev2]}, + {~"UUID5", "5", [Rev5]} + ], + {ok, _} = couch_db:purge_docs(Db, PurgeInfos), + {ok, Db2} = couch_db:reopen(Db), + + Result2 = run_query(Db2, []), + Expect2 = + {ok, [ + {meta, [{total, 2}, {offset, 0}]}, + {row, [{id, ~"3"}, {key, 3}, {value, 3}]}, + {row, [{id, ~"4"}, {key, 4}, {value, 4}]} + ]}, + ?assertEqual(Expect2, Result2). test_purge_nochange(Db) -> - ?_test(begin - Result = run_query(Db, []), - Expect = - {ok, [ - {meta, [{total, 5}, {offset, 0}]}, - {row, [{id, <<"1">>}, {key, 1}, {value, 1}]}, - {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, - {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, - {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, - {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} - ]}, - ?assertEqual(Expect, Result), - - FDI1 = couch_db:get_full_doc_info(Db, <<"1">>), - Rev1 = get_rev(FDI1), - - PurgeInfos = [ - {<<"UUID1">>, <<"6">>, [Rev1]} - ], - {ok, _} = couch_db:purge_docs(Db, PurgeInfos), - {ok, Db2} = couch_db:reopen(Db), - - Result2 = run_query(Db2, []), - Expect2 = - {ok, [ - {meta, [{total, 5}, {offset, 0}]}, - {row, [{id, <<"1">>}, {key, 1}, {value, 1}]}, - {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, - {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, - {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, - {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} - ]}, - ?assertEqual(Expect2, Result2) - end). + Result = run_query(Db, []), + Expect = + {ok, [ + {meta, [{total, 5}, {offset, 0}]}, + {row, [{id, ~"1"}, {key, 1}, {value, 1}]}, + {row, [{id, ~"2"}, {key, 2}, {value, 2}]}, + {row, [{id, ~"3"}, {key, 3}, {value, 3}]}, + {row, [{id, ~"4"}, {key, 4}, {value, 4}]}, + {row, [{id, ~"5"}, {key, 5}, {value, 5}]} + ]}, + ?assertEqual(Expect, Result), + + FDI1 = couch_db:get_full_doc_info(Db, ~"1"), + Rev1 = get_rev(FDI1), + + PurgeInfos = [ + {~"UUID1", ~"6", [Rev1]} + ], + {ok, _} = couch_db:purge_docs(Db, PurgeInfos), + {ok, Db2} = couch_db:reopen(Db), + + Result2 = run_query(Db2, []), + Expect2 = + {ok, [ + {meta, [{total, 5}, {offset, 0}]}, + {row, [{id, ~"1"}, {key, 1}, {value, 1}]}, + {row, [{id, ~"2"}, {key, 2}, {value, 2}]}, + {row, [{id, ~"3"}, {key, 3}, {value, 3}]}, + {row, [{id, ~"4"}, {key, 4}, {value, 4}]}, + {row, [{id, ~"5"}, {key, 5}, {value, 5}]} + ]}, + ?assertEqual(Expect2, Result2). + +test_purge_nochange_non_leaf_rev(Db) -> + FDI1 = couch_db:get_full_doc_info(Db, ~"1"), + Rev1 = get_rev(FDI1), + Update = + {[ + {'_id', ~"1"}, + {'_rev', couch_doc:rev_to_str(Rev1)}, + {'val', 42} + ]}, + {ok, [_Rev2]} = save_docs(Db, [Update], []), + {ok, Db1} = couch_db:reopen(Db), + + Result = run_query(Db1, []), + Expect = + {ok, [ + {meta, [{total, 5}, {offset, 0}]}, + {row, [{id, ~"2"}, {key, 2}, {value, 2}]}, + {row, [{id, ~"3"}, {key, 3}, {value, 3}]}, + {row, [{id, ~"4"}, {key, 4}, {value, 4}]}, + {row, [{id, ~"5"}, {key, 5}, {value, 5}]}, + {row, [{id, ~"1"}, {key, 42}, {value, 42}]} + ]}, + ?assertEqual(Expect, Result), + + % Rev1 is not a leaf any longer so nothing is purged, but the + % purge request is still recorded and the index has to process it + PurgeInfos = [{~"UUID1", ~"1", [Rev1]}], + ?assertEqual({ok, [{ok, []}]}, couch_db:purge_docs(Db1, PurgeInfos)), + {ok, Db2} = couch_db:reopen(Db1), + ?assertEqual(1, couch_db:get_purge_seq(Db2)), + + Result2 = run_query(Db2, []), + ?assertEqual(Expect, Result2), + ?assertEqual(1, get_local_purge_seq(Db2)). + +test_purge_nochange_unknown_rev(Db) -> + Result = run_query(Db, []), + Expect = + {ok, [ + {meta, [{total, 5}, {offset, 0}]}, + {row, [{id, ~"1"}, {key, 1}, {value, 1}]}, + {row, [{id, ~"2"}, {key, 2}, {value, 2}]}, + {row, [{id, ~"3"}, {key, 3}, {value, 3}]}, + {row, [{id, ~"4"}, {key, 4}, {value, 4}]}, + {row, [{id, ~"5"}, {key, 5}, {value, 5}]} + ]}, + ?assertEqual(Expect, Result), + + % A revision which never existed, so nothing is purged + UnknownRev = {1, couch_hash:md5_hash(~"unknown")}, + PurgeInfos = [{~"UUID1", ~"1", [UnknownRev]}], + ?assertEqual({ok, [{ok, []}]}, couch_db:purge_docs(Db, PurgeInfos)), + {ok, Db2} = couch_db:reopen(Db), + ?assertEqual(1, couch_db:get_purge_seq(Db2)), + + Result2 = run_query(Db2, []), + ?assertEqual(Expect, Result2), + ?assertEqual(1, get_local_purge_seq(Db2)). + +test_purge_partial_indexed_winner(Db) -> + % Add a longer conflicting branch so it becomes the winner + WinningRev = {2, couch_hash:md5_hash(~"1.2-2")}, + Conflict = #doc{ + id = ~"1", + revs = {2, [element(2, WinningRev), couch_hash:md5_hash(~"1.2-1")]}, + body = {[{~"val", 1.2}]} + }, + {ok, []} = couch_db:update_docs(Db, [Conflict], [], ?REPLICATED_CHANGES), + {ok, Db1} = couch_db:reopen(Db), + + Result = run_query(Db1, []), + Expect = + {ok, [ + {meta, [{total, 5}, {offset, 0}]}, + {row, [{id, ~"1"}, {key, 1.2}, {value, 1.2}]}, + {row, [{id, ~"2"}, {key, 2}, {value, 2}]}, + {row, [{id, ~"3"}, {key, 3}, {value, 3}]}, + {row, [{id, ~"4"}, {key, 4}, {value, 4}]}, + {row, [{id, ~"5"}, {key, 5}, {value, 5}]} + ]}, + ?assertEqual(Expect, Result), + + % Purge the indexed winner so the original revision wins again + PurgeInfos = [{~"UUID1", ~"1", [WinningRev]}], + ?assertEqual({ok, [{ok, [WinningRev]}]}, couch_db:purge_docs(Db1, PurgeInfos)), + {ok, Db2} = couch_db:reopen(Db1), + + Result2 = run_query(Db2, []), + Expect2 = + {ok, [ + {meta, [{total, 5}, {offset, 0}]}, + {row, [{id, ~"1"}, {key, 1}, {value, 1}]}, + {row, [{id, ~"2"}, {key, 2}, {value, 2}]}, + {row, [{id, ~"3"}, {key, 3}, {value, 3}]}, + {row, [{id, ~"4"}, {key, 4}, {value, 4}]}, + {row, [{id, ~"5"}, {key, 5}, {value, 5}]} + ]}, + ?assertEqual(Expect2, Result2). + +test_purge_recreate_indexed(Db) -> + Result = run_query(Db, []), + Expect = + {ok, [ + {meta, [{total, 5}, {offset, 0}]}, + {row, [{id, ~"1"}, {key, 1}, {value, 1}]}, + {row, [{id, ~"2"}, {key, 2}, {value, 2}]}, + {row, [{id, ~"3"}, {key, 3}, {value, 3}]}, + {row, [{id, ~"4"}, {key, 4}, {value, 4}]}, + {row, [{id, ~"5"}, {key, 5}, {value, 5}]} + ]}, + ?assertEqual(Expect, Result), + + FDI1 = couch_db:get_full_doc_info(Db, ~"1"), + Rev1 = get_rev(FDI1), + PurgeInfos = [{~"UUID1", ~"1", [Rev1]}], + ?assertEqual({ok, [{ok, [Rev1]}]}, couch_db:purge_docs(Db, PurgeInfos)), + {ok, Db1} = couch_db:reopen(Db), + + % Recreate doc before index processes the purge + {ok, [_]} = save_docs(Db1, [{[{'_id', ~"1"}, {'val', 42}]}], []), + {ok, Db2} = couch_db:reopen(Db1), + + Result2 = run_query(Db2, []), + Expect2 = + {ok, [ + {meta, [{total, 5}, {offset, 0}]}, + {row, [{id, ~"2"}, {key, 2}, {value, 2}]}, + {row, [{id, ~"3"}, {key, 3}, {value, 3}]}, + {row, [{id, ~"4"}, {key, 4}, {value, 4}]}, + {row, [{id, ~"5"}, {key, 5}, {value, 5}]}, + {row, [{id, ~"1"}, {key, 42}, {value, 42}]} + ]}, + ?assertEqual(Expect2, Result2). test_purge_index_reset(Db) -> - ?_test(begin - ok = couch_db:set_purge_infos_limit(Db, 2), - {ok, Db1} = couch_db:reopen(Db), - - Result = run_query(Db1, []), - Expect = - {ok, [ - {meta, [{total, 5}, {offset, 0}]}, - {row, [{id, <<"1">>}, {key, 1}, {value, 1}]}, - {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, - {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, - {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, - {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} - ]}, - ?assertEqual(Expect, Result), - - PurgeInfos = lists:map( - fun(I) -> - DocId = integer_to_binary(I), - FDI = couch_db:get_full_doc_info(Db, DocId), - Rev = get_rev(FDI), - {couch_uuids:random(), DocId, [Rev]} - end, - lists:seq(1, 5) - ), - {ok, _} = couch_db:purge_docs(Db1, PurgeInfos), - - {ok, Db2} = couch_db:reopen(Db1), - - % Forcibly set the purge doc to a newer purge - % sequence to force an index reset. This should - % never happen in real life but the reset - % is required for correctness. - {ok, #doc{body = {OldProps}} = LocalDoc} = get_local_purge_doc(Db2), - NewPurgeSeq = {<<"purge_seq">>, 5}, - NewProps = lists:keyreplace(<<"purge_seq">>, 1, OldProps, NewPurgeSeq), - RewindDoc = LocalDoc#doc{body = {NewProps}}, - {ok, _} = couch_db:update_doc(Db2, RewindDoc, []), - - % Compact the database to remove purge infos - {ok, _} = couch_db:start_compact(Db2), - wait_compaction(couch_db:name(Db), "database", ?LINE), - - {ok, Db3} = couch_db:reopen(Db2), - Result2 = run_query(Db3, []), - Expect2 = - {ok, [ - {meta, [{total, 0}, {offset, 0}]} - ]}, - ?assertEqual(Expect2, Result2), - - % Assert that we had a reset - meck:wait( - 1, - couch_index_updater, - handle_info, - [{'EXIT', '_', {reset, '_'}}, '_'], - 5000 - ) - end). + ok = couch_db:set_purge_infos_limit(Db, 2), + {ok, Db1} = couch_db:reopen(Db), + + Result = run_query(Db1, []), + Expect = + {ok, [ + {meta, [{total, 5}, {offset, 0}]}, + {row, [{id, ~"1"}, {key, 1}, {value, 1}]}, + {row, [{id, ~"2"}, {key, 2}, {value, 2}]}, + {row, [{id, ~"3"}, {key, 3}, {value, 3}]}, + {row, [{id, ~"4"}, {key, 4}, {value, 4}]}, + {row, [{id, ~"5"}, {key, 5}, {value, 5}]} + ]}, + ?assertEqual(Expect, Result), + + PurgeInfos = lists:map( + fun(I) -> + DocId = integer_to_binary(I), + FDI = couch_db:get_full_doc_info(Db, DocId), + Rev = get_rev(FDI), + {couch_uuids:random(), DocId, [Rev]} + end, + lists:seq(1, 5) + ), + {ok, _} = couch_db:purge_docs(Db1, PurgeInfos), + + {ok, Db2} = couch_db:reopen(Db1), + + % Forcibly set the purge doc to a newer purge + % sequence to force an index reset. This should + % never happen in real life but the reset + % is required for correctness. + {ok, #doc{body = {OldProps}} = LocalDoc} = get_local_purge_doc(Db2), + NewPurgeSeq = {~"purge_seq", 5}, + NewProps = lists:keyreplace(~"purge_seq", 1, OldProps, NewPurgeSeq), + RewindDoc = LocalDoc#doc{body = {NewProps}}, + {ok, _} = couch_db:update_doc(Db2, RewindDoc, []), + + % Compact the database to remove purge infos + {ok, _} = couch_db:start_compact(Db2), + wait_compaction(couch_db:name(Db), "database", ?LINE), + + {ok, Db3} = couch_db:reopen(Db2), + Result2 = run_query(Db3, []), + Expect2 = + {ok, [ + {meta, [{total, 0}, {offset, 0}]} + ]}, + ?assertEqual(Expect2, Result2), + + % Assert that we had a reset + meck:wait( + 1, + couch_index_updater, + handle_info, + [{'EXIT', '_', {reset, '_'}}, '_'], + 5000 + ). test_purge_compact_size_check(Db) -> - ?_test(begin - DbName = couch_db:name(Db), - Docs = couch_mrview_test_util:make_docs(normal, 6, 200), - {ok, Db1} = couch_mrview_test_util:save_docs(Db, Docs), - _Result = run_query(Db1, []), - DiskSizeBefore = db_disk_size(DbName), - - PurgedDocsNum = 150, - IdsRevs = lists:foldl( - fun(Id, CIdRevs) -> - Id1 = docid(Id), - FDI1 = couch_db:get_full_doc_info(Db1, Id1), - Rev1 = get_rev(FDI1), - UUID1 = uuid(Id), - [{UUID1, Id1, [Rev1]} | CIdRevs] - end, - [], - lists:seq(1, PurgedDocsNum) - ), - {ok, _} = couch_db:purge_docs(Db1, IdsRevs), - - {ok, Db2} = couch_db:reopen(Db1), - _Result1 = run_query(Db2, []), - {ok, PurgedIdRevs} = couch_db:fold_purge_infos( - Db2, - 0, - fun fold_fun/2, - [], - [] - ), - ?assertEqual(PurgedDocsNum, length(PurgedIdRevs)), - config:set("couchdb", "file_compression", "snappy", false), - - {ok, Db3} = couch_db:open_int(DbName, []), - {ok, _CompactPid} = couch_db:start_compact(Db3), - wait_compaction(DbName, "database", ?LINE), - ok = couch_db:close(Db3), - DiskSizeAfter = db_disk_size(DbName), - ?assert(DiskSizeBefore > DiskSizeAfter) - end). + DbName = couch_db:name(Db), + Docs = couch_mrview_test_util:make_docs(normal, 6, 200), + {ok, Db1} = couch_mrview_test_util:save_docs(Db, Docs), + _Result = run_query(Db1, []), + DiskSizeBefore = db_disk_size(DbName), + + PurgedDocsNum = 150, + IdsRevs = lists:foldl( + fun(Id, CIdRevs) -> + Id1 = docid(Id), + FDI1 = couch_db:get_full_doc_info(Db1, Id1), + Rev1 = get_rev(FDI1), + UUID1 = uuid(Id), + [{UUID1, Id1, [Rev1]} | CIdRevs] + end, + [], + lists:seq(1, PurgedDocsNum) + ), + {ok, _} = couch_db:purge_docs(Db1, IdsRevs), + + {ok, Db2} = couch_db:reopen(Db1), + _Result1 = run_query(Db2, []), + {ok, PurgedIdRevs} = couch_db:fold_purge_infos( + Db2, + 0, + fun fold_fun/2, + [], + [] + ), + ?assertEqual(PurgedDocsNum, length(PurgedIdRevs)), + config:set("couchdb", "file_compression", "snappy", false), + + {ok, Db3} = couch_db:open_int(DbName, []), + {ok, _CompactPid} = couch_db:start_compact(Db3), + wait_compaction(DbName, "database", ?LINE), + ok = couch_db:close(Db3), + DiskSizeAfter = db_disk_size(DbName), + ?assert(DiskSizeBefore > DiskSizeAfter). test_purge_compact_for_stale_purge_cp_without_client(Db) -> - ?_test(begin - DbName = couch_db:name(Db), - % add more documents to database for purge - Docs = couch_mrview_test_util:make_docs(normal, 6, 200), - {ok, Db1} = couch_mrview_test_util:save_docs(Db, Docs), - - % change PurgedDocsLimit to 10 from 1000 to - % avoid timeout of eunit test - PurgedDocsLimit = 10, - couch_db:set_purge_infos_limit(Db1, PurgedDocsLimit), - - % purge 150 documents - PurgedDocsNum = 150, - PurgeInfos = lists:foldl( - fun(Id, CIdRevs) -> - Id1 = docid(Id), - FDI1 = couch_db:get_full_doc_info(Db1, Id1), - Rev1 = get_rev(FDI1), - UUID1 = uuid(Id), - [{UUID1, Id1, [Rev1]} | CIdRevs] - end, - [], - lists:seq(1, PurgedDocsNum) - ), - {ok, _} = couch_db:purge_docs(Db1, PurgeInfos), - - {ok, Db2} = couch_db:reopen(Db1), - {ok, PurgedIdRevs} = couch_db:fold_purge_infos( - Db2, - 0, - fun fold_fun/2, - [], - [] - ), - ?assertEqual(PurgedDocsNum, length(PurgedIdRevs)), - - % run compaction to trigger pruning of purge tree - {ok, Db3} = couch_db:open_int(DbName, []), - {ok, _CompactPid} = couch_db:start_compact(Db3), - wait_compaction(DbName, "database", ?LINE), - ok = couch_db:close(Db3), - - % check the remaining purge requests in purge tree - {ok, Db4} = couch_db:reopen(Db3), - OldestPSeq = couch_db:get_oldest_purge_seq(Db4), - {ok, PurgedIdRevs2} = couch_db:fold_purge_infos( - Db4, - OldestPSeq - 1, - fun fold_fun/2, - [], - [] - ), - ?assertEqual(PurgedDocsLimit, length(PurgedIdRevs2)) - end). + DbName = couch_db:name(Db), + % add more documents to database for purge + Docs = couch_mrview_test_util:make_docs(normal, 6, 200), + {ok, Db1} = couch_mrview_test_util:save_docs(Db, Docs), + + % change PurgedDocsLimit to 10 from 1000 to + % avoid timeout of eunit test + PurgedDocsLimit = 10, + couch_db:set_purge_infos_limit(Db1, PurgedDocsLimit), + + % purge 150 documents + PurgedDocsNum = 150, + PurgeInfos = lists:foldl( + fun(Id, CIdRevs) -> + Id1 = docid(Id), + FDI1 = couch_db:get_full_doc_info(Db1, Id1), + Rev1 = get_rev(FDI1), + UUID1 = uuid(Id), + [{UUID1, Id1, [Rev1]} | CIdRevs] + end, + [], + lists:seq(1, PurgedDocsNum) + ), + {ok, _} = couch_db:purge_docs(Db1, PurgeInfos), + + {ok, Db2} = couch_db:reopen(Db1), + {ok, PurgedIdRevs} = couch_db:fold_purge_infos( + Db2, + 0, + fun fold_fun/2, + [], + [] + ), + ?assertEqual(PurgedDocsNum, length(PurgedIdRevs)), + + % run compaction to trigger pruning of purge tree + {ok, Db3} = couch_db:open_int(DbName, []), + {ok, _CompactPid} = couch_db:start_compact(Db3), + wait_compaction(DbName, "database", ?LINE), + ok = couch_db:close(Db3), + + % check the remaining purge requests in purge tree + {ok, Db4} = couch_db:reopen(Db3), + OldestPSeq = couch_db:get_oldest_purge_seq(Db4), + {ok, PurgedIdRevs2} = couch_db:fold_purge_infos( + Db4, + OldestPSeq - 1, + fun fold_fun/2, + [], + [] + ), + ?assertEqual(PurgedDocsLimit, length(PurgedIdRevs2)). test_purge_compact_for_stale_purge_cp_with_client(Db) -> - ?_test(begin - DbName = couch_db:name(Db), - % add more documents to database for purge - Docs = couch_mrview_test_util:make_docs(normal, 6, 200), - {ok, Db1} = couch_mrview_test_util:save_docs(Db, Docs), - - % change PurgedDocsLimit to 10 from 1000 to - % avoid timeout of eunit test - PurgedDocsLimit = 10, - couch_db:set_purge_infos_limit(Db1, PurgedDocsLimit), - _Result = run_query(Db1, []), - - % first purge 30 documents - PurgedDocsNum1 = 30, - IdsRevs = lists:foldl( - fun(Id, CIdRevs) -> - Id1 = docid(Id), - FDI1 = couch_db:get_full_doc_info(Db1, Id1), - Rev1 = get_rev(FDI1), - UUID1 = uuid(Id), - [{UUID1, Id1, [Rev1]} | CIdRevs] - end, - [], - lists:seq(1, PurgedDocsNum1) - ), - {ok, _} = couch_db:purge_docs(Db1, IdsRevs), - - {ok, Db2} = couch_db:reopen(Db1), - % run query again to reflect purge request to mrview - _Result1 = run_query(Db2, []), - {ok, PurgedIdRevs} = couch_db:fold_purge_infos( - Db2, - 0, - fun fold_fun/2, - [], - [] - ), - ?assertEqual(PurgedDocsNum1, length(PurgedIdRevs)), - - % then purge 120 documents - PurgedDocsNum2 = 150, - IdsRevs2 = lists:foldl( - fun(Id, CIdRevs) -> - Id1 = docid(Id), - FDI1 = couch_db:get_full_doc_info(Db1, Id1), - Rev1 = get_rev(FDI1), - UUID1 = uuid(Id), - [{UUID1, Id1, [Rev1]} | CIdRevs] - end, - [], - lists:seq(PurgedDocsNum1 + 1, PurgedDocsNum2) - ), - {ok, _} = couch_db:purge_docs(Db2, IdsRevs2), - - % run compaction to trigger pruning of purge tree - % only the first 30 purge requests are pruned - {ok, Db3} = couch_db:open_int(DbName, []), - {ok, _CompactPid} = couch_db:start_compact(Db3), - wait_compaction(DbName, "database", ?LINE), - ok = couch_db:close(Db3), - - % check the remaining purge requests in purge tree - {ok, Db4} = couch_db:reopen(Db3), - OldestPSeq = couch_db:get_oldest_purge_seq(Db4), - {ok, PurgedIdRevs2} = couch_db:fold_purge_infos( - Db4, - OldestPSeq - 1, - fun fold_fun/2, - [], - [] - ), - ?assertEqual(PurgedDocsNum2 - PurgedDocsNum1, length(PurgedIdRevs2)) - end). + DbName = couch_db:name(Db), + % add more documents to database for purge + Docs = couch_mrview_test_util:make_docs(normal, 6, 200), + {ok, Db1} = couch_mrview_test_util:save_docs(Db, Docs), + + % change PurgedDocsLimit to 10 from 1000 to + % avoid timeout of eunit test + PurgedDocsLimit = 10, + couch_db:set_purge_infos_limit(Db1, PurgedDocsLimit), + _Result = run_query(Db1, []), + + % first purge 30 documents + PurgedDocsNum1 = 30, + IdsRevs = lists:foldl( + fun(Id, CIdRevs) -> + Id1 = docid(Id), + FDI1 = couch_db:get_full_doc_info(Db1, Id1), + Rev1 = get_rev(FDI1), + UUID1 = uuid(Id), + [{UUID1, Id1, [Rev1]} | CIdRevs] + end, + [], + lists:seq(1, PurgedDocsNum1) + ), + {ok, _} = couch_db:purge_docs(Db1, IdsRevs), + + {ok, Db2} = couch_db:reopen(Db1), + % run query again to reflect purge request to mrview + _Result1 = run_query(Db2, []), + {ok, PurgedIdRevs} = couch_db:fold_purge_infos( + Db2, + 0, + fun fold_fun/2, + [], + [] + ), + ?assertEqual(PurgedDocsNum1, length(PurgedIdRevs)), + + % then purge 120 documents + PurgedDocsNum2 = 150, + IdsRevs2 = lists:foldl( + fun(Id, CIdRevs) -> + Id1 = docid(Id), + FDI1 = couch_db:get_full_doc_info(Db1, Id1), + Rev1 = get_rev(FDI1), + UUID1 = uuid(Id), + [{UUID1, Id1, [Rev1]} | CIdRevs] + end, + [], + lists:seq(PurgedDocsNum1 + 1, PurgedDocsNum2) + ), + {ok, _} = couch_db:purge_docs(Db2, IdsRevs2), + + % run compaction to trigger pruning of purge tree + % only the first 30 purge requests are pruned + {ok, Db3} = couch_db:open_int(DbName, []), + {ok, _CompactPid} = couch_db:start_compact(Db3), + wait_compaction(DbName, "database", ?LINE), + ok = couch_db:close(Db3), + + % check the remaining purge requests in purge tree + {ok, Db4} = couch_db:reopen(Db3), + OldestPSeq = couch_db:get_oldest_purge_seq(Db4), + {ok, PurgedIdRevs2} = couch_db:fold_purge_infos( + Db4, + OldestPSeq - 1, + fun fold_fun/2, + [], + [] + ), + ?assertEqual(PurgedDocsNum2 - PurgedDocsNum1, length(PurgedIdRevs2)). get_local_purge_doc(Db) -> - {ok, DDoc} = couch_db:open_doc(Db, <<"_design/bar">>, []), + {ok, DDoc} = couch_db:open_doc(Db, ~"_design/bar", []), {ok, IdxState} = couch_mrview_util:ddoc_to_mrst(couch_db:name(Db), DDoc), Sig = IdxState#mrst.sig, HexSig = list_to_binary(couch_index_util:hexsig(Sig)), DocId = couch_mrview_util:get_local_purge_doc_id(HexSig), couch_db:open_doc(Db, DocId, []). +get_local_purge_seq(Db) -> + {ok, Db1} = couch_db:reopen(Db), + {ok, #doc{body = {Props}}} = get_local_purge_doc(Db1), + couch_util:get_value(~"purge_seq", Props). + run_query(Db, Opts) -> - couch_mrview:query_view(Db, <<"_design/bar">>, <<"baz">>, Opts). + couch_mrview:query_view(Db, ~"_design/bar", ~"baz", Opts). save_docs(Db, JsonDocs, Options) -> Docs = lists:map(
