This is an automated email from the ASF dual-hosted git repository. vatamane pushed a commit to branch fix-mrview-purge-checkpoints in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit bb4e1982cbc042987e4329818cab504ac27f40bd Author: Nick Vatamaniuc <[email protected]> AuthorDate: Thu Mar 5 16:35:28 2026 -0500 Fix signatures in mrview purge checkpoints Switch from lists of integers (see example below) to a binary. ```json { "_id": "_local/purge-mrview-e758e0655818718ba98d8f53168f848f", "_rev": "0-1", "type": "mrview", "purge_seq": 393, "updated_on": 1772740238, "ddoc_id": "_design/mr", "signature": [ 101, 55, 53, 56, 101, 48, 54, 53, 53, 56, 49, 56, 55, 49, 56, 98, 97, 57, 56, 100, 56, 102, 53, 51, 49, 54, 56, 102, 56, 52, 56, 102 ] } ``` Since the old format was written to disk leave in a compatibility clause. At some point in the future maybe opt to auto-upgrade those docs, but for now, to allow effortless downgrading keep them as is. While at it, update the doc writer to use the newer time function and update the tests to use the ?TDEF_FE macro and save two indentation levels that way. --- src/couch_mrview/src/couch_mrview_index.erl | 26 +- .../eunit/couch_mrview_purge_docs_fabric_tests.erl | 362 +++++++++++---------- 2 files changed, 199 insertions(+), 189 deletions(-) diff --git a/src/couch_mrview/src/couch_mrview_index.erl b/src/couch_mrview/src/couch_mrview_index.erl index a5080ed76..26b4f1aac 100644 --- a/src/couch_mrview/src/couch_mrview_index.erl +++ b/src/couch_mrview/src/couch_mrview_index.erl @@ -240,14 +240,10 @@ verify_index_exists(DbName, Props) -> couch_util:with_db(DbName, fun(Db) -> case couch_db:get_design_doc(Db, DDocId) of {ok, #doc{} = DDoc} -> - {ok, IdxState} = couch_mrview_util:ddoc_to_mrst( - DbName, DDoc - ), + {ok, IdxState} = couch_mrview_util:ddoc_to_mrst(DbName, DDoc), IdxSig = IdxState#mrst.sig, - SigInLocal = couch_util:get_value( - <<"signature">>, Props - ), - couch_index_util:hexsig(IdxSig) == SigInLocal; + DocSig = couch_util:get_value(<<"signature">>, Props), + match_signatures(IdxSig, DocSig); {not_found, _} -> false end @@ -258,6 +254,16 @@ verify_index_exists(DbName, Props) -> false end. +match_signatures(IdxSig, DocSig) when is_binary(IdxSig), is_list(DocSig) -> + % Compatibility clause. In versions =< 3.5.1 mvrview signatures in purge + % checkpoints where written as lists of integers instead of a binary. After + % a few versions 3.6, 3.7 consider automatically upgrading them to binaries + % on open if find the older format. For now we don't as we'd like to be + % able to roll back to a previous version on upgrade. + couch_index_util:hexsig(IdxSig) =:= DocSig; +match_signatures(IdxSig, DocSig) when is_binary(IdxSig), is_binary(DocSig) -> + couch_util:to_hex_bin(IdxSig) =:= DocSig. + set_partitioned(Db, State) -> #mrst{ design_opts = DesignOpts @@ -305,16 +311,14 @@ update_local_purge_doc(Db, State) -> update_local_purge_doc(Db, State, PSeq) -> Sig = couch_index_util:hexsig(State#mrst.sig), DocId = couch_mrview_util:get_local_purge_doc_id(Sig), - {Mega, Secs, _} = os:timestamp(), - NowSecs = Mega * 1000000 + Secs, BaseDoc = couch_doc:from_json_obj( {[ {<<"_id">>, DocId}, {<<"type">>, <<"mrview">>}, {<<"purge_seq">>, PSeq}, - {<<"updated_on">>, NowSecs}, + {<<"updated_on">>, erlang:system_time(second)}, {<<"ddoc_id">>, get(idx_name, State)}, - {<<"signature">>, Sig} + {<<"signature">>, list_to_binary(Sig)} ]} ), Doc = diff --git a/src/couch_mrview/test/eunit/couch_mrview_purge_docs_fabric_tests.erl b/src/couch_mrview/test/eunit/couch_mrview_purge_docs_fabric_tests.erl index f3452b55a..8a963622f 100644 --- a/src/couch_mrview/test/eunit/couch_mrview_purge_docs_fabric_tests.erl +++ b/src/couch_mrview/test/eunit/couch_mrview_purge_docs_fabric_tests.erl @@ -53,192 +53,198 @@ view_purge_fabric_test_() -> fun setup/0, fun teardown/1, [ - fun test_purge_verify_index/1, - fun test_purge_hook_before_compaction/1 + ?TDEF_FE(test_purge_verify_index, ?TIMEOUT), + ?TDEF_FE(test_purge_hook_before_compaction, ?TIMEOUT) ] } } }. test_purge_verify_index(DbName) -> - {timeout, ?TIMEOUT, - ?_test(begin - Docs1 = couch_mrview_test_util:make_docs(normal, 5), - {ok, _} = fabric:update_docs(DbName, Docs1, [?ADMIN_CTX]), - {ok, _} = fabric:update_doc( - DbName, - couch_mrview_test_util:ddoc(map), - [?ADMIN_CTX] - ), - - Result1 = fabric:query_view(DbName, <<"bar">>, <<"baz">>, #mrargs{}), - Expect1 = - {ok, [ - {meta, [{total, 5}, {offset, 0}]}, - {row, [{id, <<"1">>}, {key, 1}, {value, 1}]}, - {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, - {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, - {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, - {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} - ]}, - ?assertEqual(Expect1, Result1), - - {ok, #doc{body = {Props1}}} = get_local_purge_doc(DbName), - ?assertEqual(0, couch_util:get_value(<<"purge_seq">>, Props1)), - ShardNames = [Sh || #shard{name = Sh} <- mem3:local_shards(DbName)], - [ShardDbName | _Rest] = ShardNames, - ?assertEqual( - true, - couch_mrview_index:verify_index_exists( - ShardDbName, Props1 - ) - ), - - purge_docs(DbName, [<<"1">>]), - - Result2 = fabric:query_view(DbName, <<"bar">>, <<"baz">>, #mrargs{}), - Expect2 = - {ok, [ - {meta, [{total, 4}, {offset, 0}]}, - {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, - {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, - {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, - {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} - ]}, - ?assertEqual(Expect2, Result2), - - {ok, #doc{body = {Props2}}} = get_local_purge_doc(DbName), - ?assertEqual(1, couch_util:get_value(<<"purge_seq">>, Props2)), - ?assertEqual( - true, - couch_mrview_index:verify_index_exists( - ShardDbName, Props2 - ) - ) - end)}. + Docs1 = couch_mrview_test_util:make_docs(normal, 5), + {ok, _} = fabric:update_docs(DbName, Docs1, [?ADMIN_CTX]), + {ok, _} = fabric:update_doc( + DbName, + couch_mrview_test_util:ddoc(map), + [?ADMIN_CTX] + ), + + Result1 = fabric:query_view(DbName, <<"bar">>, <<"baz">>, #mrargs{}), + Expect1 = + {ok, [ + {meta, [{total, 5}, {offset, 0}]}, + {row, [{id, <<"1">>}, {key, 1}, {value, 1}]}, + {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, + {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, + {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, + {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} + ]}, + ?assertEqual(Expect1, Result1), + + {ok, #doc{body = {Props1}}} = get_local_purge_doc(DbName), + ?assertEqual(0, couch_util:get_value(<<"purge_seq">>, Props1)), + ShardNames = [Sh || #shard{name = Sh} <- mem3:local_shards(DbName)], + [ShardDbName | _Rest] = ShardNames, + ?assertEqual( + true, + couch_mrview_index:verify_index_exists( + ShardDbName, Props1 + ) + ), + + purge_docs(DbName, [<<"1">>]), + + Result2 = fabric:query_view(DbName, <<"bar">>, <<"baz">>, #mrargs{}), + Expect2 = + {ok, [ + {meta, [{total, 4}, {offset, 0}]}, + {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, + {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, + {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, + {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} + ]}, + ?assertEqual(Expect2, Result2), + + {ok, #doc{body = {Props2}}} = get_local_purge_doc(DbName), + ?assertEqual(1, couch_util:get_value(<<"purge_seq">>, Props2)), + ?assertEqual( + true, + couch_mrview_index:verify_index_exists( + ShardDbName, Props2 + ) + ), + + % Make sure we can verify an older format in versions =< 3.5.1 + % where signatures were lists (arrays) of integers + DocSig = couch_util:get_value(<<"signature">>, Props2), + DocSig1 = binary_to_list(DocSig), + Props3 = lists:keyreplace(<<"signature">>, 1, Props2, {<<"signature">>, DocSig1}), + ?assertEqual( + true, + couch_mrview_index:verify_index_exists( + ShardDbName, Props3 + ) + ). test_purge_hook_before_compaction(DbName) -> - {timeout, ?TIMEOUT, - ?_test(begin - Docs1 = couch_mrview_test_util:make_docs(normal, 5), - {ok, _} = fabric:update_docs(DbName, Docs1, [?ADMIN_CTX]), - {ok, _} = fabric:update_doc( - DbName, - couch_mrview_test_util:ddoc(map), - [?ADMIN_CTX] - ), - - Result1 = fabric:query_view(DbName, <<"bar">>, <<"baz">>, #mrargs{}), - Expect1 = - {ok, [ - {meta, [{total, 5}, {offset, 0}]}, - {row, [{id, <<"1">>}, {key, 1}, {value, 1}]}, - {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, - {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, - {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, - {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} - ]}, - ?assertEqual(Expect1, Result1), - - purge_docs(DbName, [<<"1">>]), - - Result2 = fabric:query_view(DbName, <<"bar">>, <<"baz">>, #mrargs{}), - Expect2 = - {ok, [ - {meta, [{total, 4}, {offset, 0}]}, - {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, - {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, - {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, - {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} - ]}, - ?assertEqual(Expect2, Result2), - - {ok, #doc{body = {Props1}}} = get_local_purge_doc(DbName), - ?assertEqual(1, couch_util:get_value(<<"purge_seq">>, Props1)), - - [ShardName | _] = local_shards(DbName), - couch_util:with_db(ShardName, fun(Db) -> - {ok, _} = couch_db:start_compact(Db) - end), - wait_compaction(ShardName, ?LINE), - - ?assertEqual( - ok, - meck:wait( - 1, - couch_mrview_index, - ensure_local_purge_docs, - '_', - 5000 - ) - ), - - % Make sure compaction didn't change the update seq - {ok, #doc{body = {Props1}}} = get_local_purge_doc(DbName), - ?assertEqual(1, couch_util:get_value(<<"purge_seq">>, Props1)), - - purge_docs(DbName, [<<"2">>]), - - couch_util:with_db(ShardName, fun(Db) -> - {ok, _} = couch_db:start_compact(Db) - end), - wait_compaction(ShardName, ?LINE), - - ?assertEqual( - ok, - meck:wait( - 2, - couch_mrview_index, - ensure_local_purge_docs, - '_', - 5000 - ) - ), - - % Make sure compaction after a purge didn't overwrite - % the local purge doc for the index - {ok, #doc{body = {Props2}}} = get_local_purge_doc(DbName), - ?assertEqual(1, couch_util:get_value(<<"purge_seq">>, Props2)), - - % Force another update to ensure that we update - % the local doc appropriate after compaction - Result3 = fabric:query_view(DbName, <<"bar">>, <<"baz">>, #mrargs{}), - Expect3 = - {ok, [ - {meta, [{total, 3}, {offset, 0}]}, - {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, - {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, - {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} - ]}, - ?assertEqual(Expect3, Result3), - - {ok, #doc{body = {Props3}}} = get_local_purge_doc(DbName), - ?assertEqual(2, couch_util:get_value(<<"purge_seq">>, Props3)), - - % Check that if the local doc doesn't exist that one - % is created for the index on compaction - delete_local_purge_doc(DbName), - ?assertMatch({not_found, _}, get_local_purge_doc(DbName)), - - couch_util:with_db(ShardName, fun(Db) -> - {ok, _} = couch_db:start_compact(Db) - end), - wait_compaction(ShardName, ?LINE), - - ?assertEqual( - ok, - meck:wait( - 3, - couch_mrview_index, - ensure_local_purge_docs, - '_', - 5000 - ) - ), - - {ok, #doc{body = {Props4}}} = get_local_purge_doc(DbName), - ?assertEqual(2, couch_util:get_value(<<"purge_seq">>, Props4)) - end)}. + Docs1 = couch_mrview_test_util:make_docs(normal, 5), + {ok, _} = fabric:update_docs(DbName, Docs1, [?ADMIN_CTX]), + {ok, _} = fabric:update_doc( + DbName, + couch_mrview_test_util:ddoc(map), + [?ADMIN_CTX] + ), + + Result1 = fabric:query_view(DbName, <<"bar">>, <<"baz">>, #mrargs{}), + Expect1 = + {ok, [ + {meta, [{total, 5}, {offset, 0}]}, + {row, [{id, <<"1">>}, {key, 1}, {value, 1}]}, + {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, + {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, + {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, + {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} + ]}, + ?assertEqual(Expect1, Result1), + + purge_docs(DbName, [<<"1">>]), + + Result2 = fabric:query_view(DbName, <<"bar">>, <<"baz">>, #mrargs{}), + Expect2 = + {ok, [ + {meta, [{total, 4}, {offset, 0}]}, + {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, + {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, + {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, + {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} + ]}, + ?assertEqual(Expect2, Result2), + + {ok, #doc{body = {Props1}}} = get_local_purge_doc(DbName), + ?assertEqual(1, couch_util:get_value(<<"purge_seq">>, Props1)), + + [ShardName | _] = local_shards(DbName), + couch_util:with_db(ShardName, fun(Db) -> + {ok, _} = couch_db:start_compact(Db) + end), + wait_compaction(ShardName, ?LINE), + + ?assertEqual( + ok, + meck:wait( + 1, + couch_mrview_index, + ensure_local_purge_docs, + '_', + 5000 + ) + ), + + % Make sure compaction didn't change the update seq + {ok, #doc{body = {Props1}}} = get_local_purge_doc(DbName), + ?assertEqual(1, couch_util:get_value(<<"purge_seq">>, Props1)), + + purge_docs(DbName, [<<"2">>]), + + couch_util:with_db(ShardName, fun(Db) -> + {ok, _} = couch_db:start_compact(Db) + end), + wait_compaction(ShardName, ?LINE), + + ?assertEqual( + ok, + meck:wait( + 2, + couch_mrview_index, + ensure_local_purge_docs, + '_', + 5000 + ) + ), + + % Make sure compaction after a purge didn't overwrite + % the local purge doc for the index + {ok, #doc{body = {Props2}}} = get_local_purge_doc(DbName), + ?assertEqual(1, couch_util:get_value(<<"purge_seq">>, Props2)), + + % Force another update to ensure that we update + % the local doc appropriate after compaction + Result3 = fabric:query_view(DbName, <<"bar">>, <<"baz">>, #mrargs{}), + Expect3 = + {ok, [ + {meta, [{total, 3}, {offset, 0}]}, + {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, + {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, + {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} + ]}, + ?assertEqual(Expect3, Result3), + + {ok, #doc{body = {Props3}}} = get_local_purge_doc(DbName), + ?assertEqual(2, couch_util:get_value(<<"purge_seq">>, Props3)), + + % Check that if the local doc doesn't exist that one + % is created for the index on compaction + delete_local_purge_doc(DbName), + ?assertMatch({not_found, _}, get_local_purge_doc(DbName)), + + couch_util:with_db(ShardName, fun(Db) -> + {ok, _} = couch_db:start_compact(Db) + end), + wait_compaction(ShardName, ?LINE), + + ?assertEqual( + ok, + meck:wait( + 3, + couch_mrview_index, + ensure_local_purge_docs, + '_', + 5000 + ) + ), + + {ok, #doc{body = {Props4}}} = get_local_purge_doc(DbName), + ?assertEqual(2, couch_util:get_value(<<"purge_seq">>, Props4)). get_local_purge_doc(DbName) -> {ok, DDoc} = fabric:open_doc(DbName, <<"_design/bar">>, []),
