This is an automated email from the ASF dual-hosted git repository. nickva pushed a commit to branch fix-recreated-doc-att-revpos in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 21dd2269667c095365678621ab9e6cfff8af6f4d Author: Nick Vatamaniuc <[email protected]> AuthorDate: Thu Sep 24 01:28:56 2026 -0400 Fix attachment revisions when recreating a deleted doc When we re-create a deleted doc, we forgot to update the revpos of the attachments. They were left at 1- when they should be at 3- Fixes: https://github.com/apache/couchdb/issues/6126 --- src/couch/src/couch_att.erl | 49 +++++- src/couch/src/couch_db_updater.erl | 48 +++++- .../test/eunit/couch_db_recreate_att_tests.erl | 182 +++++++++++++++++++++ 3 files changed, 269 insertions(+), 10 deletions(-) diff --git a/src/couch/src/couch_att.erl b/src/couch/src/couch_att.erl index c7e051a28..bf28875e0 100644 --- a/src/couch/src/couch_att.erl +++ b/src/couch/src/couch_att.erl @@ -29,7 +29,9 @@ -export([ size_info/1, to_disk_term/1, - from_disk_term/2 + from_disk_term/2, + disk_term_set_revpos/2, + stub_from_disk_term/1 ]). -export([ @@ -365,6 +367,28 @@ to_disk_term(Att) -> ), {list_to_tuple(lists:reverse(Base)), Extended}. +%% Set revpos in a disk term +disk_term_set_revpos({Base, Extended}, RevPos) when is_tuple(Base), is_list(Extended) -> + {disk_term_set_revpos(Base, RevPos), Extended}; +disk_term_set_revpos({_Name, _Type, _Sp, _AttLen, _DiskLen, _RevPos, _Md5, _Enc} = Term, RevPos) -> + setelement(6, Term, RevPos). + +%% Make a stub from a disk term the same way as to_disk_term/1. It has everything +%% except the data so we don't need to open a stream here +stub_from_disk_term({Base, Extended}) when is_tuple(Base), is_list(Extended) -> + store(Extended, stub_from_disk_term(Base)); +stub_from_disk_term({Name, Type, _Sp, AttLen, DiskLen, RevPos, Md5, Enc}) -> + #att{ + name = Name, + type = Type, + att_len = AttLen, + disk_len = DiskLen, + md5 = Md5, + revpos = RevPos, + data = stub, + encoding = upgrade_encoding(Enc) + }. + %% The new disk term format is a simple wrapper around the legacy format. Base %% properties will remain in a tuple while the new fields and possibly data from %% future extensions will be stored in a list of atom/value pairs. While this is @@ -897,6 +921,29 @@ attachment_stub_merge_test_() -> %% attachment errors. {"Attachment stub merging tests", []}. +disk_term_set_revpos_test() -> + {Base, Ext} = disk_term_set_revpos({{<<"a">>, t, sp, 1, 1, 1, m, e}, []}, 5), + ?assertEqual({<<"a">>, t, sp, 1, 1, 5, m, e}, Base), + ?assertEqual([], Ext), + Term = disk_term_set_revpos({<<"a">>, t, sp, 1, 1, 1, m, e}, 7), + ?assertEqual(7, element(6, Term)). + +stub_from_disk_term_test() -> + Att = new([ + {name, <<"a.txt">>}, + {type, <<"text/plain">>}, + {att_len, 3}, + {disk_len, 5}, + {md5, <<"md5">>}, + {revpos, 2}, + {data, {stream, {couch_bt_engine_stream, {fake_fd, fake_sp}}}}, + {encoding, gzip} + ]), + ?assertEqual(Att#att{data = stub}, stub_from_disk_term(to_disk_term(Att))), + ExtendedAtt = store(headers, [{<<"X-Foo">>, <<"bar">>}], Att), + ExtendedStub = store(data, stub, ExtendedAtt), + ?assertEqual(ExtendedStub, stub_from_disk_term(to_disk_term(ExtendedAtt))). + %% Test generators test_non_upgrading_fields(Attachment) -> diff --git a/src/couch/src/couch_db_updater.erl b/src/couch/src/couch_db_updater.erl index 8e697039f..3229a2db9 100644 --- a/src/couch/src/couch_db_updater.erl +++ b/src/couch/src/couch_db_updater.erl @@ -28,7 +28,8 @@ add_infos = [], rem_seqs = [], cur_seq, - full_partitions = [] + full_partitions = [], + db }). init({Engine, DbName, FilePath, Options0}) -> @@ -474,14 +475,15 @@ merge_rev_trees([NewDocs | RestDocsList], [OldDocInfo | RestOldInfo], Acc) -> #merge_acc{ revs_limit = Limit, replicated_changes = ReplicatedChanges, - full_partitions = FullPartitions + full_partitions = FullPartitions, + db = Db } = Acc, % Track doc ids so we can debug large revision trees erlang:put(last_id_merged, OldDocInfo#full_doc_info.id), NewDocInfo0 = lists:foldl( fun({Client, NewDoc}, OldInfoAcc) -> - NewInfo = merge_rev_tree(OldInfoAcc, NewDoc, Client, ReplicatedChanges), + NewInfo = merge_rev_tree(Db, OldInfoAcc, NewDoc, Client, ReplicatedChanges), case is_overflowed(NewInfo, OldInfoAcc, FullPartitions) of true when not ReplicatedChanges -> DocId = NewInfo#full_doc_info.id, @@ -535,7 +537,7 @@ merge_rev_trees([NewDocs | RestDocsList], [OldDocInfo | RestOldInfo], Acc) -> merge_rev_trees(RestDocsList, RestOldInfo, NewAcc) end. -merge_rev_tree(OldInfo, NewDoc, Client, false) when +merge_rev_tree(Db, OldInfo, NewDoc, Client, false) when OldInfo#full_doc_info.deleted -> % We're recreating a document that was previously @@ -561,12 +563,17 @@ merge_rev_tree(OldInfo, NewDoc, Client, false) when _ -> NewDoc#doc.body end, + % Attachments here are disk terms. new_revid/1 can't read them, it + % wants stubs. So we'll give it stubs with the right digests. + Atts = [couch_att:stub_from_disk_term(Att) || Att <- NewDoc#doc.atts], RevIdDoc = NewDoc#doc{ revs = {OldPos, [OldRev]}, - body = Body + body = Body, + atts = Atts }, NewRevId = couch_db:new_revid(RevIdDoc), - NewDoc2 = NewDoc#doc{revs = {OldPos + 1, [NewRevId, OldRev]}}, + NewDoc1 = set_recreated_att_revpos(Db, NewDoc, OldPos + 1), + NewDoc2 = NewDoc1#doc{revs = {OldPos + 1, [NewRevId, OldRev]}}, % Merge our modified new doc into the tree #full_doc_info{rev_tree = OldTree} = OldInfo, @@ -586,7 +593,7 @@ merge_rev_tree(OldInfo, NewDoc, Client, false) when send_result(Client, NewDoc, conflict), OldInfo end; -merge_rev_tree(OldInfo, NewDoc, Client, false) -> +merge_rev_tree(_Db, OldInfo, NewDoc, Client, false) -> % We're attempting to merge a new revision into an % undeleted document. To not be a conflict we require % that the merge results in extending a branch. @@ -612,7 +619,7 @@ merge_rev_tree(OldInfo, NewDoc, Client, false) -> send_result(Client, NewDoc, conflict), OldInfo end; -merge_rev_tree(OldInfo, NewDoc, _Client, true) -> +merge_rev_tree(_Db, OldInfo, NewDoc, _Client, true) -> % We're merging in revisions without caring about % conflicts. Most likely this is a replication update. OldTree = OldInfo#full_doc_info.rev_tree, @@ -620,6 +627,28 @@ merge_rev_tree(OldInfo, NewDoc, _Client, true) -> {NewTree, _} = couch_key_tree:merge(OldTree, NewTree0), OldInfo#full_doc_info{rev_tree = NewTree}. +% Revpos were set before the document was grafted onto the deleted revision and +% point to rev 1. Update them to the new revpos. This is needed when we +% re-create a document which was previously deleted. +set_recreated_att_revpos(_Db, #doc{atts = []} = Doc, _RevPos) -> + Doc; +set_recreated_att_revpos(Db, #doc{atts = Atts, meta = Meta} = Doc, RevPos) -> + Atts1 = [couch_att:disk_term_set_revpos(Att, RevPos) || Att <- Atts], + case lists:keytake(comp_body, 1, Meta) of + {value, {comp_body, CompBody}, Meta1} -> + % comp_body existence means the doc summary was already serialized + % with old atts. We can't use that, we need to re-serialize it with + % the new atts. keytake removes the old comp_body and then + % serialize_doc will re-create it. + Doc1 = Doc#doc{body = CompBody, atts = Atts1, meta = Meta1}, + couch_db_engine:serialize_doc(Db, Doc1); + false -> + % This is in the odd case we use a different engine than the b-tree + % engine which doesn't need the comp_body hack. Just update the + % atts then. + Doc#doc{atts = Atts1} + end. + is_overflowed(_New, _Old, []) -> false; is_overflowed(Old, Old, _FullPartitions) -> @@ -697,7 +726,8 @@ update_docs_int(Db, DocsList, LocalDocs, ReplicatedChanges) -> add_infos = [], rem_seqs = [], cur_seq = UpdateSeq, - full_partitions = FullPartitions + full_partitions = FullPartitions, + db = Db }, {ok, AccOut} = merge_rev_trees(DocsList, OldDocInfos, AccIn), #merge_acc{ diff --git a/src/couch/test/eunit/couch_db_recreate_att_tests.erl b/src/couch/test/eunit/couch_db_recreate_att_tests.erl new file mode 100644 index 000000000..33a72f4d6 --- /dev/null +++ b/src/couch/test/eunit/couch_db_recreate_att_tests.erl @@ -0,0 +1,182 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(couch_db_recreate_att_tests). + +-include_lib("couch/include/couch_eunit.hrl"). +-include_lib("couch/include/couch_db.hrl"). + +-define(DOC_ID, ~"doc"). +-define(ATT_NAME, ~"att"). +-define(ATT_TYPE, ~"application/octet-stream"). +-define(ADATA, <<"aaaaa">>). +-define(BDATA, <<"bbbbbb">>). + +setup() -> + Src = ?tempdb(), + Tgt = ?tempdb(), + {ok, SrcDb} = couch_db:create(Src, [?ADMIN_CTX]), + {ok, TgtDb} = couch_db:create(Tgt, [?ADMIN_CTX]), + ok = couch_db:close(SrcDb), + ok = couch_db:close(TgtDb), + {Src, Tgt}. + +teardown({Src, Tgt}) -> + ok = couch_server:delete(Src, [?ADMIN_CTX]), + ok = couch_server:delete(Tgt, [?ADMIN_CTX]). + +recreate_att_test_() -> + { + "Recreating doc attachments", + { + setup, + fun() -> test_util:start_couch([ioq]) end, + fun test_util:stop_couch/1, + { + foreach, + fun setup/0, + fun teardown/1, + [ + ?TDEF_FE(t_recreate_revpos), + ?TDEF_FE(t_recreate_multiple_deletes), + ?TDEF_FE(t_atts_since_works), + ?TDEF_FE(t_recreate_replicates), + ?TDEF_FE(t_recreate_no_atts), + ?TDEF_FE(t_revs_depend_on_att_content), + ?TDEF_FE(t_rev_is_deterministic) + ] + } + } + }. + +t_recreate_revpos({Src, _}) -> + {1, _} = Rev1 = create(Src, ?ADATA), + {2, _} = delete(Src, Rev1), + {3, _} = Rev3 = create(Src, ?BDATA), + Att = open_att(Src, Rev3, []), + ?assertEqual(3, couch_att:fetch(revpos, Att)), + ?assertEqual(?BDATA, couch_att:to_binary(Att)). + +t_recreate_multiple_deletes({Src, _}) -> + Rev1 = create(Src, ?ADATA), + Rev2 = delete(Src, Rev1), + Rev3 = create(Src, ?BDATA), + {4, _} = delete(Src, Rev3), + {5, _} = Rev5 = create(Src, ?ADATA), + ?assertNotEqual(Rev2, Rev5), + Att = open_att(Src, Rev5, []), + ?assertEqual(5, couch_att:fetch(revpos, Att)), + % a's then b's then a's again + ?assertEqual(?ADATA, couch_att:to_binary(Att)). + +t_atts_since_works({Src, _}) -> + Rev1 = create(Src, ?ADATA), + delete(Src, Rev1), + Rev3 = create(Src, ?BDATA), + Att = open_att(Src, Rev3, [{atts_since, [Rev1]}]), + ?assertNotEqual(stub, couch_att:fetch(data, Att)), + ?assertEqual(?BDATA, couch_att:to_binary(Att)). + +t_recreate_replicates({Src, Tgt}) -> + % Replicate a's + Rev1 = create(Src, ?ADATA), + replicate(Src, Tgt, Rev1, []), + ?assertEqual(?ADATA, couch_att:to_binary(open_att(Tgt, Rev1, []))), + % Delete, recreate and re-replicate + delete(Src, Rev1), + Rev3 = create(Src, ?BDATA), + replicate(Src, Tgt, Rev3, [Rev1]), + % Expect b's now + ?assertEqual(?BDATA, couch_att:to_binary(open_att(Tgt, Rev3, []))). + +t_recreate_no_atts({Src, _}) -> + Rev1 = create(Src, ?ADATA), + delete(Src, Rev1), + {ok, Db} = couch_db:open_int(Src, []), + Doc = #doc{id = ?DOC_ID, body = {[{<<"foo">>, 1}]}}, + {ok, {3, _}} = couch_db:update_doc(Db, Doc, []), + ok = couch_db:close(Db), + {ok, Db1} = couch_db:open_int(Src, []), + {ok, #doc{atts = Atts, body = Body}} = couch_db:open_doc(Db1, ?DOC_ID, []), + ok = couch_db:close(Db1), + ?assertEqual([], Atts), + ?assertEqual({[{<<"foo">>, 1}]}, Body). + +t_revs_depend_on_att_content({Db1, Db2}) -> + % Recreate with different attachment content in each db. We should be + % getting different revisions then, too. + do_recreate(Db1, Db2), + {3, _} = Rev1 = create(Db1, ?ADATA), + {3, _} = Rev2 = create(Db2, ?BDATA), + ?assertNotEqual(Rev1, Rev2). + +t_rev_is_deterministic({Db1, Db2}) -> + % Write some random attachment to get a different file position in db2 and + % we should still get the same revision for same content. The idea is we + % can generate the data independently on multuple servers but their revs + % should match (as opposed to creating conflicts) + create(Db2, <<"junk">>, [{?ATT_NAME, ?BDATA}]), + do_recreate(Db1, Db2), + {3, _} = Rev = create(Db1, ?BDATA), + ?assertEqual(Rev, create(Db2, ?BDATA)). + +% The revisions should match, so a recreated doc will branch off the same +% deleted revision in both dbs. +do_recreate(Db1, Db2) -> + Rev1 = create(Db1, ?ADATA), + ?assertEqual(Rev1, create(Db2, ?ADATA)), + Rev2 = delete(Db1, Rev1), + ?assertEqual(Rev2, delete(Db2, Rev1)). + +create(DbName, Data) -> + create(DbName, ?DOC_ID, [{?ATT_NAME, Data}]). + +create(DbName, DocId, Atts) -> + AttsJson = [ + {Name, {[{~"content_type", ?ATT_TYPE}, {~"data", base64:encode(Data)}]}} + || {Name, Data} <- Atts + ], + Doc = couch_doc:from_json_obj({[ + {~"_id", DocId}, + {~"_attachments", {AttsJson}} + ]}), + {ok, Db} = couch_db:open_int(DbName, []), + {ok, Rev} = couch_db:update_doc(Db, Doc, []), + ok = couch_db:close(Db), + Rev. + +delete(DbName, {Pos, RevId}) -> + {ok, Db} = couch_db:open_int(DbName, []), + Doc = #doc{id = ?DOC_ID, revs = {Pos, [RevId]}, deleted = true}, + {ok, Rev} = couch_db:update_doc(Db, Doc, []), + ok = couch_db:close(Db), + Rev. + +open_att(DbName, Rev, Opts) -> + {ok, Db} = couch_db:open_int(DbName, []), + {ok, [{ok, #doc{atts = [Att]}}]} = couch_db:open_doc_revs(Db, ?DOC_ID, [Rev], Opts), + ok = couch_db:close(Db), + Att. + +% Pretend we're replicating. Fetch revs with atts_since that target has. Write +% to target with new_edits=false. Attachments which the source thinks the +% target already has are sent as stubs. +replicate(Src, Tgt, Rev, AttsSince) -> + {ok, SDb} = couch_db:open_int(Src, []), + Opts = [revs, {atts_since, AttsSince}], + {ok, [{ok, Doc}]} = couch_db:open_doc_revs(SDb, ?DOC_ID, [Rev], Opts), + ok = couch_db:close(SDb), + Json = couch_doc:to_json_obj(Doc, [revs, attachments]), + Doc1 = couch_doc:from_json_obj(Json), + {ok, TDb} = couch_db:open_int(Tgt, []), + {ok, []} = couch_db:update_docs(TDb, [Doc1], [], ?REPLICATED_CHANGES), + ok = couch_db:close(TDb).
