This is an automated email from the ASF dual-hosted git repository. nickva pushed a commit to branch split-batch-writes in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 61d5cc006df46fe6ab3aa4972f300946dc92e5f8 Author: Nick Vatamaniuc <[email protected]> AuthorDate: Thu Nov 12 02:15:58 2026 -0500 Batch up writes for shard splitting Similar to what we did for reads (d61d3e0c4) just for writes. Export append_raw_chunks/2 couch_file call and add write_doc_bodies/2` to the couch engine API. Trying it with Q=1 shards 1GB file with about 1KB docs: main: 58 sec pr: 45 sec --- src/couch/src/couch_bt_engine.erl | 7 ++ src/couch/src/couch_db_engine.erl | 13 +++ src/couch/src/couch_db_split.erl | 93 +++++++++++++++------- src/couch/src/couch_file.erl | 2 +- .../src/cpse_test_read_write_docs.erl | 38 +++++++++ 5 files changed, 122 insertions(+), 31 deletions(-) diff --git a/src/couch/src/couch_bt_engine.erl b/src/couch/src/couch_bt_engine.erl index da7e34bb4..9c15a32a0 100644 --- a/src/couch/src/couch_bt_engine.erl +++ b/src/couch/src/couch_bt_engine.erl @@ -61,6 +61,7 @@ serialize_doc/2, write_doc_body/2, + write_doc_bodies/2, write_doc_infos/3, purge_docs/3, copy_purge_infos/2, @@ -452,6 +453,12 @@ write_doc_body(St, #doc{} = Doc) -> {ok, Ptr, Written} = couch_file:append_raw_chunk(Fd, Doc#doc.body), {ok, Doc#doc{body = Ptr}, Written}. +write_doc_bodies(#st{fd = Fd}, Docs) when is_list(Docs) -> + Chunks = [Chunk || #doc{body = Chunk} <- Docs], + {ok, Resps} = couch_file:append_raw_chunks(Fd, Chunks), + Fun = fun(Doc, {Ptr, Written}) -> {Doc#doc{body = Ptr}, Written} end, + {ok, lists:zipwith(Fun, Docs, Resps)}. + write_doc_infos(#st{} = St, Pairs, LocalDocs) -> #st{ id_tree = IdTree, diff --git a/src/couch/src/couch_db_engine.erl b/src/couch/src/couch_db_engine.erl index 91f5a9859..f175e63b7 100644 --- a/src/couch/src/couch_db_engine.erl +++ b/src/couch/src/couch_db_engine.erl @@ -384,6 +384,14 @@ -callback write_doc_body(DbHandle :: db_handle(), Doc :: doc()) -> {ok, FlushedDoc :: doc(), BytesWritten :: non_neg_integer()}. +% Batch version of write_doc_body/2. It should be called with a list of +% serialized docs. It should return a {FlushedDoc, BytesWritten} pair per input +% document, in order. Engines implementations maybe write all the bodies +% together (couch_bt_engine appends them with a one couch_file call, for +% example). +-callback write_doc_bodies(DbHandle :: db_handle(), Docs :: [doc()]) -> + {ok, [{FlushedDoc :: doc(), BytesWritten :: non_neg_integer()}]}. + % This function is called from the context of couch_db_updater % and as such is guaranteed single threaded for the given % DbHandle. @@ -717,6 +725,7 @@ serialize_doc/2, write_doc_body/2, + write_doc_bodies/2, write_doc_infos/3, purge_docs/3, copy_purge_infos/2, @@ -928,6 +937,10 @@ write_doc_body(#db{} = Db, #doc{} = Doc) -> #db{engine = {Engine, EngineState}} = Db, Engine:write_doc_body(EngineState, Doc). +write_doc_bodies(#db{} = Db, Docs) when is_list(Docs) -> + #db{engine = {Engine, EngineState}} = Db, + Engine:write_doc_bodies(EngineState, Docs). + write_doc_infos(#db{} = Db, DocUpdates, LocalDocs) -> #db{engine = {Engine, EngineState}} = Db, {ok, NewSt} = Engine:write_doc_infos(EngineState, DocUpdates, LocalDocs), diff --git a/src/couch/src/couch_db_split.erl b/src/couch/src/couch_db_split.erl index f41f88eb6..184e71b8b 100644 --- a/src/couch/src/couch_db_split.erl +++ b/src/couch/src/couch_db_split.erl @@ -58,6 +58,7 @@ source_db, target_db, bodies = #{}, + docs = [], active = 0, external = 0, atts = [] @@ -426,25 +427,32 @@ changes_cb(#full_doc_info{} = FDI, #state{} = State) -> % Use batched make_docs read to get all the pending docs bodies copy_pending(#state{pending = []} = State) -> State; -copy_pending(#state{pending = Pending, source_db = SourceDb} = State) -> +copy_pending(#state{pending = Pending, source_db = SourceDb, targets = Targets} = State) -> FDIs = lists:reverse(Pending), Bodies = get_bodies(SourceDb, FDIs), - State1 = State#state{pending = [], pending_count = 0, pending_bytes = 0}, - lists:foldl( - fun(FDI, StateAcc) -> - copy_fdi(FDI, Bodies, StateAcc) + KeyFun = fun(#full_doc_info{id = Id}) -> + {Key, #target{}} = pick_target(Id, State, Targets, changes), + Key + end, + Targets1 = maps:fold( + fun(Key, KeyFDIs, Acc) -> + Acc#{Key := copy_fdis(KeyFDIs, Bodies, map_get(Key, Acc), State)} end, - State1, - FDIs - ). - -copy_fdi(#full_doc_info{id = Id} = FDI, #{} = Bodies, #state{} = State) -> - #state{source_db = SourceDb, targets = Targets} = State, - {Key, Target} = pick_target(Id, State, Targets, changes), - FDI1 = process_fdi(FDI, Bodies, SourceDb, Target#target.db), - MaxBuffer = State#state.max_buffer_size, - Target1 = acc_and_flush(FDI1, Target, MaxBuffer, fun commit_docs/1), - State#state{targets = Targets#{Key => Target1}}. + Targets, + maps:groups_from_list(KeyFun, FDIs) + ), + State#state{targets = Targets1, pending = [], pending_count = 0, pending_bytes = 0}. + +% Write the docs bodies as a batch. First serialize the bodies of all the +% leafs, then batch write them. +copy_fdis(FDIs, #{} = Bodies, #target{db = TargetDb} = Target, #state{} = State) -> + #state{source_db = SourceDb, max_buffer_size = MaxBuffer} = State, + MapF = fun(FDI, Acc) -> serialize_fdi(FDI, Bodies, SourceDb, TargetDb, Acc) end, + {FDIs1, DocsRev} = lists:mapfoldl(MapF, [], FDIs), + {ok, Written} = couch_db_engine:write_doc_bodies(TargetDb, lists:reverse(DocsRev)), + {FDIs2, []} = lists:mapfoldl(fun finish_fdi/2, Written, FDIs1), + FoldF = fun(FDI, Acc) -> acc_and_flush(FDI, Acc, MaxBuffer, fun commit_docs/1) end, + lists:foldl(FoldF, Target, FDIs2). % Get a map of Ptr => #doc{} from a list if FDIs get_bodies(SourceDb, FDIs) -> @@ -476,19 +484,23 @@ commit_docs(#target{buffer = FDIs, db = Db} = Target) -> {ok, Db2} = couch_db_engine:commit_data(Db1), Target#target{buffer = [], buffer_size = 0, db = Db2}. -process_fdi(FDI, #{} = Bodies, SourceDb, TargetDb) -> +% Copy attachments and serialize bodies. Leafs get their external and +% attachment sizes here, then keep the source pointer until finish_fdi/2 call. +serialize_fdi(FDI, #{} = Bodies, SourceDb, TargetDb, DocsAcc) -> #full_doc_info{id = Id, rev_tree = RTree} = FDI, - Acc = #racc{id = Id, source_db = SourceDb, target_db = TargetDb, bodies = Bodies}, - {NewRTree, NewAcc} = couch_key_tree:mapfold(fun revtree_cb/4, Acc, RTree), - {Active, External} = total_sizes(NewAcc), - FDI#full_doc_info{ - rev_tree = NewRTree, - sizes = #size_info{active = Active, external = External} - }. + Acc = #racc{ + id = Id, + source_db = SourceDb, + target_db = TargetDb, + bodies = Bodies, + docs = DocsAcc + }, + {NewRTree, #racc{docs = DocsAcc1}} = couch_key_tree:mapfold(fun serialize_cb/4, Acc, RTree), + {FDI#full_doc_info{rev_tree = NewRTree}, DocsAcc1}. -revtree_cb(_Rev, _Leaf, branch, Acc) -> +serialize_cb(_Rev, _Leaf, branch, Acc) -> {[], Acc}; -revtree_cb({Pos, RevId}, Leaf, leaf, Acc) -> +serialize_cb({Pos, RevId}, Leaf, leaf, Acc) -> #racc{id = Id, source_db = SourceDb, target_db = TargetDb, bodies = Bodies} = Acc, #leaf{deleted = Deleted, ptr = Ptr, sizes = LeafSizes} = Leaf, #doc{body = Body, atts = AttInfos0} = maps:get(Ptr, Bodies), @@ -516,16 +528,37 @@ revtree_cb({Pos, RevId}, Leaf, leaf, Acc) -> Atts = [process_attachment(Att, SourceDb, TargetDb) || Att <- AttInfos], Doc2 = Doc1#doc{atts = Atts}, Doc3 = couch_db_engine:serialize_doc(TargetDb, Doc2), - {ok, Doc4, Active} = couch_db_engine:write_doc_body(TargetDb, Doc3), % element(3,...) and (4,...) are the stream pointer and size respecitively % (see couch_att.erl) They are numeric for compatibility with older formats AttSizes = [{element(3, A), element(4, A)} || A <- Atts], NewLeaf = Leaf#leaf{ - ptr = Doc4#doc.body, - sizes = #size_info{active = Active, external = External}, + sizes = #size_info{external = External}, atts = AttSizes }, - {NewLeaf, add_sizes(Active, External, AttSizes, Acc)}. + {NewLeaf, Acc#racc{docs = [Doc3 | Acc#racc.docs]}}. + +% Second pass over the leafs. Bodies are written already. Update sizes and +% turn #doc{} records into proper leafs with ptr and size data. +finish_fdi(#full_doc_info{rev_tree = RTree} = FDI, Written) -> + {NewRTree, {Written1, Acc}} = couch_key_tree:mapfold( + fun finish_cb/4, {Written, #racc{}}, RTree + ), + {Active, External} = total_sizes(Acc), + NewFDI = FDI#full_doc_info{ + rev_tree = NewRTree, + sizes = #size_info{active = Active, external = External} + }, + {NewFDI, Written1}. + +finish_cb(_Rev, Value, branch, Acc) -> + {Value, Acc}; +finish_cb(_Rev, #leaf{} = Leaf, leaf, {[{#doc{body = Ptr}, Active} | Rest], #racc{} = Acc}) -> + #leaf{sizes = #size_info{external = External} = Sizes, atts = AttSizes} = Leaf, + NewLeaf = Leaf#leaf{ + ptr = Ptr, + sizes = Sizes#size_info{active = Active} + }, + {NewLeaf, {Rest, add_sizes(Active, External, AttSizes, Acc)}}. % This is copied almost verbatim from the compactor process_attachment( diff --git a/src/couch/src/couch_file.erl b/src/couch/src/couch_file.erl index 1a69fa017..112d8ba6a 100644 --- a/src/couch/src/couch_file.erl +++ b/src/couch/src/couch_file.erl @@ -43,7 +43,7 @@ -export([open/1, open/2, close/1, bytes/1, sync/1, truncate/2, set_db_pid/2]). -export([pread_term/2, pread_binary/2]). -export([append_binary/2]). --export([append_raw_chunk/2, assemble_file_chunk_and_checksum/1]). +-export([append_raw_chunk/2, append_raw_chunks/2, assemble_file_chunk_and_checksum/1]). -export([append_term/2, append_term/3]). -export([pread_terms/2]). -export([append_terms/2, append_terms/3]). diff --git a/src/couch_pse_tests/src/cpse_test_read_write_docs.erl b/src/couch_pse_tests/src/cpse_test_read_write_docs.erl index 16f4283a9..8b1084027 100644 --- a/src/couch_pse_tests/src/cpse_test_read_write_docs.erl +++ b/src/couch_pse_tests/src/cpse_test_read_write_docs.erl @@ -304,3 +304,41 @@ cpse_delete_local_doc(Db1) -> [not_found], couch_db_engine:open_local_docs(Db3, [<<"_local/foo">>]) ). + +cpse_write_doc_bodies(Db1) -> + {ok, DbBatch} = cpse_util:create_db(), + try + Docs = [ + #doc{id = ~"1", body = {[{~"a", 1}]}}, + #doc{id = ~"2", body = {[{~"b", binary:copy(~"x", 10000)}]}}, + #doc{id = ~"3", body = {[{~"c", 3}]}} + ], + SerializeF = fun(Doc) -> couch_db_engine:serialize_doc(Db1, Doc) end, + SDocs = lists:map(SerializeF, Docs), + IndividualF = fun(Doc) -> + {ok, Doc1, Written} = couch_db_engine:write_doc_body(Db1, Doc), + {Doc1, Written} + end, + Individual = lists:map(IndividualF, SDocs), + {ok, Batch} = couch_db_engine:write_doc_bodies(DbBatch, SDocs), + ?assertEqual(3, length(Batch)), + ?assertEqual( + [Written || {_, Written} <- Individual], + [Written || {_, Written} <- Batch] + ), + ?assertEqual( + [Doc#doc.body || {Doc, _} <- Individual], + [Doc#doc.body || {Doc, _} <- Batch] + ), + ReadBodyF = fun({Doc, _}) -> + Doc1 = couch_db_engine:read_doc_body(DbBatch, Doc), + couch_compress:decompress(Doc1#doc.body) + end, + Expected = [Doc#doc.body || Doc <- Docs], + Written = lists:map(ReadBodyF, Batch), + ?assertEqual(Expected, Written), + % Check the empty case + ?assertEqual({ok, []}, couch_db_engine:write_doc_bodies(DbBatch, [])) + after + ok = couch_server:delete(couch_db:name(DbBatch), []) + end.
