This is an automated email from the ASF dual-hosted git repository.
nickva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
The following commit(s) were added to refs/heads/main by this push:
new d61d3e0c4 Speed up shard splitting using the new batched couch engine
call
d61d3e0c4 is described below
commit d61d3e0c483f9e01624c25bc100c4a019e83ffea
Author: Nick Vatamaniuc <[email protected]>
AuthorDate: Sat Nov 7 13:52:32 2026 -0500
Speed up shard splitting using the new batched couch engine call
Same idea as in #6206: when we read docs from the source use the new
`read_doc_bodies/2` batched call. `read_doc_bodies/2` will then use a single
preads call for couch_file.
Use separate batching parameters (doc count and size) vs the exisiting
buffering ones. Those were for target commit scheduling and this is for
source
batching so it didn't feel like combining the two made sense.
Trying to split 1GB Q=1 shard on my mac got a bit of speed-up 1.2x: 94sec ->
75sec.
---
src/couch/src/couch_db_split.erl | 88 +++++++++++++++++++++----
src/couch/test/eunit/couch_db_split_tests.erl | 94 ++++++++++++++++++++++++++-
2 files changed, 165 insertions(+), 17 deletions(-)
diff --git a/src/couch/src/couch_db_split.erl b/src/couch/src/couch_db_split.erl
index 3c01e576b..67f31ea1b 100644
--- a/src/couch/src/couch_db_split.erl
+++ b/src/couch/src/couch_db_split.erl
@@ -22,13 +22,20 @@
-define(DEFAULT_BUFFER_SIZE, 16777216).
+% Source doc batching parameters
+-define(BODY_BATCH_COUNT, 1000).
+-define(BODY_BATCH_BYTES, 16777216).
+
-record(state, {
source_db,
source_uuid,
targets,
pickfun,
max_buffer_size = ?DEFAULT_BUFFER_SIZE,
- hashfun
+ hashfun,
+ pending = [],
+ pending_count = 0,
+ pending_bytes = 0
}).
-record(target, {
@@ -42,6 +49,7 @@
id,
source_db,
target_db,
+ bodies = #{},
active = 0,
external = 0,
atts = []
@@ -378,26 +386,78 @@ commit_purge_infos(#target{buffer = PIs0, db = Db} =
Target) ->
Target#target{buffer = [], buffer_size = 0, db = Db2}.
copy_docs(#state{source_db = Db} = State) ->
- {ok, NewState} = couch_db:fold_changes(Db, 0, fun changes_cb/2, State),
+ {ok, State1} = couch_db:fold_changes(Db, 0, fun changes_cb/2, State),
+ State2 = copy_pending(State1),
CommitTargets = maps:map(
fun(_, #target{} = T) ->
commit_docs(T)
end,
- NewState#state.targets
+ State2#state.targets
),
- NewState#state{targets = CommitTargets}.
+ State2#state{targets = CommitTargets}.
% Backwards compatibility clause. Seq trees used to hold #doc_infos at one time
changes_cb(#doc_info{id = Id}, #state{source_db = Db} = State) ->
[FDI = #full_doc_info{}] = couch_db_engine:open_docs(Db, [Id]),
changes_cb(FDI, State);
-changes_cb(#full_doc_info{id = Id} = FDI, #state{} = State) ->
+changes_cb(#full_doc_info{} = FDI, #state{} = State) ->
+ #state{pending = Pending, pending_count = Count, pending_bytes = Bytes} =
State,
+ Bytes1 = Bytes + active_size(FDI),
+ Count1 = Count + 1,
+ State1 = State#state{
+ pending = [FDI | Pending],
+ pending_count = Count1,
+ pending_bytes = Bytes1
+ },
+ case Count1 >= ?BODY_BATCH_COUNT orelse Bytes1 >= ?BODY_BATCH_BYTES of
+ true -> {ok, copy_pending(State1)};
+ false -> {ok, State1}
+ end.
+
+% Use batched make_docs read to get all the pending docs bodies
+copy_pending(#state{pending = []} = State) ->
+ State;
+copy_pending(#state{pending = Pending, source_db = SourceDb} = State) ->
+ FDIs = lists:reverse(Pending),
+ Bodies = get_bodies(SourceDb, FDIs),
+ State1 = State#state{pending = [], pending_count = 0, pending_bytes = 0},
+ lists:foldl(
+ fun(FDI, StateAcc) ->
+ copy_fdi(FDI, Bodies, StateAcc)
+ end,
+ State1,
+ FDIs
+ ).
+
+copy_fdi(#full_doc_info{id = Id} = FDI, #{} = Bodies, #state{} = State) ->
#state{source_db = SourceDb, targets = Targets} = State,
{Key, Target} = pick_target(Id, State, Targets, changes),
- FDI1 = process_fdi(FDI, SourceDb, Target#target.db),
+ FDI1 = process_fdi(FDI, Bodies, SourceDb, Target#target.db),
MaxBuffer = State#state.max_buffer_size,
Target1 = acc_and_flush(FDI1, Target, MaxBuffer, fun commit_docs/1),
- {ok, State#state{targets = Targets#{Key => Target1}}}.
+ State#state{targets = Targets#{Key => Target1}}.
+
+% Get a map of Ptr => #doc{} from a list if FDIs
+get_bodies(SourceDb, FDIs) ->
+ Docs = lists:flatmap(
+ fun(#full_doc_info{rev_tree = Tree}) ->
+ [#doc{body = Ptr} || {#leaf{ptr = Ptr}, _} <-
couch_key_tree:get_all_leafs(Tree)]
+ end,
+ FDIs
+ ),
+ Docs1 = couch_db_engine:read_doc_bodies(SourceDb, Docs),
+ maps:from_list(
+ lists:zipwith(
+ fun(#doc{body = Ptr}, #doc{} = Doc) -> {Ptr, Doc} end,
+ Docs,
+ Docs1
+ )
+ ).
+
+active_size(#full_doc_info{sizes = #size_info{active = Active}}) when
is_integer(Active) ->
+ Active;
+active_size(#full_doc_info{}) ->
+ 0.
commit_docs(#target{buffer = [], db = Db} = Target) ->
Target#target{db = Db};
@@ -407,9 +467,9 @@ commit_docs(#target{buffer = FDIs, db = Db} = Target) ->
{ok, Db2} = couch_db_engine:commit_data(Db1),
Target#target{buffer = [], buffer_size = 0, db = Db2}.
-process_fdi(FDI, SourceDb, TargetDb) ->
+process_fdi(FDI, #{} = Bodies, SourceDb, TargetDb) ->
#full_doc_info{id = Id, rev_tree = RTree} = FDI,
- Acc = #racc{id = Id, source_db = SourceDb, target_db = TargetDb},
+ Acc = #racc{id = Id, source_db = SourceDb, target_db = TargetDb, bodies =
Bodies},
{NewRTree, NewAcc} = couch_key_tree:mapfold(fun revtree_cb/4, Acc, RTree),
{Active, External} = total_sizes(NewAcc),
FDI#full_doc_info{
@@ -420,16 +480,16 @@ process_fdi(FDI, SourceDb, TargetDb) ->
revtree_cb(_Rev, _Leaf, branch, Acc) ->
{[], Acc};
revtree_cb({Pos, RevId}, Leaf, leaf, Acc) ->
- #racc{id = Id, source_db = SourceDb, target_db = TargetDb} = Acc,
+ #racc{id = Id, source_db = SourceDb, target_db = TargetDb, bodies =
Bodies} = Acc,
#leaf{deleted = Deleted, ptr = Ptr, sizes = LeafSizes} = Leaf,
- Doc0 = #doc{
+ #doc{body = Body, atts = AttInfos0} = maps:get(Ptr, Bodies),
+ Doc1 = #doc{
id = Id,
revs = {Pos, [RevId]},
deleted = Deleted,
- body = Ptr
+ body = Body,
+ atts = AttInfos0
},
- Doc1 = couch_db_engine:read_doc_body(SourceDb, Doc0),
- #doc{body = Body, atts = AttInfos0} = Doc1,
External =
case LeafSizes#size_info.external of
0 when is_binary(Body) ->
diff --git a/src/couch/test/eunit/couch_db_split_tests.erl
b/src/couch/test/eunit/couch_db_split_tests.erl
index f0baa920b..510a2321f 100644
--- a/src/couch/test/eunit/couch_db_split_tests.erl
+++ b/src/couch/test/eunit/couch_db_split_tests.erl
@@ -36,7 +36,8 @@ split_test_() ->
{"Should split an empty shard", 0, 2},
{"Should split shard in half", 100, 2},
{"Should split shard in three", 99, 3},
- {"Should split shard in four", 100, 4}
+ {"Should split shard in four", 100, 4},
+ {"Should split shard with many docs", 2500, 2}
],
{
setup,
@@ -58,7 +59,8 @@ split_test_() ->
fun should_fail_on_existing_target/1,
fun should_fail_on_invalid_target_name/1,
fun should_crash_on_invalid_tmap/1,
- fun should_fail_on_opened_target/1
+ fun should_fail_on_opened_target/1,
+ fun should_keep_all_the_docs/1
]
}
]
@@ -102,6 +104,33 @@ should_split_shard({Desc, TotalDocs, Q}, DbName) ->
)
end)}.
+should_keep_all_the_docs(DbName) ->
+ bunch_of_docs(DbName, 1200),
+ Before = all_the_docs(DbName),
+ Ranges = make_ranges(2),
+ TMap = make_targets(Ranges),
+ PickFun = fun(DocId, Rs, _HashFun) -> lists:nth(1 + erlang:phash2(DocId,
2), Rs) end,
+ {timeout, ?TIMEOUT,
+ ?_test(begin
+ ?assertEqual(1204, length(Before)),
+ ?assertEqual(2, length(proplists:get_value(~"conflict", Before))),
+ {ok, _} = couch_db_split:split(DbName, TMap, PickFun),
+ [SnapA, SnapB] = [all_the_docs(Name) || Name <- maps:values(TMap)],
+ ?assert(length(SnapA) > 0),
+ ?assert(length(SnapB) > 0),
+ ?assertEqual([], [Id || {Id, _} <- SnapA, lists:keymember(Id, 1,
SnapB)]),
+ ?assertEqual(Before, lists:sort(SnapA ++ SnapB)),
+ lists:foreach(
+ fun(Name) ->
+ {ok, Db} = couch_db:open_int(Name, []),
+ FilePath = couch_db:get_filepath(Db),
+ ok = couch_db:close(Db),
+ ok = file:delete(FilePath)
+ end,
+ maps:values(TMap)
+ )
+ end)}.
+
should_fail_on_missing_source(_DbName) ->
DbName = ?tempdb(),
Ranges = make_ranges(2),
@@ -349,9 +378,68 @@ create_local_doc(I) ->
create_prefix_id_doc(I, "_local/").
create_prefix_id_doc(I, Prefix) ->
- Id = iolist_to_binary(io_lib:format(Prefix ++ "~3..0B", [I])),
+ Id = iolist_to_binary(io_lib:format(Prefix ++ "~4..0B", [I])),
couch_doc:from_json_obj({[{<<"_id">>, Id}, {<<"value">>, I}]}).
+% Some plain, some large, conflicted and attachments in the mix
+bunch_of_docs(DbName, N) ->
+ Plain = [
+ couch_doc:from_json_obj(
+ {[
+ {~"_id", list_to_binary(io_lib:format("doc~6..0B", [I]))},
+ {~"v", I},
+ {~"s", binary:copy(~"x", I rem 500)}
+ ]}
+ )
+ || I <- lists:seq(1, N)
+ ],
+ Att = couch_doc:from_json_obj(
+ {[
+ {~"_id", ~"att"},
+ {
+ ~"_attachments",
+ {[
+ {
+ ~"a.txt",
+ {[
+ {~"content_type", ~"text/plain"},
+ {~"data", base64:encode(~"xkcd")}
+ ]}
+ }
+ ]}
+ }
+ ]}
+ ),
+ Big = couch_doc:from_json_obj({[{~"_id", ~"big"}, {~"blob",
binary:copy(~"y", 200000)}]}),
+ Del = couch_doc:from_json_obj({[{~"_id", ~"del"}, {~"v", 1}]}),
+ Conflicts = [
+ #doc{id = ~"conflict", revs = {1, [~"aaa"]}, body = {[{~"v", ~"a"}]}},
+ #doc{id = ~"conflict", revs = {1, [~"bbb"]}, body = {[{~"v", ~"b"}]}}
+ ],
+ couch_util:with_db(DbName, fun(Db) ->
+ {ok, _} = couch_db:update_docs(Db, Plain ++ [Att, Big]),
+ {ok, Db1} = couch_db:reopen(Db),
+ {ok, [{ok, {Pos, RevId}}]} = couch_db:update_docs(Db1, [Del]),
+ {ok, Db2} = couch_db:reopen(Db1),
+ Deleted = Del#doc{revs = {Pos, [RevId]}, deleted = true},
+ {ok, [{ok, _}]} = couch_db:update_docs(Db2, [Deleted]),
+ {ok, Db3} = couch_db:reopen(Db2),
+ {ok, []} = couch_db:update_docs(Db3, Conflicts, [],
?REPLICATED_CHANGES)
+ end).
+
+% This is like a snapshot of the db to compare before and after.
+all_the_docs(DbName) ->
+ couch_util:with_db(DbName, fun(Db) ->
+ FoldFun = fun(#full_doc_info{id = Id}, Acc) -> {ok, [Id | Acc]} end,
+ {ok, Ids} = couch_db:fold_docs(Db, FoldFun, [], [include_deleted]),
+ lists:sort([{Id, all_the_revs(Db, Id)} || Id <- Ids])
+ end).
+
+all_the_revs(Db, Id) ->
+ {ok, Results} = couch_db:open_doc_revs(Db, Id, all, []),
+ Opts = [attachments, revs, conflicts, deleted_conflicts],
+ [couch_doc:to_json_obj(Doc, Opts) || {ok, #doc{} = Doc} <- Results].
+
docid_to_integer(<<"_local/", DocId/binary>>) ->
docid_to_integer(DocId);
docid_to_integer(DocId) ->