This is an automated email from the ASF dual-hosted git repository. nickva pushed a commit to branch split-batch-bytes-vheap in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 322a39847d96e58a318a90013e93babac127f1a5 Author: Nick Vatamaniuc <[email protected]> AuthorDate: Wed Nov 11 18:24:32 2026 -0500 Adjust a few shard splitting parameters After seeing an OTP issue [1] related to bin_vheap_size I played with the batch size and min_bin_vheap_size on the split process and noticed a nice speedup if we both lowered the batch size and used a higher min_bin_vheap_size size. [1] https://github.com/erlang/otp/issues/11841 --- src/couch/src/couch_db_split.erl | 17 ++++++++++++++++- src/couch/test/eunit/couch_db_split_tests.erl | 14 ++++++++++++-- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/src/couch/src/couch_db_split.erl b/src/couch/src/couch_db_split.erl index 67f31ea1b..f41f88eb6 100644 --- a/src/couch/src/couch_db_split.erl +++ b/src/couch/src/couch_db_split.erl @@ -24,7 +24,15 @@ % Source doc batching parameters -define(BODY_BATCH_COUNT, 1000). --define(BODY_BATCH_BYTES, 16777216). +-define(BODY_BATCH_BYTES, 1048576). + +% Min binary virtual heap size of the splitting process. The default of ~370KB +% is garbage collected a few times for each batch of bodies. Anything still in +% use at that point will move to the old generation and only a full sweep will +% free it after that. So we're increasing the minimum here to keep them in the +% young heap. Benchmarking on OTP 27 showed a good speedup vs the default (63s +% -> 44sec). +-define(MIN_BIN_VHEAP_BYTES, 8388608). -record(state, { source_db, @@ -65,6 +73,7 @@ split(Source, #{} = Targets, PickFun) when Engine = get_engine(SourceDb), Partitioned = couch_db:is_partitioned(SourceDb), HashFun = mem3_hash:get_hash_fun(couch_db:name(SourceDb)), + set_min_bin_vheap_size(), try split(SourceDb, Partitioned, Engine, Targets, PickFun, HashFun) catch @@ -572,6 +581,12 @@ total_sizes(#racc{active = Active, external = External, atts = Atts}) -> TotalAtts = lists:foldl(fun({_, S}, A) -> S + A end, 0, Atts), {Active + TotalAtts, External + TotalAtts}. +% We have to set it as a process flag as spawn_opt has a bug currently +% (see https://github.com/erlang/otp/issues/11840) +set_min_bin_vheap_size() -> + Words = ?MIN_BIN_VHEAP_BYTES div erlang:system_info(wordsize), + process_flag(min_bin_vheap_size, Words). + get_max_buffer_size() -> config:get_integer("reshard", "split_buffer_size", ?DEFAULT_BUFFER_SIZE). diff --git a/src/couch/test/eunit/couch_db_split_tests.erl b/src/couch/test/eunit/couch_db_split_tests.erl index 510a2321f..ff816b612 100644 --- a/src/couch/test/eunit/couch_db_split_tests.erl +++ b/src/couch/test/eunit/couch_db_split_tests.erl @@ -112,7 +112,7 @@ should_keep_all_the_docs(DbName) -> PickFun = fun(DocId, Rs, _HashFun) -> lists:nth(1 + erlang:phash2(DocId, 2), Rs) end, {timeout, ?TIMEOUT, ?_test(begin - ?assertEqual(1204, length(Before)), + ?assertEqual(1209, length(Before)), ?assertEqual(2, length(proplists:get_value(~"conflict", Before))), {ok, _} = couch_db_split:split(DbName, TMap, PickFun), [SnapA, SnapB] = [all_the_docs(Name) || Name <- maps:values(TMap)], @@ -411,13 +411,23 @@ bunch_of_docs(DbName, N) -> ]} ), Big = couch_doc:from_json_obj({[{~"_id", ~"big"}, {~"blob", binary:copy(~"y", 200000)}]}), + % Incompressible, to get over the source batch byte limit + Rand = [ + couch_doc:from_json_obj( + {[ + {~"_id", <<"rand", (integer_to_binary(I))/binary>>}, + {~"blob", base64:encode(crypto:strong_rand_bytes(300000))} + ]} + ) + || I <- lists:seq(1, 5) + ], Del = couch_doc:from_json_obj({[{~"_id", ~"del"}, {~"v", 1}]}), Conflicts = [ #doc{id = ~"conflict", revs = {1, [~"aaa"]}, body = {[{~"v", ~"a"}]}}, #doc{id = ~"conflict", revs = {1, [~"bbb"]}, body = {[{~"v", ~"b"}]}} ], couch_util:with_db(DbName, fun(Db) -> - {ok, _} = couch_db:update_docs(Db, Plain ++ [Att, Big]), + {ok, _} = couch_db:update_docs(Db, Plain ++ [Att, Big] ++ Rand), {ok, Db1} = couch_db:reopen(Db), {ok, [{ok, {Pos, RevId}}]} = couch_db:update_docs(Db1, [Del]), {ok, Db2} = couch_db:reopen(Db1),
