This is an automated email from the ASF dual-hosted git repository.
nickva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
The following commit(s) were added to refs/heads/main by this push:
new 8de7bf138 Add a batched version of make_docs
8de7bf138 is described below
commit 8de7bf13843c00ddf862a08c2ddfbc6f873364dd
Author: Nick Vatamaniuc <[email protected]>
AuthorDate: Thu Nov 5 23:20:53 2026 -0500
Add a batched version of make_docs
For a while now we were able to issue a batch of preads to the couch_file
[1].
We can use that to batch read `#doc{}` records. With this we can speed up
the
internal replicator, especially when copying lots of small documents between
shards.
Tested with small 1GB of 1KB docs, blew away a shard copy then re-created
with
the internal replicator:
main: 113-118 sec
pr: 81-85 sec
About 1.4x faster or so.
[1]
https://github.com/apache/couchdb/blob/eafed9939fe88de296d8875cf3b2ace5be3293cb/src/couch/src/couch_file.erl#L199
---
src/couch/src/couch_bt_engine.erl | 10 ++++
src/couch/src/couch_db.erl | 29 +++++++---
src/couch/src/couch_db_engine.erl | 10 ++++
src/couch/test/eunit/couch_db_doc_tests.erl | 88 ++++++++++++++++++++++++-----
src/mem3/src/mem3_rep.erl | 16 ++----
5 files changed, 123 insertions(+), 30 deletions(-)
diff --git a/src/couch/src/couch_bt_engine.erl
b/src/couch/src/couch_bt_engine.erl
index 55dcfd409..da7e34bb4 100644
--- a/src/couch/src/couch_bt_engine.erl
+++ b/src/couch/src/couch_bt_engine.erl
@@ -56,6 +56,7 @@
open_docs/2,
open_local_docs/2,
read_doc_body/2,
+ read_doc_bodies/2,
load_purge_infos/2,
serialize_doc/2,
@@ -404,6 +405,15 @@ read_doc_body(#st{} = St, #doc{} = Doc) ->
atts = Atts
}.
+read_doc_bodies(#st{} = St, Docs) when is_list(Docs) ->
+ Ptrs = [Ptr || #doc{body = Ptr} <- Docs],
+ {ok, Terms} = couch_file:pread_terms(St#st.fd, Ptrs),
+ lists:zipwith(
+ fun(#doc{} = Doc, {Body, Atts}) -> Doc#doc{body = Body, atts = Atts}
end,
+ Docs,
+ Terms
+ ).
+
load_purge_infos(St, UUIDs) ->
Results = couch_btree:lookup(St#st.purge_tree, UUIDs),
lists:map(
diff --git a/src/couch/src/couch_db.erl b/src/couch/src/couch_db.erl
index 131b58c96..438d4a3c5 100644
--- a/src/couch/src/couch_db.erl
+++ b/src/couch/src/couch_db.erl
@@ -138,6 +138,7 @@
validate_dbname/1,
make_doc/5,
+ make_docs/2,
new_revid/1
]).
@@ -2017,14 +2018,28 @@ make_doc(_Db, Id, Deleted, nil = _Bp, RevisionPath) ->
atts = [],
deleted = Deleted
};
-make_doc(#db{} = Db, Id, Deleted, Bp, {Pos, Revs}) ->
+make_doc(#db{} = Db, Id, Deleted, Bp, RevisionPath) ->
+ [Doc] = make_docs(Db, [{Id, Deleted, Bp, RevisionPath}]),
+ Doc.
+
+% Specs is [{Id, Deleted, BodyPtr, RevPath}], every spec with a body pointer
+make_docs(#db{} = Db, Specs) when is_list(Specs) ->
RevsLimit = get_revs_limit(Db),
- Doc0 = couch_db_engine:read_doc_body(Db, #doc{
- id = Id,
- revs = {Pos, lists:sublist(Revs, 1, RevsLimit)},
- body = Bp,
- deleted = Deleted
- }),
+ Docs0 = lists:map(
+ fun({Id, Del, Bp, {Pos, Revs}}) ->
+ #doc{
+ id = Id,
+ revs = {Pos, lists:sublist(Revs, 1, RevsLimit)},
+ body = Bp,
+ deleted = Del
+ }
+ end,
+ Specs
+ ),
+ Docs1 = couch_db_engine:read_doc_bodies(Db, Docs0),
+ [doc_atts_and_opts(Db, Doc) || Doc <- Docs1].
+
+doc_atts_and_opts(#db{} = Db, #doc{} = Doc0) ->
Doc1 =
case Doc0#doc.atts of
BinAtts when is_binary(BinAtts) ->
diff --git a/src/couch/src/couch_db_engine.erl
b/src/couch/src/couch_db_engine.erl
index d83e0ed9d..91f5a9859 100644
--- a/src/couch/src/couch_db_engine.erl
+++ b/src/couch/src/couch_db_engine.erl
@@ -345,6 +345,11 @@
-callback read_doc_body(DbHandle :: db_handle(), RawDoc :: doc()) ->
doc().
+% Batch version of read_doc_body/2. Returns the list of #doc{} records
+% in the same order as the RawDocs specs
+-callback read_doc_bodies(DbHandle :: db_handle(), RawDocs :: [doc()]) ->
+ [doc()].
+
% This function will be called from many contexts concurrently.
% If the storage engine has a purge_info() record for any of the
% provided UUIDs, those purge_info() records should be returned. The
@@ -707,6 +712,7 @@
open_docs/2,
open_local_docs/2,
read_doc_body/2,
+ read_doc_bodies/2,
load_purge_infos/2,
serialize_doc/2,
@@ -906,6 +912,10 @@ read_doc_body(#db{} = Db, RawDoc) ->
#db{engine = {Engine, EngineState}} = Db,
Engine:read_doc_body(EngineState, RawDoc).
+read_doc_bodies(#db{} = Db, RawDocs) when is_list(RawDocs) ->
+ #db{engine = {Engine, EngineState}} = Db,
+ Engine:read_doc_bodies(EngineState, RawDocs).
+
load_purge_infos(#db{} = Db, UUIDs) ->
#db{engine = {Engine, EngineState}} = Db,
Engine:load_purge_infos(EngineState, UUIDs).
diff --git a/src/couch/test/eunit/couch_db_doc_tests.erl
b/src/couch/test/eunit/couch_db_doc_tests.erl
index 75005df76..58b2910e7 100644
--- a/src/couch/test/eunit/couch_db_doc_tests.erl
+++ b/src/couch/test/eunit/couch_db_doc_tests.erl
@@ -40,24 +40,25 @@ couch_db_doc_test_() ->
fun setup/0,
fun teardown/1,
[
- fun should_truncate_number_of_revisions/1,
- fun should_raise_bad_request_on_invalid_rev/1,
- fun should_allow_access_in_doc_keys_test/1
+ ?TDEF_FE(t_truncate_number_of_revisions),
+ ?TDEF_FE(t_raise_bad_request_on_invalid_rev),
+ ?TDEF_FE(t_allow_access_in_doc_keys),
+ ?TDEF_FE(t_make_docs_in_batch)
]
}
}
}.
-should_truncate_number_of_revisions(DbName) ->
+t_truncate_number_of_revisions(DbName) ->
DocId = <<"foo">>,
Db = open_db(DbName),
couch_db:set_revs_limit(Db, 5),
Rev = create_doc(Db, DocId),
Rev10 = add_revisions(Db, DocId, Rev, 10),
{ok, [{ok, #doc{revs = {11, Revs}}}]} = open_doc_rev(Db, DocId, Rev10),
- ?_assertEqual(5, length(Revs)).
+ ?assertEqual(5, length(Revs)).
-should_raise_bad_request_on_invalid_rev(DbName) ->
+t_raise_bad_request_on_invalid_rev(DbName) ->
DocId = <<"foo">>,
InvalidRev1 = <<"foo">>,
InvalidRev2 = <<"a-foo">>,
@@ -65,25 +66,86 @@ should_raise_bad_request_on_invalid_rev(DbName) ->
Expect = {bad_request, <<"Invalid rev format">>},
Db = open_db(DbName),
create_doc(Db, DocId),
- [
- {InvalidRev1, ?_assertThrow(Expect, add_revisions(Db, DocId,
InvalidRev1, 1))},
- {InvalidRev2, ?_assertThrow(Expect, add_revisions(Db, DocId,
InvalidRev2, 1))},
- {InvalidRev3, ?_assertThrow(Expect, add_revisions(Db, DocId,
InvalidRev3, 1))}
- ].
+ ?assertThrow(Expect, add_revisions(Db, DocId, InvalidRev1, 1)),
+ ?assertThrow(Expect, add_revisions(Db, DocId, InvalidRev2, 1)),
+ ?assertThrow(Expect, add_revisions(Db, DocId, InvalidRev3, 1)).
-should_allow_access_in_doc_keys_test(_DbName) ->
+t_allow_access_in_doc_keys(_DbName) ->
Json = <<"{\"_id\":\"foo\",\"_access\":[\"test\"]}">>,
EJson = couch_util:json_decode(Json),
Expected = {[{<<"_id">>, <<"foo">>}, {<<"_access">>, [<<"test">>]}]},
EJson = Expected,
Doc = couch_doc:from_json_obj(EJson),
NewEJson = couch_doc:to_json_obj(Doc, []),
- ?_assertEqual(NewEJson, Expected).
+ ?assertEqual(NewEJson, Expected).
+
+t_make_docs_in_batch(DbName) ->
+ Db0 = open_db(DbName),
+ Rev = create_doc(Db0, ~"somedoc"),
+ add_revisions(Db0, ~"somedoc", Rev, 3),
+ del_doc(Db0, ~"deleted", create_doc(Db0, ~"deleted")),
+ create_att_doc(Db0, ~"att"),
+ {ok, Db} = couch_db:reopen(Db0),
+ Ids = [~"somedoc", ~"deleted", ~"att"],
+ Specs = doc_specs(Db, Ids),
+ ?assertEqual(3, length(Specs)),
+
+ Expected = [
+ couch_db:make_doc(Db, Id, Del, Ptr, Path)
+ || {Id, Del, Ptr, Path} <- Specs
+ ],
+ ?assertEqual(Expected, couch_db:make_docs(Db, Specs)),
+ ?assertEqual(lists:reverse(Expected), couch_db:make_docs(Db,
lists:reverse(Specs))),
+ ?assertEqual([], couch_db:make_docs(Db, [])),
+
+ % And the docs are the ones written
+ [#doc{revs = {4, RevIds}}] = couch_db:make_docs(Db, doc_specs(Db,
[~"somedoc"])),
+ ?assertEqual(4, length(RevIds)),
+ [#doc{deleted = true}] = couch_db:make_docs(Db, doc_specs(Db,
[~"deleted"])),
+ [#doc{atts = [Att]}] = couch_db:make_docs(Db, doc_specs(Db, [~"att"])),
+ ?assertEqual(~"a.txt", couch_att:fetch(name, Att)).
open_db(DbName) ->
{ok, Db} = couch_db:open_int(DbName, [?ADMIN_CTX]),
Db.
+doc_specs(Db, Ids) ->
+ lists:flatmap(
+ fun(Id) ->
+ #full_doc_info{rev_tree = Tree} = couch_db:get_full_doc_info(Db,
Id),
+ Leafs = couch_key_tree:get_all_leafs(Tree),
+ [{Id, Del, Ptr, Path} || {#leaf{deleted = Del, ptr = Ptr}, Path}
<- Leafs]
+ end,
+ Ids
+ ).
+
+del_doc(Db0, DocId, Rev) ->
+ {ok, Db} = couch_db:reopen(Db0),
+ {Pos, RevId} = couch_doc:parse_rev(Rev),
+ Doc = #doc{id = DocId, revs = {Pos, [RevId]}, deleted = true, body = {[]}},
+ {ok, _} = couch_db:update_doc(Db, Doc, []).
+
+create_att_doc(Db0, DocId) ->
+ {ok, Db} = couch_db:reopen(Db0),
+ Doc = couch_doc:from_json_obj(
+ {[
+ {~"_id", DocId},
+ {
+ ~"_attachments",
+ {[
+ {
+ ~"a.txt",
+ {[
+ {~"content_type", ~"text/plain"},
+ {~"data", base64:encode(~"xkcd")}
+ ]}
+ }
+ ]}
+ }
+ ]}
+ ),
+ {ok, _} = couch_db:update_doc(Db, Doc, []).
+
create_doc(Db, DocId) ->
add_revision(Db, DocId, undefined).
diff --git a/src/mem3/src/mem3_rep.erl b/src/mem3/src/mem3_rep.erl
index 55ac938cd..659adf275 100644
--- a/src/mem3/src/mem3_rep.erl
+++ b/src/mem3/src/mem3_rep.erl
@@ -844,19 +844,15 @@ chunk_revs([{Id, R, A} | Revs], {Count, Chunk}, Chunks,
Limit) ->
open_docs(Db, Infos, Missing) ->
InfoMap = #{Id => FDI || #full_doc_info{id = Id} = FDI <- Infos},
- lists:flatmap(
+ Specs = lists:flatmap(
fun({Id, Revs, _}) ->
- #full_doc_info{rev_tree = RevTree} = map_get(Id, InfoMap),
- {FoundRevs, _} = couch_key_tree:get_key_leafs(RevTree, Revs),
- lists:map(
- fun({#leaf{deleted = IsDel, ptr = SummaryPtr}, FoundRevPath})
->
- couch_db:make_doc(Db, Id, IsDel, SummaryPtr, FoundRevPath)
- end,
- FoundRevs
- )
+ #full_doc_info{rev_tree = Tree} = map_get(Id, InfoMap),
+ {Revs1, _} = couch_key_tree:get_key_leafs(Tree, Revs),
+ [{Id, Del, Ptr, Path} || {#leaf{deleted = Del, ptr = Ptr}, Path}
<- Revs1]
end,
Missing
- ).
+ ),
+ couch_db:make_docs(Db, Specs).
save_on_target(Node, Name, Docs, Timeout) ->
mem3_rpc:update_docs(