This is an automated email from the ASF dual-hosted git repository.

nickva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git


The following commit(s) were added to refs/heads/main by this push:
     new 8de7bf138 Add a batched version of make_docs
8de7bf138 is described below

commit 8de7bf13843c00ddf862a08c2ddfbc6f873364dd
Author: Nick Vatamaniuc <[email protected]>
AuthorDate: Thu Nov 5 23:20:53 2026 -0500

    Add a batched version of make_docs
    
    For a while now we were able to issue a batch of preads to the couch_file 
[1].
    We can use that to batch read `#doc{}` records. With this we can speed up 
the
    internal replicator, especially when copying lots of small documents between
    shards.
    
    Tested with small 1GB of 1KB docs, blew away a shard copy then re-created 
with
    the internal replicator:
    
      main: 113-118 sec
      pr: 81-85 sec
    
    About 1.4x faster or so.
    
    [1]
    
https://github.com/apache/couchdb/blob/eafed9939fe88de296d8875cf3b2ace5be3293cb/src/couch/src/couch_file.erl#L199
---
 src/couch/src/couch_bt_engine.erl           | 10 ++++
 src/couch/src/couch_db.erl                  | 29 +++++++---
 src/couch/src/couch_db_engine.erl           | 10 ++++
 src/couch/test/eunit/couch_db_doc_tests.erl | 88 ++++++++++++++++++++++++-----
 src/mem3/src/mem3_rep.erl                   | 16 ++----
 5 files changed, 123 insertions(+), 30 deletions(-)

diff --git a/src/couch/src/couch_bt_engine.erl 
b/src/couch/src/couch_bt_engine.erl
index 55dcfd409..da7e34bb4 100644
--- a/src/couch/src/couch_bt_engine.erl
+++ b/src/couch/src/couch_bt_engine.erl
@@ -56,6 +56,7 @@
     open_docs/2,
     open_local_docs/2,
     read_doc_body/2,
+    read_doc_bodies/2,
     load_purge_infos/2,
 
     serialize_doc/2,
@@ -404,6 +405,15 @@ read_doc_body(#st{} = St, #doc{} = Doc) ->
         atts = Atts
     }.
 
+read_doc_bodies(#st{} = St, Docs) when is_list(Docs) ->
+    Ptrs = [Ptr || #doc{body = Ptr} <- Docs],
+    {ok, Terms} = couch_file:pread_terms(St#st.fd, Ptrs),
+    lists:zipwith(
+        fun(#doc{} = Doc, {Body, Atts}) -> Doc#doc{body = Body, atts = Atts} 
end,
+        Docs,
+        Terms
+    ).
+
 load_purge_infos(St, UUIDs) ->
     Results = couch_btree:lookup(St#st.purge_tree, UUIDs),
     lists:map(
diff --git a/src/couch/src/couch_db.erl b/src/couch/src/couch_db.erl
index 131b58c96..438d4a3c5 100644
--- a/src/couch/src/couch_db.erl
+++ b/src/couch/src/couch_db.erl
@@ -138,6 +138,7 @@
     validate_dbname/1,
 
     make_doc/5,
+    make_docs/2,
     new_revid/1
 ]).
 
@@ -2017,14 +2018,28 @@ make_doc(_Db, Id, Deleted, nil = _Bp, RevisionPath) ->
         atts = [],
         deleted = Deleted
     };
-make_doc(#db{} = Db, Id, Deleted, Bp, {Pos, Revs}) ->
+make_doc(#db{} = Db, Id, Deleted, Bp, RevisionPath) ->
+    [Doc] = make_docs(Db, [{Id, Deleted, Bp, RevisionPath}]),
+    Doc.
+
+% Specs is [{Id, Deleted, BodyPtr, RevPath}], every spec with a body pointer
+make_docs(#db{} = Db, Specs) when is_list(Specs) ->
     RevsLimit = get_revs_limit(Db),
-    Doc0 = couch_db_engine:read_doc_body(Db, #doc{
-        id = Id,
-        revs = {Pos, lists:sublist(Revs, 1, RevsLimit)},
-        body = Bp,
-        deleted = Deleted
-    }),
+    Docs0 = lists:map(
+        fun({Id, Del, Bp, {Pos, Revs}}) ->
+            #doc{
+                id = Id,
+                revs = {Pos, lists:sublist(Revs, 1, RevsLimit)},
+                body = Bp,
+                deleted = Del
+            }
+        end,
+        Specs
+    ),
+    Docs1 = couch_db_engine:read_doc_bodies(Db, Docs0),
+    [doc_atts_and_opts(Db, Doc) || Doc <- Docs1].
+
+doc_atts_and_opts(#db{} = Db, #doc{} = Doc0) ->
     Doc1 =
         case Doc0#doc.atts of
             BinAtts when is_binary(BinAtts) ->
diff --git a/src/couch/src/couch_db_engine.erl 
b/src/couch/src/couch_db_engine.erl
index d83e0ed9d..91f5a9859 100644
--- a/src/couch/src/couch_db_engine.erl
+++ b/src/couch/src/couch_db_engine.erl
@@ -345,6 +345,11 @@
 -callback read_doc_body(DbHandle :: db_handle(), RawDoc :: doc()) ->
     doc().
 
+% Batch version of read_doc_body/2. Returns the list of #doc{} records
+% in the same order as the RawDocs specs
+-callback read_doc_bodies(DbHandle :: db_handle(), RawDocs :: [doc()]) ->
+    [doc()].
+
 % This function will be called from many contexts concurrently.
 % If the storage engine has a purge_info() record for any of the
 % provided UUIDs, those purge_info() records should be returned. The
@@ -707,6 +712,7 @@
     open_docs/2,
     open_local_docs/2,
     read_doc_body/2,
+    read_doc_bodies/2,
     load_purge_infos/2,
 
     serialize_doc/2,
@@ -906,6 +912,10 @@ read_doc_body(#db{} = Db, RawDoc) ->
     #db{engine = {Engine, EngineState}} = Db,
     Engine:read_doc_body(EngineState, RawDoc).
 
+read_doc_bodies(#db{} = Db, RawDocs) when is_list(RawDocs) ->
+    #db{engine = {Engine, EngineState}} = Db,
+    Engine:read_doc_bodies(EngineState, RawDocs).
+
 load_purge_infos(#db{} = Db, UUIDs) ->
     #db{engine = {Engine, EngineState}} = Db,
     Engine:load_purge_infos(EngineState, UUIDs).
diff --git a/src/couch/test/eunit/couch_db_doc_tests.erl 
b/src/couch/test/eunit/couch_db_doc_tests.erl
index 75005df76..58b2910e7 100644
--- a/src/couch/test/eunit/couch_db_doc_tests.erl
+++ b/src/couch/test/eunit/couch_db_doc_tests.erl
@@ -40,24 +40,25 @@ couch_db_doc_test_() ->
                 fun setup/0,
                 fun teardown/1,
                 [
-                    fun should_truncate_number_of_revisions/1,
-                    fun should_raise_bad_request_on_invalid_rev/1,
-                    fun should_allow_access_in_doc_keys_test/1
+                    ?TDEF_FE(t_truncate_number_of_revisions),
+                    ?TDEF_FE(t_raise_bad_request_on_invalid_rev),
+                    ?TDEF_FE(t_allow_access_in_doc_keys),
+                    ?TDEF_FE(t_make_docs_in_batch)
                 ]
             }
         }
     }.
 
-should_truncate_number_of_revisions(DbName) ->
+t_truncate_number_of_revisions(DbName) ->
     DocId = <<"foo">>,
     Db = open_db(DbName),
     couch_db:set_revs_limit(Db, 5),
     Rev = create_doc(Db, DocId),
     Rev10 = add_revisions(Db, DocId, Rev, 10),
     {ok, [{ok, #doc{revs = {11, Revs}}}]} = open_doc_rev(Db, DocId, Rev10),
-    ?_assertEqual(5, length(Revs)).
+    ?assertEqual(5, length(Revs)).
 
-should_raise_bad_request_on_invalid_rev(DbName) ->
+t_raise_bad_request_on_invalid_rev(DbName) ->
     DocId = <<"foo">>,
     InvalidRev1 = <<"foo">>,
     InvalidRev2 = <<"a-foo">>,
@@ -65,25 +66,86 @@ should_raise_bad_request_on_invalid_rev(DbName) ->
     Expect = {bad_request, <<"Invalid rev format">>},
     Db = open_db(DbName),
     create_doc(Db, DocId),
-    [
-        {InvalidRev1, ?_assertThrow(Expect, add_revisions(Db, DocId, 
InvalidRev1, 1))},
-        {InvalidRev2, ?_assertThrow(Expect, add_revisions(Db, DocId, 
InvalidRev2, 1))},
-        {InvalidRev3, ?_assertThrow(Expect, add_revisions(Db, DocId, 
InvalidRev3, 1))}
-    ].
+    ?assertThrow(Expect, add_revisions(Db, DocId, InvalidRev1, 1)),
+    ?assertThrow(Expect, add_revisions(Db, DocId, InvalidRev2, 1)),
+    ?assertThrow(Expect, add_revisions(Db, DocId, InvalidRev3, 1)).
 
-should_allow_access_in_doc_keys_test(_DbName) ->
+t_allow_access_in_doc_keys(_DbName) ->
     Json = <<"{\"_id\":\"foo\",\"_access\":[\"test\"]}">>,
     EJson = couch_util:json_decode(Json),
     Expected = {[{<<"_id">>, <<"foo">>}, {<<"_access">>, [<<"test">>]}]},
     EJson = Expected,
     Doc = couch_doc:from_json_obj(EJson),
     NewEJson = couch_doc:to_json_obj(Doc, []),
-    ?_assertEqual(NewEJson, Expected).
+    ?assertEqual(NewEJson, Expected).
+
+t_make_docs_in_batch(DbName) ->
+    Db0 = open_db(DbName),
+    Rev = create_doc(Db0, ~"somedoc"),
+    add_revisions(Db0, ~"somedoc", Rev, 3),
+    del_doc(Db0, ~"deleted", create_doc(Db0, ~"deleted")),
+    create_att_doc(Db0, ~"att"),
+    {ok, Db} = couch_db:reopen(Db0),
+    Ids = [~"somedoc", ~"deleted", ~"att"],
+    Specs = doc_specs(Db, Ids),
+    ?assertEqual(3, length(Specs)),
+
+    Expected = [
+        couch_db:make_doc(Db, Id, Del, Ptr, Path)
+     || {Id, Del, Ptr, Path} <- Specs
+    ],
+    ?assertEqual(Expected, couch_db:make_docs(Db, Specs)),
+    ?assertEqual(lists:reverse(Expected), couch_db:make_docs(Db, 
lists:reverse(Specs))),
+    ?assertEqual([], couch_db:make_docs(Db, [])),
+
+    % And the docs are the ones written
+    [#doc{revs = {4, RevIds}}] = couch_db:make_docs(Db, doc_specs(Db, 
[~"somedoc"])),
+    ?assertEqual(4, length(RevIds)),
+    [#doc{deleted = true}] = couch_db:make_docs(Db, doc_specs(Db, 
[~"deleted"])),
+    [#doc{atts = [Att]}] = couch_db:make_docs(Db, doc_specs(Db, [~"att"])),
+    ?assertEqual(~"a.txt", couch_att:fetch(name, Att)).
 
 open_db(DbName) ->
     {ok, Db} = couch_db:open_int(DbName, [?ADMIN_CTX]),
     Db.
 
+doc_specs(Db, Ids) ->
+    lists:flatmap(
+        fun(Id) ->
+            #full_doc_info{rev_tree = Tree} = couch_db:get_full_doc_info(Db, 
Id),
+            Leafs = couch_key_tree:get_all_leafs(Tree),
+            [{Id, Del, Ptr, Path} || {#leaf{deleted = Del, ptr = Ptr}, Path} 
<- Leafs]
+        end,
+        Ids
+    ).
+
+del_doc(Db0, DocId, Rev) ->
+    {ok, Db} = couch_db:reopen(Db0),
+    {Pos, RevId} = couch_doc:parse_rev(Rev),
+    Doc = #doc{id = DocId, revs = {Pos, [RevId]}, deleted = true, body = {[]}},
+    {ok, _} = couch_db:update_doc(Db, Doc, []).
+
+create_att_doc(Db0, DocId) ->
+    {ok, Db} = couch_db:reopen(Db0),
+    Doc = couch_doc:from_json_obj(
+        {[
+            {~"_id", DocId},
+            {
+                ~"_attachments",
+                {[
+                    {
+                        ~"a.txt",
+                        {[
+                            {~"content_type", ~"text/plain"},
+                            {~"data", base64:encode(~"xkcd")}
+                        ]}
+                    }
+                ]}
+            }
+        ]}
+    ),
+    {ok, _} = couch_db:update_doc(Db, Doc, []).
+
 create_doc(Db, DocId) ->
     add_revision(Db, DocId, undefined).
 
diff --git a/src/mem3/src/mem3_rep.erl b/src/mem3/src/mem3_rep.erl
index 55ac938cd..659adf275 100644
--- a/src/mem3/src/mem3_rep.erl
+++ b/src/mem3/src/mem3_rep.erl
@@ -844,19 +844,15 @@ chunk_revs([{Id, R, A} | Revs], {Count, Chunk}, Chunks, 
Limit) ->
 
 open_docs(Db, Infos, Missing) ->
     InfoMap = #{Id => FDI || #full_doc_info{id = Id} = FDI <- Infos},
-    lists:flatmap(
+    Specs = lists:flatmap(
         fun({Id, Revs, _}) ->
-            #full_doc_info{rev_tree = RevTree} = map_get(Id, InfoMap),
-            {FoundRevs, _} = couch_key_tree:get_key_leafs(RevTree, Revs),
-            lists:map(
-                fun({#leaf{deleted = IsDel, ptr = SummaryPtr}, FoundRevPath}) 
->
-                    couch_db:make_doc(Db, Id, IsDel, SummaryPtr, FoundRevPath)
-                end,
-                FoundRevs
-            )
+            #full_doc_info{rev_tree = Tree} = map_get(Id, InfoMap),
+            {Revs1, _} = couch_key_tree:get_key_leafs(Tree, Revs),
+            [{Id, Del, Ptr, Path} || {#leaf{deleted = Del, ptr = Ptr}, Path} 
<- Revs1]
         end,
         Missing
-    ).
+    ),
+    couch_db:make_docs(Db, Specs).
 
 save_on_target(Node, Name, Docs, Timeout) ->
     mem3_rpc:update_docs(

Reply via email to