This is an automated email from the ASF dual-hosted git repository.

nickva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git


The following commit(s) were added to refs/heads/main by this push:
     new 0185416bf Optimize B-tree chunk size default
0185416bf is described below

commit 0185416bf0cf8427d295de096742bd668ce330b9
Author: Nick Vatamaniuc <[email protected]>
AuthorDate: Wed Sep 16 10:46:56 2026 -0400

    Optimize B-tree chunk size default
    
    Switch from 1279 to 4096
    
    The chunk size is the limit at which B-tree nodes are split. 1279 default 
was
    set 18+ [0] years ago by Damian after benchmarking on his personal laptop 
with
    a spinning hard drive [1]. Nowadays most disks are SSDs and usually have a 
4KB
    block size. We may read and write 1279 bytes but a modern disk will still do
    the IO work to read 4KB blocks then throw the rest away. So it makes sense 
to
    adjust it a bit to avoid wasting all that IO bandwidth for nothing.
    
    Besides just the increased IO having a such a low default makes our B-tree
    deeper which in turn generates even more IO and inflates our "active" size.
    Raising the threshold to 4096 reduces the tree depth. For example on 200K 
doc
    couch file Id tree went from depth 6 to 4 and the Seq tree from 4 to 3. This
    also reduced the active size by about 25%. (To get an idea of what the 
B-tree
    depths are there is a new depth function in `couch_btree` module and a 
bunch of
    helper functions in `couch_debug` calling it).
    
    The value of 4096 was first published in a research paper [2] by Chung et 
al.
    on optimizing CouchDB for their embedded storage application. They saw a 20%
    TPS improvement with 4096 chunk size. To validate the setting we ran a k6
    benchmark test on a large cluster with concurrent doc updates and reads.
    
    Our test setup:
      12 node cluster
      Q=32, N=3
      10M docs, size 256B
      1000 reads/s
      200 writes/s
      1000 writes/s via _bulk_docs
      4 1 hour trials with each setting
    
    Setting changed:
     btree_chunk_size 1279 -> 4096
    
    Results:
      | Scenario   | P50 (msec)                | P90 (msec)             |
      |------------|---------------------------|------------------------|
      | doc_get    | 6.9 -> 5.1 (-27%)         | 11.1 -> 6.4 (-43%)     |
      | doc_insert | 12.2 -> 8.3 (-32%)        | 37.5 -> 10.9 (-71%)    |
      | doc_update | 14.1 -> 9.0 (-36%)        | 44.4 -> 12.2 (-73%)    |
      | bulk_docs  | 101 -> 64 (-37%)          | 136 -> 83 (-39%)       |
    
    We do even better than what Chung et al. found. It could be because we now 
have
    a B-tree block cache which favors shallow B-trees as it prioritizes nodes
    closer to the root.
    
    [0]
    
https://github.com/apache/couchdb/commit/10881200f1d6761d504badb2e12a16a38033c721
    > Damien F. Katz committed on Jul 28, 2008
    
    [1]
    
https://web.archive.org/web/20120315120536/http://ayende.com/blog/3609/more-couchdb-reading-btree-query-modify
    
    > The ideal size of CHUNK_THRESHOLD was determined to be 0x4FF when timing 
some
    test runs on my MacBook Pro.
    
    [2] https://arxiv.org/pdf/1912.11456 Performance Tuning and Scaling 
Enterprise
    Blockchain Applications. Grant Chung et al.
---
 rel/overlay/etc/default.ini                |  2 +-
 src/couch/src/couch_btree.erl              | 16 +++++++-
 src/couch/src/couch_debug.erl              | 59 +++++++++++++++++++++++++++++-
 src/couch/test/eunit/couch_btree_tests.erl | 54 ++++++++++++++++++++++++++-
 4 files changed, 125 insertions(+), 6 deletions(-)

diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini
index 866b0c79a..57263c8a9 100644
--- a/rel/overlay/etc/default.ini
+++ b/rel/overlay/etc/default.ini
@@ -47,7 +47,7 @@ view_index_dir = {{view_index_dir}}
 ;               local dbs on :5986 are read/write for admins only
 ;default_security = admin_only
 
-;btree_chunk_size = 1279
+;btree_chunk_size = 4096
 
 ; When set to "true" the node will stop processing interactive requests and
 ; return "status":"maintenance_mode" from the /_up endpoint. If a load balancer
diff --git a/src/couch/src/couch_btree.erl b/src/couch/src/couch_btree.erl
index 1c349d002..0b4522f56 100644
--- a/src/couch/src/couch_btree.erl
+++ b/src/couch/src/couch_btree.erl
@@ -17,11 +17,11 @@
 -export([fold_reduce/4, lookup/2, set_options/2]).
 -export([is_btree/1, get_state/1, get_fd/1, get_reduce_fun/1]).
 -export([extract/2, assemble/3, less/3]).
--export([get_chunk_size/0]).
+-export([get_chunk_size/0, depth/1]).
 
 -include_lib("couch/include/couch_db.hrl").
 
--define(DEFAULT_CHUNK_SIZE, 1279).
+-define(DEFAULT_CHUNK_SIZE, 4096).
 
 % For the btree cache, the priority of the root node will be
 % this value. The priority is roughly how many cleanup interval
@@ -158,6 +158,18 @@ size(#btree{root = {_P, _Red}}) ->
 size(#btree{root = {_P, _Red, Size}}) ->
     Size.
 
+depth(#btree{root = nil}) ->
+    0;
+depth(#btree{} = Bt) ->
+    depth(Bt, Bt#btree.root, 1).
+
+depth(Bt, Node, Depth) ->
+    Pointer = element(1, Node),
+    case get_node(Bt, Pointer, Depth) of
+        {kv_node, _} -> Depth;
+        {kp_node, [{_Key, First} | _]} -> depth(Bt, First, Depth + 1)
+    end.
+
 get_group_fun(Bt, Options) ->
     case couch_util:get_value(key_group_level, Options) of
         exact ->
diff --git a/src/couch/src/couch_debug.erl b/src/couch/src/couch_debug.erl
index 2cc0a94b4..257d311a5 100644
--- a/src/couch/src/couch_debug.erl
+++ b/src/couch/src/couch_debug.erl
@@ -62,6 +62,15 @@
     print_tree/2
 ]).
 
+-export([
+    btree_depths/1,
+    btree_depths_local/1
+]).
+
+-include_lib("couch/include/couch_db.hrl").
+-include("couch_db_int.hrl").
+-include("couch_bt_engine.hrl").
+
 -define(PING_TIMEOUT_IN_MS, 60000).
 
 -type throw(_Reason) :: no_return().
@@ -105,7 +114,10 @@ help() ->
         dead_nodes,
         ping,
         ping_nodes,
-        node_events
+        node_events,
+        btree_depths,
+        btree_depths_local,
+        btree_depths_cluster
     ].
 
 -spec help(Function :: function_name()) -> ok.
@@ -509,6 +521,22 @@ help(node_events) ->
 
         ---
     ", []);
+help(btree_depths) ->
+    io:format("
+    btree_depths(ShardName)
+    -----------------------
+
+    Get btree depths of ID and Seq trees.
+    ---
+    ", []);
+help(btree_depths_local) ->
+    io:format("
+    btree_depths_local(DbName)
+    --------------------------
+
+    Get btree depths of ID and Seq trees for all local shards of db.
+    ---
+    ", []);
 help(Unknown) ->
     io:format("Unknown function: `~p`. Please try one of the following:~n", 
[Unknown]),
     [io:format("    - ~s~n", [Function]) || Function <- help()],
@@ -1007,6 +1035,35 @@ ping_nodes(Nodes) ->
 ping_nodes(Nodes, Timeout) ->
     mem3:ping_nodes(Nodes, Timeout).
 
+btree_depths(DbName) when is_binary(DbName) ->
+    couch_util:with_db(DbName, fun db_btree_depths/1).
+
+btree_depths_local(DbName) when is_binary(DbName) ->
+    try mem3:local_shards(DbName) of
+        Shards ->
+            [shard_btree_depths(mem3:name(S)) || S <- Shards]
+    catch
+        error:database_does_not_exist ->
+            {error, database_does_not_exist}
+    end.
+
+shard_btree_depths(ShardName) ->
+    try
+        {ShardName, btree_depths(ShardName)}
+    catch
+        Tag:Err ->
+            {ShardName, {error, {Tag, Err}}}
+    end.
+
+db_btree_depths(#db{engine = {couch_bt_engine, St}}) ->
+    Trees = [
+        {id_tree, St#st.id_tree},
+        {seq_tree, St#st.seq_tree}
+    ],
+    [{Name, couch_btree:depth(Tree)} || {Name, Tree} <- Trees, 
couch_btree:is_btree(Tree)];
+db_btree_depths(#db{engine = {Engine, _}}) ->
+    {error, {unsupported_engine, Engine}}.
+
 node_events() ->
     mem3_distribution:events().
 
diff --git a/src/couch/test/eunit/couch_btree_tests.erl 
b/src/couch/test/eunit/couch_btree_tests.erl
index f365a37cf..18bc06576 100644
--- a/src/couch/test/eunit/couch_btree_tests.erl
+++ b/src/couch/test/eunit/couch_btree_tests.erl
@@ -39,7 +39,7 @@ setup_kvs_with_cache(_) ->
     {Fd, Btree}.
 
 setup_kvs_with_small_chunk_size(_) ->
-    % Less than a 1/4 than current default 1279
+    % Well below the default chunk size
     config:set("couchdb", "btree_chunk_size", "300", false),
     {ok, Fd} = couch_file:open(?tempfile(), [create, overwrite]),
     {ok, Btree} = couch_btree:open(nil, Fd, [
@@ -49,7 +49,7 @@ setup_kvs_with_small_chunk_size(_) ->
     {Fd, Btree}.
 
 setup_kvs_with_large_chunk_size(_) ->
-    % About 4x than current default 1279
+    % Above the default chunk size
     config:set("couchdb", "btree_chunk_size", "5000", false),
     {ok, Fd} = couch_file:open(?tempfile(), [create, overwrite]),
     {ok, Btree} = couch_btree:open(nil, Fd, [
@@ -771,3 +771,53 @@ test_traversal_callbacks(Btree, _KeyValues) ->
     % With 250 items the root is a kp. Always skipping should reduce to true.
     {ok, _, true} = couch_btree:fold(Btree, FoldFun, true, [{dir, fwd}]),
     ok.
+
+depth_test_() ->
+    {
+        setup,
+        fun() -> test_util:start(?MODULE, [ioq]) end,
+        fun test_util:stop/1,
+        {
+            foreach,
+            fun setup/0,
+            fun teardown/1,
+            [
+                fun should_have_zero_depth_for_empty_btree/1,
+                fun should_have_depth_one_after_insert/1,
+                fun should_depth_increase_with_more_keys/1,
+                fun should_depth_return_zero_after_all_keys_removed/1
+            ]
+        }
+    }.
+
+should_have_zero_depth_for_empty_btree({_, Btree}) ->
+    ?_assertEqual(0, couch_btree:depth(Btree)).
+
+should_have_depth_one_after_insert({_, Btree}) ->
+    ?_test(begin
+        {ok, Btree1} = couch_btree:add_remove(Btree, [{1, a}], []),
+        ?assertEqual(1, couch_btree:depth(Btree1))
+    end).
+
+should_depth_increase_with_more_keys({_, Btree}) ->
+    ?_test(begin
+        config:set("couchdb", "btree_chunk_size", "1279", false),
+        % A few keys fit in a single kv_node
+        SmallKVs = [{K, K} || K <- lists:seq(1, 5)],
+        {ok, Btree1} = couch_btree:add_remove(Btree, SmallKVs, []),
+        ?assertEqual(1, couch_btree:depth(Btree1)),
+        % More kv should spill into level 2
+        LargeKVs = [{K, K} || K <- lists:seq(1, ?ROWS)],
+        {ok, Btree2} = couch_btree:add_remove(Btree1, LargeKVs, []),
+        ?assertEqual(2, couch_btree:depth(Btree2))
+    end).
+
+should_depth_return_zero_after_all_keys_removed({_, Btree}) ->
+    ?_test(begin
+        KVs = [{K, K} || K <- lists:seq(1, 10)],
+        {ok, Btree1} = couch_btree:add_remove(Btree, KVs, []),
+        ?assertEqual(1, couch_btree:depth(Btree1)),
+        Keys = [K || {K, _} <- KVs],
+        {ok, Btree2} = couch_btree:add_remove(Btree1, [], Keys),
+        ?assertEqual(0, couch_btree:depth(Btree2))
+    end).

Reply via email to