This is an automated email from the ASF dual-hosted git repository. vatamane pushed a commit to branch use-decentralized-counters-for-folsom-ets-tables in repository https://gitbox.apache.org/repos/asf/couchdb-folsom.git
commit 41b47d2b6f89d1a41a01711df057511ae6d5556f Author: Nick Vatamaniuc <[email protected]> AuthorDate: Tue Jun 13 18:20:32 2023 -0400 Use {decentralized_counters, true} for folsom ets tables Benchmarking CouchDB in a Kubernetes cluster with lock counting enabled, noticed `folsom_slide_uniform` ets folsom histogram table locks were at the top with lot of collisions and time spent waiting on them. Folsom sliding window histograms add their data using an `ets:insert/2` operation. `ets:insert/2` has to acquire an ets metadata lock in order to atomically update the ets size and memory usage. In our case, we don't need to atomically track `ets:info/2` so we can use the new (since Erlang 23+) `{decentralized_counters, true}` to reduce some of the lock contention. ``` > lcnt:rt_opt({copy_save, true}), lcnt:clear(), timer:sleep(10000), lcnt:collect(), lcnt:conflicts(). lock id #tries #collisions collisions [%] time [us] duration [%] ----- --- ------- ------------ --------------- ---------- ------------- db_hash_slot 2432 3073630 155606 5.0626 80812927 807.3885 run_queue 22 12050789 1823812 15.1344 74547468 744.7913 proc_main 64973 6287153 2020053 32.1299 19640746 196.2274 pix_lock 1024 320419 272 0.0849 3071989 30.6917 proc_status 64973 5666139 92365 1.6301 1792422 17.9078 alcu_allocator 10 1146957 108469 9.4571 1674152 16.7262 dirty_run_queue_sleep_list 2 1920837 276037 14.3707 1355878 13.5464 proc_msgq 64973 7011355 24101 0.3437 78924 0.7885 dist_entry_out_queue 7 367096 1676 0.4566 6023 0.0602 atom_tab 1 4760261 46 0.0010 4167 0.0416 port_sched_lock 101 390590 353 0.0904 1705 0.0170 dist_entry_links 6 23389 14 0.0599 323 0.0032 db_tab 399 4968232 13 0.0003 146 0.0015 drv_ev_state 128 72977 6 0.0082 40 0.0004 ``` ``` > lcnt:inspect(db_hash_slot). lock id #tries #collisions collisions [%] time [us] duration [%] histogram [log2(us)] ----- --- ------- ------------ --------------- ---------- ------------- --------------------- db_hash_slot folsom_slide_uniform 21808 13020 59.7029 7075598 70.6911 | ............XX. . | db_hash_slot folsom_slide_uniform 24165 12706 52.5802 6640890 66.3480 | .............XX.. | db_hash_slot folsom_slide_uniform 25239 10766 42.6562 5779694 57.7440 | ............XX.. | db_hash_slot folsom_slide_uniform 23089 9987 43.2544 5082403 50.7774 | ............XX... | db_hash_slot folsom_slide_uniform 23590 9598 40.6867 4994044 49.8947 | .............XX... | db_hash_slot folsom_slide_uniform 19252 6916 35.9235 4676219 46.7193 | .............Xx.... | db_hash_slot folsom_slide_uniform 26561 7277 27.3973 4173067 41.6924 | .............XX.. .. | db_hash_slot folsom_slide_uniform 19250 5136 26.6805 4018223 40.1454 | ............XX.... | db_hash_slot folsom_slide_uniform 23604 8058 34.1383 3995173 39.9151 | ............XX.. | db_hash_slot folsom_slide_uniform 22218 4877 21.9507 3647108 36.4376 | ............XX..... | db_hash_slot folsom_slide_uniform 26549 6959 26.2119 3558691 35.5543 | ............XX... | db_hash_slot folsom_slide_uniform 22716 6124 26.9590 3097839 30.9500 | ............XX.. | db_hash_slot folsom_slide_uniform 25615 5936 23.1739 2921810 29.1913 | .............XX. | db_hash_slot folsom_slide_uniform 26012 5798 22.2897 2853857 28.5124 | ............XX.. | db_hash_slot folsom_slide_uniform 25615 5137 20.0547 2486628 24.8435 | .............XX. | db_hash_slot folsom_slide_uniform 27787 4392 15.8060 2110240 21.0831 | ............XX.. | db_hash_slot folsom_slide_uniform 23168 4041 17.4422 2056501 20.5462 | .............XX. | db_hash_slot folsom_slide_uniform 21923 3927 17.9127 2055016 20.5313 | ............XX.. | db_hash_slot folsom_slide_uniform 27816 4092 14.7110 1878684 18.7696 | .............XX. | db_hash_slot folsom_slide_uniform 26001 3802 14.6225 1774354 17.7273 | ............XX.. | ``` https://www.erlang.org/blog/scalable-ets-counters/ https://www.erlang.org/doc/man/ets.html#new-2 --- include/folsom.hrl | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/include/folsom.hrl b/include/folsom.hrl index 1e0f8dd..9a498e7 100644 --- a/include/folsom.hrl +++ b/include/folsom.hrl @@ -15,25 +15,28 @@ -define(DEFAULT_INTERVAL, 5000). -define(DEFAULT_SAMPLE_TYPE, uniform). +% {decentralized_counters, true} is supported since Erlang 23+ +% In Erlang 25+ it's possible to use {write_concurrency, auto} as a replacement +% for both {write_concurrency, true}, {decentralized_counters, true} options. +% See https://www.erlang.org/doc/man/ets.html#new-2 for more info. +% +-define(ETS_OPTS, [{write_concurrency, true}, {decentralized_counters, true}, public]). + -record(spiral, { - tid = folsom_metrics_histogram_ets:new(folsom_spiral, - [set, - {write_concurrency, true}, - public]), + tid = folsom_metrics_histogram_ets:new(folsom_spiral, [set] ++ ?ETS_OPTS), server }). -record(slide, { window = ?DEFAULT_SLIDING_WINDOW, - reservoir = folsom_metrics_histogram_ets:new(folsom_slide, - [duplicate_bag, {write_concurrency, true}, public]), + reservoir = folsom_metrics_histogram_ets:new(folsom_slide, [duplicate_bag] ++ ?ETS_OPTS), server }). -record(slide_uniform, { window = ?DEFAULT_SLIDING_WINDOW, size = ?DEFAULT_SIZE, - reservoir = folsom_metrics_histogram_ets:new(folsom_slide_uniform,[set, {write_concurrency, true}, public]), + reservoir = folsom_metrics_histogram_ets:new(folsom_slide_uniform,[set] ++ ?ETS_OPTS), seed = rand:seed_s(exsplus, os:timestamp()), server }). @@ -41,7 +44,7 @@ -record(uniform, { size = ?DEFAULT_SIZE, n = 1, - reservoir = folsom_metrics_histogram_ets:new(folsom_uniform,[set, {write_concurrency, true}, public]), + reservoir = folsom_metrics_histogram_ets:new(folsom_uniform,[set] ++ ?ETS_OPTS), seed = rand:seed_s(exsplus, os:timestamp()) }). @@ -52,19 +55,19 @@ size = ?DEFAULT_SIZE, seed = rand:seed_s(exsplus, os:timestamp()), n = 1, - reservoir = folsom_metrics_histogram_ets:new(folsom_exdec,[ordered_set, {write_concurrency, true}, public]) + reservoir = folsom_metrics_histogram_ets:new(folsom_exdec,[ordered_set] ++ ?ETS_OPTS) }). -record(none, { size = ?DEFAULT_SIZE, n = 1, - reservoir = folsom_metrics_histogram_ets:new(folsom_none,[ordered_set, {write_concurrency, true}, public]) + reservoir = folsom_metrics_histogram_ets:new(folsom_none,[ordered_set] ++ ?ETS_OPTS) }). -record(slide_sorted, { size = ?DEFAULT_SIZE, n = 0, - reservoir = folsom_metrics_histogram_ets:new(folsom_slide_sorted,[ordered_set, {write_concurrency, true}, public]) + reservoir = folsom_metrics_histogram_ets:new(folsom_slide_sorted,[ordered_set] ++ ?ETS_OPTS) }). -record(histogram, {
