This is an automated email from the ASF dual-hosted git repository. chewbranca pushed a commit to branch couch-stats-resource-tracker-v2 in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 03fa0cbb106e1985d2bc43fc6f292f5d800dca96 Author: Russell Branca <[email protected]> AuthorDate: Wed Aug 21 14:10:36 2024 -0700 Rework btree stats allowing for write_node stats --- src/couch/priv/stats_descriptions.cfg | 12 ++++++-- src/couch/src/couch_btree.erl | 4 +-- .../src/couch_stats_resource_tracker.erl | 35 +++++++++++++++------- 3 files changed, 36 insertions(+), 15 deletions(-) diff --git a/src/couch/priv/stats_descriptions.cfg b/src/couch/priv/stats_descriptions.cfg index 33c556521..9d5bd1133 100644 --- a/src/couch/priv/stats_descriptions.cfg +++ b/src/couch/priv/stats_descriptions.cfg @@ -442,14 +442,22 @@ {type, counter}, {desc, <<"number of couch btree kv fold callback invocations">>} ]}. -{[couchdb, btree, kp_node], [ +{[couchdb, btree, get_node, kp_node], [ {type, counter}, {desc, <<"number of couch btree kp_nodes read">>} ]}. -{[couchdb, btree, kv_node], [ +{[couchdb, btree, get_node, kv_node], [ {type, counter}, {desc, <<"number of couch btree kv_nodes read">>} ]}. +{[couchdb, btree, write_node, kp_node], [ + {type, counter}, + {desc, <<"number of couch btree kp_nodes written">>} +]}. +{[couchdb, btree, write_node, kv_node], [ + {type, counter}, + {desc, <<"number of couch btree kv_nodes written">>} +]}. %% CSRT (couch_stats_resource_tracker) stats {[couchdb, csrt, delta_missing_t0], [ {type, counter}, diff --git a/src/couch/src/couch_btree.erl b/src/couch/src/couch_btree.erl index 27b5bc18b..ba176cca2 100644 --- a/src/couch/src/couch_btree.erl +++ b/src/couch/src/couch_btree.erl @@ -473,7 +473,7 @@ reduce_tree_size(kp_node, NodeSize, [{_K, {_P, _Red, Sz}} | NodeList]) -> get_node(#btree{fd = Fd}, NodePos) -> {ok, {NodeType, NodeList}} = couch_file:pread_term(Fd, NodePos), %% TODO: wire in csrt tracking - couch_stats:increment_counter([couchdb, btree, NodeType]), + couch_stats:increment_counter([couchdb, btree, get_node, NodeType]), {NodeType, NodeList}. write_node(#btree{fd = Fd, compression = Comp} = Bt, NodeType, NodeList) -> @@ -482,6 +482,7 @@ write_node(#btree{fd = Fd, compression = Comp} = Bt, NodeType, NodeList) -> % now write out each chunk and return the KeyPointer pairs for those nodes ToWrite = [{NodeType, Chunk} || Chunk <- Chunks], WriteOpts = [{compression, Comp}], + couch_stats:increment_counter([couchdb, btree, write_node, NodeType]), {ok, PtrSizes} = couch_file:append_terms(Fd, ToWrite, WriteOpts), {ok, group_kps(Bt, NodeType, Chunks, PtrSizes)}. @@ -1165,7 +1166,6 @@ stream_kv_node2(Bt, Reds, PrevKVs, [{K, V} | RestKVs], InRange, Dir, Fun, Acc) - false -> {stop, {PrevKVs, Reds}, Acc}; true -> - couch_stats:increment_counter([couchdb, btree, folds]), AssembledKV = assemble(Bt, K, V), case Fun(visit, AssembledKV, {PrevKVs, Reds}, Acc) of {ok, Acc2} -> diff --git a/src/couch_stats/src/couch_stats_resource_tracker.erl b/src/couch_stats/src/couch_stats_resource_tracker.erl index c3817cb90..3e75be427 100644 --- a/src/couch_stats/src/couch_stats_resource_tracker.erl +++ b/src/couch_stats/src/couch_stats_resource_tracker.erl @@ -100,6 +100,8 @@ -define(COUCH_SERVER_OPEN, db_open). -define(COUCH_BT_GET_KP_NODE, get_kp_node). -define(COUCH_BT_GET_KV_NODE, get_kv_node). +-define(COUCH_BT_WRITE_KP_NODE, write_kp_node). +-define(COUCH_BT_WRITE_KV_NODE, write_kv_node). -define(COUCH_JS_FILTER, js_filter). -define(COUCH_JS_FILTER_ERROR, js_filter_error). -define(COUCH_JS_FILTERED_DOCS, js_filtered_docs). @@ -143,7 +145,9 @@ %% TODO: switch record definitions to be macro based, eg: %% ?COUCH_BT_GET_KP_NODE = 0, get_kv_node = 0, - get_kp_node = 0 + get_kp_node = 0, + write_kv_node = 0, + write_kp_node = 0 }). %% @@ -390,8 +394,10 @@ to_json(#rctx{}=Rctx) -> js_filter_error = JSFilterErrors, js_filtered_docs = JSFilteredDocss, type = Type, - get_kp_node = KpNodes, - get_kv_node = KvNodes, + get_kp_node = GetKpNodes, + get_kv_node = GetKvNodes, + %%write_kp_node = WriteKpNodes, + %%write_kv_node = WriteKvNodes, changes_returned = ChangesReturned, ioq_calls = IoqCalls } = Rctx, @@ -412,8 +418,10 @@ to_json(#rctx{}=Rctx) -> js_filtered_docs => JSFilteredDocss, rows_read => RowsRead, type => convert_type(Type), - kp_nodes => KpNodes, - kv_nodes => KvNodes, + get_kp_nodes => GetKpNodes, + get_kv_nodes => GetKvNodes, + %%write_kp_nodes => WriteKpNodes, + %%write_kv_nodes => WriteKvNodes, changes_returned => ChangesReturned, ioq_calls => IoqCalls }. @@ -564,10 +572,17 @@ maybe_inc([fabric_rpc, view, rows_read], Val) -> inc(?ROWS_READ, Val); maybe_inc([couchdb, couch_server, open], Val) -> inc(?DB_OPEN, Val); -maybe_inc([couchdb, btree, kp_node], Val) -> +maybe_inc([couchdb, btree, get_node, kp_node], Val) -> inc(?COUCH_BT_GET_KP_NODE, Val); -maybe_inc([couchdb, btree, kv_node], Val) -> +maybe_inc([couchdb, btree, get_node, kv_node], Val) -> inc(?COUCH_BT_GET_KV_NODE, Val); +%% The write_node logic won't pickup writes as none of the RPC +%% processes actually perform the write operation +%% TODO: bubble up induced work from other processes +maybe_inc([couchdb, btree, write_node, kp_node], Val) -> + inc(?COUCH_BT_WRITE_KP_NODE, Val); +maybe_inc([couchdb, btree, write_node, kv_node], Val) -> + inc(?COUCH_BT_WRITE_KV_NODE, Val); maybe_inc([couchdb, query_server, js_filter_error], Val) -> inc(?COUCH_JS_FILTER_ERROR, Val); maybe_inc([couchdb, query_server, js_filter], Val) -> @@ -762,9 +777,6 @@ tracker({Pid, _Ref}=PidRef) -> end. log_process_lifetime_report(PidRef) -> - %% TODO: catch error out of here, report crashes on depth>1 json - %%io:format("CSRT RCTX: ~p~n", [to_flat_json(Rctx)]), - %% TODO: clean this up case is_enabled() andalso is_logging_enabled() of true -> Rctx = get_resource_raw(PidRef), @@ -782,7 +794,8 @@ is_logging_enabled() -> logging_enabled() =/= false. logging_enabled() -> - case conf_get("log_pid_usage_report", "coordinator") of + %%case conf_get("log_pid_usage_report", "coordinator") of + case conf_get("log_pid_usage_report", "true") of "coordinator" -> coordinator; "true" ->
