This is an automated email from the ASF dual-hosted git repository. ronny pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/couchdb.git
The following commit(s) were added to refs/heads/main by this push: new b6ea325cc Add new HTTP endpoint `/_node/_local/_smoosh/status`. (#4766) b6ea325cc is described below commit b6ea325ccb1e3de45f8541ca4fde225ac75f7d54 Author: Ronny Berndt <ro...@apache.org> AuthorDate: Fri Sep 22 20:44:50 2023 +0200 Add new HTTP endpoint `/_node/_local/_smoosh/status`. (#4766) Introduce a new HTTP endpoint `/_node/_local/_smoosh/status` to get status information from the CouchDB auto-compaction daemon. Previously, this was only possible by starting a `remsh` session and manually calling the `smoosh:status/1` function. The internal data structures of `smoosh:status/1` are migrated into Erlang maps to send them directly as json to the client. To add more status information to smoosh in the future, the available information will be stored under the json key `channels`. Example: { "channels": { "ratio_dbs": { ... }, "slack_dbs": { ... }, ... } --- src/chttpd/src/chttpd_node.erl | 6 ++ src/docs/src/api/server/common.rst | 110 +++++++++++++++++++++++++++++++ src/smoosh/src/smoosh_channel.erl | 14 ++-- src/smoosh/src/smoosh_persist.erl | 10 +-- src/smoosh/src/smoosh_priority_queue.erl | 31 ++++----- src/smoosh/src/smoosh_server.erl | 16 +++-- src/smoosh/test/smoosh_tests.erl | 43 ++++++------ 7 files changed, 174 insertions(+), 56 deletions(-) diff --git a/src/chttpd/src/chttpd_node.erl b/src/chttpd/src/chttpd_node.erl index 46850fc4e..165b85a42 100644 --- a/src/chttpd/src/chttpd_node.erl +++ b/src/chttpd/src/chttpd_node.erl @@ -39,6 +39,12 @@ handle_node_req(#httpd{path_parts = [_, <<"_local">>]} = Req) -> send_json(Req, 200, {[{name, node()}]}); handle_node_req(#httpd{path_parts = [A, <<"_local">> | Rest]} = Req) -> handle_node_req(Req#httpd{path_parts = [A, node()] ++ Rest}); +% GET /_node/$node/_smoosh/status +handle_node_req(#httpd{method = 'GET', path_parts = [_, _Node, <<"_smoosh">>, <<"status">>]} = Req) -> + {ok, Status} = smoosh:status(), + send_json(Req, 200, Status); +handle_node_req(#httpd{path_parts = [_, _Node, <<"_smoosh">>, <<"status">>]} = Req) -> + send_method_not_allowed(Req, "GET"); % GET /_node/$node/_versions handle_node_req(#httpd{method = 'GET', path_parts = [_, _Node, <<"_versions">>]} = Req) -> IcuVer = couch_ejson_compare:get_icu_version(), diff --git a/src/docs/src/api/server/common.rst b/src/docs/src/api/server/common.rst index 9e645f649..783b8120a 100644 --- a/src/docs/src/api/server/common.rst +++ b/src/docs/src/api/server/common.rst @@ -1902,6 +1902,116 @@ See :ref:`Configuration of Prometheus Endpoint <config/prometheus>` for details. Accept: text/plain Host: localhost:17986 +.. _api/server/smoosh/status: + +===================================== +``/_node/{node-name}/_smoosh/status`` +===================================== + +.. versionadded:: 3.4 + +.. http:get:: /_node/{node-name}/_smoosh/status + :synopsis: Returns metrics of the CouchDB's auto-compaction daemon + + This prints the state of each channel, how many jobs they are + currently running and how many jobs are enqueued (as well as the + lowest and highest priority of those enqueued items). The idea is to + provide, at a glance, sufficient insight into ``smoosh`` that an operator + can assess whether ``smoosh`` is adequately targeting the reclaimable + space in the cluster. + + In general, a healthy status output will have + items in the ``ratio_dbs`` and ``ratio_views`` channels. Owing to the default + settings, the ``slack_dbs`` and ``slack_views`` will almost certainly have + items in them. Historically, we've not found that the slack channels, + on their own, are particularly adept at keeping things well compacted. + + :code 200: Request completed successfully + :code 401: CouchDB Server Administrator privileges required + + **Request**: + + .. code-block:: http + + GET /_node/_local/_smoosh/status HTTP/1.1 + Host: 127.0.0.1:5984 + Accept: */* + + **Response**: + + .. code-block:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "channels": { + "slack_dbs": { + "starting": 0, + "waiting": { + "size": 0, + "min": 0, + "max": 0 + }, + "active": 0 + }, + "ratio_dbs": { + "starting": 0, + "waiting": { + "size": 56, + "min": 1.125, + "max": 11.0625 + }, + "active": 0 + }, + "ratio_views": { + "starting": 0, + "waiting": { + "size": 0, + "min": 0, + "max": 0 + }, + "active": 0 + }, + "upgrade_dbs": { + "starting": 0, + "waiting": { + "size": 0, + "min": 0, + "max": 0 + }, + "active": 0 + }, + "slack_views": { + "starting": 0, + "waiting": { + "size": 0, + "min": 0, + "max": 0 + }, + "active": 0 + }, + "upgrade_views": { + "starting": 0, + "waiting": { + "size": 0, + "min": 0, + "max": 0 + }, + "active": 0 + }, + "index_cleanup": { + "starting": 0, + "waiting": { + "size": 0, + "min": 0, + "max": 0 + }, + "active": 0 + } + } + } + .. _api/server/system: ============================== diff --git a/src/smoosh/src/smoosh_channel.erl b/src/smoosh/src/smoosh_channel.erl index 92fd3413b..3cfbcdec6 100644 --- a/src/smoosh/src/smoosh_channel.erl +++ b/src/smoosh/src/smoosh_channel.erl @@ -77,10 +77,10 @@ enqueue(ServerRef, Object, Priority) -> get_status(StatusTab) when is_reference(StatusTab) -> try ets:lookup(StatusTab, status) of [{status, Status}] -> Status; - [] -> [] + [] -> #{} catch error:badarg -> - [] + #{} end. close(ServerRef) -> @@ -235,11 +235,11 @@ unpersist(Name) -> % set_status(#state{} = State) -> #state{active = Active, starting = Starting, waiting = Waiting} = State, - Status = [ - {active, map_size(Active)}, - {starting, map_size(Starting)}, - {waiting, smoosh_priority_queue:info(Waiting)} - ], + Status = #{ + active => map_size(Active), + starting => map_size(Starting), + waiting => smoosh_priority_queue:info(Waiting) + }, true = ets:insert(State#state.stab, {status, Status}), State. diff --git a/src/smoosh/src/smoosh_persist.erl b/src/smoosh/src/smoosh_persist.erl index 2feab7e69..c1519f65f 100644 --- a/src/smoosh/src/smoosh_persist.erl +++ b/src/smoosh/src/smoosh_persist.erl @@ -225,7 +225,7 @@ t_persist_unpersist_disabled(_) -> Q2 = unpersist(Name), ?assertEqual(Name, smoosh_priority_queue:name(Q2)), - ?assertEqual([{size, 0}], smoosh_priority_queue:info(Q2)). + ?assertEqual(#{max => 0, min => 0, size => 0}, smoosh_priority_queue:info(Q2)). t_persist_unpersist_enabled(_) -> Name = "chan2", @@ -241,7 +241,7 @@ t_persist_unpersist_enabled(_) -> Q2 = unpersist(Name), ?assertEqual(Name, smoosh_priority_queue:name(Q2)), Info2 = smoosh_priority_queue:info(Q2), - ?assertEqual([{size, 3}, {min, 1.0}, {max, infinity}], Info2), + ?assertEqual(#{max => infinity, min => 1.0, size => 3}, Info2), ?assertEqual(Keys, drain_q(Q2)), % Try to persist the already unpersisted queue @@ -249,7 +249,7 @@ t_persist_unpersist_enabled(_) -> Q3 = unpersist(Name), ?assertEqual(Name, smoosh_priority_queue:name(Q3)), Info3 = smoosh_priority_queue:info(Q2), - ?assertEqual([{size, 3}, {min, 1.0}, {max, infinity}], Info3), + ?assertEqual(#{max => infinity, min => 1.0, size => 3}, Info3), ?assertEqual(Keys, drain_q(Q3)). t_persist_unpersist_errors(_) -> @@ -267,7 +267,7 @@ t_persist_unpersist_errors(_) -> Q2 = unpersist(Name), ?assertEqual(Name, smoosh_priority_queue:name(Q2)), - ?assertEqual([{size, 0}], smoosh_priority_queue:info(Q2)), + ?assertEqual(#{max => 0, min => 0, size => 0}, smoosh_priority_queue:info(Q2)), Dir = state_dir(), ok = file:make_dir(Dir), @@ -278,7 +278,7 @@ t_persist_unpersist_errors(_) -> Q3 = unpersist(Name), ?assertEqual(Name, smoosh_priority_queue:name(Q3)), - ?assertEqual([{size, 0}], smoosh_priority_queue:info(Q3)), + ?assertEqual(#{max => 0, min => 0, size => 0}, smoosh_priority_queue:info(Q3)), ok = file:del_dir_r(Dir). diff --git a/src/smoosh/src/smoosh_priority_queue.erl b/src/smoosh/src/smoosh_priority_queue.erl index b2ef4393d..2f2fba687 100644 --- a/src/smoosh/src/smoosh_priority_queue.erl +++ b/src/smoosh/src/smoosh_priority_queue.erl @@ -64,17 +64,14 @@ qsize(#priority_queue{tree = Tree}) -> gb_trees:size(Tree). info(#priority_queue{tree = Tree} = Q) -> - [ - {size, qsize(Q)} - | case gb_trees:is_empty(Tree) of - true -> - []; - false -> - {{Min, _}, _} = gb_trees:smallest(Tree), - {{Max, _}, _} = gb_trees:largest(Tree), - [{min, Min}, {max, Max}] - end - ]. + case gb_trees:is_empty(Tree) of + true -> + #{size => qsize(Q), min => 0, max => 0}; + false -> + {{Min, _}, _} = gb_trees:smallest(Tree), + {{Max, _}, _} = gb_trees:largest(Tree), + #{size => qsize(Q), min => Min, max => Max} + end. insert(Key, Priority, Capacity, #priority_queue{tree = Tree, map = Map} = Q) -> TreeKey = {Priority, make_ref()}, @@ -122,7 +119,7 @@ basics_test() -> Q = new("foo"), ?assertMatch(#priority_queue{}, Q), ?assertEqual("foo", name(Q)), - ?assertEqual([{size, 0}], info(Q)). + ?assertEqual(0, maps:get(size, info(Q))). empty_test() -> Q = new("foo"), @@ -136,7 +133,7 @@ one_element_test() -> Q0 = new("foo"), Q = in(?K1, ?P1, 1, Q0), ?assertMatch(#priority_queue{}, Q), - ?assertEqual([{size, 1}, {min, 1}, {max, 1}], info(Q)), + ?assertEqual(#{max => 1, min => 1, size => 1}, info(Q)), ?assertEqual(Q, truncate(1, Q)), ?assertMatch({?K1, #priority_queue{}}, out(Q)), {?K1, Q2} = out(Q), @@ -144,7 +141,7 @@ one_element_test() -> ?assertEqual(#{?K1 => ?P1}, to_map(Q)), Q3 = from_map("foo", 1, to_map(Q)), ?assertEqual("foo", name(Q3)), - ?assertEqual([{size, 1}, {min, ?P1}, {max, ?P1}], info(Q3)), + ?assertEqual(#{max => ?P1, min => ?P1, size => 1}, info(Q3)), ?assertEqual(to_map(Q), to_map(Q3)), ?assertEqual(Q0, flush(Q)). @@ -153,7 +150,7 @@ multiple_elements_basics_test() -> Q1 = in(?K1, ?P1, 10, Q0), Q2 = in(?K2, ?P2, 10, Q1), Q3 = in(?K3, ?P3, 10, Q2), - ?assertEqual([{size, 3}, {min, ?P1}, {max, ?P3}], info(Q3)), + ?assertEqual(#{max => ?P3, min => ?P1, size => 3}, info(Q3)), ?assertEqual([?K3, ?K2, ?K1], drain(Q3)). update_element_same_priority_test() -> @@ -166,7 +163,7 @@ update_element_new_priority_test() -> Q1 = in(?K1, ?P1, 10, Q0), Q2 = in(?K2, ?P2, 10, Q1), Q3 = in(?K1, ?P3, 10, Q2), - ?assertEqual([{size, 2}, {min, ?P2}, {max, ?P3}], info(Q3)), + ?assertEqual(#{max => ?P3, min => ?P2, size => 2}, info(Q3)), ?assertEqual([?K1, ?K2], drain(Q3)). capacity_test() -> @@ -189,7 +186,7 @@ a_lot_of_elements_test() -> lists:seq(1, N) ), Q = from_map("foo", N, maps:from_list(KVs)), - ?assertMatch([{size, N} | _], info(Q)), + ?assertMatch(N, maps:get(size, info(Q))), {_, Priorities} = lists:unzip(drain(Q)), ?assertEqual(lists:reverse(lists:sort(Priorities)), Priorities). diff --git a/src/smoosh/src/smoosh_server.erl b/src/smoosh/src/smoosh_server.erl index 10368a549..3b0b86808 100644 --- a/src/smoosh/src/smoosh_server.erl +++ b/src/smoosh/src/smoosh_server.erl @@ -96,12 +96,14 @@ flush() -> gen_server:call(?MODULE, flush, infinity). status() -> - try ets:foldl(fun get_channel_status/2, [], ?MODULE) of - Res -> {ok, Res} - catch - error:badarg -> - {ok, []} - end. + ChannelsStatus = + try ets:foldl(fun get_channel_status/2, #{}, ?MODULE) of + Res -> Res + catch + error:badarg -> + #{} + end, + {ok, #{channels => ChannelsStatus}}. enqueue(Object0) -> Object = smoosh_utils:validate_arg(Object0), @@ -286,7 +288,7 @@ remove_enqueue_ref(Ref, #state{} = State) when is_reference(Ref) -> get_channel_status(#channel{name = Name, stab = Tab}, Acc) -> Status = smoosh_channel:get_status(Tab), - [{Name, Status} | Acc]; + Acc#{list_to_atom(Name) => Status}; get_channel_status(_, Acc) -> Acc. diff --git a/src/smoosh/test/smoosh_tests.erl b/src/smoosh/test/smoosh_tests.erl index 622cabc8e..6861db5e1 100644 --- a/src/smoosh/test/smoosh_tests.erl +++ b/src/smoosh/test/smoosh_tests.erl @@ -82,21 +82,22 @@ teardown(DbName) -> config:delete("smoosh", "cleanup_index_files", false). t_default_channels(_) -> + ChannelStatus = maps:get(channels, status()), ?assertMatch( [ - {"index_cleanup", _}, - {"ratio_dbs", _}, - {"ratio_views", _}, - {"slack_dbs", _}, - {"slack_views", _}, - {"upgrade_dbs", _}, - {"upgrade_views", _} + index_cleanup, + ratio_dbs, + ratio_views, + slack_dbs, + slack_views, + upgrade_dbs, + upgrade_views ], - status() + lists:sort(maps:keys(ChannelStatus)) ), % If app hasn't started status won't crash application:stop(smoosh), - ?assertEqual([], status()). + ?assertEqual(#{channels => #{}}, status()). t_channels_recreated_on_crash(_) -> RatioDbsPid = get_channel_pid("ratio_dbs"), @@ -104,7 +105,8 @@ t_channels_recreated_on_crash(_) -> exit(RatioDbsPid, kill), meck:wait(1, smoosh_channel, start_link, 1, 3000), wait_for_channels(7), - ?assertMatch([_, {"ratio_dbs", _} | _], status()), + ChannelStatus = maps:get(channels, status()), + ?assertMatch(true, maps:is_key(ratio_dbs, ChannelStatus)), ?assertNotEqual(RatioDbsPid, get_channel_pid("ratio_dbs")). t_can_create_and_delete_channels(_) -> @@ -402,17 +404,17 @@ delete_doc(DbName, DDocId) -> status() -> {ok, Props} = smoosh:status(), - lists:keysort(1, Props). + Props. status(Channel) -> - case lists:keyfind(Channel, 1, status()) of - {_, Val} -> - Val, - Active = proplists:get_value(active, Val), - Starting = proplists:get_value(starting, Val), - WaitingInfo = proplists:get_value(waiting, Val), - Waiting = proplists:get_value(size, WaitingInfo), - {Active, Starting, Waiting}; + ChannelStatus = maps:get(channels, status()), + ChannelAtom = list_to_atom(Channel), + case maps:is_key(ChannelAtom, ChannelStatus) of + true -> + #{active := Active, starting := Starting, waiting := Waiting} = maps:get( + ChannelAtom, ChannelStatus + ), + {Active, Starting, maps:get(size, Waiting)}; false -> false end. @@ -443,7 +445,8 @@ wait_for_channels() -> wait_for_channels(N) when is_integer(N), N >= 0 -> WaitFun = fun() -> - case length(status()) of + ChannelStatus = maps:get(channels, status()), + case length(maps:keys(ChannelStatus)) of N -> ok; _ -> wait end