This is an automated email from the ASF dual-hosted git repository.
ronny pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
The following commit(s) were added to refs/heads/main by this push:
new b6ea325cc Add new HTTP endpoint `/_node/_local/_smoosh/status`. (#4766)
b6ea325cc is described below
commit b6ea325ccb1e3de45f8541ca4fde225ac75f7d54
Author: Ronny Berndt <ro...@apache.org>
AuthorDate: Fri Sep 22 20:44:50 2023 +0200
Add new HTTP endpoint `/_node/_local/_smoosh/status`. (#4766)
Introduce a new HTTP endpoint `/_node/_local/_smoosh/status`
to get status information from the CouchDB auto-compaction daemon.
Previously, this was only possible by starting a `remsh` session
and manually calling the `smoosh:status/1` function.
The internal data structures of `smoosh:status/1` are migrated
into Erlang maps to send them directly as json to the client.
To add more status information to smoosh in the future, the available
information will be stored under the json key `channels`.
Example:
{
"channels": {
"ratio_dbs": { ... },
"slack_dbs": { ... },
...
}
---
src/chttpd/src/chttpd_node.erl | 6 ++
src/docs/src/api/server/common.rst | 110 +++++++++++++++++++++++++++++++
src/smoosh/src/smoosh_channel.erl | 14 ++--
src/smoosh/src/smoosh_persist.erl | 10 +--
src/smoosh/src/smoosh_priority_queue.erl | 31 ++++-----
src/smoosh/src/smoosh_server.erl | 16 +++--
src/smoosh/test/smoosh_tests.erl | 43 ++++++------
7 files changed, 174 insertions(+), 56 deletions(-)
diff --git a/src/chttpd/src/chttpd_node.erl b/src/chttpd/src/chttpd_node.erl
index 46850fc4e..165b85a42 100644
--- a/src/chttpd/src/chttpd_node.erl
+++ b/src/chttpd/src/chttpd_node.erl
@@ -39,6 +39,12 @@ handle_node_req(#httpd{path_parts = [_, <<"_local">>]} =
Req) ->
send_json(Req, 200, {[{name, node()}]});
handle_node_req(#httpd{path_parts = [A, <<"_local">> | Rest]} = Req) ->
handle_node_req(Req#httpd{path_parts = [A, node()] ++ Rest});
+% GET /_node/$node/_smoosh/status
+handle_node_req(#httpd{method = 'GET', path_parts = [_, _Node, <<"_smoosh">>,
<<"status">>]} = Req) ->
+ {ok, Status} = smoosh:status(),
+ send_json(Req, 200, Status);
+handle_node_req(#httpd{path_parts = [_, _Node, <<"_smoosh">>, <<"status">>]} =
Req) ->
+ send_method_not_allowed(Req, "GET");
% GET /_node/$node/_versions
handle_node_req(#httpd{method = 'GET', path_parts = [_, _Node,
<<"_versions">>]} = Req) ->
IcuVer = couch_ejson_compare:get_icu_version(),
diff --git a/src/docs/src/api/server/common.rst
b/src/docs/src/api/server/common.rst
index 9e645f649..783b8120a 100644
--- a/src/docs/src/api/server/common.rst
+++ b/src/docs/src/api/server/common.rst
@@ -1902,6 +1902,116 @@ See :ref:`Configuration of Prometheus Endpoint
<config/prometheus>` for details.
Accept: text/plain
Host: localhost:17986
+.. _api/server/smoosh/status:
+
+=====================================
+``/_node/{node-name}/_smoosh/status``
+=====================================
+
+.. versionadded:: 3.4
+
+.. http:get:: /_node/{node-name}/_smoosh/status
+ :synopsis: Returns metrics of the CouchDB's auto-compaction daemon
+
+ This prints the state of each channel, how many jobs they are
+ currently running and how many jobs are enqueued (as well as the
+ lowest and highest priority of those enqueued items). The idea is to
+ provide, at a glance, sufficient insight into ``smoosh`` that an operator
+ can assess whether ``smoosh`` is adequately targeting the reclaimable
+ space in the cluster.
+
+ In general, a healthy status output will have
+ items in the ``ratio_dbs`` and ``ratio_views`` channels. Owing to the
default
+ settings, the ``slack_dbs`` and ``slack_views`` will almost certainly have
+ items in them. Historically, we've not found that the slack channels,
+ on their own, are particularly adept at keeping things well compacted.
+
+ :code 200: Request completed successfully
+ :code 401: CouchDB Server Administrator privileges required
+
+ **Request**:
+
+ .. code-block:: http
+
+ GET /_node/_local/_smoosh/status HTTP/1.1
+ Host: 127.0.0.1:5984
+ Accept: */*
+
+ **Response**:
+
+ .. code-block:: http
+
+ HTTP/1.1 200 OK
+ Content-Type: application/json
+
+ {
+ "channels": {
+ "slack_dbs": {
+ "starting": 0,
+ "waiting": {
+ "size": 0,
+ "min": 0,
+ "max": 0
+ },
+ "active": 0
+ },
+ "ratio_dbs": {
+ "starting": 0,
+ "waiting": {
+ "size": 56,
+ "min": 1.125,
+ "max": 11.0625
+ },
+ "active": 0
+ },
+ "ratio_views": {
+ "starting": 0,
+ "waiting": {
+ "size": 0,
+ "min": 0,
+ "max": 0
+ },
+ "active": 0
+ },
+ "upgrade_dbs": {
+ "starting": 0,
+ "waiting": {
+ "size": 0,
+ "min": 0,
+ "max": 0
+ },
+ "active": 0
+ },
+ "slack_views": {
+ "starting": 0,
+ "waiting": {
+ "size": 0,
+ "min": 0,
+ "max": 0
+ },
+ "active": 0
+ },
+ "upgrade_views": {
+ "starting": 0,
+ "waiting": {
+ "size": 0,
+ "min": 0,
+ "max": 0
+ },
+ "active": 0
+ },
+ "index_cleanup": {
+ "starting": 0,
+ "waiting": {
+ "size": 0,
+ "min": 0,
+ "max": 0
+ },
+ "active": 0
+ }
+ }
+ }
+
.. _api/server/system:
==============================
diff --git a/src/smoosh/src/smoosh_channel.erl
b/src/smoosh/src/smoosh_channel.erl
index 92fd3413b..3cfbcdec6 100644
--- a/src/smoosh/src/smoosh_channel.erl
+++ b/src/smoosh/src/smoosh_channel.erl
@@ -77,10 +77,10 @@ enqueue(ServerRef, Object, Priority) ->
get_status(StatusTab) when is_reference(StatusTab) ->
try ets:lookup(StatusTab, status) of
[{status, Status}] -> Status;
- [] -> []
+ [] -> #{}
catch
error:badarg ->
- []
+ #{}
end.
close(ServerRef) ->
@@ -235,11 +235,11 @@ unpersist(Name) ->
%
set_status(#state{} = State) ->
#state{active = Active, starting = Starting, waiting = Waiting} = State,
- Status = [
- {active, map_size(Active)},
- {starting, map_size(Starting)},
- {waiting, smoosh_priority_queue:info(Waiting)}
- ],
+ Status = #{
+ active => map_size(Active),
+ starting => map_size(Starting),
+ waiting => smoosh_priority_queue:info(Waiting)
+ },
true = ets:insert(State#state.stab, {status, Status}),
State.
diff --git a/src/smoosh/src/smoosh_persist.erl
b/src/smoosh/src/smoosh_persist.erl
index 2feab7e69..c1519f65f 100644
--- a/src/smoosh/src/smoosh_persist.erl
+++ b/src/smoosh/src/smoosh_persist.erl
@@ -225,7 +225,7 @@ t_persist_unpersist_disabled(_) ->
Q2 = unpersist(Name),
?assertEqual(Name, smoosh_priority_queue:name(Q2)),
- ?assertEqual([{size, 0}], smoosh_priority_queue:info(Q2)).
+ ?assertEqual(#{max => 0, min => 0, size => 0},
smoosh_priority_queue:info(Q2)).
t_persist_unpersist_enabled(_) ->
Name = "chan2",
@@ -241,7 +241,7 @@ t_persist_unpersist_enabled(_) ->
Q2 = unpersist(Name),
?assertEqual(Name, smoosh_priority_queue:name(Q2)),
Info2 = smoosh_priority_queue:info(Q2),
- ?assertEqual([{size, 3}, {min, 1.0}, {max, infinity}], Info2),
+ ?assertEqual(#{max => infinity, min => 1.0, size => 3}, Info2),
?assertEqual(Keys, drain_q(Q2)),
% Try to persist the already unpersisted queue
@@ -249,7 +249,7 @@ t_persist_unpersist_enabled(_) ->
Q3 = unpersist(Name),
?assertEqual(Name, smoosh_priority_queue:name(Q3)),
Info3 = smoosh_priority_queue:info(Q2),
- ?assertEqual([{size, 3}, {min, 1.0}, {max, infinity}], Info3),
+ ?assertEqual(#{max => infinity, min => 1.0, size => 3}, Info3),
?assertEqual(Keys, drain_q(Q3)).
t_persist_unpersist_errors(_) ->
@@ -267,7 +267,7 @@ t_persist_unpersist_errors(_) ->
Q2 = unpersist(Name),
?assertEqual(Name, smoosh_priority_queue:name(Q2)),
- ?assertEqual([{size, 0}], smoosh_priority_queue:info(Q2)),
+ ?assertEqual(#{max => 0, min => 0, size => 0},
smoosh_priority_queue:info(Q2)),
Dir = state_dir(),
ok = file:make_dir(Dir),
@@ -278,7 +278,7 @@ t_persist_unpersist_errors(_) ->
Q3 = unpersist(Name),
?assertEqual(Name, smoosh_priority_queue:name(Q3)),
- ?assertEqual([{size, 0}], smoosh_priority_queue:info(Q3)),
+ ?assertEqual(#{max => 0, min => 0, size => 0},
smoosh_priority_queue:info(Q3)),
ok = file:del_dir_r(Dir).
diff --git a/src/smoosh/src/smoosh_priority_queue.erl
b/src/smoosh/src/smoosh_priority_queue.erl
index b2ef4393d..2f2fba687 100644
--- a/src/smoosh/src/smoosh_priority_queue.erl
+++ b/src/smoosh/src/smoosh_priority_queue.erl
@@ -64,17 +64,14 @@ qsize(#priority_queue{tree = Tree}) ->
gb_trees:size(Tree).
info(#priority_queue{tree = Tree} = Q) ->
- [
- {size, qsize(Q)}
- | case gb_trees:is_empty(Tree) of
- true ->
- [];
- false ->
- {{Min, _}, _} = gb_trees:smallest(Tree),
- {{Max, _}, _} = gb_trees:largest(Tree),
- [{min, Min}, {max, Max}]
- end
- ].
+ case gb_trees:is_empty(Tree) of
+ true ->
+ #{size => qsize(Q), min => 0, max => 0};
+ false ->
+ {{Min, _}, _} = gb_trees:smallest(Tree),
+ {{Max, _}, _} = gb_trees:largest(Tree),
+ #{size => qsize(Q), min => Min, max => Max}
+ end.
insert(Key, Priority, Capacity, #priority_queue{tree = Tree, map = Map} = Q) ->
TreeKey = {Priority, make_ref()},
@@ -122,7 +119,7 @@ basics_test() ->
Q = new("foo"),
?assertMatch(#priority_queue{}, Q),
?assertEqual("foo", name(Q)),
- ?assertEqual([{size, 0}], info(Q)).
+ ?assertEqual(0, maps:get(size, info(Q))).
empty_test() ->
Q = new("foo"),
@@ -136,7 +133,7 @@ one_element_test() ->
Q0 = new("foo"),
Q = in(?K1, ?P1, 1, Q0),
?assertMatch(#priority_queue{}, Q),
- ?assertEqual([{size, 1}, {min, 1}, {max, 1}], info(Q)),
+ ?assertEqual(#{max => 1, min => 1, size => 1}, info(Q)),
?assertEqual(Q, truncate(1, Q)),
?assertMatch({?K1, #priority_queue{}}, out(Q)),
{?K1, Q2} = out(Q),
@@ -144,7 +141,7 @@ one_element_test() ->
?assertEqual(#{?K1 => ?P1}, to_map(Q)),
Q3 = from_map("foo", 1, to_map(Q)),
?assertEqual("foo", name(Q3)),
- ?assertEqual([{size, 1}, {min, ?P1}, {max, ?P1}], info(Q3)),
+ ?assertEqual(#{max => ?P1, min => ?P1, size => 1}, info(Q3)),
?assertEqual(to_map(Q), to_map(Q3)),
?assertEqual(Q0, flush(Q)).
@@ -153,7 +150,7 @@ multiple_elements_basics_test() ->
Q1 = in(?K1, ?P1, 10, Q0),
Q2 = in(?K2, ?P2, 10, Q1),
Q3 = in(?K3, ?P3, 10, Q2),
- ?assertEqual([{size, 3}, {min, ?P1}, {max, ?P3}], info(Q3)),
+ ?assertEqual(#{max => ?P3, min => ?P1, size => 3}, info(Q3)),
?assertEqual([?K3, ?K2, ?K1], drain(Q3)).
update_element_same_priority_test() ->
@@ -166,7 +163,7 @@ update_element_new_priority_test() ->
Q1 = in(?K1, ?P1, 10, Q0),
Q2 = in(?K2, ?P2, 10, Q1),
Q3 = in(?K1, ?P3, 10, Q2),
- ?assertEqual([{size, 2}, {min, ?P2}, {max, ?P3}], info(Q3)),
+ ?assertEqual(#{max => ?P3, min => ?P2, size => 2}, info(Q3)),
?assertEqual([?K1, ?K2], drain(Q3)).
capacity_test() ->
@@ -189,7 +186,7 @@ a_lot_of_elements_test() ->
lists:seq(1, N)
),
Q = from_map("foo", N, maps:from_list(KVs)),
- ?assertMatch([{size, N} | _], info(Q)),
+ ?assertMatch(N, maps:get(size, info(Q))),
{_, Priorities} = lists:unzip(drain(Q)),
?assertEqual(lists:reverse(lists:sort(Priorities)), Priorities).
diff --git a/src/smoosh/src/smoosh_server.erl b/src/smoosh/src/smoosh_server.erl
index 10368a549..3b0b86808 100644
--- a/src/smoosh/src/smoosh_server.erl
+++ b/src/smoosh/src/smoosh_server.erl
@@ -96,12 +96,14 @@ flush() ->
gen_server:call(?MODULE, flush, infinity).
status() ->
- try ets:foldl(fun get_channel_status/2, [], ?MODULE) of
- Res -> {ok, Res}
- catch
- error:badarg ->
- {ok, []}
- end.
+ ChannelsStatus =
+ try ets:foldl(fun get_channel_status/2, #{}, ?MODULE) of
+ Res -> Res
+ catch
+ error:badarg ->
+ #{}
+ end,
+ {ok, #{channels => ChannelsStatus}}.
enqueue(Object0) ->
Object = smoosh_utils:validate_arg(Object0),
@@ -286,7 +288,7 @@ remove_enqueue_ref(Ref, #state{} = State) when
is_reference(Ref) ->
get_channel_status(#channel{name = Name, stab = Tab}, Acc) ->
Status = smoosh_channel:get_status(Tab),
- [{Name, Status} | Acc];
+ Acc#{list_to_atom(Name) => Status};
get_channel_status(_, Acc) ->
Acc.
diff --git a/src/smoosh/test/smoosh_tests.erl b/src/smoosh/test/smoosh_tests.erl
index 622cabc8e..6861db5e1 100644
--- a/src/smoosh/test/smoosh_tests.erl
+++ b/src/smoosh/test/smoosh_tests.erl
@@ -82,21 +82,22 @@ teardown(DbName) ->
config:delete("smoosh", "cleanup_index_files", false).
t_default_channels(_) ->
+ ChannelStatus = maps:get(channels, status()),
?assertMatch(
[
- {"index_cleanup", _},
- {"ratio_dbs", _},
- {"ratio_views", _},
- {"slack_dbs", _},
- {"slack_views", _},
- {"upgrade_dbs", _},
- {"upgrade_views", _}
+ index_cleanup,
+ ratio_dbs,
+ ratio_views,
+ slack_dbs,
+ slack_views,
+ upgrade_dbs,
+ upgrade_views
],
- status()
+ lists:sort(maps:keys(ChannelStatus))
),
% If app hasn't started status won't crash
application:stop(smoosh),
- ?assertEqual([], status()).
+ ?assertEqual(#{channels => #{}}, status()).
t_channels_recreated_on_crash(_) ->
RatioDbsPid = get_channel_pid("ratio_dbs"),
@@ -104,7 +105,8 @@ t_channels_recreated_on_crash(_) ->
exit(RatioDbsPid, kill),
meck:wait(1, smoosh_channel, start_link, 1, 3000),
wait_for_channels(7),
- ?assertMatch([_, {"ratio_dbs", _} | _], status()),
+ ChannelStatus = maps:get(channels, status()),
+ ?assertMatch(true, maps:is_key(ratio_dbs, ChannelStatus)),
?assertNotEqual(RatioDbsPid, get_channel_pid("ratio_dbs")).
t_can_create_and_delete_channels(_) ->
@@ -402,17 +404,17 @@ delete_doc(DbName, DDocId) ->
status() ->
{ok, Props} = smoosh:status(),
- lists:keysort(1, Props).
+ Props.
status(Channel) ->
- case lists:keyfind(Channel, 1, status()) of
- {_, Val} ->
- Val,
- Active = proplists:get_value(active, Val),
- Starting = proplists:get_value(starting, Val),
- WaitingInfo = proplists:get_value(waiting, Val),
- Waiting = proplists:get_value(size, WaitingInfo),
- {Active, Starting, Waiting};
+ ChannelStatus = maps:get(channels, status()),
+ ChannelAtom = list_to_atom(Channel),
+ case maps:is_key(ChannelAtom, ChannelStatus) of
+ true ->
+ #{active := Active, starting := Starting, waiting := Waiting} =
maps:get(
+ ChannelAtom, ChannelStatus
+ ),
+ {Active, Starting, maps:get(size, Waiting)};
false ->
false
end.
@@ -443,7 +445,8 @@ wait_for_channels() ->
wait_for_channels(N) when is_integer(N), N >= 0 ->
WaitFun = fun() ->
- case length(status()) of
+ ChannelStatus = maps:get(channels, status()),
+ case length(maps:keys(ChannelStatus)) of
N -> ok;
_ -> wait
end
