This is an automated email from the ASF dual-hosted git repository. garren pushed a commit to branch add-partition-info-endpoint in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit fda7ee8994befded280105fe97d854450f2bb932 Author: Garren Smith <[email protected]> AuthorDate: Thu Sep 20 14:37:08 2018 +0200 Add /_partition/:partition/_info endpoint Add an endpoint that returns the partition size and doc count --- src/chttpd/src/chttpd_db.erl | 4 ++ src/couch/src/couch_bt_engine.erl | 8 ++- src/couch/src/couch_db.erl | 11 ++- src/couch/src/couch_db_engine.erl | 24 +++++++ src/fabric/src/fabric.erl | 9 ++- src/fabric/src/fabric_db_partition_info.erl | 100 ++++++++++++++++++++++++++++ src/fabric/src/fabric_rpc.erl | 5 +- 7 files changed, 156 insertions(+), 5 deletions(-) diff --git a/src/chttpd/src/chttpd_db.erl b/src/chttpd/src/chttpd_db.erl index 9dab4b5..c06d224 100644 --- a/src/chttpd/src/chttpd_db.erl +++ b/src/chttpd/src/chttpd_db.erl @@ -252,6 +252,10 @@ handle_view_cleanup_req(Req, Db) -> send_json(Req, 202, {[{ok, true}]}). +handle_partition_req(#httpd{method='GET',path_parts=[DbName, <<"_partition">>, Partition, <<"_info">>]}=Req, _Db) -> + {ok, PartitionInfo} = fabric:get_partition_info(DbName, Partition), + send_json(Req, {PartitionInfo}); + handle_partition_req(#httpd{ path_parts=[DbName, <<"_partition">>, Partition, _Design, Name, <<"_",_/binary>> = Action | _Rest] }=Req, Db) -> diff --git a/src/couch/src/couch_bt_engine.erl b/src/couch/src/couch_bt_engine.erl index e24d55e..9cd4f3f 100644 --- a/src/couch/src/couch_bt_engine.erl +++ b/src/couch/src/couch_bt_engine.erl @@ -299,8 +299,12 @@ get_partition_info(#st{} = St, Partition) -> {partition, Partition}, {doc_count, DocCount}, {doc_del_count, DocDelCount}, - {active, SizeInfo#size_info.active}, - {external, SizeInfo#size_info.external} + {size, + [ + {active, SizeInfo#size_info.active}, + {external, SizeInfo#size_info.external} + ] + } ]. diff --git a/src/couch/src/couch_db.erl b/src/couch/src/couch_db.erl index a5c71d7..a2a7534 100644 --- a/src/couch/src/couch_db.erl +++ b/src/couch/src/couch_db.erl @@ -113,7 +113,9 @@ validate_dbname/1, make_doc/5, - new_revid/1 + new_revid/1, + + get_partition_info/2 ]). @@ -475,6 +477,13 @@ get_db_info(Db) -> ], {ok, InfoList}. +get_partition_info(#db{} = Db, Partition) when is_binary(Partition) -> + Sizes = couch_db_engine:get_partition_info(Db, Partition), + {ok, Sizes}; +get_partition_info(_Db, _Partition) -> + throw({badrequest, <<"`partition` is not valid">>}). + + get_design_docs(#db{name = <<"shards/", _:18/binary, DbName/binary>>}) -> {_, Ref} = spawn_monitor(fun() -> exit(fabric:design_docs(DbName)) end), receive {'DOWN', Ref, _, _, Response} -> diff --git a/src/couch/src/couch_db_engine.erl b/src/couch/src/couch_db_engine.erl index 9b28812..910c34e 100644 --- a/src/couch/src/couch_db_engine.erl +++ b/src/couch/src/couch_db_engine.erl @@ -42,6 +42,12 @@ -type purge_info() :: [{docid(), revs()}]. -type epochs() :: [{Node::atom(), UpdateSeq::non_neg_integer()}]. -type size_info() :: [{Name::atom(), Size::non_neg_integer()}]. +-type partition_info() :: [ + {Partition::atom(), Partition::binary()} | + {DocCount::atom(), DocCount::non_neg_integer()} | + {DocDelCount::atom(), DocDelCount::non_neg_integer()} | + {Size::atom(), size_info()} +]. -type write_stream_options() :: [ {buffer_size, Size::pos_integer()} | @@ -254,6 +260,18 @@ -callback get_size_info(DbHandle::db_handle()) -> SizeInfo::size_info(). +% This returns the information for the given partition. +% It should just be a list of {Name::atom(), Size::non_neg_integer()} +% It returns the partition name, doc count, deleted doc count and two sizes: +% +% active - Theoretical minimum number of bytes to store this partition on disk +% +% external - Number of bytes that would be required to represent the +% contents of this partition outside of the database +-callback get_partition_info(DbHandle::db_handle(), Partition::binary()) -> + partition_info(). + + % The current update sequence of the database. The update % sequence should be incrememnted for every revision added to % the database. @@ -627,6 +645,7 @@ get_prop/2, get_prop/3, get_size_info/1, + get_partition_info/2, get_update_seq/1, get_uuid/1, @@ -803,6 +822,11 @@ get_size_info(#db{} = Db) -> Engine:get_size_info(EngineState). +get_partition_info(#db{} = Db, Partition) -> + #db{engine = {Engine, EngineState}} = Db, + Engine:get_partition_info(EngineState, Partition). + + get_update_seq(#db{} = Db) -> #db{engine = {Engine, EngineState}} = Db, Engine:get_update_seq(EngineState). diff --git a/src/fabric/src/fabric.erl b/src/fabric/src/fabric.erl index f5c7937..c6b7c5b 100644 --- a/src/fabric/src/fabric.erl +++ b/src/fabric/src/fabric.erl @@ -21,7 +21,7 @@ delete_db/2, get_db_info/1, get_doc_count/1, set_revs_limit/3, set_security/2, set_security/3, get_revs_limit/1, get_security/1, get_security/2, get_all_security/1, get_all_security/2, - compact/1, compact/2]). + compact/1, compact/2, get_partition_info/2]). % Documents -export([open_doc/3, open_revs/4, get_doc_info/3, get_full_doc_info/3, @@ -84,6 +84,13 @@ all_dbs(Prefix) when is_list(Prefix) -> get_db_info(DbName) -> fabric_db_info:go(dbname(DbName)). +%% @doc returns the size of a given partition +-spec get_partition_info(dbname(), Partition::binary()) -> + {ok, {partition_info, non_neg_integer()}}. +get_partition_info(DbName, Partition) -> + fabric_db_partition_info:go(dbname(DbName), Partition). + + %% @doc the number of docs in a database -spec get_doc_count(dbname()) -> {ok, non_neg_integer()} | diff --git a/src/fabric/src/fabric_db_partition_info.erl b/src/fabric/src/fabric_db_partition_info.erl new file mode 100644 index 0000000..a742b06 --- /dev/null +++ b/src/fabric/src/fabric_db_partition_info.erl @@ -0,0 +1,100 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(fabric_db_partition_info). + +-export([go/2]). + +-include_lib("fabric/include/fabric.hrl"). +-include_lib("mem3/include/mem3.hrl"). + +go(DbName, Partition) -> + Shards = mem3:shards(DbName, <<Partition/binary, ":foo">>), + Workers = fabric_util:submit_jobs(Shards, get_partition_info, [Partition]), + RexiMon = fabric_util:create_monitors(Shards), + Fun = fun handle_message/3, + Acc0 = {fabric_dict:init(Workers, nil), []}, + try + case fabric_util:recv(Workers, #shard.ref, Fun, Acc0) of + {ok, Acc} -> {ok, Acc}; + {timeout, {WorkersDict, _}} -> + DefunctWorkers = fabric_util:remove_done_workers( + WorkersDict, + nil + ), + fabric_util:log_timeout( + DefunctWorkers, + "get_partition_info" + ), + {error, timeout}; + {error, Error} -> throw(Error) + end + after + rexi_monitor:stop(RexiMon) + end. + +handle_message({rexi_DOWN, _, {_,NodeRef},_}, _Shard, {Counters, Acc}) -> + case fabric_util:remove_down_workers(Counters, NodeRef) of + {ok, NewCounters} -> + {ok, {NewCounters, Acc}}; + error -> + {error, {nodedown, <<"progress not possible">>}} + end; + +handle_message({rexi_EXIT, Reason}, Shard, {Counters, Acc}) -> + NewCounters = fabric_dict:erase(Shard, Counters), + case fabric_view:is_progress_possible(NewCounters) of + true -> + {ok, {NewCounters, Acc}}; + false -> + {error, Reason} + end; + +handle_message({ok, Sizes}, #shard{dbname=Name} = Shard, {Counters, Acc}) -> + Acc2 = [Sizes | Acc], + Counters1 = fabric_dict:erase(Shard, Counters), + case fabric_dict:size(Counters1) =:= 0 of + true -> + [FirstInfo | RestInfos] = Acc2, + PartitionInfo = get_max_partition_size(FirstInfo, RestInfos), + {stop, + [{db_name, Name} | format_partition(PartitionInfo)] + }; + false -> + {ok, {Counters1, Acc2}} + end; + +handle_message(_, _, Acc) -> + {ok, Acc}. + +get_max_partition_size(Max, []) -> + Max; +get_max_partition_size(MaxInfo, [NextInfo | Rest]) -> + {size, MaxSize} = lists:keyfind(size, 1, MaxInfo), + {size, NextSize} = lists:keyfind(size, 1, NextInfo), + + {external, MaxExtSize} = lists:keyfind(external, 1, MaxSize), + {external, NextExtSize} = lists:keyfind(external, 1, NextSize), + case NextExtSize > MaxExtSize of + true -> + get_max_partition_size(NextInfo, Rest); + false -> + get_max_partition_size(MaxInfo, Rest) + end. + + +% for JS to work nicely we need to convert the size list +% to a jiffy object +format_partition(PartitionInfo) -> + {value, {size, Size}, PartitionInfo1} = lists:keytake(size, 1, PartitionInfo), + [{size, {Size}} | PartitionInfo1]. + diff --git a/src/fabric/src/fabric_rpc.erl b/src/fabric/src/fabric_rpc.erl index e538a9d..20bca0e 100644 --- a/src/fabric/src/fabric_rpc.erl +++ b/src/fabric/src/fabric_rpc.erl @@ -18,7 +18,7 @@ -export([all_docs/3, changes/3, map_view/4, reduce_view/4, group_info/2]). -export([create_db/1, create_db/2, delete_db/1, reset_validation_funs/1, set_security/3, set_revs_limit/3, create_shard_db_doc/2, - delete_shard_db_doc/2]). + delete_shard_db_doc/2, get_partition_info/2]). -export([get_all_security/2, open_shard/2]). -export([compact/1, compact/2]). @@ -167,6 +167,9 @@ get_db_info(DbName) -> get_db_info(DbName, DbOptions) -> with_db(DbName, DbOptions, {couch_db, get_db_info, []}). +get_partition_info(DbName, Partition) -> + with_db(DbName, [], {couch_db, get_partition_info, [Partition]}). + %% equiv get_doc_count(DbName, []) get_doc_count(DbName) -> get_doc_count(DbName, []).
