Use the new couch_stats application
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch/repo Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch/commit/f1013bfa Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch/tree/f1013bfa Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch/diff/f1013bfa Branch: refs/heads/windsor-merge Commit: f1013bfa222469856ce5e877407af9eeee49051f Parents: 9573e90 Author: Paul J. Davis <[email protected]> Authored: Thu Aug 21 01:05:11 2014 -0500 Committer: Robert Newson <[email protected]> Committed: Tue Aug 26 10:44:12 2014 +0100 ---------------------------------------------------------------------- priv/stat_descriptions.cfg | 194 ++++++++++++++++---- src/couch.app.src | 24 ++- src/couch_auth_cache.erl | 6 +- src/couch_db.erl | 29 ++- src/couch_db_updater.erl | 11 ++ src/couch_file.erl | 4 +- src/couch_httpd.erl | 14 +- src/couch_httpd_cors.erl | 2 +- src/couch_httpd_db.erl | 14 +- src/couch_httpd_stats_handlers.erl | 75 ++++---- src/couch_lru.erl | 1 + src/couch_query_servers.erl | 1 + src/couch_server.erl | 6 +- src/couch_stats_aggregator.erl | 312 -------------------------------- src/couch_stats_collector.erl | 133 -------------- 15 files changed, 268 insertions(+), 558 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/f1013bfa/priv/stat_descriptions.cfg ---------------------------------------------------------------------- diff --git a/priv/stat_descriptions.cfg b/priv/stat_descriptions.cfg index b80d768..d6e1586 100644 --- a/priv/stat_descriptions.cfg +++ b/priv/stat_descriptions.cfg @@ -14,37 +14,163 @@ % a trailing full-stop / period % Please keep this in alphabetical order -{couchdb, database_writes, "number of times a database was changed"}. -{couchdb, database_reads, "number of times a document was read from a database"}. -{couchdb, open_databases, "number of open databases"}. -{couchdb, open_os_files, "number of file descriptors CouchDB has open"}. -{couchdb, request_time, "length of a request inside CouchDB without MochiWeb"}. -{couchdb, auth_cache_hits, "number of authentication cache hits"}. -{couchdb, auth_cache_misses, "number of authentication cache misses"}. - -{httpd, bulk_requests, "number of bulk requests"}. -{httpd, requests, "number of HTTP requests"}. -{httpd, temporary_view_reads, "number of temporary view reads"}. -{httpd, view_reads, "number of view reads"}. -{httpd, clients_requesting_changes, "number of clients for continuous _changes"}. - -{httpd_request_methods, 'COPY', "number of HTTP COPY requests"}. -{httpd_request_methods, 'DELETE', "number of HTTP DELETE requests"}. -{httpd_request_methods, 'GET', "number of HTTP GET requests"}. -{httpd_request_methods, 'HEAD', "number of HTTP HEAD requests"}. -{httpd_request_methods, 'POST', "number of HTTP POST requests"}. -{httpd_request_methods, 'PUT', "number of HTTP PUT requests"}. - -{httpd_status_codes, '200', "number of HTTP 200 OK responses"}. -{httpd_status_codes, '201', "number of HTTP 201 Created responses"}. -{httpd_status_codes, '202', "number of HTTP 202 Accepted responses"}. -{httpd_status_codes, '301', "number of HTTP 301 Moved Permanently responses"}. -{httpd_status_codes, '304', "number of HTTP 304 Not Modified responses"}. -{httpd_status_codes, '400', "number of HTTP 400 Bad Request responses"}. -{httpd_status_codes, '401', "number of HTTP 401 Unauthorized responses"}. -{httpd_status_codes, '403', "number of HTTP 403 Forbidden responses"}. -{httpd_status_codes, '404', "number of HTTP 404 Not Found responses"}. -{httpd_status_codes, '405', "number of HTTP 405 Method Not Allowed responses"}. -{httpd_status_codes, '409', "number of HTTP 409 Conflict responses"}. -{httpd_status_codes, '412', "number of HTTP 412 Precondition Failed responses"}. -{httpd_status_codes, '500', "number of HTTP 500 Internal Server Error responses"}. +{[couchdb, auth_cache_hits], [ + {type, counter}, + {desc, <<"number of authentication cache hits">>} +]}. +{[couchdb, auth_cache_misses], [ + {type, counter}, + {desc, <<"number of authentication cache misses">>} +]}. +{[couchdb, collect_results_time], [ + {type, histogram}, + {desc, <<"microsecond latency for calls to couch_db:collect_results/3">>} +]}. +{[couchdb, database_writes], [ + {type, counter}, + {desc, <<"number of times a database was changed">>} +]}. +{[couchdb, database_reads], [ + {type, counter}, + {desc, <<"number of times a document was read from a database">>} +]}. +{[couchdb, db_open_time], [ + {type, histogram}, + {desc, <<"milliseconds required to open a database">>} +]}. +{[couchdb, document_inserts], [ + {type, counter}, + {desc, <<"number of documents inserted">>} +]}. +{[couchdb, document_writes], [ + {type, counter}, + {desc, <<"number of document write operations">>} +]}. +{[couchdb, local_document_writes], [ + {type, counter}, + {desc, <<"number of _local document write operations">>} +]}. +{[couchdb, httpd, bulk_requests], [ + {type, counter}, + {desc, <<"number of bulk requests">>} +]}. +{[couchdb, httpd, requests], [ + {type, counter}, + {desc, <<"number of HTTP requests">>} +]}. +{[couchdb, httpd, temporary_view_reads], [ + {type, counter}, + {desc, <<"number of temporary view reads">>} +]}. +{[couchdb, httpd, view_reads], [ + {type, counter}, + {desc, <<"number of view reads">>} +]}. +{[couchdb, httpd, clients_requesting_changes], [ + {type, counter}, + {desc, <<"number of clients for continuous _changes">>} +]}. +{[couchdb, httpd_request_methods, 'COPY'], [ + {type, counter}, + {desc, <<"number of HTTP COPY requests">>} +]}. +{[couchdb, httpd_request_methods, 'DELETE'], [ + {type, counter}, + {desc, <<"number of HTTP DELETE requests">>} +]}. +{[couchdb, httpd_request_methods, 'GET'], [ + {type, counter}, + {desc, <<"number of HTTP GET requests">>} +]}. +{[couchdb, httpd_request_methods, 'HEAD'], [ + {type, counter}, + {desc, <<"number of HTTP HEAD requests">>} +]}. +{[couchdb, httpd_request_methods, 'POST'], [ + {type, counter}, + {desc, <<"number of HTTP POST requests">>} +]}. +{[couchdb, httpd_request_methods, 'PUT'], [ + {type, counter}, + {desc, <<"number of HTTP PUT requests">>} +]}. +{[couchdb, httpd_status_codes, '200'], [ + {type, counter}, + {desc, <<"number of HTTP 200 OK responses">>} +]}. +{[couchdb, httpd_status_codes, '201'], [ + {type, counter}, + {desc, <<"number of HTTP 201 Created responses">>} +]}. +{[couchdb, httpd_status_codes, '202'], [ + {type, counter}, + {desc, <<"number of HTTP 202 Accepted responses">>} +]}. +{[couchdb, httpd_status_codes, '301'], [ + {type, counter}, + {desc, <<"number of HTTP 301 Moved Permanently responses">>} +]}. +{[couchdb, httpd_status_codes, '302'], [ + {type, counter}, + {desc, <<"number of HTTP 302 Found responses">>} +]}. +{[couchdb, httpd_status_codes, '304'], [ + {type, counter}, + {desc, <<"number of HTTP 304 Not Modified responses">>} +]}. +{[couchdb, httpd_status_codes, '400'], [ + {type, counter}, + {desc, <<"number of HTTP 400 Bad Request responses">>} +]}. +{[couchdb, httpd_status_codes, '401'], [ + {type, counter}, + {desc, <<"number of HTTP 401 Unauthorized responses">>} +]}. +{[couchdb, httpd_status_codes, '403'], [ + {type, counter}, + {desc, <<"number of HTTP 403 Forbidden responses">>} +]}. +{[couchdb, httpd_status_codes, '404'], [ + {type, counter}, + {desc, <<"number of HTTP 404 Not Found responses">>} +]}. +{[couchdb, httpd_status_codes, '405'], [ + {type, counter}, + {desc, <<"number of HTTP 405 Method Not Allowed responses">>} +]}. +{[couchdb, httpd_status_codes, '409'], [ + {type, counter}, + {desc, <<"number of HTTP 409 Conflict responses">>} +]}. +{[couchdb, httpd_status_codes, '412'], [ + {type, counter}, + {desc, <<"number of HTTP 412 Precondition Failed responses">>} +]}. +{[couchdb, httpd_status_codes, '500'], [ + {type, counter}, + {desc, <<"number of HTTP 500 Internal Server Error responses">>} +]}. +{[couchdb, open_databases], [ + {type, counter}, + {desc, <<"number of open databases">>} +]}. +{[couchdb, open_os_files], [ + {type, counter}, + {desc, <<"number of file descriptors CouchDB has open">>} +]}. +{[couchdb, request_time], [ + {type, histogram}, + {desc, <<"length of a request inside CouchDB without MochiWeb">>} +]}. +{[couchdb, couch_server, lru_skip], [ + {type, counter}, + {desc, <<"number of couch_server LRU operations skipped">>} +]}. +{[couchdb, couchjs, map_doc], [ + {type, counter}, + {desc, <<"number of documents mapped in the couchjs view server">>} +]}. +{[couchdb, couchjs, emits], [ + {type, counter}, + {desc, <<"number of invocations of `emit' in map functions in the couchjs view server">>} +]}. http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/f1013bfa/src/couch.app.src ---------------------------------------------------------------------- diff --git a/src/couch.app.src b/src/couch.app.src index 30f3b2a..ba12f41 100644 --- a/src/couch.app.src +++ b/src/couch.app.src @@ -23,11 +23,27 @@ couch_secondary_services, couch_server, couch_sup, - couch_stats_aggregator, - couch_stats_collector, couch_task_status ]}, {mod, {couch_app, []}}, - {applications, [kernel, stdlib, crypto, sasl, inets, oauth, ibrowse, - mochiweb, ssl, couch_log, couch_event, b64url]} + {applications, [ + % stdlib + kernel, + stdlib, + crypto, + sasl, + inets, + ssl, + + % Upstream deps + ibrowse, + mochiweb, + oauth, + + % ASF deps + b64url, + couch_log, + couch_event, + couch_stats + ]} ]}. http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/f1013bfa/src/couch_auth_cache.erl ---------------------------------------------------------------------- diff --git a/src/couch_auth_cache.erl b/src/couch_auth_cache.erl index d53cff0..8cf631b 100644 --- a/src/couch_auth_cache.erl +++ b/src/couch_auth_cache.erl @@ -102,7 +102,7 @@ get_from_cache(UserName) -> [] -> gen_server:call(?MODULE, {fetch, UserName}, infinity); [{UserName, {Credentials, _ATime}}] -> - couch_stats_collector:increment({couchdb, auth_cache_hits}), + couch_stats:increment_counter([couchdb, auth_cache_hits]), gen_server:cast(?MODULE, {cache_hit, UserName}), Credentials end @@ -182,11 +182,11 @@ handle_call({new_max_cache_size, NewSize}, _From, State) -> handle_call({fetch, UserName}, _From, State) -> {Credentials, NewState} = case ets:lookup(?BY_USER, UserName) of [{UserName, {Creds, ATime}}] -> - couch_stats_collector:increment({couchdb, auth_cache_hits}), + couch_stats:increment_counter([couchdb, auth_cache_hits]), cache_hit(UserName, Creds, ATime), {Creds, State}; [] -> - couch_stats_collector:increment({couchdb, auth_cache_misses}), + couch_stats:increment_counter([couchdb, auth_cache_misses]), Creds = get_user_props_from_db(UserName), State1 = add_cache_entry(UserName, Creds, erlang:now(), State), {Creds, State1} http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/f1013bfa/src/couch_db.erl ---------------------------------------------------------------------- diff --git a/src/couch_db.erl b/src/couch_db.erl index 8a9f9e4..bd250ef 100644 --- a/src/couch_db.erl +++ b/src/couch_db.erl @@ -125,7 +125,8 @@ is_idle(#db{compactor_pid=nil, waiting_delayed_commit=nil} = Db) -> undefined -> true; {monitored_by, Pids} -> - (Pids -- [Db#db.main_pid, whereis(couch_stats_collector)]) =:= [] + PidTracker = whereis(couch_stats_process_tracker), + (Pids -- [Db#db.main_pid, PidTracker]) =:= [] end; is_idle(_Db) -> false. @@ -172,7 +173,7 @@ open_doc(Db, IdOrDocInfo) -> open_doc(Db, IdOrDocInfo, []). open_doc(Db, Id, Options) -> - increment_stat(Db, {couchdb, database_reads}), + increment_stat(Db, [couchdb, database_reads]), case open_doc_int(Db, Id, Options) of {ok, #doc{deleted=true}=Doc} -> case lists:member(deleted, Options) of @@ -221,7 +222,7 @@ find_ancestor_rev_pos({RevPos, [RevId|Rest]}, AttsSinceRevs) -> end. open_doc_revs(Db, Id, Revs, Options) -> - increment_stat(Db, {couchdb, database_reads}), + increment_stat(Db, [couchdb, database_reads]), [{ok, Results}] = open_doc_revs_int(Db, [{Id, Revs}], Options), {ok, [apply_open_options(Result, Options) || Result <- Results]}. @@ -840,7 +841,7 @@ doc_tag(#doc{meta=Meta}) -> end. update_docs(Db, Docs0, Options, replicated_changes) -> - increment_stat(Db, {couchdb, database_writes}), + increment_stat(Db, [couchdb, database_writes]), Docs = tag_docs(Docs0), DocBuckets = before_docs_update(Db, group_alike_docs(Docs)), @@ -867,7 +868,7 @@ update_docs(Db, Docs0, Options, replicated_changes) -> {ok, DocErrors}; update_docs(Db, Docs0, Options, interactive_edit) -> - increment_stat(Db, {couchdb, database_writes}), + increment_stat(Db, [couchdb, database_writes]), AllOrNothing = lists:member(all_or_nothing, Options), Docs = tag_docs(Docs0), @@ -960,6 +961,18 @@ set_commit_option(Options) -> [full_commit|Options] end. +collect_results_with_metrics(Pid, MRef, []) -> + Begin = os:timestamp(), + try + collect_results(Pid, MRef, []) + after + ResultsTime = timer:now_diff(os:timestamp(), Begin) div 1000, + couch_stats:update_histogram( + [couchdb, collect_results_time], + ResultsTime + ) + end. + collect_results(Pid, MRef, ResultsAcc) -> receive {result, Pid, Result} -> @@ -981,7 +994,7 @@ write_and_commit(#db{main_pid=Pid, user_ctx=Ctx}=Db, DocBuckets1, MRef = erlang:monitor(process, Pid), try Pid ! {update_docs, self(), DocBuckets, NonRepDocs, MergeConflicts, FullCommit}, - case collect_results(Pid, MRef, []) of + case collect_results_with_metrics(Pid, MRef, []) of {ok, Results} -> {ok, Results}; retry -> % This can happen if the db file we wrote to was swapped out by @@ -995,7 +1008,7 @@ write_and_commit(#db{main_pid=Pid, user_ctx=Ctx}=Db, DocBuckets1, DocBuckets3 = prepare_doc_summaries(Db2, DocBuckets2), close(Db2), Pid ! {update_docs, self(), DocBuckets3, NonRepDocs, MergeConflicts, FullCommit}, - case collect_results(Pid, MRef, []) of + case collect_results_with_metrics(Pid, MRef, []) of {ok, Results} -> {ok, Results}; retry -> throw({update_error, compaction_retry}) end @@ -1338,7 +1351,7 @@ increment_stat(#db{options = Options}, Stat) -> true -> ok; false -> - couch_stats_collector:increment(Stat) + couch_stats:increment_counter(Stat) end. skip_deleted(FoldFun) -> http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/f1013bfa/src/couch_db_updater.erl ---------------------------------------------------------------------- diff --git a/src/couch_db_updater.erl b/src/couch_db_updater.erl index 2c1e808..cd434df 100644 --- a/src/couch_db_updater.erl +++ b/src/couch_db_updater.erl @@ -60,6 +60,7 @@ init({DbName, Filepath, Fd, Options}) -> end end, Db = init_db(DbName, Filepath, Fd, Header, Options), + couch_stats_process_tracker:track([couchdb, open_databases]), % we don't load validation funs here because the fabric query is liable to % race conditions. Instead see couch_db:validate_doc_update, which loads % them lazily @@ -784,6 +785,16 @@ update_docs_int(Db, DocsList, NonRepDocs, MergeConflicts, FullCommit) -> {ok, DocInfoByIdBTree2} = couch_btree:add_remove(DocInfoByIdBTree, IndexFullDocInfos, []), {ok, DocInfoBySeqBTree2} = couch_btree:add_remove(DocInfoBySeqBTree, IndexFullDocInfos, RemoveSeqs), + + WriteCount = length(IndexFullDocInfos), + couch_stats:increment_counter([couchdb, document_inserts], + WriteCount - length(RemoveSeqs)), + couch_stats:increment_counter([couchdb, document_writes], WriteCount), + couch_stats:increment_counter( + [couchdb, local_doc_writes], + length(NonRepDocs) + ), + Db3 = Db2#db{ id_tree = DocInfoByIdBTree2, seq_tree = DocInfoBySeqBTree2, http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/f1013bfa/src/couch_file.erl ---------------------------------------------------------------------- diff --git a/src/couch_file.erl b/src/couch_file.erl index 4fda603..be6b634 100644 --- a/src/couch_file.erl +++ b/src/couch_file.erl @@ -347,7 +347,7 @@ file_open_options(Options) -> maybe_track_open_os_files(Options) -> case not lists:member(sys_db, Options) of true -> - couch_stats_collector:track_process_count({couchdb, open_os_files}); + couch_stats_process_tracker:track([couchdb, open_os_files]); false -> ok end. @@ -574,7 +574,7 @@ split_iolist([Byte | Rest], SplitAt, BeginAcc) when is_integer(Byte) -> split_iolist(Rest, SplitAt - 1, [Byte | BeginAcc]). -% System dbs aren't monitored by couch_stats_collector +% System dbs aren't monitored by couch_stats_process_tracker is_idle(#file{is_sys=true}) -> case process_info(self(), monitored_by) of {monitored_by, []} -> true; http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/f1013bfa/src/couch_httpd.erl ---------------------------------------------------------------------- diff --git a/src/couch_httpd.erl b/src/couch_httpd.erl index 0c5af59..32950eb 100644 --- a/src/couch_httpd.erl +++ b/src/couch_httpd.erl @@ -350,8 +350,8 @@ handle_request_int(MochiReq, DefaultFun, send_error(HttpReq, Error) end, RequestTime = round(timer:now_diff(os:timestamp(), Begin)/1000), - couch_stats_collector:record({couchdb, request_time}, RequestTime), - couch_stats_collector:increment({httpd, requests}), + couch_stats:update_histogram([couchdb, request_time], RequestTime), + couch_stats:increment_counter([httpd, requests]), {ok, Resp}. check_request_uri_length(Uri) -> @@ -390,7 +390,7 @@ authenticate_request(Response, _AuthSrcs) -> Response. increment_method_stats(Method) -> - couch_stats_collector:increment({httpd_request_methods, Method}). + couch_stats:increment_counter([httpd_request_methods, Method]). validate_referer(Req) -> Host = host_for_request(Req), @@ -614,7 +614,7 @@ log_request(#httpd{mochi_req=MochiReq,peer=Peer}=Req, Code) -> start_response_length(#httpd{mochi_req=MochiReq}=Req, Code, Headers, Length) -> log_request(Req, Code), - couch_stats_collector:increment({httpd_status_codes, Code}), + couch_stats:increment_counter([httpd_status_codes, Code]), Headers1 = Headers ++ server_header() ++ couch_httpd_auth:cookie_auth_header(Req, Headers), Headers2 = couch_httpd_cors:cors_headers(Req, Headers1), @@ -627,7 +627,7 @@ start_response_length(#httpd{mochi_req=MochiReq}=Req, Code, Headers, Length) -> start_response(#httpd{mochi_req=MochiReq}=Req, Code, Headers) -> log_request(Req, Code), - couch_stats_collector:increment({httpd_status_codes, Code}), + couch_stats:increment_counter([httpd_status_codes, Code]), CookieHeader = couch_httpd_auth:cookie_auth_header(Req, Headers), Headers1 = Headers ++ server_header() ++ CookieHeader, Headers2 = couch_httpd_cors:cors_headers(Req, Headers1), @@ -661,7 +661,7 @@ http_1_0_keep_alive(Req, Headers) -> start_chunked_response(#httpd{mochi_req=MochiReq}=Req, Code, Headers) -> log_request(Req, Code), - couch_stats_collector:increment({httpd_status_codes, Code}), + couch_stats:increment_counter([httpd_status_codes, Code]), Headers1 = http_1_0_keep_alive(MochiReq, Headers), Headers2 = Headers1 ++ server_header() ++ couch_httpd_auth:cookie_auth_header(Req, Headers1), @@ -686,7 +686,7 @@ last_chunk(Resp) -> send_response(#httpd{mochi_req=MochiReq}=Req, Code, Headers, Body) -> log_request(Req, Code), - couch_stats_collector:increment({httpd_status_codes, Code}), + couch_stats:increment_counter([httpd_status_codes, Code]), Headers1 = http_1_0_keep_alive(MochiReq, Headers), if Code >= 500 -> ?LOG_ERROR("httpd ~p error response:~n ~s", [Code, Body]); http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/f1013bfa/src/couch_httpd_cors.erl ---------------------------------------------------------------------- diff --git a/src/couch_httpd_cors.erl b/src/couch_httpd_cors.erl index 15b838e..66878d0 100644 --- a/src/couch_httpd_cors.erl +++ b/src/couch_httpd_cors.erl @@ -174,7 +174,7 @@ handle_preflight_request(Origin, Host, MochiReq) -> send_preflight_response(#httpd{mochi_req=MochiReq}=Req, Headers) -> couch_httpd:log_request(Req, 204), - couch_stats_collector:increment({httpd_status_codes, 204}), + couch_stats:increment_counter([httpd_status_codes, 204]), Headers1 = couch_httpd:http_1_0_keep_alive(MochiReq, Headers), Headers2 = Headers1 ++ couch_httpd:server_header() ++ couch_httpd_auth:cookie_auth_header(Req, Headers1), http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/f1013bfa/src/couch_httpd_db.erl ---------------------------------------------------------------------- diff --git a/src/couch_httpd_db.erl b/src/couch_httpd_db.erl index 77d8788..4fdc9aa 100644 --- a/src/couch_httpd_db.erl +++ b/src/couch_httpd_db.erl @@ -145,16 +145,8 @@ handle_changes_req2(Req, Db) -> FeedChangesFun(MakeCallback(Resp)) end end, - couch_stats_collector:increment( - {httpd, clients_requesting_changes} - ), - try - WrapperFun(ChangesFun) - after - couch_stats_collector:decrement( - {httpd, clients_requesting_changes} - ) - end. + couch_stats_process_tracker:track([httpd, clients_requesting_changes]), + WrapperFun(ChangesFun). handle_compact_req(#httpd{method='POST'}=Req, Db) -> case Req#httpd.path_parts of @@ -293,7 +285,7 @@ db_req(#httpd{path_parts=[_,<<"_ensure_full_commit">>]}=Req, _Db) -> send_method_not_allowed(Req, "POST"); db_req(#httpd{method='POST',path_parts=[_,<<"_bulk_docs">>]}=Req, Db) -> - couch_stats_collector:increment({httpd, bulk_requests}), + couch_stats:increment_counter([httpd, bulk_requests]), couch_httpd:validate_ctype(Req, "application/json"), {JsonProps} = couch_httpd:json_body_obj(Req), case couch_util:get_value(<<"docs">>, JsonProps) of http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/f1013bfa/src/couch_httpd_stats_handlers.erl ---------------------------------------------------------------------- diff --git a/src/couch_httpd_stats_handlers.erl b/src/couch_httpd_stats_handlers.erl index cd357ea..88376ab 100644 --- a/src/couch_httpd_stats_handlers.erl +++ b/src/couch_httpd_stats_handlers.erl @@ -11,46 +11,43 @@ % the License. -module(couch_httpd_stats_handlers). --include_lib("couch/include/couch_db.hrl"). +-include("couch_db.hrl"). -export([handle_stats_req/1]). --import(couch_httpd, [ - send_json/2, send_json/3, send_json/4, send_method_not_allowed/2, - start_json_response/2, send_chunk/2, end_json_response/1, - start_chunked_response/3, send_error/4 -]). handle_stats_req(#httpd{method='GET', path_parts=[_]}=Req) -> - flush(Req), - send_json(Req, couch_stats_aggregator:all(range(Req))); - -handle_stats_req(#httpd{method='GET', path_parts=[_, _Mod]}) -> - throw({bad_request, <<"Stat names must have exactly two parts.">>}); - -handle_stats_req(#httpd{method='GET', path_parts=[_, Mod, Key]}=Req) -> - flush(Req), - Stats = couch_stats_aggregator:get_json({list_to_atom(binary_to_list(Mod)), - list_to_atom(binary_to_list(Key))}, range(Req)), - send_json(Req, {[{Mod, {[{Key, Stats}]}}]}); - -handle_stats_req(#httpd{method='GET', path_parts=[_, _Mod, _Key | _Extra]}) -> - throw({bad_request, <<"Stat names must have exactly two parts.">>}); - -handle_stats_req(Req) -> - send_method_not_allowed(Req, "GET"). - -range(Req) -> - case couch_util:get_value("range", couch_httpd:qs(Req)) of - undefined -> - 0; - Value -> - list_to_integer(Value) - end. - -flush(Req) -> - case couch_util:get_value("flush", couch_httpd:qs(Req)) of - "true" -> - couch_stats_aggregator:collect_sample(); - _Else -> - ok - end. + Stats = couch_stats:fetch(), + Nested = nest(Stats), + EJSON = to_ejson(Nested), + couch_httpd:send_json(Req, EJSON). + +nest(Proplist) -> + nest(Proplist, []). + +nest([], Acc) -> + Acc; +nest([{[Key|Keys], Value}|Rest], Acc) -> + Acc1 = case proplists:lookup(Key, Acc) of + {Key, Old} -> + [{Key, nest([{Keys, Value}], Old)}|proplists:delete(Key, Acc)]; + none -> + Term = lists:foldr(fun(K, A) -> [{K, A}] end, Value, Keys), + [{Key, Term}|Acc] + end, + nest(Rest, Acc1). + +to_ejson([{_, _}|_]=Proplist) -> + EJSONProps = lists:map( + fun({Key, Value}) -> {maybe_format_key(Key), to_ejson(Value)} end, + Proplist + ), + {EJSONProps}; +to_ejson(NotAProplist) -> + NotAProplist. + +maybe_format_key(Key) when is_integer(Key) -> + maybe_format_key(integer_to_list(Key)); +maybe_format_key(Key) when is_list(Key) -> + list_to_binary(Key); +maybe_format_key(Key) -> + Key. http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/f1013bfa/src/couch_lru.erl ---------------------------------------------------------------------- diff --git a/src/couch_lru.erl b/src/couch_lru.erl index ad432ec..d58eb69 100644 --- a/src/couch_lru.erl +++ b/src/couch_lru.erl @@ -52,6 +52,7 @@ close_int({Lru, DbName, Iter}, {Tree, Dict} = Cache) -> {gb_trees:delete(Lru, Tree), dict:erase(DbName, Dict)}; false -> true = ets:update_element(couch_dbs, DbName, {#db.fd_monitor, nil}), + couch_stats:increment_counter([couchdb, couch_server, lru_skip]), close_int(gb_trees:next(Iter), update(DbName, Cache)) end; false -> http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/f1013bfa/src/couch_query_servers.erl ---------------------------------------------------------------------- diff --git a/src/couch_query_servers.erl b/src/couch_query_servers.erl index c84ff7e..13b0b91 100644 --- a/src/couch_query_servers.erl +++ b/src/couch_query_servers.erl @@ -76,6 +76,7 @@ map_docs(Proc, Docs) -> FunsResults) end, Docs), + couch_stats:increment_counter([couchdb, couchjs, map_doc], length(Docs)), {ok, Results}. map_doc_raw(Proc, Doc) -> http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/f1013bfa/src/couch_server.erl ---------------------------------------------------------------------- diff --git a/src/couch_server.erl b/src/couch_server.erl index 964ecad..66a3c74 100644 --- a/src/couch_server.erl +++ b/src/couch_server.erl @@ -345,8 +345,8 @@ handle_call({open_result, DbName, {ok, Db}}, {FromPid, _Tag}, Server) -> link(Db#db.main_pid), true = ets:delete(couch_dbs_pid_to_name, FromPid), case erase({async_open, DbName}) of undefined -> ok; T0 -> - ?LOG_INFO("needed ~p ms to open new ~s", [timer:now_diff(os:timestamp(),T0)/1000, - DbName]) + OpenTime = timer:now_diff(os:timestamp(), T0) / 1000, + couch_stats:update_histogram([couchdb, db_open_time], OpenTime) end, % icky hack of field values - compactor_pid used to store clients % and fd used to possibly store a creation request @@ -363,8 +363,6 @@ handle_call({open_result, DbName, {ok, Db}}, {FromPid, _Tag}, Server) -> true = ets:insert(couch_dbs_pid_to_name, {Db#db.main_pid, DbName}), Lru = case couch_db:is_system_db(Db) of false -> - Stat = {couchdb, open_databases}, - couch_stats_collector:track_process_count(Db#db.main_pid, Stat), couch_lru:insert(DbName, Server#server.lru); true -> Server#server.lru http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/f1013bfa/src/couch_stats_aggregator.erl ---------------------------------------------------------------------- diff --git a/src/couch_stats_aggregator.erl b/src/couch_stats_aggregator.erl deleted file mode 100644 index 45987d6..0000000 --- a/src/couch_stats_aggregator.erl +++ /dev/null @@ -1,312 +0,0 @@ -% Licensed under the Apache License, Version 2.0 (the "License"); you may not -% use this file except in compliance with the License. You may obtain a copy of -% the License at -% -% http://www.apache.org/licenses/LICENSE-2.0 -% -% Unless required by applicable law or agreed to in writing, software -% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -% License for the specific language governing permissions and limitations under -% the License. - --module(couch_stats_aggregator). --behaviour(gen_server). --behaviour(config_listener). - --export([start/0, start/1, stop/0]). --export([all/0, all/1, get/1, get/2, get_json/1, get_json/2, collect_sample/0]). - --export([init/1, terminate/2, code_change/3]). --export([handle_call/3, handle_cast/2, handle_info/2]). - -% config_listener api --export([handle_config_change/5]). - - --record(aggregate, { - description = <<"">>, - seconds = 0, - count = 0, - current = null, - sum = null, - mean = null, - variance = null, - stddev = null, - min = null, - max = null, - samples = [] -}). - - -start() -> - PrivDir = couch_util:priv_dir(), - start(filename:join(PrivDir, "stat_descriptions.cfg")). - -start(FileName) -> - gen_server:start_link({local, ?MODULE}, ?MODULE, [FileName], []). - -stop() -> - gen_server:cast(?MODULE, stop). - -all() -> - ?MODULE:all(0). -all(Time) when is_binary(Time) -> - ?MODULE:all(list_to_integer(binary_to_list(Time))); -all(Time) when is_atom(Time) -> - ?MODULE:all(list_to_integer(atom_to_list(Time))); -all(Time) when is_integer(Time) -> - Aggs = ets:match(?MODULE, {{'$1', Time}, '$2'}), - Stats = lists:map(fun([Key, Agg]) -> {Key, Agg} end, Aggs), - case Stats of - [] -> - {[]}; - _ -> - Ret = lists:foldl(fun({{Mod, Key}, Agg}, Acc) -> - CurrKeys = case proplists:lookup(Mod, Acc) of - none -> []; - {Mod, {Keys}} -> Keys - end, - NewMod = {[{Key, to_json_term(Agg)} | CurrKeys]}, - [{Mod, NewMod} | proplists:delete(Mod, Acc)] - end, [], Stats), - {Ret} - end. - -get(Key) -> - ?MODULE:get(Key, 0). -get(Key, Time) when is_binary(Time) -> - ?MODULE:get(Key, list_to_integer(binary_to_list(Time))); -get(Key, Time) when is_atom(Time) -> - ?MODULE:get(Key, list_to_integer(atom_to_list(Time))); -get(Key, Time) when is_integer(Time) -> - case ets:lookup(?MODULE, {make_key(Key), Time}) of - [] -> #aggregate{seconds=Time}; - [{_, Agg}] -> Agg - end. - -get_json(Key) -> - get_json(Key, 0). -get_json(Key, Time) -> - to_json_term(?MODULE:get(Key, Time)). - -collect_sample() -> - gen_server:call(?MODULE, collect_sample, infinity). - - -init(StatDescsFileName) -> - % Create an aggregate entry for each {description, rate} pair. - ets:new(?MODULE, [named_table, set, protected]), - SampleStr = config:get("stats", "samples", "[0]"), - {ok, Samples} = couch_util:parse_term(SampleStr), - {ok, Descs} = file:consult(StatDescsFileName), - lists:foreach(fun({Sect, Key, Value}) -> - lists:foreach(fun(Secs) -> - Agg = #aggregate{ - description=list_to_binary(Value), - seconds=Secs - }, - ets:insert(?MODULE, {{{Sect, Key}, Secs}, Agg}) - end, Samples) - end, Descs), - - ok = config:listen_for_changes(?MODULE, nil), - - Rate = list_to_integer(config:get("stats", "rate", "1000")), - % TODO: Add timer_start to kernel start options. - {ok, TRef} = timer:apply_after(Rate, ?MODULE, collect_sample, []), - {ok, {TRef, Rate}}. - -terminate(_Reason, {TRef, _Rate}) -> - timer:cancel(TRef), - ok. - -handle_call(collect_sample, _, {OldTRef, SampleInterval}) -> - timer:cancel(OldTRef), - {ok, TRef} = timer:apply_after(SampleInterval, ?MODULE, collect_sample, []), - % Gather new stats values to add. - Incs = lists:map(fun({Key, Value}) -> - {Key, {incremental, Value}} - end, couch_stats_collector:all(incremental)), - Abs = lists:map(fun({Key, Values}) -> - couch_stats_collector:clear(Key), - Values2 = case Values of - X when is_list(X) -> X; - Else -> [Else] - end, - {_, Mean} = lists:foldl(fun(Val, {Count, Curr}) -> - {Count+1, Curr + (Val - Curr) / (Count+1)} - end, {0, 0}, Values2), - {Key, {absolute, Mean}} - end, couch_stats_collector:all(absolute)), - - Values = Incs ++ Abs, - Now = os:timestamp(), - lists:foreach(fun({{Key, Rate}, Agg}) -> - NewAgg = case proplists:lookup(Key, Values) of - none -> - rem_values(Now, Agg); - {Key, {Type, Value}} -> - NewValue = new_value(Type, Value, Agg#aggregate.current), - Agg2 = add_value(Now, NewValue, Agg), - rem_values(Now, Agg2) - end, - ets:insert(?MODULE, {{Key, Rate}, NewAgg}) - end, ets:tab2list(?MODULE)), - {reply, ok, {TRef, SampleInterval}}. - -handle_cast(stop, State) -> - {stop, normal, State}. - -handle_info({gen_event_EXIT, {config_listener, ?MODULE}, _Reason}, State) -> - erlang:send_after(5000, self(), restart_config_listener), - {noreply, State}; -handle_info(restart_config_listener, State) -> - ok = config:listen_for_changes(?MODULE, nil), - {noreply, State}; -handle_info(_Info, State) -> - {noreply, State}. - -code_change(_OldVersion, State, _Extra) -> - {ok, State}. - - -handle_config_change("stats", _, _, _, _) -> - exit(whereis(?MODULE), config_change), - remove_handler; -handle_config_change(_, _, _, _, _) -> - {ok, nil}. - - -new_value(incremental, Value, null) -> - Value; -new_value(incremental, Value, Current) -> - Value - Current; -new_value(absolute, Value, _Current) -> - Value. - -add_value(Time, Value, #aggregate{count=Count, seconds=Secs}=Agg) when Count < 1 -> - Samples = case Secs of - 0 -> []; - _ -> [{Time, Value}] - end, - Agg#aggregate{ - count=1, - current=Value, - sum=Value, - mean=Value, - variance=0.0, - stddev=null, - min=Value, - max=Value, - samples=Samples - }; -add_value(Time, Value, Agg) -> - #aggregate{ - count=Count, - current=Current, - sum=Sum, - mean=Mean, - variance=Variance, - samples=Samples - } = Agg, - - NewCount = Count + 1, - NewMean = Mean + (Value - Mean) / NewCount, - NewVariance = Variance + (Value - Mean) * (Value - NewMean), - StdDev = case NewCount > 1 of - false -> null; - _ -> math:sqrt(NewVariance / (NewCount - 1)) - end, - Agg2 = Agg#aggregate{ - count=NewCount, - current=Current + Value, - sum=Sum + Value, - mean=NewMean, - variance=NewVariance, - stddev=StdDev, - min=lists:min([Agg#aggregate.min, Value]), - max=lists:max([Agg#aggregate.max, Value]) - }, - case Agg2#aggregate.seconds of - 0 -> Agg2; - _ -> Agg2#aggregate{samples=[{Time, Value} | Samples]} - end. - -rem_values(Time, Agg) -> - Seconds = Agg#aggregate.seconds, - Samples = Agg#aggregate.samples, - Pred = fun({When, _Value}) -> - timer:now_diff(Time, When) =< (Seconds * 1000000) - end, - {Keep, Remove} = lists:splitwith(Pred, Samples), - Agg2 = lists:foldl(fun({_, Value}, Acc) -> - rem_value(Value, Acc) - end, Agg, Remove), - Agg2#aggregate{samples=Keep}. - -rem_value(_Value, #aggregate{count=Count, seconds=Secs}) when Count =< 1 -> - #aggregate{seconds=Secs}; -rem_value(Value, Agg) -> - #aggregate{ - count=Count, - sum=Sum, - mean=Mean, - variance=Variance - } = Agg, - - OldMean = (Mean * Count - Value) / (Count - 1), - OldVariance = Variance - (Value - OldMean) * (Value - Mean), - OldCount = Count - 1, - StdDev = case OldCount > 1 of - false -> null; - _ -> math:sqrt(clamp_value(OldVariance / (OldCount - 1))) - end, - Agg#aggregate{ - count=OldCount, - sum=Sum-Value, - mean=clamp_value(OldMean), - variance=clamp_value(OldVariance), - stddev=StdDev - }. - -to_json_term(Agg) -> - {Min, Max} = case Agg#aggregate.seconds > 0 of - false -> - {Agg#aggregate.min, Agg#aggregate.max}; - _ -> - case length(Agg#aggregate.samples) > 0 of - true -> - Extract = fun({_Time, Value}) -> Value end, - Samples = lists:map(Extract, Agg#aggregate.samples), - {lists:min(Samples), lists:max(Samples)}; - _ -> - {null, null} - end - end, - {[ - {description, Agg#aggregate.description}, - {current, round_value(Agg#aggregate.sum)}, - {sum, round_value(Agg#aggregate.sum)}, - {mean, round_value(Agg#aggregate.mean)}, - {stddev, round_value(Agg#aggregate.stddev)}, - {min, Min}, - {max, Max} - ]}. - -make_key({Mod, Val}) when is_integer(Val) -> - {Mod, list_to_atom(integer_to_list(Val))}; -make_key(Key) -> - Key. - -round_value(Val) when not is_number(Val) -> - Val; -round_value(Val) when Val == 0 -> - Val; -round_value(Val) -> - erlang:round(Val * 1000.0) / 1000.0. - -clamp_value(Val) when Val > 0.00000000000001 -> - Val; -clamp_value(_) -> - 0.0. http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/f1013bfa/src/couch_stats_collector.erl ---------------------------------------------------------------------- diff --git a/src/couch_stats_collector.erl b/src/couch_stats_collector.erl deleted file mode 100644 index 5bf4864..0000000 --- a/src/couch_stats_collector.erl +++ /dev/null @@ -1,133 +0,0 @@ -% Licensed under the Apache License, Version 2.0 (the "License"); you may not -% use this file except in compliance with the License. You may obtain a copy of -% the License at -% -% http://www.apache.org/licenses/LICENSE-2.0 -% -% Unless required by applicable law or agreed to in writing, software -% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -% License for the specific language governing permissions and limitations under -% the License. - -% todo -% - remove existance check on increment(), decrement() and record(). have -% modules initialize counters on startup. - --module(couch_stats_collector). - --behaviour(gen_server). --vsn(1). - --export([start/0, stop/0]). --export([all/0, all/1, get/1, increment/1, decrement/1, record/2, clear/1]). --export([track_process_count/1, track_process_count/2]). - --export([init/1, terminate/2, code_change/3]). --export([handle_call/3, handle_cast/2, handle_info/2]). - --define(HIT_TABLE, stats_hit_table). --define(ABS_TABLE, stats_abs_table). - -start() -> - gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). - -stop() -> - gen_server:call(?MODULE, stop). - -all() -> - ets:tab2list(?HIT_TABLE) ++ abs_to_list(). - -all(Type) -> - case Type of - incremental -> ets:tab2list(?HIT_TABLE); - absolute -> abs_to_list() - end. - -get(Key) -> - case ets:lookup(?HIT_TABLE, Key) of - [] -> - case ets:lookup(?ABS_TABLE, Key) of - [] -> - nil; - AbsVals -> - lists:map(fun({_, Value}) -> Value end, AbsVals) - end; - [{_, Counter}] -> - Counter - end. - -increment(Key) -> - Key2 = make_key(Key), - case catch ets:update_counter(?HIT_TABLE, Key2, 1) of - {'EXIT', {badarg, _}} -> - catch ets:insert(?HIT_TABLE, {Key2, 1}), - ok; - _ -> - ok - end. - -decrement(Key) -> - Key2 = make_key(Key), - case catch ets:update_counter(?HIT_TABLE, Key2, -1) of - {'EXIT', {badarg, _}} -> - catch ets:insert(?HIT_TABLE, {Key2, -1}), - ok; - _ -> ok - end. - -record(Key, Value) -> - catch ets:insert(?ABS_TABLE, {make_key(Key), Value}). - -clear(Key) -> - catch ets:delete(?ABS_TABLE, make_key(Key)). - -track_process_count(Stat) -> - track_process_count(self(), Stat). - -track_process_count(Pid, Stat) -> - ok = couch_stats_collector:increment(Stat), - gen_server:cast(?MODULE, {track_process_count, Pid, Stat}). - - -init(_) -> - ets:new(?HIT_TABLE, [named_table, set, public]), - ets:new(?ABS_TABLE, [named_table, duplicate_bag, public]), - {ok, dict:new()}. - -terminate(_Reason, _State) -> - ok. - -handle_call(stop, _, State) -> - {stop, normal, stopped, State}. - -handle_cast({track_process_count, Pid, Stat}, State) -> - Ref = erlang:monitor(process, Pid), - {noreply, dict:store(Ref, Stat, State)}. - -handle_info({'DOWN', Ref, _, _, _}, State) -> - Stat = dict:fetch(Ref, State), - couch_stats_collector:decrement(Stat), - {noreply, dict:erase(Ref, State)}. - -code_change(_, State, _Extra) -> - {ok, State}. - - -make_key({Module, Key}) when is_integer(Key) -> - {Module, list_to_atom(integer_to_list(Key))}; -make_key(Key) -> - Key. - -abs_to_list() -> - SortedKVs = lists:sort(ets:tab2list(?ABS_TABLE)), - lists:foldl(fun({Key, Val}, Acc) -> - case Acc of - [] -> - [{Key, [Val]}]; - [{Key, Prev} | Rest] -> - [{Key, [Val | Prev]} | Rest]; - Others -> - [{Key, [Val]} | Others] - end - end, [], SortedKVs).
