This is an automated email from the ASF dual-hosted git repository.
pgj pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
The following commit(s) were added to refs/heads/main by this push:
new 4f3dc9526 mango: communicate rows read for global stats collection
4f3dc9526 is described below
commit 4f3dc9526329bbb5bc6f15d5683da856b195a674
Author: Gabor Pali <[email protected]>
AuthorDate: Sat Dec 16 15:07:26 2023 +0100
mango: communicate rows read for global stats collection
Besides the execution statistics that could be shown for the user,
the number of rows and documents read are tracked separately on
the global level. (Which is more like a bug than a feature, but
that is out of the scope of this change.)
With the introduction of covering indexes, these two have been
become different, therefore global counting has to be taught about
the number of times when no documents were read but rows only. For
text indexes, the number of rows and documents should be the same,
the distinction makes sense only for json indexes.
---
src/mango/src/mango_cursor_text.erl | 223 +++++++++++++++++-------------
src/mango/src/mango_cursor_view.erl | 84 ++++++++---
src/mango/src/mango_execution_stats.erl | 6 +-
src/mango/test/15-execution-stats-test.py | 2 +-
4 files changed, 198 insertions(+), 117 deletions(-)
diff --git a/src/mango/src/mango_cursor_text.erl
b/src/mango/src/mango_cursor_text.erl
index 5f13499b9..ee1d962f7 100644
--- a/src/mango/src/mango_cursor_text.erl
+++ b/src/mango/src/mango_cursor_text.erl
@@ -181,13 +181,14 @@ handle_hit(CAcc0, Sort, Doc) ->
#cacc{
limit = Limit,
skip = Skip,
- execution_stats = Stats,
+ execution_stats = Stats0,
documents_seen = Seen
} = CAcc0,
CAcc1 = update_bookmark(CAcc0, Sort),
- Stats1 = mango_execution_stats:incr_docs_examined(Stats),
+ Stats1 = mango_execution_stats:incr_keys_examined(Stats0),
+ Stats = mango_execution_stats:incr_docs_examined(Stats1),
couch_stats:increment_counter([mango, docs_examined]),
- CAcc2 = CAcc1#cacc{execution_stats = Stats1},
+ CAcc2 = CAcc1#cacc{execution_stats = Stats},
case mango_selector:match(CAcc2#cacc.selector, Doc) of
true ->
DocId = mango_doc:get_field(Doc, <<"_id">>),
@@ -430,8 +431,18 @@ execute_test_() ->
foreach,
fun() ->
meck:new(foo, [non_strict]),
+ meck:new(couch_stats),
meck:expect(couch_db, name, [db], meck:val(db_name)),
- meck:expect(couch_stats, increment_counter, [[mango,
docs_examined]], meck:val(ok)),
+ meck:expect(
+ couch_stats,
+ increment_counter,
+ [
+ {[[mango, docs_examined]], meck:val(ok)},
+ {[[mango, results_returned]], meck:val(ok)},
+ {[[couch_log, level, report]], meck:val(ok)}
+ ]
+ ),
+ meck:expect(couch_stats, update_histogram, [[mango, query_time],
'_'], meck:val(ok)),
% Dummy mock functions to progressively update the
% respective states therefore their results could be
% asserted later on.
@@ -466,28 +477,6 @@ execute_test_() ->
{ok, IdDocs}
end
),
- meck:expect(mango_execution_stats, log_start, [stats],
meck:val({stats, 0})),
- meck:expect(
- mango_execution_stats,
- maybe_add_stats,
- fun(_Options, _UserFun, {stats, N}, {acc, M}) -> {{acc, M +
1}, {stats, N + 1}} end
- ),
- meck:expect(mango_execution_stats, log_stats, [{stats, '_'}],
meck:val(ok)),
- meck:expect(
- mango_cursor,
- maybe_add_warning,
- fun(_UserFun, _Cursor, {stats, _}, {acc, M}) -> {acc, M + 1}
end
- ),
- meck:expect(
- mango_execution_stats,
- incr_docs_examined,
- fun({stats, N}) -> {stats, N + 1} end
- ),
- meck:expect(
- mango_execution_stats,
- incr_results_returned,
- fun({stats, N}) -> {stats, N + 1} end
- ),
meck:expect(
mango_selector_text,
append_sort_type,
@@ -541,13 +530,14 @@ t_execute_empty(_) ->
fields = fields,
selector = selector,
opts = Options,
- execution_stats = stats
+ execution_stats = #execution_stats{executionStartTime = {0, 0, 0}}
},
meck:expect(mango_selector_text, convert, [selector], meck:val(<<>>)),
- ?assertEqual({ok, {acc, 3}}, execute(Cursor, fun foo:add_key_only/2, {acc,
0})),
- ?assertEqual(0, meck:num_calls(couch_stats, increment_counter, 1)),
- ?assertEqual(0, meck:num_calls(mango_execution_stats, incr_docs_examined,
1)),
- ?assertEqual(0, meck:num_calls(mango_execution_stats,
incr_results_returned, 1)).
+ meck:expect(chttpd_stats, incr_rows, [0], meck:val(ok)),
+ meck:expect(chttpd_stats, incr_reads, [0], meck:val(ok)),
+ ?assertEqual({ok, {acc, 1}}, execute(Cursor, fun foo:add_key_only/2, {acc,
0})),
+ ?assertEqual(0, meck:num_calls(couch_stats, increment_counter, [[mango,
docs_examined]])),
+ ?assertEqual(0, meck:num_calls(couch_stats, increment_counter, [[mango,
results_returned]])).
t_execute_no_results(_) ->
Limit = 10,
@@ -564,7 +554,7 @@ t_execute_no_results(_) ->
fields = fields,
selector = selector,
opts = Options,
- execution_stats = stats
+ execution_stats = #execution_stats{executionStartTime = {0, 0, 0}}
},
QueryArgs =
#index_query_args{
@@ -583,13 +573,13 @@ t_execute_no_results(_) ->
),
meck:expect(mango_selector_text, convert, [selector], meck:val(query)),
meck:expect(mango_selector, match, fun(selector, {doc, _}) -> true end),
- ?assertEqual({ok, {acc, 3}}, execute(Cursor, fun foo:add_key_only/2, {acc,
0})),
- ?assertEqual(0, meck:num_calls(dreyfus_fabric, get_json_docs, 2)),
- ?assertEqual(0, meck:num_calls(couch_stats, increment_counter, 1)),
- ?assertEqual(0, meck:num_calls(mango_execution_stats, incr_docs_examined,
1)),
- ?assertEqual(0, meck:num_calls(mango_execution_stats,
incr_results_returned, 1)).
+ meck:expect(chttpd_stats, incr_rows, [0], meck:val(ok)),
+ meck:expect(chttpd_stats, incr_reads, [0], meck:val(ok)),
+ ?assertEqual({ok, {acc, 1}}, execute(Cursor, fun foo:add_key_only/2, {acc,
0})),
+ ?assertEqual(0, meck:num_calls(dreyfus_fabric, get_json_docs, 2)).
t_execute_more_results(_) ->
+ AllHits = 3,
Options = [{partition, partition}, {sort, {[]}}, {bookmark, [bookmark,
0]}],
Cursor = #cursor{
db = db,
@@ -599,7 +589,7 @@ t_execute_more_results(_) ->
fields = fields,
selector = selector,
opts = Options,
- execution_stats = stats
+ execution_stats = #execution_stats{executionStartTime = {0, 0, 0}}
},
meck:expect(
dreyfus_fabric_search,
@@ -631,12 +621,16 @@ t_execute_more_results(_) ->
),
meck:expect(mango_selector_text, convert, [selector], meck:val(query)),
meck:expect(mango_selector, match, fun(selector, {doc, _}) -> true end),
- ?assertEqual({ok, {acc, 6}}, execute(Cursor, fun foo:normal/2, {acc, 0})),
- ?assertEqual(3, meck:num_calls(couch_stats, increment_counter, 1)),
- ?assertEqual(3, meck:num_calls(mango_execution_stats, incr_docs_examined,
1)),
- ?assertEqual(3, meck:num_calls(mango_execution_stats,
incr_results_returned, 1)).
+ meck:expect(chttpd_stats, incr_rows, [AllHits], meck:val(ok)),
+ meck:expect(chttpd_stats, incr_reads, [AllHits], meck:val(ok)),
+ ?assertEqual({ok, {acc, 4}}, execute(Cursor, fun foo:normal/2, {acc, 0})),
+ ?assertEqual(AllHits, meck:num_calls(couch_stats, increment_counter,
[[mango, docs_examined]])),
+ ?assertEqual(
+ AllHits, meck:num_calls(couch_stats, increment_counter, [[mango,
results_returned]])
+ ).
t_execute_unique_results(_) ->
+ AllHits = 6,
UniqueHits = 3,
Options = [{partition, partition}, {sort, {[]}}, {bookmark, []}],
Cursor = #cursor{
@@ -647,7 +641,7 @@ t_execute_unique_results(_) ->
fields = fields,
selector = selector,
opts = Options,
- execution_stats = stats
+ execution_stats = #execution_stats{executionStartTime = {0, 0, 0}}
},
meck:expect(
dreyfus_fabric_search,
@@ -680,10 +674,13 @@ t_execute_unique_results(_) ->
),
meck:expect(mango_selector_text, convert, [selector], meck:val(query)),
meck:expect(mango_selector, match, fun(selector, {doc, _}) -> true end),
- ?assertEqual({ok, {acc, 6}}, execute(Cursor, fun foo:normal/2, {acc, 0})),
- ?assertEqual(6, meck:num_calls(couch_stats, increment_counter, 1)),
- ?assertEqual(6, meck:num_calls(mango_execution_stats, incr_docs_examined,
1)),
- ?assertEqual(UniqueHits, meck:num_calls(mango_execution_stats,
incr_results_returned, 1)).
+ meck:expect(chttpd_stats, incr_rows, [AllHits], meck:val(ok)),
+ meck:expect(chttpd_stats, incr_reads, [AllHits], meck:val(ok)),
+ ?assertEqual({ok, {acc, 4}}, execute(Cursor, fun foo:normal/2, {acc, 0})),
+ ?assertEqual(AllHits, meck:num_calls(couch_stats, increment_counter,
[[mango, docs_examined]])),
+ ?assertEqual(
+ UniqueHits, meck:num_calls(couch_stats, increment_counter, [[mango,
results_returned]])
+ ).
t_execute_limit_cutoff(_) ->
Limit = 2,
@@ -699,7 +696,7 @@ t_execute_limit_cutoff(_) ->
fields = fields,
selector = selector,
opts = Options,
- execution_stats = stats
+ execution_stats = #execution_stats{executionStartTime = {0, 0, 0}}
},
QueryArgs =
#index_query_args{
@@ -722,10 +719,13 @@ t_execute_limit_cutoff(_) ->
),
meck:expect(mango_selector_text, convert, [selector], meck:val(query)),
meck:expect(mango_selector, match, fun(selector, {doc, _}) -> true end),
- ?assertEqual({ok, {acc, 5}}, execute(Cursor, fun foo:normal/2, {acc, 0})),
- ?assertEqual(Limit, meck:num_calls(couch_stats, increment_counter, 1)),
- ?assertEqual(Limit, meck:num_calls(mango_execution_stats,
incr_docs_examined, 1)),
- ?assertEqual(Limit, meck:num_calls(mango_execution_stats,
incr_results_returned, 1)).
+ meck:expect(chttpd_stats, incr_rows, [Limit], meck:val(ok)),
+ meck:expect(chttpd_stats, incr_reads, [Limit], meck:val(ok)),
+ ?assertEqual({ok, {acc, 3}}, execute(Cursor, fun foo:normal/2, {acc, 0})),
+ ?assertEqual(Limit, meck:num_calls(couch_stats, increment_counter,
[[mango, docs_examined]])),
+ ?assertEqual(
+ Limit, meck:num_calls(couch_stats, increment_counter, [[mango,
results_returned]])
+ ).
t_execute_limit_cutoff_unique(_) ->
Limit = 4,
@@ -740,7 +740,7 @@ t_execute_limit_cutoff_unique(_) ->
fields = fields,
selector = selector,
opts = Options,
- execution_stats = stats
+ execution_stats = #execution_stats{executionStartTime = {0, 0, 0}}
},
meck:expect(
dreyfus_fabric_search,
@@ -773,10 +773,13 @@ t_execute_limit_cutoff_unique(_) ->
),
meck:expect(mango_selector_text, convert, [selector], meck:val(query)),
meck:expect(mango_selector, match, fun(selector, {doc, _}) -> true end),
- ?assertEqual({ok, {acc, 6}}, execute(Cursor, fun foo:normal/2, {acc, 0})),
- ?assertEqual(AllHits, meck:num_calls(couch_stats, increment_counter, 1)),
- ?assertEqual(AllHits, meck:num_calls(mango_execution_stats,
incr_docs_examined, 1)),
- ?assertEqual(ActualHits, meck:num_calls(mango_execution_stats,
incr_results_returned, 1)).
+ meck:expect(chttpd_stats, incr_rows, [AllHits], meck:val(ok)),
+ meck:expect(chttpd_stats, incr_reads, [AllHits], meck:val(ok)),
+ ?assertEqual({ok, {acc, 4}}, execute(Cursor, fun foo:normal/2, {acc, 0})),
+ ?assertEqual(AllHits, meck:num_calls(couch_stats, increment_counter,
[[mango, docs_examined]])),
+ ?assertEqual(
+ ActualHits, meck:num_calls(couch_stats, increment_counter, [[mango,
results_returned]])
+ ).
t_execute_limit_zero(_) ->
Limit = 0,
@@ -791,7 +794,7 @@ t_execute_limit_zero(_) ->
fields = fields,
selector = selector,
opts = Options,
- execution_stats = stats
+ execution_stats = #execution_stats{executionStartTime = {0, 0, 0}}
},
QueryArgs =
#index_query_args{
@@ -814,10 +817,13 @@ t_execute_limit_zero(_) ->
),
meck:expect(mango_selector_text, convert, [selector], meck:val(query)),
meck:expect(mango_selector, match, fun(selector, {doc, _}) -> true end),
- ?assertEqual({ok, {acc, 3}}, execute(Cursor, fun foo:add_key_only/2, {acc,
0})),
- ?assertEqual(1, meck:num_calls(couch_stats, increment_counter, 1)),
- ?assertEqual(1, meck:num_calls(mango_execution_stats, incr_docs_examined,
1)),
- ?assertEqual(Limit, meck:num_calls(mango_execution_stats,
incr_results_returned, 1)).
+ meck:expect(chttpd_stats, incr_rows, [Limit], meck:val(ok)),
+ meck:expect(chttpd_stats, incr_reads, [Limit], meck:val(ok)),
+ ?assertEqual({ok, {acc, 1}}, execute(Cursor, fun foo:add_key_only/2, {acc,
0})),
+ ?assertEqual(1, meck:num_calls(couch_stats, increment_counter, [[mango,
docs_examined]])),
+ ?assertEqual(
+ Limit, meck:num_calls(couch_stats, increment_counter, [[mango,
results_returned]])
+ ).
t_execute_limit_unique(_) ->
Limit = 5,
@@ -832,7 +838,7 @@ t_execute_limit_unique(_) ->
fields = fields,
selector = selector,
opts = Options,
- execution_stats = stats
+ execution_stats = #execution_stats{executionStartTime = {0, 0, 0}}
},
meck:expect(
dreyfus_fabric_search,
@@ -865,10 +871,13 @@ t_execute_limit_unique(_) ->
),
meck:expect(mango_selector_text, convert, [selector], meck:val(query)),
meck:expect(mango_selector, match, fun(selector, {doc, _}) -> true end),
- ?assertEqual({ok, {acc, 6}}, execute(Cursor, fun foo:normal/2, {acc, 0})),
- ?assertEqual(AllHits, meck:num_calls(couch_stats, increment_counter, 1)),
- ?assertEqual(AllHits, meck:num_calls(mango_execution_stats,
incr_docs_examined, 1)),
- ?assertEqual(UniqueHits, meck:num_calls(mango_execution_stats,
incr_results_returned, 1)).
+ meck:expect(chttpd_stats, incr_rows, [AllHits], meck:val(ok)),
+ meck:expect(chttpd_stats, incr_reads, [AllHits], meck:val(ok)),
+ ?assertEqual({ok, {acc, 4}}, execute(Cursor, fun foo:normal/2, {acc, 0})),
+ ?assertEqual(AllHits, meck:num_calls(couch_stats, increment_counter,
[[mango, docs_examined]])),
+ ?assertEqual(
+ UniqueHits, meck:num_calls(couch_stats, increment_counter, [[mango,
results_returned]])
+ ).
t_execute_skip(_) ->
UniqueHits = 3,
@@ -882,7 +891,7 @@ t_execute_skip(_) ->
fields = fields,
selector = selector,
opts = Options,
- execution_stats = stats
+ execution_stats = #execution_stats{executionStartTime = {0, 0, 0}}
},
meck:expect(
dreyfus_fabric_search,
@@ -910,11 +919,15 @@ t_execute_skip(_) ->
),
meck:expect(mango_selector_text, convert, [selector], meck:val(query)),
meck:expect(mango_selector, match, fun(selector, {doc, _}) -> true end),
- ?assertEqual({ok, {acc, 4}}, execute(Cursor, fun foo:normal/2, {acc, 0})),
- ?assertEqual(UniqueHits, meck:num_calls(couch_stats, increment_counter,
1)),
- ?assertEqual(UniqueHits, meck:num_calls(mango_execution_stats,
incr_docs_examined, 1)),
+ meck:expect(chttpd_stats, incr_rows, [UniqueHits], meck:val(ok)),
+ meck:expect(chttpd_stats, incr_reads, [UniqueHits], meck:val(ok)),
+ ?assertEqual({ok, {acc, 2}}, execute(Cursor, fun foo:normal/2, {acc, 0})),
?assertEqual(
- UniqueHits - Skip, meck:num_calls(mango_execution_stats,
incr_results_returned, 1)
+ UniqueHits, meck:num_calls(couch_stats, increment_counter, [[mango,
docs_examined]])
+ ),
+ ?assertEqual(
+ UniqueHits - Skip,
+ meck:num_calls(couch_stats, increment_counter, [[mango,
results_returned]])
).
t_execute_skip_unique(_) ->
@@ -930,7 +943,7 @@ t_execute_skip_unique(_) ->
fields = fields,
selector = selector,
opts = Options,
- execution_stats = stats
+ execution_stats = #execution_stats{executionStartTime = {0, 0, 0}}
},
meck:expect(
dreyfus_fabric_search,
@@ -963,11 +976,13 @@ t_execute_skip_unique(_) ->
),
meck:expect(mango_selector_text, convert, [selector], meck:val(query)),
meck:expect(mango_selector, match, fun(selector, {doc, _}) -> true end),
- ?assertEqual({ok, {acc, 4}}, execute(Cursor, fun foo:normal/2, {acc, 0})),
- ?assertEqual(AllHits, meck:num_calls(couch_stats, increment_counter, 1)),
- ?assertEqual(AllHits, meck:num_calls(mango_execution_stats,
incr_docs_examined, 1)),
+ meck:expect(chttpd_stats, incr_rows, [AllHits], meck:val(ok)),
+ meck:expect(chttpd_stats, incr_reads, [AllHits], meck:val(ok)),
+ ?assertEqual({ok, {acc, 2}}, execute(Cursor, fun foo:normal/2, {acc, 0})),
+ ?assertEqual(AllHits, meck:num_calls(couch_stats, increment_counter,
[[mango, docs_examined]])),
?assertEqual(
- UniqueHits - Skip, meck:num_calls(mango_execution_stats,
incr_results_returned, 1)
+ UniqueHits - Skip,
+ meck:num_calls(couch_stats, increment_counter, [[mango,
results_returned]])
).
t_execute_no_matches(_) ->
@@ -982,7 +997,7 @@ t_execute_no_matches(_) ->
fields = fields,
selector = selector,
opts = Options,
- execution_stats = stats
+ execution_stats = #execution_stats{executionStartTime = {0, 0, 0}}
},
meck:expect(
dreyfus_fabric_search,
@@ -1010,10 +1025,15 @@ t_execute_no_matches(_) ->
),
meck:expect(mango_selector_text, convert, [selector], meck:val(query)),
meck:expect(mango_selector, match, fun(selector, {doc, _}) -> false end),
- ?assertEqual({ok, {acc, 3}}, execute(Cursor, fun foo:add_key_only/2, {acc,
0})),
- ?assertEqual(UniqueHits, meck:num_calls(couch_stats, increment_counter,
1)),
- ?assertEqual(UniqueHits, meck:num_calls(mango_execution_stats,
incr_docs_examined, 1)),
- ?assertEqual(Matches, meck:num_calls(mango_execution_stats,
incr_results_returned, 1)).
+ meck:expect(chttpd_stats, incr_rows, [UniqueHits], meck:val(ok)),
+ meck:expect(chttpd_stats, incr_reads, [UniqueHits], meck:val(ok)),
+ ?assertEqual({ok, {acc, 1}}, execute(Cursor, fun foo:add_key_only/2, {acc,
0})),
+ ?assertEqual(
+ UniqueHits, meck:num_calls(couch_stats, increment_counter, [[mango,
docs_examined]])
+ ),
+ ?assertEqual(
+ Matches, meck:num_calls(couch_stats, increment_counter, [[mango,
results_returned]])
+ ).
t_execute_mixed_matches(_) ->
UniqueHits = 3,
@@ -1027,7 +1047,7 @@ t_execute_mixed_matches(_) ->
fields = fields,
selector = selector,
opts = Options,
- execution_stats = stats
+ execution_stats = #execution_stats{executionStartTime = {0, 0, 0}}
},
meck:expect(
dreyfus_fabric_search,
@@ -1055,10 +1075,15 @@ t_execute_mixed_matches(_) ->
),
meck:expect(mango_selector_text, convert, [selector], meck:val(query)),
meck:expect(mango_selector, match, fun(selector, {doc, N}) -> N == 2 end),
- ?assertEqual({ok, {acc, 4}}, execute(Cursor, fun foo:normal/2, {acc, 0})),
- ?assertEqual(UniqueHits, meck:num_calls(couch_stats, increment_counter,
1)),
- ?assertEqual(UniqueHits, meck:num_calls(mango_execution_stats,
incr_docs_examined, 1)),
- ?assertEqual(Matches, meck:num_calls(mango_execution_stats,
incr_results_returned, 1)).
+ meck:expect(chttpd_stats, incr_rows, [UniqueHits], meck:val(ok)),
+ meck:expect(chttpd_stats, incr_reads, [UniqueHits], meck:val(ok)),
+ ?assertEqual({ok, {acc, 2}}, execute(Cursor, fun foo:normal/2, {acc, 0})),
+ ?assertEqual(
+ UniqueHits, meck:num_calls(couch_stats, increment_counter, [[mango,
docs_examined]])
+ ),
+ ?assertEqual(
+ Matches, meck:num_calls(couch_stats, increment_counter, [[mango,
results_returned]])
+ ).
t_execute_user_fun_returns_stop(_) ->
UniqueHits = 3,
@@ -1075,7 +1100,7 @@ t_execute_user_fun_returns_stop(_) ->
fields = fields,
selector = selector,
opts = Options,
- execution_stats = stats
+ execution_stats = #execution_stats{executionStartTime = {0, 0, 0}}
},
QueryArgs =
#index_query_args{
@@ -1114,10 +1139,15 @@ t_execute_user_fun_returns_stop(_) ->
{Status, {acc, N + 1}}
end
),
- ?assertEqual({ok, {acc, 6}}, execute(Cursor, fun foo:stops/2, {acc, 0})),
- ?assertEqual(UniqueHits, meck:num_calls(couch_stats, increment_counter,
1)),
- ?assertEqual(UniqueHits, meck:num_calls(mango_execution_stats,
incr_docs_examined, 1)),
- ?assertEqual(3, meck:num_calls(mango_execution_stats,
incr_results_returned, 1)).
+ meck:expect(chttpd_stats, incr_rows, [UniqueHits], meck:val(ok)),
+ meck:expect(chttpd_stats, incr_reads, [UniqueHits], meck:val(ok)),
+ ?assertEqual({ok, {acc, 4}}, execute(Cursor, fun foo:stops/2, {acc, 0})),
+ ?assertEqual(
+ UniqueHits, meck:num_calls(couch_stats, increment_counter, [[mango,
docs_examined]])
+ ),
+ ?assertEqual(
+ UniqueHits, meck:num_calls(couch_stats, increment_counter, [[mango,
results_returned]])
+ ).
t_execute_search_error(_) ->
Limit = 10,
@@ -1133,7 +1163,7 @@ t_execute_search_error(_) ->
fields = fields,
selector = selector,
opts = Options,
- execution_stats = stats
+ execution_stats = #execution_stats{executionStartTime = {0, 0, 0}}
},
QueryArgs =
#index_query_args{
@@ -1151,13 +1181,14 @@ t_execute_search_error(_) ->
meck:val({error, reason})
),
meck:expect(mango_selector_text, convert, [selector], meck:val(query)),
+ meck:expect(chttpd_stats, incr_rows, [0], meck:val(ok)),
+ meck:expect(chttpd_stats, incr_reads, [0], meck:val(ok)),
Exception = {mango_error, mango_cursor_text, {text_search_error, {error,
reason}}},
?assertThrow(Exception, execute(Cursor, fun foo:normal/2, {acc, 0})),
?assertEqual(0, meck:num_calls(dreyfus_fabric, get_json_docs, 2)),
?assertEqual(0, meck:num_calls(foo, normal, 2)),
- ?assertEqual(0, meck:num_calls(couch_stats, increment_counter, 1)),
- ?assertEqual(0, meck:num_calls(mango_execution_stats, incr_docs_examined,
1)),
- ?assertEqual(0, meck:num_calls(mango_execution_stats,
incr_results_returned, 1)).
+ ?assertEqual(0, meck:num_calls(couch_stats, increment_counter, [[mango,
docs_examined]])),
+ ?assertEqual(0, meck:num_calls(couch_stats, increment_counter, [[mango,
results_returned]])).
explain_test_() ->
{
diff --git a/src/mango/src/mango_cursor_view.erl
b/src/mango/src/mango_cursor_view.erl
index b1357ac68..b103d869d 100644
--- a/src/mango/src/mango_cursor_view.erl
+++ b/src/mango/src/mango_cursor_view.erl
@@ -992,12 +992,10 @@ execute_test_() ->
foreach,
fun() ->
meck:new(foo, [non_strict]),
- meck:new(fabric)
- end,
- fun(_) ->
- meck:unload(fabric),
- meck:unload(foo)
+ meck:new(fabric),
+ meck:new(chttpd_stats)
end,
+ fun(_) -> meck:unload() end,
[
?TDEF_FE(t_execute_empty),
?TDEF_FE(t_execute_ok_all_docs),
@@ -1014,7 +1012,9 @@ t_execute_empty(_) ->
meck:expect(fabric, query_view, ['_', '_', '_', '_', '_', '_'],
meck:val(error)),
?assertEqual({ok, accumulator}, execute(Cursor, undefined, accumulator)),
?assertNot(meck:called(fabric, all_docs, '_')),
- ?assertNot(meck:called(fabric, query_view, '_')).
+ ?assertNot(meck:called(fabric, query_view, '_')),
+ ?assertNot(meck:called(chttpd_stats, incr_rows, '_')),
+ ?assertNot(meck:called(chttpd_stats, incr_reads, '_')).
t_execute_ok_all_docs(_) ->
Bookmark = bookmark,
@@ -1039,12 +1039,19 @@ t_execute_ok_all_docs(_) ->
user_fun = fun foo:bar/2,
execution_stats = '_'
},
+ TotalKeysExamined = 9,
+ TotalDocsExamined = 3,
Cursor2 =
Cursor1#cursor{
bookmark = Bookmark,
bookmark_docid = undefined,
bookmark_key = undefined,
- execution_stats = #execution_stats{executionStartTime = {0, 0, 0}}
+ execution_stats =
+ #execution_stats{
+ totalKeysExamined = TotalKeysExamined,
+ totalDocsExamined = TotalDocsExamined,
+ executionStartTime = {0, 0, 0}
+ }
},
Extra =
[
@@ -1071,13 +1078,25 @@ t_execute_ok_all_docs(_) ->
db, [{user_ctx, user_ctx}], fun
mango_cursor_view:handle_all_docs_message/2, Cursor1, Args
],
meck:expect(fabric, all_docs, Parameters, meck:val({ok, Cursor2})),
+ meck:expect(chttpd_stats, incr_rows, [TotalKeysExamined], meck:val(ok)),
+ meck:expect(chttpd_stats, incr_reads, [TotalDocsExamined], meck:val(ok)),
?assertEqual({ok, updated_accumulator}, execute(Cursor, fun foo:bar/2,
accumulator)),
?assert(meck:called(fabric, all_docs, '_')).
t_execute_ok_query_view(_) ->
Bookmark = bookmark,
- UserFnParameters = [{add_key, bookmark, Bookmark}, accumulator],
- meck:expect(foo, bar, UserFnParameters, meck:val({undefined,
updated_accumulator})),
+ UserFnParameters1 = [{add_key, bookmark, Bookmark}, accumulator1],
+ IndexScanWarning =
+ <<"The number of documents examined is high in proportion to the
number of results returned. Consider adding a more specific index to improve
this.">>,
+ UserFnParameters2 = [{add_key, warning, IndexScanWarning}, accumulator2],
+ meck:expect(
+ foo,
+ bar,
+ [
+ {UserFnParameters1, meck:val({undefined, accumulator2})},
+ {UserFnParameters2, meck:val({undefined, accumulator3})}
+ ]
+ ),
Index =
#idx{
type = <<"json">>,
@@ -1099,16 +1118,23 @@ t_execute_ok_query_view(_) ->
},
Cursor1 =
Cursor#cursor{
- user_acc = accumulator,
+ user_acc = accumulator1,
user_fun = fun foo:bar/2,
execution_stats = '_'
},
+ TotalKeysExamined = 99,
+ TotalDocsExamined = 33,
Cursor2 =
Cursor1#cursor{
bookmark = Bookmark,
bookmark_docid = undefined,
bookmark_key = undefined,
- execution_stats = #execution_stats{executionStartTime = {0, 0, 0}}
+ execution_stats =
+ #execution_stats{
+ totalKeysExamined = TotalKeysExamined,
+ totalDocsExamined = TotalDocsExamined,
+ executionStartTime = {0, 0, 0}
+ }
},
Extra =
[
@@ -1141,17 +1167,31 @@ t_execute_ok_query_view(_) ->
Args
],
meck:expect(fabric, query_view, Parameters, meck:val({ok, Cursor2})),
- ?assertEqual({ok, updated_accumulator}, execute(Cursor, fun foo:bar/2,
accumulator)),
+ meck:expect(chttpd_stats, incr_rows, [TotalKeysExamined], meck:val(ok)),
+ meck:expect(chttpd_stats, incr_reads, [TotalDocsExamined], meck:val(ok)),
+ ?assertEqual({ok, accumulator3}, execute(Cursor, fun foo:bar/2,
accumulator1)),
?assert(meck:called(fabric, query_view, '_')).
t_execute_ok_all_docs_with_execution_stats(_) ->
Bookmark = bookmark,
+ TotalKeysExamined = 33,
+ TotalDocsExamined = 12,
+ TotalQuorumDocsExamined = 0,
+ ResultsReturned = 20,
+ ExecutionStats =
+ #execution_stats{
+ totalKeysExamined = TotalKeysExamined,
+ totalDocsExamined = TotalDocsExamined,
+ totalQuorumDocsExamined = TotalQuorumDocsExamined,
+ resultsReturned = ResultsReturned,
+ executionStartTime = {0, 0, 0}
+ },
Stats =
{[
- {total_keys_examined, 0},
- {total_docs_examined, 0},
- {total_quorum_docs_examined, 0},
- {results_returned, 0},
+ {total_keys_examined, TotalKeysExamined},
+ {total_docs_examined, TotalDocsExamined},
+ {total_quorum_docs_examined, TotalQuorumDocsExamined},
+ {results_returned, ResultsReturned},
{execution_time_ms, '_'}
]},
UserFnDefinition =
@@ -1187,7 +1227,7 @@ t_execute_ok_all_docs_with_execution_stats(_) ->
bookmark = Bookmark,
bookmark_docid = undefined,
bookmark_key = undefined,
- execution_stats = #execution_stats{executionStartTime = {0, 0, 0}}
+ execution_stats = ExecutionStats
},
Extra =
[
@@ -1213,6 +1253,8 @@ t_execute_ok_all_docs_with_execution_stats(_) ->
Parameters = [
db, [{user_ctx, user_ctx}], fun
mango_cursor_view:handle_all_docs_message/2, Cursor1, Args
],
+ meck:expect(chttpd_stats, incr_rows, [TotalKeysExamined], meck:val(ok)),
+ meck:expect(chttpd_stats, incr_reads, [TotalDocsExamined], meck:val(ok)),
meck:expect(fabric, all_docs, Parameters, meck:val({ok, Cursor2})),
?assertEqual({ok, updated_accumulator2}, execute(Cursor, fun foo:bar/2,
accumulator)),
?assert(meck:called(fabric, all_docs, '_')).
@@ -1232,7 +1274,9 @@ t_execute_error_1(_) ->
db, '_', <<"ghibli">>, index_name, fun
mango_cursor_view:handle_message/2, '_', '_'
],
meck:expect(fabric, query_view, Parameters, meck:val({error, reason})),
- ?assertEqual({error, reason}, execute(Cursor, undefined, accumulator)).
+ ?assertEqual({error, reason}, execute(Cursor, undefined, accumulator)),
+ ?assertNot(meck:called(chttpd_stats, incr_rows, '_')),
+ ?assertNot(meck:called(chttpd_stats, incr_reads, '_')).
t_execute_error_2(_) ->
Cursor =
@@ -1249,7 +1293,9 @@ t_execute_error_2(_) ->
db, '_', <<"ghibli">>, index_name, fun
mango_cursor_view:handle_message/2, '_', '_'
],
meck:expect(fabric, query_view, Parameters, meck:val({ok, {error,
reason}})),
- ?assertEqual({error, {error, reason}}, execute(Cursor, undefined,
accumulator)).
+ ?assertEqual({error, {error, reason}}, execute(Cursor, undefined,
accumulator)),
+ ?assertNot(meck:called(chttpd_stats, incr_rows, '_')),
+ ?assertNot(meck:called(chttpd_stats, incr_reads, '_')).
view_cb_test_() ->
{
diff --git a/src/mango/src/mango_execution_stats.erl
b/src/mango/src/mango_execution_stats.erl
index 350b58bda..dce7c087b 100644
--- a/src/mango/src/mango_execution_stats.erl
+++ b/src/mango/src/mango_execution_stats.erl
@@ -15,6 +15,7 @@
-export([
to_json/1,
to_map/1,
+ incr_keys_examined/1,
incr_keys_examined/2,
incr_docs_examined/1,
incr_docs_examined/2,
@@ -53,6 +54,9 @@ to_map(Stats) ->
execution_time_ms => Stats#execution_stats.executionTimeMs
}.
+incr_keys_examined(Stats) ->
+ incr_keys_examined(Stats, 1).
+
incr_keys_examined(Stats, N) ->
Stats#execution_stats{
totalKeysExamined = Stats#execution_stats.totalKeysExamined + N
@@ -92,8 +96,8 @@ log_end(Stats) ->
maybe_add_stats(Opts, UserFun, Stats0, UserAcc) ->
Stats1 = log_end(Stats0),
couch_stats:update_histogram([mango, query_time],
Stats1#execution_stats.executionTimeMs),
- %% TODO: add rows read when we collect the stats
%% TODO: add docs vs quorum docs
+ chttpd_stats:incr_rows(Stats1#execution_stats.totalKeysExamined),
chttpd_stats:incr_reads(Stats1#execution_stats.totalDocsExamined),
FinalAcc =
diff --git a/src/mango/test/15-execution-stats-test.py
b/src/mango/test/15-execution-stats-test.py
index a8f996136..caa542d7a 100644
--- a/src/mango/test/15-execution-stats-test.py
+++ b/src/mango/test/15-execution-stats-test.py
@@ -81,7 +81,7 @@ class ExecutionStatsTests_Text(mango.UserDocsTextTests):
{"$text": "Stephanie"}, return_raw=True, executionStats=True
)
self.assertEqual(len(resp["docs"]), 1)
- self.assertEqual(resp["execution_stats"]["total_keys_examined"], 0)
+ self.assertEqual(resp["execution_stats"]["total_keys_examined"], 1)
self.assertEqual(resp["execution_stats"]["total_docs_examined"], 1)
self.assertEqual(resp["execution_stats"]["total_quorum_docs_examined"], 0)
self.assertEqual(resp["execution_stats"]["results_returned"], 1)
