This is an automated email from the ASF dual-hosted git repository.
pgj pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
The following commit(s) were added to refs/heads/main by this push:
new caa5e2320 mango: rolling execution statistics
caa5e2320 is described below
commit caa5e23204f56da71eb57b98d238d829a7509a2f
Author: Gabor Pali
AuthorDate: Fri Mar 15 01:37:52 2024 +0100
mango: rolling execution statistics
In case of map-reduce views, the arrival of the `complete` message
is not guaranteed for the view callback (at the shard) when a
`stop` is issued during the aggregation (at the coordinator). Due
to that, internally collected shard-level statistics may not be
fed back to the coordinator which can cause data loss hence
inaccuracy in the overall execution statistics.
Address this issue by switching to a "rolling" model where
row-level statistics are immediately streamed back to the
coordinator. Support mixed-version cluster upgrades by activating
this model only if requested through the map-reduce arguments and
the given shard supports that.
Fixes #4560
---
src/fabric/src/fabric_view_row.erl| 17 +++-
src/mango/src/mango_cursor_view.erl | 146 ++
src/mango/test/15-execution-stats-test.py | 69 ++
3 files changed, 194 insertions(+), 38 deletions(-)
diff --git a/src/fabric/src/fabric_view_row.erl
b/src/fabric/src/fabric_view_row.erl
index 8a313a11d..09dc3cff0 100644
--- a/src/fabric/src/fabric_view_row.erl
+++ b/src/fabric/src/fabric_view_row.erl
@@ -18,9 +18,11 @@
get_value/1,
get_doc/1,
get_worker/1,
+get_stats/1,
set_key/2,
set_doc/2,
set_worker/2,
+set_stats/2,
transform/1
]).
@@ -91,6 +93,14 @@ set_worker(#view_row{} = Row, Worker) ->
set_worker({view_row, #{} = Row}, Worker) ->
{view_row, Row#{worker => Worker}}.
+get_stats({view_row, #{stats := Stats}}) ->
+Stats;
+get_stats({view_row, #{}}) ->
+undefined.
+
+set_stats({view_row, #{} = Row}, Stats) ->
+{view_row, Row#{stats => Stats}}.
+
transform(#view_row{value = {[{reduce_overflow_error, Msg}]}}) ->
{row, [{key, null}, {id, error}, {value, reduce_overflow_error}, {reason,
Msg}]};
transform(#view_row{key = Key, id = reduced, value = Value}) ->
@@ -109,8 +119,13 @@ transform({view_row, #{} = Row0}) ->
Value = maps:get(value, Row0, undefined),
Doc = maps:get(doc, Row0, undefined),
Worker = maps:get(worker, Row0, undefined),
+Stats = maps:get(stats, Row0, undefined),
Row = #view_row{id = Id, key = Key, value = Value, doc = Doc, worker =
Worker},
-transform(Row).
+{row, Props} = RowProps = transform(Row),
+case Stats of
+undefined -> RowProps;
+#{} -> {row, [{stats, Stats} | Props]}
+end.
-ifdef(TEST).
-include_lib("couch/include/couch_eunit.hrl").
diff --git a/src/mango/src/mango_cursor_view.erl
b/src/mango/src/mango_cursor_view.erl
index 1f3943083..e42b14136 100644
--- a/src/mango/src/mango_cursor_view.erl
+++ b/src/mango/src/mango_cursor_view.erl
@@ -55,6 +55,15 @@
covering_index => 'maybe'(#idx{})
}.
+-type mrargs_extra_item() ::
+{callback, {atom(), atom()}}
+| {selector, any()}
+| {callback_args, viewcbargs()}
+| {ignore_partition_query_limit, boolean()}
+| {execution_stats_map, boolean()}
+| {execution_stats_rolling, boolean()}.
+-type mrargs_extra() :: [mrargs_extra_item()].
+
-spec viewcbargs_new(Selector, Fields, CoveringIndex) -> ViewCBArgs when
Selector :: selector(),
Fields :: fields(),
@@ -207,7 +216,9 @@ base_args(#cursor{index = Idx, selector = Selector, fields
= Fields} = Cursor) -
% - Return execution statistics in a map
{execution_stats_map, true},
% - Return view rows in a map
-{view_row_map, true}
+{view_row_map, true},
+% - Stream execution statistics
+{execution_stats_rolling, true}
]
}.
@@ -341,6 +352,43 @@ choose_best_index(IndexRanges) ->
{SelectedIndex, SelectedIndexRanges, _} = hd(SortedIndexRanges),
{{SelectedIndex, SelectedIndexRanges}, SortedIndexRanges}.
+-spec format_stats(RawStats, Options) -> FormattedStats when
+RawStats :: shard_stats_v2(),
+Options :: mrargs_extra(),
+FormattedStats :: shard_stats_v1() | shard_stats_v2().
+format_stats(Stats, Options) when is_list(Options) ->
+case couch_util:get_value(execution_stats_map, Options, false) of
+true ->
+Stats;
+false ->
+#{docs_examined := DocsExamined} = Stats,
+{docs_examined, DocsExamined}
+end.
+
+-spec submit_stats(Options) -> ok when
+Options :: mrargs_extra().
+submit_stats(Options) when is_list(Options) ->
+ShardStats = mango_execution_stats:shard_get_stats(),
+Stats = format_stats(ShardStats,
