This is an automated email from the ASF dual-hosted git repository. rnewson pushed a commit to branch enhance_reduce_limit-2 in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 7d33ebf304fddbc02ed4a24ff1dfc17b287944cd Author: Robert Newson <[email protected]> AuthorDate: Mon Sep 22 14:32:26 2025 +0100 parameterise reduce_limit threshold and ratio --- share/server/views.js | 8 ++++---- src/couch/src/couch_proc_manager.erl | 2 ++ src/couch/src/couch_query_servers.erl | 9 ++++++++- src/couch/test/eunit/couch_query_servers_tests.erl | 22 +++++++++++++++++++++- 4 files changed, 35 insertions(+), 6 deletions(-) diff --git a/share/server/views.js b/share/server/views.js index b59c9912e..b80c997d9 100644 --- a/share/server/views.js +++ b/share/server/views.js @@ -36,9 +36,9 @@ var Views = (function() { var reduce_line = JSON.stringify(reductions); var reduce_length = reduce_line.length; var input_length = State.line_length - code_size - // TODO make reduce_limit config into a number if (State.query_config && State.query_config.reduce_limit && - reduce_length > 4096 && ((reduce_length * 2) > input_length)) { + reduce_length > State.query_config.reduce_limit_threshold && + ((reduce_length * State.query_config.reduce_limit_ratio) > input_length)) { var log_message = [ "Reduce output must shrink more rapidly:", "input size:", input_length, @@ -65,8 +65,8 @@ var Views = (function() { // fatal_error. But by default if they don't do error handling we // just eat the exception and carry on. // - // In this case we abort map processing but don't destroy the - // JavaScript process. If you need to destroy the JavaScript + // In this case we abort map processing but don't destroy the + // JavaScript process. If you need to destroy the JavaScript // process, throw the error form matched by the block below. throw(["error", "map_runtime_error", "function raised 'fatal_error'"]); } else if (err[0] == "fatal") { diff --git a/src/couch/src/couch_proc_manager.erl b/src/couch/src/couch_proc_manager.erl index 6b9eb817c..aa538e23e 100644 --- a/src/couch/src/couch_proc_manager.erl +++ b/src/couch/src/couch_proc_manager.erl @@ -732,6 +732,8 @@ remove_waiting_client(#client{wait_key = Key}) -> get_proc_config() -> {[ {<<"reduce_limit">>, get_reduce_limit()}, + {<<"reduce_limit_threshold">>, couch_query_servers:reduce_limit_threshold()}, + {<<"reduce_limit_ratio">>, couch_query_servers:reduce_limit_ratio()}, {<<"timeout">>, get_os_process_timeout()} ]}. diff --git a/src/couch/src/couch_query_servers.erl b/src/couch/src/couch_query_servers.erl index c29f13a5e..ef9e28f9b 100644 --- a/src/couch/src/couch_query_servers.erl +++ b/src/couch/src/couch_query_servers.erl @@ -19,6 +19,7 @@ -export([filter_view/4]). -export([finalize/2]). -export([rewrite/3]). +-export([reduce_limit_threshold/0, reduce_limit_ratio/0]). -export([with_ddoc_proc/3, proc_prompt/2, ddoc_prompt/4, ddoc_proc_prompt/3, json_doc/1]). @@ -278,7 +279,7 @@ sum_arrays(Else, _) -> throw_sum_error(Else). check_sum_overflow(InSize, OutSize, Sum) -> - Overflowed = OutSize > 4906 andalso OutSize * 2 > InSize, + Overflowed = OutSize > reduce_limit_threshold() andalso OutSize * reduce_limit_ratio() > InSize, case config:get("query_server_config", "reduce_limit", "true") of "true" when Overflowed -> Msg = log_sum_overflow(InSize, OutSize), @@ -302,6 +303,12 @@ log_sum_overflow(InSize, OutSize) -> couch_log:error(Msg, []), Msg. +reduce_limit_threshold() -> + config:get_integer("query_server_config", "reduce_limit_threshold", 4906). + +reduce_limit_ratio() -> + config:get_float("query_server_config", "reduce_limit_ratio", 2.0). + builtin_stats(_, []) -> {0, 0, 0, 0, 0}; builtin_stats(_, [[_, First] | Rest]) -> diff --git a/src/couch/test/eunit/couch_query_servers_tests.erl b/src/couch/test/eunit/couch_query_servers_tests.erl index 369e82e88..a2fffb446 100644 --- a/src/couch/test/eunit/couch_query_servers_tests.erl +++ b/src/couch/test/eunit/couch_query_servers_tests.erl @@ -23,7 +23,9 @@ setup() -> Ctx. teardown(Ctx) -> - config:delete("query_server_config", "reduce_limit", "true", false), + config:delete("query_server_config", "reduce_limit", true), + config:delete("query_server_config", "reduce_limit_threshold", true), + config:delete("query_server_config", "reduce_limit_ratio", true), config:delete("log", "level", false), test_util:stop_couch(Ctx), meck:unload(). @@ -37,6 +39,8 @@ query_server_limits_test_() -> fun teardown/1, [ ?TDEF_FE(builtin_should_return_error_on_overflow), + ?TDEF_FE(builtin_should_not_return_error_with_generous_overflow_threshold), + ?TDEF_FE(builtin_should_not_return_error_with_generous_overflow_ratio), ?TDEF_FE(builtin_should_return_object_on_log), ?TDEF_FE(builtin_should_return_object_on_false), ?TDEF_FE(js_reduce_should_return_error_on_overflow), @@ -55,6 +59,22 @@ builtin_should_return_error_on_overflow(_) -> ?assertMatch({[{<<"error">>, <<"builtin_reduce_error">>} | _]}, Result), ?assert(meck:called(couch_log, error, '_')). +builtin_should_not_return_error_with_generous_overflow_threshold(_) -> + config:set("query_server_config", "reduce_limit", "true", false), + config:set_integer("query_server_config", "reduce_limit_threshold", 1000000, false), + meck:reset(couch_log), + KVs = gen_sum_kvs(), + {ok, [Result]} = couch_query_servers:reduce(<<"foo">>, [<<"_sum">>], KVs), + ?assertNotMatch({[{<<"error">>, <<"builtin_reduce_error">>} | _]}, Result). + +builtin_should_not_return_error_with_generous_overflow_ratio(_) -> + config:set("query_server_config", "reduce_limit", "true", false), + config:set_float("query_server_config", "reduce_limit_ratio", 0.1, false), + meck:reset(couch_log), + KVs = gen_sum_kvs(), + {ok, [Result]} = couch_query_servers:reduce(<<"foo">>, [<<"_sum">>], KVs), + ?assertNotMatch({[{<<"error">>, <<"builtin_reduce_error">>} | _]}, Result). + builtin_should_return_object_on_log(_) -> config:set("query_server_config", "reduce_limit", "log", false), meck:reset(couch_log),
