This is an automated email from the ASF dual-hosted git repository. jan pushed a commit to branch improve-reduce-limit-errors in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit b030a8648214d88411191dd74f65ee4edc0ed7a2 Author: Paul J. Davis <[email protected]> AuthorDate: Thu Sep 7 14:37:36 2017 -0500 Return reduce overflow errors to the client This changes the reduce overflow error to return an error to the client rather than blowing up the view build. This allows views that have a single bad reduce to build while not crushing the server's RAM usage. --- src/couch/src/couch_query_servers.erl | 29 ++++++++++++++++++++++++----- src/couch_mrview/src/couch_mrview_http.erl | 6 +++++- src/fabric/src/fabric_util.erl | 2 -- src/fabric/src/fabric_view.erl | 2 ++ test/javascript/tests/view_errors.js | 4 ++-- 5 files changed, 33 insertions(+), 10 deletions(-) diff --git a/src/couch/src/couch_query_servers.erl b/src/couch/src/couch_query_servers.erl index 63b0e39..4928eea 100644 --- a/src/couch/src/couch_query_servers.erl +++ b/src/couch/src/couch_query_servers.erl @@ -125,20 +125,39 @@ os_reduce(Lang, OsRedSrcs, KVs) -> Proc = get_os_process(Lang), OsResults = try proc_prompt(Proc, [<<"reduce">>, OsRedSrcs, KVs]) of [true, Reductions] -> Reductions + catch + throw:{reduce_overflow_error, Msg} -> + [{[{reduce_overflow_error, Msg}]} || _ <- OsRedSrcs] after ok = ret_os_process(Proc) end, {ok, OsResults}. os_rereduce(Lang, OsRedSrcs, KVs) -> - Proc = get_os_process(Lang), - try proc_prompt(Proc, [<<"rereduce">>, OsRedSrcs, KVs]) of - [true, [Reduction]] -> Reduction - after - ok = ret_os_process(Proc) + case get_overflow_error(KVs) of + undefined -> + Proc = get_os_process(Lang), + try proc_prompt(Proc, [<<"rereduce">>, OsRedSrcs, KVs]) of + [true, [Reduction]] -> Reduction + catch + throw:{reduce_overflow_error, Msg} -> + {[{reduce_overflow_error, Msg}]} + after + ok = ret_os_process(Proc) + end; + Error -> + Error end. +get_overflow_error([]) -> + undefined; +get_overflow_error([{[{reduce_overflow_error, _}]} = Error | _]) -> + Error; +get_overflow_error([_ | Rest]) -> + get_overflow_error(Rest). + + builtin_reduce(_Re, [], _KVs, Acc) -> {ok, lists:reverse(Acc)}; builtin_reduce(Re, [<<"_sum",_/binary>>|BuiltinReds], KVs, Acc) -> diff --git a/src/couch_mrview/src/couch_mrview_http.erl b/src/couch_mrview/src/couch_mrview_http.erl index 9ad50ee..004caef 100644 --- a/src/couch_mrview/src/couch_mrview_http.erl +++ b/src/couch_mrview/src/couch_mrview_http.erl @@ -440,7 +440,11 @@ row_to_json(error, Row) -> % match prior behavior. Key = couch_util:get_value(key, Row), Val = couch_util:get_value(value, Row), - Obj = {[{key, Key}, {error, Val}]}, + Reason = couch_util:get_value(reason, Row), + ReasonProp = if Reason == undefined -> []; true -> + [{reason, Reason}] + end, + Obj = {[{key, Key}, {error, Val}] ++ ReasonProp}, ?JSON_ENCODE(Obj); row_to_json(Id0, Row) -> Id = case Id0 of diff --git a/src/fabric/src/fabric_util.erl b/src/fabric/src/fabric_util.erl index bf3f023..49f4c89 100644 --- a/src/fabric/src/fabric_util.erl +++ b/src/fabric/src/fabric_util.erl @@ -203,8 +203,6 @@ get_shard([#shard{node = Node, name = Name} | Rest], Opts, Timeout, Factor) -> rexi_monitor:stop(Mon) end. -error_info({{<<"reduce_overflow_error">>, _} = Error, _Stack}) -> - Error; error_info({{timeout, _} = Error, _Stack}) -> Error; error_info({{Error, Reason}, Stack}) -> diff --git a/src/fabric/src/fabric_view.erl b/src/fabric/src/fabric_view.erl index 45262e4..dd0fcfd 100644 --- a/src/fabric/src/fabric_view.erl +++ b/src/fabric/src/fabric_view.erl @@ -258,6 +258,8 @@ find_next_key([], _, _, _) -> find_next_key([Key|Rest], _, _, _) -> {Key, Rest}. +transform_row(#view_row{value={[{reduce_overflow_error, Msg}]}}) -> + {row, [{key,null}, {id,error}, {value,reduce_overflow_error}, {reason,Msg}]}; transform_row(#view_row{key=Key, id=reduced, value=Value}) -> {row, [{key,Key}, {value,Value}]}; transform_row(#view_row{key=Key, id=undefined}) -> diff --git a/test/javascript/tests/view_errors.js b/test/javascript/tests/view_errors.js index b53a3c7..0d9cd79 100644 --- a/test/javascript/tests/view_errors.js +++ b/test/javascript/tests/view_errors.js @@ -174,9 +174,9 @@ couchTests.view_errors = function(debug) { // if the reduce grows to fast, throw an overflow error var path = "/" + db_name + "/_design/testbig/_view/reduce_too_big"; xhr = CouchDB.request("GET", path); - T(xhr.status == 500); + T(xhr.status == 200); result = JSON.parse(xhr.responseText); - T(result.error == "reduce_overflow_error"); + T(result.rows[0].error == "reduce_overflow_error"); try { db.query(function() {emit(null, null)}, null, {startkey: 2, endkey:1}); -- To stop receiving notification emails like this one, please contact "[email protected]" <[email protected]>.
