nickva commented on code in PR #4144:
URL: https://github.com/apache/couchdb/pull/4144#discussion_r959022533
##########
src/couch_replicator/src/couch_replicator_scheduler_job.erl:
##########
@@ -25,7 +25,9 @@
handle_info/2,
handle_cast/2,
code_change/3,
- format_status/2
+ format_status/2,
+ add_stats/2,
Review Comment:
Agree, that would be better!
##########
src/couch_replicator/src/couch_replicator_api_wrap.erl:
##########
@@ -206,6 +207,74 @@ get_missing_revs(#httpdb{} = Db, IdRevs) ->
end
).
+bulk_get(#httpdb{} = Db, #{} = IdRevs, Options) ->
+ FoldFun = fun({Id, Rev}, PAs, Acc) -> [{Id, Rev, PAs} | Acc] end,
+ ReqDocsList = lists:sort(maps:fold(FoldFun, [], IdRevs)),
+ MapFun = fun({Id, Rev, PAs}) ->
+ #{
+ <<"id">> => Id,
+ <<"rev">> => couch_doc:rev_to_str(Rev),
+ <<"atts_since">> => couch_doc:revs_to_strs(PAs)
+ }
+ end,
+ ReqDocsMaps = lists:map(MapFun, ReqDocsList),
+ % We are also sending request parameters in the doc body with the hopes
+ % that at some point in the future we could make that the default, instead
+ % of having to send query parameters with a POST request as we do today
+ Body = options_to_json_map(Options, #{<<"docs">> => ReqDocsMaps}),
+ Req = [
+ {method, post},
+ {path, "_bulk_get"},
+ {qs, options_to_query_args(Options, [])},
+ {body, ?JSON_ENCODE(Body)},
+ {headers, [
+ {"Content-Type", "application/json"},
+ {"Accept", "application/json"}
+ ]}
+ ],
+ try
+ send_req(Db, Req, fun
+ (200, _, {[{<<"results">>, Res}]}) when is_list(Res) ->
+ Zip = lists:zipwith(fun bulk_get_zip/2, ReqDocsList, Res),
+ {ok, maps:from_list(Zip)};
+ (200, _, _) ->
+ {error, {bulk_get_failed, invalid_results}};
+ (ErrCode, _, _) when is_integer(ErrCode) ->
+ % On older Apache CouchDB instances where _bulk_get is not
+ % implemented we would hit the POST db/doc form uploader
+ % handler. When that fails the request body is not consumed and
+ % we'd end up recycling a worker with an unsent body in the
+ % connection stream. Instead of waiting for it to blow up
+ % eventually and consuming an extra retry attempt, proactively
+ % advise httpc logic to stop this worker and not return back to
+ % the pool.
+ couch_replicator_httpc:stop_http_worker(),
+ {error, {bulk_get_failed, ErrCode}}
+ end)
+ catch
+ exit:{http_request_failed, _, _, {error, {code, ErrCode}}} ->
+ % We are being a bit more tollerant of _bulk_get errors as we can
Review Comment:
Oh thanks, good find!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
