jaydoane commented on code in PR #4144:
URL: https://github.com/apache/couchdb/pull/4144#discussion_r958810825
##########
src/couch_replicator/src/couch_replicator_api_wrap.erl:
##########
@@ -206,6 +207,74 @@ get_missing_revs(#httpdb{} = Db, IdRevs) ->
end
).
+bulk_get(#httpdb{} = Db, #{} = IdRevs, Options) ->
+ FoldFun = fun({Id, Rev}, PAs, Acc) -> [{Id, Rev, PAs} | Acc] end,
+ ReqDocsList = lists:sort(maps:fold(FoldFun, [], IdRevs)),
+ MapFun = fun({Id, Rev, PAs}) ->
+ #{
+ <<"id">> => Id,
+ <<"rev">> => couch_doc:rev_to_str(Rev),
+ <<"atts_since">> => couch_doc:revs_to_strs(PAs)
+ }
+ end,
+ ReqDocsMaps = lists:map(MapFun, ReqDocsList),
+ % We are also sending request parameters in the doc body with the hopes
+ % that at some point in the future we could make that the default, instead
+ % of having to send query parameters with a POST request as we do today
+ Body = options_to_json_map(Options, #{<<"docs">> => ReqDocsMaps}),
+ Req = [
+ {method, post},
+ {path, "_bulk_get"},
+ {qs, options_to_query_args(Options, [])},
+ {body, ?JSON_ENCODE(Body)},
+ {headers, [
+ {"Content-Type", "application/json"},
+ {"Accept", "application/json"}
+ ]}
+ ],
+ try
+ send_req(Db, Req, fun
+ (200, _, {[{<<"results">>, Res}]}) when is_list(Res) ->
+ Zip = lists:zipwith(fun bulk_get_zip/2, ReqDocsList, Res),
+ {ok, maps:from_list(Zip)};
+ (200, _, _) ->
+ {error, {bulk_get_failed, invalid_results}};
+ (ErrCode, _, _) when is_integer(ErrCode) ->
+ % On older Apache CouchDB instances where _bulk_get is not
+ % implemented we would hit the POST db/doc form uploader
+ % handler. When that fails the request body is not consumed and
+ % we'd end up recycling a worker with an unsent body in the
+ % connection stream. Instead of waiting for it to blow up
+ % eventually and consuming an extra retry attempt, proactively
+ % advise httpc logic to stop this worker and not return back to
+ % the pool.
+ couch_replicator_httpc:stop_http_worker(),
+ {error, {bulk_get_failed, ErrCode}}
+ end)
+ catch
+ exit:{http_request_failed, _, _, {error, {code, ErrCode}}} ->
+ % We are being a bit more tollerant of _bulk_get errors as we can
Review Comment:
s/tollerant/tolerant/
##########
src/couch_replicator/src/couch_replicator_scheduler_job.erl:
##########
@@ -25,7 +25,9 @@
handle_info/2,
handle_cast/2,
code_change/3,
- format_status/2
+ format_status/2,
+ add_stats/2,
Review Comment:
might this be more accurately `sum_stats/2`?
##########
src/couch_replicator/src/couch_replicator_api_wrap.erl:
##########
@@ -206,6 +207,74 @@ get_missing_revs(#httpdb{} = Db, IdRevs) ->
end
).
+bulk_get(#httpdb{} = Db, #{} = IdRevs, Options) ->
+ FoldFun = fun({Id, Rev}, PAs, Acc) -> [{Id, Rev, PAs} | Acc] end,
+ ReqDocsList = lists:sort(maps:fold(FoldFun, [], IdRevs)),
+ MapFun = fun({Id, Rev, PAs}) ->
+ #{
+ <<"id">> => Id,
+ <<"rev">> => couch_doc:rev_to_str(Rev),
+ <<"atts_since">> => couch_doc:revs_to_strs(PAs)
+ }
+ end,
+ ReqDocsMaps = lists:map(MapFun, ReqDocsList),
+ % We are also sending request parameters in the doc body with the hopes
+ % that at some point in the future we could make that the default, instead
+ % of having to send query parameters with a POST request as we do today
Review Comment:
Thanks for this comment
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
