Negotiate max uri length in response to 414's
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/a8add8b9 Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/a8add8b9 Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/a8add8b9 Branch: refs/heads/1843-feature-bigcouch Commit: a8add8b926b724828fdd24f62ae7a6aed053e5b8 Parents: 15d75c3 Author: Robert Newson <[email protected]> Authored: Thu Feb 13 17:09:58 2014 +0000 Committer: Robert Newson <[email protected]> Committed: Thu Apr 24 17:36:51 2014 +0100 ---------------------------------------------------------------------- src/couch_replicator_api_wrap.erl | 40 ++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/a8add8b9/src/couch_replicator_api_wrap.erl ---------------------------------------------------------------------- diff --git a/src/couch_replicator_api_wrap.erl b/src/couch_replicator_api_wrap.erl index 590f6ef..f898d0a 100644 --- a/src/couch_replicator_api_wrap.erl +++ b/src/couch_replicator_api_wrap.erl @@ -50,6 +50,9 @@ -define(MAX_WAIT, 5 * 60 * 1000). +-define(MAX_URL_LEN, 7000). +-define(MIN_URL_LEN, 200). + db_uri(#httpdb{url = Url}) -> couch_util:url_strip_password(Url); @@ -171,13 +174,16 @@ open_doc_revs(#httpdb{} = HttpDb, Id, Revs, Options, Fun, Acc) -> QS = options_to_query_args(HttpDb, Path, [revs, {open_revs, Revs} | Options]), {Pid, Ref} = spawn_monitor(fun() -> Self = self(), - Callback = fun(200, Headers, StreamDataFun) -> + Callback = fun + (200, Headers, StreamDataFun) -> remote_open_doc_revs_streamer_start(Self), {<<"--">>, _, _} = couch_httpd:parse_multipart_request( get_value("Content-Type", Headers), StreamDataFun, fun mp_parse_mixed/1 - ) + ); + (414, _, _) -> + exit(request_uri_too_long) end, Streamer = spawn_link(fun() -> Params = [ @@ -217,6 +223,17 @@ open_doc_revs(#httpdb{} = HttpDb, Id, Revs, Options, Fun, Acc) -> Ret; {'DOWN', Ref, process, Pid, {{nocatch, {missing_stub,_} = Stub}, _}} -> throw(Stub); + {'DOWN', Ref, process, Pid, request_uri_too_long} -> + NewMaxLen = get_value(max_url_len, Options, ?MAX_URL_LEN) div 2, + case NewMaxLen < ?MIN_URL_LEN of + true -> + throw(request_uri_too_long); + false -> + ?LOG_INFO("Reducing url length to ~B because of 414 response", [NewMaxLen]), + Options1 = lists:keystore(max_url_len, 1, Options, + {max_url_len, NewMaxLen}), + open_doc_revs(HttpDb, Id, Revs, Options1, Fun, Acc) + end; {'DOWN', Ref, process, Pid, Else} -> Url = couch_util:url_strip_password( couch_replicator_httpc:full_url(HttpDb, [{path,Path}, {qs,QS}]) @@ -489,7 +506,11 @@ changes_json_req(Db, FilterName, {QueryParams}, _Options) -> ]}. -options_to_query_args(HttpDb, Path, Options) -> +options_to_query_args(HttpDb, Path, Options0) -> + case lists:keytake(max_url_len, 1, Options0) of + false -> MaxLen = ?MAX_URL_LEN, Options = Options0; + {value, {max_url_len, MaxLen}, Options} -> ok + end, case lists:keytake(atts_since, 1, Options) of false -> options_to_query_args(Options, []); @@ -502,7 +523,7 @@ options_to_query_args(HttpDb, Path, Options) -> RevList = atts_since_arg( length("GET " ++ FullUrl ++ " HTTP/1.1\r\n") + length("&atts_since=") + 6, % +6 = % encoded [ and ] - PAs, []), + PAs, MaxLen, []), [{"atts_since", ?JSON_ENCODE(RevList)} | QueryArgs1] end. @@ -523,12 +544,9 @@ options_to_query_args([{open_revs, Revs} | Rest], Acc) -> JsonRevs = ?b2l(iolist_to_binary(?JSON_ENCODE(couch_doc:revs_to_strs(Revs)))), options_to_query_args(Rest, [{"open_revs", JsonRevs} | Acc]). - --define(MAX_URL_LEN, 7000). - -atts_since_arg(_UrlLen, [], Acc) -> +atts_since_arg(_UrlLen, [], _MaxLen, Acc) -> lists:reverse(Acc); -atts_since_arg(UrlLen, [PA | Rest], Acc) -> +atts_since_arg(UrlLen, [PA | Rest], MaxLen, Acc) -> RevStr = couch_doc:rev_to_str(PA), NewUrlLen = case Rest of [] -> @@ -538,11 +556,11 @@ atts_since_arg(UrlLen, [PA | Rest], Acc) -> % plus 2 double quotes and a comma (% encoded) UrlLen + size(RevStr) + 9 end, - case NewUrlLen >= ?MAX_URL_LEN of + case NewUrlLen >= MaxLen of true -> lists:reverse(Acc); false -> - atts_since_arg(NewUrlLen, Rest, [RevStr | Acc]) + atts_since_arg(NewUrlLen, Rest, MaxLen, [RevStr | Acc]) end.
