Repository: couchdb Updated Branches: refs/heads/2041-update-ibrowse 8b6319f2a -> fbb4491dd (forced update)
ibrowse: update to 4.0.3 - for COUCHDB-2041 - should be re-pushed with amended VSN when upstream tags a release - ibrowse sha#108c131d at 20140306 Project: http://git-wip-us.apache.org/repos/asf/couchdb/repo Commit: http://git-wip-us.apache.org/repos/asf/couchdb/commit/fbb4491d Tree: http://git-wip-us.apache.org/repos/asf/couchdb/tree/fbb4491d Diff: http://git-wip-us.apache.org/repos/asf/couchdb/diff/fbb4491d Branch: refs/heads/2041-update-ibrowse Commit: fbb4491dd2195ad3397a457e1cee96dff9a24c88 Parents: 5c9f9a9 Author: Dave Cottlehuber <[email protected]> Authored: Wed Jan 29 11:05:23 2014 +0100 Committer: Dave Cottlehuber <[email protected]> Committed: Thu Mar 6 14:46:51 2014 +0000 ---------------------------------------------------------------------- src/ibrowse/ibrowse.app.in | 2 +- src/ibrowse/ibrowse.erl | 51 +++++---- src/ibrowse/ibrowse_http_client.erl | 190 ++++++++++++++++--------------- src/ibrowse/ibrowse_lb.erl | 28 +++-- src/ibrowse/ibrowse_lib.erl | 7 +- src/ibrowse/ibrowse_socks5.erl | 143 ++++++++--------------- 6 files changed, 198 insertions(+), 223 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/couchdb/blob/fbb4491d/src/ibrowse/ibrowse.app.in ---------------------------------------------------------------------- diff --git a/src/ibrowse/ibrowse.app.in b/src/ibrowse/ibrowse.app.in index 1d88084..813192b 100644 --- a/src/ibrowse/ibrowse.app.in +++ b/src/ibrowse/ibrowse.app.in @@ -1,6 +1,6 @@ {application, ibrowse, [{description, "Erlang HTTP client application"}, - {vsn, "4.0.1"}, + {vsn, "4.0.3pre"}, {registered, [ibrowse_sup, ibrowse]}, {applications, [kernel,stdlib]}, {env, []}, http://git-wip-us.apache.org/repos/asf/couchdb/blob/fbb4491d/src/ibrowse/ibrowse.erl ---------------------------------------------------------------------- diff --git a/src/ibrowse/ibrowse.erl b/src/ibrowse/ibrowse.erl index 80a4282..42030af 100644 --- a/src/ibrowse/ibrowse.erl +++ b/src/ibrowse/ibrowse.erl @@ -175,9 +175,11 @@ send_req(Url, Headers, Method) -> send_req(Url, Headers, Method, Body) -> send_req(Url, Headers, Method, Body, []). -%% @doc Same as send_req/4. -%% For a description of SSL Options, look in the <a href="http://www.erlang.org/doc/apps/ssl/index.html";>ssl</a> manpage. If the -%% HTTP Version to use is not specified, the default is 1.1. +%% @doc Same as send_req/4. + +%% For a description of SSL Options, look in the <a href="http://www.erlang.org/doc/apps/ssl/index.html";>ssl</a> manpage. +%% For a description of Process Options, look in the <a href="http://www.erlang.org/doc/man/gen_server.html";>gen_server</a> manpage. +%% If the HTTP Version to use is not specified, the default is 1.1. %% <br/> %% <ul> %% <li>The <code>host_header</code> option is useful in the case where ibrowse is @@ -286,7 +288,8 @@ send_req(Url, Headers, Method, Body) -> %% {headers_as_is, boolean()} | %% {give_raw_headers, boolean()} | %% {preserve_chunked_encoding,boolean()} | -%% {workaround, head_response_with_body} +%% {workaround, head_response_with_body} | +%% {worker_process_options, list()} %% %% stream_to() = process() | {process(), once} %% process() = pid() | atom() @@ -340,10 +343,12 @@ try_routing_request(Lb_pid, Parsed_url, Max_pipeline_size, {SSLOptions, IsSSL}, Headers, Method, Body, Options_1, Timeout, Try_count) when Try_count < 3 -> + ProcessOptions = get_value(worker_process_options, Options_1, []), case ibrowse_lb:spawn_connection(Lb_pid, Parsed_url, Max_sessions, Max_pipeline_size, - {SSLOptions, IsSSL}) of + {SSLOptions, IsSSL}, + ProcessOptions) of {ok, Conn_Pid} -> case do_send_req(Conn_Pid, Parsed_url, Headers, Method, Body, Options_1, Timeout) of @@ -438,6 +443,8 @@ do_send_req(Conn_Pid, Parsed_url, Headers, Method, Body, Options, Timeout) -> {error, sel_conn_closed}; {'EXIT', {normal, _}} -> {error, req_timedout}; + {'EXIT', {connection_closed, _}} -> + {error, sel_conn_closed}; {error, connection_closed} -> {error, sel_conn_closed}; {'EXIT', Reason} -> @@ -470,28 +477,32 @@ ensure_bin({Fun, _} = Body) when is_function(Fun) -> Body. %% request is sent via any of the send_req_direct/4,5,6,7 functions.<br/> %% <b>Note:</b> It is the responsibility of the calling process to control %% pipeline size on such connections. -%% -%% @spec spawn_worker_process(Url::string()) -> {ok, pid()} -spawn_worker_process(Url) -> - ibrowse_http_client:start(Url). -%% @doc Same as spawn_worker_process/1 but takes as input a Host and Port -%% instead of a URL. +%% @spec spawn_worker_process(Url::string() | {Host::string(), Port::integer()}) -> {ok, pid()} +spawn_worker_process(Args) -> + spawn_worker_process(Args, []). + +%% @doc Same as spawn_worker_process/1 except with Erlang process options. %% @spec spawn_worker_process(Host::string(), Port::integer()) -> {ok, pid()} -spawn_worker_process(Host, Port) -> - ibrowse_http_client:start({Host, Port}). +spawn_worker_process(Host, Port) when is_list(Host), is_integer(Port) -> + %% Convert old API calls to new API format. + spawn_worker_process({Host, Port}, []); +spawn_worker_process(Args, Options) -> + ibrowse_http_client:start(Args, Options). %% @doc Same as spawn_worker_process/1 except the the calling process %% is linked to the worker process which is spawned. -%% @spec spawn_link_worker_process(Url::string()) -> {ok, pid()} -spawn_link_worker_process(Url) -> - ibrowse_http_client:start_link(Url). +%% @spec spawn_link_worker_process(Url::string() | {Host::string(), Port::integer()}) -> {ok, pid()} +spawn_link_worker_process(Args) -> + spawn_link_worker_process(Args, []). -%% @doc Same as spawn_worker_process/2 except the the calling process -%% is linked to the worker process which is spawned. +%% @doc Same as spawn_link_worker_process/1 except with Erlang process options. %% @spec spawn_link_worker_process(Host::string(), Port::integer()) -> {ok, pid()} -spawn_link_worker_process(Host, Port) -> - ibrowse_http_client:start_link({Host, Port}). +spawn_link_worker_process(Host, Port) when is_list(Host), is_integer(Port) -> + %% Convert old API calls to new API format. + spawn_link_worker_process({Host, Port}, []); +spawn_link_worker_process(Args, Options) -> + ibrowse_http_client:start_link(Args, Options). %% @doc Terminate a worker process spawned using %% spawn_worker_process/2 or spawn_link_worker_process/2. Requests in http://git-wip-us.apache.org/repos/asf/couchdb/blob/fbb4491d/src/ibrowse/ibrowse_http_client.erl ---------------------------------------------------------------------- diff --git a/src/ibrowse/ibrowse_http_client.erl b/src/ibrowse/ibrowse_http_client.erl index a1cf6eb..0410c08 100644 --- a/src/ibrowse/ibrowse_http_client.erl +++ b/src/ibrowse/ibrowse_http_client.erl @@ -15,7 +15,9 @@ %% External exports -export([ start_link/1, + start_link/2, start/1, + start/2, stop/1, send_req/7 ]). @@ -39,8 +41,7 @@ -record(state, {host, port, connect_timeout, inactivity_timer_ref, - use_http_proxy = false, http_proxy_auth_digest, - socks5_host, socks5_port, socks5_user, socks5_password, + use_proxy = false, proxy_auth_digest, ssl_options = [], is_ssl = false, socket, proxy_tunnel_setup = false, tunnel_setup_queue = [], @@ -80,10 +81,16 @@ %% Description: Starts the server %%-------------------------------------------------------------------- start(Args) -> - gen_server:start(?MODULE, Args, []). + start(Args, []). + +start(Args, Options) -> + gen_server:start(?MODULE, Args, Options). start_link(Args) -> - gen_server:start_link(?MODULE, Args, []). + start_link(Args, []). + +start_link(Args, Options) -> + gen_server:start_link(?MODULE, Args, Options). stop(Conn_pid) -> case catch gen_server:call(Conn_pid, stop) of @@ -187,7 +194,7 @@ handle_info({ssl, _Sock, Data}, State) -> handle_info({stream_next, Req_id}, #state{socket = Socket, cur_req = #request{req_id = Req_id}} = State) -> - do_setopts(Socket, [{active, once}], State), + _ = do_setopts(Socket, [{active, once}], State), {noreply, set_inac_timer(State)}; handle_info({stream_next, _Req_id}, State) -> @@ -208,11 +215,11 @@ handle_info({stream_close, _Req_id}, State) -> handle_info({tcp_closed, _Sock}, State) -> do_trace("TCP connection closed by peer!~n", []), handle_sock_closed(State), - {stop, normal, State}; + {stop, connection_closed, State}; handle_info({ssl_closed, _Sock}, State) -> do_trace("SSL connection closed by peer!~n", []), handle_sock_closed(State), - {stop, normal, State}; + {stop, connection_closed, State}; handle_info({tcp_error, _Sock, Reason}, State) -> do_trace("Error on connection to ~1000.p:~1000.p -> ~1000.p~n", @@ -226,12 +233,12 @@ handle_info({ssl_error, _Sock, Reason}, State) -> {stop, normal, State}; handle_info({req_timedout, From}, State) -> - case lists:keymember(From, #request.from, queue:to_list(State#state.reqs)) of + case lists:keysearch(From, #request.from, queue:to_list(State#state.reqs)) of false -> {noreply, State}; - true -> + {value, #request{stream_to = StreamTo, req_id = ReqId}} -> + catch StreamTo ! {ibrowse_async_response_timeout, ReqId}, shutting_down(State), -%% do_error_reply(State, req_timedout), {stop, normal, State} end; @@ -288,12 +295,12 @@ handle_sock_data(Data, #state{status = get_header}=State) -> shutting_down(State), {stop, normal, State}; #state{socket = Socket, status = Status, cur_req = CurReq} = State_1 -> - case {Status, CurReq} of - {get_header, #request{caller_controls_socket = true}} -> - do_setopts(Socket, [{active, once}], State_1); - _ -> - active_once(State_1) - end, + _ = case {Status, CurReq} of + {get_header, #request{caller_controls_socket = true}} -> + do_setopts(Socket, [{active, once}], State_1); + _ -> + active_once(State_1) + end, {noreply, set_inac_timer(State_1)} end; @@ -312,7 +319,7 @@ handle_sock_data(Data, #state{status = get_body, {error, {Reason, {stat_code, StatCode}, Headers}}), {stop, normal, State}; State_1 -> - active_once(State_1), + _ = active_once(State_1), State_2 = set_inac_timer(State_1), {noreply, State_2} end; @@ -325,14 +332,14 @@ handle_sock_data(Data, #state{status = get_body, {stop, normal, State}; #state{cur_req = #request{caller_controls_socket = Ccs}, interim_reply_sent = Irs} = State_1 -> - case Irs of - true -> - active_once(State_1); - false when Ccs == true -> - do_setopts(Socket, [{active, once}], State); - false -> - active_once(State_1) - end, + _ = case Irs of + true -> + active_once(State_1); + false when Ccs == true -> + do_setopts(Socket, [{active, once}], State); + false -> + active_once(State_1) + end, State_2 = State_1#state{interim_reply_sent = false}, case Ccs of true -> @@ -342,7 +349,7 @@ handle_sock_data(Data, #state{status = get_body, {noreply, set_inac_timer(State_2)} end; State_1 -> - active_once(State_1), + _ = active_once(State_1), State_2 = set_inac_timer(State_1), {noreply, State_2} end @@ -489,25 +496,19 @@ handle_sock_closed(#state{reply_buffer = Buf, reqs = Reqs, http_status_code = SC State end. -do_connect(Host, Port, Options, #state{socks5_host = SocksHost}=State, Timeout) - when SocksHost /= undefined -> - ProxyOptions = [ - {user, State#state.socks5_user}, - {password, State#state.socks5_password}, - {host, SocksHost}, - {port, State#state.socks5_port}, - {is_ssl, State#state.is_ssl}, - {ssl_opts, State#state.ssl_options}], - ibrowse_socks5:connect(Host, Port, ProxyOptions, - get_sock_options(SocksHost, Options, []), - Timeout); -do_connect(Host, Port, Options, #state{is_ssl = true, - use_http_proxy = false, - ssl_options = SSLOptions}, +do_connect(Host, Port, Options, #state{is_ssl = true, + use_proxy = false, + ssl_options = SSLOptions}, Timeout) -> ssl:connect(Host, Port, get_sock_options(Host, Options, SSLOptions), Timeout); do_connect(Host, Port, Options, _State, Timeout) -> - gen_tcp:connect(Host, Port, get_sock_options(Host, Options, []), Timeout). + Socks5Host = get_value(socks5_host, Options, undefined), + case Socks5Host of + undefined -> + gen_tcp:connect(Host, Port, get_sock_options(Host, Options, []), Timeout); + _ -> + catch ibrowse_socks5:connect(Host, Port, Options) + end. get_sock_options(Host, Options, SSLOptions) -> Caller_socket_options = get_value(socket_options, Options, []), @@ -554,7 +555,7 @@ filter_sock_options(Opts) -> do_send(Req, #state{socket = Sock, is_ssl = true, - use_http_proxy = true, + use_proxy = true, proxy_tunnel_setup = Pts}) when Pts /= done -> gen_tcp:send(Sock, Req); do_send(Req, #state{socket = Sock, is_ssl = true}) -> ssl:send(Sock, Req); do_send(Req, #state{socket = Sock, is_ssl = false}) -> gen_tcp:send(Sock, Req). @@ -578,16 +579,16 @@ do_send_body1(Source, Resp, State, TE) -> {ok, Data} when Data == []; Data == <<>> -> do_send_body({Source}, State, TE); {ok, Data} -> - do_send(maybe_chunked_encode(Data, TE), State), + _ = do_send(maybe_chunked_encode(Data, TE), State), do_send_body({Source}, State, TE); {ok, Data, New_source_state} when Data == []; Data == <<>> -> do_send_body({Source, New_source_state}, State, TE); {ok, Data, New_source_state} -> - do_send(maybe_chunked_encode(Data, TE), State), + _ = do_send(maybe_chunked_encode(Data, TE), State), do_send_body({Source, New_source_state}, State, TE); eof when TE == true -> - do_send(<<"0\r\n\r\n">>, State), - ok; + _ = do_send(<<"0\r\n\r\n">>, State), + ok; eof -> ok; Err -> @@ -602,7 +603,7 @@ maybe_chunked_encode(Data, true) -> do_close(#state{socket = undefined}) -> ok; do_close(#state{socket = Sock, is_ssl = true, - use_http_proxy = true, + use_proxy = true, proxy_tunnel_setup = Pts }) when Pts /= done -> catch gen_tcp:close(Sock); do_close(#state{socket = Sock, is_ssl = true}) -> catch ssl:close(Sock); @@ -611,11 +612,11 @@ do_close(#state{socket = Sock, is_ssl = false}) -> catch gen_tcp:close(Sock). active_once(#state{cur_req = #request{caller_controls_socket = true}}) -> ok; active_once(#state{socket = Socket} = State) -> - do_setopts(Socket, [{active, once}], State). + _ = do_setopts(Socket, [{active, once}], State). do_setopts(_Sock, [], _) -> ok; do_setopts(Sock, Opts, #state{is_ssl = true, - use_http_proxy = true, + use_proxy = true, proxy_tunnel_setup = Pts} ) when Pts /= done -> inet:setopts(Sock, Opts); do_setopts(Sock, Opts, #state{is_ssl = true}) -> ssl:setopts(Sock, Opts); @@ -634,28 +635,17 @@ send_req_1(From, port = Port} = Url, Headers, Method, Body, Options, Timeout, #state{socket = undefined} = State) -> - ProxyHost = get_value(proxy_host, Options, false), - ProxyProtocol = get_value(proxy_protocol, Options, http), {Host_1, Port_1, State_1} = - case {ProxyHost, ProxyProtocol} of - {false, _} -> + case get_value(proxy_host, Options, false) of + false -> {Host, Port, State}; - {_, http} -> + PHost -> ProxyUser = get_value(proxy_user, Options, []), ProxyPassword = get_value(proxy_password, Options, []), Digest = http_auth_digest(ProxyUser, ProxyPassword), - {ProxyHost, get_value(proxy_port, Options, 80), - State#state{use_http_proxy = true, - http_proxy_auth_digest = Digest}}; - {_, socks5} -> - ProxyUser = list_to_binary(get_value(proxy_user, Options, [])), - ProxyPassword = list_to_binary(get_value(proxy_password, Options, [])), - ProxyPort = get_value(proxy_port, Options, 1080), - {Host, Port, - State#state{socks5_host = ProxyHost, - socks5_port = ProxyPort, - socks5_user = ProxyUser, - socks5_password = ProxyPassword}} + {PHost, get_value(proxy_port, Options, 80), + State#state{use_proxy = true, + proxy_auth_digest = Digest}} end, State_2 = check_ssl_options(Options, State_1), do_trace("Connecting...~n", []), @@ -686,7 +676,7 @@ send_req_1(From, Headers, Method, Body, Options, Timeout, #state{ proxy_tunnel_setup = false, - use_http_proxy = true, + use_proxy = true, is_ssl = true} = State) -> Ref = case Timeout of infinity -> @@ -713,7 +703,7 @@ send_req_1(From, case do_send_body(Body_1, State_1, TE) of ok -> trace_request_body(Body_1), - active_once(State_1), + _ = active_once(State_1), State_1_1 = inc_pipeline_counter(State_1), State_2 = State_1_1#state{status = get_header, cur_req = NewReq, @@ -796,7 +786,7 @@ send_req_1(From, AbsPath, RelPath, Body, Options, State_1, ReqId), trace_request(Req), - do_setopts(Socket, Caller_socket_options, State_1), + _ = do_setopts(Socket, Caller_socket_options, State_1), TE = is_chunked_encoding_specified(Options), case do_send(Req, State_1) of ok -> @@ -804,7 +794,7 @@ send_req_1(From, ok -> trace_request_body(Body_1), State_2 = inc_pipeline_counter(State_1), - active_once(State_2), + _ = active_once(State_2), State_3 = case Status of idle -> State_2#state{status = get_header, @@ -874,11 +864,11 @@ add_auth_headers(#url{username = User, end, add_proxy_auth_headers(State, Headers_1). -add_proxy_auth_headers(#state{use_http_proxy = false}, Headers) -> +add_proxy_auth_headers(#state{use_proxy = false}, Headers) -> Headers; -add_proxy_auth_headers(#state{http_proxy_auth_digest = []}, Headers) -> +add_proxy_auth_headers(#state{proxy_auth_digest = []}, Headers) -> Headers; -add_proxy_auth_headers(#state{http_proxy_auth_digest = Auth_digest}, Headers) -> +add_proxy_auth_headers(#state{proxy_auth_digest = Auth_digest}, Headers) -> [{"Proxy-Authorization", ["Basic ", Auth_digest]} | Headers]. http_auth_digest([], []) -> @@ -887,7 +877,7 @@ http_auth_digest(Username, Password) -> ibrowse_lib:encode_base64(Username ++ [$: | Password]). make_request(Method, Headers, AbsPath, RelPath, Body, Options, - #state{use_http_proxy = UseHttpProxy, is_ssl = Is_ssl}, ReqId) -> + #state{use_proxy = UseProxy, is_ssl = Is_ssl}, ReqId) -> HttpVsn = http_vsn_string(get_value(http_vsn, Options, {1,1})), Fun1 = fun({X, Y}) when is_atom(X) -> {to_lower(atom_to_list(X)), X, Y}; @@ -930,7 +920,7 @@ make_request(Method, Headers, AbsPath, RelPath, Body, Options, Headers_2 end, Headers_4 = cons_headers(Headers_3), - Uri = case get_value(use_absolute_uri, Options, false) or UseHttpProxy of + Uri = case get_value(use_absolute_uri, Options, false) or UseProxy of true -> case Is_ssl of true -> @@ -1061,7 +1051,9 @@ parse_response(Data, #state{reply_buffer = Acc, reqs = Reqs, http_status_code=StatCode} end, put(conn_close, ConnClose), - TransferEncoding = to_lower(get_value("transfer-encoding", LCHeaders, "false")), + TransferEncodings = to_lower(get_value("transfer-encoding", LCHeaders, "false")), + IsChunked = lists:any(fun(Enc) -> string:strip(Enc) =:= "chunked" end, + string:tokens(TransferEncodings, ",")), Head_response_with_body = lists:member({workaround, head_response_with_body}, Options), case get_value("content-length", LCHeaders, undefined) of _ when Method == connect, @@ -1112,7 +1104,7 @@ parse_response(Data, #state{reply_buffer = Acc, reqs = Reqs, State_2 = reset_state(State_1_1), State_3 = set_cur_request(State_2#state{reqs = Reqs_1}), parse_response(Data_1, State_3); - _ when TransferEncoding =:= "chunked" -> + _ when IsChunked -> do_trace("Chunked encoding detected...~n",[]), send_async_headers(ReqId, StreamTo, Give_raw_headers, State_1), case parse_11_response(Data_1, State_1#state{transfer_encoding=chunked, @@ -1130,6 +1122,25 @@ parse_response(Data, #state{reply_buffer = Acc, reqs = Reqs, ConnClose =:= "close" -> send_async_headers(ReqId, StreamTo, Give_raw_headers, State_1), State_1#state{reply_buffer = Data_1}; + undefined when StatCode =:= "303" -> + %% Some servers send 303 requests without a body. + %% RFC2616 says that they SHOULD, but they dont. + case ibrowse:get_config_value(allow_303_with_no_body, false) of + false -> + fail_pipelined_requests(State_1, + {error, {content_length_undefined, + {stat_code, StatCode}, Headers}}), + {error, content_length_undefined}; + true -> + {_, Reqs_1} = queue:out(Reqs), + send_async_headers(ReqId, StreamTo, Give_raw_headers, State_1), + State_1_1 = do_reply(State_1, From, StreamTo, ReqId, Resp_format, + {ok, StatCode, Headers_1, []}), + cancel_timer(T_ref, {eat_message, {req_timedout, From}}), + State_2 = reset_state(State_1_1), + State_3 = set_cur_request(State_2#state{reqs = Reqs_1}), + parse_response(Data_1, State_3) + end; undefined -> fail_pipelined_requests(State_1, {error, {content_length_undefined, @@ -1407,12 +1418,8 @@ set_cur_request(#state{reqs = Reqs, socket = Socket} = State) -> empty -> State#state{cur_req = undefined}; {value, #request{caller_controls_socket = Ccs} = NextReq} -> - case Ccs of - true -> - do_setopts(Socket, [{active, once}], State); - _ -> - ok - end, + _ = Ccs =:= true + andalso do_setopts(Socket, [{active, once}], State), State#state{cur_req = NextReq} end. @@ -1586,6 +1593,7 @@ get_crlf_pos(<<>>, _) -> no. fmt_val(L) when is_list(L) -> L; fmt_val(I) when is_integer(I) -> integer_to_list(I); fmt_val(A) when is_atom(A) -> atom_to_list(A); +fmt_val(B) when is_binary(B) -> B; fmt_val(Term) -> io_lib:format("~p", [Term]). crnl() -> "\r\n". @@ -1863,13 +1871,13 @@ dec_pipeline_counter(#state{lb_ets_tid = undefined} = State) -> State; dec_pipeline_counter(#state{cur_pipeline_size = Pipe_sz, lb_ets_tid = Tid} = State) -> - try - update_counter(Tid, self(), {2,-1,0,0}), - update_counter(Tid, self(), {3,-1,0,0}) - catch - _:_ -> - ok - end, + _ = try + update_counter(Tid, self(), {2,-1,0,0}), + update_counter(Tid, self(), {3,-1,0,0}) + catch + _:_ -> + ok + end, State#state{cur_pipeline_size = Pipe_sz - 1}. flatten([H | _] = L) when is_integer(H) -> http://git-wip-us.apache.org/repos/asf/couchdb/blob/fbb4491d/src/ibrowse/ibrowse_lb.erl ---------------------------------------------------------------------- diff --git a/src/ibrowse/ibrowse_lb.erl b/src/ibrowse/ibrowse_lb.erl index d98cf32..f5a9aef 100644 --- a/src/ibrowse/ibrowse_lb.erl +++ b/src/ibrowse/ibrowse_lb.erl @@ -16,7 +16,7 @@ %% External exports -export([ start_link/1, - spawn_connection/5, + spawn_connection/6, stop/1 ]). @@ -81,13 +81,14 @@ init([Host, Port]) -> spawn_connection(Lb_pid, Url, Max_sessions, Max_pipeline_size, - SSL_options) + SSL_options, + Process_options) when is_pid(Lb_pid), is_record(Url, url), is_integer(Max_pipeline_size), is_integer(Max_sessions) -> gen_server:call(Lb_pid, - {spawn_connection, Url, Max_sessions, Max_pipeline_size, SSL_options}). + {spawn_connection, Url, Max_sessions, Max_pipeline_size, SSL_options, Process_options}). stop(Lb_pid) -> case catch gen_server:call(Lb_pid, stop) of @@ -123,19 +124,19 @@ handle_call(_, _From, #state{proc_state = shutting_down} = State) -> {reply, {error, shutting_down}, State}; %% Update max_sessions in #state with supplied value -handle_call({spawn_connection, _Url, Max_sess, Max_pipe, _}, _From, - #state{num_cur_sessions = Num} = State) +handle_call({spawn_connection, _Url, Max_sess, Max_pipe, _, _}, _From, + #state{num_cur_sessions = Num} = State) when Num >= Max_sess -> State_1 = maybe_create_ets(State), Reply = find_best_connection(State_1#state.ets_tid, Max_pipe), {reply, Reply, State_1#state{max_sessions = Max_sess, max_pipeline_size = Max_pipe}}; -handle_call({spawn_connection, Url, Max_sess, Max_pipe, SSL_options}, _From, +handle_call({spawn_connection, Url, Max_sess, Max_pipe, SSL_options, Process_options}, _From, #state{num_cur_sessions = Cur} = State) -> State_1 = maybe_create_ets(State), Tid = State_1#state.ets_tid, - {ok, Pid} = ibrowse_http_client:start_link({Tid, Url, SSL_options}), + {ok, Pid} = ibrowse_http_client:start_link({Tid, Url, SSL_options}, Process_options), ets:insert(Tid, {Pid, 0, 0}), {reply, {ok, Pid}, State_1#state{num_cur_sessions = Cur + 1, max_sessions = Max_sess, @@ -230,7 +231,9 @@ code_change(_OldVsn, State, _Extra) -> %%% Internal functions %%-------------------------------------------------------------------- find_best_connection(Tid, Max_pipe) -> + ets:safe_fixtable(Tid, true), Res = find_best_connection(ets:first(Tid), Tid, Max_pipe), + ets:safe_fixtable(Tid, false), Res. find_best_connection('$end_of_table', _, _) -> @@ -239,9 +242,14 @@ find_best_connection(Pid, Tid, Max_pipe) -> case ets:lookup(Tid, Pid) of [{Pid, Cur_sz, Speculative_sz}] when Cur_sz < Max_pipe, Speculative_sz < Max_pipe -> - ets:update_counter(Tid, Pid, {3, 1, 9999999, 9999999}), - {ok, Pid}; - _ -> + case catch ets:update_counter(Tid, Pid, {3, 1, 9999999, 9999999}) of + {'EXIT', _} -> + %% The selected process has shutdown + find_best_connection(ets:next(Tid, Pid), Tid, Max_pipe); + _ -> + {ok, Pid} + end; + _ -> find_best_connection(ets:next(Tid, Pid), Tid, Max_pipe) end. http://git-wip-us.apache.org/repos/asf/couchdb/blob/fbb4491d/src/ibrowse/ibrowse_lib.erl ---------------------------------------------------------------------- diff --git a/src/ibrowse/ibrowse_lib.erl b/src/ibrowse/ibrowse_lib.erl index 7b12cb3..1ce6bd4 100644 --- a/src/ibrowse/ibrowse_lib.erl +++ b/src/ibrowse/ibrowse_lib.erl @@ -362,10 +362,9 @@ parse_url([], get_password, Url, TmpAcc) -> parse_url([], State, Url, TmpAcc) -> {invalid_uri_2, State, Url, TmpAcc}. -default_port(socks5) -> 1080; -default_port(http) -> 80; -default_port(https) -> 443; -default_port(ftp) -> 21. +default_port(http) -> 80; +default_port(https) -> 443; +default_port(ftp) -> 21. printable_date() -> {{Y,Mo,D},{H, M, S}} = calendar:local_time(), http://git-wip-us.apache.org/repos/asf/couchdb/blob/fbb4491d/src/ibrowse/ibrowse_socks5.erl ---------------------------------------------------------------------- diff --git a/src/ibrowse/ibrowse_socks5.erl b/src/ibrowse/ibrowse_socks5.erl index d00df44..e6d8913 100644 --- a/src/ibrowse/ibrowse_socks5.erl +++ b/src/ibrowse/ibrowse_socks5.erl @@ -1,109 +1,58 @@ -% Licensed under the Apache License, Version 2.0 (the "License"); you may not -% use this file except in compliance with the License. You may obtain a copy of -% the License at -% -% http://www.apache.org/licenses/LICENSE-2.0 -% -% Unless required by applicable law or agreed to in writing, software -% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -% License for the specific language governing permissions and limitations under -% the License. - -module(ibrowse_socks5). --define(VERSION, 5). --define(CONNECT, 1). +-include_lib("kernel/src/inet_dns.hrl"). --define(NO_AUTH, 0). --define(USERPASS, 2). --define(UNACCEPTABLE, 16#FF). --define(RESERVED, 0). +-export([connect/3]). --define(ATYP_IPV4, 1). --define(ATYP_DOMAINNAME, 3). --define(ATYP_IPV6, 4). +-define(TIMEOUT, 2000). --define(SUCCEEDED, 0). +-define(SOCKS5, 5). +-define(AUTH_METHOD_NO, 0). +-define(AUTH_METHOD_USERPASS, 2). +-define(ADDRESS_TYPE_IP4, 1). +-define(COMMAND_TYPE_TCPIP_STREAM, 1). +-define(RESERVER, 0). +-define(STATUS_GRANTED, 0). --export([connect/5]). +-define(DNS_IP, {8,8,8,8}). --import(ibrowse_lib, [get_value/2, get_value/3]). +connect(Host, Port, Options) -> + Socks5Host = proplists:get_value(socks5_host, Options), + Socks5Port = proplists:get_value(socks5_port, Options), -connect(TargetHost, TargetPort, ProxyOptions, Options, Timeout) -> - case gen_tcp:connect(get_value(host, ProxyOptions), - get_value(port, ProxyOptions), - Options, Timeout) of - {ok, Socket} -> - case handshake(Socket, Options) of - ok -> - case connect(TargetHost, TargetPort, Socket) of - ok -> - maybe_ssl(Socket, ProxyOptions, Timeout); - Else -> - gen_tcp:close(Socket), - Else - end; - Else -> - gen_tcp:close(Socket), - Else - end; - Else -> - Else - end. + {ok, Socket} = gen_tcp:connect(Socks5Host, Socks5Port, [binary, {packet, 0}, {keepalive, true}, {active, false}]), -handshake(Socket, ProxyOptions) when is_port(Socket) -> - {Handshake, Success} = case get_value(user, ProxyOptions, <<>>) of - <<>> -> - {<<?VERSION, 1, ?NO_AUTH>>, ?NO_AUTH}; - User -> - Password = get_value(password, ProxyOptions, <<>>), - {<<?VERSION, 1, ?USERPASS, (byte_size(User)), User, - (byte_size(Password)), Password>>, ?USERPASS} - end, - ok = gen_tcp:send(Socket, Handshake), - case gen_tcp:recv(Socket, 0) of - {ok, <<?VERSION, Success>>} -> - ok; - {ok, <<?VERSION, ?UNACCEPTABLE>>} -> - {error, unacceptable}; - {error, Reason} -> - {error, Reason} - end. + {ok, _Bin} = + case proplists:get_value(socks5_user, Options, undefined) of + undefined -> + ok = gen_tcp:send(Socket, <<?SOCKS5, 1, ?AUTH_METHOD_NO>>), + {ok, <<?SOCKS5, ?AUTH_METHOD_NO>>} = gen_tcp:recv(Socket, 2, ?TIMEOUT); + _Else -> + Socks5User = list_to_binary(proplists:get_value(socks5_user, Options)), + Socks5Pass = list_to_binary(proplists:get_value(socks5_pass, Options)), + + ok = gen_tcp:send(Socket, <<?SOCKS5, 1, ?AUTH_METHOD_USERPASS>>), + {ok, <<?SOCKS5, ?AUTH_METHOD_USERPASS>>} = gen_tcp:recv(Socket, 2, ?TIMEOUT), -connect(Host, Port, Via) when is_list(Host) -> - connect(list_to_binary(Host), Port, Via); -connect(Host, Port, Via) when is_binary(Host), is_integer(Port), - is_port(Via) -> - ok = gen_tcp:send(Via, - <<?VERSION, ?CONNECT, ?RESERVED, ?ATYP_DOMAINNAME, - (byte_size(Host)), Host/binary, - (Port):16>>), - case gen_tcp:recv(Via, 0) of - {ok, <<?VERSION, ?SUCCEEDED, ?RESERVED, _/binary>>} -> - ok; - {ok, <<?VERSION, Rep, ?RESERVED, _/binary>>} -> - {error, rep(Rep)}; - {error, Reason} -> - {error, Reason} - end. + UserLength = byte_size(Socks5User), -maybe_ssl(Socket, ProxyOptions, Timeout) -> - IsSsl = get_value(is_ssl, ProxyOptions, false), - SslOpts = get_value(ssl_opts, ProxyOptions, []), - case IsSsl of - false -> - {ok, Socket}; - true -> - ssl:connect(Socket, SslOpts, Timeout) - end. + ok = gen_tcp:send(Socket, << 1, UserLength >>), + ok = gen_tcp:send(Socket, Socks5User), + PassLength = byte_size(Socks5Pass), + ok = gen_tcp:send(Socket, << PassLength >>), + ok = gen_tcp:send(Socket, Socks5Pass), + {ok, <<1, 0>>} = gen_tcp:recv(Socket, 2, ?TIMEOUT) + end, + + {IP1,IP2,IP3,IP4} = case inet_parse:address(Host) of + {ok, IP} -> + IP; + _Other -> + {ok, NsData} = inet_res:nslookup(Host, in, a, [{?DNS_IP, 53}]), + [Addr | _NewAnList] = [D || #dns_rr{data=D, type=a} <- NsData#dns_rec.anlist], + Addr + end, -rep(0) -> succeeded; -rep(1) -> server_fail; -rep(2) -> disallowed_by_ruleset; -rep(3) -> network_unreachable; -rep(4) -> host_unreachable; -rep(5) -> connection_refused; -rep(6) -> ttl_expired; -rep(7) -> command_not_supported; -rep(8) -> address_type_not_supported. + ok = gen_tcp:send(Socket, <<?SOCKS5, ?COMMAND_TYPE_TCPIP_STREAM, ?RESERVER, ?ADDRESS_TYPE_IP4, IP1, IP2, IP3, IP4, Port:16>>), + {ok, << ?SOCKS5, ?STATUS_GRANTED, ?RESERVER, ?ADDRESS_TYPE_IP4, IP1, IP2, IP3, IP4, Port:16 >>} = gen_tcp:recv(Socket, 10, ?TIMEOUT), + {ok, Socket}.
