http://git-wip-us.apache.org/repos/asf/couchdb-ibrowse/blob/4af2d408/ibrowse_lb.erl ---------------------------------------------------------------------- diff --git a/ibrowse_lb.erl b/ibrowse_lb.erl deleted file mode 100644 index d98cf32..0000000 --- a/ibrowse_lb.erl +++ /dev/null @@ -1,252 +0,0 @@ -%%%------------------------------------------------------------------- -%%% File : ibrowse_lb.erl -%%% Author : chandru <[email protected]> -%%% Description : -%%% -%%% Created : 6 Mar 2008 by chandru <[email protected]> -%%%------------------------------------------------------------------- --module(ibrowse_lb). --author(chandru). --behaviour(gen_server). -%%-------------------------------------------------------------------- -%% Include files -%%-------------------------------------------------------------------- - -%%-------------------------------------------------------------------- -%% External exports --export([ - start_link/1, - spawn_connection/5, - stop/1 - ]). - -%% gen_server callbacks --export([ - init/1, - handle_call/3, - handle_cast/2, - handle_info/2, - terminate/2, - code_change/3 - ]). - --record(state, {parent_pid, - ets_tid, - host, - port, - max_sessions, - max_pipeline_size, - num_cur_sessions = 0, - proc_state - }). - --include("ibrowse.hrl"). - -%%==================================================================== -%% External functions -%%==================================================================== -%%-------------------------------------------------------------------- -%% Function: start_link/0 -%% Description: Starts the server -%%-------------------------------------------------------------------- -start_link(Args) -> - gen_server:start_link(?MODULE, Args, []). - -%%==================================================================== -%% Server functions -%%==================================================================== - -%%-------------------------------------------------------------------- -%% Function: init/1 -%% Description: Initiates the server -%% Returns: {ok, State} | -%% {ok, State, Timeout} | -%% ignore | -%% {stop, Reason} -%%-------------------------------------------------------------------- -init([Host, Port]) -> - process_flag(trap_exit, true), - Max_sessions = ibrowse:get_config_value({max_sessions, Host, Port}, 10), - Max_pipe_sz = ibrowse:get_config_value({max_pipeline_size, Host, Port}, 10), - put(my_trace_flag, ibrowse_lib:get_trace_status(Host, Port)), - put(ibrowse_trace_token, ["LB: ", Host, $:, integer_to_list(Port)]), - Tid = ets:new(ibrowse_lb, [public, ordered_set]), - {ok, #state{parent_pid = whereis(ibrowse), - host = Host, - port = Port, - ets_tid = Tid, - max_pipeline_size = Max_pipe_sz, - max_sessions = Max_sessions}}. - -spawn_connection(Lb_pid, Url, - Max_sessions, - Max_pipeline_size, - SSL_options) - when is_pid(Lb_pid), - is_record(Url, url), - is_integer(Max_pipeline_size), - is_integer(Max_sessions) -> - gen_server:call(Lb_pid, - {spawn_connection, Url, Max_sessions, Max_pipeline_size, SSL_options}). - -stop(Lb_pid) -> - case catch gen_server:call(Lb_pid, stop) of - {'EXIT', {timeout, _}} -> - exit(Lb_pid, kill); - ok -> - ok - end. -%%-------------------------------------------------------------------- -%% Function: handle_call/3 -%% Description: Handling call messages -%% Returns: {reply, Reply, State} | -%% {reply, Reply, State, Timeout} | -%% {noreply, State} | -%% {noreply, State, Timeout} | -%% {stop, Reason, Reply, State} | (terminate/2 is called) -%% {stop, Reason, State} (terminate/2 is called) -%%-------------------------------------------------------------------- - -handle_call(stop, _From, #state{ets_tid = undefined} = State) -> - gen_server:reply(_From, ok), - {stop, normal, State}; - -handle_call(stop, _From, #state{ets_tid = Tid} = State) -> - ets:foldl(fun({Pid, _, _}, Acc) -> - ibrowse_http_client:stop(Pid), - Acc - end, [], Tid), - gen_server:reply(_From, ok), - {stop, normal, State}; - -handle_call(_, _From, #state{proc_state = shutting_down} = State) -> - {reply, {error, shutting_down}, State}; - -%% Update max_sessions in #state with supplied value -handle_call({spawn_connection, _Url, Max_sess, Max_pipe, _}, _From, - #state{num_cur_sessions = Num} = State) - when Num >= Max_sess -> - State_1 = maybe_create_ets(State), - Reply = find_best_connection(State_1#state.ets_tid, Max_pipe), - {reply, Reply, State_1#state{max_sessions = Max_sess, - max_pipeline_size = Max_pipe}}; - -handle_call({spawn_connection, Url, Max_sess, Max_pipe, SSL_options}, _From, - #state{num_cur_sessions = Cur} = State) -> - State_1 = maybe_create_ets(State), - Tid = State_1#state.ets_tid, - {ok, Pid} = ibrowse_http_client:start_link({Tid, Url, SSL_options}), - ets:insert(Tid, {Pid, 0, 0}), - {reply, {ok, Pid}, State_1#state{num_cur_sessions = Cur + 1, - max_sessions = Max_sess, - max_pipeline_size = Max_pipe}}; - -handle_call(Request, _From, State) -> - Reply = {unknown_request, Request}, - {reply, Reply, State}. - -%%-------------------------------------------------------------------- -%% Function: handle_cast/2 -%% Description: Handling cast messages -%% Returns: {noreply, State} | -%% {noreply, State, Timeout} | -%% {stop, Reason, State} (terminate/2 is called) -%%-------------------------------------------------------------------- -handle_cast(_Msg, State) -> - {noreply, State}. - -%%-------------------------------------------------------------------- -%% Function: handle_info/2 -%% Description: Handling all non call/cast messages -%% Returns: {noreply, State} | -%% {noreply, State, Timeout} | -%% {stop, Reason, State} (terminate/2 is called) -%%-------------------------------------------------------------------- -handle_info({'EXIT', Parent, _Reason}, #state{parent_pid = Parent} = State) -> - {stop, normal, State}; - -handle_info({'EXIT', _Pid, _Reason}, #state{ets_tid = undefined} = State) -> - {noreply, State}; - -handle_info({'EXIT', Pid, _Reason}, - #state{num_cur_sessions = Cur, - ets_tid = Tid} = State) -> - ets:match_delete(Tid, {{'_', Pid}, '_'}), - Cur_1 = Cur - 1, - case Cur_1 of - 0 -> - ets:delete(Tid), - {noreply, State#state{ets_tid = undefined, num_cur_sessions = 0}, 10000}; - _ -> - {noreply, State#state{num_cur_sessions = Cur_1}} - end; - -handle_info({trace, Bool}, #state{ets_tid = undefined} = State) -> - put(my_trace_flag, Bool), - {noreply, State}; - -handle_info({trace, Bool}, #state{ets_tid = Tid} = State) -> - ets:foldl(fun({{_, Pid}, _}, Acc) when is_pid(Pid) -> - catch Pid ! {trace, Bool}, - Acc; - (_, Acc) -> - Acc - end, undefined, Tid), - put(my_trace_flag, Bool), - {noreply, State}; - -handle_info(timeout, State) -> - %% We can't shutdown the process immediately because a request - %% might be in flight. So we first remove the entry from the - %% ibrowse_lb ets table, and then shutdown a couple of seconds - %% later - ets:delete(ibrowse_lb, {State#state.host, State#state.port}), - erlang:send_after(2000, self(), shutdown), - {noreply, State#state{proc_state = shutting_down}}; - -handle_info(shutdown, State) -> - {stop, normal, State}; - -handle_info(_Info, State) -> - {noreply, State}. - -%%-------------------------------------------------------------------- -%% Function: terminate/2 -%% Description: Shutdown the server -%% Returns: any (ignored by gen_server) -%%-------------------------------------------------------------------- -terminate(_Reason, _State) -> - ok. - -%%-------------------------------------------------------------------- -%% Func: code_change/3 -%% Purpose: Convert process state when code is changed -%% Returns: {ok, NewState} -%%-------------------------------------------------------------------- -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -%%-------------------------------------------------------------------- -%%% Internal functions -%%-------------------------------------------------------------------- -find_best_connection(Tid, Max_pipe) -> - Res = find_best_connection(ets:first(Tid), Tid, Max_pipe), - Res. - -find_best_connection('$end_of_table', _, _) -> - {error, retry_later}; -find_best_connection(Pid, Tid, Max_pipe) -> - case ets:lookup(Tid, Pid) of - [{Pid, Cur_sz, Speculative_sz}] when Cur_sz < Max_pipe, - Speculative_sz < Max_pipe -> - ets:update_counter(Tid, Pid, {3, 1, 9999999, 9999999}), - {ok, Pid}; - _ -> - find_best_connection(ets:next(Tid, Pid), Tid, Max_pipe) - end. - -maybe_create_ets(#state{ets_tid = undefined} = State) -> - Tid = ets:new(ibrowse_lb, [public, ordered_set]), - State#state{ets_tid = Tid}; -maybe_create_ets(State) -> - State.
http://git-wip-us.apache.org/repos/asf/couchdb-ibrowse/blob/4af2d408/ibrowse_lib.erl ---------------------------------------------------------------------- diff --git a/ibrowse_lib.erl b/ibrowse_lib.erl deleted file mode 100644 index 7b12cb3..0000000 --- a/ibrowse_lib.erl +++ /dev/null @@ -1,442 +0,0 @@ -%%% File : ibrowse_lib.erl -%%% Author : Chandrashekhar Mullaparthi <[email protected]> -%%% Description : -%%% Created : 27 Feb 2004 by Chandrashekhar Mullaparthi <[email protected]> -%% @doc Module with a few useful functions - --module(ibrowse_lib). --author('chandru'). --ifdef(debug). --compile(export_all). --endif. - --include("ibrowse.hrl"). - --ifdef(EUNIT). --include_lib("eunit/include/eunit.hrl"). --endif. - --export([ - get_trace_status/2, - do_trace/2, - do_trace/3, - url_encode/1, - decode_rfc822_date/1, - status_code/1, - encode_base64/1, - decode_base64/1, - get_value/2, - get_value/3, - parse_url/1, - printable_date/0 - ]). - -get_trace_status(Host, Port) -> - ibrowse:get_config_value({trace, Host, Port}, false). - -%% @doc URL-encodes a string based on RFC 1738. Returns a flat list. -%% @spec url_encode(Str) -> UrlEncodedStr -%% Str = string() -%% UrlEncodedStr = string() -url_encode(Str) when is_list(Str) -> - url_encode_char(lists:reverse(Str), []). - -url_encode_char([X | T], Acc) when X >= $0, X =< $9 -> - url_encode_char(T, [X | Acc]); -url_encode_char([X | T], Acc) when X >= $a, X =< $z -> - url_encode_char(T, [X | Acc]); -url_encode_char([X | T], Acc) when X >= $A, X =< $Z -> - url_encode_char(T, [X | Acc]); -url_encode_char([X | T], Acc) when X == $-; X == $_; X == $. -> - url_encode_char(T, [X | Acc]); -url_encode_char([32 | T], Acc) -> - url_encode_char(T, [$+ | Acc]); -url_encode_char([X | T], Acc) -> - url_encode_char(T, [$%, d2h(X bsr 4), d2h(X band 16#0f) | Acc]); -url_encode_char([], Acc) -> - Acc. - -d2h(N) when N<10 -> N+$0; -d2h(N) -> N+$a-10. - -decode_rfc822_date(String) when is_list(String) -> - case catch decode_rfc822_date_1(string:tokens(String, ", \t\r\n")) of - {'EXIT', _} -> - {error, invalid_date}; - Res -> - Res - end. - -% TODO: Have to handle the Zone -decode_rfc822_date_1([_,DayInt,Month,Year, Time,Zone]) -> - decode_rfc822_date_1([DayInt,Month,Year, Time,Zone]); -decode_rfc822_date_1([Day,Month,Year, Time,_Zone]) -> - DayI = list_to_integer(Day), - MonthI = month_int(Month), - YearI = list_to_integer(Year), - TimeTup = case string:tokens(Time, ":") of - [H,M] -> - {list_to_integer(H), - list_to_integer(M), - 0}; - [H,M,S] -> - {list_to_integer(H), - list_to_integer(M), - list_to_integer(S)} - end, - {{YearI,MonthI,DayI}, TimeTup}. - -month_int("Jan") -> 1; -month_int("Feb") -> 2; -month_int("Mar") -> 3; -month_int("Apr") -> 4; -month_int("May") -> 5; -month_int("Jun") -> 6; -month_int("Jul") -> 7; -month_int("Aug") -> 8; -month_int("Sep") -> 9; -month_int("Oct") -> 10; -month_int("Nov") -> 11; -month_int("Dec") -> 12. - -%% @doc Given a status code, returns an atom describing the status code. -%% @spec status_code(StatusCode::status_code()) -> StatusDescription -%% status_code() = string() | integer() -%% StatusDescription = atom() -status_code(100) -> continue; -status_code(101) -> switching_protocols; -status_code(102) -> processing; -status_code(200) -> ok; -status_code(201) -> created; -status_code(202) -> accepted; -status_code(203) -> non_authoritative_information; -status_code(204) -> no_content; -status_code(205) -> reset_content; -status_code(206) -> partial_content; -status_code(207) -> multi_status; -status_code(300) -> multiple_choices; -status_code(301) -> moved_permanently; -status_code(302) -> found; -status_code(303) -> see_other; -status_code(304) -> not_modified; -status_code(305) -> use_proxy; -status_code(306) -> unused; -status_code(307) -> temporary_redirect; -status_code(400) -> bad_request; -status_code(401) -> unauthorized; -status_code(402) -> payment_required; -status_code(403) -> forbidden; -status_code(404) -> not_found; -status_code(405) -> method_not_allowed; -status_code(406) -> not_acceptable; -status_code(407) -> proxy_authentication_required; -status_code(408) -> request_timeout; -status_code(409) -> conflict; -status_code(410) -> gone; -status_code(411) -> length_required; -status_code(412) -> precondition_failed; -status_code(413) -> request_entity_too_large; -status_code(414) -> request_uri_too_long; -status_code(415) -> unsupported_media_type; -status_code(416) -> requested_range_not_satisfiable; -status_code(417) -> expectation_failed; -status_code(422) -> unprocessable_entity; -status_code(423) -> locked; -status_code(424) -> failed_dependency; -status_code(500) -> internal_server_error; -status_code(501) -> not_implemented; -status_code(502) -> bad_gateway; -status_code(503) -> service_unavailable; -status_code(504) -> gateway_timeout; -status_code(505) -> http_version_not_supported; -status_code(507) -> insufficient_storage; -status_code(X) when is_list(X) -> status_code(list_to_integer(X)); -status_code(_) -> unknown_status_code. - -%% @doc Implements the base64 encoding algorithm. The output data type matches in the input data type. -%% @spec encode_base64(In) -> Out -%% In = string() | binary() -%% Out = string() | binary() -encode_base64(List) when is_list(List) -> - binary_to_list(base64:encode(List)); -encode_base64(Bin) when is_binary(Bin) -> - base64:encode(Bin). - -%% @doc Implements the base64 decoding algorithm. The output data type matches in the input data type. -%% @spec decode_base64(In) -> Out | exit({error, invalid_input}) -%% In = string() | binary() -%% Out = string() | binary() -decode_base64(List) when is_list(List) -> - binary_to_list(base64:decode(List)); -decode_base64(Bin) when is_binary(Bin) -> - base64:decode(Bin). - -get_value(Tag, TVL, DefVal) -> - case lists:keysearch(Tag, 1, TVL) of - false -> - DefVal; - {value, {_, Val}} -> - Val - end. - -get_value(Tag, TVL) -> - {value, {_, V}} = lists:keysearch(Tag,1,TVL), - V. - -parse_url(Url) -> - try - case parse_url(Url, get_protocol, #url{abspath=Url}, []) of - #url{host_type = undefined, host = Host} = UrlRec -> - case inet_parse:address(Host) of - {ok, {_, _, _, _, _, _, _, _}} -> - UrlRec#url{host_type = ipv6_address}; - {ok, {_, _, _, _}} -> - UrlRec#url{host_type = ipv4_address}; - _ -> - UrlRec#url{host_type = hostname} - end; - #url{} = UrlRec -> - UrlRec; - _ -> - {error, invalid_uri} - end - catch _:_ -> - {error, invalid_uri} - end. - -parse_url([$:, $/, $/ | _], get_protocol, Url, []) -> - {invalid_uri_1, Url}; -parse_url([$:, $/, $/ | T], get_protocol, Url, TmpAcc) -> - Prot = list_to_existing_atom(lists:reverse(TmpAcc)), - parse_url(T, get_username, - Url#url{protocol = Prot}, - []); -parse_url([H | T], get_username, Url, TmpAcc) when H == $/; - H == $? -> - Path = case H of - $/ -> - [$/ | T]; - $? -> - [$/, $? | T] - end, - %% No username/password. No port number - Url#url{host = lists:reverse(TmpAcc), - port = default_port(Url#url.protocol), - path = Path}; -parse_url([$: | T], get_username, Url, TmpAcc) -> - %% It is possible that no username/password has been - %% specified. But we'll continue with the assumption that there is - %% a username/password. If we encounter a '@' later on, there is a - %% username/password indeed. If we encounter a '/', it was - %% actually the hostname - parse_url(T, get_password, - Url#url{username = lists:reverse(TmpAcc)}, - []); -parse_url([$@ | T], get_username, Url, TmpAcc) -> - parse_url(T, get_host, - Url#url{username = lists:reverse(TmpAcc), - password = ""}, - []); -parse_url([$[ | T], get_username, Url, []) -> - % IPv6 address literals are enclosed by square brackets: - % http://www.ietf.org/rfc/rfc2732.txt - parse_url(T, get_ipv6_address, Url#url{host_type = ipv6_address}, []); -parse_url([$[ | T], get_username, _Url, TmpAcc) -> - {error, {invalid_username_or_host, lists:reverse(TmpAcc) ++ "[" ++ T}}; -parse_url([$[ | _], get_password, _Url, []) -> - {error, missing_password}; -parse_url([$[ | T], get_password, Url, TmpAcc) -> - % IPv6 address literals are enclosed by square brackets: - % http://www.ietf.org/rfc/rfc2732.txt - parse_url(T, get_ipv6_address, - Url#url{host_type = ipv6_address, - password = lists:reverse(TmpAcc)}, - []); -parse_url([$@ | T], get_password, Url, TmpAcc) -> - parse_url(T, get_host, - Url#url{password = lists:reverse(TmpAcc)}, - []); -parse_url([H | T], get_password, Url, TmpAcc) when H == $/; - H == $? -> - %% Ok, what we thought was the username/password was the hostname - %% and portnumber - #url{username=User} = Url, - Port = list_to_integer(lists:reverse(TmpAcc)), - Path = case H of - $/ -> - [$/ | T]; - $? -> - [$/, $? | T] - end, - Url#url{host = User, - port = Port, - username = undefined, - password = undefined, - path = Path}; -parse_url([$] | T], get_ipv6_address, #url{protocol = Prot} = Url, TmpAcc) -> - Addr = lists:reverse(TmpAcc), - case inet_parse:address(Addr) of - {ok, {_, _, _, _, _, _, _, _}} -> - Url2 = Url#url{host = Addr, port = default_port(Prot)}, - case T of - [$: | T2] -> - parse_url(T2, get_port, Url2, []); - [$/ | T2] -> - Url2#url{path = [$/ | T2]}; - [$? | T2] -> - Url2#url{path = [$/, $? | T2]}; - [] -> - Url2#url{path = "/"}; - _ -> - {error, {invalid_host, "[" ++ Addr ++ "]" ++ T}} - end; - _ -> - {error, {invalid_ipv6_address, Addr}} - end; -parse_url([$[ | T], get_host, #url{} = Url, []) -> - parse_url(T, get_ipv6_address, Url#url{host_type = ipv6_address}, []); -parse_url([$: | T], get_host, #url{} = Url, TmpAcc) -> - parse_url(T, get_port, - Url#url{host = lists:reverse(TmpAcc)}, - []); -parse_url([H | T], get_host, #url{protocol=Prot} = Url, TmpAcc) when H == $/; - H == $? -> - Path = case H of - $/ -> - [$/ | T]; - $? -> - [$/, $? | T] - end, - Url#url{host = lists:reverse(TmpAcc), - port = default_port(Prot), - path = Path}; -parse_url([H | T], get_port, #url{protocol=Prot} = Url, TmpAcc) when H == $/; - H == $? -> - Path = case H of - $/ -> - [$/ | T]; - $? -> - [$/, $? | T] - end, - Port = case TmpAcc of - [] -> - default_port(Prot); - _ -> - list_to_integer(lists:reverse(TmpAcc)) - end, - Url#url{port = Port, path = Path}; -parse_url([H | T], State, Url, TmpAcc) -> - parse_url(T, State, Url, [H | TmpAcc]); -parse_url([], get_host, Url, TmpAcc) when TmpAcc /= [] -> - Url#url{host = lists:reverse(TmpAcc), - port = default_port(Url#url.protocol), - path = "/"}; -parse_url([], get_username, Url, TmpAcc) when TmpAcc /= [] -> - Url#url{host = lists:reverse(TmpAcc), - port = default_port(Url#url.protocol), - path = "/"}; -parse_url([], get_port, #url{protocol=Prot} = Url, TmpAcc) -> - Port = case TmpAcc of - [] -> - default_port(Prot); - _ -> - list_to_integer(lists:reverse(TmpAcc)) - end, - Url#url{port = Port, - path = "/"}; -parse_url([], get_password, Url, TmpAcc) -> - %% Ok, what we thought was the username/password was the hostname - %% and portnumber - #url{username=User} = Url, - Port = case TmpAcc of - [] -> - default_port(Url#url.protocol); - _ -> - list_to_integer(lists:reverse(TmpAcc)) - end, - Url#url{host = User, - port = Port, - username = undefined, - password = undefined, - path = "/"}; -parse_url([], State, Url, TmpAcc) -> - {invalid_uri_2, State, Url, TmpAcc}. - -default_port(socks5) -> 1080; -default_port(http) -> 80; -default_port(https) -> 443; -default_port(ftp) -> 21. - -printable_date() -> - {{Y,Mo,D},{H, M, S}} = calendar:local_time(), - {_,_,MicroSecs} = now(), - [integer_to_list(Y), - $-, - integer_to_list(Mo), - $-, - integer_to_list(D), - $_, - integer_to_list(H), - $:, - integer_to_list(M), - $:, - integer_to_list(S), - $:, - integer_to_list(MicroSecs div 1000)]. - -do_trace(Fmt, Args) -> - do_trace(get(my_trace_flag), Fmt, Args). - --ifdef(DEBUG). -do_trace(_, Fmt, Args) -> - io:format("~s -- (~s) - "++Fmt, - [printable_date(), - get(ibrowse_trace_token) | Args]). --else. -do_trace(true, Fmt, Args) -> - io:format("~s -- (~s) - "++Fmt, - [printable_date(), - get(ibrowse_trace_token) | Args]); -do_trace(_, _, _) -> - ok. --endif. - --ifdef(EUNIT). - -parse_url_test() -> - Urls = [{"http://[FEDC:BA98:7654:3210:FEDC:BA98:7654:3210]:80/index.html";, - #url{abspath = "http://[FEDC:BA98:7654:3210:FEDC:BA98:7654:3210]:80/index.html";, - host = "FEDC:BA98:7654:3210:FEDC:BA98:7654:3210", - port = 80, protocol = http, path = "/index.html", - host_type = ipv6_address}}, - {"http://[1080:0:0:0:8:800:200C:417A]/index.html";, - #url{abspath = "http://[1080:0:0:0:8:800:200C:417A]/index.html";, - host_type = ipv6_address, port = 80, protocol = http, - host = "1080:0:0:0:8:800:200C:417A", path = "/index.html"}}, - {"http://[3ffe:2a00:100:7031::1]";, - #url{abspath = "http://[3ffe:2a00:100:7031::1]";, - host_type = ipv6_address, port = 80, protocol = http, - host = "3ffe:2a00:100:7031::1", path = "/"}}, - {"http://[1080::8:800:200C:417A]/foo";, - #url{abspath = "http://[1080::8:800:200C:417A]/foo";, - host_type = ipv6_address, port = 80, protocol = http, - host = "1080::8:800:200C:417A", path = "/foo"}}, - {"http://[::192.9.5.5]/ipng";, - #url{abspath = "http://[::192.9.5.5]/ipng";, - host_type = ipv6_address, port = 80, protocol = http, - host = "::192.9.5.5", path = "/ipng"}}, - {"http://[::FFFF:129.144.52.38]:80/index.html";, - #url{abspath = "http://[::FFFF:129.144.52.38]:80/index.html";, - host_type = ipv6_address, port = 80, protocol = http, - host = "::FFFF:129.144.52.38", path = "/index.html"}}, - {"http://[2010:836B:4179::836B:4179]";, - #url{abspath = "http://[2010:836B:4179::836B:4179]";, - host_type = ipv6_address, port = 80, protocol = http, - host = "2010:836B:4179::836B:4179", path = "/"}} - ], - lists:foreach( - fun({Url, Expected_result}) -> - ?assertMatch(Expected_result, parse_url(Url)) - end, Urls). - --endif. http://git-wip-us.apache.org/repos/asf/couchdb-ibrowse/blob/4af2d408/ibrowse_socks5.erl ---------------------------------------------------------------------- diff --git a/ibrowse_socks5.erl b/ibrowse_socks5.erl deleted file mode 100644 index d00df44..0000000 --- a/ibrowse_socks5.erl +++ /dev/null @@ -1,109 +0,0 @@ -% Licensed under the Apache License, Version 2.0 (the "License"); you may not -% use this file except in compliance with the License. You may obtain a copy of -% the License at -% -% http://www.apache.org/licenses/LICENSE-2.0 -% -% Unless required by applicable law or agreed to in writing, software -% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -% License for the specific language governing permissions and limitations under -% the License. - --module(ibrowse_socks5). - --define(VERSION, 5). --define(CONNECT, 1). - --define(NO_AUTH, 0). --define(USERPASS, 2). --define(UNACCEPTABLE, 16#FF). --define(RESERVED, 0). - --define(ATYP_IPV4, 1). --define(ATYP_DOMAINNAME, 3). --define(ATYP_IPV6, 4). - --define(SUCCEEDED, 0). - --export([connect/5]). - --import(ibrowse_lib, [get_value/2, get_value/3]). - -connect(TargetHost, TargetPort, ProxyOptions, Options, Timeout) -> - case gen_tcp:connect(get_value(host, ProxyOptions), - get_value(port, ProxyOptions), - Options, Timeout) of - {ok, Socket} -> - case handshake(Socket, Options) of - ok -> - case connect(TargetHost, TargetPort, Socket) of - ok -> - maybe_ssl(Socket, ProxyOptions, Timeout); - Else -> - gen_tcp:close(Socket), - Else - end; - Else -> - gen_tcp:close(Socket), - Else - end; - Else -> - Else - end. - -handshake(Socket, ProxyOptions) when is_port(Socket) -> - {Handshake, Success} = case get_value(user, ProxyOptions, <<>>) of - <<>> -> - {<<?VERSION, 1, ?NO_AUTH>>, ?NO_AUTH}; - User -> - Password = get_value(password, ProxyOptions, <<>>), - {<<?VERSION, 1, ?USERPASS, (byte_size(User)), User, - (byte_size(Password)), Password>>, ?USERPASS} - end, - ok = gen_tcp:send(Socket, Handshake), - case gen_tcp:recv(Socket, 0) of - {ok, <<?VERSION, Success>>} -> - ok; - {ok, <<?VERSION, ?UNACCEPTABLE>>} -> - {error, unacceptable}; - {error, Reason} -> - {error, Reason} - end. - -connect(Host, Port, Via) when is_list(Host) -> - connect(list_to_binary(Host), Port, Via); -connect(Host, Port, Via) when is_binary(Host), is_integer(Port), - is_port(Via) -> - ok = gen_tcp:send(Via, - <<?VERSION, ?CONNECT, ?RESERVED, ?ATYP_DOMAINNAME, - (byte_size(Host)), Host/binary, - (Port):16>>), - case gen_tcp:recv(Via, 0) of - {ok, <<?VERSION, ?SUCCEEDED, ?RESERVED, _/binary>>} -> - ok; - {ok, <<?VERSION, Rep, ?RESERVED, _/binary>>} -> - {error, rep(Rep)}; - {error, Reason} -> - {error, Reason} - end. - -maybe_ssl(Socket, ProxyOptions, Timeout) -> - IsSsl = get_value(is_ssl, ProxyOptions, false), - SslOpts = get_value(ssl_opts, ProxyOptions, []), - case IsSsl of - false -> - {ok, Socket}; - true -> - ssl:connect(Socket, SslOpts, Timeout) - end. - -rep(0) -> succeeded; -rep(1) -> server_fail; -rep(2) -> disallowed_by_ruleset; -rep(3) -> network_unreachable; -rep(4) -> host_unreachable; -rep(5) -> connection_refused; -rep(6) -> ttl_expired; -rep(7) -> command_not_supported; -rep(8) -> address_type_not_supported. http://git-wip-us.apache.org/repos/asf/couchdb-ibrowse/blob/4af2d408/ibrowse_sup.erl ---------------------------------------------------------------------- diff --git a/ibrowse_sup.erl b/ibrowse_sup.erl deleted file mode 100644 index ace33d1..0000000 --- a/ibrowse_sup.erl +++ /dev/null @@ -1,63 +0,0 @@ -%%%------------------------------------------------------------------- -%%% File : ibrowse_sup.erl -%%% Author : Chandrashekhar Mullaparthi <[email protected]> -%%% Description : -%%% -%%% Created : 15 Oct 2003 by Chandrashekhar Mullaparthi <[email protected]> -%%%------------------------------------------------------------------- --module(ibrowse_sup). --behaviour(supervisor). -%%-------------------------------------------------------------------- -%% Include files -%%-------------------------------------------------------------------- - -%%-------------------------------------------------------------------- -%% External exports -%%-------------------------------------------------------------------- --export([ - start_link/0 - ]). - -%%-------------------------------------------------------------------- -%% Internal exports -%%-------------------------------------------------------------------- --export([ - init/1 - ]). - -%%-------------------------------------------------------------------- -%% Macros -%%-------------------------------------------------------------------- --define(SERVER, ?MODULE). - -%%-------------------------------------------------------------------- -%% Records -%%-------------------------------------------------------------------- - -%%==================================================================== -%% External functions -%%==================================================================== -%%-------------------------------------------------------------------- -%% Function: start_link/0 -%% Description: Starts the supervisor -%%-------------------------------------------------------------------- -start_link() -> - supervisor:start_link({local, ?SERVER}, ?MODULE, []). - -%%==================================================================== -%% Server functions -%%==================================================================== -%%-------------------------------------------------------------------- -%% Func: init/1 -%% Returns: {ok, {SupFlags, [ChildSpec]}} | -%% ignore | -%% {error, Reason} -%%-------------------------------------------------------------------- -init([]) -> - AChild = {ibrowse,{ibrowse,start_link,[]}, - permanent,2000,worker,[ibrowse, ibrowse_http_client]}, - {ok,{{one_for_all,10,1}, [AChild]}}. - -%%==================================================================== -%% Internal functions -%%==================================================================== http://git-wip-us.apache.org/repos/asf/couchdb-ibrowse/blob/4af2d408/ibrowse_test.erl ---------------------------------------------------------------------- diff --git a/ibrowse_test.erl b/ibrowse_test.erl deleted file mode 100644 index d97f76c..0000000 --- a/ibrowse_test.erl +++ /dev/null @@ -1,625 +0,0 @@ -%%% File : ibrowse_test.erl -%%% Author : Chandrashekhar Mullaparthi <[email protected]> -%%% Description : Test ibrowse -%%% Created : 14 Oct 2003 by Chandrashekhar Mullaparthi <[email protected]> - --module(ibrowse_test). --export([ - load_test/3, - send_reqs_1/3, - do_send_req/2, - unit_tests/0, - unit_tests/1, - unit_tests_1/2, - ue_test/0, - ue_test/1, - verify_chunked_streaming/0, - verify_chunked_streaming/1, - test_chunked_streaming_once/0, - i_do_async_req_list/4, - test_stream_once/3, - test_stream_once/4, - test_20122010/0, - test_20122010/1, - test_pipeline_head_timeout/0, - test_pipeline_head_timeout/1, - do_test_pipeline_head_timeout/4, - test_head_transfer_encoding/0, - test_head_transfer_encoding/1, - test_head_response_with_body/0, - test_head_response_with_body/1 - ]). - -test_stream_once(Url, Method, Options) -> - test_stream_once(Url, Method, Options, 5000). - -test_stream_once(Url, Method, Options, Timeout) -> - case ibrowse:send_req(Url, [], Method, [], [{stream_to, {self(), once}} | Options], Timeout) of - {ibrowse_req_id, Req_id} -> - case ibrowse:stream_next(Req_id) of - ok -> - test_stream_once(Req_id); - Err -> - Err - end; - Err -> - Err - end. - -test_stream_once(Req_id) -> - receive - {ibrowse_async_headers, Req_id, StatCode, Headers} -> - io:format("Recvd headers~n~p~n", [{ibrowse_async_headers, Req_id, StatCode, Headers}]), - case ibrowse:stream_next(Req_id) of - ok -> - test_stream_once(Req_id); - Err -> - Err - end; - {ibrowse_async_response, Req_id, {error, Err}} -> - io:format("Recvd error: ~p~n", [Err]); - {ibrowse_async_response, Req_id, Body_1} -> - io:format("Recvd body part: ~n~p~n", [{ibrowse_async_response, Req_id, Body_1}]), - case ibrowse:stream_next(Req_id) of - ok -> - test_stream_once(Req_id); - Err -> - Err - end; - {ibrowse_async_response_end, Req_id} -> - ok - end. -%% Use ibrowse:set_max_sessions/3 and ibrowse:set_max_pipeline_size/3 to -%% tweak settings before running the load test. The defaults are 10 and 10. -load_test(Url, NumWorkers, NumReqsPerWorker) when is_list(Url), - is_integer(NumWorkers), - is_integer(NumReqsPerWorker), - NumWorkers > 0, - NumReqsPerWorker > 0 -> - proc_lib:spawn(?MODULE, send_reqs_1, [Url, NumWorkers, NumReqsPerWorker]). - -send_reqs_1(Url, NumWorkers, NumReqsPerWorker) -> - Start_time = now(), - ets:new(pid_table, [named_table, public]), - ets:new(ibrowse_test_results, [named_table, public]), - ets:new(ibrowse_errors, [named_table, public, ordered_set]), - init_results(), - process_flag(trap_exit, true), - log_msg("Starting spawning of workers...~n", []), - spawn_workers(Url, NumWorkers, NumReqsPerWorker), - log_msg("Finished spawning workers...~n", []), - do_wait(Url), - End_time = now(), - log_msg("All workers are done...~n", []), - log_msg("ibrowse_test_results table: ~n~p~n", [ets:tab2list(ibrowse_test_results)]), - log_msg("Start time: ~1000.p~n", [calendar:now_to_local_time(Start_time)]), - log_msg("End time : ~1000.p~n", [calendar:now_to_local_time(End_time)]), - Elapsed_time_secs = trunc(timer:now_diff(End_time, Start_time) / 1000000), - log_msg("Elapsed : ~p~n", [Elapsed_time_secs]), - log_msg("Reqs/sec : ~p~n", [round(trunc((NumWorkers*NumReqsPerWorker) / Elapsed_time_secs))]), - dump_errors(). - -init_results() -> - ets:insert(ibrowse_test_results, {crash, 0}), - ets:insert(ibrowse_test_results, {send_failed, 0}), - ets:insert(ibrowse_test_results, {other_error, 0}), - ets:insert(ibrowse_test_results, {success, 0}), - ets:insert(ibrowse_test_results, {retry_later, 0}), - ets:insert(ibrowse_test_results, {trid_mismatch, 0}), - ets:insert(ibrowse_test_results, {success_no_trid, 0}), - ets:insert(ibrowse_test_results, {failed, 0}), - ets:insert(ibrowse_test_results, {timeout, 0}), - ets:insert(ibrowse_test_results, {req_id, 0}). - -spawn_workers(_Url, 0, _) -> - ok; -spawn_workers(Url, NumWorkers, NumReqsPerWorker) -> - Pid = proc_lib:spawn_link(?MODULE, do_send_req, [Url, NumReqsPerWorker]), - ets:insert(pid_table, {Pid, []}), - spawn_workers(Url, NumWorkers - 1, NumReqsPerWorker). - -do_wait(Url) -> - receive - {'EXIT', _, normal} -> - catch ibrowse:show_dest_status(Url), - catch ibrowse:show_dest_status(), - do_wait(Url); - {'EXIT', Pid, Reason} -> - ets:delete(pid_table, Pid), - ets:insert(ibrowse_errors, {Pid, Reason}), - ets:update_counter(ibrowse_test_results, crash, 1), - do_wait(Url); - Msg -> - io:format("Recvd unknown message...~p~n", [Msg]), - do_wait(Url) - after 1000 -> - case ets:info(pid_table, size) of - 0 -> - done; - _ -> - catch ibrowse:show_dest_status(Url), - catch ibrowse:show_dest_status(), - do_wait(Url) - end - end. - -do_send_req(Url, NumReqs) -> - do_send_req_1(Url, NumReqs). - -do_send_req_1(_Url, 0) -> - ets:delete(pid_table, self()); -do_send_req_1(Url, NumReqs) -> - Counter = integer_to_list(ets:update_counter(ibrowse_test_results, req_id, 1)), - case ibrowse:send_req(Url, [{"ib_req_id", Counter}], get, [], [], 10000) of - {ok, _Status, Headers, _Body} -> - case lists:keysearch("ib_req_id", 1, Headers) of - {value, {_, Counter}} -> - ets:update_counter(ibrowse_test_results, success, 1); - {value, _} -> - ets:update_counter(ibrowse_test_results, trid_mismatch, 1); - false -> - ets:update_counter(ibrowse_test_results, success_no_trid, 1) - end; - {error, req_timedout} -> - ets:update_counter(ibrowse_test_results, timeout, 1); - {error, send_failed} -> - ets:update_counter(ibrowse_test_results, send_failed, 1); - {error, retry_later} -> - ets:update_counter(ibrowse_test_results, retry_later, 1); - Err -> - ets:insert(ibrowse_errors, {now(), Err}), - ets:update_counter(ibrowse_test_results, other_error, 1), - ok - end, - do_send_req_1(Url, NumReqs-1). - -dump_errors() -> - case ets:info(ibrowse_errors, size) of - 0 -> - ok; - _ -> - {A, B, C} = now(), - Filename = lists:flatten( - io_lib:format("ibrowse_errors_~p_~p_~p.txt" , [A, B, C])), - case file:open(Filename, [write, delayed_write, raw]) of - {ok, Iod} -> - dump_errors(ets:first(ibrowse_errors), Iod); - Err -> - io:format("failed to create file ~s. Reason: ~p~n", [Filename, Err]), - ok - end - end. - -dump_errors('$end_of_table', Iod) -> - file:close(Iod); -dump_errors(Key, Iod) -> - [{_, Term}] = ets:lookup(ibrowse_errors, Key), - file:write(Iod, io_lib:format("~p~n", [Term])), - dump_errors(ets:next(ibrowse_errors, Key), Iod). - -%%------------------------------------------------------------------------------ -%% Unit Tests -%%------------------------------------------------------------------------------ --define(TEST_LIST, [{"http://intranet/messenger";, get}, - {"http://www.google.co.uk";, get}, - {"http://www.google.com";, get}, - {"http://www.google.com";, options}, - {"https://mail.google.com";, get}, - {"http://www.sun.com";, get}, - {"http://www.oracle.com";, get}, - {"http://www.bbc.co.uk";, get}, - {"http://www.bbc.co.uk";, trace}, - {"http://www.bbc.co.uk";, options}, - {"http://yaws.hyber.org";, get}, - {"http://jigsaw.w3.org/HTTP/ChunkedScript";, get}, - {"http://jigsaw.w3.org/HTTP/TE/foo.txt";, get}, - {"http://jigsaw.w3.org/HTTP/TE/bar.txt";, get}, - {"http://jigsaw.w3.org/HTTP/connection.html";, get}, - {"http://jigsaw.w3.org/HTTP/cc.html";, get}, - {"http://jigsaw.w3.org/HTTP/cc-private.html";, get}, - {"http://jigsaw.w3.org/HTTP/cc-proxy-revalidate.html";, get}, - {"http://jigsaw.w3.org/HTTP/cc-nocache.html";, get}, - {"http://jigsaw.w3.org/HTTP/h-content-md5.html";, get}, - {"http://jigsaw.w3.org/HTTP/h-retry-after.html";, get}, - {"http://jigsaw.w3.org/HTTP/h-retry-after-date.html";, get}, - {"http://jigsaw.w3.org/HTTP/neg";, get}, - {"http://jigsaw.w3.org/HTTP/negbad";, get}, - {"http://jigsaw.w3.org/HTTP/400/toolong/";, get}, - {"http://jigsaw.w3.org/HTTP/300/";, get}, - {"http://jigsaw.w3.org/HTTP/Basic/";, get, [{basic_auth, {"guest", "guest"}}]}, - {"http://jigsaw.w3.org/HTTP/CL/";, get}, - {"http://www.httpwatch.com/httpgallery/chunked/";, get}, - {"https://github.com";, get, [{ssl_options, [{depth, 2}]}]}, - {local_test_fun, test_20122010, []}, - {local_test_fun, test_pipeline_head_timeout, []}, - {local_test_fun, test_head_transfer_encoding, []}, - {local_test_fun, test_head_response_with_body, []} - ]). - -unit_tests() -> - unit_tests([]). - -unit_tests(Options) -> - application:start(crypto), - application:start(public_key), - application:start(ssl), - (catch ibrowse_test_server:start_server(8181, tcp)), - ibrowse:start(), - Options_1 = Options ++ [{connect_timeout, 5000}], - Test_timeout = proplists:get_value(test_timeout, Options, 60000), - {Pid, Ref} = erlang:spawn_monitor(?MODULE, unit_tests_1, [self(), Options_1]), - receive - {done, Pid} -> - ok; - {'DOWN', Ref, _, _, Info} -> - io:format("Test process crashed: ~p~n", [Info]) - after Test_timeout -> - exit(Pid, kill), - io:format("Timed out waiting for tests to complete~n", []) - end, - catch ibrowse_test_server:stop_server(8181), - ok. - -unit_tests_1(Parent, Options) -> - lists:foreach(fun({local_test_fun, Fun_name, Args}) -> - execute_req(local_test_fun, Fun_name, Args); - ({Url, Method}) -> - execute_req(Url, Method, Options); - ({Url, Method, X_Opts}) -> - execute_req(Url, Method, X_Opts ++ Options) - end, ?TEST_LIST), - Parent ! {done, self()}. - -verify_chunked_streaming() -> - verify_chunked_streaming([]). - -verify_chunked_streaming(Options) -> - io:format("~nVerifying that chunked streaming is working...~n", []), - Url = "http://www.httpwatch.com/httpgallery/chunked/";, - io:format(" URL: ~s~n", [Url]), - io:format(" Fetching data without streaming...~n", []), - Result_without_streaming = ibrowse:send_req( - Url, [], get, [], - [{response_format, binary} | Options]), - io:format(" Fetching data with streaming as list...~n", []), - Async_response_list = do_async_req_list( - Url, get, [{response_format, list} | Options]), - io:format(" Fetching data with streaming as binary...~n", []), - Async_response_bin = do_async_req_list( - Url, get, [{response_format, binary} | Options]), - io:format(" Fetching data with streaming as binary, {active, once}...~n", []), - Async_response_bin_once = do_async_req_list( - Url, get, [once, {response_format, binary} | Options]), - Res1 = compare_responses(Result_without_streaming, Async_response_list, Async_response_bin), - Res2 = compare_responses(Result_without_streaming, Async_response_list, Async_response_bin_once), - case {Res1, Res2} of - {success, success} -> - io:format(" Chunked streaming working~n", []); - _ -> - ok - end. - -test_chunked_streaming_once() -> - test_chunked_streaming_once([]). - -test_chunked_streaming_once(Options) -> - io:format("~nTesting chunked streaming with the {stream_to, {Pid, once}} option...~n", []), - Url = "http://www.httpwatch.com/httpgallery/chunked/";, - io:format(" URL: ~s~n", [Url]), - io:format(" Fetching data with streaming as binary, {active, once}...~n", []), - case do_async_req_list(Url, get, [once, {response_format, binary} | Options]) of - {ok, _, _, _} -> - io:format(" Success!~n", []); - Err -> - io:format(" Fail: ~p~n", [Err]) - end. - -compare_responses({ok, St_code, _, Body}, {ok, St_code, _, Body}, {ok, St_code, _, Body}) -> - success; -compare_responses({ok, St_code, _, Body_1}, {ok, St_code, _, Body_2}, {ok, St_code, _, Body_3}) -> - case Body_1 of - Body_2 -> - io:format("Body_1 and Body_2 match~n", []); - Body_3 -> - io:format("Body_1 and Body_3 match~n", []); - _ when Body_2 == Body_3 -> - io:format("Body_2 and Body_3 match~n", []); - _ -> - io:format("All three bodies are different!~n", []) - end, - io:format("Body_1 -> ~p~n", [Body_1]), - io:format("Body_2 -> ~p~n", [Body_2]), - io:format("Body_3 -> ~p~n", [Body_3]), - fail_bodies_mismatch; -compare_responses(R1, R2, R3) -> - io:format("R1 -> ~p~n", [R1]), - io:format("R2 -> ~p~n", [R2]), - io:format("R3 -> ~p~n", [R3]), - fail. - -%% do_async_req_list(Url) -> -%% do_async_req_list(Url, get). - -%% do_async_req_list(Url, Method) -> -%% do_async_req_list(Url, Method, [{stream_to, self()}, -%% {stream_chunk_size, 1000}]). - -do_async_req_list(Url, Method, Options) -> - {Pid,_} = erlang:spawn_monitor(?MODULE, i_do_async_req_list, - [self(), Url, Method, - Options ++ [{stream_chunk_size, 1000}]]), -%% io:format("Spawned process ~p~n", [Pid]), - wait_for_resp(Pid). - -wait_for_resp(Pid) -> - receive - {async_result, Pid, Res} -> - Res; - {async_result, Other_pid, _} -> - io:format("~p: Waiting for result from ~p: got from ~p~n", [self(), Pid, Other_pid]), - wait_for_resp(Pid); - {'DOWN', _, _, Pid, Reason} -> - {'EXIT', Reason}; - {'DOWN', _, _, _, _} -> - wait_for_resp(Pid); - Msg -> - io:format("Recvd unknown message: ~p~n", [Msg]), - wait_for_resp(Pid) - after 100000 -> - {error, timeout} - end. - -i_do_async_req_list(Parent, Url, Method, Options) -> - Options_1 = case lists:member(once, Options) of - true -> - [{stream_to, {self(), once}} | (Options -- [once])]; - false -> - [{stream_to, self()} | Options] - end, - Res = ibrowse:send_req(Url, [], Method, [], Options_1), - case Res of - {ibrowse_req_id, Req_id} -> - Result = wait_for_async_resp(Req_id, Options, undefined, undefined, []), - Parent ! {async_result, self(), Result}; - Err -> - Parent ! {async_result, self(), Err} - end. - -wait_for_async_resp(Req_id, Options, Acc_Stat_code, Acc_Headers, Body) -> - receive - {ibrowse_async_headers, Req_id, StatCode, Headers} -> - %% io:format("Recvd headers...~n", []), - maybe_stream_next(Req_id, Options), - wait_for_async_resp(Req_id, Options, StatCode, Headers, Body); - {ibrowse_async_response_end, Req_id} -> - %% io:format("Recvd end of response.~n", []), - Body_1 = list_to_binary(lists:reverse(Body)), - {ok, Acc_Stat_code, Acc_Headers, Body_1}; - {ibrowse_async_response, Req_id, Data} -> - maybe_stream_next(Req_id, Options), - %% io:format("Recvd data...~n", []), - wait_for_async_resp(Req_id, Options, Acc_Stat_code, Acc_Headers, [Data | Body]); - {ibrowse_async_response, Req_id, {error, _} = Err} -> - {ok, Acc_Stat_code, Acc_Headers, Err}; - Err -> - {ok, Acc_Stat_code, Acc_Headers, Err} - after 10000 -> - {timeout, Acc_Stat_code, Acc_Headers, Body} - end. - -maybe_stream_next(Req_id, Options) -> - case lists:member(once, Options) of - true -> - ibrowse:stream_next(Req_id); - false -> - ok - end. - -execute_req(local_test_fun, Method, Args) -> - io:format(" ~-54.54w: ", [Method]), - Result = (catch apply(?MODULE, Method, Args)), - io:format("~p~n", [Result]); -execute_req(Url, Method, Options) -> - io:format("~7.7w, ~50.50s: ", [Method, Url]), - Result = (catch ibrowse:send_req(Url, [], Method, [], Options)), - case Result of - {ok, SCode, _H, _B} -> - io:format("Status code: ~p~n", [SCode]); - Err -> - io:format("~p~n", [Err]) - end. - -ue_test() -> - ue_test(lists:duplicate(1024, $?)). -ue_test(Data) -> - {Time, Res} = timer:tc(ibrowse_lib, url_encode, [Data]), - io:format("Time -> ~p~n", [Time]), - io:format("Data Length -> ~p~n", [length(Data)]), - io:format("Res Length -> ~p~n", [length(Res)]). -% io:format("Result -> ~s~n", [Res]). - -log_msg(Fmt, Args) -> - io:format("~s -- " ++ Fmt, - [ibrowse_lib:printable_date() | Args]). - -%%------------------------------------------------------------------------------ -%% Test what happens when the response to a HEAD request is a -%% Chunked-Encoding response with a non-empty body. Issue #67 on -%% Github -%% ------------------------------------------------------------------------------ -test_head_transfer_encoding() -> - clear_msg_q(), - test_head_transfer_encoding("http://localhost:8181/ibrowse_head_test";). - -test_head_transfer_encoding(Url) -> - case ibrowse:send_req(Url, [], head) of - {ok, "200", _, _} -> - success; - Res -> - {test_failed, Res} - end. - -%%------------------------------------------------------------------------------ -%% Test what happens when the response to a HEAD request is a -%% Chunked-Encoding response with a non-empty body. Issue #67 on -%% Github -%% ------------------------------------------------------------------------------ -test_head_response_with_body() -> - clear_msg_q(), - test_head_response_with_body("http://localhost:8181/ibrowse_head_transfer_enc";). - -test_head_response_with_body(Url) -> - case ibrowse:send_req(Url, [], head, [], [{workaround, head_response_with_body}]) of - {ok, "400", _, _} -> - success; - Res -> - {test_failed, Res} - end. - -%%------------------------------------------------------------------------------ -%% Test what happens when the request at the head of a pipeline times out -%%------------------------------------------------------------------------------ -test_pipeline_head_timeout() -> - clear_msg_q(), - test_pipeline_head_timeout("http://localhost:8181/ibrowse_inac_timeout_test";). - -test_pipeline_head_timeout(Url) -> - {ok, Pid} = ibrowse:spawn_worker_process(Url), - Test_parent = self(), - Fun = fun({fixed, Timeout}) -> - spawn(fun() -> - do_test_pipeline_head_timeout(Url, Pid, Test_parent, Timeout) - end); - (Timeout_mult) -> - spawn(fun() -> - Timeout = 1000 + Timeout_mult*1000, - do_test_pipeline_head_timeout(Url, Pid, Test_parent, Timeout) - end) - end, - Pids = [Fun(X) || X <- [{fixed, 32000} | lists:seq(1,10)]], - Result = accumulate_worker_resp(Pids), - case lists:all(fun({_, X_res}) -> - X_res == {error,req_timedout} - end, Result) of - true -> - success; - false -> - {test_failed, Result} - end. - -do_test_pipeline_head_timeout(Url, Pid, Test_parent, Req_timeout) -> - Resp = ibrowse:send_req_direct( - Pid, - Url, - [], get, [], - [{socket_options,[{keepalive,true}]}, - {inactivity_timeout,180000}, - {connect_timeout,180000}], Req_timeout), - Test_parent ! {self(), Resp}. - -accumulate_worker_resp(Pids) -> - accumulate_worker_resp(Pids, []). - -accumulate_worker_resp([_ | _] = Pids, Acc) -> - receive - {Pid, Res} when is_pid(Pid) -> - accumulate_worker_resp(Pids -- [Pid], [{Pid, Res} | Acc]); - Err -> - io:format("Received unexpected: ~p~n", [Err]) - end; -accumulate_worker_resp([], Acc) -> - lists:reverse(Acc). - -clear_msg_q() -> - receive - _ -> - clear_msg_q() - after 0 -> - ok - end. -%%------------------------------------------------------------------------------ -%% -%%------------------------------------------------------------------------------ - -test_20122010() -> - test_20122010("http://localhost:8181";). - -test_20122010(Url) -> - {ok, Pid} = ibrowse:spawn_worker_process(Url), - Expected_resp = <<"1-2-3-4-5-6-7-8-9-10-11-12-13-14-15-16-17-18-19-20-21-22-23-24-25-26-27-28-29-30-31-32-33-34-35-36-37-38-39-40-41-42-43-44-45-46-47-48-49-50-51-52-53-54-55-56-57-58-59-60-61-62-63-64-65-66-67-68-69-70-71-72-73-74-75-76-77-78-79-80-81-82-83-84-85-86-87-88-89-90-91-92-93-94-95-96-97-98-99-100">>, - Test_parent = self(), - Fun = fun() -> - do_test_20122010(Url, Pid, Expected_resp, Test_parent) - end, - Pids = [erlang:spawn_monitor(Fun) || _ <- lists:seq(1,10)], - wait_for_workers(Pids). - -wait_for_workers([{Pid, _Ref} | Pids]) -> - receive - {Pid, success} -> - wait_for_workers(Pids) - after 60000 -> - test_failed - end; -wait_for_workers([]) -> - success. - -do_test_20122010(Url, Pid, Expected_resp, Test_parent) -> - do_test_20122010(10, Url, Pid, Expected_resp, Test_parent). - -do_test_20122010(0, _Url, _Pid, _Expected_resp, Test_parent) -> - Test_parent ! {self(), success}; -do_test_20122010(Rem_count, Url, Pid, Expected_resp, Test_parent) -> - {ibrowse_req_id, Req_id} = ibrowse:send_req_direct( - Pid, - Url ++ "/ibrowse_stream_once_chunk_pipeline_test", - [], get, [], - [{stream_to, {self(), once}}, - {inactivity_timeout, 10000}, - {include_ibrowse_req_id, true}]), - do_trace("~p -- sent request ~1000.p~n", [self(), Req_id]), - Req_id_str = lists:flatten(io_lib:format("~1000.p",[Req_id])), - receive - {ibrowse_async_headers, Req_id, "200", Headers} -> - case lists:keysearch("x-ibrowse-request-id", 1, Headers) of - {value, {_, Req_id_str}} -> - ok; - {value, {_, Req_id_1}} -> - do_trace("~p -- Sent req-id: ~1000.p. Recvd: ~1000.p~n", - [self(), Req_id, Req_id_1]), - exit(req_id_mismatch) - end - after 5000 -> - do_trace("~p -- response headers not received~n", [self()]), - exit({timeout, test_failed}) - end, - do_trace("~p -- response headers received~n", [self()]), - ok = ibrowse:stream_next(Req_id), - case do_test_20122010_1(Expected_resp, Req_id, []) of - true -> - do_test_20122010(Rem_count - 1, Url, Pid, Expected_resp, Test_parent); - false -> - Test_parent ! {self(), failed} - end. - -do_test_20122010_1(Expected_resp, Req_id, Acc) -> - receive - {ibrowse_async_response, Req_id, Body_part} -> - ok = ibrowse:stream_next(Req_id), - do_test_20122010_1(Expected_resp, Req_id, [Body_part | Acc]); - {ibrowse_async_response_end, Req_id} -> - Acc_1 = list_to_binary(lists:reverse(Acc)), - Result = Acc_1 == Expected_resp, - do_trace("~p -- End of response. Result: ~p~n", [self(), Result]), - Result - after 1000 -> - exit({timeout, test_failed}) - end. - -do_trace(Fmt, Args) -> - do_trace(get(my_trace_flag), Fmt, Args). - -do_trace(true, Fmt, Args) -> - io:format("~s -- " ++ Fmt, [ibrowse_lib:printable_date() | Args]); -do_trace(_, _, _) -> - ok. http://git-wip-us.apache.org/repos/asf/couchdb-ibrowse/blob/4af2d408/include/ibrowse.hrl ---------------------------------------------------------------------- diff --git a/include/ibrowse.hrl b/include/ibrowse.hrl new file mode 100644 index 0000000..18dde82 --- /dev/null +++ b/include/ibrowse.hrl @@ -0,0 +1,21 @@ +-ifndef(IBROWSE_HRL). +-define(IBROWSE_HRL, "ibrowse.hrl"). + +-record(url, { + abspath, + host, + port, + username, + password, + path, + protocol, + host_type % 'hostname', 'ipv4_address' or 'ipv6_address' +}). + +-record(lb_pid, {host_port, pid}). + +-record(client_conn, {key, cur_pipeline_size = 0, reqs_served = 0}). + +-record(ibrowse_conf, {key, value}). + +-endif. http://git-wip-us.apache.org/repos/asf/couchdb-ibrowse/blob/4af2d408/src/ibrowse.app.src ---------------------------------------------------------------------- diff --git a/src/ibrowse.app.src b/src/ibrowse.app.src new file mode 100644 index 0000000..1d88084 --- /dev/null +++ b/src/ibrowse.app.src @@ -0,0 +1,7 @@ +{application, ibrowse, + [{description, "Erlang HTTP client application"}, + {vsn, "4.0.1"}, + {registered, [ibrowse_sup, ibrowse]}, + {applications, [kernel,stdlib]}, + {env, []}, + {mod, {ibrowse_app, []}}]}.
