Filipe, Fixing now.
On Fri, Nov 5, 2010 at 8:01 PM, Filipe David Manana <[email protected]> wrote: > Paul, > > Here: > > +stream_chunked_response(Req, ReqId, Resp) -> > + receive > + {ibrowse_async_response, ReqId, Chunk} -> > + couch_httpd:send_chunk(Resp, Chunk), > + ibrowse:stream_next(ReqId), > + stream_chunked_response(Req, ReqId, Resp); > + {ibrowse_async_response, ReqId, {error, Reason}} -> > + throw({error, Reason}); > + {ibrowse_async_response_end, ReqId} -> > + couch_httpd:last_chunk(Resp) > + end. > + > + > +stream_length_response(Req, ReqId, Resp) -> > + receive > + {ibrowse_async_response, ReqId, Chunk} -> > + couch_httpd:send(Resp, Chunk), > + ibrowse:stream_next(ReqId), > + stream_length_response(Req, ReqId, Resp); > + {ibrowse_async_response, {error, Reason}} -> > + throw({error, Reason}); > + {ibrowse_async_response_end, ReqId} -> > + ok > + end. > > The " {ibrowse_async_response, ReqId, {error, Reason}} " clauses > should come before the " {ibrowse_async_response, ReqId, Chunk} " > clauses, since the last always mask the former. > > Also, in the stream_length_response function you're missing the ReqId > element before the {error, Reason} element. > > Good work :D > > On Fri, Nov 5, 2010 at 11:26 PM, <[email protected]> wrote: >> Author: davisp >> Date: Fri Nov 5 23:26:21 2010 >> New Revision: 1031877 >> >> URL: http://svn.apache.org/viewvc?rev=1031877&view=rev >> Log: >> HTTP proxy handler. >> >> The second of two new features to replace the _externals protocols. This >> allows users to configure CouchDB to proxy requests to an external HTTP >> server. The external HTTP server is not required to be on the same host >> running CouchDB. >> >> The configuration looks like such: >> >> [httpd_global_handlers] >> _google = {couch_httpd_proxy, handle_proxy_req, <<"http://www.google.com";>>} >> >> You can then hit this proxy at the url: >> >> http://127.0.0.1:5984/_google >> >> If you add any path after the proxy name, or make a request with a query >> string, those will be appended to the URL specified in the configuration. >> >> Ie: >> >> http://127.0.0.1:5984/_google/search?q=plankton >> >> would translate to: >> >> http://www.google.com/search?q=plankton >> >> Obviously, request bodies are handled as expected. >> >> >> Added: >> couchdb/trunk/src/couchdb/couch_httpd_proxy.erl >> couchdb/trunk/test/etap/180-http-proxy.ini >> couchdb/trunk/test/etap/180-http-proxy.t >> couchdb/trunk/test/etap/test_web.erl >> Modified: >> couchdb/trunk/etc/couchdb/local.ini >> couchdb/trunk/share/www/script/test/basics.js >> couchdb/trunk/src/couchdb/Makefile.am >> couchdb/trunk/src/couchdb/couch_httpd.erl >> couchdb/trunk/test/etap/ (props changed) >> couchdb/trunk/test/etap/Makefile.am >> >> Modified: couchdb/trunk/etc/couchdb/local.ini >> URL: >> http://svn.apache.org/viewvc/couchdb/trunk/etc/couchdb/local.ini?rev=1031877&r1=1031876&r2=1031877&view=diff >> ============================================================================== >> --- couchdb/trunk/etc/couchdb/local.ini (original) >> +++ couchdb/trunk/etc/couchdb/local.ini Fri Nov 5 23:26:21 2010 >> @@ -20,6 +20,9 @@ >> ; the whitelist. >> ;config_whitelist = [{httpd,config_whitelist}, {log,level}, {etc,etc}] >> >> +[httpd_global_handlers] >> +;_google = {couch_httpd_proxy, handle_proxy_req, >> <<"http://www.google.com";>>} >> + >> [couch_httpd_auth] >> ; If you set this to true, you should also uncomment the WWW-Authenticate >> line >> ; above. If you don't configure a WWW-Authenticate header, CouchDB will send >> >> Modified: couchdb/trunk/share/www/script/test/basics.js >> URL: >> http://svn.apache.org/viewvc/couchdb/trunk/share/www/script/test/basics.js?rev=1031877&r1=1031876&r2=1031877&view=diff >> ============================================================================== >> --- couchdb/trunk/share/www/script/test/basics.js (original) >> +++ couchdb/trunk/share/www/script/test/basics.js Fri Nov 5 23:26:21 2010 >> @@ -159,8 +159,8 @@ couchTests.basics = function(debug) { >> var loc = xhr.getResponseHeader("Location"); >> T(loc, "should have a Location header"); >> var locs = loc.split('/'); >> - T(locs[4] == resp.id); >> - T(locs[3] == "test_suite_db"); >> + T(locs[locs.length-1] == resp.id); >> + T(locs[locs.length-2] == "test_suite_db"); >> >> // test that that POST's with an _id aren't overriden with a UUID. >> var xhr = CouchDB.request("POST", "/test_suite_db", { >> >> Modified: couchdb/trunk/src/couchdb/Makefile.am >> URL: >> http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/Makefile.am?rev=1031877&r1=1031876&r2=1031877&view=diff >> ============================================================================== >> --- couchdb/trunk/src/couchdb/Makefile.am (original) >> +++ couchdb/trunk/src/couchdb/Makefile.am Fri Nov 5 23:26:21 2010 >> @@ -50,6 +50,7 @@ source_files = \ >> couch_httpd_show.erl \ >> couch_httpd_view.erl \ >> couch_httpd_misc_handlers.erl \ >> + couch_httpd_proxy.erl \ >> couch_httpd_rewrite.erl \ >> couch_httpd_stats_handlers.erl \ >> couch_httpd_vhost.erl \ >> @@ -107,6 +108,7 @@ compiled_files = \ >> couch_httpd_db.beam \ >> couch_httpd_auth.beam \ >> couch_httpd_oauth.beam \ >> + couch_httpd_proxy.beam \ >> couch_httpd_external.beam \ >> couch_httpd_show.beam \ >> couch_httpd_view.beam \ >> >> Modified: couchdb/trunk/src/couchdb/couch_httpd.erl >> URL: >> http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_httpd.erl?rev=1031877&r1=1031876&r2=1031877&view=diff >> ============================================================================== >> --- couchdb/trunk/src/couchdb/couch_httpd.erl (original) >> +++ couchdb/trunk/src/couchdb/couch_httpd.erl Fri Nov 5 23:26:21 2010 >> @@ -22,7 +22,7 @@ >> -export([parse_form/1,json_body/1,json_body_obj/1,body/1,doc_etag/1, >> make_etag/1, etag_respond/3]). >> -export([primary_header_value/2,partition/1,serve_file/3,serve_file/4, >> server_header/0]). >> -export([start_chunked_response/3,send_chunk/2,log_request/2]). >> --export([start_response_length/4, send/2]). >> +-export([start_response_length/4, start_response/3, send/2]). >> -export([start_json_response/2, start_json_response/3, >> end_json_response/1]). >> -export([send_response/4,send_method_not_allowed/2,send_error/4, >> send_redirect/2,send_chunked_error/2]). >> -export([send_json/2,send_json/3,send_json/4,last_chunk/1,parse_multipart_request/3]). >> @@ -526,6 +526,18 @@ start_response_length(#httpd{mochi_req=M >> end, >> {ok, Resp}. >> >> +start_response(#httpd{mochi_req=MochiReq}=Req, Code, Headers) -> >> + log_request(Req, Code), >> + couch_stats_collector:increment({httpd_status_cdes, Code}), >> + CookieHeader = couch_httpd_auth:cookie_auth_header(Req, Headers), >> + Headers2 = Headers ++ server_header() ++ CookieHeader, >> + Resp = MochiReq:start_response({Code, Headers2}), >> + case MochiReq:get(method) of >> + 'HEAD' -> throw({http_head_abort, Resp}); >> + _ -> ok >> + end, >> + {ok, Resp}. >> + >> send(Resp, Data) -> >> Resp:send(Data), >> {ok, Resp}. >> >> Added: couchdb/trunk/src/couchdb/couch_httpd_proxy.erl >> URL: >> http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_httpd_proxy.erl?rev=1031877&view=auto >> ============================================================================== >> --- couchdb/trunk/src/couchdb/couch_httpd_proxy.erl (added) >> +++ couchdb/trunk/src/couchdb/couch_httpd_proxy.erl Fri Nov 5 23:26:21 2010 >> @@ -0,0 +1,425 @@ >> +% Licensed under the Apache License, Version 2.0 (the "License"); you may >> not >> +% use this file except in compliance with the License. You may obtain a >> copy of >> +% the License at >> +% >> +% http://www.apache.org/licenses/LICENSE-2.0 >> +% >> +% Unless required by applicable law or agreed to in writing, software >> +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT >> +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the >> +% License for the specific language governing permissions and limitations >> under >> +% the License. >> +-module(couch_httpd_proxy). >> + >> +-export([handle_proxy_req/2]). >> + >> +-include("couch_db.hrl"). >> +-include("../ibrowse/ibrowse.hrl"). >> + >> +-define(TIMEOUT, infinity). >> +-define(PKT_SIZE, 4096). >> + >> + >> +handle_proxy_req(Req, ProxyDest) -> >> + >> + %% Bug in Mochiweb? >> + %% Reported here: http://github.com/mochi/mochiweb/issues/issue/16 >> + erase(mochiweb_request_body_length), >> + >> + Method = get_method(Req), >> + Url = get_url(Req, ProxyDest), >> + Version = get_version(Req), >> + Headers = get_headers(Req), >> + Body = get_body(Req), >> + Options = [ >> + {http_vsn, Version}, >> + {headers_as_is, true}, >> + {response_format, binary}, >> + {stream_to, {self(), once}} >> + ], >> + case ibrowse:send_req(Url, Headers, Method, Body, Options, ?TIMEOUT) of >> + {ibrowse_req_id, ReqId} -> >> + stream_response(Req, ProxyDest, ReqId); >> + {error, Reason} -> >> + throw({error, Reason}) >> + end. >> + >> + >> +get_method(#httpd{mochi_req=MochiReq}) -> >> + case MochiReq:get(method) of >> + Method when is_atom(Method) -> >> + list_to_atom(string:to_lower(atom_to_list(Method))); >> + Method when is_list(Method) -> >> + list_to_atom(string:to_lower(Method)); >> + Method when is_binary(Method) -> >> + list_to_atom(string:to_lower(?b2l(Method))) >> + end. >> + >> + >> +get_url(Req, ProxyDest) when is_binary(ProxyDest) -> >> + get_url(Req, ?b2l(ProxyDest)); >> +get_url(#httpd{mochi_req=MochiReq}=Req, ProxyDest) -> >> + BaseUrl = case mochiweb_util:partition(ProxyDest, "/") of >> + {[], "/", _} -> couch_httpd:absolute_uri(Req, ProxyDest); >> + _ -> ProxyDest >> + end, >> + ProxyPrefix = "/" ++ ?b2l(hd(Req#httpd.path_parts)), >> + RequestedPath = MochiReq:get(raw_path), >> + case mochiweb_util:partition(RequestedPath, ProxyPrefix) of >> + {[], ProxyPrefix, []} -> >> + BaseUrl; >> + {[], ProxyPrefix, [$/ | DestPath]} -> >> + remove_trailing_slash(BaseUrl) ++ "/" ++ DestPath; >> + {[], ProxyPrefix, DestPath} -> >> + remove_trailing_slash(BaseUrl) ++ "/" ++ DestPath; >> + _Else -> >> + throw({invalid_url_path, {ProxyPrefix, RequestedPath}}) >> + end. >> + >> +get_version(#httpd{mochi_req=MochiReq}) -> >> + MochiReq:get(version). >> + >> + >> +get_headers(#httpd{mochi_req=MochiReq}) -> >> + to_ibrowse_headers(mochiweb_headers:to_list(MochiReq:get(headers)), []). >> + >> +to_ibrowse_headers([], Acc) -> >> + lists:reverse(Acc); >> +to_ibrowse_headers([{K, V} | Rest], Acc) when is_atom(K) -> >> + to_ibrowse_headers([{atom_to_list(K), V} | Rest], Acc); >> +to_ibrowse_headers([{K, V} | Rest], Acc) when is_list(K) -> >> + case string:to_lower(K) of >> + "content-length" -> >> + to_ibrowse_headers(Rest, [{content_length, V} | Acc]); >> + % This appears to make ibrowse too smart. >> + %"transfer-encoding" -> >> + % to_ibrowse_headers(Rest, [{transfer_encoding, V} | Acc]); >> + _ -> >> + to_ibrowse_headers(Rest, [{K, V} | Acc]) >> + end. >> + >> +get_body(#httpd{method='GET'}) -> >> + fun() -> eof end; >> +get_body(#httpd{method='HEAD'}) -> >> + fun() -> eof end; >> +get_body(#httpd{method='DELETE'}) -> >> + fun() -> eof end; >> +get_body(#httpd{mochi_req=MochiReq}) -> >> + case MochiReq:get(body_length) of >> + undefined -> >> + <<>>; >> + {unknown_transfer_encoding, Unknown} -> >> + exit({unknown_transfer_encoding, Unknown}); >> + chunked -> >> + {fun stream_chunked_body/1, {init, MochiReq, 0}}; >> + 0 -> >> + <<>>; >> + Length when is_integer(Length) andalso Length > 0 -> >> + {fun stream_length_body/1, {init, MochiReq, Length}}; >> + Length -> >> + exit({invalid_body_length, Length}) >> + end. >> + >> + >> +remove_trailing_slash(Url) -> >> + rem_slash(lists:reverse(Url)). >> + >> +rem_slash([]) -> >> + []; >> +rem_slash([$\s | RevUrl]) -> >> + rem_slash(RevUrl); >> +rem_slash([$\t | RevUrl]) -> >> + rem_slash(RevUrl); >> +rem_slash([$\r | RevUrl]) -> >> + rem_slash(RevUrl); >> +rem_slash([$\n | RevUrl]) -> >> + rem_slash(RevUrl); >> +rem_slash([$/ | RevUrl]) -> >> + rem_slash(RevUrl); >> +rem_slash(RevUrl) -> >> + lists:reverse(RevUrl). >> + >> + >> +stream_chunked_body({init, MReq, 0}) -> >> + % First chunk, do expect-continue dance. >> + init_body_stream(MReq), >> + stream_chunked_body({stream, MReq, 0, [], ?PKT_SIZE}); >> +stream_chunked_body({stream, MReq, 0, Buf, BRem}) -> >> + % Finished a chunk, get next length. If next length >> + % is 0, its time to try and read trailers. >> + {CRem, Data} = read_chunk_length(MReq), >> + case CRem of >> + 0 -> >> + BodyData = iolist_to_binary(lists:reverse(Buf, Data)), >> + {ok, BodyData, {trailers, MReq, [], ?PKT_SIZE}}; >> + _ -> >> + stream_chunked_body( >> + {stream, MReq, CRem, [Data | Buf], BRem-size(Data)} >> + ) >> + end; >> +stream_chunked_body({stream, MReq, CRem, Buf, BRem}) when BRem =< 0 -> >> + % Time to empty our buffers to the upstream socket. >> + BodyData = iolist_to_binary(lists:reverse(Buf)), >> + {ok, BodyData, {stream, MReq, CRem, [], ?PKT_SIZE}}; >> +stream_chunked_body({stream, MReq, CRem, Buf, BRem}) -> >> + % Buffer some more data from the client. >> + Length = lists:min([CRem, BRem]), >> + Socket = MReq:get(socket), >> + NewState = case mochiweb_socket:recv(Socket, Length, ?TIMEOUT) of >> + {ok, Data} when size(Data) == CRem -> >> + case mochiweb_socket:recv(Socket, 2, ?TIMEOUT) of >> + {ok, <<"\r\n">>} -> >> + {stream, MReq, 0, [<<"\r\n">>, Data | Buf], >> BRem-Length-2}; >> + _ -> >> + exit(normal) >> + end; >> + {ok, Data} -> >> + {stream, MReq, CRem-Length, [Data | Buf], BRem-Length}; >> + _ -> >> + exit(normal) >> + end, >> + stream_chunked_body(NewState); >> +stream_chunked_body({trailers, MReq, Buf, BRem}) when BRem =< 0 -> >> + % Empty our buffers and send data upstream. >> + BodyData = iolist_to_binary(lists:reverse(Buf)), >> + {ok, BodyData, {trailers, MReq, [], ?PKT_SIZE}}; >> +stream_chunked_body({trailers, MReq, Buf, BRem}) -> >> + % Read another trailer into the buffer or stop on an >> + % empty line. >> + Socket = MReq:get(socket), >> + mochiweb_socket:setopts(Socket, [{packet, line}]), >> + case mochiweb_socket:recv(Socket, 0, ?TIMEOUT) of >> + {ok, <<"\r\n">>} -> >> + mochiweb_socket:setopts(Socket, [{packet, raw}]), >> + BodyData = iolist_to_binary(lists:reverse(Buf, <<"\r\n">>)), >> + {ok, BodyData, eof}; >> + {ok, Footer} -> >> + mochiweb_socket:setopts(Socket, [{packet, raw}]), >> + NewState = {trailers, MReq, [Footer | Buf], BRem-size(Footer)}, >> + stream_chunked_body(NewState); >> + _ -> >> + exit(normal) >> + end; >> +stream_chunked_body(eof) -> >> + % Tell ibrowse we're done sending data. >> + eof. >> + >> + >> +stream_length_body({init, MochiReq, Length}) -> >> + % Do the expect-continue dance >> + init_body_stream(MochiReq), >> + stream_length_body({stream, MochiReq, Length}); >> +stream_length_body({stream, _MochiReq, 0}) -> >> + % Finished streaming. >> + eof; >> +stream_length_body({stream, MochiReq, Length}) -> >> + BufLen = lists:min([Length, ?PKT_SIZE]), >> + case MochiReq:recv(BufLen) of >> + <<>> -> eof; >> + Bin -> {ok, Bin, {stream, MochiReq, Length-BufLen}} >> + end. >> + >> + >> +init_body_stream(MochiReq) -> >> + Expect = case MochiReq:get_header_value("expect") of >> + undefined -> >> + undefined; >> + Value when is_list(Value) -> >> + string:to_lower(Value) >> + end, >> + case Expect of >> + "100-continue" -> >> + MochiReq:start_raw_response({100, gb_trees:empty()}); >> + _Else -> >> + ok >> + end. >> + >> + >> +read_chunk_length(MochiReq) -> >> + Socket = MochiReq:get(socket), >> + mochiweb_socket:setopts(Socket, [{packet, line}]), >> + case mochiweb_socket:recv(Socket, 0, ?TIMEOUT) of >> + {ok, Header} -> >> + mochiweb_socket:setopts(Socket, [{packet, raw}]), >> + Splitter = fun(C) -> >> + C =/= $\r andalso C =/= $\n andalso C =/= $\s >> + end, >> + {Hex, _Rest} = lists:splitwith(Splitter, ?b2l(Header)), >> + {mochihex:to_int(Hex), Header}; >> + _ -> >> + exit(normal) >> + end. >> + >> + >> +stream_response(Req, ProxyDest, ReqId) -> >> + receive >> + {ibrowse_async_headers, ReqId, "100", _} -> >> + % ibrowse doesn't handle 100 Continue responses which >> + % means we have to discard them so the proxy client >> + % doesn't get confused. >> + ibrowse:stream_next(ReqId), >> + stream_response(Req, ProxyDest, ReqId); >> + {ibrowse_async_headers, ReqId, Status, Headers} -> >> + {Source, Dest} = get_urls(Req, ProxyDest), >> + FixedHeaders = fix_headers(Source, Dest, Headers, []), >> + case body_length(FixedHeaders) of >> + chunked -> >> + {ok, Resp} = couch_httpd:start_chunked_response( >> + Req, list_to_integer(Status), FixedHeaders >> + ), >> + ibrowse:stream_next(ReqId), >> + stream_chunked_response(Req, ReqId, Resp), >> + {ok, Resp}; >> + Length when is_integer(Length) -> >> + {ok, Resp} = couch_httpd:start_response_length( >> + Req, list_to_integer(Status), FixedHeaders, Length >> + ), >> + ibrowse:stream_next(ReqId), >> + stream_length_response(Req, ReqId, Resp), >> + {ok, Resp}; >> + _ -> >> + {ok, Resp} = couch_httpd:start_response( >> + Req, list_to_integer(Status), FixedHeaders >> + ), >> + ibrowse:stream_next(ReqId), >> + stream_length_response(Req, ReqId, Resp), >> + % XXX: MochiWeb apparently doesn't look at the >> + % response to see if it must force close the >> + % connection. So we help it out here. >> + erlang:put(mochiweb_request_force_close, true), >> + {ok, Resp} >> + end >> + end. >> + >> + >> +stream_chunked_response(Req, ReqId, Resp) -> >> + receive >> + {ibrowse_async_response, ReqId, Chunk} -> >> + couch_httpd:send_chunk(Resp, Chunk), >> + ibrowse:stream_next(ReqId), >> + stream_chunked_response(Req, ReqId, Resp); >> + {ibrowse_async_response, ReqId, {error, Reason}} -> >> + throw({error, Reason}); >> + {ibrowse_async_response_end, ReqId} -> >> + couch_httpd:last_chunk(Resp) >> + end. >> + >> + >> +stream_length_response(Req, ReqId, Resp) -> >> + receive >> + {ibrowse_async_response, ReqId, Chunk} -> >> + couch_httpd:send(Resp, Chunk), >> + ibrowse:stream_next(ReqId), >> + stream_length_response(Req, ReqId, Resp); >> + {ibrowse_async_response, {error, Reason}} -> >> + throw({error, Reason}); >> + {ibrowse_async_response_end, ReqId} -> >> + ok >> + end. >> + >> + >> +get_urls(Req, ProxyDest) -> >> + SourceUrl = couch_httpd:absolute_uri(Req, "/" ++ >> hd(Req#httpd.path_parts)), >> + Source = parse_url(?b2l(iolist_to_binary(SourceUrl))), >> + case (catch parse_url(ProxyDest)) of >> + Dest when is_record(Dest, url) -> >> + {Source, Dest}; >> + _ -> >> + DestUrl = couch_httpd:absolute_uri(Req, ProxyDest), >> + {Source, parse_url(DestUrl)} >> + end. >> + >> + >> +fix_headers(_, _, [], Acc) -> >> + lists:reverse(Acc); >> +fix_headers(Source, Dest, [{K, V} | Rest], Acc) -> >> + Fixed = case string:to_lower(K) of >> + "location" -> rewrite_location(Source, Dest, V); >> + "content-location" -> rewrite_location(Source, Dest, V); >> + "uri" -> rewrite_location(Source, Dest, V); >> + "destination" -> rewrite_location(Source, Dest, V); >> + "set-cookie" -> rewrite_cookie(Source, Dest, V); >> + _ -> V >> + end, >> + fix_headers(Source, Dest, Rest, [{K, Fixed} | Acc]). >> + >> + >> +rewrite_location(Source, #url{host=Host, port=Port, protocol=Proto}, Url) -> >> + case (catch parse_url(Url)) of >> + #url{host=Host, port=Port, protocol=Proto} = Location -> >> + DestLoc = #url{ >> + protocol=Source#url.protocol, >> + host=Source#url.host, >> + port=Source#url.port, >> + path=join_url_path(Source#url.path, Location#url.path) >> + }, >> + url_to_url(DestLoc); >> + #url{} -> >> + Url; >> + _ -> >> + url_to_url(Source#url{path=join_url_path(Source#url.path, Url)}) >> + end. >> + >> + >> +rewrite_cookie(_Source, _Dest, Cookie) -> >> + Cookie. >> + >> + >> +parse_url(Url) when is_binary(Url) -> >> + ibrowse_lib:parse_url(?b2l(Url)); >> +parse_url(Url) when is_list(Url) -> >> + ibrowse_lib:parse_url(?b2l(iolist_to_binary(Url))). >> + >> + >> +join_url_path(Src, Dst) -> >> + Src2 = case lists:reverse(Src) of >> + "/" ++ RestSrc -> lists:reverse(RestSrc); >> + _ -> Src >> + end, >> + Dst2 = case Dst of >> + "/" ++ RestDst -> RestDst; >> + _ -> Dst >> + end, >> + Src2 ++ "/" ++ Dst2. >> + >> + >> +url_to_url(#url{host=Host, port=Port, path=Path, protocol=Proto}) -> >> + LPort = case {Proto, Port} of >> + {http, 80} -> ""; >> + {https, 443} -> ""; >> + _ -> ":" ++ integer_to_list(Port) >> + end, >> + LPath = case Path of >> + "/" ++ _RestPath -> Path; >> + _ -> "/" ++ Path >> + end, >> + atom_to_list(Proto) ++ "://" ++ Host ++ LPort ++ LPath. >> + >> + >> +body_length(Headers) -> >> + case is_chunked(Headers) of >> + true -> chunked; >> + _ -> content_length(Headers) >> + end. >> + >> + >> +is_chunked([]) -> >> + false; >> +is_chunked([{K, V} | Rest]) -> >> + case string:to_lower(K) of >> + "transfer-encoding" -> >> + string:to_lower(V) == "chunked"; >> + _ -> >> + is_chunked(Rest) >> + end. >> + >> +content_length([]) -> >> + undefined; >> +content_length([{K, V} | Rest]) -> >> + case string:to_lower(K) of >> + "content-length" -> >> + list_to_integer(V); >> + _ -> >> + content_length(Rest) >> + end. >> + >> >> Propchange: couchdb/trunk/test/etap/ >> ------------------------------------------------------------------------------ >> --- svn:ignore (original) >> +++ svn:ignore Fri Nov 5 23:26:21 2010 >> @@ -3,4 +3,5 @@ Makefile >> Makefile.in >> test_util.erl >> test_util.beam >> +test_web.beam >> run >> >> Added: couchdb/trunk/test/etap/180-http-proxy.ini >> URL: >> http://svn.apache.org/viewvc/couchdb/trunk/test/etap/180-http-proxy.ini?rev=1031877&view=auto >> ============================================================================== >> --- couchdb/trunk/test/etap/180-http-proxy.ini (added) >> +++ couchdb/trunk/test/etap/180-http-proxy.ini Fri Nov 5 23:26:21 2010 >> @@ -0,0 +1,20 @@ >> +; Licensed to the Apache Software Foundation (ASF) under one >> +; or more contributor license agreements. See the NOTICE file >> +; distributed with this work for additional information >> +; regarding copyright ownership. The ASF licenses this file >> +; to you under the Apache License, Version 2.0 (the >> +; "License"); you may not use this file except in compliance >> +; with the License. You may obtain a copy of the License at >> +; >> +; http://www.apache.org/licenses/LICENSE-2.0 >> +; >> +; Unless required by applicable law or agreed to in writing, >> +; software distributed under the License is distributed on an >> +; "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY >> +; KIND, either express or implied. See the License for the >> +; specific language governing permissions and limitations >> +; under the License. >> + >> +[httpd_global_handlers] >> +_test = {couch_httpd_proxy, handle_proxy_req, <<"http://127.0.0.1:5985/";>>} >> +_error = {couch_httpd_proxy, handle_proxy_req, <<"http://127.0.0.1:5986/";>>} >> \ No newline at end of file >> >> Added: couchdb/trunk/test/etap/180-http-proxy.t >> URL: >> http://svn.apache.org/viewvc/couchdb/trunk/test/etap/180-http-proxy.t?rev=1031877&view=auto >> ============================================================================== >> --- couchdb/trunk/test/etap/180-http-proxy.t (added) >> +++ couchdb/trunk/test/etap/180-http-proxy.t Fri Nov 5 23:26:21 2010 >> @@ -0,0 +1,357 @@ >> +#!/usr/bin/env escript >> +% Licensed under the Apache License, Version 2.0 (the "License"); you may >> not >> +% use this file except in compliance with the License. You may obtain a >> copy of >> +% the License at >> +% >> +% http://www.apache.org/licenses/LICENSE-2.0 >> +% >> +% Unless required by applicable law or agreed to in writing, software >> +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT >> +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the >> +% License for the specific language governing permissions and limitations >> under >> +% the License. >> + >> +-record(req, {method=get, path="", headers=[], body="", opts=[]}). >> + >> +default_config() -> >> + [ >> + test_util:build_file("etc/couchdb/default_dev.ini"), >> + test_util:source_file("test/etap/180-http-proxy.ini") >> + ]. >> + >> +server() -> "http://127.0.0.1:5984/_test/";. >> +proxy() -> "http://127.0.0.1:5985/";. >> +external() -> "https://www.google.com/";. >> + >> +main(_) -> >> + test_util:init_code_path(), >> + >> + etap:plan(61), >> + case (catch test()) of >> + ok -> >> + etap:end_tests(); >> + Other -> >> + etap:diag("Test died abnormally: ~p", [Other]), >> + etap:bail("Bad return value.") >> + end, >> + ok. >> + >> +check_request(Name, Req, Remote, Local) -> >> + case Remote of >> + no_remote -> ok; >> + _ -> test_web:set_assert(Remote) >> + end, >> + Url = case proplists:lookup(url, Req#req.opts) of >> + none -> server() ++ Req#req.path; >> + {url, DestUrl} -> DestUrl >> + end, >> + Opts = [{headers_as_is, true} | Req#req.opts], >> + Resp =ibrowse:send_req( >> + Url, Req#req.headers, Req#req.method, Req#req.body, Opts >> + ), >> + %etap:diag("ibrowse response: ~p", [Resp]), >> + case Local of >> + no_local -> ok; >> + _ -> etap:fun_is(Local, Resp, Name) >> + end, >> + case {Remote, Local} of >> + {no_remote, _} -> >> + ok; >> + {_, no_local} -> >> + ok; >> + _ -> >> + etap:is(test_web:check_last(), was_ok, Name ++ " - request >> handled") >> + end, >> + Resp. >> + >> +test() -> >> + couch_server_sup:start_link(default_config()), >> + ibrowse:start(), >> + crypto:start(), >> + test_web:start_link(), >> + >> + test_basic(), >> + test_alternate_status(), >> + test_trailing_slash(), >> + test_passes_header(), >> + test_passes_host_header(), >> + test_passes_header_back(), >> + test_rewrites_location_headers(), >> + test_doesnt_rewrite_external_locations(), >> + test_rewrites_relative_location(), >> + test_uses_same_version(), >> + test_passes_body(), >> + test_passes_eof_body_back(), >> + test_passes_chunked_body(), >> + test_passes_chunked_body_back(), >> + >> + test_connect_error(), >> + >> + ok. >> + >> +test_basic() -> >> + Remote = fun(Req) -> >> + 'GET' = Req:get(method), >> + "/" = Req:get(path), >> + undefined = Req:get(body_length), >> + undefined = Req:recv_body(), >> + {ok, {200, [{"Content-Type", "text/plain"}], "ok"}} >> + end, >> + Local = fun({ok, "200", _, "ok"}) -> true; (_) -> false end, >> + check_request("Basic proxy test", #req{}, Remote, Local). >> + >> +test_alternate_status() -> >> + Remote = fun(Req) -> >> + "/alternate_status" = Req:get(path), >> + {ok, {201, [], "ok"}} >> + end, >> + Local = fun({ok, "201", _, "ok"}) -> true; (_) -> false end, >> + Req = #req{path="alternate_status"}, >> + check_request("Alternate status", Req, Remote, Local). >> + >> +test_trailing_slash() -> >> + Remote = fun(Req) -> >> + "/trailing_slash/" = Req:get(path), >> + {ok, {200, [], "ok"}} >> + end, >> + Local = fun({ok, "200", _, "ok"}) -> true; (_) -> false end, >> + Req = #req{path="trailing_slash/"}, >> + check_request("Trailing slash", Req, Remote, Local). >> + >> +test_passes_header() -> >> + Remote = fun(Req) -> >> + "/passes_header" = Req:get(path), >> + "plankton" = Req:get_header_value("X-CouchDB-Ralph"), >> + {ok, {200, [], "ok"}} >> + end, >> + Local = fun({ok, "200", _, "ok"}) -> true; (_) -> false end, >> + Req = #req{ >> + path="passes_header", >> + headers=[{"X-CouchDB-Ralph", "plankton"}] >> + }, >> + check_request("Passes header", Req, Remote, Local). >> + >> +test_passes_host_header() -> >> + Remote = fun(Req) -> >> + "/passes_host_header" = Req:get(path), >> + "www.google.com" = Req:get_header_value("Host"), >> + {ok, {200, [], "ok"}} >> + end, >> + Local = fun({ok, "200", _, "ok"}) -> true; (_) -> false end, >> + Req = #req{ >> + path="passes_host_header", >> + headers=[{"Host", "www.google.com"}] >> + }, >> + check_request("Passes host header", Req, Remote, Local). >> + >> +test_passes_header_back() -> >> + Remote = fun(Req) -> >> + "/passes_header_back" = Req:get(path), >> + {ok, {200, [{"X-CouchDB-Plankton", "ralph"}], "ok"}} >> + end, >> + Local = fun >> + ({ok, "200", Headers, "ok"}) -> >> + lists:member({"X-CouchDB-Plankton", "ralph"}, Headers); >> + (_) -> >> + false >> + end, >> + Req = #req{path="passes_header_back"}, >> + check_request("Passes header back", Req, Remote, Local). >> + >> +test_rewrites_location_headers() -> >> + etap:diag("Testing location header rewrites."), >> + do_rewrite_tests([ >> + {"Location", proxy() ++ "foo/bar", server() ++ "foo/bar"}, >> + {"Content-Location", proxy() ++ "bing?q=2", server() ++ "bing?q=2"}, >> + {"Uri", proxy() ++ "zip#frag", server() ++ "zip#frag"}, >> + {"Destination", proxy(), server()} >> + ]). >> + >> +test_doesnt_rewrite_external_locations() -> >> + etap:diag("Testing no rewrite of external locations."), >> + do_rewrite_tests([ >> + {"Location", external() ++ "search", external() ++ "search"}, >> + {"Content-Location", external() ++ "s?q=2", external() ++ "s?q=2"}, >> + {"Uri", external() ++ "f#f", external() ++ "f#f"}, >> + {"Destination", external() ++ "f?q=2#f", external() ++ "f?q=2#f"} >> + ]). >> + >> +test_rewrites_relative_location() -> >> + etap:diag("Testing relative rewrites."), >> + do_rewrite_tests([ >> + {"Location", "/foo", server() ++ "foo"}, >> + {"Content-Location", "bar", server() ++ "bar"}, >> + {"Uri", "/zing?q=3", server() ++ "zing?q=3"}, >> + {"Destination", "bing?q=stuff#yay", server() ++ "bing?q=stuff#yay"} >> + ]). >> + >> +do_rewrite_tests(Tests) -> >> + lists:foreach(fun({Header, Location, Url}) -> >> + do_rewrite_test(Header, Location, Url) >> + end, Tests). >> + >> +do_rewrite_test(Header, Location, Url) -> >> + Remote = fun(Req) -> >> + "/rewrite_test" = Req:get(path), >> + {ok, {302, [{Header, Location}], "ok"}} >> + end, >> + Local = fun >> + ({ok, "302", Headers, "ok"}) -> >> + etap:is( >> + couch_util:get_value(Header, Headers), >> + Url, >> + "Header rewritten correctly." >> + ), >> + true; >> + (_) -> >> + false >> + end, >> + Req = #req{path="rewrite_test"}, >> + Label = "Rewrite test for ", >> + check_request(Label ++ Header, Req, Remote, Local). >> + >> +test_uses_same_version() -> >> + Remote = fun(Req) -> >> + "/uses_same_version" = Req:get(path), >> + {1, 0} = Req:get(version), >> + {ok, {200, [], "ok"}} >> + end, >> + Local = fun({ok, "200", _, "ok"}) -> true; (_) -> false end, >> + Req = #req{ >> + path="uses_same_version", >> + opts=[{http_vsn, {1, 0}}] >> + }, >> + check_request("Uses same version", Req, Remote, Local). >> + >> +test_passes_body() -> >> + Remote = fun(Req) -> >> + 'PUT' = Req:get(method), >> + "/passes_body" = Req:get(path), >> + <<"Hooray!">> = Req:recv_body(), >> + {ok, {201, [], "ok"}} >> + end, >> + Local = fun({ok, "201", _, "ok"}) -> true; (_) -> false end, >> + Req = #req{ >> + method=put, >> + path="passes_body", >> + body="Hooray!" >> + }, >> + check_request("Passes body", Req, Remote, Local). >> + >> +test_passes_eof_body_back() -> >> + BodyChunks = [<<"foo">>, <<"bar">>, <<"bazinga">>], >> + Remote = fun(Req) -> >> + 'GET' = Req:get(method), >> + "/passes_eof_body" = Req:get(path), >> + {raw, {200, [{"Connection", "close"}], BodyChunks}} >> + end, >> + Local = fun({ok, "200", _, "foobarbazinga"}) -> true; (_) -> false end, >> + Req = #req{path="passes_eof_body"}, >> + check_request("Passes eof body", Req, Remote, Local). >> + >> +test_passes_chunked_body() -> >> + BodyChunks = [<<"foo">>, <<"bar">>, <<"bazinga">>], >> + Remote = fun(Req) -> >> + 'POST' = Req:get(method), >> + "/passes_chunked_body" = Req:get(path), >> + RecvBody = fun >> + ({Length, Chunk}, [Chunk | Rest]) -> >> + Length = size(Chunk), >> + Rest; >> + ({0, []}, []) -> >> + ok >> + end, >> + ok = Req:stream_body(1024*1024, RecvBody, BodyChunks), >> + {ok, {201, [], "ok"}} >> + end, >> + Local = fun({ok, "201", _, "ok"}) -> true; (_) -> false end, >> + Req = #req{ >> + method=post, >> + path="passes_chunked_body", >> + headers=[{"Transfer-Encoding", "chunked"}], >> + body=mk_chunked_body(BodyChunks) >> + }, >> + check_request("Passes chunked body", Req, Remote, Local). >> + >> +test_passes_chunked_body_back() -> >> + Name = "Passes chunked body back", >> + Remote = fun(Req) -> >> + 'GET' = Req:get(method), >> + "/passes_chunked_body_back" = Req:get(path), >> + BodyChunks = [<<"foo">>, <<"bar">>, <<"bazinga">>], >> + {chunked, {200, [{"Transfer-Encoding", "chunked"}], BodyChunks}} >> + end, >> + Req = #req{ >> + path="passes_chunked_body_back", >> + opts=[{stream_to, self()}] >> + }, >> + >> + Resp = check_request(Name, Req, Remote, no_local), >> + >> + etap:fun_is( >> + fun({ibrowse_req_id, _}) -> true; (_) -> false end, >> + Resp, >> + "Received an ibrowse request id." >> + ), >> + {_, ReqId} = Resp, >> + >> + % Grab headers from response >> + receive >> + {ibrowse_async_headers, ReqId, "200", Headers} -> >> + etap:is( >> + proplists:get_value("Transfer-Encoding", Headers), >> + "chunked", >> + "Response included the Transfer-Encoding: chunked header" >> + ), >> + ibrowse:stream_next(ReqId) >> + after 1000 -> >> + throw({error, timeout}) >> + end, >> + >> + % Check body received >> + % TODO: When we upgrade to ibrowse >= 2.0.0 this check needs to >> + % check that the chunks returned are what we sent from the >> + % Remote test. >> + etap:diag("TODO: UPGRADE IBROWSE"), >> + etap:is(recv_body(ReqId, []), <<"foobarbazinga">>, "Decoded chunked >> body."), >> + >> + % Check test_web server. >> + etap:is(test_web:check_last(), was_ok, Name ++ " - request handled"). >> + >> +test_connect_error() -> >> + Local = fun({ok, "500", _Headers, _Body}) -> true; (_) -> false end, >> + Req = #req{opts=[{url, "http://127.0.0.1:5984/_error"}]}, >> + check_request("Connect error", Req, no_remote, Local). >> + >> + >> +mk_chunked_body(Chunks) -> >> + mk_chunked_body(Chunks, []). >> + >> +mk_chunked_body([], Acc) -> >> + iolist_to_binary(lists:reverse(Acc, "0\r\n\r\n")); >> +mk_chunked_body([Chunk | Rest], Acc) -> >> + Size = to_hex(size(Chunk)), >> + mk_chunked_body(Rest, ["\r\n", Chunk, "\r\n", Size | Acc]). >> + >> +to_hex(Val) -> >> + to_hex(Val, []). >> + >> +to_hex(0, Acc) -> >> + Acc; >> +to_hex(Val, Acc) -> >> + to_hex(Val div 16, [hex_char(Val rem 16) | Acc]). >> + >> +hex_char(V) when V < 10 -> $0 + V; >> +hex_char(V) -> $A + V - 10. >> + >> +recv_body(ReqId, Acc) -> >> + receive >> + {ibrowse_async_response, ReqId, Data} -> >> + recv_body(ReqId, [Data | Acc]); >> + {ibrowse_async_response_end, ReqId} -> >> + iolist_to_binary(lists:reverse(Acc)); >> + Else -> >> + throw({error, unexpected_mesg, Else}) >> + after 5000 -> >> + throw({error, timeout}) >> + end. >> >> Modified: couchdb/trunk/test/etap/Makefile.am >> URL: >> http://svn.apache.org/viewvc/couchdb/trunk/test/etap/Makefile.am?rev=1031877&r1=1031876&r2=1031877&view=diff >> ============================================================================== >> --- couchdb/trunk/test/etap/Makefile.am (original) >> +++ couchdb/trunk/test/etap/Makefile.am Fri Nov 5 23:26:21 2010 >> @@ -11,7 +11,7 @@ >> ## the License. >> >> noinst_SCRIPTS = run >> -noinst_DATA = test_util.beam >> +noinst_DATA = test_util.beam test_web.beam >> >> %.beam: %.erl >> $(ERLC) $< >> @@ -27,6 +27,7 @@ DISTCLEANFILES = temp.* >> >> EXTRA_DIST = \ >> run.tpl \ >> + test_web.erl \ >> 001-load.t \ >> 002-icu-driver.t \ >> 010-file-basics.t \ >> @@ -77,4 +78,6 @@ EXTRA_DIST = \ >> 172-os-daemon-errors.4.es \ >> 172-os-daemon-errors.t \ >> 173-os-daemon-cfg-register.es \ >> - 173-os-daemon-cfg-register.t >> + 173-os-daemon-cfg-register.t \ >> + 180-http-proxy.ini \ >> + 180-http-proxy.t >> >> Added: couchdb/trunk/test/etap/test_web.erl >> URL: >> http://svn.apache.org/viewvc/couchdb/trunk/test/etap/test_web.erl?rev=1031877&view=auto >> ============================================================================== >> --- couchdb/trunk/test/etap/test_web.erl (added) >> +++ couchdb/trunk/test/etap/test_web.erl Fri Nov 5 23:26:21 2010 >> @@ -0,0 +1,99 @@ >> +% Licensed under the Apache License, Version 2.0 (the "License"); you may >> not >> +% use this file except in compliance with the License. You may obtain a >> copy of >> +% the License at >> +% >> +% http://www.apache.org/licenses/LICENSE-2.0 >> +% >> +% Unless required by applicable law or agreed to in writing, software >> +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT >> +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the >> +% License for the specific language governing permissions and limitations >> under >> +% the License. >> + >> +-module(test_web). >> +-behaviour(gen_server). >> + >> +-export([start_link/0, loop/1, get_port/0, set_assert/1, check_last/0]). >> +-export([init/1, terminate/2, code_change/3]). >> +-export([handle_call/3, handle_cast/2, handle_info/2]). >> + >> +-define(SERVER, test_web_server). >> +-define(HANDLER, test_web_handler). >> + >> +start_link() -> >> + gen_server:start({local, ?HANDLER}, ?MODULE, [], []), >> + mochiweb_http:start([ >> + {name, ?SERVER}, >> + {loop, {?MODULE, loop}}, >> + {port, 5985} >> + ]). >> + >> +loop(Req) -> >> + %etap:diag("Handling request: ~p", [Req]), >> + case gen_server:call(?HANDLER, {check_request, Req}) of >> + {ok, RespInfo} -> >> + {ok, Req:respond(RespInfo)}; >> + {raw, {Status, Headers, BodyChunks}} -> >> + Resp = Req:start_response({Status, Headers}), >> + lists:foreach(fun(C) -> Resp:send(C) end, BodyChunks), >> + erlang:put(mochiweb_request_force_close, true), >> + {ok, Resp}; >> + {chunked, {Status, Headers, BodyChunks}} -> >> + Resp = Req:respond({Status, Headers, chunked}), >> + timer:sleep(500), >> + lists:foreach(fun(C) -> Resp:write_chunk(C) end, BodyChunks), >> + Resp:write_chunk([]), >> + {ok, Resp}; >> + {error, Reason} -> >> + etap:diag("Error: ~p", [Reason]), >> + Body = lists:flatten(io_lib:format("Error: ~p", [Reason])), >> + {ok, Req:respond({200, [], Body})} >> + end. >> + >> +get_port() -> >> + mochiweb_socket_server:get(?SERVER, port). >> + >> +set_assert(Fun) -> >> + ok = gen_server:call(?HANDLER, {set_assert, Fun}). >> + >> +check_last() -> >> + gen_server:call(?HANDLER, last_status). >> + >> +init(_) -> >> + {ok, nil}. >> + >> +terminate(_Reason, _State) -> >> + ok. >> + >> +handle_call({check_request, Req}, _From, State) when is_function(State, 1) >> -> >> + Resp2 = case (catch State(Req)) of >> + {ok, Resp} -> {reply, {ok, Resp}, was_ok}; >> + {raw, Resp} -> {reply, {raw, Resp}, was_ok}; >> + {chunked, Resp} -> {reply, {chunked, Resp}, was_ok}; >> + Error -> {reply, {error, Error}, not_ok} >> + end, >> + Req:cleanup(), >> + Resp2; >> +handle_call({check_request, _Req}, _From, _State) -> >> + {reply, {error, no_assert_function}, not_ok}; >> +handle_call(last_status, _From, State) when is_atom(State) -> >> + {reply, State, nil}; >> +handle_call(last_status, _From, State) -> >> + {reply, {error, not_checked}, State}; >> +handle_call({set_assert, Fun}, _From, nil) -> >> + {reply, ok, Fun}; >> +handle_call({set_assert, _}, _From, State) -> >> + {reply, {error, assert_function_set}, State}; >> +handle_call(Msg, _From, State) -> >> + {reply, {ignored, Msg}, State}. >> + >> +handle_cast(Msg, State) -> >> + etap:diag("Ignoring cast message: ~p", [Msg]), >> + {noreply, State}. >> + >> +handle_info(Msg, State) -> >> + etap:diag("Ignoring info message: ~p", [Msg]), >> + {noreply, State}. >> + >> +code_change(_OldVsn, State, _Extra) -> >> + {ok, State}. >> >> >> > > > > -- > Filipe David Manana, > [email protected], [email protected] > > "Reasonable men adapt themselves to the world. > Unreasonable men adapt the world to themselves. > That's why all progress depends on unreasonable men." >
