Author: fdmanana
Date: Mon Sep 6 23:08:17 2010
New Revision: 993173
URL: http://svn.apache.org/viewvc?rev=993173&view=rev
Log:
New replicator: use the API "GET /db/doc?open_revs=all" to fetch the documents
for a replication by doc ID list.
Modified:
couchdb/branches/new_replicator/src/couchdb/couch_api_wrap.erl
couchdb/branches/new_replicator/src/couchdb/couch_doc.erl
couchdb/branches/new_replicator/src/couchdb/couch_httpd.erl
couchdb/branches/new_replicator/src/couchdb/couch_replicate.erl
Modified: couchdb/branches/new_replicator/src/couchdb/couch_api_wrap.erl
URL:
http://svn.apache.org/viewvc/couchdb/branches/new_replicator/src/couchdb/couch_api_wrap.erl?rev=993173&r1=993172&r2=993173&view=diff
==============================================================================
--- couchdb/branches/new_replicator/src/couchdb/couch_api_wrap.erl (original)
+++ couchdb/branches/new_replicator/src/couchdb/couch_api_wrap.erl Mon Sep 6
23:08:17 2010
@@ -41,7 +41,6 @@
ensure_full_commit/1,
get_missing_revs/2,
open_doc/3,
- open_doc/4,
open_doc_revs/6,
update_doc/4,
changes_since/5,
@@ -162,10 +161,15 @@ get_missing_revs(Db, IdRevs) ->
open_doc_revs(#httpdb{} = HttpDb, Id, Revs, Options, Fun, Acc) ->
+ RevStr = case Revs of
+ all ->
+ "all";
+ _ ->
+ ?JSON_ENCODE(couch_doc:revs_to_strs(Revs))
+ end,
Self = self(),
QArgs = [
- {"revs", "true"},
- {"open_revs", ?JSON_ENCODE(couch_doc:revs_to_strs(Revs))} |
+ {"revs", "true"}, {"open_revs", RevStr} |
options_to_query_args(Options, [])
],
Streamer = spawn_link(fun() ->
@@ -187,59 +191,16 @@ open_doc_revs(Db, Id, Revs, Options, Fun
{ok, Results} = couch_db:open_doc_revs(Db, Id, Revs, Options),
{ok, lists:foldl(Fun, Acc, Results)}.
-open_doc(Db, Id, Options, Fun) ->
- Fun(open_doc(Db, Id, Options)).
-open_doc(#httpdb{} = HttpDb, Id, Options) ->
- QArgs = [
- {"attachments", "true"},
- {"revs", "true"} |
- options_to_query_args(Options, [])
- ],
- Self = self(),
- Streamer = spawn_link(fun() ->
- send_req(
- HttpDb,
- [{headers, [{"accept", "application/json,
multipart/related"}]},
- {path, encode_doc_id(Id)}, {qs, QArgs},
- {ibrowse_options, [{stream_to, {self(), once}}]}],
- fun(Code, Headers, StreamDataFun) ->
- CType = get_value("Content-Type", Headers),
- Self ! {self(), CType},
- case CType of
- "application/json" ->
- receive
- {get, From} ->
- EJson = json_stream_parse:to_ejson(StreamDataFun),
- case Code of
- 200 ->
- Doc = couch_doc:from_json_obj(EJson),
- From ! {data, self(), Doc};
- _ErrorCode ->
- From ! {data, self(), EJson}
- end
- end;
- "multipart/related;" ++ _ ->
- couch_httpd:parse_multipart_request(
- CType,
- StreamDataFun,
- fun(Ev)-> couch_doc:mp_parse_doc(Ev, []) end)
- end
- end),
- unlink(Self)
- end),
- receive
- {Streamer, "application/json"} ->
- Streamer ! {get, self()},
- receive
- {data, Streamer, #doc{} = Doc} ->
- {ok, Doc};
- {data, Streamer, Error} ->
- Error
- end;
- {Streamer, "multipart/related;" ++ _} ->
- couch_doc:doc_from_multi_part_stream(Streamer)
- end;
+open_doc(#httpdb{} = Db, Id, Options) ->
+ send_req(
+ Db,
+ [{path, encode_doc_id(Id)}, {qs, options_to_query_args(Options, [])}],
+ fun(200, _, Body) ->
+ {ok, couch_doc:from_json_obj(Body)};
+ (_, _, {Props}) ->
+ {error, get_value(<<"error">>, Props)}
+ end);
open_doc(Db, Id, Options) ->
couch_db:open_doc(Db, Id, Options).
Modified: couchdb/branches/new_replicator/src/couchdb/couch_doc.erl
URL:
http://svn.apache.org/viewvc/couchdb/branches/new_replicator/src/couchdb/couch_doc.erl?rev=993173&r1=993172&r2=993173&view=diff
==============================================================================
--- couchdb/branches/new_replicator/src/couchdb/couch_doc.erl (original)
+++ couchdb/branches/new_replicator/src/couchdb/couch_doc.erl Mon Sep 6
23:08:17 2010
@@ -16,9 +16,8 @@
-export([att_foldl/3,att_foldl_decode/3,get_validate_doc_fun/1]).
-export([from_json_obj/1,to_json_obj/2,has_stubs/1, merge_stubs/2]).
-export([validate_docid/1]).
--export([doc_from_multi_part_stream/1, doc_from_multi_part_stream/2]).
+-export([doc_from_multi_part_stream/2]).
-export([doc_to_multi_part_stream/5, len_doc_to_multi_part_stream/4]).
--export([mp_parse_doc/2]).
-include("couch_db.hrl").
@@ -442,7 +441,13 @@ atts_to_mp([Att | RestAtts], Boundary, W
atts_to_mp(RestAtts, Boundary, WriteFun, SendEncodedAtts).
-doc_from_multi_part_stream(Parser) ->
+doc_from_multi_part_stream(ContentType, DataFun) ->
+ Self = self(),
+ Parser = spawn_link(fun() ->
+ couch_httpd:parse_multipart_request(ContentType, DataFun,
+ fun(Next)-> mp_parse_doc(Next, []) end),
+ unlink(Self)
+ end),
Parser ! {get_doc_bytes, self()},
receive
{doc_bytes, DocBytes} ->
@@ -462,15 +467,6 @@ doc_from_multi_part_stream(Parser) ->
{ok, Doc#doc{atts=Atts2}}
end.
-doc_from_multi_part_stream(ContentType, DataFun) ->
- Self = self(),
- Parser = spawn_link(fun() ->
- couch_httpd:parse_multipart_request(ContentType, DataFun,
- fun(Next)-> mp_parse_doc(Next, []) end),
- unlink(Self)
- end),
- doc_from_multi_part_stream(Parser).
-
mp_parse_doc({headers, H}, []) ->
case couch_util:get_value("content-type", H) of
{"application/json", _} ->
Modified: couchdb/branches/new_replicator/src/couchdb/couch_httpd.erl
URL:
http://svn.apache.org/viewvc/couchdb/branches/new_replicator/src/couchdb/couch_httpd.erl?rev=993173&r1=993172&r2=993173&view=diff
==============================================================================
--- couchdb/branches/new_replicator/src/couchdb/couch_httpd.erl (original)
+++ couchdb/branches/new_replicator/src/couchdb/couch_httpd.erl Mon Sep 6
23:08:17 2010
@@ -823,17 +823,20 @@ read_until(#mp{data_fun=DataFun, buffer=
end.
-parse_part_header(#mp{callback=UserCallBack}=Mp) ->
- {Mp2, AccCallback} = read_until(Mp, <<"\r\n\r\n">>,
- fun(Next) -> acc_callback(Next, []) end),
- HeaderData = AccCallback(get_data),
-
- Headers =
- lists:foldl(fun(Line, Acc) ->
- split_header(Line) ++ Acc
- end, [], re:split(HeaderData,<<"\r\n">>, [])),
- NextCallback = UserCallBack({headers, Headers}),
- parse_part_body(Mp2#mp{callback=NextCallback}).
+parse_part_header(Mp) ->
+ case check_for_last(Mp) of
+ {last, #mp{callback=UserCallBack} = Mp2} ->
+ Mp2#mp{callback=UserCallBack(eof)};
+ {more, #mp{callback=UserCallBack} = Mp2} ->
+ {Mp3, AccCallback} = read_until(Mp2, <<"\r\n\r\n">>,
+ fun(Next) -> acc_callback(Next, []) end),
+ HeaderData = AccCallback(get_data),
+ Headers = lists:foldl(fun(Line, Acc) ->
+ split_header(Line) ++ Acc
+ end, [], re:split(HeaderData,<<"\r\n">>, [])),
+ NextCallback = UserCallBack({headers, Headers}),
+ parse_part_body(Mp3#mp{callback=NextCallback})
+ end.
parse_part_body(#mp{boundary=Prefix, callback=Callback}=Mp) ->
{Mp2, WrappedCallback} = read_until(Mp, Prefix,
Modified: couchdb/branches/new_replicator/src/couchdb/couch_replicate.erl
URL:
http://svn.apache.org/viewvc/couchdb/branches/new_replicator/src/couchdb/couch_replicate.erl?rev=993173&r1=993172&r2=993173&view=diff
==============================================================================
--- couchdb/branches/new_replicator/src/couchdb/couch_replicate.erl (original)
+++ couchdb/branches/new_replicator/src/couchdb/couch_replicate.erl Mon Sep 6
23:08:17 2010
@@ -618,8 +618,9 @@ doc_copy_loop(CopierId, Cp, Source, Targ
Cp ! {done, CopierId};
{ok, [{doc_id, Id}]} ->
?LOG_DEBUG("Doc copier ~p got {doc_id, ~p}", [CopierId, Id]),
- couch_api_wrap:open_doc(
- Source, Id, [], fun(R) -> doc_handler(R, Target, Cp) end),
+ couch_api_wrap:open_doc_revs(
+ Source, Id, all, [],
+ fun(R, _) -> doc_handler(R, Target, Cp) end, []),
doc_copy_loop(CopierId, Cp, Source, Target, MissingRevsQueue);
{ok, [{Id, Revs, PossibleAncestors, Seq}]} ->
?LOG_DEBUG("Doc copier ~p got {~p, ~p, ~p, ~p}",
