eiri closed pull request #1237: [WIP] Re-introduce etags in view
URL: https://github.com/apache/couchdb/pull/1237
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/src/chttpd/src/chttpd_db.erl b/src/chttpd/src/chttpd_db.erl
index ed0adead92..0e1495d95d 100644
--- a/src/chttpd/src/chttpd_db.erl
+++ b/src/chttpd/src/chttpd_db.erl
@@ -682,7 +682,11 @@ multi_all_docs_view(Req, Db, OP, Queries) ->
all_docs_view(Req, Db, Keys, OP) ->
Args0 = couch_mrview_http:parse_params(Req, Keys),
- Args1 = Args0#mrargs{view_type=map},
+ ETagFun = fun(Sig, Acc) ->
+ ETag = couch_httpd:make_etag(Sig),
+ {ok, Acc#vacc{etag=ETag}}
+ end,
+ Args1 = Args0#mrargs{view_type=map, preflight_fun=ETagFun},
Args2 = couch_mrview_util:validate_args(Args1),
Args3 = set_namespace(OP, Args2),
Options = [{user_ctx, Req#httpd.user_ctx}],
diff --git a/src/chttpd/src/chttpd_show.erl b/src/chttpd/src/chttpd_show.erl
index c6d232c969..3a97b87b07 100644
--- a/src/chttpd/src/chttpd_show.erl
+++ b/src/chttpd/src/chttpd_show.erl
@@ -202,7 +202,12 @@ handle_view_list(Req, Db, DDoc, LName, {ViewDesignName,
ViewName}, Keys) ->
DbName = couch_db:name(Db),
{ok, VDoc} = ddoc_cache:open(DbName, <<"_design/",
ViewDesignName/binary>>),
CB = fun couch_mrview_show:list_cb/2,
- QueryArgs = couch_mrview_http:parse_params(Req, Keys),
+ QueryArgs0 = couch_mrview_http:parse_params(Req, Keys),
+ ETagFun = fun(Sig, Acc) ->
+ ETag = couch_httpd:make_etag(Sig),
+ {ok, Acc#vacc{etag=ETag}}
+ end,
+ QueryArgs = QueryArgs0#mrargs{preflight_fun=ETagFun},
Options = [{user_ctx, Req#httpd.user_ctx}],
couch_query_servers:with_ddoc_proc(DDoc, fun(QServer) ->
Acc = #lacc{
diff --git a/src/chttpd/src/chttpd_view.erl b/src/chttpd/src/chttpd_view.erl
index 3c05c64ca7..c3781a1528 100644
--- a/src/chttpd/src/chttpd_view.erl
+++ b/src/chttpd/src/chttpd_view.erl
@@ -41,7 +41,12 @@ multi_query_view(Req, Db, DDoc, ViewName, Queries) ->
design_doc_view(Req, Db, DDoc, ViewName, Keys) ->
- Args = couch_mrview_http:parse_params(Req, Keys),
+ Args0 = couch_mrview_http:parse_params(Req, Keys),
+ ETagFun = fun(Sig, Acc) ->
+ ETag = couch_httpd:make_etag(Sig),
+ {ok, Acc#vacc{etag=ETag}}
+ end,
+ Args = Args0#mrargs{preflight_fun=ETagFun},
Max = chttpd:chunked_response_buffer_size(),
VAcc = #vacc{db=Db, req=Req, threshold=Max},
Options = [{user_ctx, Req#httpd.user_ctx}],
diff --git a/src/couch_mrview/src/couch_mrview.erl
b/src/couch_mrview/src/couch_mrview.erl
index a099f377ec..9647e97c05 100644
--- a/src/couch_mrview/src/couch_mrview.erl
+++ b/src/couch_mrview/src/couch_mrview.erl
@@ -216,7 +216,9 @@ query_all_docs(Db, Args, Callback, Acc) when is_list(Args)
->
query_all_docs(Db, to_mrargs(Args), Callback, Acc);
query_all_docs(Db, Args0, Callback, Acc) ->
Sig = couch_util:with_db(Db, fun(WDb) ->
- {ok, Info} = couch_db:get_db_info(WDb),
+ {ok, Info0} = couch_db:get_db_info(WDb),
+ Keys = [db_name, doc_count, doc_del_count, update_seq, purge_seq],
+ Info = lists:filter(fun({K, _}) -> lists:member(K, Keys) end, Info0),
couch_index_util:hexsig(crypto:hash(md5, term_to_binary(Info)))
end),
Args1 = Args0#mrargs{view_type=map},
diff --git a/src/couch_mrview/src/couch_mrview_http.erl
b/src/couch_mrview/src/couch_mrview_http.erl
index 004caef09f..2cdf568062 100644
--- a/src/couch_mrview/src/couch_mrview_http.erl
+++ b/src/couch_mrview/src/couch_mrview_http.erl
@@ -344,12 +344,23 @@ view_cb(complete, #vacc{resp=undefined}=Acc) ->
{ok, Resp} = chttpd:send_json(Acc#vacc.req, 200, {[{rows, []}]}),
{ok, Acc#vacc{resp=Resp}};
-view_cb(Msg, #vacc{resp=undefined}=Acc) ->
+view_cb(Msg, #vacc{resp=undefined, etag=undefined}=Acc) ->
%% Start response
Headers = [],
{ok, Resp} = chttpd:start_delayed_json_response(Acc#vacc.req, 200,
Headers),
view_cb(Msg, Acc#vacc{resp=Resp, should_close=true});
+view_cb(Msg, #vacc{req=Req, resp=undefined, etag=ETag}=Acc) ->
+ Headers = [{"ETag", ETag}, {<<"Vary">>, <<"Accept">>}],
+ case chttpd:etag_match(Req, ETag) of
+ true ->
+ {ok, Resp} = chttpd:send_response(Req, 304, Headers, <<>>),
+ {stop, Acc#vacc{resp=Resp}};
+ false ->
+ {ok, Resp} = chttpd:start_delayed_json_response(Req, 200, Headers),
+ view_cb(Msg, Acc#vacc{resp=Resp, should_close=true})
+ end;
+
%% ---------------------------------------------------
%% From here on down, the response has been started.
diff --git a/src/couch_mrview/src/couch_mrview_show.erl
b/src/couch_mrview/src/couch_mrview_show.erl
index 2411c2ca2b..a33fcb40f5 100644
--- a/src/couch_mrview/src/couch_mrview_show.erl
+++ b/src/couch_mrview/src/couch_mrview_show.erl
@@ -244,13 +244,20 @@ list_cb(complete, Acc) ->
{ok, Resp2}.
start_list_resp(Head, Acc) ->
- #lacc{db=Db, req=Req, qserver=QServer, lname=LName} = Acc,
- JsonReq = json_req_obj(Req, Db),
-
- [<<"start">>,Chunk,JsonResp] =
couch_query_servers:ddoc_proc_prompt(QServer,
- [<<"lists">>, LName], [Head, JsonReq]),
- Acc2 = send_non_empty_chunk(fixup_headers(JsonResp, Acc), Chunk),
- {ok, Acc2}.
+ #lacc{db=Db, req=Req, etag=ETag, qserver=QServer, lname=LName} = Acc,
+ case chttpd:etag_match(Req, ETag) of
+ true ->
+ Headers = [{"ETag", ETag}, {<<"Vary">>, <<"Accept">>}],
+ {ok, Resp} = chttpd:send_response(Req, 304, Headers, <<>>),
+ {stop, Resp};
+ false ->
+ JsonReq = json_req_obj(Req, Db),
+ [<<"start">>, Chunk, JsonResp]
+ = couch_query_servers:ddoc_proc_prompt(QServer,
+ [<<"lists">>, LName], [Head, JsonReq]),
+ Acc2 = send_non_empty_chunk(fixup_headers(JsonResp, Acc), Chunk),
+ {ok, Acc2}
+ end.
fixup_headers(Headers, #lacc{etag=ETag} = Acc) ->
Headers2 = apply_etag(Headers, ETag),
diff --git a/src/fabric/include/fabric.hrl b/src/fabric/include/fabric.hrl
index be1d63926b..77028ea430 100644
--- a/src/fabric/include/fabric.hrl
+++ b/src/fabric/include/fabric.hrl
@@ -31,7 +31,8 @@
lang,
sorted,
user_acc,
- update_seq
+ update_seq,
+ etag = []
}).
-record(stream_acc, {
diff --git a/src/fabric/src/fabric_rpc.erl b/src/fabric/src/fabric_rpc.erl
index 4a69e7ea16..5df93400b9 100644
--- a/src/fabric/src/fabric_rpc.erl
+++ b/src/fabric/src/fabric_rpc.erl
@@ -302,9 +302,9 @@ get_or_create_db(DbName, Options) ->
couch_db:open_int(DbName, [{create_if_missing, true} | Options]).
-view_cb({meta, Meta}, Acc) ->
+view_cb({meta, Meta}, #vacc{etag = ETag} = Acc) ->
% Map function starting
- ok = rexi:stream2({meta, Meta}),
+ ok = rexi:stream2([{meta, Meta}, {etag, ETag}]),
{ok, Acc};
view_cb({row, Row}, Acc) ->
% Adding another row
@@ -324,9 +324,9 @@ view_cb(ok, ddoc_updated) ->
rexi:reply({ok, ddoc_updated}).
-reduce_cb({meta, Meta}, Acc) ->
+reduce_cb({meta, Meta}, #vacc{etag = ETag} = Acc) ->
% Map function starting
- ok = rexi:stream2({meta, Meta}),
+ ok = rexi:stream2([{meta, Meta}, {etag, ETag}]),
{ok, Acc};
reduce_cb({row, Row}, Acc) ->
% Adding another row
diff --git a/src/fabric/src/fabric_util.erl b/src/fabric/src/fabric_util.erl
index dd4b80da60..b9d6c5786e 100644
--- a/src/fabric/src/fabric_util.erl
+++ b/src/fabric/src/fabric_util.erl
@@ -19,6 +19,7 @@
-export([stream_start/2, stream_start/4]).
-export([log_timeout/2, remove_done_workers/2]).
-export([is_users_db/1, is_replicator_db/1, fake_db/2]).
+-export([make_etag/1, maybe_set_etag/2]).
-export([upgrade_mrargs/1]).
-compile({inline, [{doc_id_and_rev,1}]}).
@@ -308,6 +309,23 @@ fake_db(DbName, Opts) ->
{ok, Db} = couch_db:clustered_db(DbName, UserCtx, SecProps),
Db.
+maybe_set_etag(undefined, Acc) ->
+ {ok, Acc};
+maybe_set_etag(ETag, #vacc{} = Acc) ->
+ {ok, Acc#vacc{etag = ETag}};
+maybe_set_etag(ETag, #lacc{} = Acc) ->
+ {ok, Acc#lacc{etag = ETag}};
+maybe_set_etag(_, Acc) ->
+ {ok, Acc}.
+
+make_etag([]) ->
+ undefined;
+make_etag(ETags) ->
+ case lists:member(undefined, ETags) of
+ true -> undefined;
+ false -> ?b2l(chttpd:make_etag(lists:usort(ETags)))
+ end.
+
%% test function
kv(Item, Count) ->
{make_key(Item), {Item,Count}}.
@@ -374,3 +392,40 @@ upgrade_mrargs({mrargs,
sorted = Sorted,
extra = Extra
}.
+
+
+-ifdef(TEST).
+-include_lib("eunit/include/eunit.hrl").
+
+make_etag_test() ->
+ ETagsEmpty = [],
+ ETagsUndef1 = [undefined],
+ ETagsUndef2 = [chttpd:make_etag(a), undefined],
+ ETags1 = [
+ <<"\"2F332B4YM6IYB0S4TL61WB07T\"">>,
+ <<"\"AVHZC4PRSDE1I6C13FMROYFER\"">>,
+ <<"\"D7Q054EV4MVV6G57TRLUVQUNG\"">>,
+ <<"\"CT8FCP4VF9AHGS75L255B4TAN\"">>,
+ <<"\"8WRGVGKV4IQWH5JELHHBD8SCP\"">>,
+ <<"\"1YTAPUW4HXKHL3JO3Y487O5H\"">>,
+ <<"\"189S9QEX2P5CG3XSYHX0DMIUM\"">>,
+ <<"\"DXK436MEITBQGE9CY61MH6OFD\"">>
+ ],
+ ETags2 = [
+ <<"\"189S9QEX2P5CG3XSYHX0DMIUM\"">>,
+ <<"\"2F332B4YM6IYB0S4TL61WB07T\"">>,
+ <<"\"8WRGVGKV4IQWH5JELHHBD8SCP\"">>,
+ <<"\"D7Q054EV4MVV6G57TRLUVQUNG\"">>,
+ <<"\"DXK436MEITBQGE9CY61MH6OFD\"">>,
+ <<"\"CT8FCP4VF9AHGS75L255B4TAN\"">>,
+ <<"\"AVHZC4PRSDE1I6C13FMROYFER\"">>,
+ <<"\"1YTAPUW4HXKHL3JO3Y487O5H\"">>
+ ],
+ [
+ ?assertEqual(undefined, make_etag(ETagsEmpty)),
+ ?assertEqual(undefined, make_etag(ETagsUndef1)),
+ ?assertEqual(undefined, make_etag(ETagsUndef2)),
+ ?assertEqual(make_etag(ETags1), make_etag(ETags2))
+ ].
+
+-endif.
diff --git a/src/fabric/src/fabric_view_all_docs.erl
b/src/fabric/src/fabric_view_all_docs.erl
index ac16dac526..7782c6ca6a 100644
--- a/src/fabric/src/fabric_view_all_docs.erl
+++ b/src/fabric/src/fabric_view_all_docs.erl
@@ -143,6 +143,9 @@ handle_message({rexi_EXIT, Reason}, Worker, State) ->
fabric_view:handle_worker_exit(State, Worker, Reason);
handle_message({meta, Meta0}, {Worker, From}, State) ->
+ handle_message([{meta, Meta0}, {etag, undefined}], {Worker, From}, State);
+
+handle_message([{meta, Meta0}, {etag, ETag0}], {Worker, From}, State) ->
Tot = couch_util:get_value(total, Meta0, 0),
Off = couch_util:get_value(offset, Meta0, 0),
Seq = couch_util:get_value(update_seq, Meta0, 0),
@@ -151,8 +154,9 @@ handle_message({meta, Meta0}, {Worker, From}, State) ->
counters = Counters0,
total_rows = Total0,
offset = Offset0,
- user_acc = AccIn,
- update_seq = UpdateSeq0
+ user_acc = AccIn0,
+ update_seq = UpdateSeq0,
+ etag = ETags0
} = State,
% Assert that we don't have other messages from this
% worker when the total_and_offset message arrives.
@@ -166,13 +170,15 @@ handle_message({meta, Meta0}, {Worker, From}, State) ->
{_, null} -> null;
_ -> [{Worker, Seq} | UpdateSeq0]
end,
+ ETags = [ETag0 | ETags0],
case fabric_dict:any(0, Counters1) of
true ->
{ok, State#collector{
counters = Counters1,
total_rows = Total,
update_seq = UpdateSeq,
- offset = Offset
+ offset = Offset,
+ etag = ETags
}};
false ->
FinalOffset = case Offset of
@@ -188,11 +194,14 @@ handle_message({meta, Meta0}, {Worker, From}, State) ->
_ ->
[{update_seq, fabric_view_changes:pack_seqs(UpdateSeq)}]
end,
+ ETag = fabric_util:make_etag(ETags),
+ {ok, AccIn} = fabric_util:maybe_set_etag(ETag, AccIn0),
{Go, Acc} = Callback({meta, Meta}, AccIn),
{Go, State#collector{
counters = fabric_dict:decrement_all(Counters1),
total_rows = Total,
offset = FinalOffset,
+ etag = ETag,
user_acc = Acc,
update_seq = UpdateSeq0
}}
diff --git a/src/fabric/src/fabric_view_map.erl
b/src/fabric/src/fabric_view_map.erl
index b6a3d6f837..09b20a8bf4 100644
--- a/src/fabric/src/fabric_view_map.erl
+++ b/src/fabric/src/fabric_view_map.erl
@@ -95,6 +95,9 @@ handle_message({rexi_EXIT, Reason}, Worker, State) ->
fabric_view:handle_worker_exit(State, Worker, Reason);
handle_message({meta, Meta0}, {Worker, From}, State) ->
+ handle_message([{meta, Meta0}, {etag, undefined}], {Worker, From}, State);
+
+handle_message([{meta, Meta0}, {etag, ETag0}], {Worker, From}, State) ->
Tot = couch_util:get_value(total, Meta0, 0),
Off = couch_util:get_value(offset, Meta0, 0),
Seq = couch_util:get_value(update_seq, Meta0, 0),
@@ -103,8 +106,9 @@ handle_message({meta, Meta0}, {Worker, From}, State) ->
counters = Counters0,
total_rows = Total0,
offset = Offset0,
- user_acc = AccIn,
- update_seq = UpdateSeq0
+ user_acc = AccIn0,
+ update_seq = UpdateSeq0,
+ etag = ETags0
} = State,
% Assert that we don't have other messages from this
% worker when the total_and_offset message arrives.
@@ -117,13 +121,15 @@ handle_message({meta, Meta0}, {Worker, From}, State) ->
nil -> nil;
_ -> [{Worker, Seq} | UpdateSeq0]
end,
+ ETags = [ETag0 | ETags0],
case fabric_dict:any(0, Counters1) of
true ->
{ok, State#collector{
counters = Counters1,
total_rows = Total,
update_seq = UpdateSeq,
- offset = Offset
+ offset = Offset,
+ etag = ETags
}};
false ->
FinalOffset = erlang:min(Total, Offset+State#collector.skip),
@@ -134,11 +140,14 @@ handle_message({meta, Meta0}, {Worker, From}, State) ->
_ ->
[{update_seq, fabric_view_changes:pack_seqs(UpdateSeq)}]
end,
+ ETag = fabric_util:make_etag(ETags),
+ {ok, AccIn} = fabric_util:maybe_set_etag(ETag, AccIn0),
{Go, Acc} = Callback({meta, Meta}, AccIn),
{Go, State#collector{
counters = fabric_dict:decrement_all(Counters1),
total_rows = Total,
offset = FinalOffset,
+ etag = ETag,
user_acc = Acc
}}
end;
diff --git a/src/fabric/src/fabric_view_reduce.erl
b/src/fabric/src/fabric_view_reduce.erl
index a74be10733..022b346d25 100644
--- a/src/fabric/src/fabric_view_reduce.erl
+++ b/src/fabric/src/fabric_view_reduce.erl
@@ -105,12 +105,16 @@ handle_message({rexi_EXIT, Reason}, Worker, State) ->
fabric_view:handle_worker_exit(State, Worker, Reason);
handle_message({meta, Meta0}, {Worker, From}, State) ->
+ handle_message([{meta, Meta0}, {etag, undefined}], {Worker, From}, State);
+
+handle_message([{meta, Meta0}, {etag, ETag0}], {Worker, From}, State) ->
Seq = couch_util:get_value(update_seq, Meta0, 0),
#collector{
callback = Callback,
counters = Counters0,
- user_acc = AccIn,
- update_seq = UpdateSeq0
+ user_acc = AccIn0,
+ update_seq = UpdateSeq0,
+ etag = ETags0
} = State,
% Assert that we don't have other messages from this
% worker when the total_and_offset message arrives.
@@ -121,11 +125,13 @@ handle_message({meta, Meta0}, {Worker, From}, State) ->
nil -> nil;
_ -> [{Worker, Seq} | UpdateSeq0]
end,
+ ETags = [ETag0 | ETags0],
case fabric_dict:any(0, Counters1) of
true ->
{ok, State#collector{
counters = Counters1,
- update_seq = UpdateSeq
+ update_seq = UpdateSeq,
+ etag = ETags
}};
false ->
Meta = case UpdateSeq of
@@ -134,9 +140,12 @@ handle_message({meta, Meta0}, {Worker, From}, State) ->
_ ->
[{update_seq, fabric_view_changes:pack_seqs(UpdateSeq)}]
end,
+ ETag = fabric_util:make_etag(ETags),
+ {ok, AccIn} = fabric_util:maybe_set_etag(ETag, AccIn0),
{Go, Acc} = Callback({meta, Meta}, AccIn),
{Go, State#collector{
counters = fabric_dict:decrement_all(Counters1),
+ etag = ETag,
user_acc = Acc
}}
end;
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
