Repository: couchdb-couch-mrview Updated Branches: refs/heads/master 2341c38c6 -> 6ec35971e
Refactor to add an interface for view_state of #mrheader The goal of this refactoring is to make it easier to understand and maintain view_state data structure of #mrheader. Unlike record #mrview in #mrst, the data structure that represents view_states in #mrheader is a tuple (of tuples) and its definition and application spreads across all couch_mrview_util.erl module. This makes it hard to follow and understand how it used or troubleshoot mrview in case of the problems. This change introduces an interface for generation and accessing view_state and also separates generation of reduction closures from mrview's `reduce`, `reduce_to_count` and `fold_reduce` functions as they rely on view_state's structure. Note that this refactoring also removes explicit set of generic ordering function on collation option "raw" as it is a default of couch_btree:open on missing option for `less` function. Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/6ec35971 Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/6ec35971 Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/6ec35971 Branch: refs/heads/master Commit: 6ec35971ebd96133cc6a34fa29f1d364049a7ecd Parents: 2341c38 Author: Eric Avdey <e...@eiri.ca> Authored: Fri Jan 15 16:12:55 2016 -0400 Committer: Eric Avdey <e...@eiri.ca> Committed: Tue Jan 19 11:39:40 2016 -0400 ---------------------------------------------------------------------- src/couch_mrview_util.erl | 235 +++++++++++++++++++++++++---------------- 1 file changed, 143 insertions(+), 92 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/6ec35971/src/couch_mrview_util.erl ---------------------------------------------------------------------- diff --git a/src/couch_mrview_util.erl b/src/couch_mrview_util.erl index c30460c..51e51d8 100644 --- a/src/couch_mrview_util.erl +++ b/src/couch_mrview_util.erl @@ -200,7 +200,7 @@ init_state(Db, Fd, #mrst{views=Views}=State, nil) -> purge_seq=couch_db:get_purge_seq(Db), id_btree_state=nil, log_btree_state=nil, - view_states=[{nil, nil, nil, 0, 0} || _ <- Views] + view_states=[make_view_state(#mrview{}) || _ <- Views] }, init_state(Db, Fd, State, Header); % read <= 1.2.x header record and transpile it to >=1.3.x @@ -215,8 +215,8 @@ init_state(Db, Fd, State, #index_header{ purge_seq=PurgeSeq, id_btree_state=IdBtreeState, log_btree_state=nil, - view_states=[{Bt, nil, nil, USeq, PSeq} || {Bt, USeq, PSeq} <- ViewStates] - }); + view_states=[make_view_state(V) || V <- ViewStates] + }); init_state(Db, Fd, State, Header) -> #mrst{ language=Lang, @@ -232,12 +232,6 @@ init_state(Db, Fd, State, Header) -> view_states=ViewStates } = Header, - StateUpdate = fun - ({_, _, _, _, _}=St) -> St; - (St) -> {St, nil, nil, 0, 0} - end, - ViewStates2 = lists:map(StateUpdate, ViewStates), - IdBtOpts = [{compression, couch_db:compression(Db)}], {ok, IdBtree} = couch_btree:open(IdBtreeState, Fd, IdBtOpts), {ok, LogBtree} = case SeqIndexed orelse KeySeqIndexed of @@ -246,7 +240,7 @@ init_state(Db, Fd, State, Header) -> end, OpenViewFun = fun(St, View) -> open_view(Db, Fd, Lang, St, View) end, - Views2 = lists:zipwith(OpenViewFun, ViewStates2, Views), + Views2 = lists:zipwith(OpenViewFun, ViewStates, Views), State#mrst{ fd=Fd, @@ -258,45 +252,33 @@ init_state(Db, Fd, State, Header) -> views=Views2 }. -open_view(Db, Fd, Lang, {BTState, SeqBTState, KSeqBTState, USeq, PSeq}, View) -> - FunSrcs = [FunSrc || {_Name, FunSrc} <- View#mrview.reduce_funs], - ReduceFun = - fun(reduce, KVs) -> - KVs2 = detuple_kvs(expand_dups(KVs, []), []), - {ok, Result} = couch_query_servers:reduce(Lang, FunSrcs, KVs2), - {length(KVs2), Result}; - (rereduce, Reds) -> - Count = lists:sum([Count0 || {Count0, _} <- Reds]), - UsrReds = [UsrRedsList || {_, UsrRedsList} <- Reds], - {ok, Result} = couch_query_servers:rereduce(Lang, FunSrcs, UsrReds), - {Count, Result} - end, - - Less = case couch_util:get_value(<<"collation">>, View#mrview.options) of - <<"raw">> -> fun(A, B) -> A < B end; - _ -> fun couch_ejson_compare:less_json_ids/2 - end, - +open_view(Db, Fd, Lang, ViewState, View) -> + ReduceFun = make_reduce_fun(Lang, View#mrview.reduce_funs), + LessFun = maybe_define_less_fun(View), + Compression = couch_db:compression(Db), + BTState = get_key_btree_state(ViewState), ViewBtOpts = [ - {less, Less}, + {less, LessFun}, {reduce, ReduceFun}, - {compression, couch_db:compression(Db)} + {compression, Compression} ], {ok, Btree} = couch_btree:open(BTState, Fd, ViewBtOpts), BySeqReduceFun = fun couch_db_updater:btree_by_seq_reduce/2, {ok, SeqBtree} = if View#mrview.seq_indexed -> + SeqBTState = get_seq_btree_state(ViewState), ViewSeqBtOpts = [{reduce, BySeqReduceFun}, - {compression, couch_db:compression(Db)}], + {compression, Compression}], couch_btree:open(SeqBTState, Fd, ViewSeqBtOpts); true -> {ok, nil} end, {ok, KeyBySeqBtree} = if View#mrview.keyseq_indexed -> - KeyBySeqBtOpts = [{less, Less}, + KSeqBTState = get_kseq_btree_state(ViewState), + KeyBySeqBtOpts = [{less, LessFun}, {reduce, BySeqReduceFun}, - {compression, couch_db:compression(Db)}], + {compression, Compression}], couch_btree:open(KSeqBTState, Fd, KeyBySeqBtOpts); true -> {ok, nil} @@ -305,8 +287,8 @@ open_view(Db, Fd, Lang, {BTState, SeqBTState, KSeqBTState, USeq, PSeq}, View) -> View#mrview{btree=Btree, seq_btree=SeqBtree, key_byseq_btree=KeyBySeqBtree, - update_seq=USeq, - purge_seq=PSeq}. + update_seq=get_update_seq(ViewState), + purge_seq=get_purge_seq(ViewState)}. temp_view_to_ddoc({Props}) -> @@ -341,18 +323,9 @@ all_docs_reduce_to_count(Reductions) -> reduce_to_count(nil) -> 0; reduce_to_count(Reductions) -> - Reduce = fun - (reduce, KVs) -> - Counts = [ - case V of {dups, Vals} -> length(Vals); _ -> 1 end - || {_,V} <- KVs - ], - {lists:sum(Counts), []}; - (rereduce, Reds) -> - {lists:sum([Count0 || {Count0, _} <- Reds]), []} - end, - {Count, _} = couch_btree:final_reduce(Reduce, Reductions), - Count. + CountReduceFun = fun count_reduce/2, + FinalReduction = couch_btree:final_reduce(CountReduceFun, Reductions), + get_count(FinalReduction). %% @doc get all changes for a view get_view_changes_count(View) -> @@ -414,25 +387,13 @@ fold_reduce({NthRed, Lang, View}, Fun, Acc, Options) -> btree=Bt, reduce_funs=RedFuns } = View, - LPad = lists:duplicate(NthRed - 1, []), - RPad = lists:duplicate(length(RedFuns) - NthRed, []), - {_Name, FunSrc} = lists:nth(NthRed,RedFuns), - ReduceFun = fun - (reduce, KVs0) -> - KVs1 = detuple_kvs(expand_dups(KVs0, []), []), - {ok, Red} = couch_query_servers:reduce(Lang, [FunSrc], KVs1), - {0, LPad ++ Red ++ RPad}; - (rereduce, Reds) -> - ExtractRed = fun({_, UReds0}) -> [lists:nth(NthRed, UReds0)] end, - UReds = lists:map(ExtractRed, Reds), - {ok, Red} = couch_query_servers:rereduce(Lang, [FunSrc], UReds), - {0, LPad ++ Red ++ RPad} - end, + ReduceFun = make_user_reds_reduce_fun(Lang, RedFuns, NthRed), WrapperFun = fun({GroupedKey, _}, PartialReds, Acc0) -> - {_, Reds} = couch_btree:final_reduce(ReduceFun, PartialReds), - Fun(GroupedKey, lists:nth(NthRed, Reds), Acc0) + FinalReduction = couch_btree:final_reduce(ReduceFun, PartialReds), + UserReductions = get_user_reds(FinalReduction), + Fun(GroupedKey, lists:nth(NthRed, UserReductions), Acc0) end, couch_btree:fold_reduce(Bt, WrapperFun, Acc, Options). @@ -610,37 +571,12 @@ make_header(State) -> views=Views } = State, - ViewStates = lists:foldr(fun(V, Acc) -> - SeqBtState = case V#mrview.seq_indexed of - true -> - couch_btree:get_state(V#mrview.seq_btree); - _ -> - nil - end, - KSeqBtState = case V#mrview.keyseq_indexed of - true -> - couch_btree:get_state(V#mrview.key_byseq_btree); - _ -> - nil - end, - [{couch_btree:get_state(V#mrview.btree), - SeqBtState, - KSeqBtState, - V#mrview.update_seq, - V#mrview.purge_seq} | Acc] - end, [], Views), - - LogBtreeState = case LogBtree of - nil -> nil; - _ -> couch_btree:get_state(LogBtree) - end, - #mrheader{ seq=Seq, purge_seq=PurgeSeq, - id_btree_state=couch_btree:get_state(IdBtree), - log_btree_state= LogBtreeState, - view_states=ViewStates + id_btree_state=get_btree_state(IdBtree), + log_btree_state=get_btree_state(LogBtree), + view_states=[make_view_state(V) || V <- Views] }. @@ -1004,6 +940,121 @@ old_view_format(View) -> %% End of <= 1.2.x upgrade code. +make_view_state(#mrview{} = View) -> + BTState = get_btree_state(View#mrview.btree), + SeqBTState = case View#mrview.seq_indexed of + true -> + get_btree_state(View#mrview.seq_btree); + _ -> + nil + end, + KSeqBTState = case View#mrview.keyseq_indexed of + true -> + get_btree_state(View#mrview.key_byseq_btree); + _ -> + nil + end, + { + BTState, + SeqBTState, + KSeqBTState, + View#mrview.update_seq, + View#mrview.purge_seq + }; +make_view_state({BTState, UpdateSeq, PurgeSeq}) -> + {BTState, nil, nil, UpdateSeq, PurgeSeq}; +make_view_state(nil) -> + {nil, nil, nil, 0, 0}. + + +get_key_btree_state(ViewState) -> + element(1, ViewState). + +get_seq_btree_state(ViewState) -> + element(2, ViewState). + +get_kseq_btree_state(ViewState) -> + element(3, ViewState). + +get_update_seq(ViewState) -> + element(4, ViewState). + +get_purge_seq(ViewState) -> + element(5, ViewState). + +get_count(Reduction) -> + element(1, Reduction). + +get_user_reds(Reduction) -> + element(2, Reduction). + + +make_reduce_fun(Lang, ReduceFuns) -> + FunSrcs = [FunSrc || {_, FunSrc} <- ReduceFuns], + fun + (reduce, KVs0) -> + KVs = detuple_kvs(expand_dups(KVs0, []), []), + {ok, Result} = couch_query_servers:reduce(Lang, FunSrcs, KVs), + {length(KVs), Result}; + (rereduce, Reds) -> + ExtractFun = fun(Red, {CountsAcc0, URedsAcc0}) -> + CountsAcc = CountsAcc0 + get_count(Red), + URedsAcc = lists:append(URedsAcc0, [get_user_reds(Red)]), + {CountsAcc, URedsAcc} + end, + {Counts, UReds} = lists:foldl(ExtractFun, {0, []}, Reds), + {ok, Result} = couch_query_servers:rereduce(Lang, FunSrcs, UReds), + {Counts, Result} + end. + + +maybe_define_less_fun(#mrview{options = Options}) -> + case couch_util:get_value(<<"collation">>, Options) of + <<"raw">> -> undefined; + _ -> fun couch_ejson_compare:less_json_ids/2 + end. + + +count_reduce(reduce, KVs) -> + CountFun = fun + ({_, {dups, Vals}}, Acc) -> Acc + length(Vals); + (_, Acc) -> Acc + 1 + end, + Count = lists:foldl(CountFun, 0, KVs), + {Count, []}; +count_reduce(rereduce, Reds) -> + CountFun = fun(Red, Acc) -> + Acc + get_count(Red) + end, + Count = lists:foldl(CountFun, 0, Reds), + {Count, []}. + + +make_user_reds_reduce_fun(Lang, ReduceFuns, NthRed) -> + LPad = lists:duplicate(NthRed - 1, []), + RPad = lists:duplicate(length(ReduceFuns) - NthRed, []), + {_, FunSrc} = lists:nth(NthRed, ReduceFuns), + fun + (reduce, KVs0) -> + KVs = detuple_kvs(expand_dups(KVs0, []), []), + {ok, Result} = couch_query_servers:reduce(Lang, [FunSrc], KVs), + {0, LPad ++ Result ++ RPad}; + (rereduce, Reds) -> + ExtractFun = fun(Reds0) -> + [lists:nth(NthRed, get_user_reds(Reds0))] + end, + UReds = lists:map(ExtractFun, Reds), + {ok, Result} = couch_query_servers:rereduce(Lang, [FunSrc], UReds), + {0, LPad ++ Result ++ RPad} + end. + + +get_btree_state(nil) -> + nil; +get_btree_state(#btree{} = Btree) -> + couch_btree:get_state(Btree). + + extract_view_reduce({red, {N, _Lang, #mrview{reduce_funs=Reds}}, _Ref}) -> {_Name, FunSrc} = lists:nth(N, Reds), FunSrc.
