couch_mrview: fix compaction with seqs_indexed=true This change smakes sure to compact the view indexes when using the view changes.
Project: http://git-wip-us.apache.org/repos/asf/couchdb/repo Commit: http://git-wip-us.apache.org/repos/asf/couchdb/commit/6f173d5b Tree: http://git-wip-us.apache.org/repos/asf/couchdb/tree/6f173d5b Diff: http://git-wip-us.apache.org/repos/asf/couchdb/diff/6f173d5b Branch: refs/heads/1994-merge-rcouch Commit: 6f173d5b394c39542571bf133b5b90d7f5b0651f Parents: 8997ab1 Author: Benoit Chesneau <[email protected]> Authored: Mon Jan 27 15:29:10 2014 +0100 Committer: Benoit Chesneau <[email protected]> Committed: Mon Jan 27 15:29:10 2014 +0100 ---------------------------------------------------------------------- .../couch_mrview/src/couch_mrview_compactor.erl | 84 ++++++++++++++++++-- apps/couch_mrview/src/couch_mrview_util.erl | 12 ++- 2 files changed, 88 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/couchdb/blob/6f173d5b/apps/couch_mrview/src/couch_mrview_compactor.erl ---------------------------------------------------------------------- diff --git a/apps/couch_mrview/src/couch_mrview_compactor.erl b/apps/couch_mrview/src/couch_mrview_compactor.erl index 4ab8dd1..a7e594f 100644 --- a/apps/couch_mrview/src/couch_mrview_compactor.erl +++ b/apps/couch_mrview/src/couch_mrview_compactor.erl @@ -40,6 +40,8 @@ compact(State) -> sig=Sig, update_seq=Seq, id_btree=IdBtree, + log_btree=LogBtree, + seq_indexed=SeqIndexed, views=Views } = State, @@ -56,15 +58,28 @@ compact(State) -> #mrst{ id_btree = EmptyIdBtree, + log_btree = EmptyLogBtree, views = EmptyViews } = EmptyState, + TotalChanges0 = case SeqIndexed of + true -> NumDocIds * 2; + _ -> NumDocIds + end, + TotalChanges = lists:foldl( fun(View, Acc) -> {ok, Kvs} = couch_mrview_util:get_row_count(View), - Acc + Kvs + case SeqIndexed of + true -> + {ok, SKvs} = couch_mrview_util:get_view_changes_count(View), + Acc + Kvs + SKvs * 2; + false -> + Acc + Kvs + end end, - NumDocIds, Views), + TotalChanges0, Views), + couch_task_status:add_task([ {type, view_compaction}, {database, DbName}, @@ -106,13 +121,25 @@ compact(State) -> {ok, NewIdBtree} = couch_btree:add(Bt3, lists:reverse(Uncopied)), FinalAcc2 = update_task(FinalAcc, length(Uncopied)), + + {NewLogBtree, FinalAcc3} = case SeqIndexed of + true -> + compact_log(LogBtree, BufferSize, + FinalAcc2#acc{kvs=[], + kvs_size=0, + btree=EmptyLogBtree}); + _ -> + {nil, FinalAcc2} + end, + {NewViews, _} = lists:mapfoldl(fun({View, EmptyView}, Acc) -> compact_view(View, EmptyView, BufferSize, Acc) - end, FinalAcc2, lists:zip(Views, EmptyViews)), + end, FinalAcc3, lists:zip(Views, EmptyViews)), unlink(EmptyState#mrst.fd), {ok, EmptyState#mrst{ id_btree=NewIdBtree, + log_btree=NewLogBtree, views=NewViews, update_seq=Seq }}. @@ -129,9 +156,53 @@ recompact(State) -> {ok, State2} end. +compact_log(LogBtree, BufferSize, Acc0) -> + FoldFun = fun({DocId, _} = KV, Acc) -> + #acc{btree = Bt, kvs = Kvs, kvs_size = KvsSize} = Acc, + KvsSize2 = KvsSize + ?term_size(KV), + case KvsSize2 >= BufferSize of + true -> + {ok, Bt2} = couch_btree:add(Bt, lists:reverse([KV | Kvs])), + Acc2 = update_task(Acc, 1 + length(Kvs)), + {ok, Acc2#acc{ + btree = Bt2, kvs = [], kvs_size = 0}}; + _ -> + {ok, Acc#acc{ + kvs = [KV | Kvs], kvs_size = KvsSize2}} + end + end, + + {ok, _, FinalAcc} = couch_btree:foldl(LogBtree, FoldFun, Acc0), + #acc{btree = Bt3, kvs = Uncopied} = FinalAcc, + {ok, NewLogBtree} = couch_btree:add(Bt3, lists:reverse(Uncopied)), + FinalAcc2 = update_task(FinalAcc, length(Uncopied)), + {NewLogBtree, FinalAcc2}. %% @spec compact_view(View, EmptyView, Retry, Acc) -> {CompactView, NewAcc} compact_view(View, EmptyView, BufferSize, Acc0) -> + + {NewBt, Acc1} = compact_view_btree(View#mrview.btree, + EmptyView#mrview.btree, + BufferSize, Acc0), + + %% are we indexing changes by sequences? + {NewSeqBt, NewKeyBySeqBt, FinalAcc} = case View#mrview.seq_indexed of + true -> + {SBt, Acc2} = compact_view_btree(View#mrview.seq_btree, + EmptyView#mrview.seq_btree, + BufferSize, Acc1), + {KSBt, Acc3} = compact_view_btree(View#mrview.key_byseq_btree, + EmptyView#mrview.key_byseq_btree, + BufferSize, Acc2), + {SBt, KSBt, Acc3}; + _ -> + {nil, nil, Acc1} + end, + {EmptyView#mrview{btree=NewBt, + seq_btree=NewSeqBt, + key_byseq_btree=NewKeyBySeqBt}, FinalAcc}. + +compact_view_btree(Btree, EmptyBtree, BufferSize, Acc0) -> Fun = fun(KV, #acc{btree = Bt, kvs = Kvs, kvs_size = KvsSize} = Acc) -> KvsSize2 = KvsSize + ?term_size(KV), if KvsSize2 >= BufferSize -> @@ -143,13 +214,12 @@ compact_view(View, EmptyView, BufferSize, Acc0) -> end end, - InitAcc = Acc0#acc{kvs = [], kvs_size = 0, btree = EmptyView#mrview.btree}, - {ok, _, FinalAcc} = couch_btree:foldl(View#mrview.btree, Fun, InitAcc), + InitAcc = Acc0#acc{kvs = [], kvs_size = 0, btree = EmptyBtree}, + {ok, _, FinalAcc} = couch_btree:foldl(Btree, Fun, InitAcc), #acc{btree = Bt3, kvs = Uncopied} = FinalAcc, {ok, NewBt} = couch_btree:add(Bt3, lists:reverse(Uncopied)), FinalAcc2 = update_task(FinalAcc, length(Uncopied)), - {EmptyView#mrview{btree=NewBt}, FinalAcc2}. - + {NewBt, FinalAcc2}. update_task(#acc{changes = Changes, total_changes = Total} = Acc, ChangesInc) -> Changes2 = Changes + ChangesInc, http://git-wip-us.apache.org/repos/asf/couchdb/blob/6f173d5b/apps/couch_mrview/src/couch_mrview_util.erl ---------------------------------------------------------------------- diff --git a/apps/couch_mrview/src/couch_mrview_util.erl b/apps/couch_mrview/src/couch_mrview_util.erl index 8cb4fae..7c42de8 100644 --- a/apps/couch_mrview/src/couch_mrview_util.erl +++ b/apps/couch_mrview/src/couch_mrview_util.erl @@ -18,6 +18,7 @@ -export([index_file/2, compaction_file/2, open_file/1]). -export([delete_files/2, delete_index_file/2, delete_compaction_file/2]). -export([get_row_count/1, all_docs_reduce_to_count/1, reduce_to_count/1]). +-export([get_view_changes_count/1]). -export([all_docs_key_opts/1, all_docs_key_opts/2, key_opts/1, key_opts/2]). -export([fold/4, fold_reduce/4]). -export([temp_view_to_ddoc/1]). @@ -330,6 +331,12 @@ reduce_to_count(Reductions) -> {Count, _} = couch_btree:final_reduce(Reduce, Reductions), Count. +%% @doc get all changes for a view +get_view_changes_count(#mrview{seq_btree=Btree}) -> + couch_btree:fold_reduce( + Btree, fun(_SeqStart, PartialReds, 0) -> + {ok, couch_btree:final_reduce(Btree, PartialReds)} + end,0, []). fold(#mrview{btree=Bt}, Fun, Acc, Opts) -> WrapperFun = fun(KV, Reds, Acc2) -> @@ -625,10 +632,13 @@ reset_state(State) -> State#mrst{ fd=nil, qserver=nil, + seq_indexed=State#mrst.seq_indexed, update_seq=0, id_btree=nil, log_btree=nil, - views=[View#mrview{btree=nil, seq_btree=nil, key_byseq_btree=nil} + views=[View#mrview{btree=nil, seq_btree=nil, + key_byseq_btree=nil, + seq_indexed=View#mrview.seq_indexed} || View <- State#mrst.views] }.
