This is an automated email from the ASF dual-hosted git repository. garren pushed a commit to branch fdb-mango-indexes in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 506adf0e99624a9fe1ed74605b413ef3e2a2cea2 Author: Garren Smith <[email protected]> AuthorDate: Tue Feb 18 14:44:04 2020 +0200 code clean up --- src/couch_views/src/couch_views_server.erl | 1 - src/fabric/src/fabric2_fdb.erl | 1 - src/mango/src/mango_app.erl | 1 + src/mango/src/mango_cursor_view.erl | 60 +++++----------------- src/mango/src/mango_fdb.erl | 80 +++--------------------------- src/mango/src/mango_fdb_special.erl | 64 ++++++++++++++++++++++-- src/mango/src/mango_fdb_view.erl | 72 +++++++++++++++++++++++++-- src/mango/src/mango_indexer.erl | 3 +- 8 files changed, 150 insertions(+), 132 deletions(-) diff --git a/src/couch_views/src/couch_views_server.erl b/src/couch_views/src/couch_views_server.erl index 6299407..1a8caf3 100644 --- a/src/couch_views/src/couch_views_server.erl +++ b/src/couch_views/src/couch_views_server.erl @@ -92,7 +92,6 @@ spawn_workers(St) -> max_workers := MaxWorkers, worker_module := WorkerModule } = St, - io:format("BOOM COUCH VIEWS SERVER ~p ~n", [WorkerModule]), case maps:size(Workers) < MaxWorkers of true -> Pid = WorkerModule:spawn_link(), diff --git a/src/fabric/src/fabric2_fdb.erl b/src/fabric/src/fabric2_fdb.erl index cf73244..4249701 100644 --- a/src/fabric/src/fabric2_fdb.erl +++ b/src/fabric/src/fabric2_fdb.erl @@ -811,7 +811,6 @@ write_doc(#{} = Db0, Doc, NewWinner0, OldWinner, ToUpdate, ToRemove) -> % Update database size AddSize = sum_add_rev_sizes([NewWinner | ToUpdate]), RemSize = sum_rem_rev_sizes(ToRemove), -%% TODO: causing mango indexes to fail with fdb error 1036 %% incr_stat(Db, <<"sizes">>, <<"external">>, AddSize - RemSize), ok. diff --git a/src/mango/src/mango_app.erl b/src/mango/src/mango_app.erl index 7a0c39d..221d57d 100644 --- a/src/mango/src/mango_app.erl +++ b/src/mango/src/mango_app.erl @@ -15,6 +15,7 @@ -export([start/2, stop/1]). start(_Type, StartArgs) -> + mango_jobs:set_timeout(), mango_sup:start_link(StartArgs). stop(_State) -> diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl index 22ff6a8..96d1fb5 100644 --- a/src/mango/src/mango_cursor_view.erl +++ b/src/mango/src/mango_cursor_view.erl @@ -19,9 +19,7 @@ ]). -export([ -%% view_cb/2, handle_message/2, - handle_all_docs_message/2, composite_indexes/2, choose_best_index/2 ]). @@ -140,7 +138,7 @@ index_args(#cursor{} = Cursor) -> mango_json_bookmark:update_args(Bookmark, Args1). -execute(#cursor{db = Db, index = Idx, execution_stats = Stats} = Cursor0, UserFun, UserAcc) -> +execute(#cursor{db = Db, execution_stats = Stats} = Cursor0, UserFun, UserAcc) -> Cursor = Cursor0#cursor{ user_fun = UserFun, user_acc = UserAcc, @@ -152,26 +150,21 @@ execute(#cursor{db = Db, index = Idx, execution_stats = Stats} = Cursor0, UserFu {ok, UserAcc}; _ -> Args = index_args(Cursor), - #cursor{opts = Opts} = Cursor, - Result = case mango_idx:def(Idx) of - all_docs -> - CB = fun ?MODULE:handle_all_docs_message/2, - % all_docs - mango_fdb:query_all_docs(Db, CB, Cursor, Args); - _ -> - CB = fun ?MODULE:handle_message/2, - % json index - mango_fdb:query(Db, CB, Cursor, Args) - end, + CB = fun ?MODULE:handle_message/2, + Result = mango_fdb:query(Db, CB, Cursor, Args), case Result of {ok, LastCursor} -> NewBookmark = mango_json_bookmark:create(LastCursor), Arg = {add_key, bookmark, NewBookmark}, - {_Go, FinalUserAcc} = UserFun(Arg, LastCursor#cursor.user_acc), - Stats0 = LastCursor#cursor.execution_stats, - FinalUserAcc0 = mango_execution_stats:maybe_add_stats(Opts, UserFun, Stats0, FinalUserAcc), - FinalUserAcc1 = mango_cursor:maybe_add_warning(UserFun, Cursor, FinalUserAcc0), - {ok, FinalUserAcc1}; + #cursor{ + opts = Opts, + execution_stats = Stats0, + user_acc = FinalUserAcc0 + } = LastCursor, + {_Go, FinalUserAcc1} = UserFun(Arg, FinalUserAcc0), + FinalUserAcc2 = mango_execution_stats:maybe_add_stats(Opts, UserFun, Stats0, FinalUserAcc1), + FinalUserAcc3 = mango_cursor:maybe_add_warning(UserFun, Cursor, FinalUserAcc2), + {ok, FinalUserAcc3}; {error, Reason} -> {error, Reason} end @@ -334,20 +327,6 @@ handle_message({error, Reason}, _Cursor) -> {error, Reason}. -handle_all_docs_message({row, Props}, Cursor) -> - io:format("ALL DOCS ~p ~n", [Props]), - case is_design_doc(Props) of - true -> {ok, Cursor}; - false -> - Doc = couch_util:get_value(doc, Props), - Key = couch_util:get_value(key, Props), - - handle_message({doc, Key, Doc}, Cursor) - end; -handle_all_docs_message(Message, Cursor) -> - handle_message(Message, Cursor). - - handle_doc(#cursor{skip = S} = C, _) when S > 0 -> {ok, C#cursor{skip = S - 1}}; handle_doc(#cursor{limit = L, execution_stats = Stats} = C, Doc) when L > 0 -> @@ -362,16 +341,6 @@ handle_doc(#cursor{limit = L, execution_stats = Stats} = C, Doc) when L > 0 -> handle_doc(C, _Doc) -> {stop, C}. - -%%ddocid(Idx) -> -%% case mango_idx:ddoc(Idx) of -%% <<"_design/", Rest/binary>> -> -%% Rest; -%% Else -> -%% Else -%% end. - - %%apply_opts([], Args) -> %% Args; %%apply_opts([{r, RStr} | Rest], Args) -> @@ -485,11 +454,6 @@ match_doc(Selector, Doc, ExecutionStats) -> end. -is_design_doc(RowProps) -> - case couch_util:get_value(id, RowProps) of - <<"_design/", _/binary>> -> true; - _ -> false - end. update_bookmark_keys(#cursor{limit = Limit} = Cursor, {Key, Props}) when Limit > 0 -> diff --git a/src/mango/src/mango_fdb.erl b/src/mango/src/mango_fdb.erl index edbb27f..9b5206e 100644 --- a/src/mango/src/mango_fdb.erl +++ b/src/mango/src/mango_fdb.erl @@ -18,7 +18,6 @@ -include("mango.hrl"). -include("mango_idx.hrl"). -include("mango_cursor.hrl"). --include("mango_idx_view.hrl"). -export([ @@ -30,8 +29,9 @@ set_update_seq/3, remove_doc/3, write_doc/3, - query_all_docs/4, - query/4 + query/4, + base_fold_opts/1, + mango_idx_prefix/2 ]). @@ -118,90 +118,26 @@ write_doc(TxDb, DocId, IdxResults) -> add_key(TxDb, MangoIdxPrefix, Results, DocId) end, IdxResults). - -query_all_docs(Db, CallBack, Cursor, Args) -> +query(Db, CallBack, Cursor, Args) -> #cursor{ index = Idx } = Cursor, - Opts = args_to_fdb_opts(Args, Idx) ++ [{include_docs, true}], - io:format("ALL DOC OPTS ~p ~n", [Opts]), - fabric2_db:fold_docs(Db, CallBack, Cursor, Opts). + Mod = mango_idx:fdb_mod(Idx), + Mod:query(Db, CallBack, Cursor, Args). -query(Db, CallBack, Cursor, Args) -> - #cursor{ - index = Idx - } = Cursor, - MangoIdxPrefix = mango_idx_prefix(Db, Idx#idx.ddoc), - fabric2_fdb:transactional(Db, fun (TxDb) -> - Acc0 = #{ - cursor => Cursor, - prefix => MangoIdxPrefix, - db => TxDb, - callback => CallBack - }, - - Opts = args_to_fdb_opts(Args, Idx), - io:format("OPTS ~p ~n", [Opts]), - try - Acc1 = fabric2_fdb:fold_range(TxDb, MangoIdxPrefix, fun fold_cb/2, Acc0, Opts), - #{ - cursor := Cursor1 - } = Acc1, - {ok, Cursor1} - catch - throw:{stop, StopCursor} -> - {ok, StopCursor} - end - end). - - -args_to_fdb_opts(Args, Idx) -> +base_fold_opts(Args) -> #{ - start_key := StartKey, - start_key_docid := StartKeyDocId, - end_key := EndKey, - end_key_docid := EndKeyDocId, dir := Direction, skip := Skip } = Args, - io:format("ARGS ~p ~n", [Args]), - io:format("START ~p ~n End ~p ~n", [StartKey, EndKey]), - Mod = mango_idx:fdb_mod(Idx), - - StartKeyOpts = Mod:start_key_opts(StartKey, StartKeyDocId), - EndKeyOpts = Mod:end_key_opts(EndKey, EndKeyDocId), - [ {skip, Skip}, {dir, Direction}, {streaming_mode, want_all}, {restart_tx, true} - ] ++ StartKeyOpts ++ EndKeyOpts. - - -fold_cb({Key, Val}, Acc) -> - #{ - prefix := MangoIdxPrefix, - db := Db, - callback := Callback, - cursor := Cursor - - } = Acc, - {{_, DocId}} = erlfdb_tuple:unpack(Key, MangoIdxPrefix), - SortKeys = couch_views_encoding:decode(Val), - {ok, Doc} = fabric2_db:open_doc(Db, DocId, [{conflicts, true}]), - JSONDoc = couch_doc:to_json_obj(Doc, []), - io:format("PRINT ~p ~p ~n", [DocId, JSONDoc]), - case Callback({doc, SortKeys, JSONDoc}, Cursor) of - {ok, Cursor1} -> - Acc#{ - cursor := Cursor1 - }; - {stop, Cursor1} -> - throw({stop, Cursor1}) - end. + ]. mango_idx_prefix(TxDb, Id) -> diff --git a/src/mango/src/mango_fdb_special.erl b/src/mango/src/mango_fdb_special.erl index e8fd6c1..ef55bc4 100644 --- a/src/mango/src/mango_fdb_special.erl +++ b/src/mango/src/mango_fdb_special.erl @@ -14,19 +14,73 @@ -module(mango_fdb_special). -include_lib("couch/include/couch_db.hrl"). +-include("mango_cursor.hrl"). -export([ - start_key_opts/2, - end_key_opts/2 + query/4 ]). -start_key_opts(StartKey, _StartKeyDocId) -> + +query(Db, CallBack, Cursor, Args) -> + Acc = #{ + cursor => Cursor, + callback => CallBack + }, + Opts = args_to_fdb_opts(Args), + io:format("ALL DOC OPTS ~p ~n", [Opts]), + {ok, Acc1} = fabric2_db:fold_docs(Db, fun fold_cb/2, Acc, Opts), + {ok, maps:get(cursor, Acc1)}. + + +args_to_fdb_opts(Args) -> + #{ + start_key := StartKey, + end_key := EndKey + } = Args, + BaseOpts = mango_fdb:base_fold_opts(Args), + BaseOpts ++ [{include_docs, true}] + ++ start_key_opts(StartKey) ++ end_key_opts(EndKey). + + +start_key_opts(StartKey) -> [{start_key, fabric2_util:encode_all_doc_key(StartKey)}]. -end_key_opts(?MAX_STR, _EndKeyDocId) -> +end_key_opts(?MAX_STR) -> []; -end_key_opts(EndKey, _EndKeyDocId) -> +end_key_opts(EndKey) -> [{end_key, fabric2_util:encode_all_doc_key(EndKey)}]. + + +fold_cb({row, Props}, Acc) -> + #{ + cursor := Cursor, + callback := Callback + } = Acc, + io:format("ALL DOCS ~p ~n", [Props]), + case is_design_doc(Props) of + true -> + {ok, Acc}; + false -> + Doc = couch_util:get_value(doc, Props), + Key = couch_util:get_value(key, Props), + {Go, Cursor1} = Callback({doc, Key, Doc}, Cursor), + {Go, Acc#{cursor := Cursor1}} + end; + +fold_cb(Message, Acc) -> + #{ + cursor := Cursor, + callback := Callback + } = Acc, + {Go, Cursor1} = Callback(Message, Cursor), + {Go, Acc#{cursor := Cursor1}}. + + +is_design_doc(RowProps) -> + case couch_util:get_value(id, RowProps) of + <<"_design/", _/binary>> -> true; + _ -> false + end. diff --git a/src/mango/src/mango_fdb_view.erl b/src/mango/src/mango_fdb_view.erl index faab91b..329087f 100644 --- a/src/mango/src/mango_fdb_view.erl +++ b/src/mango/src/mango_fdb_view.erl @@ -15,10 +15,55 @@ -export([ - start_key_opts/2, - end_key_opts/2 + query/4 ]). + +-include("mango_idx.hrl"). +-include("mango_cursor.hrl"). + + +query(Db, CallBack, Cursor, Args) -> + #cursor{ + index = Idx + } = Cursor, + MangoIdxPrefix = mango_fdb:mango_idx_prefix(Db, Idx#idx.ddoc), + fabric2_fdb:transactional(Db, fun (TxDb) -> + Acc0 = #{ + cursor => Cursor, + prefix => MangoIdxPrefix, + db => TxDb, + callback => CallBack + }, + + Opts = args_to_fdb_opts(Args), + io:format("OPTS ~p ~n", [Opts]), + try + Acc1 = fabric2_fdb:fold_range(TxDb, MangoIdxPrefix, + fun fold_cb/2, Acc0, Opts), + #{ + cursor := Cursor1 + } = Acc1, + {ok, Cursor1} + catch + throw:{stop, StopCursor} -> + {ok, StopCursor} + end + end). + + +args_to_fdb_opts(Args) -> + #{ + start_key := StartKey, + start_key_docid := StartKeyDocId, + end_key := EndKey, + end_key_docid := EndKeyDocId + } = Args, + BaseOpts = mango_fdb:base_fold_opts(Args), + BaseOpts ++ start_key_opts(StartKey, StartKeyDocId) + ++ end_key_opts(EndKey, EndKeyDocId). + + start_key_opts([], _StartKeyDocId) -> []; @@ -32,6 +77,27 @@ end_key_opts([], _EndKeyDocId) -> []; end_key_opts(EndKey, EndKeyDocId) -> - io:format("ENDKEY ~p ~n", [EndKey]), EndKey1 = couch_views_encoding:encode(EndKey, key), [{end_key, {EndKey1, EndKeyDocId}}]. + +fold_cb({Key, Val}, Acc) -> + #{ + prefix := MangoIdxPrefix, + db := Db, + callback := Callback, + cursor := Cursor + + } = Acc, + {{_, DocId}} = erlfdb_tuple:unpack(Key, MangoIdxPrefix), + SortKeys = couch_views_encoding:decode(Val), + {ok, Doc} = fabric2_db:open_doc(Db, DocId, [{conflicts, true}]), + JSONDoc = couch_doc:to_json_obj(Doc, []), + io:format("PRINT ~p ~p ~n", [DocId, JSONDoc]), + case Callback({doc, SortKeys, JSONDoc}, Cursor) of + {ok, Cursor1} -> + Acc#{ + cursor := Cursor1 + }; + {stop, Cursor1} -> + throw({stop, Cursor1}) + end. diff --git a/src/mango/src/mango_indexer.erl b/src/mango/src/mango_indexer.erl index d00a254..b0b119f 100644 --- a/src/mango/src/mango_indexer.erl +++ b/src/mango/src/mango_indexer.erl @@ -66,8 +66,7 @@ doc_id(#doc{id = DocId}, _) -> DocId. -% Design doc -% Todo: Check if design doc is mango index and kick off background worker +% Check if design doc is mango index and kick off background worker % to build new index modify_int(Db, _Change, #doc{id = <<?DESIGN_DOC_PREFIX, _/binary>>} = Doc, _PrevDoc) ->
