This is an automated email from the ASF dual-hosted git repository. rnewson pushed a commit to branch user-partitioned-dbs-6 in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit cb96435dfd0f99d11e7badfc6e8ec92ae339887a Author: Robert Newson <[email protected]> AuthorDate: Mon Sep 10 11:56:56 2018 +0100 allow partitioned:true in _index post. merge somewhere --- src/mango/src/mango_cursor_view.erl | 21 +++++++----------- src/mango/src/mango_httpd.erl | 4 ++-- src/mango/src/mango_idx.erl | 44 +++++++++++++++++++++++++++++++++---- src/mango/src/mango_idx.hrl | 1 + src/mango/src/mango_idx_view.erl | 16 +++++++++++++- src/mango/src/mango_opts.erl | 10 +++++++-- 6 files changed, 74 insertions(+), 22 deletions(-) diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl index 708a73e..3eedf99 100644 --- a/src/mango/src/mango_cursor_view.erl +++ b/src/mango/src/mango_cursor_view.erl @@ -103,20 +103,15 @@ base_args(#cursor{index = Idx, opts = Opts} = Cursor) -> end_key = mango_idx:end_key(Idx, Cursor#cursor.ranges), include_docs = true }, - Args2 = case mem3:is_partitioned(Idx#idx.dbname) of - true -> - Partition = couch_util:get_value(partition, Opts), - add_partition_opts(Args1, Partition); - false -> - Args1 + Partitioned = couch_util:get_value(partitioned, Idx#idx.design_opts), + Args2 = couch_mrview_util:set_extra(Args1, partitioned, Partitioned), + Args3 = case couch_util:get_value(partition, Opts) of + <<>> -> + Args2; + Partition -> + couch_mrview_util:set_extra(Args2, partition, Partition) end, - add_style(Idx, Args2). - -add_partition_opts(#mrargs{} = Args, <<>>) -> - Args; -add_partition_opts(#mrargs{} = Args, Partition) -> - Args1 = couch_mrview_util:set_extra(Args, partitioned, true), - couch_mrview_util:set_extra(Args1, partition, Partition). + add_style(Idx, Args3). add_style(#idx{def = all_docs}, Args) -> couch_mrview_util:set_extra(Args, style, all_docs); diff --git a/src/mango/src/mango_httpd.erl b/src/mango/src/mango_httpd.erl index d9589b3..77a73c1 100644 --- a/src/mango/src/mango_httpd.erl +++ b/src/mango/src/mango_httpd.erl @@ -104,9 +104,9 @@ handle_index_req(#httpd{method='GET', path_parts=[_, _]}=Req, Db) -> JsonIdxs = lists:sublist(JsonIdxs0, Skip+1, Limit), chttpd:send_json(Req, {[{total_rows, TotalRows}, {indexes, JsonIdxs}]}); -handle_index_req(#httpd{method='POST', path_parts=[_, _]}=Req, Db) -> +handle_index_req(#httpd{method='POST', path_parts=[DbName, _]}=Req, Db) -> chttpd:validate_ctype(Req, "application/json"), - {ok, Opts} = mango_opts:validate_idx_create(chttpd:json_body_obj(Req)), + {ok, Opts} = mango_opts:validate_idx_create(DbName, chttpd:json_body_obj(Req)), {ok, Idx0} = mango_idx:new(Db, Opts), {ok, Idx} = mango_idx:validate_new(Idx0, Db), DbOpts = [{user_ctx, Req#httpd.user_ctx}, deleted, ejson_body], diff --git a/src/mango/src/mango_idx.erl b/src/mango/src/mango_idx.erl index 8af92b9..e051218 100644 --- a/src/mango/src/mango_idx.erl +++ b/src/mango/src/mango_idx.erl @@ -58,8 +58,9 @@ list(Db) -> get_usable_indexes(Db, Selector, Opts) -> - ExistingIndexes = mango_idx:list(Db), - + PQ = is_partitioned_query(Opts), + ExistingIndexes = filter_indexes_by_partitioned( + mango_idx:list(Db), PQ), GlobalIndexes = mango_cursor:remove_indexes_with_partial_filter_selector(ExistingIndexes), UserSpecifiedIndex = mango_cursor:maybe_filter_indexes_by_ddoc(ExistingIndexes, Opts), UsableIndexes0 = lists:usort(GlobalIndexes ++ UserSpecifiedIndex), @@ -68,13 +69,36 @@ get_usable_indexes(Db, Selector, Opts) -> UsableFilter = fun(I) -> is_usable(I, Selector, SortFields) end, case lists:filter(UsableFilter, UsableIndexes0) of - [] -> + [] -> ?MANGO_ERROR({no_usable_index, missing_sort_index}); - UsableIndexes -> + UsableIndexes -> UsableIndexes end. +filter_indexes_by_partitioned(Indexes, PQ) -> + filter_indexes_by_partitioned(Indexes, PQ, []). + + +filter_indexes_by_partitioned([], _PQ, Acc) -> + lists:reverse(Acc); +filter_indexes_by_partitioned([Idx | Rest], PQ, Acc) -> + {partitioned, PI} = lists:keyfind(partitioned, 1, Idx#idx.design_opts), + case {Idx#idx.def, PI, PQ} of + {all_docs, _, _} -> + % all_docs works both ways. + filter_indexes_by_partitioned(Rest, PQ, [Idx | Acc]); + {_, Same, Same} -> + filter_indexes_by_partitioned(Rest, PQ, [Idx | Acc]); + {_, _, _} -> + filter_indexes_by_partitioned(Rest, PQ, Acc) + end. + + +is_partitioned_query(Opts) -> + lists:keyfind(partition, 1, Opts) /= {partition, <<>>}. + + recover(Db) -> {ok, DDocs0} = mango_util:open_ddocs(Db), Pred = fun({Props}) -> @@ -101,6 +125,7 @@ get_sort_fields(Opts) -> new(Db, Opts) -> Def = get_idx_def(Opts), + DesignOpts = get_idx_design_opts(Db, Opts), Type = get_idx_type(Opts), IdxName = get_idx_name(Def, Opts), DDoc = get_idx_ddoc(Def, Opts), @@ -110,6 +135,7 @@ new(Db, Opts) -> name = IdxName, type = Type, def = Def, + design_opts = DesignOpts, opts = filter_opts(Opts) }}. @@ -182,11 +208,13 @@ from_ddoc(Db, {Props}) -> special(Db) -> + Partitioned = mem3:is_partitioned(Db), AllDocs = #idx{ dbname = db_to_name(Db), name = <<"_all_docs">>, type = <<"special">>, def = all_docs, + design_opts = [{partitioned, Partitioned}], opts = [] }, % Add one for _update_seq @@ -285,6 +313,12 @@ get_idx_def(Opts) -> end. +get_idx_design_opts(Db, Opts) -> + DbPartitioned = mem3:is_partitioned(couch_db:name(Db)), + Partitioned = proplists:get_value(partitioned, Opts, DbPartitioned), + [{partitioned, Partitioned}]. + + get_idx_type(Opts) -> case proplists:get_value(type, Opts) of <<"json">> -> <<"json">>; @@ -341,6 +375,8 @@ filter_opts([{type, _} | Rest]) -> filter_opts(Rest); filter_opts([{w, _} | Rest]) -> filter_opts(Rest); +filter_opts([{partitioned, _} | Rest]) -> + filter_opts(Rest); filter_opts([Opt | Rest]) -> [Opt | filter_opts(Rest)]. diff --git a/src/mango/src/mango_idx.hrl b/src/mango/src/mango_idx.hrl index 712031b..179911f 100644 --- a/src/mango/src/mango_idx.hrl +++ b/src/mango/src/mango_idx.hrl @@ -16,5 +16,6 @@ name, type, def, + design_opts, opts }). diff --git a/src/mango/src/mango_idx_view.erl b/src/mango/src/mango_idx_view.erl index 2d784b6..ef1ca59 100644 --- a/src/mango/src/mango_idx_view.erl +++ b/src/mango/src/mango_idx_view.erl @@ -54,7 +54,8 @@ add(#doc{body={Props0}}=DDoc, Idx) -> NewView = make_view(Idx), Views2 = lists:keystore(element(1, NewView), 1, Views1, NewView), Props1 = lists:keystore(<<"views">>, 1, Props0, {<<"views">>, {Views2}}), - {ok, DDoc#doc{body={Props1}}}. + Props2 = lists:keystore(<<"options">>, 1, Props1, {<<"options">>, {Idx#idx.design_opts}}), + {ok, DDoc#doc{body={Props2}}}. remove(#doc{body={Props0}}=DDoc, Idx) -> @@ -78,6 +79,7 @@ remove(#doc{body={Props0}}=DDoc, Idx) -> from_ddoc({Props}) -> + DesignOpts = validate_design_opts(Props), case lists:keyfind(<<"views">>, 1, Props) of {<<"views">>, {Views}} when is_list(Views) -> lists:flatmap(fun({Name, {VProps}}) -> @@ -89,6 +91,7 @@ from_ddoc({Props}) -> type = <<"json">>, name = Name, def = Def, + design_opts = DesignOpts, opts = Opts }, [I] @@ -104,6 +107,7 @@ to_json(Idx) -> {ddoc, Idx#idx.ddoc}, {name, Idx#idx.name}, {type, Idx#idx.type}, + {design_opts, {Idx#idx.design_opts}}, {def, {def_to_json(Idx#idx.def)}} ]}. @@ -248,6 +252,16 @@ validate_ddoc(VProps) -> end. +validate_design_opts(Props) -> + case lists:keyfind(<<"options">>, 1, Props) of + {<<"options">>, {[{<<"partitioned">>, P}]}} + when is_boolean(P) -> + [{partitioned, P}]; + _ -> + [] + end. + + % This function returns a list of indexes that % can be used to restrict this query. This works by % searching the selector looking for field names that diff --git a/src/mango/src/mango_opts.erl b/src/mango/src/mango_opts.erl index f1b7b60..365e248 100644 --- a/src/mango/src/mango_opts.erl +++ b/src/mango/src/mango_opts.erl @@ -13,7 +13,7 @@ -module(mango_opts). -export([ - validate_idx_create/1, + validate_idx_create/2, validate_find/1 ]). @@ -43,11 +43,17 @@ -include("mango.hrl"). -validate_idx_create({Props}) -> +validate_idx_create(DbName, {Props}) -> Opts = [ {<<"index">>, [ {tag, def} ]}, + {<<"partitioned">>, [ + {tag, partitioned}, + {optional, true}, + {default, mem3:is_partitioned(DbName)}, + {validator, fun is_boolean/1} + ]}, {<<"type">>, [ {tag, type}, {optional, true},
