This is an automated email from the ASF dual-hosted git repository. rnewson pushed a commit to branch user-partitioned-dbs-6 in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit cdf155b959ecddef530b467ac2857dcc678db096 Author: Robert Newson <[email protected]> AuthorDate: Thu Aug 9 13:43:17 2018 +0100 Implement _all_docs and _find support --- src/chttpd/src/chttpd_db.erl | 9 ++-- src/couch_mrview/src/couch_mrview.erl | 7 +-- src/couch_mrview/src/couch_mrview_updater.erl | 4 +- src/couch_mrview/src/couch_mrview_util.erl | 77 ++++++++++++++++++++++----- src/fabric/src/fabric.erl | 7 ++- src/fabric/src/fabric_view.erl | 2 +- src/fabric/src/fabric_view_all_docs.erl | 24 ++++++++- src/mango/src/mango_cursor_view.erl | 23 ++++++-- src/mango/src/mango_opts.erl | 6 +++ src/mem3/src/mem3.erl | 5 +- 10 files changed, 135 insertions(+), 29 deletions(-) diff --git a/src/chttpd/src/chttpd_db.erl b/src/chttpd/src/chttpd_db.erl index 57d85e1..83d0489 100644 --- a/src/chttpd/src/chttpd_db.erl +++ b/src/chttpd/src/chttpd_db.erl @@ -684,12 +684,15 @@ multi_all_docs_view(Req, Db, OP, Queries) -> all_docs_view(Req, Db, Keys, OP) -> Args0 = couch_mrview_http:parse_params(Req, Keys), Args1 = Args0#mrargs{view_type=map}, - Args2 = couch_mrview_util:validate_args(Args1), - Args3 = set_namespace(OP, Args2), + DbPartitioned = mem3:is_partitioned(couch_db:name(Db)), + Args2 = couch_mrview_util:set_extra(Args1, partitioned, DbPartitioned), + Args3 = couch_mrview_util:set_extra(Args2, style, all_docs), + Args4 = couch_mrview_util:validate_args(Args3), + Args5 = set_namespace(OP, Args4), Options = [{user_ctx, Req#httpd.user_ctx}], Max = chttpd:chunked_response_buffer_size(), VAcc = #vacc{db=Db, req=Req, threshold=Max}, - {ok, Resp} = fabric:all_docs(Db, Options, fun couch_mrview_http:view_cb/2, VAcc, Args3), + {ok, Resp} = fabric:all_docs(Db, Options, fun couch_mrview_http:view_cb/2, VAcc, Args5), {ok, Resp#vacc.resp}. db_doc_req(#httpd{method='DELETE'}=Req, Db, DocId) -> diff --git a/src/couch_mrview/src/couch_mrview.erl b/src/couch_mrview/src/couch_mrview.erl index 09945f5..f5963e7 100644 --- a/src/couch_mrview/src/couch_mrview.erl +++ b/src/couch_mrview/src/couch_mrview.erl @@ -228,12 +228,13 @@ query_all_docs(Db, Args0, Callback, Acc) -> couch_index_util:hexsig(couch_hash:md5_hash(term_to_binary(Info))) end), Args1 = Args0#mrargs{view_type=map}, - Args2 = couch_mrview_util:validate_args(Args1), - {ok, Acc1} = case Args2#mrargs.preflight_fun of + Args2 = couch_mrview_util:set_extra(Args1, style, all_docs), + Args3 = couch_mrview_util:validate_args(Args2), + {ok, Acc1} = case Args3#mrargs.preflight_fun of PFFun when is_function(PFFun, 2) -> PFFun(Sig, Acc); _ -> {ok, Acc} end, - all_docs_fold(Db, Args2, Callback, Acc1). + all_docs_fold(Db, Args3, Callback, Acc1). query_view(Db, DDoc, VName) -> diff --git a/src/couch_mrview/src/couch_mrview_updater.erl b/src/couch_mrview/src/couch_mrview_updater.erl index 2b69eee..0c1a17c 100644 --- a/src/couch_mrview/src/couch_mrview_updater.erl +++ b/src/couch_mrview/src/couch_mrview_updater.erl @@ -315,7 +315,9 @@ write_kvs(State, UpdateSeq, ViewKVs, DocIdKeys, Seqs, Log0) -> design_opts=DesignOpts } = State, - Partitioned = couch_util:get_value(<<"partitioned">>, DesignOpts, false), + DbPartitioned = mem3:is_partitioned(State#mrst.db_name), + Partitioned = couch_util:get_value(<<"partitioned">>, DesignOpts, DbPartitioned), + Revs = dict:from_list(dict:fetch_keys(Log0)), Log = dict:fold(fun({Id, _Rev}, DIKeys, Acc) -> diff --git a/src/couch_mrview/src/couch_mrview_util.erl b/src/couch_mrview/src/couch_mrview_util.erl index 8a05535..acfb282 100644 --- a/src/couch_mrview/src/couch_mrview_util.erl +++ b/src/couch_mrview/src/couch_mrview_util.erl @@ -39,8 +39,11 @@ -define(GET_VIEW_RETRY_COUNT, 1). -define(GET_VIEW_RETRY_DELAY, 50). -define(LOWEST_KEY, null). --define(HIGHEST_KEY, {[{<<239, 191, 176>>, null}]}). % is {"\ufff0": null} - +-define(HIGHEST_KEY, {<<255, 255, 255, 255>>}). +-define(PARTITION_START(P), <<P/binary, $:>>). +-define(PARTITION_END(P), <<P/binary, $;>>). +-define(LOWEST(A, B), (if A < B -> A; true -> B end)). +-define(HIGHEST(A, B), (if A > B -> A; true -> B end)). -include_lib("couch/include/couch_db.hrl"). -include_lib("couch_mrview/include/couch_mrview.hrl"). @@ -580,20 +583,38 @@ validate_args(Args) -> _ -> mrverror(<<"Invalid value for `sorted`.">>) end, - case {get_extra(Args, partitioned, false), get_extra(Args, partition)} of - {true, undefined} -> + Style = get_extra(Args, style, normal), + Partitioned = get_extra(Args, partitioned, false), + Partition = get_extra(Args, partition), + + case {Style, Partitioned, Partition} of + {all_docs, true, _} -> + ok; % _all_docs can be called with or without partition parameter. + {all_docs, false, undefined} -> + ok; + {all_docs, false, _Partition} -> + mrverror(<<"`partition` parameter is not supported in this db.">>); + {normal, true, undefined} -> mrverror(<<"`partition` parameter is mandatory for queries to this view.">>); - {true, _Partition} -> + {normal, true, _Partition} -> ok; - {false, undefined} -> + {normal, false, undefined} -> ok; - {false, _Partition} -> + {normal, false, _Partition} -> mrverror(<<"`partition` parameter is not supported in this view.">>) end, - Args1 = case get_extra(Args, partitioned, false) of - true -> apply_partition(Args); - false -> Args + Args1 = case {Style, Partitioned, Partition} of + {all_docs, true, undefined} -> + Args; + {all_docs, true, Partition} -> + apply_partition(Args, all_docs); + {all_docs, false, _} -> + Args; + {normal, true, _} -> + apply_partition(Args, normal); + {normal, false, _} -> + Args end, Args1#mrargs{ @@ -614,15 +635,18 @@ determine_group_level(#mrargs{group=true, group_level=undefined}) -> determine_group_level(#mrargs{group_level=GroupLevel}) -> GroupLevel. -apply_partition(#mrargs{} = Args0) -> +apply_partition(#mrargs{} = Args0, Style) -> case get_extra(Args0, partition_applied, false) of true -> Args0; false -> Partition = get_extra(Args0, partition), - Args1 = apply_partition(Partition, Args0), + Args1 = case Style of + normal -> apply_partition(Partition, Args0); + all_docs -> apply_all_docs_partition(Partition, Args0) + end, set_extra(Args1, partition_applied, true) - end. + end; apply_partition(Partition, #mrargs{direction=fwd, start_key=undefined, end_key=undefined} = Args) -> Args#mrargs{start_key={p, Partition, ?LOWEST_KEY}, end_key={p, Partition, ?HIGHEST_KEY}}; @@ -646,6 +670,33 @@ apply_partition(Partition, #mrargs{start_key=SK0, end_key=EK0} = Args) -> Args#mrargs{start_key={p, Partition, SK0}, end_key={p, Partition, EK0}}. +%% all_docs is special as it's not really a view and is already +%% effectively partitioned as the partition is a prefix of all keys. +apply_all_docs_partition(Partition, #mrargs{direction=fwd, start_key=undefined, end_key=undefined} = Args) -> + Args#mrargs{start_key = ?PARTITION_START(Partition), end_key = ?PARTITION_END(Partition)}; + +apply_all_docs_partition(Partition, #mrargs{direction=rev, start_key=undefined, end_key=undefined} = Args) -> + Args#mrargs{start_key = ?PARTITION_END(Partition), end_key = ?PARTITION_START(Partition)}; + +apply_all_docs_partition(Partition, #mrargs{direction=fwd, start_key=SK, end_key=undefined} = Args) -> + Args#mrargs{start_key = ?HIGHEST(?PARTITION_START(Partition), SK), end_key = ?PARTITION_END(Partition)}; + +apply_all_docs_partition(Partition, #mrargs{direction=rev, start_key=SK, end_key=undefined} = Args) -> + Args#mrargs{start_key = ?LOWEST(?PARTITION_END(Partition), SK), end_key = ?PARTITION_START(Partition)}; + +apply_all_docs_partition(Partition, #mrargs{direction=fwd, start_key=undefined, end_key=EK} = Args) -> + Args#mrargs{start_key = ?PARTITION_START(Partition), end_key = ?LOWEST(?PARTITION_END(Partition), EK)}; + +apply_all_docs_partition(Partition, #mrargs{direction=rev, start_key=undefined, end_key=EK} = Args) -> + Args#mrargs{start_key = ?PARTITION_END(Partition), end_key = ?HIGHEST(?PARTITION_START(Partition), EK)}; + +apply_all_docs_partition(Partition, #mrargs{direction=fwd, start_key=SK, end_key=EK} = Args) -> + Args#mrargs{start_key = ?HIGHEST(?PARTITION_START(Partition), SK), end_key = ?LOWEST(?PARTITION_END(Partition), EK)}; + +apply_all_docs_partition(Partition, #mrargs{direction=rev, start_key=SK, end_key=EK} = Args) -> + Args#mrargs{start_key = ?LOWEST(?PARTITION_END(Partition), SK), end_key = ?HIGHEST(?PARTITION_START(Partition), EK)}. + + check_range(#mrargs{start_key=undefined}, _Cmp) -> ok; check_range(#mrargs{end_key=undefined}, _Cmp) -> diff --git a/src/fabric/src/fabric.erl b/src/fabric/src/fabric.erl index 97e323e..4de365f 100644 --- a/src/fabric/src/fabric.erl +++ b/src/fabric/src/fabric.erl @@ -294,9 +294,12 @@ all_docs(DbName, Callback, Acc, QueryArgs) -> #mrargs{} | [option()]) -> {ok, any()} | {error, Reason :: term()}. -all_docs(DbName, Options, Callback, Acc0, #mrargs{} = QueryArgs) when + +all_docs(DbName, Options, Callback, Acc0, #mrargs{} = QueryArgs0) when is_function(Callback, 2) -> - fabric_view_all_docs:go(dbname(DbName), opts(Options), QueryArgs, Callback, Acc0); + DbPartitioned = mem3:is_partitioned(dbname(DbName)), + QueryArgs1 = couch_mrview_util:set_extra(QueryArgs0, partitioned, DbPartitioned), + fabric_view_all_docs:go(dbname(DbName), opts(Options), QueryArgs1, Callback, Acc0); %% @doc convenience function that takes a keylist rather than a record %% @equiv all_docs(DbName, Callback, Acc0, kl_to_query_args(QueryArgs)) diff --git a/src/fabric/src/fabric_view.erl b/src/fabric/src/fabric_view.erl index 844b44d..c0e2974 100644 --- a/src/fabric/src/fabric_view.erl +++ b/src/fabric/src/fabric_view.erl @@ -122,7 +122,7 @@ maybe_send_row(State) -> user_acc = AccIn, query_args = QueryArgs } = State, - Partitioned = couch_mrview_util:get_extra(QueryArgs, partitioned, false), + Partitioned = couch_mrview_util:get_extra(QueryArgs, partitioned), case fabric_dict:any(0, Counters) of true -> {ok, State}; diff --git a/src/fabric/src/fabric_view_all_docs.erl b/src/fabric/src/fabric_view_all_docs.erl index ac16dac..9616394 100644 --- a/src/fabric/src/fabric_view_all_docs.erl +++ b/src/fabric/src/fabric_view_all_docs.erl @@ -118,7 +118,7 @@ go(DbName, _Options, Workers, QueryArgs, Callback, Acc0) -> #mrargs{limit = Limit, skip = Skip, update_seq = UpdateSeq} = QueryArgs, State = #collector{ db_name = DbName, - query_args = QueryArgs, + query_args = couch_mrview_util:set_extra(QueryArgs, style, all_docs), callback = Callback, counters = fabric_dict:init(Workers, 0), skip = Skip, @@ -136,6 +136,28 @@ go(DbName, _Options, Workers, QueryArgs, Callback, Acc0) -> {ok, Resp} end. +shards(DbName, Args) -> + case couch_mrview_util:get_extra(Args, partitioned) of + true -> + StartKey = partition(Args#mrargs.start_key), + EndKey = partition(Args#mrargs.end_key), + case {StartKey, EndKey} of + {Same, Same} when Same =/= undefined -> + mem3:shards(DbName, <<Same/binary, ":foo">>); + {_, _} -> + mem3:shards(DbName) + end; + _ -> + mem3:shards(DbName) + end. + +partition(undefined) -> + undefined; +partition(null) -> + null; +partition(Key) when is_binary(Key) -> + hd(binary:split(Key, <<":">>)). + handle_message({rexi_DOWN, _, {_, NodeRef}, _}, _, State) -> fabric_view:check_down_shards(State, NodeRef); diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl index dbea36e..63c010f 100644 --- a/src/mango/src/mango_cursor_view.erl +++ b/src/mango/src/mango_cursor_view.erl @@ -29,6 +29,7 @@ -include_lib("couch/include/couch_db.hrl"). -include_lib("couch_mrview/include/couch_mrview.hrl"). -include("mango_cursor.hrl"). +-include("mango_idx.hrl"). -include("mango_idx_view.hrl"). create(Db, Indexes, Selector, Opts) -> @@ -71,7 +72,8 @@ explain(Cursor) -> {direction, Args#mrargs.direction}, {stable, Args#mrargs.stable}, {update, Args#mrargs.update}, - {conflicts, Args#mrargs.conflicts} + {conflicts, Args#mrargs.conflicts}, + {extra, {Args#mrargs.extra}} ]}}]. @@ -93,14 +95,27 @@ maybe_replace_max_json([H | T] = EndKey) when is_list(EndKey) -> maybe_replace_max_json(EndKey) -> EndKey. -base_args(#cursor{index = Idx} = Cursor) -> - #mrargs{ +base_args(#cursor{index = Idx, opts = Opts} = Cursor) -> + Args1 = #mrargs{ view_type = map, reduce = false, start_key = mango_idx:start_key(Idx, Cursor#cursor.ranges), end_key = mango_idx:end_key(Idx, Cursor#cursor.ranges), include_docs = true - }. + }, + Args2 = add_partition_opts(Args1, couch_util:get_value(partition, Opts)), + add_style(Idx, Args2). + +add_partition_opts(#mrargs{} = Args, <<>>) -> + Args; +add_partition_opts(#mrargs{} = Args, Partition) -> + Args1 = couch_mrview_util:set_extra(Args, partitioned, true), + couch_mrview_util:set_extra(Args1, partition, Partition). + +add_style(#idx{def = all_docs}, Args) -> + couch_mrview_util:set_extra(Args, style, all_docs); +add_style(_, Args) -> + Args. execute(#cursor{db = Db, index = Idx, execution_stats = Stats} = Cursor0, UserFun, UserAcc) -> diff --git a/src/mango/src/mango_opts.erl b/src/mango/src/mango_opts.erl index 7bae9c9..87d876a 100644 --- a/src/mango/src/mango_opts.erl +++ b/src/mango/src/mango_opts.erl @@ -81,6 +81,12 @@ validate_find({Props}) -> {tag, selector}, {validator, fun validate_selector/1} ]}, + {<<"partition">>, [ + {tag, partition}, + {optional, true}, + {default, <<>>}, + {validator, fun is_string/1} + ]}, {<<"use_index">>, [ {tag, use_index}, {optional, true}, diff --git a/src/mem3/src/mem3.erl b/src/mem3/src/mem3.erl index aecca2f..f113357 100644 --- a/src/mem3/src/mem3.erl +++ b/src/mem3/src/mem3.erl @@ -352,7 +352,10 @@ is_partitioned(DbName0) when is_binary(DbName0) -> is_partitioned(mem3:shards(DbName)) end; -is_partitioned(Shards) when is_list(Shards) -> +is_partitioned([#shard{} | _] = Shards) -> + lists:all(fun is_partitioned/1, Shards); + +is_partitioned([#ordered_shard{} | _] = Shards) -> lists:all(fun is_partitioned/1, Shards); is_partitioned(#shard{opts=Opts}) ->
