This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch feature/user-partitioned-databases-davisp
in repository https://gitbox.apache.org/repos/asf/couchdb.git
The following commit(s) were added to
refs/heads/feature/user-partitioned-databases-davisp by this push:
new 7bbeb96 Support partitioned queries in Mango
7bbeb96 is described below
commit 7bbeb9685356fe164d80fe14df6e4faaf2d60fd1
Author: Paul J. Davis <[email protected]>
AuthorDate: Mon Oct 29 14:08:05 2018 -0500
Support partitioned queries in Mango
Co-authored-by: Garren Smith <[email protected]>
Co-authored-by: Robert Newson <[email protected]>
---
src/mango/src/mango_cursor.erl | 1 +
src/mango/src/mango_cursor_view.erl | 4 ++
src/mango/src/mango_error.erl | 7 +++
src/mango/src/mango_idx.erl | 105 +++++++++++++++++++++++++++++++++---
src/mango/src/mango_idx.hrl | 1 +
src/mango/src/mango_opts.erl | 12 +++++
6 files changed, 124 insertions(+), 6 deletions(-)
diff --git a/src/mango/src/mango_cursor.erl b/src/mango/src/mango_cursor.erl
index 5d2ea71..c6f21dd 100644
--- a/src/mango/src/mango_cursor.erl
+++ b/src/mango/src/mango_cursor.erl
@@ -71,6 +71,7 @@ explain(#cursor{}=Cursor) ->
{[
{dbname, mango_idx:dbname(Idx)},
{index, mango_idx:to_json(Idx)},
+ {partitioned, mango_idx:partitioned(Idx)},
{selector, Selector},
{opts, {Opts}},
{limit, Limit},
diff --git a/src/mango/src/mango_cursor_view.erl
b/src/mango/src/mango_cursor_view.erl
index 174381e..b28b8c2 100644
--- a/src/mango/src/mango_cursor_view.erl
+++ b/src/mango/src/mango_cursor_view.erl
@@ -71,6 +71,7 @@ explain(Cursor) ->
{include_docs, Args#mrargs.include_docs},
{view_type, Args#mrargs.view_type},
{reduce, Args#mrargs.reduce},
+ {partition, couch_mrview_util:get_extra(Args, partition)},
{start_key, maybe_replace_max_json(Args#mrargs.start_key)},
{end_key, maybe_replace_max_json(Args#mrargs.end_key)},
{direction, Args#mrargs.direction},
@@ -395,6 +396,9 @@ apply_opts([{update, false} | Rest], Args) ->
update = false
},
apply_opts(Rest, NewArgs);
+apply_opts([{partition, Partition} | Rest], Args) when is_binary(partition) ->
+ NewArgs = couch_mrview_util:set_extra(Args, partition, Partition),
+ apply_opts(Rest, NewArgs);
apply_opts([{_, _} | Rest], Args) ->
% Ignore unknown options
apply_opts(Rest, Args).
diff --git a/src/mango/src/mango_error.erl b/src/mango/src/mango_error.erl
index b2bbb39..b44ff68 100644
--- a/src/mango/src/mango_error.erl
+++ b/src/mango/src/mango_error.erl
@@ -104,6 +104,13 @@ info(mango_idx, {invalid_index_type, BadType}) ->
<<"invalid_index">>,
fmt("Invalid type for index: ~s", [BadType])
};
+info(mango_idx, {partitoned_option_mismatch, BadDDoc}) ->
+ {
+ 400,
+ <<"invalid_partitioned_option">>,
+ fmt("Requested partitioned option does not match existing value on"
+ " design document ~s", [BadDDoc])
+ };
info(mango_idx, invalid_query_ddoc_language) ->
{
400,
diff --git a/src/mango/src/mango_idx.erl b/src/mango/src/mango_idx.erl
index 8af92b9..af74b63 100644
--- a/src/mango/src/mango_idx.erl
+++ b/src/mango/src/mango_idx.erl
@@ -33,6 +33,7 @@
name/1,
type/1,
def/1,
+ partitioned/1,
opts/1,
columns/1,
is_usable/3,
@@ -59,8 +60,10 @@ list(Db) ->
get_usable_indexes(Db, Selector, Opts) ->
ExistingIndexes = mango_idx:list(Db),
-
- GlobalIndexes =
mango_cursor:remove_indexes_with_partial_filter_selector(ExistingIndexes),
+ MatchingPartitionIndexes = filter_partition_indexes(ExistingIndexes, Opts),
+ GlobalIndexes = mango_cursor:remove_indexes_with_partial_filter_selector(
+ MatchingPartitionIndexes
+ ),
UserSpecifiedIndex =
mango_cursor:maybe_filter_indexes_by_ddoc(ExistingIndexes, Opts),
UsableIndexes0 = lists:usort(GlobalIndexes ++ UserSpecifiedIndex),
@@ -110,6 +113,7 @@ new(Db, Opts) ->
name = IdxName,
type = Type,
def = Def,
+ partitioned = get_idx_partitioned(Opts),
opts = filter_opts(Opts)
}}.
@@ -121,10 +125,11 @@ validate_new(Idx, Db) ->
add(DDoc, Idx) ->
Mod = idx_mod(Idx),
- {ok, NewDDoc} = Mod:add(DDoc, Idx),
+ {ok, NewDDoc1} = Mod:add(DDoc, Idx),
+ NewDDoc2 = set_ddoc_partitioned(DDoc, Idx),
% Round trip through JSON for normalization
- Body = ?JSON_DECODE(?JSON_ENCODE(NewDDoc#doc.body)),
- {ok, NewDDoc#doc{body = Body}}.
+ Body = ?JSON_DECODE(?JSON_ENCODE(NewDDoc2#doc.body)),
+ {ok, NewDDoc2#doc{body = Body}}.
remove(DDoc, Idx) ->
@@ -176,7 +181,8 @@ from_ddoc(Db, {Props}) ->
lists:map(fun(Idx) ->
Idx#idx{
dbname = DbName,
- ddoc = DDoc
+ ddoc = DDoc,
+ partitioned = set_idx_partitioned(Db, Props)
}
end, Idxs).
@@ -213,6 +219,10 @@ def(#idx{def=Def}) ->
Def.
+partitioned(#idx{partitioned=Partitioned}) ->
+ Partitioned.
+
+
opts(#idx{opts=Opts}) ->
Opts.
@@ -329,6 +339,87 @@ gen_name(Idx, Opts0) ->
mango_util:enc_hex(Sha).
+get_idx_partitioned(Opts) ->
+ case proplists:get_value(partitioned, Opts) of
+ B when is_boolean(B) ->
+ B;
+ undefined ->
+ undefined
+ end.
+
+
+set_ddoc_partitioned(DDoc, Idx) ->
+ % We have to verify that the new index being added
+ % to this design document either matches the current
+ % ddoc's design options *or* this is a new design doc
+ #doc{
+ id = DDocId,
+ revs = Revs,
+ body = {BodyProps}
+ } = DDoc,
+ OldDOpts = couch_util:get_value(<<"options">>, BodyProps),
+ OldOpt = CurrDOpts of
+ {OldDOptProps} when is_list(OldDOptProps) ->
+ couch_util:get_value(<<"partitioned">>, OldDOptProps);
+ _ ->
+ undefined
+ end,
+ % If new matches old we're done
+ if Idx#idx.partitioned == OldOpt -> DDoc; true ->
+ % If we're creating a ddoc then we can set the options
+ NewBodyProps = case Revs == {0, []} of
+ true when Idx#idx.partitioned /= undefined ->
+ set_ddoc_partitioned_option(DDoc, Idx#idx.partitioned);
+ true when NewOpts == undefined ->
+ DDoc;
+ false ->
+ ?MANGO_ERROR({partitioned_option_mismatch, DDocId})
+ end
+ end.
+
+
+set_ddoc_partitioned_option(DDoc, Partitioned) ->
+ #doc{
+ body = {BodyProps}
+ } = DDoc,
+ NewProps = case couch_util:get_value(<<"options">>, BodyProps) of
+ {Existing} when is_list(Existing) ->
+ Opt = {<<"partitioned">>, Partitioned},
+ New = lists:keystore(<<"partitioned">>, 1, Existing, Opt),
+ lists:keystore(<<"options">>, 1, BodyProps, {<<"options">>, New});
+ undefined ->
+ New = {<<"options">>, {[{<<"partitioned">>, Partitioned}]}},
+ lists:keystore(<<"options">>, 1, BodyProps, New)
+ end,
+ DDoc#doc{body = {NewProps}}.
+
+
+set_idx_partitioned(Db, DDocProps) ->
+ Default = fabric_util:is_partitioned(Db),
+ case couch_util:get_value(<<"options">>, DDocProps) of
+ {DesignOpts} ->
+ case couch_util:get_value(<<"partitioned">>, DesignOpts) of
+ P when is_boolean(P) ->
+ P;
+ undefined ->
+ Default
+ end;
+ undefined ->
+ Default
+ end.
+
+
+filter_partition_indexes(Indexes, Opts) ->
+ PFilt = case couch_util:get_value(partition, Opts) of
+ P when is_binary(P) ->
+ fun(#idx{partitioned = P}) -> P end,
+ undefined ->
+ fun(#idx{partitioned = P}) -> not P end
+ end,
+ Filt = fun(Idx) -> type(Idx) == <<"special">> orelse PFilt(Idx) end,
+ lists:filter(Filt, Indexes).
+
+
filter_opts([]) ->
[];
filter_opts([{user_ctx, _} | Rest]) ->
@@ -341,6 +432,8 @@ filter_opts([{type, _} | Rest]) ->
filter_opts(Rest);
filter_opts([{w, _} | Rest]) ->
filter_opts(Rest);
+filter_opts([{partitioned, _} | Rest]) ->
+ filter_opts(Rest);
filter_opts([Opt | Rest]) ->
[Opt | filter_opts(Rest)].
diff --git a/src/mango/src/mango_idx.hrl b/src/mango/src/mango_idx.hrl
index 712031b..9725950 100644
--- a/src/mango/src/mango_idx.hrl
+++ b/src/mango/src/mango_idx.hrl
@@ -16,5 +16,6 @@
name,
type,
def,
+ partitioned,
opts
}).
diff --git a/src/mango/src/mango_opts.erl b/src/mango/src/mango_opts.erl
index 7bae9c9..6eb1259 100644
--- a/src/mango/src/mango_opts.erl
+++ b/src/mango/src/mango_opts.erl
@@ -70,6 +70,12 @@ validate_idx_create({Props}) ->
{optional, true},
{default, 2},
{validator, fun is_pos_integer/1}
+ ]},
+ {<<"partitioned">>, [
+ {tag, partitioned},
+ {optional, true},
+ {default, undefined},
+ {validator, fun mango_opts:is_boolean/1}
]}
],
validate(Props, Opts).
@@ -117,6 +123,12 @@ validate_find({Props}) ->
{default, []},
{validator, fun validate_fields/1}
]},
+ {<<"partition">>, [
+ {tag, partition},
+ {optional, true},
+ {default, undefined},
+ {validator, fun is_string/1}
+ ]},
{<<"r">>, [
{tag, r},
{optional, true},
