This is an automated email from the ASF dual-hosted git repository. rnewson pushed a commit to branch user-partitioned-dbs-6 in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit bededec159fde6b8bd9dd494f9c369737b587742 Author: Garren Smith <[email protected]> AuthorDate: Thu Sep 13 12:47:38 2018 +0200 add partition support to mango text and fix failing tests --- src/mango/src/mango_cursor_text.erl | 17 +++++++++++- src/mango/src/mango_cursor_view.erl | 4 ++- src/mango/src/mango_idx.erl | 44 ++++++++++++++++++++----------- src/mango/src/mango_idx_text.erl | 10 ++++--- src/mango/src/mango_idx_view.erl | 12 +-------- src/mango/test/05-index-selection-test.py | 14 +++++----- src/mango/test/user_docs.py | 3 ++- 7 files changed, 64 insertions(+), 40 deletions(-) diff --git a/src/mango/src/mango_cursor_text.erl b/src/mango/src/mango_cursor_text.erl index 3883bc8..7d78e3c 100644 --- a/src/mango/src/mango_cursor_text.erl +++ b/src/mango/src/mango_cursor_text.erl @@ -25,6 +25,7 @@ -include_lib("dreyfus/include/dreyfus.hrl"). -include("mango_cursor.hrl"). -include("mango.hrl"). +-include("mango_idx.hrl"). -record(cacc, { @@ -96,6 +97,20 @@ execute(Cursor, UserFun, UserAcc) -> sort = sort_query(Opts, Selector), raw_bookmark = true }, + + DbPartitioned = mem3:is_partitioned(couch_db:name(Db)), + Partitioned = couch_util:get_value(partitioned, Idx#idx.design_opts, DbPartitioned), + QueryArgs1 = case Partitioned of + true -> + Partition = couch_util:get_value(partition, Opts), + QueryArgs#index_query_args{ + partition = Partition, + partitioned = true + }; + false -> + QueryArgs + end, + CAcc = #cacc{ selector = Selector, dbname = couch_db:name(Db), @@ -104,7 +119,7 @@ execute(Cursor, UserFun, UserAcc) -> bookmark = get_bookmark(Opts), limit = Limit, skip = Skip, - query_args = QueryArgs, + query_args = QueryArgs1, user_fun = UserFun, user_acc = UserAcc, fields = Cursor#cursor.fields, diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl index 3eedf99..1623420 100644 --- a/src/mango/src/mango_cursor_view.erl +++ b/src/mango/src/mango_cursor_view.erl @@ -103,7 +103,9 @@ base_args(#cursor{index = Idx, opts = Opts} = Cursor) -> end_key = mango_idx:end_key(Idx, Cursor#cursor.ranges), include_docs = true }, - Partitioned = couch_util:get_value(partitioned, Idx#idx.design_opts), + + DbPartitioned = mem3:is_partitioned(couch_db:name(Cursor#cursor.db)), + Partitioned = couch_util:get_value(partitioned, Idx#idx.design_opts, DbPartitioned), Args2 = couch_mrview_util:set_extra(Args1, partitioned, Partitioned), Args3 = case couch_util:get_value(partition, Opts) of <<>> -> diff --git a/src/mango/src/mango_idx.erl b/src/mango/src/mango_idx.erl index e051218..261e8f0 100644 --- a/src/mango/src/mango_idx.erl +++ b/src/mango/src/mango_idx.erl @@ -23,6 +23,7 @@ new/2, validate_new/2, + validate_design_opts/1, add/2, remove/2, from_ddoc/2, @@ -59,7 +60,7 @@ list(Db) -> get_usable_indexes(Db, Selector, Opts) -> PQ = is_partitioned_query(Opts), - ExistingIndexes = filter_indexes_by_partitioned( + ExistingIndexes = filter_indexes_by_partitioned(mem3:is_partitioned(db_to_name(Db)), mango_idx:list(Db), PQ), GlobalIndexes = mango_cursor:remove_indexes_with_partial_filter_selector(ExistingIndexes), UserSpecifiedIndex = mango_cursor:maybe_filter_indexes_by_ddoc(ExistingIndexes, Opts), @@ -76,23 +77,24 @@ get_usable_indexes(Db, Selector, Opts) -> end. -filter_indexes_by_partitioned(Indexes, PQ) -> - filter_indexes_by_partitioned(Indexes, PQ, []). +filter_indexes_by_partitioned(false, Indexes, _PQ) -> + Indexes; +filter_indexes_by_partitioned(DbPartitioned, Indexes, PQ) -> + FilterFun = fun (Idx)-> + PartitionedIdx = case lists:keyfind(partitioned, 1, Idx#idx.design_opts) of + {partitioned, PI} -> PI; + false -> DbPartitioned + end, + filter_index_by_partitioned(Idx#idx.def, PartitionedIdx, PQ) + end, + lists:filter(FilterFun, Indexes). -filter_indexes_by_partitioned([], _PQ, Acc) -> - lists:reverse(Acc); -filter_indexes_by_partitioned([Idx | Rest], PQ, Acc) -> - {partitioned, PI} = lists:keyfind(partitioned, 1, Idx#idx.design_opts), - case {Idx#idx.def, PI, PQ} of - {all_docs, _, _} -> - % all_docs works both ways. - filter_indexes_by_partitioned(Rest, PQ, [Idx | Acc]); - {_, Same, Same} -> - filter_indexes_by_partitioned(Rest, PQ, [Idx | Acc]); - {_, _, _} -> - filter_indexes_by_partitioned(Rest, PQ, Acc) - end. +filter_index_by_partitioned(all_docs, _PartitionedIdx, _PartitionedQuery) -> + true; + +filter_index_by_partitioned(_Def, PartitionedIdx, PartitionedQuery) -> + PartitionedIdx =:= PartitionedQuery. is_partitioned_query(Opts) -> @@ -145,6 +147,16 @@ validate_new(Idx, Db) -> Mod:validate_new(Idx, Db). +validate_design_opts(Props) -> + case lists:keyfind(<<"options">>, 1, Props) of + {<<"options">>, {[{<<"partitioned">>, P}]}} + when is_boolean(P) -> + [{partitioned, P}]; + _ -> + [] + end. + + add(DDoc, Idx) -> Mod = idx_mod(Idx), {ok, NewDDoc} = Mod:add(DDoc, Idx), diff --git a/src/mango/src/mango_idx_text.erl b/src/mango/src/mango_idx_text.erl index 29b4441..906bf4d 100644 --- a/src/mango/src/mango_idx_text.erl +++ b/src/mango/src/mango_idx_text.erl @@ -51,7 +51,8 @@ add(#doc{body={Props0}}=DDoc, Idx) -> Texts2 = lists:keystore(element(1, NewText), 1, Texts1, NewText), Props1 = lists:keystore(<<"indexes">>, 1, Props0, {<<"indexes">>, {Texts2}}), - {ok, DDoc#doc{body={Props1}}}. + Props2 = lists:keystore(<<"options">>, 1, Props1, {<<"options">>, {Idx#idx.design_opts}}), + {ok, DDoc#doc{body={Props2}}}. remove(#doc{body={Props0}}=DDoc, Idx) -> @@ -75,6 +76,7 @@ remove(#doc{body={Props0}}=DDoc, Idx) -> from_ddoc({Props}) -> + DesignOpts = mango_idx:validate_design_opts(Props), case lists:keyfind(<<"indexes">>, 1, Props) of {<<"indexes">>, {Texts}} when is_list(Texts) -> lists:flatmap(fun({Name, {VProps}}) -> @@ -85,7 +87,8 @@ from_ddoc({Props}) -> I = #idx{ type = <<"text">>, name = Name, - def = Def + def = Def, + design_opts = DesignOpts }, [I] end @@ -257,7 +260,8 @@ opts() -> make_text(Idx) -> Text= {[ {<<"index">>, Idx#idx.def}, - {<<"analyzer">>, construct_analyzer(Idx#idx.def)} + {<<"analyzer">>, construct_analyzer(Idx#idx.def)}, + {<<"options">>, {Idx#idx.opts}} ]}, {Idx#idx.name, Text}. diff --git a/src/mango/src/mango_idx_view.erl b/src/mango/src/mango_idx_view.erl index ef1ca59..c0567f5 100644 --- a/src/mango/src/mango_idx_view.erl +++ b/src/mango/src/mango_idx_view.erl @@ -79,7 +79,7 @@ remove(#doc{body={Props0}}=DDoc, Idx) -> from_ddoc({Props}) -> - DesignOpts = validate_design_opts(Props), + DesignOpts = mango_idx:validate_design_opts(Props), case lists:keyfind(<<"views">>, 1, Props) of {<<"views">>, {Views}} when is_list(Views) -> lists:flatmap(fun({Name, {VProps}}) -> @@ -252,16 +252,6 @@ validate_ddoc(VProps) -> end. -validate_design_opts(Props) -> - case lists:keyfind(<<"options">>, 1, Props) of - {<<"options">>, {[{<<"partitioned">>, P}]}} - when is_boolean(P) -> - [{partitioned, P}]; - _ -> - [] - end. - - % This function returns a list of indexes that % can be used to restrict this query. This works by % searching the selector looking for field names that diff --git a/src/mango/test/05-index-selection-test.py b/src/mango/test/05-index-selection-test.py index 2a40fda..0c93410 100644 --- a/src/mango/test/05-index-selection-test.py +++ b/src/mango/test/05-index-selection-test.py @@ -40,7 +40,7 @@ class IndexSelectionTests: def test_with_or(self): # index on ["company","manager"] - ddocid = "_design/a0c425a60cf3c3c09e3c537c9ef20059dcef9198" + ddocid = "_design/company-manager" resp = self.db.find({ "company": { @@ -56,7 +56,7 @@ class IndexSelectionTests: def test_use_most_columns(self): # ddoc id for the age index - ddocid = "_design/ad3d537c03cd7c6a43cf8dff66ef70ea54c2b40f" + ddocid = "_design/age" resp = self.db.find({ "name.first": "Stephanie", "name.last": "Something or other", @@ -81,7 +81,7 @@ class IndexSelectionTests: def test_invalid_use_index(self): # ddoc id for the age index - ddocid = "_design/ad3d537c03cd7c6a43cf8dff66ef70ea54c2b40f" + ddocid = "_design/age" r = self.db.find({}, use_index=ddocid, return_raw=True) self.assertEqual(r["warning"], '{0} was not used because it does not contain a valid index for this query.'.format(ddocid)) @@ -101,7 +101,7 @@ class IndexSelectionTests: def test_reject_use_index_invalid_fields(self): # index on ["company","manager"] which should not be valid - ddocid = "_design/a0c425a60cf3c3c09e3c537c9ef20059dcef9198" + ddocid = "_design/company-manager" selector = { "company": "Pharmex" } @@ -114,8 +114,8 @@ class IndexSelectionTests: def test_reject_use_index_ddoc_and_name_invalid_fields(self): # index on ["company","manager"] which should not be valid - ddocid = "_design/a0c425a60cf3c3c09e3c537c9ef20059dcef9198" - name = "a0c425a60cf3c3c09e3c537c9ef20059dcef9198" + ddocid = "_design/company-manager" + name = "company-manager" selector = { "company": "Pharmex" } @@ -130,7 +130,7 @@ class IndexSelectionTests: def test_reject_use_index_sort_order(self): # index on ["company","manager"] which should not be valid # and there is no valid fallback (i.e. an index on ["company"]) - ddocid = "_design/a0c425a60cf3c3c09e3c537c9ef20059dcef9198" + ddocid = "_design/company-manager" selector = { "company": {"$gt": None} } diff --git a/src/mango/test/user_docs.py b/src/mango/test/user_docs.py index 02ffe9f..7455f0a 100644 --- a/src/mango/test/user_docs.py +++ b/src/mango/test/user_docs.py @@ -87,7 +87,8 @@ def add_view_indexes(db, kwargs): ["ordered"] ] for idx in indexes: - assert db.create_index(idx) is True + name = '-'.join(idx) + assert db.create_index(idx, ddoc=name, name=name) is True def add_text_indexes(db, kwargs):
