This is an automated email from the ASF dual-hosted git repository. garren pushed a commit to branch fdb-mango-indexes in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 93399f52cdae498054d5dd66916fbc6b01e9c80c Author: Garren Smith <[email protected]> AuthorDate: Mon Feb 10 18:15:00 2020 +0200 getting tests to pass --- src/mango/src/mango_cursor.erl | 7 ++++++ src/mango/src/mango_execution_stats.erl | 8 ------ src/mango/src/mango_execution_stats.hrl | 1 - src/mango/src/mango_fdb.erl | 2 +- src/mango/src/mango_idx.erl | 5 ++-- src/mango/src/mango_idx_special.erl | 4 ++- src/mango/test/12-use-correct-index-test.py | 38 +++++++++++++++++------------ src/mango/test/15-execution-stats-test.py | 6 ++--- src/mango/test/16-index-selectors-test.py | 2 ++ src/mango/test/17-multi-type-value-test.py | 4 +-- src/mango/test/18-json-sort.py | 6 ++--- src/mango/test/19-find-conflicts.py | 2 +- src/mango/test/user_docs.py | 4 ++- 13 files changed, 49 insertions(+), 40 deletions(-) diff --git a/src/mango/src/mango_cursor.erl b/src/mango/src/mango_cursor.erl index c6f21dd..8bdf022 100644 --- a/src/mango/src/mango_cursor.erl +++ b/src/mango/src/mango_cursor.erl @@ -19,6 +19,7 @@ execute/3, maybe_filter_indexes_by_ddoc/2, remove_indexes_with_partial_filter_selector/1, + remove_unbuilt_indexes/1, maybe_add_warning/3 ]). @@ -123,6 +124,12 @@ remove_indexes_with_partial_filter_selector(Indexes) -> lists:filter(FiltFun, Indexes). +remove_unbuilt_indexes(Indexes) -> + lists:filter(fun (Idx) -> + Idx#idx.build_status == ?MANGO_INDEX_READY + end, Indexes). + + create_cursor(Db, Indexes, Selector, Opts) -> [{CursorMod, CursorModIndexes} | _] = group_indexes_by_type(Indexes), CursorMod:create(Db, CursorModIndexes, Selector, Opts). diff --git a/src/mango/src/mango_execution_stats.erl b/src/mango/src/mango_execution_stats.erl index 7e8afd7..a3572a1 100644 --- a/src/mango/src/mango_execution_stats.erl +++ b/src/mango/src/mango_execution_stats.erl @@ -18,7 +18,6 @@ incr_keys_examined/1, incr_docs_examined/1, incr_docs_examined/2, - incr_quorum_docs_examined/1, incr_results_returned/1, log_start/1, log_end/1, @@ -33,7 +32,6 @@ to_json(Stats) -> {[ {total_keys_examined, Stats#execution_stats.totalKeysExamined}, {total_docs_examined, Stats#execution_stats.totalDocsExamined}, - {total_quorum_docs_examined, Stats#execution_stats.totalQuorumDocsExamined}, {results_returned, Stats#execution_stats.resultsReturned}, {execution_time_ms, Stats#execution_stats.executionTimeMs} ]}. @@ -55,12 +53,6 @@ incr_docs_examined(Stats, N) -> }. -incr_quorum_docs_examined(Stats) -> - Stats#execution_stats { - totalQuorumDocsExamined = Stats#execution_stats.totalQuorumDocsExamined + 1 - }. - - incr_results_returned(Stats) -> Stats#execution_stats { resultsReturned = Stats#execution_stats.resultsReturned + 1 diff --git a/src/mango/src/mango_execution_stats.hrl b/src/mango/src/mango_execution_stats.hrl index ea5ed5e..783c1e7 100644 --- a/src/mango/src/mango_execution_stats.hrl +++ b/src/mango/src/mango_execution_stats.hrl @@ -13,7 +13,6 @@ -record(execution_stats, { totalKeysExamined = 0, totalDocsExamined = 0, - totalQuorumDocsExamined = 0, resultsReturned = 0, executionStartTime, executionTimeMs diff --git a/src/mango/src/mango_fdb.erl b/src/mango/src/mango_fdb.erl index 9a17a85..1274f0e 100644 --- a/src/mango/src/mango_fdb.erl +++ b/src/mango/src/mango_fdb.erl @@ -226,7 +226,7 @@ fold_cb({Key, Val}, Acc) -> } = Acc, {{_, DocId}} = erlfdb_tuple:unpack(Key, MangoIdxPrefix), SortKeys = couch_views_encoding:decode(Val), - {ok, Doc} = fabric2_db:open_doc(Db, DocId), + {ok, Doc} = fabric2_db:open_doc(Db, DocId, [{conflicts, true}]), JSONDoc = couch_doc:to_json_obj(Doc, []), io:format("PRINT ~p ~p ~n", [DocId, JSONDoc]), case Callback({doc, SortKeys, JSONDoc}, Cursor) of diff --git a/src/mango/src/mango_idx.erl b/src/mango/src/mango_idx.erl index 5be8530..501d8ef 100644 --- a/src/mango/src/mango_idx.erl +++ b/src/mango/src/mango_idx.erl @@ -103,12 +103,13 @@ get_rev_info(Row) -> get_usable_indexes(Db, Selector, Opts) -> - ExistingIndexes = mango_idx:list(Db), + ExistingIndexes = mango_idx:add_build_status(Db, mango_idx:list(Db)), GlobalIndexes = mango_cursor:remove_indexes_with_partial_filter_selector( ExistingIndexes ), + GlobalIndexes1 = mango_cursor:remove_unbuilt_indexes(GlobalIndexes), UserSpecifiedIndex = mango_cursor:maybe_filter_indexes_by_ddoc(ExistingIndexes, Opts), - UsableIndexes0 = lists:usort(GlobalIndexes ++ UserSpecifiedIndex), + UsableIndexes0 = lists:usort(GlobalIndexes1 ++ UserSpecifiedIndex), UsableIndexes1 = filter_partition_indexes(UsableIndexes0, Opts), SortFields = get_sort_fields(Opts), diff --git a/src/mango/src/mango_idx_special.erl b/src/mango/src/mango_idx_special.erl index ac6efc7..844a0ba 100644 --- a/src/mango/src/mango_idx_special.erl +++ b/src/mango/src/mango_idx_special.erl @@ -27,6 +27,7 @@ -include_lib("couch/include/couch_db.hrl"). +-include("mango.hrl"). -include("mango_idx.hrl"). @@ -55,7 +56,8 @@ to_json(#idx{def=all_docs}) -> {<<"fields">>, [{[ {<<"_id">>, <<"asc">>} ]}]} - ]}} + ]}}, + {build_status, ?MANGO_INDEX_READY} ]}. diff --git a/src/mango/test/12-use-correct-index-test.py b/src/mango/test/12-use-correct-index-test.py index 987f507..d495e94 100644 --- a/src/mango/test/12-use-correct-index-test.py +++ b/src/mango/test/12-use-correct-index-test.py @@ -54,36 +54,41 @@ class ChooseCorrectIndexForDocs(mango.DbPerClass): self.db.save_docs(copy.deepcopy(DOCS)) def test_choose_index_with_one_field_in_index(self): - self.db.create_index(["name", "age", "user_id"], ddoc="aaa") - self.db.create_index(["name"], ddoc="zzz") + self.db.create_index(["name", "age", "user_id"], ddoc="aaa", wait_for_built_index=False) + self.db.create_index(["name"], ddoc="zzz", wait_for_built_index=False) + self.db.wait_for_built_indexes() explain = self.db.find({"name": "Eddie"}, explain=True) self.assertEqual(explain["index"]["ddoc"], "_design/zzz") def test_choose_index_with_two(self): - self.db.create_index(["name", "age", "user_id"], ddoc="aaa") - self.db.create_index(["name", "age"], ddoc="bbb") - self.db.create_index(["name"], ddoc="zzz") + self.db.create_index(["name", "age", "user_id"], ddoc="aaa", wait_for_built_index=False) + self.db.create_index(["name", "age"], ddoc="bbb", wait_for_built_index=False) + self.db.create_index(["name"], ddoc="zzz", wait_for_built_index=False) + self.db.wait_for_built_indexes() explain = self.db.find({"name": "Eddie", "age": {"$gte": 12}}, explain=True) self.assertEqual(explain["index"]["ddoc"], "_design/bbb") def test_choose_index_alphabetically(self): - self.db.create_index(["name"], ddoc="aaa") - self.db.create_index(["name"], ddoc="bbb") - self.db.create_index(["name"], ddoc="zzz") + self.db.create_index(["name"], ddoc="aaa", wait_for_built_index=False) + self.db.create_index(["name"], ddoc="bbb", wait_for_built_index=False) + self.db.create_index(["name"], ddoc="zzz", wait_for_built_index=False) + self.db.wait_for_built_indexes() explain = self.db.find({"name": "Eddie", "age": {"$gte": 12}}, explain=True) self.assertEqual(explain["index"]["ddoc"], "_design/aaa") def test_choose_index_most_accurate(self): - self.db.create_index(["name", "age", "user_id"], ddoc="aaa") - self.db.create_index(["name", "age"], ddoc="bbb") - self.db.create_index(["name"], ddoc="zzz") + self.db.create_index(["name", "age", "user_id"], ddoc="aaa", wait_for_built_index=False) + self.db.create_index(["name", "age"], ddoc="bbb", wait_for_built_index=False) + self.db.create_index(["name"], ddoc="zzz", wait_for_built_index=False) + self.db.wait_for_built_indexes() explain = self.db.find({"name": "Eddie", "age": {"$gte": 12}}, explain=True) self.assertEqual(explain["index"]["ddoc"], "_design/bbb") def test_choose_index_most_accurate_in_memory_selector(self): - self.db.create_index(["name", "location", "user_id"], ddoc="aaa") - self.db.create_index(["name", "age", "user_id"], ddoc="bbb") - self.db.create_index(["name"], ddoc="zzz") + self.db.create_index(["name", "location", "user_id"], ddoc="aaa", wait_for_built_index=False) + self.db.create_index(["name", "age", "user_id"], ddoc="bbb", wait_for_built_index=False) + self.db.create_index(["name"], ddoc="zzz", wait_for_built_index=False) + self.db.wait_for_built_indexes() explain = self.db.find({"name": "Eddie", "number": {"$lte": 12}}, explain=True) self.assertEqual(explain["index"]["ddoc"], "_design/zzz") @@ -100,8 +105,9 @@ class ChooseCorrectIndexForDocs(mango.DbPerClass): def test_chooses_idxA(self): DOCS2 = [{"a": 1, "b": 1, "c": 1}, {"a": 1000, "d": 1000, "e": 1000}] self.db.save_docs(copy.deepcopy(DOCS2)) - self.db.create_index(["a", "b", "c"]) - self.db.create_index(["a", "d", "e"]) + self.db.create_index(["a", "b", "c"], wait_for_built_index=False) + self.db.create_index(["a", "d", "e"], wait_for_built_index=False) + self.db.wait_for_built_indexes() explain = self.db.find( {"a": {"$gt": 0}, "b": {"$gt": 0}, "c": {"$gt": 0}}, explain=True ) diff --git a/src/mango/test/15-execution-stats-test.py b/src/mango/test/15-execution-stats-test.py index 922cadf..90430d8 100644 --- a/src/mango/test/15-execution-stats-test.py +++ b/src/mango/test/15-execution-stats-test.py @@ -22,7 +22,6 @@ class ExecutionStatsTests(mango.UserDocsTests): self.assertEqual(len(resp["docs"]), 3) self.assertEqual(resp["execution_stats"]["total_keys_examined"], 0) self.assertEqual(resp["execution_stats"]["total_docs_examined"], 3) - self.assertEqual(resp["execution_stats"]["total_quorum_docs_examined"], 0) self.assertEqual(resp["execution_stats"]["results_returned"], 3) # See https://github.com/apache/couchdb/issues/1732 # Erlang os:timestamp() only has ms accuracy on Windows! @@ -37,10 +36,10 @@ class ExecutionStatsTests(mango.UserDocsTests): resp = self.db.find( {"age": {"$lt": 35}}, return_raw=True, r=3, executionStats=True ) + print(resp) self.assertEqual(len(resp["docs"]), 3) self.assertEqual(resp["execution_stats"]["total_keys_examined"], 0) - self.assertEqual(resp["execution_stats"]["total_docs_examined"], 0) - self.assertEqual(resp["execution_stats"]["total_quorum_docs_examined"], 3) + self.assertEqual(resp["execution_stats"]["total_docs_examined"], 3) self.assertEqual(resp["execution_stats"]["results_returned"], 3) # See https://github.com/apache/couchdb/issues/1732 # Erlang os:timestamp() only has ms accuracy on Windows! @@ -63,7 +62,6 @@ class ExecutionStatsTests_Text(mango.UserDocsTextTests): self.assertEqual(len(resp["docs"]), 1) self.assertEqual(resp["execution_stats"]["total_keys_examined"], 0) self.assertEqual(resp["execution_stats"]["total_docs_examined"], 1) - self.assertEqual(resp["execution_stats"]["total_quorum_docs_examined"], 0) self.assertEqual(resp["execution_stats"]["results_returned"], 1) self.assertGreater(resp["execution_stats"]["execution_time_ms"], 0) diff --git a/src/mango/test/16-index-selectors-test.py b/src/mango/test/16-index-selectors-test.py index 4510065..a3014e9 100644 --- a/src/mango/test/16-index-selectors-test.py +++ b/src/mango/test/16-index-selectors-test.py @@ -158,6 +158,7 @@ class IndexSelectorJson(mango.DbPerClass): def test_old_selector_with_no_selector_still_supported(self): selector = {"location": {"$gte": "FRA"}} self.db.save_doc(oldschoolnoselectorddoc) + self.db.wait_for_built_indexes() resp = self.db.find(selector, explain=True, use_index="oldschoolnoselector") self.assertEqual(resp["index"]["name"], "oldschoolnoselector") docs = self.db.find(selector, use_index="oldschoolnoselector") @@ -166,6 +167,7 @@ class IndexSelectorJson(mango.DbPerClass): def test_old_selector_still_supported(self): selector = {"location": {"$gte": "FRA"}} self.db.save_doc(oldschoolddoc) + self.db.wait_for_built_indexes() resp = self.db.find(selector, explain=True, use_index="oldschool") self.assertEqual(resp["index"]["name"], "oldschool") docs = self.db.find(selector, use_index="oldschool") diff --git a/src/mango/test/17-multi-type-value-test.py b/src/mango/test/17-multi-type-value-test.py index 21e7afd..b9420a3 100644 --- a/src/mango/test/17-multi-type-value-test.py +++ b/src/mango/test/17-multi-type-value-test.py @@ -53,9 +53,9 @@ class MultiValueFieldTests: class MultiValueFieldJSONTests(mango.DbPerClass, MultiValueFieldTests): def setUp(self): self.db.recreate() + self.db.create_index(["name"], wait_for_built_index=False) + self.db.create_index(["age", "name"], wait_for_built_index=False) self.db.save_docs(copy.deepcopy(DOCS)) - self.db.create_index(["name"]) - self.db.create_index(["age", "name"]) # @unittest.skipUnless(mango.has_text_service(), "requires text service") diff --git a/src/mango/test/18-json-sort.py b/src/mango/test/18-json-sort.py index d4e60a3..62c8e29 100644 --- a/src/mango/test/18-json-sort.py +++ b/src/mango/test/18-json-sort.py @@ -15,7 +15,7 @@ import copy import unittest DOCS = [ - {"_id": "1", "name": "Jimi", "age": 10, "cars": 1}, + {"_id": "aa", "name": "Jimi", "age": 10, "cars": 1}, {"_id": "2", "name": "Eddie", "age": 20, "cars": 1}, {"_id": "3", "name": "Jane", "age": 30, "cars": 2}, {"_id": "4", "name": "Mary", "age": 40, "cars": 2}, @@ -33,7 +33,7 @@ class JSONIndexSortOptimisations(mango.DbPerClass): selector = {"cars": "2", "age": {"$gt": 10}} explain = self.db.find(selector, sort=["age"], explain=True) self.assertEqual(explain["index"]["name"], "cars-age") - self.assertEqual(explain["mrargs"]["direction"], "fwd") + self.assertEqual(explain["args"]["direction"], "fwd") def test_works_for_all_fields_specified(self): self.db.create_index(["cars", "age"], name="cars-age") @@ -52,7 +52,7 @@ class JSONIndexSortOptimisations(mango.DbPerClass): selector = {"cars": "2", "age": {"$gt": 10}} explain = self.db.find(selector, sort=[{"age": "desc"}], explain=True) self.assertEqual(explain["index"]["name"], "cars-age") - self.assertEqual(explain["mrargs"]["direction"], "rev") + self.assertEqual(explain["args"]["direction"], "rev") def test_not_work_for_non_constant_field(self): self.db.create_index(["cars", "age"], name="cars-age") diff --git a/src/mango/test/19-find-conflicts.py b/src/mango/test/19-find-conflicts.py index bf865d6..45a1e31 100644 --- a/src/mango/test/19-find-conflicts.py +++ b/src/mango/test/19-find-conflicts.py @@ -25,7 +25,7 @@ class ChooseCorrectIndexForDocs(mango.DbPerClass): self.db.save_docs_with_conflicts(copy.deepcopy(CONFLICT)) def test_retrieve_conflicts(self): - self.db.create_index(["_conflicts"]) + self.db.create_index(["_conflicts"], wait_for_built_index=False) result = self.db.find({"_conflicts": {"$exists": True}}, conflicts=True) self.assertEqual( result[0]["_conflicts"][0], "1-23202479633c2b380f79507a776743d5" diff --git a/src/mango/test/user_docs.py b/src/mango/test/user_docs.py index 0a52dee..6793601 100644 --- a/src/mango/test/user_docs.py +++ b/src/mango/test/user_docs.py @@ -90,7 +90,9 @@ def add_view_indexes(db, kwargs): (["ordered"], "ordered"), ] for (idx, name) in indexes: - assert db.create_index(idx, name=name, ddoc=name) is True + assert db.create_index(idx, name=name, ddoc=name, + wait_for_built_index=False) is True + db.wait_for_built_indexes() def add_text_indexes(db, kwargs):
