[GitHub] tonysun83 commented on a change in pull request #962: Mango: generate a warning instead of an error when an user-specified index cannot be used
tonysun83 commented on a change in pull request #962: Mango: generate a warning instead of an error when an user-specified index cannot be used URL: https://github.com/apache/couchdb/pull/962#discussion_r153708248 ## File path: src/mango/src/mango_cursor.erl ## @@ -150,12 +151,54 @@ group_indexes_by_type(Indexes) -> end, ?CURSOR_MODULES). -maybe_add_warning(UserFun, #idx{type = IndexType}, UserAcc) -> -case IndexType of +maybe_add_warning(UserFun, #cursor{index = Index, opts = Opts}, UserAcc) -> +UseIndexInvalid = case lists:keyfind(use_index, 1, Opts) of +{use_index, []} -> +[]; +{use_index, [DesignId]} -> +case filter_indexes([Index], DesignId) of +[] -> +[fmt("_design/~s was not used because it does not contain a valid index for this query.", +[ddoc_name(DesignId)])]; +_ -> +[] +end; +{use_index, [DesignId, ViewName]} -> +case filter_indexes([Index], DesignId, ViewName) of +[] -> +[fmt("_design/~s, ~s was not used because it is not a valid index for this query.", +[ddoc_name(DesignId), ViewName])]; +_ -> +[] +end +end, + +NoIndex = case Index#idx.type of <<"special">> -> -Arg = {add_key, warning, <<"no matching index found, create an index to optimize query time">>}, -{_Go, UserAcc0} = UserFun(Arg, UserAcc), -UserAcc0; +[<<"no matching index found, create an index to optimize query time">>]; _ -> -UserAcc -end. \ No newline at end of file +[] +end, + +maybe_add_warning_int(UseIndexInvalid ++ NoIndex, UserFun, UserAcc). + + +maybe_add_warning_int([], _, UserAcc) -> + UserAcc; + +maybe_add_warning_int(Warnings, UserFun, UserAcc) -> +% only include the first warning Review comment: see comment about about the warning list This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] tonysun83 commented on a change in pull request #962: Mango: generate a warning instead of an error when an user-specified index cannot be used
tonysun83 commented on a change in pull request #962: Mango: generate a warning instead of an error when an user-specified index cannot be used URL: https://github.com/apache/couchdb/pull/962#discussion_r153708120 ## File path: src/mango/src/mango_cursor.erl ## @@ -150,12 +151,54 @@ group_indexes_by_type(Indexes) -> end, ?CURSOR_MODULES). -maybe_add_warning(UserFun, #idx{type = IndexType}, UserAcc) -> -case IndexType of +maybe_add_warning(UserFun, #cursor{index = Index, opts = Opts}, UserAcc) -> +UseIndexInvalid = case lists:keyfind(use_index, 1, Opts) of +{use_index, []} -> +[]; +{use_index, [DesignId]} -> +case filter_indexes([Index], DesignId) of +[] -> +[fmt("_design/~s was not used because it does not contain a valid index for this query.", +[ddoc_name(DesignId)])]; +_ -> +[] +end; +{use_index, [DesignId, ViewName]} -> +case filter_indexes([Index], DesignId, ViewName) of +[] -> +[fmt("_design/~s, ~s was not used because it is not a valid index for this query.", +[ddoc_name(DesignId), ViewName])]; +_ -> +[] +end +end, + +NoIndex = case Index#idx.type of <<"special">> -> -Arg = {add_key, warning, <<"no matching index found, create an index to optimize query time">>}, -{_Go, UserAcc0} = UserFun(Arg, UserAcc), -UserAcc0; +[<<"no matching index found, create an index to optimize query time">>]; _ -> -UserAcc -end. \ No newline at end of file +[] +end, + +maybe_add_warning_int(UseIndexInvalid ++ NoIndex, UserFun, UserAcc). Review comment: Do we really need to append two messages and then take the head of that list later? I feel like it's either going to be a 1) "we used a diff index" warning or 2) "no indexes found, we used all docs" warning. Instead of returning empty lists, perhaps you could just move the NoIndex value as the return value for the case clauses. That way no need to add and then take the head. Ex: ``` {use_index, []} -> NoIndex; This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] tonysun83 commented on a change in pull request #962: Mango: generate a warning instead of an error when an user-specified index cannot be used
tonysun83 commented on a change in pull request #962: Mango: generate a warning instead of an error when an user-specified index cannot be used URL: https://github.com/apache/couchdb/pull/962#discussion_r153707667 ## File path: src/mango/src/mango_cursor.erl ## @@ -150,12 +151,54 @@ group_indexes_by_type(Indexes) -> end, ?CURSOR_MODULES). -maybe_add_warning(UserFun, #idx{type = IndexType}, UserAcc) -> -case IndexType of +maybe_add_warning(UserFun, #cursor{index = Index, opts = Opts}, UserAcc) -> +UseIndexInvalid = case lists:keyfind(use_index, 1, Opts) of +{use_index, []} -> +[]; +{use_index, [DesignId]} -> +case filter_indexes([Index], DesignId) of +[] -> +[fmt("_design/~s was not used because it does not contain a valid index for this query.", +[ddoc_name(DesignId)])]; +_ -> +[] +end; +{use_index, [DesignId, ViewName]} -> +case filter_indexes([Index], DesignId, ViewName) of +[] -> +[fmt("_design/~s, ~s was not used because it is not a valid index for this query.", +[ddoc_name(DesignId), ViewName])]; Review comment: same as the above comment This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] tonysun83 commented on a change in pull request #962: Mango: generate a warning instead of an error when an user-specified index cannot be used
tonysun83 commented on a change in pull request #962: Mango: generate a warning instead of an error when an user-specified index cannot be used URL: https://github.com/apache/couchdb/pull/962#discussion_r153707643 ## File path: src/mango/src/mango_cursor.erl ## @@ -150,12 +151,54 @@ group_indexes_by_type(Indexes) -> end, ?CURSOR_MODULES). -maybe_add_warning(UserFun, #idx{type = IndexType}, UserAcc) -> -case IndexType of +maybe_add_warning(UserFun, #cursor{index = Index, opts = Opts}, UserAcc) -> +UseIndexInvalid = case lists:keyfind(use_index, 1, Opts) of +{use_index, []} -> +[]; +{use_index, [DesignId]} -> +case filter_indexes([Index], DesignId) of +[] -> +[fmt("_design/~s was not used because it does not contain a valid index for this query.", +[ddoc_name(DesignId)])]; Review comment: Should make the warning message a little bit more explicit here and add "we used index " instead? This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] tonysun83 commented on a change in pull request #962: Mango: generate a warning instead of an error when an user-specified index cannot be used
tonysun83 commented on a change in pull request #962: Mango: generate a warning instead of an error when an user-specified index cannot be used URL: https://github.com/apache/couchdb/pull/962#discussion_r153702449 ## File path: src/mango/test/05-index-selection-test.py ## @@ -108,41 +103,53 @@ def test_reject_use_index_ddoc_and_name_invalid_fields(self): selector = { "company": "Pharmex" } -try: -self.db.find(selector, use_index=[ddocid,name]) -except Exception as e: -self.assertEqual(e.response.status_code, 400) -else: -raise AssertionError("did not reject bad use_index") + +resp = self.db.find(selector, use_index=[ddocid,name], return_raw=True) +self.assertEqual(resp["warning"], "{0}, {1} was not used because it is not a valid index for this query.".format(ddocid, name)) def test_reject_use_index_sort_order(self): # index on ["company","manager"] which should not be valid +# and there is no valid fallback (i.e. an index on ["company"]) ddocid = "_design/a0c425a60cf3c3c09e3c537c9ef20059dcef9198" selector = { -"company": {"$gt": None}, -"manager": {"$gt": None} +"company": {"$gt": None} } try: -self.db.find(selector, use_index=ddocid, sort=[{"manager":"desc"}]) +self.db.find(selector, use_index=ddocid, sort=[{"company":"desc"}]) except Exception as e: self.assertEqual(e.response.status_code, 400) else: raise AssertionError("did not reject bad use_index") -def test_reject_use_index_ddoc_and_name_sort_order(self): -# index on ["company","manager"] which should not be valid -ddocid = "_design/a0c425a60cf3c3c09e3c537c9ef20059dcef9198" -name = "a0c425a60cf3c3c09e3c537c9ef20059dcef9198" +def test_use_index_fallback_if_valid_sort(self): +ddocid_valid = "_design/fallbackfoo" +ddocid_invalid = "_design/fallbackfoobar" +self.db.create_index(fields=["foo"], ddoc=ddocid_invalid) +self.db.create_index(fields=["foo", "bar"], ddoc=ddocid_valid) selector = { -"company": {"$gt": None}, -"manager": {"$gt": None} +"foo": {"$gt": None} } -try: -self.db.find(selector, use_index=[ddocid,name], sort=[{"manager":"desc"}]) -except Exception as e: -self.assertEqual(e.response.status_code, 400) -else: -raise AssertionError("did not reject bad use_index") + +resp_explain = self.db.find(selector, sort=["foo", "bar"], use_index=ddocid_invalid, explain=True) +self.assertEqual(resp_explain["index"]["ddoc"], ddocid_valid) + +resp = self.db.find(selector, sort=["foo", "bar"], use_index=ddocid_invalid, return_raw=True) +self.assertEqual(resp["warning"], '{0} was not used because it does not contain a valid index for this query.'.format(ddocid_invalid)) Review comment: same as above, we should get a value back here right? It's going to fall back on the ddoc_valid and return a result, so just confirm a result? This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] tonysun83 commented on a change in pull request #962: Mango: generate a warning instead of an error when an user-specified index cannot be used
tonysun83 commented on a change in pull request #962: Mango: generate a warning instead of an error when an user-specified index cannot be used URL: https://github.com/apache/couchdb/pull/962#discussion_r153701470 ## File path: src/mango/test/05-index-selection-test.py ## @@ -108,41 +103,53 @@ def test_reject_use_index_ddoc_and_name_invalid_fields(self): selector = { "company": "Pharmex" } -try: -self.db.find(selector, use_index=[ddocid,name]) -except Exception as e: -self.assertEqual(e.response.status_code, 400) -else: -raise AssertionError("did not reject bad use_index") + +resp = self.db.find(selector, use_index=[ddocid,name], return_raw=True) +self.assertEqual(resp["warning"], "{0}, {1} was not used because it is not a valid index for this query.".format(ddocid, name)) Review comment: in the test case above, you're checking that a value is added ``` # should still return a correct result for d in r["docs"]: self.assertEqual(d["company"], "Pharmex") ``` We should be consistent? This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] tonysun83 commented on a change in pull request #962: Mango: generate a warning instead of an error when an user-specified index cannot be used
tonysun83 commented on a change in pull request #962: Mango: generate a warning instead of an error when an user-specified index cannot be used URL: https://github.com/apache/couchdb/pull/962#discussion_r153701470 ## File path: src/mango/test/05-index-selection-test.py ## @@ -108,41 +103,53 @@ def test_reject_use_index_ddoc_and_name_invalid_fields(self): selector = { "company": "Pharmex" } -try: -self.db.find(selector, use_index=[ddocid,name]) -except Exception as e: -self.assertEqual(e.response.status_code, 400) -else: -raise AssertionError("did not reject bad use_index") + +resp = self.db.find(selector, use_index=[ddocid,name], return_raw=True) +self.assertEqual(resp["warning"], "{0}, {1} was not used because it is not a valid index for this query.".format(ddocid, name)) Review comment: in the test case above, you're checking that a value is returned ``` # should still return a correct result for d in r["docs"]: self.assertEqual(d["company"], "Pharmex") ``` We should be consistent? This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] tonysun83 commented on a change in pull request #962: Mango: generate a warning instead of an error when an user-specified index cannot be used
tonysun83 commented on a change in pull request #962: Mango: generate a warning instead of an error when an user-specified index cannot be used URL: https://github.com/apache/couchdb/pull/962#discussion_r153701126 ## File path: src/mango/test/05-index-selection-test.py ## @@ -46,7 +46,7 @@ def test_use_most_columns(self): "name.last": "Something or other", "age": {"$gt": 1} }, explain=True) -self.assertNotEqual(resp["index"]["ddoc"], "_design/" + ddocid) +self.assertNotEqual(resp["index"]["ddoc"], ddocid) Review comment: this looks a random testcase fix not related to the PR? This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] tonysun83 commented on a change in pull request #962: Mango: generate a warning instead of an error when an user-specified index cannot be used
tonysun83 commented on a change in pull request #962: Mango: generate a warning instead of an error when an user-specified index cannot be used URL: https://github.com/apache/couchdb/pull/962#discussion_r149551201 ## File path: src/mango/test/05-index-selection-test.py ## @@ -77,34 +73,78 @@ def test_uses_index_when_no_range_or_equals(self): resp_explain = self.db.find(selector, explain=True) self.assertEqual(resp_explain["index"]["type"], "json") - Review comment: whitespace This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] tonysun83 commented on a change in pull request #962: Mango: generate a warning instead of an error when an user-specified index cannot be used
tonysun83 commented on a change in pull request #962: Mango: generate a warning instead of an error when an user-specified index cannot be used URL: https://github.com/apache/couchdb/pull/962#discussion_r149551044 ## File path: src/mango/src/mango_idx.erl ## @@ -59,20 +59,16 @@ list(Db) -> get_usable_indexes(Db, Selector, Opts) -> ExistingIndexes = mango_idx:list(Db), -if ExistingIndexes /= [] -> ok; true -> -?MANGO_ERROR({no_usable_index, no_indexes_defined}) -end, -FilteredIndexes = mango_cursor:maybe_filter_indexes_by_ddoc(ExistingIndexes, Opts), -if FilteredIndexes /= [] -> ok; true -> -?MANGO_ERROR({no_usable_index, no_index_matching_name}) -end, +GlobalIndexes = mango_cursor:remove_indexes_with_partial_filter_selector(ExistingIndexes), +UserSpecifiedIndex = mango_cursor:maybe_filter_indexes_by_ddoc(ExistingIndexes, Opts), +UsableIndexes0 = lists:usort(GlobalIndexes ++ UserSpecifiedIndex), Review comment: why are we sorting the indexes here? This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] tonysun83 commented on a change in pull request #962: Mango: generate a warning instead of an error when an user-specified index cannot be used
tonysun83 commented on a change in pull request #962: Mango: generate a warning instead of an error when an user-specified index cannot be used URL: https://github.com/apache/couchdb/pull/962#discussion_r149550494 ## File path: src/mango/src/mango_error.erl ## @@ -21,12 +21,6 @@ ]). -info(mango_idx, {no_usable_index, no_indexes_defined}) -> -{ -400, -<<"no_usable_index">>, -<<"There are no indexes defined in this database.">> -}; info(mango_idx, {no_usable_index, no_index_matching_name}) -> Review comment: do we need to remove this error as well? This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] tonysun83 commented on a change in pull request #962: Mango: generate a warning instead of an error when an user-specified index cannot be used
tonysun83 commented on a change in pull request #962: Mango: generate a warning instead of an error when an user-specified index cannot be used URL: https://github.com/apache/couchdb/pull/962#discussion_r149549786 ## File path: src/mango/src/mango_cursor.erl ## @@ -150,12 +151,53 @@ group_indexes_by_type(Indexes) -> end, ?CURSOR_MODULES). -maybe_add_warning(UserFun, #idx{type = IndexType}, UserAcc) -> -case IndexType of +maybe_add_warning(UserFun, #cursor{index = Index, opts = Opts}, UserAcc) -> + +UseIndexInvalid = case lists:keyfind(use_index, 1, Opts) of +{use_index, []} -> +[]; +{use_index, [DesignId]} -> +case filter_indexes([Index], DesignId) of +[] -> +[fmt("_design/~s was not used because it does not contain a valid index for this query.", [ddoc_name(DesignId)])]; +_ -> +[] +end; +{use_index, [DesignId, ViewName]} -> +case filter_indexes([Index], DesignId, ViewName) of +[] -> +[fmt("_design/~s, ~s was not used because it is not a valid index for this query.", [ddoc_name(DesignId), ViewName])]; +_ -> +[] +end +end, + +NoIndex = case Index#idx.type of <<"special">> -> -Arg = {add_key, warning, <<"no matching index found, create an index to optimize query time">>}, -{_Go, UserAcc0} = UserFun(Arg, UserAcc), -UserAcc0; +[<<"no matching index found, create an index to optimize query time">>]; _ -> -UserAcc -end. \ No newline at end of file +[] +end, + +maybe_add_warning_int(UseIndexInvalid ++ NoIndex, UserFun, UserAcc). + + +maybe_add_warning_int([], _, UserAcc) -> + UserAcc; + +maybe_add_warning_int(Warnings, UserFun, UserAcc) -> +% only include the first warning +Arg = {add_key, warning, hd(Warnings)}, +{_Go, UserAcc0} = UserFun(Arg, UserAcc), +UserAcc0. + + +fmt(Format, Args) -> +iolist_to_binary(io_lib:format(Format, Args)). + + +ddoc_name(<<"_design/", Name/binary>>) -> +Name; + +ddoc_name(Name) -> +Name. Review comment: missing endline This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] tonysun83 commented on a change in pull request #962: Mango: generate a warning instead of an error when an user-specified index cannot be used
tonysun83 commented on a change in pull request #962: Mango: generate a warning instead of an error when an user-specified index cannot be used URL: https://github.com/apache/couchdb/pull/962#discussion_r149549612 ## File path: src/mango/src/mango_cursor.erl ## @@ -150,12 +151,53 @@ group_indexes_by_type(Indexes) -> end, ?CURSOR_MODULES). -maybe_add_warning(UserFun, #idx{type = IndexType}, UserAcc) -> -case IndexType of +maybe_add_warning(UserFun, #cursor{index = Index, opts = Opts}, UserAcc) -> + +UseIndexInvalid = case lists:keyfind(use_index, 1, Opts) of +{use_index, []} -> +[]; +{use_index, [DesignId]} -> +case filter_indexes([Index], DesignId) of +[] -> +[fmt("_design/~s was not used because it does not contain a valid index for this query.", [ddoc_name(DesignId)])]; +_ -> +[] +end; +{use_index, [DesignId, ViewName]} -> +case filter_indexes([Index], DesignId, ViewName) of +[] -> +[fmt("_design/~s, ~s was not used because it is not a valid index for this query.", [ddoc_name(DesignId), ViewName])]; Review comment: super long line This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] tonysun83 commented on a change in pull request #962: Mango: generate a warning instead of an error when an user-specified index cannot be used
tonysun83 commented on a change in pull request #962: Mango: generate a warning instead of an error when an user-specified index cannot be used URL: https://github.com/apache/couchdb/pull/962#discussion_r149549547 ## File path: src/mango/src/mango_cursor.erl ## @@ -150,12 +151,53 @@ group_indexes_by_type(Indexes) -> end, ?CURSOR_MODULES). -maybe_add_warning(UserFun, #idx{type = IndexType}, UserAcc) -> -case IndexType of +maybe_add_warning(UserFun, #cursor{index = Index, opts = Opts}, UserAcc) -> + +UseIndexInvalid = case lists:keyfind(use_index, 1, Opts) of +{use_index, []} -> +[]; +{use_index, [DesignId]} -> +case filter_indexes([Index], DesignId) of +[] -> +[fmt("_design/~s was not used because it does not contain a valid index for this query.", [ddoc_name(DesignId)])]; Review comment: this line is too long This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] tonysun83 commented on a change in pull request #962: Mango: generate a warning instead of an error when an user-specified index cannot be used
tonysun83 commented on a change in pull request #962: Mango: generate a warning instead of an error when an user-specified index cannot be used URL: https://github.com/apache/couchdb/pull/962#discussion_r149549331 ## File path: src/mango/src/mango_cursor.erl ## @@ -150,12 +151,53 @@ group_indexes_by_type(Indexes) -> end, ?CURSOR_MODULES). -maybe_add_warning(UserFun, #idx{type = IndexType}, UserAcc) -> -case IndexType of +maybe_add_warning(UserFun, #cursor{index = Index, opts = Opts}, UserAcc) -> + Review comment: Is this whitespace intended? I don't think I've seen it right after a function declaration before. Might want to check styling guidelines. It's slightly hard to read. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] tonysun83 commented on a change in pull request #962: Mango: generate a warning instead of an error when an user-specified index cannot be used
tonysun83 commented on a change in pull request #962: Mango: generate a warning instead of an error when an user-specified index cannot be used URL: https://github.com/apache/couchdb/pull/962#discussion_r149549331 ## File path: src/mango/src/mango_cursor.erl ## @@ -150,12 +151,53 @@ group_indexes_by_type(Indexes) -> end, ?CURSOR_MODULES). -maybe_add_warning(UserFun, #idx{type = IndexType}, UserAcc) -> -case IndexType of +maybe_add_warning(UserFun, #cursor{index = Index, opts = Opts}, UserAcc) -> + Review comment: Is this whitespace intended? I don't think I've seen this before. Might want to check styling guidelines. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services