Repository: couchdb-mango Updated Branches: refs/heads/master 50066bc84 -> 4afd60e84
Add config parameter to reject index all text indexes Text indexes that index all fields can sometimes lead to OOM issues when users have documents with nested array fields. This change adds in a config parameter to provide the ability to log users who do this and also reject new requests. Note that we need to pass in a db record to validate_new because it contains user and db name info that will be logged. COUCHDB-3249 Project: http://git-wip-us.apache.org/repos/asf/couchdb-mango/repo Commit: http://git-wip-us.apache.org/repos/asf/couchdb-mango/commit/4afd60e8 Tree: http://git-wip-us.apache.org/repos/asf/couchdb-mango/tree/4afd60e8 Diff: http://git-wip-us.apache.org/repos/asf/couchdb-mango/diff/4afd60e8 Branch: refs/heads/master Commit: 4afd60e84d0e1c57f5d6a1e3542955faa565ca4b Parents: 50066bc Author: Tony Sun <[email protected]> Authored: Tue Dec 6 00:46:51 2016 -0800 Committer: Tony Sun <[email protected]> Committed: Tue Dec 6 12:59:55 2016 -0800 ---------------------------------------------------------------------- src/mango_error.erl | 6 ++++ src/mango_httpd.erl | 2 +- src/mango_idx.erl | 6 ++-- src/mango_idx_text.erl | 83 +++++++++++++++++++++++++++++++++++++++++++-- src/mango_idx_view.erl | 4 +-- 5 files changed, 93 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/4afd60e8/src/mango_error.erl ---------------------------------------------------------------------- diff --git a/src/mango_error.erl b/src/mango_error.erl index f0a8ee2..76a8362 100644 --- a/src/mango_error.erl +++ b/src/mango_error.erl @@ -174,6 +174,12 @@ info(mango_idx_text, {index_not_found, BadIdx}) -> <<"index_not_found">>, fmt("Text index ~s not found in this design doc.", [BadIdx]) }; +info(mango_idx_text, index_all_disabled) -> + { + 403, + <<"index_all_disabled">>, + <<"New text indexes are forbidden to index all fields.">> + }; info(mango_opts, {invalid_bulk_docs, Val}) -> { http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/4afd60e8/src/mango_httpd.erl ---------------------------------------------------------------------- diff --git a/src/mango_httpd.erl b/src/mango_httpd.erl index bde3850..a088276 100644 --- a/src/mango_httpd.erl +++ b/src/mango_httpd.erl @@ -84,7 +84,7 @@ handle_index_req(#httpd{method='POST', path_parts=[_, _]}=Req, Db) -> chttpd:validate_ctype(Req, "application/json"), {ok, Opts} = mango_opts:validate_idx_create(chttpd:json_body_obj(Req)), {ok, Idx0} = mango_idx:new(Db, Opts), - {ok, Idx} = mango_idx:validate_new(Idx0), + {ok, Idx} = mango_idx:validate_new(Idx0, Db), {ok, DDoc} = mango_util:load_ddoc(Db, mango_idx:ddoc(Idx)), Id = Idx#idx.ddoc, Name = Idx#idx.name, http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/4afd60e8/src/mango_idx.erl ---------------------------------------------------------------------- diff --git a/src/mango_idx.erl b/src/mango_idx.erl index 11c713b..bc88b97 100644 --- a/src/mango_idx.erl +++ b/src/mango_idx.erl @@ -23,7 +23,7 @@ for_sort/2, new/2, - validate_new/1, + validate_new/2, add/2, remove/2, from_ddoc/2, @@ -136,9 +136,9 @@ new(Db, Opts) -> }}. -validate_new(Idx) -> +validate_new(Idx, Db) -> Mod = idx_mod(Idx), - Mod:validate_new(Idx). + Mod:validate_new(Idx, Db). add(DDoc, Idx) -> http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/4afd60e8/src/mango_idx_text.erl ---------------------------------------------------------------------- diff --git a/src/mango_idx_text.erl b/src/mango_idx_text.erl index 40a1fd3..4cfda5a 100644 --- a/src/mango_idx_text.erl +++ b/src/mango_idx_text.erl @@ -14,7 +14,7 @@ -export([ - validate_new/1, + validate_new/2, validate_fields/1, validate_index_def/1, add/2, @@ -32,8 +32,9 @@ -include("mango_idx.hrl"). -validate_new(#idx{}=Idx) -> +validate_new(#idx{}=Idx, Db) -> {ok, Def} = do_validate(Idx#idx.def), + maybe_reject_index_all_req(Def, Db), {ok, Idx#idx{def=Def}}. @@ -327,3 +328,81 @@ indexable_fields(Fields, {op_null, {_, _}}) -> indexable_fields(Fields, {op_default, _}) -> [<<"$default">> | Fields]. + + +maybe_reject_index_all_req({Def}, #db{name=DbName, user_ctx=Ctx}) -> + User = Ctx#user_ctx.name, + Fields = couch_util:get_value(fields, Def), + case {Fields, forbid_index_all()} of + {all_fields, "true"} -> + ?MANGO_ERROR(index_all_disabled); + {all_fields, "warn"} -> + couch_log:warning("User ~p is indexing all fields in db ~p", + [User, DbName]); + _ -> + ok + end. + + +forbid_index_all() -> + config:get("mango", "index_all_disabled", "false"). + + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + + +setup() -> + test_util:start_couch(), + meck:expect(couch_log, warning, 2, + fun(_,_) -> + throw({test_error, logged_warning}) + end), + %default index all def that generates {fields, all_fields} + Index = #idx{def={[]}}, + Db = #db{name = <<"testdb">>, user_ctx=#user_ctx{name = <<"u1">>}}, + {Index, Db}. + + +teardown(_) -> + ok = config:delete("mango", "index_all_disabled"), + test_util:stop_couch(). + + +index_all_test_() -> + { + foreach, + fun setup/0, + fun teardown/1, + [ + fun forbid_index_all/1, + fun default_and_false_index_all/1, + fun warn_index_all/1 + ] + + }. + + +forbid_index_all({Idx, Db}) -> + ok = config:set("mango", "index_all_disabled", "true"), + ?_assertThrow({mango_error, ?MODULE, index_all_disabled}, + validate_new(Idx, Db) + ). + + +default_and_false_index_all({Idx, Db}) -> + {ok, #idx{def={Def}}} = validate_new(Idx, Db), + Fields = couch_util:get_value(fields, Def), + ?_assertEqual(all_fields, Fields), + ok = config:set("mango", "index_all_disabled", "false"), + {ok, #idx{def={Def2}}} = validate_new(Idx, Db), + Fields2 = couch_util:get_value(fields, Def2), + ?_assertEqual(all_fields, Fields2). + + +warn_index_all({Idx, Db}) -> + ok = config:set("mango", "index_all_disabled", "warn"), + ?_assertThrow({test_error, logged_warning}, validate_new(Idx, Db)). + + +-endif. http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/4afd60e8/src/mango_idx_view.erl ---------------------------------------------------------------------- diff --git a/src/mango_idx_view.erl b/src/mango_idx_view.erl index 0cc713f..8bad34c 100644 --- a/src/mango_idx_view.erl +++ b/src/mango_idx_view.erl @@ -14,7 +14,7 @@ -export([ - validate_new/1, + validate_new/2, validate_index_def/1, add/2, remove/2, @@ -36,7 +36,7 @@ -include("mango_idx.hrl"). -validate_new(#idx{}=Idx) -> +validate_new(#idx{}=Idx, _Db) -> {ok, Def} = do_validate(Idx#idx.def), {ok, Idx#idx{def=Def}}.
