Repository: couchdb-mango Updated Branches: refs/heads/79577-add-config-index-all [created] a4cce0032
Add config parameter to reject index all text indexes Text indexes that index all fields can sometimes lead to OOM issues when users have documents with nested array fields. This change adds in a config parameter to provide the ability to log users who do this and also reject new requests. Note that we need to pass in a db record to validate_new because it contains user and db name info that will be logged. COUCHDB-3249 Project: http://git-wip-us.apache.org/repos/asf/couchdb-mango/repo Commit: http://git-wip-us.apache.org/repos/asf/couchdb-mango/commit/a4cce003 Tree: http://git-wip-us.apache.org/repos/asf/couchdb-mango/tree/a4cce003 Diff: http://git-wip-us.apache.org/repos/asf/couchdb-mango/diff/a4cce003 Branch: refs/heads/79577-add-config-index-all Commit: a4cce0032a9ca43301038eb46bfd4dc1dd6a4a49 Parents: 50066bc Author: Tony Sun <[email protected]> Authored: Tue Dec 6 00:46:51 2016 -0800 Committer: Tony Sun <[email protected]> Committed: Tue Dec 6 12:51:25 2016 -0800 ---------------------------------------------------------------------- src/mango_error.erl | 6 ++++ src/mango_httpd.erl | 2 +- src/mango_idx.erl | 6 ++-- src/mango_idx_text.erl | 83 +++++++++++++++++++++++++++++++++++++++++++-- src/mango_idx_view.erl | 4 +-- 5 files changed, 93 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/a4cce003/src/mango_error.erl ---------------------------------------------------------------------- diff --git a/src/mango_error.erl b/src/mango_error.erl index f0a8ee2..76a8362 100644 --- a/src/mango_error.erl +++ b/src/mango_error.erl @@ -174,6 +174,12 @@ info(mango_idx_text, {index_not_found, BadIdx}) -> <<"index_not_found">>, fmt("Text index ~s not found in this design doc.", [BadIdx]) }; +info(mango_idx_text, index_all_disabled) -> + { + 403, + <<"index_all_disabled">>, + <<"New text indexes are forbidden to index all fields.">> + }; info(mango_opts, {invalid_bulk_docs, Val}) -> { http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/a4cce003/src/mango_httpd.erl ---------------------------------------------------------------------- diff --git a/src/mango_httpd.erl b/src/mango_httpd.erl index bde3850..a088276 100644 --- a/src/mango_httpd.erl +++ b/src/mango_httpd.erl @@ -84,7 +84,7 @@ handle_index_req(#httpd{method='POST', path_parts=[_, _]}=Req, Db) -> chttpd:validate_ctype(Req, "application/json"), {ok, Opts} = mango_opts:validate_idx_create(chttpd:json_body_obj(Req)), {ok, Idx0} = mango_idx:new(Db, Opts), - {ok, Idx} = mango_idx:validate_new(Idx0), + {ok, Idx} = mango_idx:validate_new(Idx0, Db), {ok, DDoc} = mango_util:load_ddoc(Db, mango_idx:ddoc(Idx)), Id = Idx#idx.ddoc, Name = Idx#idx.name, http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/a4cce003/src/mango_idx.erl ---------------------------------------------------------------------- diff --git a/src/mango_idx.erl b/src/mango_idx.erl index 11c713b..bc88b97 100644 --- a/src/mango_idx.erl +++ b/src/mango_idx.erl @@ -23,7 +23,7 @@ for_sort/2, new/2, - validate_new/1, + validate_new/2, add/2, remove/2, from_ddoc/2, @@ -136,9 +136,9 @@ new(Db, Opts) -> }}. -validate_new(Idx) -> +validate_new(Idx, Db) -> Mod = idx_mod(Idx), - Mod:validate_new(Idx). + Mod:validate_new(Idx, Db). add(DDoc, Idx) -> http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/a4cce003/src/mango_idx_text.erl ---------------------------------------------------------------------- diff --git a/src/mango_idx_text.erl b/src/mango_idx_text.erl index 40a1fd3..4cfda5a 100644 --- a/src/mango_idx_text.erl +++ b/src/mango_idx_text.erl @@ -14,7 +14,7 @@ -export([ - validate_new/1, + validate_new/2, validate_fields/1, validate_index_def/1, add/2, @@ -32,8 +32,9 @@ -include("mango_idx.hrl"). -validate_new(#idx{}=Idx) -> +validate_new(#idx{}=Idx, Db) -> {ok, Def} = do_validate(Idx#idx.def), + maybe_reject_index_all_req(Def, Db), {ok, Idx#idx{def=Def}}. @@ -327,3 +328,81 @@ indexable_fields(Fields, {op_null, {_, _}}) -> indexable_fields(Fields, {op_default, _}) -> [<<"$default">> | Fields]. + + +maybe_reject_index_all_req({Def}, #db{name=DbName, user_ctx=Ctx}) -> + User = Ctx#user_ctx.name, + Fields = couch_util:get_value(fields, Def), + case {Fields, forbid_index_all()} of + {all_fields, "true"} -> + ?MANGO_ERROR(index_all_disabled); + {all_fields, "warn"} -> + couch_log:warning("User ~p is indexing all fields in db ~p", + [User, DbName]); + _ -> + ok + end. + + +forbid_index_all() -> + config:get("mango", "index_all_disabled", "false"). + + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + + +setup() -> + test_util:start_couch(), + meck:expect(couch_log, warning, 2, + fun(_,_) -> + throw({test_error, logged_warning}) + end), + %default index all def that generates {fields, all_fields} + Index = #idx{def={[]}}, + Db = #db{name = <<"testdb">>, user_ctx=#user_ctx{name = <<"u1">>}}, + {Index, Db}. + + +teardown(_) -> + ok = config:delete("mango", "index_all_disabled"), + test_util:stop_couch(). + + +index_all_test_() -> + { + foreach, + fun setup/0, + fun teardown/1, + [ + fun forbid_index_all/1, + fun default_and_false_index_all/1, + fun warn_index_all/1 + ] + + }. + + +forbid_index_all({Idx, Db}) -> + ok = config:set("mango", "index_all_disabled", "true"), + ?_assertThrow({mango_error, ?MODULE, index_all_disabled}, + validate_new(Idx, Db) + ). + + +default_and_false_index_all({Idx, Db}) -> + {ok, #idx{def={Def}}} = validate_new(Idx, Db), + Fields = couch_util:get_value(fields, Def), + ?_assertEqual(all_fields, Fields), + ok = config:set("mango", "index_all_disabled", "false"), + {ok, #idx{def={Def2}}} = validate_new(Idx, Db), + Fields2 = couch_util:get_value(fields, Def2), + ?_assertEqual(all_fields, Fields2). + + +warn_index_all({Idx, Db}) -> + ok = config:set("mango", "index_all_disabled", "warn"), + ?_assertThrow({test_error, logged_warning}, validate_new(Idx, Db)). + + +-endif. http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/a4cce003/src/mango_idx_view.erl ---------------------------------------------------------------------- diff --git a/src/mango_idx_view.erl b/src/mango_idx_view.erl index 0cc713f..8bad34c 100644 --- a/src/mango_idx_view.erl +++ b/src/mango_idx_view.erl @@ -14,7 +14,7 @@ -export([ - validate_new/1, + validate_new/2, validate_index_def/1, add/2, remove/2, @@ -36,7 +36,7 @@ -include("mango_idx.hrl"). -validate_new(#idx{}=Idx) -> +validate_new(#idx{}=Idx, _Db) -> {ok, Def} = do_validate(Idx#idx.def), {ok, Idx#idx{def=Def}}.
