Implement config parameter max_pread_size
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch/repo Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch/commit/8ea500ef Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch/tree/8ea500ef Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch/diff/8ea500ef Branch: refs/heads/master Commit: 8ea500ef413d09f862609d34bdd8ac6737cd26a3 Parents: 89990e1 Author: Eric Avdey <e...@eiri.ca> Authored: Mon May 16 13:55:52 2016 -0300 Committer: Eric Avdey <e...@eiri.ca> Committed: Mon May 16 16:50:50 2016 -0300 ---------------------------------------------------------------------- src/couch_file.erl | 37 +++++++++++++++++++++++++++++++------ test/couch_file_tests.erl | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/8ea500ef/src/couch_file.erl ---------------------------------------------------------------------- diff --git a/src/couch_file.erl b/src/couch_file.erl index 4bb8be8..03c2628 100644 --- a/src/couch_file.erl +++ b/src/couch_file.erl @@ -12,7 +12,7 @@ -module(couch_file). -behaviour(gen_server). --vsn(1). +-vsn(2). -include_lib("couch/include/couch_db.hrl"). @@ -21,13 +21,15 @@ -define(MONITOR_CHECK, 10000). -define(SIZE_BLOCK, 16#1000). % 4 KiB -define(READ_AHEAD, 2 * ?SIZE_BLOCK). +-define(IS_OLD_STATE(S), tuple_size(S) /= tuple_size(#file{})). -record(file, { fd, is_sys, eof = 0, - db_pid + db_pid, + pread_limit = 0 }). % public API @@ -337,6 +339,8 @@ init_status_error(ReturnPid, Ref, Error) -> init({Filepath, Options, ReturnPid, Ref}) -> process_flag(trap_exit, true), OpenOptions = file_open_options(Options), + Limit = get_pread_limit(), + IsSys = lists:member(sys_db, Options), case lists:member(create, Options) of true -> filelib:ensure_dir(Filepath), @@ -357,7 +361,7 @@ init({Filepath, Options, ReturnPid, Ref}) -> ok = file:sync(Fd), maybe_track_open_os_files(Options), erlang:send_after(?INITIAL_WAIT, self(), maybe_close), - {ok, #file{fd=Fd, is_sys=lists:member(sys_db, Options)}}; + {ok, #file{fd=Fd, is_sys=IsSys, pread_limit=Limit}}; false -> ok = file:close(Fd), init_status_error(ReturnPid, Ref, {error, eexist}) @@ -365,7 +369,7 @@ init({Filepath, Options, ReturnPid, Ref}) -> false -> maybe_track_open_os_files(Options), erlang:send_after(?INITIAL_WAIT, self(), maybe_close), - {ok, #file{fd=Fd, is_sys=lists:member(sys_db, Options)}} + {ok, #file{fd=Fd, is_sys=IsSys, pread_limit=Limit}} end; Error -> init_status_error(ReturnPid, Ref, Error) @@ -381,7 +385,7 @@ init({Filepath, Options, ReturnPid, Ref}) -> maybe_track_open_os_files(Options), {ok, Eof} = file:position(Fd, eof), erlang:send_after(?INITIAL_WAIT, self(), maybe_close), - {ok, #file{fd=Fd, eof=Eof, is_sys=lists:member(sys_db, Options)}}; + {ok, #file{fd=Fd, eof=Eof, is_sys=IsSys, pread_limit=Limit}}; Error -> init_status_error(ReturnPid, Ref, Error) end @@ -408,6 +412,9 @@ terminate(_Reason, #file{fd = nil}) -> terminate(_Reason, #file{fd = Fd}) -> ok = file:close(Fd). +handle_call(Msg, From, File) when ?IS_OLD_STATE(File) -> + handle_call(Msg, From, upgrade_state(File)); + handle_call(close, _From, #file{fd=Fd}=File) -> {stop, normal, file:close(Fd), File#file{fd = nil}}; @@ -418,6 +425,8 @@ handle_call({pread_iolist, Pos}, _From, File) -> catch throw:read_beyond_eof -> throw(read_beyond_eof); + throw:{exceed_pread_limit, Limit} -> + throw({exceed_pread_limit, Limit}); _:_ -> read_raw_iolist_int(File, Pos, 4) end, @@ -491,6 +500,9 @@ handle_cast(close, Fd) -> code_change(_OldVsn, State, _Extra) -> {ok, State}. +handle_info(Msg, File) when ?IS_OLD_STATE(File) -> + handle_info(Msg, upgrade_state(File)); + handle_info(maybe_close, File) -> case is_idle(File) of true -> @@ -553,12 +565,14 @@ maybe_read_more_iolist(Buffer, DataSize, NextPos, File) -> {Data::iolist(), CurPos::non_neg_integer()}. read_raw_iolist_int(Fd, {Pos, _Size}, Len) -> % 0110 UPGRADE CODE read_raw_iolist_int(Fd, Pos, Len); -read_raw_iolist_int(#file{fd = Fd} = F, Pos, Len) -> +read_raw_iolist_int(#file{fd = Fd, pread_limit = Limit} = F, Pos, Len) -> BlockOffset = Pos rem ?SIZE_BLOCK, TotalBytes = calculate_total_read_len(BlockOffset, Len), case Pos + TotalBytes of Size when Size > F#file.eof + ?READ_AHEAD -> throw(read_beyond_eof); + Size when Size > Limit -> + throw({exceed_pread_limit, Limit}); Size -> {ok, <<RawBin:TotalBytes/binary>>} = file:pread(Fd, Pos, TotalBytes), {remove_block_prefixes(BlockOffset, RawBin), Size} @@ -659,6 +673,17 @@ process_info(Pid) -> {Fd, InitialName} end. +upgrade_state({file, Fd, IsSys, Eof, DbPid}) -> + Limit = get_pread_limit(), + #file{fd=Fd, is_sys=IsSys, eof=Eof, db_pid=DbPid, pread_limit=Limit}; +upgrade_state(State) -> + State. + +get_pread_limit() -> + case config:get_integer("couchdb", "max_pread_size", 0) of + N when N > 0 -> N; + _ -> infinity + end. -ifdef(TEST). -include_lib("couch/include/couch_eunit.hrl"). http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/8ea500ef/test/couch_file_tests.erl ---------------------------------------------------------------------- diff --git a/test/couch_file_tests.erl b/test/couch_file_tests.erl index 24edf46..497999e 100644 --- a/test/couch_file_tests.erl +++ b/test/couch_file_tests.erl @@ -150,6 +150,38 @@ should_truncate(Fd) -> ok = couch_file:truncate(Fd, Size), ?_assertMatch({ok, foo}, couch_file:pread_term(Fd, 0)). +pread_limit_test_() -> + { + "Read limit tests", + { + setup, + fun() -> + Ctx = test_util:start(?MODULE), + config:set("couchdb", "max_pread_size", "50000"), + Ctx + end, + fun(Ctx) -> + config:delete("couchdb", "max_pread_size"), + test_util:stop(Ctx) + end, + ?foreach([ + fun should_increase_file_size_on_write/1, + fun should_return_current_file_size_on_write/1, + fun should_write_and_read_term/1, + fun should_write_and_read_binary/1, + fun should_not_read_more_than_pread_limit/1 + ]) + } + }. + +should_not_read_more_than_pread_limit(Fd) -> + BigBin = list_to_binary(lists:duplicate(100000, 0)), + {ok, Pos, _Size} = couch_file:append_binary(Fd, BigBin), + unlink(Fd), + ExpectedError = {badmatch, {'EXIT', {bad_return_value, + {exceed_pread_limit, 50000}}}}, + ?_assertError(ExpectedError, couch_file:pread_binary(Fd, Pos)). + header_test_() -> {
