Updated Branches: refs/heads/COUCHDB-1342 62dde7e9f -> 4f7336ed3
Make couch_file:flush/1 calls automatic Project: http://git-wip-us.apache.org/repos/asf/couchdb/repo Commit: http://git-wip-us.apache.org/repos/asf/couchdb/commit/4f7336ed Tree: http://git-wip-us.apache.org/repos/asf/couchdb/tree/4f7336ed Diff: http://git-wip-us.apache.org/repos/asf/couchdb/diff/4f7336ed Branch: refs/heads/COUCHDB-1342 Commit: 4f7336ed30c0a34823cb332190b57a608725a21c Parents: 0b89f5a Author: Filipe David Borba Manana <[email protected]> Authored: Tue Nov 29 00:12:10 2011 +0000 Committer: Filipe David Borba Manana <[email protected]> Committed: Tue Nov 29 13:58:43 2011 +0000 ---------------------------------------------------------------------- src/couch_mrview/src/couch_mrview_index.erl | 3 +- src/couch_mrview/src/couch_mrview_updater.erl | 8 +--- src/couch_mrview/src/couch_mrview_util.erl | 1 - src/couchdb/couch_db_updater.erl | 9 +---- src/couchdb/couch_file.erl | 37 +++++++++++++++---- test/etap/010-file-basics.t | 6 --- test/etap/011-file-headers.t | 7 ---- test/etap/020-btree-basics.t | 10 ----- test/etap/021-btree-reductions.t | 1 - test/etap/050-stream.t | 1 - 10 files changed, 33 insertions(+), 50 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/couchdb/blob/4f7336ed/src/couch_mrview/src/couch_mrview_index.erl ---------------------------------------------------------------------- diff --git a/src/couch_mrview/src/couch_mrview_index.erl b/src/couch_mrview/src/couch_mrview_index.erl index 21d38d1..1e7b7a3 100644 --- a/src/couch_mrview/src/couch_mrview_index.erl +++ b/src/couch_mrview/src/couch_mrview_index.erl @@ -128,8 +128,7 @@ finish_update(State) -> commit(State) -> Header = {State#mrst.sig, couch_mrview_util:make_header(State)}, - ok = couch_file:write_header(State#mrst.fd, Header), - ok = couch_file:flush(State#mrst.fd). + ok = couch_file:write_header(State#mrst.fd, Header). compact(Db, State, Opts) -> http://git-wip-us.apache.org/repos/asf/couchdb/blob/4f7336ed/src/couch_mrview/src/couch_mrview_updater.erl ---------------------------------------------------------------------- diff --git a/src/couch_mrview/src/couch_mrview_updater.erl b/src/couch_mrview/src/couch_mrview_updater.erl index ef6f2e7..3014664 100644 --- a/src/couch_mrview/src/couch_mrview_updater.erl +++ b/src/couch_mrview/src/couch_mrview_updater.erl @@ -55,8 +55,7 @@ start_update(Partial, State, NumChanges) -> purge(_Db, PurgeSeq, PurgedIdRevs, State) -> #mrst{ id_btree=IdBtree, - views=Views, - fd=Fd + views=Views } = State, Ids = [Id || {Id, _Revs} <- PurgedIdRevs], @@ -88,7 +87,6 @@ purge(_Db, PurgeSeq, PurgedIdRevs, State) -> end, Views2 = lists:map(RemKeysFun, Views), - ok = couch_file:flush(Fd), {ok, State#mrst{ id_btree=IdBtree2, views=Views2, @@ -231,8 +229,7 @@ insert_results(DocId, [KVs | RKVs], [{Id, VKVs} | RVKVs], VKVAcc, VIdKeys) -> write_kvs(State, UpdateSeq, ViewKVs, DocIdKeys) -> #mrst{ id_btree=IdBtree, - first_build=FirstBuild, - fd=Fd + first_build=FirstBuild } = State, {ok, ToRemove, IdBtree2} = update_id_btree(IdBtree, DocIdKeys, FirstBuild), @@ -248,7 +245,6 @@ write_kvs(State, UpdateSeq, ViewKVs, DocIdKeys) -> View#mrview{btree=VBtree2, update_seq=NewUpdateSeq} end, - ok = couch_file:flush(Fd), State#mrst{ views=lists:zipwith(UpdateView, State#mrst.views, ViewKVs), update_seq=UpdateSeq, http://git-wip-us.apache.org/repos/asf/couchdb/blob/4f7336ed/src/couch_mrview/src/couch_mrview_util.erl ---------------------------------------------------------------------- diff --git a/src/couch_mrview/src/couch_mrview_util.erl b/src/couch_mrview/src/couch_mrview_util.erl index 065a2cd..a13b154 100644 --- a/src/couch_mrview/src/couch_mrview_util.erl +++ b/src/couch_mrview/src/couch_mrview_util.erl @@ -546,7 +546,6 @@ delete_file(FName) -> reset_index(Db, Fd, #mrst{sig=Sig}=State) -> ok = couch_file:truncate(Fd, 0), ok = couch_file:write_header(Fd, {Sig, nil}), - ok = couch_file:flush(Fd), init_state(Db, Fd, reset_state(State), nil). http://git-wip-us.apache.org/repos/asf/couchdb/blob/4f7336ed/src/couchdb/couch_db_updater.erl ---------------------------------------------------------------------- diff --git a/src/couchdb/couch_db_updater.erl b/src/couchdb/couch_db_updater.erl index 03a04aa..a4d4202 100644 --- a/src/couchdb/couch_db_updater.erl +++ b/src/couchdb/couch_db_updater.erl @@ -659,8 +659,7 @@ update_docs_int(Db, DocsList, NonRepDocs, MergeConflicts, FullCommit) -> fulldocinfo_by_id_btree = DocInfoByIdBTree, docinfo_by_seq_btree = DocInfoBySeqBTree, update_seq = LastSeq, - revs_limit = RevsLimit, - fd = Fd + revs_limit = RevsLimit } = Db, Ids = [Id || [{_Client, #doc{id=Id}}|_] <- DocsList], % lookup up the old documents, if they exist. @@ -695,7 +694,6 @@ update_docs_int(Db, DocsList, NonRepDocs, MergeConflicts, FullCommit) -> fulldocinfo_by_id_btree = DocInfoByIdBTree2, docinfo_by_seq_btree = DocInfoBySeqBTree2, update_seq = NewSeq}, - couch_file:flush(Fd), % Check if we just updated any design documents, and update the validation % funs if we did. @@ -787,7 +785,6 @@ commit_data(Db, _) -> end, ok = couch_file:write_header(Fd, Header), - ok = couch_file:flush(Fd), case lists:member(after_header, FsyncOptions) of true -> ok = couch_file:sync(Fd); @@ -838,7 +835,6 @@ copy_docs(Db, #db{fd = DestFd} = NewDb, InfoBySeq0, Retry) -> InfoBySeq = lists:usort(fun(#doc_info{id=A}, #doc_info{id=B}) -> A =< B end, InfoBySeq0), Ids = [Id || #doc_info{id=Id} <- InfoBySeq], - ok = couch_file:flush(DestFd), LookupResults = couch_btree:lookup(Db#db.fulldocinfo_by_id_btree, Ids), NewFullDocInfos1 = lists:map( @@ -888,8 +884,6 @@ copy_docs(Db, #db{fd = DestFd} = NewDb, InfoBySeq0, Retry) -> copy_compact(Db, NewDb0, Retry) -> FsyncOptions = [Op || Op <- NewDb0#db.fsync_options, Op == before_header], NewDb = NewDb0#db{fsync_options=FsyncOptions}, - ok = couch_file:flush(Db#db.fd), - ok = couch_file:flush(NewDb#db.fd), TotalChanges = couch_db:count_changes_since(Db, NewDb#db.update_seq), BufferSize = list_to_integer( couch_config:get("database_compaction", "doc_buffer_size", "524288")), @@ -955,7 +949,6 @@ copy_compact(Db, NewDb0, Retry) -> NewDb4 = NewDb3 end, - ok = couch_file:flush(NewDb4#db.fd), commit_data(NewDb4#db{update_seq=Db#db.update_seq}). start_copy_compact(#db{name=Name,filepath=Filepath,header=#db_header{purge_seq=PurgeSeq}}=Db) -> http://git-wip-us.apache.org/repos/asf/couchdb/blob/4f7336ed/src/couchdb/couch_file.erl ---------------------------------------------------------------------- diff --git a/src/couchdb/couch_file.erl b/src/couchdb/couch_file.erl index 3979eb7..bb69004 100644 --- a/src/couchdb/couch_file.erl +++ b/src/couchdb/couch_file.erl @@ -132,6 +132,16 @@ pread_binary(Fd, Pos) -> pread_iolist(Fd, Pos) -> + case do_read_iolist(Fd, Pos) of + eof -> + flush(Fd), + do_read_iolist(Fd, Pos); + Else -> + Else + end. + + +do_read_iolist(Fd, Pos) -> case gen_server:call(Fd, {pread_iolist, Pos}, infinity) of {ok, _IoList} = Ok -> Ok; @@ -260,7 +270,8 @@ write_header(Fd, Data) -> Md5 = couch_util:md5(Bin), % now we assemble the final header binary and write to disk FinalBin = <<Md5/binary, Bin/binary>>, - ok = gen_server:call(Fd, {write_header, FinalBin}, infinity). + ok = gen_server:call(Fd, {write_header, FinalBin}, infinity), + ok = flush(Fd). % server functions @@ -472,8 +483,12 @@ read_raw_iolist_int(ReadFd, {Pos, _Size}, Len) -> % 0110 UPGRADE CODE read_raw_iolist_int(ReadFd, Pos, Len) -> BlockOffset = Pos rem ?SIZE_BLOCK, TotalBytes = calculate_total_read_len(BlockOffset, Len), - {ok, <<RawBin:TotalBytes/binary>>} = file:pread(ReadFd, Pos, TotalBytes), - {remove_block_prefixes(BlockOffset, RawBin), Pos + TotalBytes}. + case file:pread(ReadFd, Pos, TotalBytes) of + {ok, <<RawBin:TotalBytes/binary>>} -> + {remove_block_prefixes(BlockOffset, RawBin), Pos + TotalBytes}; + eof -> + throw(eof) + end. -spec extract_md5(iolist()) -> {binary(), iolist()}. extract_md5(FullIoList) -> @@ -649,7 +664,13 @@ write_header_blocks(Fd, Eof, Header) -> reader_loop(Fd) -> receive {read, Pos, From} -> - read_iolist(Fd, Pos, From), + Result = try + read_iolist(Fd, Pos) + catch + throw:eof -> + eof + end, + gen_server:reply(From, Result), reader_loop(Fd); {find_header, Eof, From} -> gen_server:reply(From, find_header(Fd, Eof div ?SIZE_BLOCK)), @@ -660,9 +681,9 @@ reader_loop(Fd) -> end. --compile({inline, [read_iolist/3]}). +-compile({inline, [read_iolist/2]}). -read_iolist(Fd, Pos, From) -> +read_iolist(Fd, Pos) -> {RawData, NextPos} = try % up to 8Kbs of read ahead read_raw_iolist_int(Fd, Pos, 2 * ?SIZE_BLOCK - (Pos rem ?SIZE_BLOCK)) @@ -676,8 +697,8 @@ read_iolist(Fd, Pos, From) -> 1 -> {Md5, IoList} = extract_md5( maybe_read_more_iolist(RestRawData, 16 + Len, NextPos, Fd)), - gen_server:reply(From, {ok, IoList, Md5}); + {ok, IoList, Md5}; 0 -> IoList = maybe_read_more_iolist(RestRawData, Len, NextPos, Fd), - gen_server:reply(From, {ok, IoList}) + {ok, IoList} end. http://git-wip-us.apache.org/repos/asf/couchdb/blob/4f7336ed/test/etap/010-file-basics.t ---------------------------------------------------------------------- diff --git a/test/etap/010-file-basics.t b/test/etap/010-file-basics.t index a1b9a98..fb1b29e 100755 --- a/test/etap/010-file-basics.t +++ b/test/etap/010-file-basics.t @@ -57,7 +57,6 @@ test() -> ?etap_match(couch_file:append_binary(Fd, <<"fancy!">>), {ok, Size, _}, "Appending a binary returns the current file size."), - ok = couch_file:flush(Fd), etap:is({ok, foo}, couch_file:pread_term(Fd, 0), "Reading the first term returns what we wrote: foo"), @@ -71,30 +70,25 @@ test() -> ), {ok, BinPos, _} = couch_file:append_binary(Fd, <<131,100,0,3,102,111,111>>), - ok = couch_file:flush(Fd), etap:is({ok, foo}, couch_file:pread_term(Fd, BinPos), "Reading a term from a written binary term representation succeeds."), BigBin = list_to_binary(lists:duplicate(100000, 0)), {ok, BigBinPos, _} = couch_file:append_binary(Fd, BigBin), - ok = couch_file:flush(Fd), etap:is({ok, BigBin}, couch_file:pread_binary(Fd, BigBinPos), "Reading a large term from a written representation succeeds."), ok = couch_file:write_header(Fd, hello), - ok = couch_file:flush(Fd), etap:is({ok, hello}, couch_file:read_header(Fd), "Reading a header succeeds."), {ok, BigBinPos2, _} = couch_file:append_binary(Fd, BigBin), - ok = couch_file:flush(Fd), etap:is({ok, BigBin}, couch_file:pread_binary(Fd, BigBinPos2), "Reading a large term from a written representation succeeds 2."), % append_binary == append_iolist? % Possible bug in pread_iolist or iolist() -> append_binary {ok, IOLPos, _} = couch_file:append_binary(Fd, ["foo", $m, <<"bam">>]), - ok = couch_file:flush(Fd), {ok, IoList} = couch_file:pread_iolist(Fd, IOLPos), etap:is(<<"foombam">>, iolist_to_binary(IoList), "Reading an results in a binary form of the written iolist()"), http://git-wip-us.apache.org/repos/asf/couchdb/blob/4f7336ed/test/etap/011-file-headers.t ---------------------------------------------------------------------- diff --git a/test/etap/011-file-headers.t b/test/etap/011-file-headers.t index 82c028f..a26b032 100755 --- a/test/etap/011-file-headers.t +++ b/test/etap/011-file-headers.t @@ -45,7 +45,6 @@ test() -> etap:is_greater(Size1, 0, "Writing a header allocates space in the file."), - ok = couch_file:flush(Fd), etap:is({ok, {<<"some_data">>, 32}}, couch_file:read_header(Fd), "Reading the header returns what we wrote."), @@ -56,7 +55,6 @@ test() -> etap:is_greater(Size2, Size1, "Writing a second header allocates more space."), - ok = couch_file:flush(Fd), etap:is({ok, [foo, <<"more">>]}, couch_file:read_header(Fd), "Reading the second header does not return the first header."), @@ -71,7 +69,6 @@ test() -> "Rewriting the same second header returns the same second size."), couch_file:write_header(Fd, erlang:make_tuple(5000, <<"CouchDB">>)), - ok = couch_file:flush(Fd), etap:is( couch_file:read_header(Fd), {ok, erlang:make_tuple(5000, <<"CouchDB">>)}, @@ -85,7 +82,6 @@ test() -> % Destroy the 0x1 byte that marks a header check_header_recovery(fun(CouchFd, RawFd, Expect, HeaderPos) -> - ok = couch_file:flush(CouchFd), etap:isnt(Expect, couch_file:read_header(CouchFd), "Should return a different header before corruption."), file:pwrite(RawFd, HeaderPos, <<0>>), @@ -95,7 +91,6 @@ test() -> % Corrupt the size. check_header_recovery(fun(CouchFd, RawFd, Expect, HeaderPos) -> - ok = couch_file:flush(CouchFd), etap:isnt(Expect, couch_file:read_header(CouchFd), "Should return a different header before corruption."), % +1 for 0x1 byte marker @@ -106,7 +101,6 @@ test() -> % Corrupt the MD5 signature check_header_recovery(fun(CouchFd, RawFd, Expect, HeaderPos) -> - ok = couch_file:flush(CouchFd), etap:isnt(Expect, couch_file:read_header(CouchFd), "Should return a different header before corruption."), % +5 = +1 for 0x1 byte and +4 for term size. @@ -117,7 +111,6 @@ test() -> % Corrupt the data check_header_recovery(fun(CouchFd, RawFd, Expect, HeaderPos) -> - ok = couch_file:flush(CouchFd), etap:isnt(Expect, couch_file:read_header(CouchFd), "Should return a different header before corruption."), % +21 = +1 for 0x1 byte, +4 for term size and +16 for MD5 sig http://git-wip-us.apache.org/repos/asf/couchdb/blob/4f7336ed/test/etap/020-btree-basics.t ---------------------------------------------------------------------- diff --git a/test/etap/020-btree-basics.t b/test/etap/020-btree-basics.t index 372b494..e65aeb2 100755 --- a/test/etap/020-btree-basics.t +++ b/test/etap/020-btree-basics.t @@ -97,7 +97,6 @@ test_kvs(KeyValues) -> "After removing all keys btree size is 0."), {Btree4, _} = lists:foldl(fun(KV, {BtAcc, PrevSize}) -> - ok = couch_file:flush(Fd), {ok, BtAcc2} = couch_btree:add_remove(BtAcc, [KV], []), case couch_btree:size(BtAcc2) > PrevSize of true -> @@ -107,7 +106,6 @@ test_kvs(KeyValues) -> end, {BtAcc2, couch_btree:size(BtAcc2)} end, {Btree3, couch_btree:size(Btree3)}, KeyValues), - ok = couch_file:flush(Fd), etap:ok(test_btree(Btree4, KeyValues), "Adding all keys one at a time returns a complete btree."), @@ -115,7 +113,6 @@ test_kvs(KeyValues) -> "Non empty btrees have a size > 0."), {Btree5, _} = lists:foldl(fun({K, _}, {BtAcc, PrevSize}) -> - ok = couch_file:flush(Fd), {ok, BtAcc2} = couch_btree:add_remove(BtAcc, [], [K]), case couch_btree:size(BtAcc2) < PrevSize of true -> @@ -125,7 +122,6 @@ test_kvs(KeyValues) -> end, {BtAcc2, couch_btree:size(BtAcc2)} end, {Btree4, couch_btree:size(Btree4)}, KeyValues), - ok = couch_file:flush(Fd), etap:ok(test_btree(Btree5, []), "Removing all keys one at a time returns an empty btree."), @@ -134,7 +130,6 @@ test_kvs(KeyValues) -> KeyValuesRev = lists:reverse(KeyValues), {Btree6, _} = lists:foldl(fun(KV, {BtAcc, PrevSize}) -> - ok = couch_file:flush(Fd), {ok, BtAcc2} = couch_btree:add_remove(BtAcc, [KV], []), case couch_btree:size(BtAcc2) > PrevSize of true -> @@ -177,7 +172,6 @@ test_kvs(KeyValues) -> true. test_btree(#btree{fd = Fd} = Btree, KeyValues) -> - ok = couch_file:flush(Fd), ok = test_key_access(Btree, KeyValues), ok = test_lookup_access(Btree, KeyValues), ok = test_final_reductions(Btree, KeyValues), @@ -185,7 +179,6 @@ test_btree(#btree{fd = Fd} = Btree, KeyValues) -> true. test_add_remove(#btree{fd = Fd} = Btree, OutKeyValues, RemainingKeyValues) -> - ok = couch_file:flush(Fd), Btree2 = lists:foldl(fun({K, _}, BtAcc) -> {ok, BtAcc2} = couch_btree:add_remove(BtAcc, [], [K]), BtAcc2 @@ -207,13 +200,11 @@ test_key_access(#btree{fd = Fd} = Btree, List) -> end, Length = length(List), Sorted = lists:sort(List), - ok = couch_file:flush(Fd), {ok, _, {[], Length}} = couch_btree:foldl(Btree, FoldFun, {Sorted, 0}), {ok, _, {[], Length}} = couch_btree:fold(Btree, FoldFun, {Sorted, 0}, [{dir, rev}]), ok. test_lookup_access(#btree{fd = Fd} = Btree, KeyValues) -> - ok = couch_file:flush(Fd), FoldFun = fun({Key, Value}, {Key, Value}) -> {stop, true} end, lists:foreach(fun({Key, Value}) -> [{ok, {Key, Value}}] = couch_btree:lookup(Btree, [Key]), @@ -221,7 +212,6 @@ test_lookup_access(#btree{fd = Fd} = Btree, KeyValues) -> end, KeyValues). test_final_reductions(#btree{fd = Fd} = Btree, KeyValues) -> - ok = couch_file:flush(Fd), KVLen = length(KeyValues), FoldLFun = fun(_X, LeadingReds, Acc) -> CountToStart = KVLen div 3 + Acc, http://git-wip-us.apache.org/repos/asf/couchdb/blob/4f7336ed/test/etap/021-btree-reductions.t ---------------------------------------------------------------------- diff --git a/test/etap/021-btree-reductions.t b/test/etap/021-btree-reductions.t index e597772..331e49a 100755 --- a/test/etap/021-btree-reductions.t +++ b/test/etap/021-btree-reductions.t @@ -47,7 +47,6 @@ test()-> end, {"odd", []}, lists:seq(1, rows())), {ok, Btree2} = couch_btree:add_remove(Btree, EvenOddKVs, []), - ok = couch_file:flush(Fd), GroupFun = fun({K1, _}, {K2, _}) -> K1 == K2 end, FoldFun = fun(GroupedKey, Unreduced, Acc) -> http://git-wip-us.apache.org/repos/asf/couchdb/blob/4f7336ed/test/etap/050-stream.t ---------------------------------------------------------------------- diff --git a/test/etap/050-stream.t b/test/etap/050-stream.t index 622079a..de0dfad 100755 --- a/test/etap/050-stream.t +++ b/test/etap/050-stream.t @@ -26,7 +26,6 @@ main(_) -> ok. read_all(Fd, PosList) -> - ok = couch_file:flush(Fd), Data = couch_stream:foldl(Fd, PosList, fun(Bin, Acc) -> [Bin, Acc] end, []), iolist_to_binary(Data).
