AlexanderKaraberov closed pull request #1382: Fix for a function_clause crash
in couch_native_process
URL: https://github.com/apache/couchdb/pull/1382
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/Makefile b/Makefile
index d56d4e8973..1e0ea82242 100644
--- a/Makefile
+++ b/Makefile
@@ -30,20 +30,23 @@ DESTDIR=
# Rebar options
apps=
-skip_deps=folsom,meck,mochiweb,triq,snappy
+skip_deps=folsom,meck,mochiweb,triq,snappy,bcrypt,hyper
suites=
tests=
+COMPILE_OPTS=$(shell echo "\
+ apps=$(apps) \
+ " | sed -e 's/[a-z_]\{1,\}= / /g')
EUNIT_OPTS=$(shell echo "\
apps=$(apps) \
skip_deps=$(skip_deps) \
suites=$(suites) \
tests=$(tests) \
- " | sed -e 's/[a-z]\+= / /g')
+ " | sed -e 's/[a-z]\{1,\}= / /g')
DIALYZE_OPTS=$(shell echo "\
apps=$(apps) \
skip_deps=$(skip_deps) \
- " | sed -e 's/[a-z]\+= / /g')
+ " | sed -e 's/[a-z]\{1,\}= / /g')
#ignore javascript tests
ignore_js_suites=
@@ -73,9 +76,9 @@ help:
.PHONY: couch
-# target: couch - Build CouchDB core
+# target: couch - Build CouchDB core, use ERL_OPTS to provide custom
compiler's options
couch: config.erl
- @COUCHDB_VERSION=$(COUCHDB_VERSION) $(REBAR) compile
+ @COUCHDB_VERSION=$(COUCHDB_VERSION) $(REBAR) compile $(COMPILE_OPTS)
@cp src/couch/priv/couchjs bin/
diff --git a/rebar.config.script b/rebar.config.script
index a3462c9900..82f4d70b99 100644
--- a/rebar.config.script
+++ b/rebar.config.script
@@ -87,13 +87,18 @@ MakeDep = fun
{AppName, ".*", {git, Url, Version}, Options}
end,
+ErlOpts = case os:getenv("ERL_OPTS") of
+ false -> [];
+ Opts -> [list_to_atom(O) || O <- string:tokens(Opts, ",")]
+end,
+
AddConfig = [
{require_otp_vsn, "R16B03|R16B03-1|17|18|19|20"},
{deps_dir, "src"},
{deps, lists:map(MakeDep, DepDescs)},
{sub_dirs, SubDirs},
{lib_dirs, ["src"]},
- {erl_opts, [bin_opt_info, debug_info, {i, "../"}]},
+ {erl_opts, [{i, "../"} | ErlOpts]},
{eunit_opts, [verbose]},
{plugins, [eunit_plugin]},
{dialyzer, [
diff --git a/src/chttpd/src/chttpd.erl b/src/chttpd/src/chttpd.erl
index 6be0d1848c..ae94ae6fca 100644
--- a/src/chttpd/src/chttpd.erl
+++ b/src/chttpd/src/chttpd.erl
@@ -288,11 +288,7 @@ process_request(#httpd{mochi_req = MochiReq} = HttpReq) ->
not_preflight ->
case chttpd_auth:authenticate(HttpReq, fun authenticate_request/1)
of
#httpd{} = Req ->
- HandlerFun = chttpd_handlers:url_handler(
- HandlerKey, fun chttpd_db:handle_request/1),
- AuthorizedReq = chttpd_auth:authorize(possibly_hack(Req),
- fun chttpd_auth_request:authorize_request/1),
- {AuthorizedReq, HandlerFun(AuthorizedReq)};
+ handle_req_after_auth(HandlerKey, Req);
Response ->
{HttpReq, Response}
end;
@@ -303,6 +299,17 @@ process_request(#httpd{mochi_req = MochiReq} = HttpReq) ->
{HttpReq, catch_error(HttpReq, Tag, Error)}
end.
+handle_req_after_auth(HandlerKey, HttpReq) ->
+ try
+ HandlerFun = chttpd_handlers:url_handler(HandlerKey,
+ fun chttpd_db:handle_request/1),
+ AuthorizedReq = chttpd_auth:authorize(possibly_hack(HttpReq),
+ fun chttpd_auth_request:authorize_request/1),
+ {AuthorizedReq, HandlerFun(AuthorizedReq)}
+ catch Tag:Error ->
+ {HttpReq, catch_error(HttpReq, Tag, Error)}
+ end.
+
catch_error(_HttpReq, throw, {http_head_abort, Resp}) ->
{ok, Resp};
catch_error(_HttpReq, throw, {http_abort, Resp, Reason}) ->
@@ -1238,4 +1245,38 @@ test_log_request(RawPath, UserCtx) ->
ok = meck:unload(couch_log),
Message.
+handle_req_after_auth_test() ->
+ Headers = mochiweb_headers:make([{"HOST", "127.0.0.1:15984"}]),
+ MochiReq = mochiweb_request:new(socket, [], 'PUT', "/newdb", version,
+ Headers),
+ UserCtx = #user_ctx{name = <<"retain_user">>},
+ Roles = [<<"_reader">>],
+ AuthorizedCtx = #user_ctx{name = <<"retain_user">>, roles = Roles},
+ Req = #httpd{
+ mochi_req = MochiReq,
+ begin_ts = {1458,588713,124003},
+ original_method = 'PUT',
+ peer = "127.0.0.1",
+ nonce = "nonce",
+ user_ctx = UserCtx
+ },
+ AuthorizedReq = Req#httpd{user_ctx = AuthorizedCtx},
+ ok = meck:new(chttpd_handlers, [passthrough]),
+ ok = meck:new(chttpd_auth, [passthrough]),
+ ok = meck:expect(chttpd_handlers, url_handler, fun(_Key, _Fun) ->
+ fun(_Req) -> handled_authorized_req end
+ end),
+ ok = meck:expect(chttpd_auth, authorize, fun(_Req, _Fun) ->
+ AuthorizedReq
+ end),
+ ?assertEqual({AuthorizedReq, handled_authorized_req},
+ handle_req_after_auth(foo_key, Req)),
+ ok = meck:expect(chttpd_auth, authorize, fun(_Req, _Fun) ->
+ meck:exception(throw, {http_abort, resp, some_reason})
+ end),
+ ?assertEqual({Req, {aborted, resp, some_reason}},
+ handle_req_after_auth(foo_key, Req)),
+ ok = meck:unload(chttpd_handlers),
+ ok = meck:unload(chttpd_auth).
+
-endif.
diff --git a/src/couch/src/couch_db.erl b/src/couch/src/couch_db.erl
index 93ea07e659..b47cc7eceb 100644
--- a/src/couch/src/couch_db.erl
+++ b/src/couch/src/couch_db.erl
@@ -870,16 +870,10 @@ prep_and_validate_replicated_updates(Db,
[Bucket|RestBuckets], [OldInfo|RestOldI
{[], AccErrors}, Bucket),
prep_and_validate_replicated_updates(Db, RestBuckets, RestOldInfo,
[ValidatedBucket | AccPrepped], AccErrors3);
#full_doc_info{rev_tree=OldTree} ->
- RevsLimit = get_revs_limit(Db),
OldLeafs = couch_key_tree:get_all_leafs_full(OldTree),
OldLeafsLU = [{Start, RevId} || {Start, [{RevId, _}|_]} <- OldLeafs],
- NewRevTree = lists:foldl(
- fun(NewDoc, AccTree) ->
- {NewTree, _} = couch_key_tree:merge(AccTree,
- couch_doc:to_path(NewDoc), RevsLimit),
- NewTree
- end,
- OldTree, Bucket),
+ NewPaths = lists:map(fun couch_doc:to_path/1, Bucket),
+ NewRevTree = couch_key_tree:multi_merge(OldTree, NewPaths),
Leafs = couch_key_tree:get_all_leafs_full(NewRevTree),
LeafRevsFullDict = dict:from_list( [{{Start, RevId}, FullPath} ||
{Start, [{RevId, _}|_]}=FullPath <- Leafs]),
{ValidatedBucket, AccErrors3} =
diff --git a/src/couch/src/couch_db_updater.erl
b/src/couch/src/couch_db_updater.erl
index a2de3bc606..fba99a7735 100644
--- a/src/couch/src/couch_db_updater.erl
+++ b/src/couch/src/couch_db_updater.erl
@@ -504,23 +504,24 @@ merge_rev_trees(Limit, MergeConflicts,
[NewDocs|RestDocsList],
[OldDocInfo|RestOldInfo], AccNewInfos, AccRemoveSeqs, AccSeq) ->
erlang:put(last_id_merged, OldDocInfo#full_doc_info.id), % for debugging
NewDocInfo0 = lists:foldl(fun({Client, NewDoc}, OldInfoAcc) ->
- merge_rev_tree(OldInfoAcc, NewDoc, Client, Limit, MergeConflicts)
+ merge_rev_tree(OldInfoAcc, NewDoc, Client, MergeConflicts)
end, OldDocInfo, NewDocs),
+ NewDocInfo1 = stem_full_doc_info(NewDocInfo0, Limit),
% When MergeConflicts is false, we updated #full_doc_info.deleted on every
% iteration of merge_rev_tree. However, merge_rev_tree does not update
% #full_doc_info.deleted when MergeConflicts is true, since we don't need
% to know whether the doc is deleted between iterations. Since we still
% need to know if the doc is deleted after the merge happens, we have to
% set it here.
- NewDocInfo1 = case MergeConflicts of
+ NewDocInfo2 = case MergeConflicts of
true ->
- NewDocInfo0#full_doc_info{
- deleted = couch_doc:is_deleted(NewDocInfo0)
+ NewDocInfo1#full_doc_info{
+ deleted = couch_doc:is_deleted(NewDocInfo1)
};
false ->
- NewDocInfo0
+ NewDocInfo1
end,
- if NewDocInfo1 == OldDocInfo ->
+ if NewDocInfo2 == OldDocInfo ->
% nothing changed
merge_rev_trees(Limit, MergeConflicts, RestDocsList, RestOldInfo,
AccNewInfos, AccRemoveSeqs, AccSeq);
@@ -529,7 +530,7 @@ merge_rev_trees(Limit, MergeConflicts,
[NewDocs|RestDocsList],
% important to note that the update_seq on OldDocInfo should
% be identical to the value on NewDocInfo1.
OldSeq = OldDocInfo#full_doc_info.update_seq,
- NewDocInfo2 = NewDocInfo1#full_doc_info{
+ NewDocInfo3 = NewDocInfo2#full_doc_info{
update_seq = AccSeq + 1
},
RemoveSeqs = case OldSeq of
@@ -537,10 +538,10 @@ merge_rev_trees(Limit, MergeConflicts,
[NewDocs|RestDocsList],
_ -> [OldSeq | AccRemoveSeqs]
end,
merge_rev_trees(Limit, MergeConflicts, RestDocsList, RestOldInfo,
- [NewDocInfo2|AccNewInfos], RemoveSeqs, AccSeq+1)
+ [NewDocInfo3|AccNewInfos], RemoveSeqs, AccSeq+1)
end.
-merge_rev_tree(OldInfo, NewDoc, Client, Limit, false)
+merge_rev_tree(OldInfo, NewDoc, Client, false)
when OldInfo#full_doc_info.deleted ->
% We're recreating a document that was previously
% deleted. To check that this is a recreation from
@@ -574,7 +575,7 @@ merge_rev_tree(OldInfo, NewDoc, Client, Limit, false)
% Merge our modified new doc into the tree
#full_doc_info{rev_tree=OldTree} = OldInfo,
NewTree0 = couch_doc:to_path(NewDoc2),
- case couch_key_tree:merge(OldTree, NewTree0, Limit) of
+ case couch_key_tree:merge(OldTree, NewTree0) of
{NewTree1, new_leaf} ->
% We changed the revision id so inform the caller
send_result(Client, NewDoc, {ok, {OldPos+1, NewRevId}}),
@@ -589,7 +590,7 @@ merge_rev_tree(OldInfo, NewDoc, Client, Limit, false)
send_result(Client, NewDoc, conflict),
OldInfo
end;
-merge_rev_tree(OldInfo, NewDoc, Client, Limit, false) ->
+merge_rev_tree(OldInfo, NewDoc, Client, false) ->
% We're attempting to merge a new revision into an
% undeleted document. To not be a conflict we require
% that the merge results in extending a branch.
@@ -597,7 +598,7 @@ merge_rev_tree(OldInfo, NewDoc, Client, Limit, false) ->
OldTree = OldInfo#full_doc_info.rev_tree,
NewTree0 = couch_doc:to_path(NewDoc),
NewDeleted = NewDoc#doc.deleted,
- case couch_key_tree:merge(OldTree, NewTree0, Limit) of
+ case couch_key_tree:merge(OldTree, NewTree0) of
{NewTree, new_leaf} when not NewDeleted ->
OldInfo#full_doc_info{
rev_tree = NewTree,
@@ -615,14 +616,18 @@ merge_rev_tree(OldInfo, NewDoc, Client, Limit, false) ->
send_result(Client, NewDoc, conflict),
OldInfo
end;
-merge_rev_tree(OldInfo, NewDoc, _Client, Limit, true) ->
+merge_rev_tree(OldInfo, NewDoc, _Client, true) ->
% We're merging in revisions without caring about
% conflicts. Most likely this is a replication update.
OldTree = OldInfo#full_doc_info.rev_tree,
NewTree0 = couch_doc:to_path(NewDoc),
- {NewTree, _} = couch_key_tree:merge(OldTree, NewTree0, Limit),
+ {NewTree, _} = couch_key_tree:merge(OldTree, NewTree0),
OldInfo#full_doc_info{rev_tree = NewTree}.
+stem_full_doc_info(#full_doc_info{rev_tree = Tree} = Info, Limit) ->
+ Stemmed = couch_key_tree:stem(Tree, Limit),
+ Info#full_doc_info{rev_tree = Stemmed}.
+
update_docs_int(Db, DocsList, LocalDocs, MergeConflicts, FullCommit) ->
UpdateSeq = couch_db_engine:get_update_seq(Db),
RevsLimit = couch_db_engine:get_revs_limit(Db),
diff --git a/src/couch/src/couch_key_tree.erl b/src/couch/src/couch_key_tree.erl
index cd661e29aa..5d53615072 100644
--- a/src/couch/src/couch_key_tree.erl
+++ b/src/couch/src/couch_key_tree.erl
@@ -59,6 +59,7 @@ get_key_leafs/2,
map/2,
map_leafs/2,
mapfold/3,
+multi_merge/2,
merge/3,
merge/2,
remove_leafs/2,
@@ -71,6 +72,15 @@ stem/2
-type revtree() :: [tree()].
+%% @doc Merge multiple paths into the given tree.
+-spec multi_merge(revtree(), tree()) -> revtree().
+multi_merge(RevTree, Trees) ->
+ lists:foldl(fun(Tree, RevTreeAcc) ->
+ {NewRevTree, _} = merge(RevTreeAcc, Tree),
+ NewRevTree
+ end, RevTree, lists:sort(Trees)).
+
+
%% @doc Merge a path into the given tree and then stem the result.
%% Although Tree is of type tree(), it must not contain any branches.
-spec merge(revtree(), tree() | path(), pos_integer()) ->
@@ -470,6 +480,70 @@ map_leafs_simple(Fun, Pos, [{Key, Value, SubTree} |
RestTree]) ->
stem(Trees, Limit) ->
+ try
+ {_, Branches} = lists:foldl(fun(Tree, {Seen, TreeAcc}) ->
+ {NewSeen, NewBranches} = stem_tree(Tree, Limit, Seen),
+ {NewSeen, NewBranches ++ TreeAcc}
+ end, {sets:new(), []}, Trees),
+ lists:sort(Branches)
+ catch throw:dupe_keys ->
+ repair_tree(Trees, Limit)
+ end.
+
+
+stem_tree({Depth, Child}, Limit, Seen) ->
+ case stem_tree(Depth, Child, Limit, Seen) of
+ {NewSeen, _, NewChild, NewBranches} ->
+ {NewSeen, [{Depth, NewChild} | NewBranches]};
+ {NewSeen, _, NewBranches} ->
+ {NewSeen, NewBranches}
+ end.
+
+
+stem_tree(_Depth, {Key, _Val, []} = Leaf, Limit, Seen) ->
+ {check_key(Key, Seen), Limit - 1, Leaf, []};
+
+stem_tree(Depth, {Key, Val, Children}, Limit, Seen0) ->
+ Seen1 = check_key(Key, Seen0),
+ FinalAcc = lists:foldl(fun(Child, Acc) ->
+ {SeenAcc, LimitPosAcc, ChildAcc, BranchAcc} = Acc,
+ case stem_tree(Depth + 1, Child, Limit, SeenAcc) of
+ {NewSeenAcc, LimitPos, NewChild, NewBranches} ->
+ NewLimitPosAcc = erlang:max(LimitPos, LimitPosAcc),
+ NewChildAcc = [NewChild | ChildAcc],
+ NewBranchAcc = NewBranches ++ BranchAcc,
+ {NewSeenAcc, NewLimitPosAcc, NewChildAcc, NewBranchAcc};
+ {NewSeenAcc, LimitPos, NewBranches} ->
+ NewLimitPosAcc = erlang:max(LimitPos, LimitPosAcc),
+ NewBranchAcc = NewBranches ++ BranchAcc,
+ {NewSeenAcc, NewLimitPosAcc, ChildAcc, NewBranchAcc}
+ end
+ end, {Seen1, -1, [], []}, Children),
+ {FinalSeen, FinalLimitPos, FinalChildren, FinalBranches} = FinalAcc,
+ case FinalLimitPos of
+ N when N > 0, length(FinalChildren) > 0 ->
+ FinalNode = {Key, Val, lists:reverse(FinalChildren)},
+ {FinalSeen, FinalLimitPos - 1, FinalNode, FinalBranches};
+ 0 when length(FinalChildren) > 0 ->
+ NewBranches = lists:map(fun(Child) ->
+ {Depth + 1, Child}
+ end, lists:reverse(FinalChildren)),
+ {FinalSeen, -1, NewBranches ++ FinalBranches};
+ N when N < 0, length(FinalChildren) == 0 ->
+ {FinalSeen, FinalLimitPos - 1, FinalBranches}
+ end.
+
+
+check_key(Key, Seen) ->
+ case sets:is_element(Key, Seen) of
+ true ->
+ throw(dupe_keys);
+ false ->
+ sets:add_element(Key, Seen)
+ end.
+
+
+repair_tree(Trees, Limit) ->
% flatten each branch in a tree into a tree path, sort by starting rev #
Paths = lists:sort(lists:map(fun({Pos, Path}) ->
StemmedPath = lists:sublist(Path, Limit),
diff --git a/src/couch/src/couch_native_process.erl
b/src/couch/src/couch_native_process.erl
index 6d66c936b3..8f8ce8b1d3 100644
--- a/src/couch/src/couch_native_process.erl
+++ b/src/couch/src/couch_native_process.erl
@@ -226,6 +226,18 @@ ddoc(State, {_, Fun}, [<<"filters">>|_], [Docs, Req]) ->
end,
Resp = lists:map(FilterFunWrapper, Docs),
{State, [true, Resp]};
+ddoc(State, {_, Fun}, [<<"views">>|_], [Docs]) ->
+ MapFunWrapper = fun(Doc) ->
+ case catch Fun(Doc) of
+ undefined -> true;
+ ok -> false;
+ false -> false;
+ [_|_] -> true;
+ {'EXIT', Error} -> couch_log:error("~p", [Error])
+ end
+ end,
+ Resp = lists:map(MapFunWrapper, Docs),
+ {State, [true, Resp]};
ddoc(State, {_, Fun}, [<<"shows">>|_], Args) ->
Resp = case (catch apply(Fun, Args)) of
FunResp when is_list(FunResp) ->
diff --git a/src/couch/src/couch_server.erl b/src/couch/src/couch_server.erl
index 05af0ed54a..903eb09cd5 100644
--- a/src/couch/src/couch_server.erl
+++ b/src/couch/src/couch_server.erl
@@ -210,6 +210,8 @@ close_db_if_idle(DbName) ->
init([]) ->
+ couch_util:set_mqd_off_heap(),
+
% Mark pluggable storage engines as a supported feature
config:enable_feature('pluggable-storage-engines'),
diff --git a/src/couch/src/couch_util.erl b/src/couch/src/couch_util.erl
index f3a9249f71..af1c0ef871 100644
--- a/src/couch/src/couch_util.erl
+++ b/src/couch/src/couch_util.erl
@@ -37,6 +37,7 @@
-export([unique_monotonic_integer/0]).
-export([check_config_blacklist/1]).
-export([check_md5/2]).
+-export([set_mqd_off_heap/0]).
-include_lib("couch/include/couch_db.hrl").
@@ -639,6 +640,15 @@ check_md5(Sig, Sig) -> ok;
check_md5(_, _) -> throw(md5_mismatch).
+set_mqd_off_heap() ->
+ try
+ erlang:process_flag(message_queue_data, off_heap),
+ ok
+ catch error:badarg ->
+ ok
+ end.
+
+
ensure_loaded(Module) when is_atom(Module) ->
case code:ensure_loaded(Module) of
{module, Module} ->
diff --git a/src/couch/test/couch_changes_tests.erl
b/src/couch/test/couch_changes_tests.erl
index 673f2faad3..e4ea761676 100644
--- a/src/couch/test/couch_changes_tests.erl
+++ b/src/couch/test/couch_changes_tests.erl
@@ -47,9 +47,11 @@ setup() ->
save_doc(Db1, {[{<<"_id">>, <<"doc7">>}]}),
save_doc(Db1, {[{<<"_id">>, <<"doc8">>}]})
]],
+ config:set("native_query_servers", "erlang", "{couch_native_process,
start_link, []}", _Persist=false),
{DbName, list_to_tuple(Revs2)}.
teardown({DbName, _}) ->
+ config:delete("native_query_servers", "erlang", _Persist=false),
delete_db(DbName),
ok.
@@ -153,7 +155,8 @@ filter_by_view() ->
fun setup/0, fun teardown/1,
[
fun should_filter_by_view/1,
- fun should_filter_by_fast_view/1
+ fun should_filter_by_fast_view/1,
+ fun should_filter_by_erlang_view/1
]
}
}.
@@ -733,6 +736,39 @@ should_filter_by_fast_view({DbName, _}) ->
?assertEqual(UpSeq, ViewUpSeq)
end).
+should_filter_by_erlang_view({DbName, _}) ->
+ ?_test(
+ begin
+ DDocId = <<"_design/app">>,
+ DDoc = couch_doc:from_json_obj({[
+ {<<"_id">>, DDocId},
+ {<<"language">>, <<"erlang">>},
+ {<<"views">>, {[
+ {<<"valid">>, {[
+ {<<"map">>, <<"fun({Doc}) ->"
+ " case lists:keyfind(<<\"_id\">>, 1, Doc) of"
+ " {<<\"_id\">>, <<\"doc3\">>} -> Emit(Doc,
null); "
+ " false -> ok"
+ " end "
+ "end.">>}
+ ]}}
+ ]}}
+ ]}),
+ ChArgs = #changes_args{filter = "_view"},
+ Req = {json_req, {[{
+ <<"query">>, {[
+ {<<"view">>, <<"app/valid">>}
+ ]}
+ }]}},
+ ok = update_ddoc(DbName, DDoc),
+ {Rows, LastSeq, UpSeq} = run_changes_query(DbName, ChArgs, Req),
+ ?assertEqual(1, length(Rows)),
+ [#row{seq = Seq, id = Id}] = Rows,
+ ?assertEqual(<<"doc3">>, Id),
+ ?assertEqual(6, Seq),
+ ?assertEqual(UpSeq, LastSeq)
+ end).
+
update_ddoc(DbName, DDoc) ->
{ok, Db} = couch_db:open_int(DbName, [?ADMIN_CTX]),
{ok, _} = couch_db:update_doc(Db, DDoc, []),
diff --git a/src/couch/test/couch_key_tree_tests.erl
b/src/couch/test/couch_key_tree_tests.erl
index 88d9203635..2b7d5fe62a 100644
--- a/src/couch/test/couch_key_tree_tests.erl
+++ b/src/couch/test/couch_key_tree_tests.erl
@@ -167,8 +167,23 @@ should_merge_reflexive_for_child_nodes()->
should_merge_tree_to_itself()->
TwoChildSibs = {1, {"1","foo", [{"1a", "bar", []},
{"1b", "bar", []}]}},
- ?_assertEqual({[TwoChildSibs], new_branch},
- couch_key_tree:merge([TwoChildSibs], TwoChildSibs, ?DEPTH)).
+ Leafs = couch_key_tree:get_all_leafs([TwoChildSibs]),
+ Paths = lists:map(fun leaf_to_path/1, Leafs),
+ FinalTree = lists:foldl(fun(Path, TreeAcc) ->
+ {NewTree, internal_node} = couch_key_tree:merge(TreeAcc, Path),
+ NewTree
+ end, [TwoChildSibs], Paths),
+ ?_assertEqual([TwoChildSibs], FinalTree).
+
+leaf_to_path({Value, {Start, Keys}}) ->
+ [Branch] = to_branch(Value, lists:reverse(Keys)),
+ {Start - length(Keys) + 1, Branch}.
+
+to_branch(Value, [Key]) ->
+ [{Key, Value, []}];
+to_branch(Value, [Key | RestKeys]) ->
+ [{Key, [], to_branch(Value, RestKeys)}].
+
should_merge_tree_of_odd_length()->
TwoChild = {1, {"1","foo", [{"1a", "bar", [{"1aa", "bar", []}]}]}},
@@ -176,9 +191,8 @@ should_merge_tree_of_odd_length()->
{"1b", "bar", []}]}},
TwoChildPlusSibs = {1, {"1","foo", [{"1a", "bar", [{"1aa", "bar", []}]},
{"1b", "bar", []}]}},
-
- ?_assertEqual({[TwoChildPlusSibs], new_branch},
- couch_key_tree:merge([TwoChild], TwoChildSibs, ?DEPTH)).
+ ?_assertEqual({[TwoChildPlusSibs], new_leaf},
+ couch_key_tree:merge([TwoChildSibs], TwoChild, ?DEPTH)).
should_merge_tree_with_stem()->
Stemmed = {2, {"1a", "bar", []}},
diff --git a/src/couch_log/src/couch_log_server.erl
b/src/couch_log/src/couch_log_server.erl
index be44af8ffe..ea5def8913 100644
--- a/src/couch_log/src/couch_log_server.erl
+++ b/src/couch_log/src/couch_log_server.erl
@@ -58,6 +58,7 @@ log(Entry) ->
init(_) ->
+ couch_util:set_mqd_off_heap(),
process_flag(trap_exit, true),
{ok, #st{
writer = couch_log_writer:init()
diff --git a/src/couch_mrview/src/couch_mrview_index.erl
b/src/couch_mrview/src/couch_mrview_index.erl
index aa1ee27415..5d285d6392 100644
--- a/src/couch_mrview/src/couch_mrview_index.erl
+++ b/src/couch_mrview/src/couch_mrview_index.erl
@@ -61,13 +61,14 @@ get(info, State) ->
} = State,
{ok, FileSize} = couch_file:bytes(Fd),
{ok, ExternalSize} = couch_mrview_util:calculate_external_size(Views),
+ {ok, ActiveViewSize} = couch_mrview_util:calculate_active_size(Views),
LogBtSize = case LogBtree of
nil ->
0;
_ ->
couch_btree:size(LogBtree)
end,
- ActiveSize = couch_btree:size(IdBtree) + LogBtSize + ExternalSize,
+ ActiveSize = couch_btree:size(IdBtree) + LogBtSize + ActiveViewSize,
UpdateOptions0 = get(update_options, State),
UpdateOptions = [atom_to_binary(O, latin1) || O <- UpdateOptions0],
diff --git a/src/couch_mrview/src/couch_mrview_util.erl
b/src/couch_mrview/src/couch_mrview_util.erl
index 0c6e5fc88d..eb461d0174 100644
--- a/src/couch_mrview/src/couch_mrview_util.erl
+++ b/src/couch_mrview/src/couch_mrview_util.erl
@@ -23,6 +23,7 @@
-export([fold/4, fold_reduce/4]).
-export([temp_view_to_ddoc/1]).
-export([calculate_external_size/1]).
+-export([calculate_active_size/1]).
-export([validate_args/1]).
-export([maybe_load_doc/3, maybe_load_doc/4]).
-export([maybe_update_index_file/1]).
@@ -830,6 +831,22 @@ calculate_external_size(Views) ->
{ok, lists:foldl(SumFun, 0, Views)}.
+calculate_active_size(Views) ->
+ BtSize = fun
+ (nil) -> 0;
+ (Bt) -> couch_btree:size(Bt)
+ end,
+ FoldFun = fun(View, Acc) ->
+ Sizes = [
+ BtSize(View#mrview.btree),
+ BtSize(View#mrview.seq_btree),
+ BtSize(View#mrview.key_byseq_btree)
+ ],
+ Acc + lists:sum([S || S <- Sizes, is_integer(S)])
+ end,
+ {ok, lists:foldl(FoldFun, 0, Views)}.
+
+
sum_btree_sizes(nil, _) ->
0;
sum_btree_sizes(_, nil) ->
diff --git a/src/couch_mrview/test/couch_mrview_index_info_tests.erl
b/src/couch_mrview/test/couch_mrview_index_info_tests.erl
index c994df9d38..efa03e7c01 100644
--- a/src/couch_mrview/test/couch_mrview_index_info_tests.erl
+++ b/src/couch_mrview/test/couch_mrview_index_info_tests.erl
@@ -18,14 +18,13 @@
-define(TIMEOUT, 1000).
--ifdef(run_broken_tests).
-
setup() ->
{ok, Db} = couch_mrview_test_util:init_db(?tempdb(), map),
couch_mrview:query_view(Db, <<"_design/bar">>, <<"baz">>),
{ok, Info} = couch_mrview:get_info(Db, <<"_design/bar">>),
{Db, Info}.
+
teardown({Db, _}) ->
couch_db:close(Db),
couch_server:delete(couch_db:name(Db), [?ADMIN_CTX]),
@@ -37,39 +36,86 @@ view_info_test_() ->
"Views index tests",
{
setup,
- fun test_util:start_couch/0, fun test_util:stop_couch/1,
+ fun test_util:start_couch/0,
+ fun test_util:stop_couch/1,
{
foreach,
- fun setup/0, fun teardown/1,
+ fun setup/0,
+ fun teardown/1,
[
- fun should_get_property/1
+ fun sig_is_binary/1,
+ fun language_is_js/1,
+ fun file_size_is_non_neg_int/1,
+ fun active_size_is_non_neg_int/1,
+ fun external_size_is_non_neg_int/1,
+ fun disk_size_is_file_size/1,
+ fun data_size_is_external_size/1,
+ fun active_size_less_than_file_size/1,
+ fun update_seq_is_non_neg_int/1,
+ fun purge_seq_is_non_neg_int/1,
+ fun update_opts_is_bin_list/1
]
}
}
}.
-should_get_property({_, Info}) ->
- InfoProps = [
- {signature, <<"276df562b152b3c4e5d34024f62672ed">>},
- {language, <<"javascript">>},
- {disk_size, 314},
- {data_size, 263},
- {update_seq, 11},
- {purge_seq, 0},
- {updater_running, false},
- {compact_running, false},
- {waiting_clients, 0}
- ],
- [
- {atom_to_list(Key), ?_assertEqual(Val, getval(Key, Info))}
- || {Key, Val} <- InfoProps
- ].
+sig_is_binary({_, Info}) ->
+ ?_assert(is_binary(prop(signature, Info))).
+
+
+language_is_js({_, Info}) ->
+ ?_assertEqual(<<"javascript">>, prop(language, Info)).
+
+
+file_size_is_non_neg_int({_, Info}) ->
+ ?_assert(check_non_neg_int([sizes, file], Info)).
+
+
+active_size_is_non_neg_int({_, Info}) ->
+ ?_assert(check_non_neg_int([sizes, active], Info)).
+
+
+external_size_is_non_neg_int({_, Info}) ->
+ ?_assert(check_non_neg_int([sizes, external], Info)).
+
+
+disk_size_is_file_size({_, Info}) ->
+ ?_assertEqual(prop([sizes, file], Info), prop(disk_size, Info)).
+
+
+data_size_is_external_size({_, Info}) ->
+ ?_assertEqual(prop([sizes, external], Info), prop(data_size, Info)).
+
+
+active_size_less_than_file_size({_, Info}) ->
+ ?_assert(prop([sizes, active], Info) < prop([sizes, file], Info)).
+
+
+update_seq_is_non_neg_int({_, Info}) ->
+ ?_assert(check_non_neg_int(update_seq, Info)).
+
+
+purge_seq_is_non_neg_int({_, Info}) ->
+ ?_assert(check_non_neg_int(purge_seq, Info)).
+
+
+update_opts_is_bin_list({_, Info}) ->
+ Opts = prop(update_options, Info),
+ ?_assert(is_list(Opts) andalso
+ (Opts == [] orelse lists:all([is_binary(B) || B <- Opts]))).
-getval(Key, PL) ->
- {value, {Key, Val}} = lists:keysearch(Key, 1, PL),
- Val.
+check_non_neg_int(Key, Info) ->
+ Size = prop(Key, Info),
+ is_integer(Size) andalso Size >= 0.
--endif.
+prop(Key, {Props}) when is_list(Props) ->
+ prop(Key, Props);
+prop([Key], Info) ->
+ prop(Key, Info);
+prop([Key | Rest], Info) ->
+ prop(Rest, prop(Key, Info));
+prop(Key, Info) when is_atom(Key), is_list(Info) ->
+ couch_util:get_value(Key, Info).
diff --git a/src/ddoc_cache/src/ddoc_cache_lru.erl
b/src/ddoc_cache/src/ddoc_cache_lru.erl
index e94934d048..248a76dc8e 100644
--- a/src/ddoc_cache/src/ddoc_cache_lru.erl
+++ b/src/ddoc_cache/src/ddoc_cache_lru.erl
@@ -87,6 +87,7 @@ refresh(DbName, DDocIds) ->
init(_) ->
+ couch_util:set_mqd_off_heap(),
process_flag(trap_exit, true),
BaseOpts = [public, named_table],
CacheOpts = [
diff --git a/src/fabric/src/fabric_doc_atts.erl
b/src/fabric/src/fabric_doc_atts.erl
new file mode 100644
index 0000000000..7ef5dd893d
--- /dev/null
+++ b/src/fabric/src/fabric_doc_atts.erl
@@ -0,0 +1,168 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(fabric_doc_atts).
+
+-include_lib("fabric/include/fabric.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+-export([
+ receiver/2,
+ receiver_callback/2
+]).
+
+
+receiver(_Req, undefined) ->
+ <<"">>;
+receiver(_Req, {unknown_transfer_encoding, Unknown}) ->
+ exit({unknown_transfer_encoding, Unknown});
+receiver(Req, chunked) ->
+ MiddleMan = spawn(fun() -> middleman(Req, chunked) end),
+ {fabric_attachment_receiver, MiddleMan, chunked};
+receiver(_Req, 0) ->
+ <<"">>;
+receiver(Req, Length) when is_integer(Length) ->
+ maybe_send_continue(Req),
+ Middleman = spawn(fun() -> middleman(Req, Length) end),
+ {fabric_attachment_receiver, Middleman, Length};
+receiver(_Req, Length) ->
+ exit({length_not_integer, Length}).
+
+
+receiver_callback(Middleman, chunked) ->
+ fun(4096, ChunkFun, State) ->
+ write_chunks(Middleman, ChunkFun, State)
+ end;
+receiver_callback(Middleman, Length) when is_integer(Length) ->
+ fun() ->
+ Middleman ! {self(), gimme_data},
+ Timeout = fabric_util:attachments_timeout(),
+ receive
+ {Middleman, Data} ->
+ rexi:reply(attachment_chunk_received),
+ Data
+ after Timeout ->
+ exit(timeout)
+ end
+ end.
+
+
+%%
+%% internal
+%%
+
+maybe_send_continue(#httpd{mochi_req = MochiReq} = Req) ->
+ case couch_httpd:header_value(Req, "expect") of
+ undefined ->
+ ok;
+ Expect ->
+ case string:to_lower(Expect) of
+ "100-continue" ->
+ MochiReq:start_raw_response({100, gb_trees:empty()});
+ _ ->
+ ok
+ end
+ end.
+
+write_chunks(MiddleMan, ChunkFun, State) ->
+ MiddleMan ! {self(), gimme_data},
+ Timeout = fabric_util:attachments_timeout(),
+ receive
+ {MiddleMan, ChunkRecordList} ->
+ rexi:reply(attachment_chunk_received),
+ case flush_chunks(ChunkRecordList, ChunkFun, State) of
+ {continue, NewState} ->
+ write_chunks(MiddleMan, ChunkFun, NewState);
+ {done, NewState} ->
+ NewState
+ end
+ after Timeout ->
+ exit(timeout)
+ end.
+
+flush_chunks([], _ChunkFun, State) ->
+ {continue, State};
+flush_chunks([{0, _}], _ChunkFun, State) ->
+ {done, State};
+flush_chunks([Chunk | Rest], ChunkFun, State) ->
+ NewState = ChunkFun(Chunk, State),
+ flush_chunks(Rest, ChunkFun, NewState).
+
+receive_unchunked_attachment(_Req, 0) ->
+ ok;
+receive_unchunked_attachment(Req, Length) ->
+ receive {MiddleMan, go} ->
+ Data = couch_httpd:recv(Req, 0),
+ MiddleMan ! {self(), Data}
+ end,
+ receive_unchunked_attachment(Req, Length - size(Data)).
+
+middleman(Req, chunked) ->
+ % spawn a process to actually receive the uploaded data
+ RcvFun = fun(ChunkRecord, ok) ->
+ receive {From, go} -> From ! {self(), ChunkRecord} end, ok
+ end,
+ Receiver = spawn(fun() -> couch_httpd:recv_chunked(Req,4096,RcvFun,ok)
end),
+
+ % take requests from the DB writers and get data from the receiver
+ N = erlang:list_to_integer(config:get("cluster","n")),
+ Timeout = fabric_util:attachments_timeout(),
+ middleman_loop(Receiver, N, [], [], Timeout);
+
+middleman(Req, Length) ->
+ Receiver = spawn(fun() -> receive_unchunked_attachment(Req, Length) end),
+ N = erlang:list_to_integer(config:get("cluster","n")),
+ Timeout = fabric_util:attachments_timeout(),
+ middleman_loop(Receiver, N, [], [], Timeout).
+
+middleman_loop(Receiver, N, Counters0, ChunkList0, Timeout) ->
+ receive {From, gimme_data} ->
+ % Figure out how far along this writer (From) is in the list
+ ListIndex = case fabric_dict:lookup_element(From, Counters0) of
+ undefined -> 0;
+ I -> I
+ end,
+
+ % Talk to the receiver to get another chunk if necessary
+ ChunkList1 = if ListIndex == length(ChunkList0) ->
+ Receiver ! {self(), go},
+ receive
+ {Receiver, ChunkRecord} ->
+ ChunkList0 ++ [ChunkRecord]
+ end;
+ true -> ChunkList0 end,
+
+ % reply to the writer
+ Reply = lists:nthtail(ListIndex, ChunkList1),
+ From ! {self(), Reply},
+
+ % Update the counter for this writer
+ Counters1 = fabric_dict:update_counter(From, length(Reply), Counters0),
+
+ % Drop any chunks that have been sent to all writers
+ Size = fabric_dict:size(Counters1),
+ NumToDrop = lists:min([I || {_, I} <- Counters1]),
+
+ {ChunkList3, Counters3} =
+ if Size == N andalso NumToDrop > 0 ->
+ ChunkList2 = lists:nthtail(NumToDrop, ChunkList1),
+ Counters2 = [{F, I-NumToDrop} || {F, I} <- Counters1],
+ {ChunkList2, Counters2};
+ true ->
+ {ChunkList1, Counters1}
+ end,
+
+ middleman_loop(Receiver, N, Counters3, ChunkList3, Timeout)
+ after Timeout ->
+ exit(Receiver, kill),
+ ok
+ end.
diff --git a/src/fabric/src/fabric_rpc.erl b/src/fabric/src/fabric_rpc.erl
index 913aafe0e2..60526f4950 100644
--- a/src/fabric/src/fabric_rpc.erl
+++ b/src/fabric/src/fabric_rpc.erl
@@ -440,6 +440,8 @@ make_att_reader({follows, Parser, Ref}) ->
throw({mp_parser_died, Reason})
end
end;
+make_att_reader({fabric_attachment_receiver, Middleman, Length}) ->
+ fabric_doc_atts:receiver_callback(Middleman, Length);
make_att_reader(Else) ->
Else.
diff --git a/src/mem3/src/mem3_shards.erl b/src/mem3/src/mem3_shards.erl
index c84b87397a..da3b69a610 100644
--- a/src/mem3/src/mem3_shards.erl
+++ b/src/mem3/src/mem3_shards.erl
@@ -184,6 +184,7 @@ handle_config_terminate(_Server, _Reason, _State) ->
erlang:send_after(?RELISTEN_DELAY, whereis(?MODULE),
restart_config_listener).
init([]) ->
+ couch_util:set_mqd_off_heap(),
ets:new(?SHARDS, [
bag,
public,
diff --git a/src/rexi/src/rexi_server.erl b/src/rexi/src/rexi_server.erl
index 3d3f272e45..954ca88cb9 100644
--- a/src/rexi/src/rexi_server.erl
+++ b/src/rexi/src/rexi_server.erl
@@ -39,6 +39,7 @@ start_link(ServerId) ->
gen_server:start_link({local, ServerId}, ?MODULE, [], []).
init([]) ->
+ couch_util:set_mqd_off_heap(),
{ok, #st{}}.
handle_call(get_errors, _From, #st{errors = Errors} = St) ->
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
