This is an automated email from the ASF dual-hosted git repository. nickva pushed a commit to branch fix-delete-race-cont in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 67075b0bfa81de7ef607afca081a7f0fd01c6afe Author: Nick Vatamaniuc <[email protected]> AuthorDate: Wed Oct 7 15:00:57 2026 -0400 More db delete 500 race fixes Continuation of https://github.com/apache/couchdb/pull/6145. When we ran internal tests we noticed a few 500 errors still leaked when db is opened with a call to get_security() first. Then, instead of getting `database_does_not_exist` we caught a `throw({error, missing_target})` from `get_or_create_db/2`. Issue: #6142 --- src/chttpd/test/eunit/chttpd_db_test.erl | 42 +++++++++++++++++++++----------- src/fabric/src/fabric_util.erl | 32 +++++++++++++++++++----- 2 files changed, 54 insertions(+), 20 deletions(-) diff --git a/src/chttpd/test/eunit/chttpd_db_test.erl b/src/chttpd/test/eunit/chttpd_db_test.erl index 85bec025c..73b81406f 100644 --- a/src/chttpd/test/eunit/chttpd_db_test.erl +++ b/src/chttpd/test/eunit/chttpd_db_test.erl @@ -58,6 +58,7 @@ all_test_() -> ?TDEF_FE(t_return_ok_true_on_ensure_full_commit, ?TIMEOUT), ?TDEF_FE(t_return_404_for_ensure_full_commit_on_no_db, ?TIMEOUT), ?TDEF_FE(t_return_404_for_delete_db_with_stale_shard_cache, ?TIMEOUT), + ?TDEF_FE(t_return_404_for_get_security_with_stale_shard_cache, ?TIMEOUT), ?TDEF_FE(t_accept_live_as_an_alias_for_continuous, ?TIMEOUT), ?TDEF_FE(t_return_headers_after_starting_continuous, ?TIMEOUT), ?TDEF_FE(t_return_404_for_delete_att_on_notadoc, ?TIMEOUT), @@ -147,24 +148,17 @@ t_return_404_for_ensure_full_commit_on_no_db(Db) -> % then can reach nodes after the dbs doc is gone and so it should answer with a % 404. t_return_404_for_delete_db_with_stale_shard_cache(_Db) -> - Db = ?tempdb(), - create_db(Db), - Shards = lists:sort(mem3:shards(Db)), - delete_db(Db), - test_util:wait(fun() -> - try mem3:shards(Db) of - _ -> wait - catch - error:database_does_not_exist -> ok - end - end), - % simulate the re-insert - true = ets:insert(mem3_shards, Shards), - ?assertEqual(Shards, lists:sort(mem3:shards(Db))), + Db = stale_shard_cache_db(), {Status, Response} = req(delete, url(Db)), ?assertEqual(404, Status), ?assertMatch(#{<<"error">> := <<"not_found">>}, Response). +t_return_404_for_get_security_with_stale_shard_cache(_Db) -> + Db = stale_shard_cache_db(), + {Status, Response} = req(get, url(Db, "_security")), + ?assertEqual(404, Status), + ?assertMatch(#{<<"error">> := <<"not_found">>}, Response). + t_accept_live_as_an_alias_for_continuous(Db) -> GetLastSeq = fun(Chunks) -> @@ -337,6 +331,26 @@ helper_queries_with_keys_limit_skip(Db, Path, ExpectedOffset, ExpectedRows1, Exp ?assertEqual(ExpectedRows2, length(Rows2)). %%%%%%%%%%%%%%%%%%%% Utility Functions %%%%%%%%%%%%%%%%%%%% + +% Create and delete a db, then put its shard map back into the shard cache to +% simulate a read racing the delete. Returns the db name. +stale_shard_cache_db() -> + Db = ?tempdb(), + create_db(Db), + Shards = lists:sort(mem3:shards(Db)), + delete_db(Db), + test_util:wait(fun() -> + try mem3:shards(Db) of + _ -> wait + catch + error:database_does_not_exist -> ok + end + end), + % simulate the re-insert + true = ets:insert(mem3_shards, Shards), + ?assertEqual(Shards, lists:sort(mem3:shards(Db))), + Db. + url(Db) -> Addr = config:get("chttpd", "bind_address", "127.0.0.1"), Port = mochiweb_socket_server:get(chttpd, port), diff --git a/src/fabric/src/fabric_util.erl b/src/fabric/src/fabric_util.erl index e8b998a19..5437f0c14 100644 --- a/src/fabric/src/fabric_util.erl +++ b/src/fabric/src/fabric_util.erl @@ -139,11 +139,21 @@ get_db(DbName, Options) -> MinTimeout = config:get_integer("fabric", "shard_timeout_min_msec", 100), MaxTimeout = request_timeout(), Timeout = get_db_timeout(length(Live), Factor, MinTimeout, MaxTimeout), - get_shard(Live, Options, Timeout, Factor). + case get_shard(Live, Options, Timeout, Factor, []) of + {ok, Db} -> + {ok, Db}; + {error, Failures} -> + % missing_target means the shard doc is missing. If every worker + % returns that the shard map from our cache is stale + case all_missing_target(Failures) of + true -> error(database_does_not_exist, [DbName]); + false -> error({internal_server_error, "No DB shards could be opened."}) + end + end. -get_shard([], _Opts, _Timeout, _Factor) -> - error({internal_server_error, "No DB shards could be opened."}); -get_shard([#shard{node = Node, name = Name} | Rest], Opts, Timeout, Factor) -> +get_shard([], _Opts, _Timeout, _Factor, Failures) -> + {error, Failures}; +get_shard([#shard{node = Node, name = Name} | Rest], Opts, Timeout, Factor, Failures) -> Mon = rexi_monitor:start([rexi_utils:server_pid(Node)]), MFA = {fabric_rpc, open_shard, [Name, [{timeout, Timeout} | Opts]]}, Ref = rexi:cast(Node, self(), MFA, [sync]), @@ -157,15 +167,25 @@ get_shard([#shard{node = Node, name = Name} | Rest], Opts, Timeout, Factor) -> throw(Error); {Ref, Reason} -> couch_log:debug("Failed to open shard ~p because: ~p", [Name, Reason]), - get_shard(Rest, Opts, Timeout, Factor) + get_shard(Rest, Opts, Timeout, Factor, [Reason | Failures]) after Timeout -> couch_log:debug("Failed to open shard ~p after: ~p", [Name, Timeout]), - get_shard(Rest, Opts, Factor * Timeout, Factor) + get_shard(Rest, Opts, Factor * Timeout, Factor, [timeout | Failures]) end after rexi_monitor:stop(Mon) end. +all_missing_target([]) -> + false; +all_missing_target(Failures) -> + lists:all(fun is_missing_target/1, Failures). + +is_missing_target({'rexi_EXIT', {{error, missing_target}, _}}) -> + true; +is_missing_target(_) -> + false. + get_db_timeout(N, Factor, MinTimeout, infinity) -> % MaxTimeout may be infinity so we just use the largest Erlang small int to % avoid blowing up the arithmetic
