This is an automated email from the ASF dual-hosted git repository. nickva pushed a commit to branch fix-db-deletion-race in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit f2af92258af650229b35b2f724e5a84496c7d980 Author: Nick Vatamaniuc <[email protected]> AuthorDate: Thu Oct 1 13:47:34 2026 -0400 Fix db delete 500 badarg When nodes return `not_found` for db doc deletion (the doc where the shardmap lives) `fabric_db_delete` tried to log a warnign with `~d` but passed a non-integer value which blew up with a 500 badarg instead of a nice 404. The value we tried to log was also wonky, the shard record from delete_shard_db_doc never actually set `dbname` so the warnign would have printed `undefined` anyway. Fix it to print dbname instead. In the issue #6142 this was reproduced with a racing DELETE. Presumably it's when a read raced the deletion and shoved the shard map back into mem3 cache. It later gets evicted but a second DELETE in the meantimes got a 500. Fix: #6142 --- src/chttpd/test/eunit/chttpd_db_test.erl | 24 ++++++++++++++++++++++++ src/fabric/src/fabric_db_delete.erl | 10 +++++----- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/src/chttpd/test/eunit/chttpd_db_test.erl b/src/chttpd/test/eunit/chttpd_db_test.erl index 6560cc949..85bec025c 100644 --- a/src/chttpd/test/eunit/chttpd_db_test.erl +++ b/src/chttpd/test/eunit/chttpd_db_test.erl @@ -57,6 +57,7 @@ all_test_() -> ?TDEF_FE(t_return_400_new_edits_false_no_revs_on_doc_update, ?TIMEOUT), ?TDEF_FE(t_return_ok_true_on_ensure_full_commit, ?TIMEOUT), ?TDEF_FE(t_return_404_for_ensure_full_commit_on_no_db, ?TIMEOUT), + ?TDEF_FE(t_return_404_for_delete_db_with_stale_shard_cache, ?TIMEOUT), ?TDEF_FE(t_accept_live_as_an_alias_for_continuous, ?TIMEOUT), ?TDEF_FE(t_return_headers_after_starting_continuous, ?TIMEOUT), ?TDEF_FE(t_return_404_for_delete_att_on_notadoc, ?TIMEOUT), @@ -141,6 +142,29 @@ t_return_404_for_ensure_full_commit_on_no_db(Db) -> ?assertEqual(404, Status), ?assertMatch(#{<<"error">> := <<"not_found">>}, Response). +% If we delete and then read the db, the deletion can put shard map back into +% the cache until dbs changes feed kicks it out again. A second delete request +% then can reach nodes after the dbs doc is gone and so it should answer with a +% 404. +t_return_404_for_delete_db_with_stale_shard_cache(_Db) -> + Db = ?tempdb(), + create_db(Db), + Shards = lists:sort(mem3:shards(Db)), + delete_db(Db), + test_util:wait(fun() -> + try mem3:shards(Db) of + _ -> wait + catch + error:database_does_not_exist -> ok + end + end), + % simulate the re-insert + true = ets:insert(mem3_shards, Shards), + ?assertEqual(Shards, lists:sort(mem3:shards(Db))), + {Status, Response} = req(delete, url(Db)), + ?assertEqual(404, Status), + ?assertMatch(#{<<"error">> := <<"not_found">>}, Response). + t_accept_live_as_an_alias_for_continuous(Db) -> GetLastSeq = fun(Chunks) -> diff --git a/src/fabric/src/fabric_db_delete.erl b/src/fabric/src/fabric_db_delete.erl index b57342622..1a80be315 100644 --- a/src/fabric/src/fabric_db_delete.erl +++ b/src/fabric/src/fabric_db_delete.erl @@ -36,10 +36,10 @@ go(DbName, _Options) -> fabric_util:submit_jobs(Shards, delete_db, []) end. -delete_shard_db_doc(Doc) -> - Shards = [#shard{node = N} || N <- mem3:nodes()], +delete_shard_db_doc(DbName) -> + Shards = [#shard{node = N, dbname = DbName} || N <- mem3:nodes()], RexiMon = fabric_util:create_monitors(Shards), - Workers = fabric_util:submit_jobs(Shards, delete_shard_db_doc, [Doc]), + Workers = fabric_util:submit_jobs(Shards, delete_shard_db_doc, [DbName]), Acc0 = {length(Shards), fabric_dict:init(Workers, nil)}, try fabric_util:recv(Workers, #shard.ref, fun handle_db_update/3, Acc0) of {timeout, {_, WorkersDict}} -> @@ -74,8 +74,8 @@ maybe_stop(W, Counters) -> {Ok, NotFound} = fabric_dict:fold(fun count_replies/3, {0, 0}, Counters), case {Ok + NotFound, Ok, NotFound} of {W, 0, W} -> - {#shard{dbname = Name}, _} = hd(Counters), - couch_log:warning("~p not_found ~d", [?MODULE, Name]), + {#shard{dbname = DbName}, _} = hd(Counters), + couch_log:warning("~p not_found ~s", [?MODULE, DbName]), {stop, not_found}; {W, _, _} -> {stop, ok};
