This is an automated email from the ASF dual-hosted git repository. nickva pushed a commit to branch unify-ownership in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 789f2e36f9f459248367309cff887be76e6573bb Author: Nick Vatamaniuc <[email protected]> AuthorDate: Thu Jun 11 19:27:04 2026 -0400 Unify membership hashes Previously replicator and fabric_doc_updater used different membership algorithms to consistently pick a first node order from a range of shard copies. Unifying them simplifies the code and also fixes issues like #6029. While at it move away from crc32 and used a new phash2/1 function. We don't have to create a term_to_binary for it first as it can take regular Erlang terms. (Thanks to Robert Newson for the original idea to do this) Fix #6029 --- src/couch_peruser/src/couch_peruser.erl | 5 +-- .../src/couch_replicator_utils.erl | 6 +--- src/fabric/src/fabric_doc_update.erl | 18 +++++++++- src/mem3/src/mem3.erl | 39 ++++++++++++++++++---- src/mem3/src/mem3_util.erl | 4 +-- 5 files changed, 53 insertions(+), 19 deletions(-) diff --git a/src/couch_peruser/src/couch_peruser.erl b/src/couch_peruser/src/couch_peruser.erl index 8a7cbe13a..bb868d9f7 100644 --- a/src/couch_peruser/src/couch_peruser.erl +++ b/src/couch_peruser/src/couch_peruser.erl @@ -269,10 +269,7 @@ should_handle_doc(ShardName, DocId) -> ) -> boolean(). should_handle_doc_int(ShardName, DocId) -> DbName = mem3:dbname(ShardName), - Live = [erlang:node() | nodes()], - Shards = mem3:shards(DbName, DocId), - Nodes = [N || #shard{node = N} <- Shards, lists:member(N, Live)], - case mem3:owner(DbName, DocId, Nodes) of + case mem3:owner(DbName, DocId) of ThisNode when ThisNode =:= node() -> couch_log:debug("peruser: handling ~s/~s", [DbName, DocId]), % do the database action diff --git a/src/couch_replicator/src/couch_replicator_utils.erl b/src/couch_replicator/src/couch_replicator_utils.erl index d784b8a84..1ce5b1de0 100644 --- a/src/couch_replicator/src/couch_replicator_utils.erl +++ b/src/couch_replicator/src/couch_replicator_utils.erl @@ -52,11 +52,7 @@ -spec owner(Dbname :: binary(), DocId :: binary()) -> node(). owner(<<"shards/", _/binary>> = ShardName, DocId) -> - DbName = mem3:dbname(ShardName), - Live = [node() | nodes()], - Shards = mem3:shards(DbName, DocId), - Nodes = [N || #shard{node = N} <- Shards, lists:member(N, Live)], - mem3:owner(DbName, DocId, Nodes); + mem3:owner(mem3:dbname(ShardName), DocId); owner(_ShardName, _DocId) -> node(). diff --git a/src/fabric/src/fabric_doc_update.erl b/src/fabric/src/fabric_doc_update.erl index 96f7c1b11..362d796e2 100644 --- a/src/fabric/src/fabric_doc_update.erl +++ b/src/fabric/src/fabric_doc_update.erl @@ -352,7 +352,7 @@ group_docs_by_shard(DbName, Docs) -> [] -> []; [#shard{range = Range} | _] = Shards -> - mem3_util:rotate_list({Range, DbName}, Shards) + owner_order(DbName, Range, Shards) end end ) @@ -362,6 +362,10 @@ group_docs_by_shard(DbName, Docs) -> ) ). +owner_order(DbName, Range, Shards) -> + Owners = mem3:owners(DbName, Range, [N || #shard{node = N} <- Shards]), + [S || N <- Owners, #shard{node = N1} = S <- Shards, N1 =:= N]. + append_update_replies([], [], DocReplyDict) -> DocReplyDict; append_update_replies([Doc | Rest], [], Dict0) -> @@ -1144,6 +1148,18 @@ sws_false_mode_ok_can_outvote_conflict() -> lists:sort(Reply) ). +owner_order_test() -> + S1 = #shard{name = <<"r1">>, node = n1, range = [0, 10]}, + S2 = #shard{name = <<"r1">>, node = n2, range = [0, 10]}, + S3 = #shard{name = <<"r1">>, node = n3, range = [0, 10]}, + % The rotation amounts, crc32(term_to_binary({Db, Range})) rem 3, are 0 + % for {<<"dba">>, [0, 10]} and 2 for {<<"dbb">>, [0, 10]} + ?assertEqual([S1, S2, S3], owner_order(<<"dba">>, [0, 10], [S2, S3, S1])), + ?assertEqual([S3, S1, S2], owner_order(<<"dbb">>, [0, 10], [S2, S3, S1])), + % the first copy is on the owner node + ?assertEqual(n1, hd(mem3:owners(<<"dba">>, [0, 10], [n1, n2, n3]))), + ?assertEqual(n3, hd(mem3:owners(<<"dbb">>, [0, 10], [n1, n2, n3]))). + % needed for testing to avoid having to start the mem3 application group_docs_by_shard_hack(_DbName, Shards, Docs) -> dict:to_list( diff --git a/src/mem3/src/mem3.erl b/src/mem3/src/mem3.erl index 6a96ae2a9..b16240665 100644 --- a/src/mem3/src/mem3.erl +++ b/src/mem3/src/mem3.erl @@ -31,7 +31,7 @@ -export([compare_nodelists/0, compare_shards/1]). -export([quorum/1, group_by_proximity/1]). -export([live_shards/2]). --export([belongs/2, owner/3]). +-export([belongs/2, owner/2, owners/3]). -export([get_placement/1]). -export([ping/1, ping/2]). -export([ping_nodes/0, ping_nodes/1, ping_nodes/2]). @@ -414,11 +414,26 @@ name(#shard{name = Name}) -> name(#ordered_shard{name = Name}) -> Name. -% Direct calculation of node membership. This is the algorithm part. It -% doesn't read the shard map, just picks owner based on a hash. --spec owner(binary(), binary(), [node()]) -> node(). -owner(DbName, DocId, Nodes) -> - hd(mem3_util:rotate_list({DbName, DocId}, lists:usort(Nodes))). +% Owner node order rotated by {DbName, Range}. +-spec owners(binary(), binary()) -> [node()]. +owners(DbName, DocId) -> + [#shard{range = Range} | _] = Shards = shards(DbName, DocId), + owners(DbName, Range, [N || #shard{node = N} <- Shards]). + +% Ownership calculation. This is the algorithm part. It doesn't read the shard +% map, just rotates the given nodes based on a hash of {DbName, Range}. +-spec owners(binary(), [non_neg_integer()], [node()]) -> [node()]. +owners(DbName, Range, Nodes) -> + mem3_util:rotate_list({DbName, Range}, lists:usort(Nodes)). + +% Pick the owner node as first live node in the owners/2 order. When a node +% goes down, ownership of its docs funnels to the next live node in the list. +-spec owner(binary(), binary()) -> node(). +owner(DbName, DocId) -> + live_owner(owners(DbName, DocId), [node() | erlang:nodes()]). + +live_owner(Owners, Live) -> + hd([N || N <- Owners, lists:member(N, Live)]). %% Check whether a node is up or down %% side effect: set up a connection to Node if there not yet is one. @@ -610,6 +625,18 @@ allowed_nodes_test_() -> } ]}. +live_owner_test() -> + Owners = [n2, n3, n1], + % all nodes live: the head of the try-order owns the doc + ?assertEqual(n2, live_owner(Owners, [n1, n2, n3])), + % owner down: its docs funnel to the next node in the try-order + ?assertEqual(n3, live_owner(Owners, [n1, n3])), + ?assertEqual(n1, live_owner(Owners, [n1])), + % other nodes down: ownership of n2's docs is unaffected + ?assertEqual(n2, live_owner(Owners, [n2])), + % no live copy nodes + ?assertError(badarg, live_owner(Owners, [])). + rotate_rand_degenerate_test() -> ?assertEqual([1], rotate_rand([1])). diff --git a/src/mem3/src/mem3_util.erl b/src/mem3/src/mem3_util.erl index cc728e0ee..7284f6e56 100644 --- a/src/mem3/src/mem3_util.erl +++ b/src/mem3/src/mem3_util.erl @@ -324,10 +324,8 @@ is_deleted(Change) -> rotate_list(_Key, []) -> []; -rotate_list(Key, List) when not is_binary(Key) -> - rotate_list(?term_to_bin(Key), List); rotate_list(Key, List) -> - {H, T} = lists:split(erlang:crc32(Key) rem length(List), List), + {H, T} = lists:split(erlang:phash2(Key) rem length(List), List), T ++ H. downcast(#shard{} = S) ->
