This is an automated email from the ASF dual-hosted git repository. vatamane pushed a commit to branch update-db-props in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 1b052913b59c9a521f91082b2b233ba15d68d899 Author: Nick Vatamaniuc <[email protected]> AuthorDate: Thu Sep 11 01:14:03 2025 -0400 Implement prop updates for shards When we implemented partitioned dbs we added a generic `props` features to the db shards and the shard docs. The `props` is a generic prop (KV) list which can store database metadata properties. Currently it only stores the `partitioned` and the `hash` db properties. Recently we discussed possibly storing a new TTL flag or moving some other metadata bits like security or revs limits and such to props and we'd want to them both for the clustered and local shards (for local _dbs, _nodes etc). In order to use props like that we'd want to allow dynamically updating props after the initial db creations so that's what this PR does. We still want to ensure ``partitioned`` and hash ``properties`` are "static" and we don't allow modifying them later so there an way in couch_db.erl to flag a set of properties as "static". A part of the dynamic API to set props on shards was already implemented in the form of `couch_db_engine:set_props/2` so in the PR we just build the rest of the bits in couch_db and fabric. This is also a first part which update properties for shards files, we'll follow up with another commit to allow update the props in the shard map document as well. --- src/couch/src/couch_db.erl | 40 ++++++++++++++++++++------ src/couch/src/couch_db_updater.erl | 20 +++++++++---- src/fabric/src/fabric.erl | 12 ++++++++ src/fabric/src/fabric_db_meta.erl | 25 +++++++++++++++- src/fabric/src/fabric_rpc.erl | 4 +++ src/fabric/test/eunit/fabric_db_info_tests.erl | 38 +++++++++++++++++++++++- 6 files changed, 123 insertions(+), 16 deletions(-) diff --git a/src/couch/src/couch_db.erl b/src/couch/src/couch_db.erl index 42d82204f..b8b8ee708 100644 --- a/src/couch/src/couch_db.erl +++ b/src/couch/src/couch_db.erl @@ -72,6 +72,9 @@ set_security/2, set_user_ctx/2, + get_props/1, + update_props/3, + load_validation_funs/1, reload_validation_funs/1, @@ -157,6 +160,11 @@ % Purge client max lag window in seconds (defaulting to 24 hours) -define(PURGE_LAG_SEC, 86400). +% DB props which cannot be dynamically updated after db creation +-define(PROP_PARTITIONED, partitioned). +-define(PROP_HASH, hash). +-define(STATIC_PROPS, [?PROP_PARTITIONED, ?PROP_HASH]). + start_link(Engine, DbName, Filepath, Options) -> Arg = {Engine, DbName, Filepath, Options}, proc_lib:start_link(couch_db_updater, init, [Arg]). @@ -232,9 +240,9 @@ is_clustered(#db{}) -> is_clustered(?OLD_DB_REC = Db) -> ?OLD_DB_MAIN_PID(Db) == undefined. -is_partitioned(#db{options = Options}) -> - Props = couch_util:get_value(props, Options, []), - couch_util:get_value(partitioned, Props, false). +is_partitioned(#db{} = Db) -> + Props = get_props(Db), + couch_util:get_value(?PROP_PARTITIONED, Props, false). close(#db{} = Db) -> ok = couch_db_engine:decref(Db); @@ -650,11 +658,7 @@ get_db_info(Db) -> undefined -> null; Else1 -> Else1 end, - Props = - case couch_db_engine:get_props(Db) of - undefined -> null; - Else2 -> {Else2} - end, + Props = get_props(Db), InfoList = [ {db_name, Name}, {engine, couch_db_engine:get_engine(Db)}, @@ -668,7 +672,7 @@ get_db_info(Db) -> {disk_format_version, DiskVersion}, {committed_update_seq, CommittedUpdateSeq}, {compacted_seq, CompactedSeq}, - {props, Props}, + {props, {Props}}, {uuid, Uuid} ], {ok, InfoList}. @@ -861,6 +865,24 @@ set_revs_limit(#db{main_pid = Pid} = Db, Limit) when Limit > 0 -> set_revs_limit(_Db, _Limit) -> throw(invalid_revs_limit). +get_props(#db{options = Options}) -> + couch_util:get_value(props, Options, []). + +update_props(#db{main_pid = Pid} = Db, K, V) -> + check_is_admin(Db), + case lists:member(K, ?STATIC_PROPS) of + true -> + throw({bad_request, <<"cannot update static property">>}); + false -> + Props = get_props(Db), + Props1 = + case V of + undefined -> lists:keydelete(K, 1, Props); + _ -> lists:keystore(K, 1, Props, {K, V}) + end, + gen_server:call(Pid, {set_props, Props1}, infinity) + end. + name(#db{name = Name}) -> Name; name(?OLD_DB_REC = Db) -> diff --git a/src/couch/src/couch_db_updater.erl b/src/couch/src/couch_db_updater.erl index 909f1aeb5..1d0e177d5 100644 --- a/src/couch/src/couch_db_updater.erl +++ b/src/couch/src/couch_db_updater.erl @@ -97,6 +97,12 @@ handle_call({set_time_seq, TSeq}, _From, Db) -> {ok, Db2} = couch_db_engine:commit_data(Db1#db{time_seq = TSeq}), ok = couch_server:db_updated(Db2), {reply, ok, Db2}; +handle_call({set_props, Props}, _From, Db) -> + {ok, Db1} = couch_db_engine:set_props(Db, Props), + Db2 = options_set_props(Db1, Props), + {ok, Db3} = couch_db_engine:commit_data(Db2), + ok = couch_server:db_updated(Db3), + {reply, ok, Db3}; handle_call({purge_docs, [], _}, _From, Db) -> {reply, {ok, []}, Db}; handle_call({purge_docs, PurgeReqs0, Options}, _From, Db) -> @@ -313,14 +319,18 @@ init_db(DbName, FilePath, EngineState, Options) -> after_doc_read = ADR }, - DbProps = couch_db_engine:get_props(InitDb), - - InitDb#db{ + Db = InitDb#db{ committed_update_seq = couch_db_engine:get_update_seq(InitDb), security = couch_db_engine:get_security(InitDb), time_seq = couch_db_engine:get_time_seq(InitDb), - options = lists:keystore(props, 1, NonCreateOpts, {props, DbProps}) - }. + options = NonCreateOpts + }, + DbProps = couch_db_engine:get_props(Db), + options_set_props(Db, DbProps). + +options_set_props(#db{options = Options} = Db, Props) -> + Options1 = lists:keystore(props, 1, Options, {props, Props}), + Db#db{options = Options1}. refresh_validate_doc_funs(#db{name = <<"shards/", _/binary>> = Name} = Db) -> spawn(fabric, reset_validation_funs, [mem3:dbname(Name)]), diff --git a/src/fabric/src/fabric.erl b/src/fabric/src/fabric.erl index 0a4b4de25..a2bf82482 100644 --- a/src/fabric/src/fabric.erl +++ b/src/fabric/src/fabric.erl @@ -25,6 +25,8 @@ get_db_info/1, get_doc_count/1, get_doc_count/2, set_revs_limit/3, + update_props/3, + update_props/4, set_security/2, set_security/3, get_revs_limit/1, get_security/1, get_security/2, @@ -186,6 +188,16 @@ get_revs_limit(DbName) -> catch couch_db:close(Db) end. +%% @doc update shard property. Some properties like `partitioned` or `hash` are +%% static and cannot be updated. They will return an error. +-spec update_props(dbname(), atom() | binary(), any()) -> ok. +update_props(DbName, K, V) -> + update_props(DbName, K, V, [?ADMIN_CTX]). + +-spec update_props(dbname(), atom() | binary(), any(), [option()]) -> ok. +update_props(DbName, K, V, Options) when is_atom(K) orelse is_binary(K) -> + fabric_db_meta:update_props(dbname(DbName), K, V, opts(Options)). + %% @doc sets the readers/writers/admin permissions for a database -spec set_security(dbname(), SecObj :: json_obj()) -> ok. set_security(DbName, SecObj) -> diff --git a/src/fabric/src/fabric_db_meta.erl b/src/fabric/src/fabric_db_meta.erl index 1013b958d..af4a069d4 100644 --- a/src/fabric/src/fabric_db_meta.erl +++ b/src/fabric/src/fabric_db_meta.erl @@ -16,7 +16,8 @@ set_revs_limit/3, set_security/3, get_all_security/2, - set_purge_infos_limit/3 + set_purge_infos_limit/3, + update_props/4 ]). -include_lib("fabric/include/fabric.hrl"). @@ -198,3 +199,25 @@ maybe_finish_get(#acc{workers = []} = Acc) -> {stop, Acc}; maybe_finish_get(Acc) -> {ok, Acc}. + +update_props(DbName, K, V, Options) -> + Shards = mem3:shards(DbName), + Workers = fabric_util:submit_jobs(Shards, update_props, [K, V, Options]), + Handler = fun handle_update_props_message/3, + Acc0 = {Workers, length(Workers) - 1}, + case fabric_util:recv(Workers, #shard.ref, Handler, Acc0) of + {ok, ok} -> + ok; + {timeout, {DefunctWorkers, _}} -> + fabric_util:log_timeout(DefunctWorkers, "update_props"), + {error, timeout}; + Error -> + Error + end. + +handle_update_props_message(ok, _, {_Workers, 0}) -> + {stop, ok}; +handle_update_props_message(ok, Worker, {Workers, Waiting}) -> + {ok, {lists:delete(Worker, Workers), Waiting - 1}}; +handle_update_props_message(Error, _, _Acc) -> + {error, Error}. diff --git a/src/fabric/src/fabric_rpc.erl b/src/fabric/src/fabric_rpc.erl index 18215ba34..492546b90 100644 --- a/src/fabric/src/fabric_rpc.erl +++ b/src/fabric/src/fabric_rpc.erl @@ -33,6 +33,7 @@ reset_validation_funs/1, set_security/3, set_revs_limit/3, + update_props/4, create_shard_db_doc/2, delete_shard_db_doc/2, get_partition_info/2 @@ -269,6 +270,9 @@ set_revs_limit(DbName, Limit, Options) -> set_purge_infos_limit(DbName, Limit, Options) -> with_db(DbName, Options, {couch_db, set_purge_infos_limit, [Limit]}). +update_props(DbName, K, V, Options) -> + with_db(DbName, Options, {couch_db, update_props, [K, V]}). + open_doc(DbName, DocId, Options) -> with_db(DbName, Options, {couch_db, open_doc, [DocId, Options]}). diff --git a/src/fabric/test/eunit/fabric_db_info_tests.erl b/src/fabric/test/eunit/fabric_db_info_tests.erl index e7df560a1..9a133ace5 100644 --- a/src/fabric/test/eunit/fabric_db_info_tests.erl +++ b/src/fabric/test/eunit/fabric_db_info_tests.erl @@ -20,7 +20,8 @@ main_test_() -> fun setup/0, fun teardown/1, with([ - ?TDEF(t_update_seq_has_uuids) + ?TDEF(t_update_seq_has_uuids), + ?TDEF(t_update_and_get_props) ]) }. @@ -55,3 +56,38 @@ t_update_seq_has_uuids(_) -> ?assertEqual(UuidFromShard, SeqUuid), ok = fabric:delete_db(DbName, []). + +t_update_and_get_props(_) -> + DbName = ?tempdb(), + ok = fabric:create_db(DbName, [{q, 1}, {n, 1}]), + + {ok, Info} = fabric:get_db_info(DbName), + Props = couch_util:get_value(props, Info), + ?assertEqual({[]}, Props), + + ?assertEqual(ok, fabric:update_props(DbName, <<"foo">>, 100)), + {ok, Info1} = fabric:get_db_info(DbName), + Props1 = couch_util:get_value(props, Info1), + ?assertEqual({[{<<"foo">>, 100}]}, Props1), + + ?assertEqual(ok, fabric:update_props(DbName, bar, 101)), + {ok, Info2} = fabric:get_db_info(DbName), + Props2 = couch_util:get_value(props, Info2), + ?assertEqual( + {[ + {<<"foo">>, 100}, + {bar, 101} + ]}, + Props2 + ), + + ?assertEqual(ok, fabric:update_props(DbName, <<"foo">>, undefined)), + {ok, Info3} = fabric:get_db_info(DbName), + ?assertEqual({[{bar, 101}]}, couch_util:get_value(props, Info3)), + + Res = fabric:update_props(DbName, partitioned, true), + ?assertMatch({error, {bad_request, _}}, Res), + {ok, Info4} = fabric:get_db_info(DbName), + ?assertEqual({[{bar, 101}]}, couch_util:get_value(props, Info4)), + + ok = fabric:delete_db(DbName, []).
