This is an automated email from the ASF dual-hosted git repository. rnewson pushed a commit to branch user-partitioned-dbs-4 in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 081757b33ff999dace3b6d272270ffd1f83487a0 Author: Garren Smith <[email protected]> AuthorDate: Thu Jun 21 16:30:35 2018 +0200 Add a generic 'props' list at db creation time This props list is recorded in each database shard as well as the shard document in the special _dbs database. Co-authored-by: Garren Smith <[email protected]> Co-authored-by: Robert Newson <[email protected]> --- src/couch/src/couch_bt_engine.erl | 73 +++++++++++++++++++++++++++-- src/couch/src/couch_bt_engine_compactor.erl | 8 +++- src/couch/src/couch_bt_engine_header.erl | 3 +- src/couch/src/couch_db.erl | 11 +++++ src/couch/src/couch_db_engine.erl | 47 +++++++++++++++++++ src/couch/src/couch_db_int.hrl | 3 +- src/couch/src/couch_db_updater.erl | 11 ++++- src/couch/src/test_engine_get_set_props.erl | 22 +++++++++ src/fabric/src/fabric_db_create.erl | 6 ++- src/mem3/src/mem3_util.erl | 12 ++++- 10 files changed, 185 insertions(+), 11 deletions(-) diff --git a/src/couch/src/couch_bt_engine.erl b/src/couch/src/couch_bt_engine.erl index ee0d6d8..6748e98 100644 --- a/src/couch/src/couch_bt_engine.erl +++ b/src/couch/src/couch_bt_engine.erl @@ -39,12 +39,16 @@ get_purge_seq/1, get_revs_limit/1, get_security/1, + get_props/1, + get_prop/2, + get_prop/3, get_size_info/1, get_update_seq/1, get_uuid/1, set_revs_limit/2, set_security/2, + set_prop/3, open_docs/2, open_local_docs/2, @@ -93,7 +97,8 @@ -export([ set_update_seq/2, update_header/2, - copy_security/2 + copy_security/2, + copy_props/2 ]). @@ -132,8 +137,9 @@ init(FilePath, Options) -> true -> delete_compaction_files(FilePath), Header0 = couch_bt_engine_header:new(), - ok = couch_file:write_header(Fd, Header0), - Header0; + Header1 = set_initial_props(Fd, Header0, Options), + ok = couch_file:write_header(Fd, Header1), + Header1; false -> case couch_file:read_header(Fd) of {ok, Header0} -> @@ -266,6 +272,31 @@ get_security(#st{header = Header} = St) -> end. +get_props(#st{header = Header} = St) -> + case couch_bt_engine_header:get(Header, props_ptr) of + undefined -> + []; + Pointer -> + {ok, Props} = couch_file:pread_term(St#st.fd, Pointer), + Props + end. + + +get_prop(St, Key) -> + Props = get_props(St), + case lists:keyfind(Key, 1, Props) of + false -> {error, no_value}; + {Key, Value} -> {ok, Value} + end. + + +get_prop(St, Key, DefaultValue) -> + case get_prop(St, Key) of + {error, no_value} -> DefaultValue; + Value -> Value + end. + + get_update_seq(#st{header = Header}) -> couch_bt_engine_header:get(Header, update_seq). @@ -296,6 +327,20 @@ set_security(#st{header = Header} = St, NewSecurity) -> {ok, increment_update_seq(NewSt)}. +set_prop(#st{header = Header} = St, Key, Value) -> + OldProps = get_props(St), + NewProps = lists:ukeymerge(1, [{Key, Value}], OldProps), + Options = [{compression, St#st.compression}], + {ok, Ptr, _} = couch_file:append_term(St#st.fd, NewProps, Options), + NewSt = St#st{ + header = couch_bt_engine_header:set(Header, [ + {props_ptr, Ptr} + ]), + needs_commit = true + }, + {ok, increment_update_seq(NewSt)}. + + open_docs(#st{} = St, DocIds) -> Results = couch_btree:lookup(St#st.id_tree, DocIds), lists:map(fun @@ -646,6 +691,16 @@ copy_security(#st{header = Header} = St, SecProps) -> needs_commit = true }}. +copy_props(#st{header = Header} = St, Props) -> + Options = [{compression, St#st.compression}], + {ok, Ptr, _} = couch_file:append_term(St#st.fd, Props, Options), + {ok, St#st{ + header = couch_bt_engine_header:set(Header, [ + {props_ptr, Ptr} + ]), + needs_commit = true + }}. + open_db_file(FilePath, Options) -> case couch_file:open(FilePath, Options) of @@ -763,6 +818,18 @@ set_default_security_object(Fd, Header, Compression, Options) -> end. +set_initial_props(Fd, Header, Options) -> + case couch_util:get_value(initial_props, Options) of + undefined -> + Header; + InitialProps -> + Compression = couch_compress:get_compression_method(), + AppendOpts = [{compression, Compression}], + {ok, Ptr, _} = couch_file:append_term(Fd, InitialProps, AppendOpts), + couch_bt_engine_header:set(Header, props_ptr, Ptr) + end. + + delete_compaction_files(FilePath) -> RootDir = config:get("couchdb", "database_dir", "."), DelOpts = [{context, compaction}], diff --git a/src/couch/src/couch_bt_engine_compactor.erl b/src/couch/src/couch_bt_engine_compactor.erl index 4e52064..83c9bbc 100644 --- a/src/couch/src/couch_bt_engine_compactor.erl +++ b/src/couch/src/couch_bt_engine_compactor.erl @@ -186,9 +186,13 @@ copy_compact(DbName, St, NewSt0, Retry) -> SecProps = couch_bt_engine:get_security(St), {ok, NewSt4} = couch_bt_engine:copy_security(NewSt3, SecProps), + % Copy general properties over + Props = couch_bt_engine:get_props(St), + {ok, NewSt5} = couch_bt_engine:copy_props(NewSt4, Props), + FinalUpdateSeq = couch_bt_engine:get_update_seq(St), - {ok, NewSt5} = couch_bt_engine:set_update_seq(NewSt4, FinalUpdateSeq), - commit_compaction_data(NewSt5). + {ok, NewSt6} = couch_bt_engine:set_update_seq(NewSt5, FinalUpdateSeq), + commit_compaction_data(NewSt6). copy_docs(St, #st{} = NewSt, MixedInfos, Retry) -> diff --git a/src/couch/src/couch_bt_engine_header.erl b/src/couch/src/couch_bt_engine_header.erl index 3d24f31..779bf26 100644 --- a/src/couch/src/couch_bt_engine_header.erl +++ b/src/couch/src/couch_bt_engine_header.erl @@ -66,7 +66,8 @@ revs_limit = 1000, uuid, epochs, - compacted_seq + compacted_seq, + props_ptr }). diff --git a/src/couch/src/couch_db.erl b/src/couch/src/couch_db.erl index 65ca54a..9bb68f8 100644 --- a/src/couch/src/couch_db.erl +++ b/src/couch/src/couch_db.erl @@ -47,6 +47,7 @@ get_pid/1, get_revs_limit/1, get_security/1, + get_props/1, get_update_seq/1, get_user_ctx/1, get_uuid/1, @@ -58,6 +59,7 @@ set_revs_limit/2, set_security/2, + set_prop/3, set_user_ctx/2, ensure_full_commit/1, @@ -578,6 +580,15 @@ set_security(#db{main_pid=Pid}=Db, {NewSecProps}) when is_list(NewSecProps) -> set_security(_, _) -> throw(bad_request). +get_props(#db{props=Props}) -> + Props. + +set_prop(#db{main_pid=Pid}=Db, Key, Value) -> + check_is_admin(Db), + ok = gen_server:call(Pid, {set_prop, Key, Value}, infinity), + {ok, _} = ensure_full_commit(Db), + ok. + set_user_ctx(#db{} = Db, UserCtx) -> {ok, Db#db{user_ctx = UserCtx}}. diff --git a/src/couch/src/couch_db_engine.erl b/src/couch/src/couch_db_engine.erl index 502faa7..9b28812 100644 --- a/src/couch/src/couch_db_engine.erl +++ b/src/couch/src/couch_db_engine.erl @@ -224,6 +224,19 @@ % the last value that was passed to set_security/2. -callback get_security(DbHandle::db_handle()) -> SecProps::any(). +% Get the current properties. +-callback get_props(DbHandle::db_handle()) -> Props::any(). + +% Get the current properties. This should just return +% the last value that was passed to set_prop/2. +-callback get_prop(DbHandle::db_handle(), Prop::atom()) -> + {ok, SecProps::json()} | {error, no_value}. + +% Get the current properties. If the value isn't set it will return the set default value. +% This should just return the last value that was passed to set_prop/2. +-callback get_prop(DbHandle::db_handle(), Prop::atom(), DefaultValue::any()) -> + {ok, SecProps::json()}. + % This information is displayed in the database info poperties. It % should just be a list of {Name::atom(), Size::non_neg_integer()} @@ -265,6 +278,15 @@ {ok, NewDbHandle::db_handle()}. +% This function is only called by couch_db_updater and +% as such is guaranteed to be single threaded calls. The +% database should simply store prop key and value somewhere so +% they can be returned by the corresponding get_prop calls. + +-callback set_prop(DbHandle::db_handle(), PropKey::atom(), PropValue::any()) -> + {ok, NewDbHandle::db_handle()}. + + % This function will be called by many processes concurrently. % It should return a #full_doc_info{} record or not_found for % every provided DocId in the order those DocId's appear in @@ -601,12 +623,16 @@ get_purge_seq/1, get_revs_limit/1, get_security/1, + get_props/1, + get_prop/2, + get_prop/3, get_size_info/1, get_update_seq/1, get_uuid/1, set_revs_limit/2, set_security/2, + set_prop/3, open_docs/2, open_local_docs/2, @@ -757,6 +783,21 @@ get_security(#db{} = Db) -> Engine:get_security(EngineState). +get_props(#db{} = Db) -> + #db{engine = {Engine, EngineState}} = Db, + Engine:get_props(EngineState). + + +get_prop(#db{} = Db, Prop) -> + #db{engine = {Engine, EngineState}} = Db, + Engine:get_prop(EngineState, Prop). + + +get_prop(#db{} = Db, Prop, DefaultValue) -> + #db{engine = {Engine, EngineState}} = Db, + Engine:get_prop(EngineState, Prop, DefaultValue). + + get_size_info(#db{} = Db) -> #db{engine = {Engine, EngineState}} = Db, Engine:get_size_info(EngineState). @@ -783,6 +824,12 @@ set_security(#db{} = Db, SecProps) -> {ok, Db#db{engine = {Engine, NewSt}}}. +set_prop(#db{} = Db, Key, Value) -> + #db{engine = {Engine, EngineState}} = Db, + {ok, NewSt} = Engine:set_prop(EngineState, Key, Value), + {ok, Db#db{engine = {Engine, NewSt}}}. + + open_docs(#db{} = Db, DocIds) -> #db{engine = {Engine, EngineState}} = Db, Engine:open_docs(EngineState, DocIds). diff --git a/src/couch/src/couch_db_int.hrl b/src/couch/src/couch_db_int.hrl index a412b33..f2a8e17 100644 --- a/src/couch/src/couch_db_int.hrl +++ b/src/couch/src/couch_db_int.hrl @@ -35,7 +35,8 @@ waiting_delayed_commit = nil, options = [], - compression + compression, + props = [] }). diff --git a/src/couch/src/couch_db_updater.erl b/src/couch/src/couch_db_updater.erl index acb9ec1..d8a769b 100644 --- a/src/couch/src/couch_db_updater.erl +++ b/src/couch/src/couch_db_updater.erl @@ -88,6 +88,14 @@ handle_call({set_security, NewSec}, _From, #db{} = Db) -> ok = gen_server:call(couch_server, {db_updated, NewSecDb}, infinity), {reply, ok, NewSecDb, idle_limit()}; +handle_call({set_prop, Key, Value}, _From, #db{} = Db) -> + {ok, NewDb} = couch_db_engine:set_prop(Db, Key, Value), + NewPropDb = commit_data(NewDb#db{ + props = couch_db_engine:get_props(NewDb) + }), + ok = gen_server:call(couch_server, {db_updated, NewPropDb}, infinity), + {reply, ok, NewPropDb, idle_limit()}; + handle_call({set_revs_limit, Limit}, _From, Db) -> {ok, Db2} = couch_db_engine:set_revs_limit(Db, Limit), Db3 = commit_data(Db2), @@ -375,7 +383,8 @@ init_db(DbName, FilePath, EngineState, Options) -> InitDb#db{ committed_update_seq = couch_db_engine:get_update_seq(InitDb), - security = couch_db_engine:get_security(InitDb) + security = couch_db_engine:get_security(InitDb), + props = couch_db_engine:get_props(InitDb) }. diff --git a/src/couch/src/test_engine_get_set_props.erl b/src/couch/src/test_engine_get_set_props.erl index 6d2a447..b0fbff4 100644 --- a/src/couch/src/test_engine_get_set_props.erl +++ b/src/couch/src/test_engine_get_set_props.erl @@ -50,6 +50,28 @@ cet_set_revs_limit() -> check_prop_set(get_revs_limit, set_revs_limit, 1000, 50). +cet_set_prop() -> + Engine = test_engine_util:get_engine(), + DbPath = test_engine_util:dbpath(), + + {ok, St0} = Engine:init(DbPath, [ + create, + {default_security_object, dso} + ]), + ?assertEqual({error, no_value}, Engine:get_prop(St0, shardkey)), + + ?assertEqual(false, Engine:get_prop(St0, shardkey, false)), + + {ok, St1} = Engine:set_prop(St0, shardkey, true), + ?assertEqual({ok, true}, Engine:get_prop(St1, shardkey)), + + {ok, St2} = Engine:commit_data(St1), + Engine:terminate(normal, St2), + + {ok, St3} = Engine:init(DbPath, []), + ?assertEqual({ok, true}, Engine:get_prop(St3, shardkey)). + + check_prop_set(GetFun, SetFun, Default, Value) -> Engine = test_engine_util:get_engine(), DbPath = test_engine_util:dbpath(), diff --git a/src/fabric/src/fabric_db_create.erl b/src/fabric/src/fabric_db_create.erl index 94ffd56..90ccdab 100644 --- a/src/fabric/src/fabric_db_create.erl +++ b/src/fabric/src/fabric_db_create.erl @@ -164,6 +164,10 @@ make_document([#shard{dbname=DbName}|_] = Shards, Suffix, Options) -> {[[<<"add">>, Range, Node] | Raw], orddict:append(Node, Range, ByNode), orddict:append(Range, Node, ByRange)} end, {[], [], []}, Shards), + InitialProps = case couch_util:get_value(initial_props, Options) of + I when is_list(I) -> [{<<"options">>, {I}}]; + _ -> [] + end, EngineProp = case couch_util:get_value(engine, Options) of E when is_binary(E) -> [{<<"engine">>, E}]; _ -> [] @@ -175,7 +179,7 @@ make_document([#shard{dbname=DbName}|_] = Shards, Suffix, Options) -> {<<"changelog">>, lists:sort(RawOut)}, {<<"by_node">>, {[{K,lists:sort(V)} || {K,V} <- ByNodeOut]}}, {<<"by_range">>, {[{K,lists:sort(V)} || {K,V} <- ByRangeOut]}} - ] ++ EngineProp} + ] ++ EngineProp ++ InitialProps} }. db_exists(DbName) -> is_list(catch mem3:shards(DbName)). diff --git a/src/mem3/src/mem3_util.erl b/src/mem3/src/mem3_util.erl index 0b69d79..e08d375 100644 --- a/src/mem3/src/mem3_util.erl +++ b/src/mem3/src/mem3_util.erl @@ -162,7 +162,7 @@ build_shards_by_node(DbName, DocProps) -> dbname = DbName, node = to_atom(Node), range = [Beg, End], - opts = get_engine_opt(DocProps) + opts = get_engine_opt(DocProps) ++ get_opts(DocProps) }, Suffix) end, Ranges) end, ByNode). @@ -180,7 +180,7 @@ build_shards_by_range(DbName, DocProps) -> node = to_atom(Node), range = [Beg, End], order = Order, - opts = get_engine_opt(DocProps) + opts = get_engine_opt(DocProps) ++ get_opts(DocProps) }, Suffix) end, lists:zip(Nodes, lists:seq(1, length(Nodes)))) end, ByRange). @@ -197,6 +197,14 @@ to_integer(N) when is_binary(N) -> to_integer(N) when is_list(N) -> list_to_integer(N). +get_opts(DocProps) -> + case couch_util:get_value(<<"options">>, DocProps) of + {Opts} -> + Opts; + _ -> + [] + end. + get_engine_opt(DocProps) -> case couch_util:get_value(<<"engine">>, DocProps) of Engine when is_binary(Engine) ->
