davisp closed pull request #597: Move view index files to .recovery when db is
deleted
URL: https://github.com/apache/couchdb/pull/597
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/src/couch/src/couch_file.erl b/src/couch/src/couch_file.erl
index acd4fda781..04ca44ddf3 100644
--- a/src/couch/src/couch_file.erl
+++ b/src/couch/src/couch_file.erl
@@ -43,7 +43,8 @@
-export([append_raw_chunk/2, assemble_file_chunk/1, assemble_file_chunk/2]).
-export([append_term/2, append_term/3, append_term_md5/2, append_term_md5/3]).
-export([write_header/2, read_header/1]).
--export([delete/2, delete/3, nuke_dir/2, init_delete_dir/1]).
+-export([delete/2, delete/3, nuke_dir/2, nuke_dir/3]).
+-export([init_delete_dir/1, init_recovery_dir/1]).
-export([msec_since_last_read/1]).
% gen_server callbacks
@@ -265,6 +266,18 @@ rename_file(Original) ->
Else -> Else
end.
+rename_dir(RootDelDir, Original, DbName) ->
+ DbDir = binary_to_list(DbName) ++ "_design",
+ RenamedIndexDir = filename:join(
+ [RootDelDir, ".recovery", DbDir]
+ ),
+ Now = calendar:local_time(),
+ filelib:ensure_dir(RenamedIndexDir),
+ case file:rename(Original, RenamedIndexDir) of
+ ok -> file:change_time(RenamedIndexDir, Now);
+ Else -> Else
+ end.
+
deleted_filename(Original) ->
{{Y, Mon, D}, {H, Min, S}} = calendar:universal_time(),
Suffix = lists:flatten(
@@ -273,14 +286,18 @@ deleted_filename(Original) ->
++ filename:extension(Original), [Y, Mon, D, H, Min, S])),
filename:rootname(Original) ++ Suffix.
-nuke_dir(RootDelDir, Dir) ->
+nuke_dir(RootDir, Dir) ->
+ nuke_dir(RootDir, Dir, []).
+nuke_dir(RootDelDir, Dir, Options) ->
EnableRecovery = config:get_boolean("couchdb",
"enable_database_recovery", false),
+ Context = couch_util:get_value(context, Options, compaction),
case EnableRecovery of
- true ->
- rename_file(Dir);
- false ->
- delete_dir(RootDelDir, Dir)
+ true when Context == delete ->
+ DbName = couch_util:get_value(db_name, Options),
+ rename_dir(RootDelDir, Dir, DbName);
+ true -> rename_file(Dir);
+ false -> delete_dir(RootDelDir, Dir)
end.
delete_dir(RootDelDir, Dir) ->
@@ -318,6 +335,9 @@ init_delete_dir(RootDir) ->
end),
ok.
+init_recovery_dir(RootDir) ->
+ Dir = filename:join(RootDir, ".recovery"),
+ filelib:ensure_dir(filename:join(Dir, "foo")).
read_header(Fd) ->
case ioq:call(Fd, find_header, erlang:get(io_priority)) of
@@ -762,11 +782,25 @@ make_filename_fixtures(DbNames) ->
"shards/00000000-1fffffff/~s.1458336317.couch",
".shards/00000000-1fffffff/~s.1458336317_design",
".shards/00000000-1fffffff/~s.1458336317_design"
- "/mrview/3133e28517e89a3e11435dd5ac4ad85a.view"
+ "/mrview/3133e28517e89a3e11435dd5ac4ad85a.view",
+ ".recovery/1499329402/shards/00000000-1fffffff/~s.1499329402_design",
+ ".recovery/1499329402/shards/00000000-1fffffff/~s.1499329402_design"
+ "/mrview/8fabddcb28f501d6764afd7def3bd352.view"
],
lists:flatmap(fun(DbName) ->
lists:map(fun(Format) ->
- filename:join("/srv/data", io_lib:format(Format, [DbName]))
+ % Count how many times we need to specify the database name
+ % by splitting on the ~s formatter.
+ ArgCount = erlang:length(binary:split(
+ list_to_binary(Format),
+ <<"~s">>,
+ [global])
+ ) - 1,
+ Args = case ArgCount of
+ 1 -> [DbName];
+ 2 -> [DbName, DbName]
+ end,
+ filename:join("/srv/data/", io_lib:format(Format, Args))
end, Formats)
end, DbNames).
diff --git a/src/couch/test/couch_file_tests.erl
b/src/couch/test/couch_file_tests.erl
index c16be16c41..7990533223 100644
--- a/src/couch/test/couch_file_tests.erl
+++ b/src/couch/test/couch_file_tests.erl
@@ -419,32 +419,44 @@ nuke_dir_test_() ->
RootDir = filename:dirname(File0),
BaseName = filename:basename(File0),
Seed = crypto:rand_uniform(1000000000, 9999999999),
- DDocDir = io_lib:format("db.~b_design", [Seed]),
+ DBName0 = io_lib:format("db.~b", [Seed]),
+ DBName = iolist_to_binary(DBName0),
+ DDocDir = io_lib:format("~s_design", [DBName0]),
ViewDir = filename:join([RootDir, DDocDir]),
file:make_dir(ViewDir),
File = filename:join([ViewDir, BaseName]),
file:rename(File0, File),
ok = couch_file:init_delete_dir(RootDir),
ok = file:write_file(File, <<>>),
- {RootDir, ViewDir}
+ {RootDir, ViewDir, DBName}
end,
- fun({RootDir, ViewDir}) ->
+ fun({RootDir, ViewDir, _DBName}) ->
meck:unload(config),
remove_dir(ViewDir),
Ext = filename:extension(ViewDir),
case filelib:wildcard(RootDir ++ "/*.deleted" ++ Ext) of
[DelDir] -> remove_dir(DelDir);
_ -> ok
- end
+ end,
+ RecDirPaths = RootDir ++ "/.recovery" ++ "/*_design",
+ [remove_dir(Dir) || Dir <- filelib:wildcard(RecDirPaths)]
end,
[
fun(Cfg) ->
- {"enable_database_recovery = false",
- make_rename_dir_test_case(Cfg, false)}
+ {"enable_database_recovery = false, context = delete",
+ make_rename_dir_test_case(Cfg, false, delete)}
+ end,
+ fun(Cfg) ->
+ {"enable_database_recovery = false, context = compaction",
+ make_rename_dir_test_case(Cfg, false, compaction)}
+ end,
+ fun(Cfg) ->
+ {"enable_database_recovery = true, context = delete",
+ make_rename_dir_test_case(Cfg, true, delete)}
end,
fun(Cfg) ->
- {"enable_database_recovery = true",
- make_rename_dir_test_case(Cfg, true)}
+ {"enable_database_recovery = true, context = compaction",
+ make_rename_dir_test_case(Cfg, true, compaction)}
end,
fun(Cfg) ->
{"delete_after_rename = true",
@@ -459,16 +471,26 @@ nuke_dir_test_() ->
}.
-make_rename_dir_test_case({RootDir, ViewDir}, EnableRecovery) ->
+make_rename_dir_test_case({RootDir, ViewDir, DBName}, EnableRecovery, Context)
->
meck:expect(config, get_boolean, fun
("couchdb", "enable_database_recovery", _) -> EnableRecovery;
("couchdb", "delete_after_rename", _) -> true
end),
DirExistsBefore = filelib:is_dir(ViewDir),
- couch_file:nuke_dir(RootDir, ViewDir),
+
+ couch_file:nuke_dir(
+ RootDir,
+ ViewDir,
+ [{db_name, DBName}, {context, Context}]
+ ),
DirExistsAfter = filelib:is_dir(ViewDir),
- Ext = filename:extension(ViewDir),
- RenamedDirs = filelib:wildcard(RootDir ++ "/*.deleted" ++ Ext),
+ RenamedDirs = if Context =:= delete ->
+ filelib:wildcard(RootDir ++ "/.recovery" ++ "/*_design");
+ true ->
+ Ext = filename:extension(ViewDir),
+ filelib:wildcard(RootDir ++ "/*.deleted" ++ Ext)
+ end,
+
ExpectRenamedCount = if EnableRecovery -> 1; true -> 0 end,
[
?_assert(DirExistsBefore),
@@ -476,7 +498,7 @@ make_rename_dir_test_case({RootDir, ViewDir},
EnableRecovery) ->
?_assertEqual(ExpectRenamedCount, length(RenamedDirs))
].
-make_delete_dir_test_case({RootDir, ViewDir}, DeleteAfterRename) ->
+make_delete_dir_test_case({RootDir, ViewDir, _DBName}, DeleteAfterRename) ->
meck:expect(config, get_boolean, fun
("couchdb", "enable_database_recovery", _) -> false;
("couchdb", "delete_after_rename", _) -> DeleteAfterRename
diff --git a/src/couch_index/src/couch_index_server.erl
b/src/couch_index/src/couch_index_server.erl
index 8225a90a33..c16b2a475f 100644
--- a/src/couch_index/src/couch_index_server.erl
+++ b/src/couch_index/src/couch_index_server.erl
@@ -129,6 +129,7 @@ init([]) ->
couch_event:link_listener(?MODULE, handle_db_event, nil, [all_dbs]),
RootDir = couch_index_util:root_dir(),
couch_file:init_delete_dir(RootDir),
+ couch_file:init_recovery_dir(RootDir),
{ok, #st{root_dir=RootDir}}.
@@ -161,14 +162,9 @@ handle_call({async_error, {DbName, _DDocId, Sig}, Error},
_From, State) ->
[gen_server:reply(From, Error) || From <- Waiters],
ets:delete(?BY_SIG, {DbName, Sig}),
{reply, ok, State};
-handle_call({reset_indexes, DbName}, _From, State) ->
- reset_indexes(DbName, State#st.root_dir),
+handle_call({reset_indexes, DbName, Options}, _From, State) ->
+ reset_indexes(DbName, State#st.root_dir, Options),
{reply, ok, State}.
-
-
-handle_cast({reset_indexes, DbName}, State) ->
- reset_indexes(DbName, State#st.root_dir),
- {noreply, State};
handle_cast({add_to_ets, [Pid, DbName, DDocId, Sig]}, State) ->
% check if Pid still exists
case ets:lookup(?BY_PID, Pid) of
@@ -179,6 +175,11 @@ handle_cast({add_to_ets, [Pid, DbName, DDocId, Sig]},
State) ->
{noreply, State};
handle_cast({rem_from_ets, [DbName, DDocId, Sig]}, State) ->
ets:delete_object(?BY_DB, {DbName, {DDocId, Sig}}),
+ {noreply, State};
+
+
+handle_cast({reset_indexes, DbName, Options}, State) ->
+ reset_indexes(DbName, State#st.root_dir, Options),
{noreply, State}.
handle_info({'EXIT', Pid, Reason}, Server) ->
@@ -237,7 +238,7 @@ new_index({Mod, IdxState, DbName, Sig}) ->
end.
-reset_indexes(DbName, Root) ->
+reset_indexes(DbName, Root, Options) ->
% shutdown all the updaters and clear the files, the db got changed
SigDDocIds = lists:foldl(fun({_, {DDocId, Sig}}, DDict) ->
dict:append(Sig, DDocId, DDict)
@@ -251,7 +252,8 @@ reset_indexes(DbName, Root) ->
end,
lists:foreach(Fun, dict:to_list(SigDDocIds)),
Path = couch_index_util:index_dir("", DbName),
- couch_file:nuke_dir(Root, Path).
+ Options1 = [{db_name, DbName} | Options],
+ couch_file:nuke_dir(Root, Path, Options1).
add_to_ets(DbName, Sig, DDocId, Pid) ->
@@ -269,10 +271,10 @@ rem_from_ets(DbName, Sig, DDocIds, Pid) ->
handle_db_event(DbName, created, St) ->
- gen_server:cast(?MODULE, {reset_indexes, DbName}),
+ gen_server:cast(?MODULE, {reset_indexes, DbName, []}),
{ok, St};
handle_db_event(DbName, deleted, St) ->
- gen_server:cast(?MODULE, {reset_indexes, DbName}),
+ gen_server:cast(?MODULE, {reset_indexes, DbName, [{context, delete}]}),
{ok, St};
handle_db_event(<<"shards/", _/binary>> = DbName, {ddoc_updated,
DDocId}, St) ->
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
