iilyak commented on a change in pull request #3766:
URL: https://github.com/apache/couchdb/pull/3766#discussion_r773366251
##########
File path: src/smoosh/src/smoosh_channel.erl
##########
@@ -194,21 +215,128 @@ handle_info(check_window, State0) ->
end,
erlang:send_after(60 * 1000, self(), check_window),
{noreply, FinalState};
+handle_info(persist_queue, State0) ->
+ #state{waiting = Queue} = State0,
+ write_state_to_file(State0),
+ smoosh_priority_queue:write_to_file(Queue),
+ Checkpoint =
+ config:get_integer(
+ "smoosh", "state_checkpoint_interval_in_sec",
?DEFAULT_CHECKPOINT_INTERVAL_IN_SEC
+ ) * 1000,
+ erlang:send_after(Checkpoint, self(), persist_queue),
+ {noreply, State0};
handle_info(pause, State0) ->
- {ok, State} = code_change(nil, State0, nil),
+ State = maybe_open_queue(State0),
{noreply, State#state{paused = true}};
handle_info(unpause, State0) ->
- {ok, State} = code_change(nil, State0, nil),
+ State = maybe_open_queue(State0),
{noreply, maybe_start_compaction(State#state{paused = false})}.
-terminate(_Reason, _State) ->
+terminate(_Reason, #state{name = Name, waiting = Q}) ->
+ file:delete(active_file_name(Name)),
+ file:delete(starting_file_name(Name)),
+ if
+ Q =/= nil ->
+ smoosh_priority_queue:close(Q);
+ true ->
+ nil
+ end,
ok.
-code_change(_OldVsn, #state{} = State, _Extra) ->
- {ok, State}.
+maybe_recover_state(#state{name = Name} = State) ->
+ Active = recover(active_file_name(Name)),
+ Starting = recover(starting_file_name(Name)),
+ DatabaseDir = config:get("couchdb", "database_dir"),
+ ViewDir = config:get("couchdb", "view_index_dir"),
+ Active1 = get_matching_compact_files(DatabaseDir, Active),
+ Active2 = get_matching_compact_files(ViewDir, Active),
+ Active3 = Active1 ++ Active2,
+ State#state{active = Active3, starting = Starting}.
+
+get_matching_compact_files(Dir, Active) ->
+ MatchingFiles = filelib:fold_files(
+ Dir,
+ "^[a-zA-Z0-9_.-]*.compact$",
+ true,
+ (fun(FilePath, Acc) ->
+ FilePrefix = filename:rootname(FilePath, ".compact"),
+ case lists:keyfind(FilePrefix, 1, Active) of
Review comment:
Actually I had a confusion. After re-reading the
https://github.ibm.com/cloudant/dbcore/issues/53 ticket I think we need to do
the opposite. We need to remove compaction jobs from active list if the compact
files are not present on disk.
There are two options here:
1. Iterate over Active and remove jobs if compact files are not present on
disk. This wouldn't pick compaction jobs triggered between last checkpoint and
node restart.
2. Do not persist active jobs at all and always do a directory scan on
smoosh start. Add all compact files found into Active. This would be slower
since we would be scanning a directory with hundred thousands of files. However
this PR does exactly this.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
