This is an automated email from the ASF dual-hosted git repository. vatamane pushed a commit to branch refactor-replication-tests in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 35da93ac185daf5844ee826fd24478115a8e7373 Author: Nick Vatamaniuc <[email protected]> AuthorDate: Thu Aug 18 21:10:15 2022 -0400 Update couch_replicator_compact_tests Compactor tests are the only tests which continue using the local ports since they deals with triggering and managing low level compaction processes. However, it was still possible to improve the tests somewhat by using the TDEF_FE macros and removing some left-over foreachx cruft. --- .../test/eunit/couch_replicator_compact_tests.erl | 314 +++++++++------------ 1 file changed, 131 insertions(+), 183 deletions(-) diff --git a/src/couch_replicator/test/eunit/couch_replicator_compact_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_compact_tests.erl index 1c093d58c..1f241c753 100644 --- a/src/couch_replicator/test/eunit/couch_replicator_compact_tests.erl +++ b/src/couch_replicator/test/eunit/couch_replicator_compact_tests.erl @@ -15,11 +15,7 @@ -include_lib("couch/include/couch_eunit.hrl"). -include_lib("couch/include/couch_db.hrl"). -include_lib("couch_replicator/src/couch_replicator.hrl"). - --import(couch_replicator_test_helper, [ - db_url/1, - get_pid/1 -]). +-include("couch_replicator_test.hrl"). -define(ATTFILE, filename:join([?FIXTURESDIR, "logo.png"])). -define(DELAY, 500). @@ -28,92 +24,60 @@ -define(TIMEOUT_EUNIT, ?TIMEOUT div 1000 + 70). -define(WRITE_BATCH_SIZE, 25). -setup() -> +setup_db() -> DbName = ?tempdb(), {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]), ok = couch_db:close(Db), DbName. -setup(remote) -> - {remote, setup()}; -setup({A, B}) -> - Ctx = test_util:start_couch([couch_replicator]), - Source = setup(A), - Target = setup(B), - {Ctx, {Source, Target}}. - -teardown({remote, DbName}) -> - teardown(DbName); -teardown(DbName) -> +teardown_db(DbName) -> ok = couch_server:delete(DbName, [?ADMIN_CTX]), ok. -teardown(_, {Ctx, {Source, Target}}) -> - teardown(Source), - teardown(Target), - ok = application:stop(couch_replicator), +test_setup() -> + Ctx = test_util:start_couch([couch_replicator]), + Source = setup_db(), + Target = setup_db(), + {Ctx, {Source, Target}}. + +test_teardown({Ctx, {Source, Target}}) -> + teardown_db(Source), + teardown_db(Target), ok = test_util:stop_couch(Ctx). compact_test_() -> - Pairs = [{remote, remote}], { "Compaction during replication tests", { - foreachx, - fun setup/1, - fun teardown/2, + foreach, + fun test_setup/0, + fun test_teardown/1, [ - {Pair, fun should_populate_replicate_compact/2} - || Pair <- Pairs + ?TDEF_FE(populate_replicate_compact, ?TIMEOUT_EUNIT) ] } }. -should_populate_replicate_compact({From, To}, {_Ctx, {Source, Target}}) -> +populate_replicate_compact({_Ctx, {Source, Target}}) -> {ok, RepPid, RepId} = replicate(Source, Target), - { - lists:flatten(io_lib:format("~p -> ~p", [From, To])), - {inorder, [ - should_run_replication(RepPid, RepId, Source, Target), - should_all_processes_be_alive(RepPid, Source, Target), - should_populate_and_compact(RepPid, Source, Target, 50, 3), - should_wait_target_in_sync(Source, Target), - should_ensure_replication_still_running(RepPid, RepId, Source, Target), - should_cancel_replication(RepId, RepPid), - should_compare_databases(Source, Target) - ]} - }. - -should_all_processes_be_alive(RepPid, Source, Target) -> - ?_test(begin - {ok, SourceDb} = reopen_db(Source), - {ok, TargetDb} = reopen_db(Target), - ?assert(is_process_alive(RepPid)), - ?assert(is_process_alive(couch_db:get_pid(SourceDb))), - ?assert(is_process_alive(couch_db:get_pid(TargetDb))) - end). - -should_run_replication(RepPid, RepId, Source, Target) -> - ?_test(check_active_tasks(RepPid, RepId, Source, Target)). - -should_ensure_replication_still_running(RepPid, RepId, Source, Target) -> - ?_test(check_active_tasks(RepPid, RepId, Source, Target)). + check_active_tasks(RepPid, RepId, Source, Target), + all_processes_are_alive(RepPid, Source, Target), + populate_and_compact(RepPid, Source, Target, 50, 3), + wait_target_in_sync(Source, Target), + check_active_tasks(RepPid, RepId, Source, Target), + cancel_replication(RepId, RepPid), + compare_databases(Source, Target). + +all_processes_are_alive(RepPid, Source, Target) -> + {ok, SourceDb} = reopen_db(Source), + {ok, TargetDb} = reopen_db(Target), + ?assert(is_process_alive(RepPid)), + ?assert(is_process_alive(couch_db:get_pid(SourceDb))), + ?assert(is_process_alive(couch_db:get_pid(TargetDb))). check_active_tasks(RepPid, {BaseId, Ext} = _RepId, Src, Tgt) -> - Source = - case Src of - {remote, NameSrc} -> - <<(db_url(NameSrc))/binary, $/>>; - _ -> - Src - end, - Target = - case Tgt of - {remote, NameTgt} -> - <<(db_url(NameTgt))/binary, $/>>; - _ -> - Tgt - end, + Source = <<(db_url(Src))/binary, $/>>, + Target = <<(db_url(Tgt))/binary, $/>>, FullRepId = ?l2b(BaseId ++ Ext), Pid = ?l2b(pid_to_list(RepPid)), RepTasks = wait_for_task_status(), @@ -152,71 +116,59 @@ wait_for_task_status() -> end end). -should_cancel_replication(RepId, RepPid) -> - ?_assertNot(begin - ok = couch_replicator_scheduler:remove_job(RepId), - is_process_alive(RepPid) - end). +cancel_replication(RepId, RepPid) -> + ok = couch_replicator_scheduler:remove_job(RepId), + ?assertNot(is_process_alive(RepPid)). + +populate_and_compact(RepPid, Source, Target, BatchSize, Rounds) -> + {ok, SourceDb0} = reopen_db(Source), + Writer = spawn_writer(SourceDb0), + lists:foreach( + fun(N) -> + {ok, SourceDb} = reopen_db(Source), + {ok, TargetDb} = reopen_db(Target), + pause_writer(Writer), + + compact_db("source", SourceDb), + ?assert(is_process_alive(RepPid)), + ?assert(is_process_alive(couch_db:get_pid(SourceDb))), + wait_for_compaction("source", SourceDb), + + compact_db("target", TargetDb), + ?assert(is_process_alive(RepPid)), + ?assert(is_process_alive(couch_db:get_pid(TargetDb))), + wait_for_compaction("target", TargetDb), + + {ok, SourceDb2} = reopen_db(SourceDb), + {ok, TargetDb2} = reopen_db(TargetDb), + + resume_writer(Writer), + wait_writer(Writer, BatchSize * N), + + compact_db("source", SourceDb2), + ?assert(is_process_alive(RepPid)), + ?assert(is_process_alive(couch_db:get_pid(SourceDb2))), + pause_writer(Writer), + wait_for_compaction("source", SourceDb2), + resume_writer(Writer), + + compact_db("target", TargetDb2), + ?assert(is_process_alive(RepPid)), + ?assert(is_process_alive(couch_db:get_pid(TargetDb2))), + pause_writer(Writer), + wait_for_compaction("target", TargetDb2), + resume_writer(Writer) + end, + lists:seq(1, Rounds) + ), + stop_writer(Writer). -should_populate_and_compact(RepPid, Source, Target, BatchSize, Rounds) -> - {timeout, ?TIMEOUT_EUNIT, - ?_test(begin - {ok, SourceDb0} = reopen_db(Source), - Writer = spawn_writer(SourceDb0), - lists:foreach( - fun(N) -> - {ok, SourceDb} = reopen_db(Source), - {ok, TargetDb} = reopen_db(Target), - pause_writer(Writer), - - compact_db("source", SourceDb), - ?assert(is_process_alive(RepPid)), - ?assert(is_process_alive(couch_db:get_pid(SourceDb))), - wait_for_compaction("source", SourceDb), - - compact_db("target", TargetDb), - ?assert(is_process_alive(RepPid)), - ?assert(is_process_alive(couch_db:get_pid(TargetDb))), - wait_for_compaction("target", TargetDb), - - {ok, SourceDb2} = reopen_db(SourceDb), - {ok, TargetDb2} = reopen_db(TargetDb), - - resume_writer(Writer), - wait_writer(Writer, BatchSize * N), - - compact_db("source", SourceDb2), - ?assert(is_process_alive(RepPid)), - ?assert(is_process_alive(couch_db:get_pid(SourceDb2))), - pause_writer(Writer), - wait_for_compaction("source", SourceDb2), - resume_writer(Writer), - - compact_db("target", TargetDb2), - ?assert(is_process_alive(RepPid)), - ?assert(is_process_alive(couch_db:get_pid(TargetDb2))), - pause_writer(Writer), - wait_for_compaction("target", TargetDb2), - resume_writer(Writer) - end, - lists:seq(1, Rounds) - ), - stop_writer(Writer) - end)}. - -should_wait_target_in_sync({remote, Source}, Target) -> - should_wait_target_in_sync(Source, Target); -should_wait_target_in_sync(Source, {remote, Target}) -> - should_wait_target_in_sync(Source, Target); -should_wait_target_in_sync(Source, Target) -> - {timeout, ?TIMEOUT_EUNIT, - ?_assert(begin - {ok, SourceDb} = couch_db:open_int(Source, []), - {ok, SourceInfo} = couch_db:get_db_info(SourceDb), - ok = couch_db:close(SourceDb), - SourceDocCount = couch_util:get_value(doc_count, SourceInfo), - wait_target_in_sync_loop(SourceDocCount, Target, 300) - end)}. +wait_target_in_sync(Source, Target) -> + {ok, SourceDb} = couch_db:open_int(Source, []), + {ok, SourceInfo} = couch_db:get_db_info(SourceDb), + ok = couch_db:close(SourceDb), + SourceDocCount = couch_util:get_value(doc_count, SourceInfo), + wait_target_in_sync_loop(SourceDocCount, Target, 300). wait_target_in_sync_loop(_DocCount, _TargetName, 0) -> erlang:error( @@ -226,8 +178,6 @@ wait_target_in_sync_loop(_DocCount, _TargetName, 0) -> {reason, "Could not get source and target databases in sync"} ]} ); -wait_target_in_sync_loop(DocCount, {remote, TargetName}, RetriesLeft) -> - wait_target_in_sync_loop(DocCount, TargetName, RetriesLeft); wait_target_in_sync_loop(DocCount, TargetName, RetriesLeft) -> {ok, Target} = couch_db:open_int(TargetName, []), {ok, TargetInfo} = couch_db:get_db_info(Target), @@ -241,49 +191,40 @@ wait_target_in_sync_loop(DocCount, TargetName, RetriesLeft) -> wait_target_in_sync_loop(DocCount, TargetName, RetriesLeft - 1) end. -should_compare_databases({remote, Source}, Target) -> - should_compare_databases(Source, Target); -should_compare_databases(Source, {remote, Target}) -> - should_compare_databases(Source, Target); -should_compare_databases(Source, Target) -> - {timeout, 35, - ?_test(begin - {ok, SourceDb} = couch_db:open_int(Source, []), - {ok, TargetDb} = couch_db:open_int(Target, []), - Fun = fun(FullDocInfo, Acc) -> - {ok, Doc} = couch_db:open_doc(SourceDb, FullDocInfo), - {Props} = DocJson = couch_doc:to_json_obj(Doc, [attachments]), - DocId = couch_util:get_value(<<"_id">>, Props), - DocTarget = - case couch_db:open_doc(TargetDb, DocId) of - {ok, DocT} -> - DocT; - Error -> - erlang:error( - {assertion_failed, [ - {module, ?MODULE}, - {line, ?LINE}, - {reason, - lists:concat([ - "Error opening document '", - ?b2l(DocId), - "' from target: ", - couch_util:to_list(Error) - ])} - ]} - ) - end, - DocTargetJson = couch_doc:to_json_obj(DocTarget, [attachments]), - ?assertEqual(DocJson, DocTargetJson), - {ok, Acc} +compare_databases(Source, Target) -> + {ok, SourceDb} = couch_db:open_int(Source, []), + {ok, TargetDb} = couch_db:open_int(Target, []), + Fun = fun(FullDocInfo, Acc) -> + {ok, Doc} = couch_db:open_doc(SourceDb, FullDocInfo), + {Props} = DocJson = couch_doc:to_json_obj(Doc, [attachments]), + DocId = couch_util:get_value(<<"_id">>, Props), + DocTarget = + case couch_db:open_doc(TargetDb, DocId) of + {ok, DocT} -> + DocT; + Error -> + erlang:error( + {assertion_failed, [ + {module, ?MODULE}, + {line, ?LINE}, + {reason, + lists:concat([ + "Error opening document '", + ?b2l(DocId), + "' from target: ", + couch_util:to_list(Error) + ])} + ]} + ) end, - {ok, _} = couch_db:fold_docs(SourceDb, Fun, [], []), - ok = couch_db:close(SourceDb), - ok = couch_db:close(TargetDb) - end)}. + DocTargetJson = couch_doc:to_json_obj(DocTarget, [attachments]), + ?assertEqual(DocJson, DocTargetJson), + {ok, Acc} + end, + {ok, _} = couch_db:fold_docs(SourceDb, Fun, [], []), + ok = couch_db:close(SourceDb), + ok = couch_db:close(TargetDb). -reopen_db({remote, Db}) -> - reopen_db(Db); reopen_db(DbName) when is_binary(DbName) -> {ok, Db} = couch_db:open_int(DbName, []), ok = couch_db:close(Db), @@ -357,21 +298,17 @@ wait_for_compaction(Type, Db) -> ) end. -replicate({remote, Db}, Target) -> - replicate(db_url(Db), Target); -replicate(Source, {remote, Db}) -> - replicate(Source, db_url(Db)); replicate(Source, Target) -> RepObject = {[ - {<<"source">>, Source}, - {<<"target">>, Target}, + {<<"source">>, db_url(Source)}, + {<<"target">>, db_url(Target)}, {<<"continuous">>, true} ]}, {ok, Rep} = couch_replicator_utils:parse_rep_doc(RepObject, ?ADMIN_USER), ok = couch_replicator_scheduler:add_job(Rep), couch_replicator_scheduler:reschedule(), - Pid = get_pid(Rep#rep.id), + Pid = couch_replicator_test_helper:get_pid(Rep#rep.id), {ok, Pid, Rep#rep.id}. wait_writer(Pid, NumDocs) -> @@ -521,3 +458,14 @@ maybe_pause(Parent, Counter) -> after 0 -> ok end. + +db_url(DbName) -> + % Note we're returning the backend (local) URL here + iolist_to_binary([ + "http://";, + config:get("httpd", "bind_address", "127.0.0.1"), + ":", + integer_to_list(mochiweb_socket_server:get(couch_httpd, port)), + "/", + DbName + ]).
