This is an automated email from the ASF dual-hosted git repository.
jiahuili430 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
The following commit(s) were added to refs/heads/main by this push:
new f79d9514a Add `_purged_infos` endpoint
f79d9514a is described below
commit f79d9514af75e2f3a9c6947146180fdf6240055b
Author: jiahuili <[email protected]>
AuthorDate: Thu Aug 17 13:51:31 2023 -0500
Add `_purged_infos` endpoint
- Expose the internal purged_infos API to users
- Add tests and documentation for `_purged_infos`
---
src/chttpd/src/chttpd_db.erl | 13 +-
src/chttpd/test/eunit/chttpd_purge_tests.erl | 742 +++++++++++----------------
src/docs/src/api/database/misc.rst | 50 ++
src/fabric/src/fabric.erl | 7 +
src/fabric/src/fabric_db_purged_infos.erl | 79 +++
src/fabric/src/fabric_rpc.erl | 6 +-
6 files changed, 454 insertions(+), 443 deletions(-)
diff --git a/src/chttpd/src/chttpd_db.erl b/src/chttpd/src/chttpd_db.erl
index e2de301b2..9b1aff54f 100644
--- a/src/chttpd/src/chttpd_db.erl
+++ b/src/chttpd/src/chttpd_db.erl
@@ -726,6 +726,15 @@ db_req(#httpd{method = 'POST', path_parts = [_,
<<"_purge">>]} = Req, Db) ->
send_json(Req, Code, {[{<<"purge_seq">>, null}, {<<"purged">>, {Json}}]});
db_req(#httpd{path_parts = [_, <<"_purge">>]} = Req, _Db) ->
send_method_not_allowed(Req, "POST");
+db_req(#httpd{method = 'GET', path_parts = [_, <<"_purged_infos">>]} = Req,
Db) ->
+ {ok, PurgedInfosRaw} = fabric:get_purged_infos(Db),
+ PurgedInfos = [
+ {[{id, Id}, {revs, [couch_doc:rev_to_str(Rev) || Rev <- Revs]}]}
+ || {Id, Revs} <- PurgedInfosRaw
+ ],
+ send_json(Req, {[{purged_infos, PurgedInfos}]});
+db_req(#httpd{path_parts = [_, <<"_purged_infos">>]} = Req, _Db) ->
+ send_method_not_allowed(Req, "GET");
db_req(#httpd{method = 'GET', path_parts = [_, OP]} = Req, Db) when
?IS_ALL_DOCS(OP) ->
case chttpd:qs_json_value(Req, "keys", nil) of
Keys when is_list(Keys) ->
@@ -855,10 +864,12 @@ db_req(#httpd{method = 'PUT', path_parts = [_,
<<"_purged_infos_limit">>]} = Req
throw(Error)
end;
_ ->
- throw({bad_request, "`purge_infos_limit` must be positive
integer"})
+ throw({bad_request, "`purged_infos_limit` must be positive
integer"})
end;
db_req(#httpd{method = 'GET', path_parts = [_, <<"_purged_infos_limit">>]} =
Req, Db) ->
send_json(Req, fabric:get_purge_infos_limit(Db));
+db_req(#httpd{path_parts = [_, <<"_purged_infos_limit">>]} = Req, _Db) ->
+ send_method_not_allowed(Req, "GET,PUT");
% Special case to enable using an unencoded slash in the URL of design docs,
% as slashes in document IDs must otherwise be URL encoded.
db_req(
diff --git a/src/chttpd/test/eunit/chttpd_purge_tests.erl
b/src/chttpd/test/eunit/chttpd_purge_tests.erl
index a8e1a955d..9514125be 100644
--- a/src/chttpd/test/eunit/chttpd_purge_tests.erl
+++ b/src/chttpd/test/eunit/chttpd_purge_tests.erl
@@ -23,48 +23,20 @@
setup() ->
Hashed = couch_passwords:hash_admin_password(?PASS),
ok = config:set("admins", ?USER, ?b2l(Hashed), _Persist = false),
- TmpDb = ?tempdb(),
+ Db = ?tempdb(),
Addr = config:get("chttpd", "bind_address", "127.0.0.1"),
Port = mochiweb_socket_server:get(chttpd, port),
- Url = lists:concat(["http://";, Addr, ":", Port, "/", ?b2l(TmpDb)]),
- create_db(Url),
- Url.
+ DbUrl = lists:concat(["http://";, Addr, ":", Port, "/", ?b2l(Db)]),
+ create_db(DbUrl),
+ DbUrl.
-teardown(Url) ->
- delete_db(Url),
+teardown(DbUrl) ->
+ delete_db(DbUrl),
ok = config:delete("admins", ?USER, _Persist = false).
-create_db(Url) ->
- {ok, Status, _, _} = test_request:put(Url, [?CONTENT_JSON, ?AUTH], "{}"),
- ?assert(Status =:= 201 orelse Status =:= 202).
-
-create_doc(Url, Id) ->
- test_request:put(
- Url ++ "/" ++ Id,
- [?CONTENT_JSON, ?AUTH],
- "{\"mr\": \"rockoartischocko\"}"
- ).
-
-create_doc(Url, Id, Content) ->
- test_request:put(
- Url ++ "/" ++ Id,
- [?CONTENT_JSON, ?AUTH],
- "{\"mr\": \"" ++ Content ++ "\"}"
- ).
-
-create_docs(Url, Docs) ->
- test_request:post(
- Url ++ "/_bulk_docs",
- [?CONTENT_JSON, ?AUTH],
- ?JSON_ENCODE({[{docs, Docs}]})
- ).
-
-delete_db(Url) ->
- {ok, 200, _, _} = test_request:delete(Url, [?AUTH]).
-
purge_test_() ->
{
- "chttpd db tests",
+ "chttpd purge tests",
{
setup,
fun chttpd_test_util:start_couch/0,
@@ -74,418 +46,306 @@ purge_test_() ->
fun setup/0,
fun teardown/1,
[
- fun test_empty_purge_request/1,
- fun test_ok_purge_request/1,
- fun test_ok_purge_request_with_101_docid/1,
- fun test_accepted_purge_request/1,
- fun test_partial_purge_request/1,
- fun test_mixed_purge_request/1,
- fun test_overmany_ids_or_revs_purge_request/1,
- fun test_exceed_limits_on_purge_infos/1,
- fun should_error_set_purged_docs_limit_to0/1,
- fun test_timeout_set_purged_infos_limit/1
+ ?TDEF_FE(t_purge_only_post_allowed),
+ ?TDEF_FE(t_empty_purge_request),
+ ?TDEF_FE(t_ok_purge_request),
+ ?TDEF_FE(t_ok_purge_with_max_document_id_number),
+ ?TDEF_FE(t_accepted_purge_request),
+ ?TDEF_FE(t_partial_purge_request),
+ ?TDEF_FE(t_mixed_purge_request),
+ ?TDEF_FE(t_over_many_ids_or_revs_purge_request),
+ ?TDEF_FE(t_purged_infos_limit_only_get_put_allowed),
+ ?TDEF_FE(t_exceed_limits_on_purge_infos),
+ ?TDEF_FE(t_should_error_set_purged_docs_limit_to_0),
+ ?TDEF_FE(t_timeout_set_purged_infos_limit),
+ ?TDEF_FE(t_purged_infos_only_get_allowed),
+ ?TDEF_FE(t_empty_purged_infos),
+ ?TDEF_FE(t_purged_infos_after_purge_request),
+ ?TDEF_FE(t_purged_infos_after_multiple_purge_requests)
]
}
}
}.
-test_empty_purge_request(Url) ->
- ?_test(begin
- IdsRevs = "{}",
- {ok, Status, _, ResultBody} = test_request:post(
- Url ++ "/_purge/",
- [?CONTENT_JSON, ?AUTH],
- IdsRevs
- ),
- ResultJson = ?JSON_DECODE(ResultBody),
- ?assert(Status =:= 201 orelse Status =:= 202),
- ?assertEqual(
- {[
- {<<"purge_seq">>, null},
- {<<"purged">>, {[]}}
- ]},
- ResultJson
- )
- end).
-
-test_ok_purge_request(Url) ->
- ?_test(begin
- {ok, _, _, Body} = create_doc(Url, "doc1"),
- {Json} = ?JSON_DECODE(Body),
- Rev1 = couch_util:get_value(<<"rev">>, Json, undefined),
- {ok, _, _, Body2} = create_doc(Url, "doc2"),
- {Json2} = ?JSON_DECODE(Body2),
- Rev2 = couch_util:get_value(<<"rev">>, Json2, undefined),
- {ok, _, _, Body3} = create_doc(Url, "doc3"),
- {Json3} = ?JSON_DECODE(Body3),
- Rev3 = couch_util:get_value(<<"rev">>, Json3, undefined),
-
- IdsRevsEJson =
- {[
- {<<"doc1">>, [Rev1]},
- {<<"doc2">>, [Rev2]},
- {<<"doc3">>, [Rev3]}
- ]},
- IdsRevs = binary_to_list(?JSON_ENCODE(IdsRevsEJson)),
-
- {ok, Status, _, ResultBody} = test_request:post(
- Url ++ "/_purge/",
- [?CONTENT_JSON, ?AUTH],
- IdsRevs
- ),
- ResultJson = ?JSON_DECODE(ResultBody),
- ?assert(Status =:= 201 orelse Status =:= 202),
- ?assertEqual(
- {[
- {<<"purge_seq">>, null},
- {<<"purged">>,
- {[
- {<<"doc1">>, [Rev1]},
- {<<"doc2">>, [Rev2]},
- {<<"doc3">>, [Rev3]}
- ]}}
- ]},
- ResultJson
- )
- end).
-
-test_ok_purge_request_with_101_docid(Url) ->
- ?_test(begin
- PurgedDocsNum = 101,
- Docs = lists:foldl(
- fun(I, Acc) ->
- Id = list_to_binary(integer_to_list(I)),
- Doc = {[{<<"_id">>, Id}, {value, I}]},
- [Doc | Acc]
- end,
- [],
- lists:seq(1, PurgedDocsNum)
- ),
-
- {ok, _, _, Body} = create_docs(Url, Docs),
- BodyJson = ?JSON_DECODE(Body),
-
- PurgeBody = lists:map(
- fun({DocResp}) ->
- Id = couch_util:get_value(<<"id">>, DocResp, undefined),
- Rev = couch_util:get_value(<<"rev">>, DocResp, undefined),
- {Id, [Rev]}
- end,
- BodyJson
- ),
-
- ok = config:set("purge", "max_document_id_number", "101"),
- try
- {ok, Status, _, _} = test_request:post(
- Url ++ "/_purge/",
- [?CONTENT_JSON, ?AUTH],
- ?JSON_ENCODE({PurgeBody})
- ),
- ?assert(Status =:= 201 orelse Status =:= 202)
- after
- ok = config:delete("purge", "max_document_id_number")
- end
- end).
-
-test_accepted_purge_request(Url) ->
- ?_test(begin
- {ok, _, _, Body} = create_doc(Url, "doc1"),
- {Json} = ?JSON_DECODE(Body),
- Rev1 = couch_util:get_value(<<"rev">>, Json, undefined),
- IdsRevsEJson =
- {[
- {<<"doc1">>, [Rev1]}
- ]},
- IdsRevs = binary_to_list(?JSON_ENCODE(IdsRevsEJson)),
+t_purge_only_post_allowed(DbUrl) ->
+ {Status, Response} = req(put, url(DbUrl, "_purge")),
+ ?assertMatch(#{<<"reason">> := <<"Only POST allowed">>}, Response),
+ ?assert(Status =:= 405).
+
+t_empty_purge_request(DbUrl) ->
+ {Status, Response} = req(post, url(DbUrl, "_purge"), #{}),
+ ?assertMatch(#{<<"purge_seq">> := null, <<"purged">> := #{}}, Response),
+ ?assert(Status =:= 201 orelse Status =:= 202).
+
+t_ok_purge_request(DbUrl) ->
+ {201, Response1} = create_docs(DbUrl, docs(3)),
+ IdsRevs = ids_revs(Response1),
+ {Status, Response2} = req(post, url(DbUrl, "_purge"), IdsRevs),
+ ?assertMatch(#{<<"purge_seq">> := null, <<"purged">> := IdsRevs},
Response2),
+ ?assert(Status =:= 201 orelse Status =:= 202).
+
+t_ok_purge_with_max_document_id_number(DbUrl) ->
+ PurgedDocsNum = 101,
+ {201, Response1} = create_docs(DbUrl, docs(PurgedDocsNum)),
+ IdsRevs = ids_revs(Response1),
+
+ {400, #{<<"reason">> := Error}} = req(post, url(DbUrl, "_purge"), IdsRevs),
+ ?assertEqual(<<"Exceeded maximum number of documents.">>, Error),
+
+ ok = config:set("purge", "max_document_id_number", "101", _Persist =
false),
+ try
+ {Status, Response2} = req(post, url(DbUrl, "_purge"), IdsRevs),
+ ?assertMatch(#{<<"purge_seq">> := null, <<"purged">> := IdsRevs},
Response2),
+ ?assert(Status =:= 201 orelse Status =:= 202)
+ after
+ ok = config:delete("purge", "max_document_id_number", _Persist)
+ end.
+
+t_accepted_purge_request(DbUrl) ->
+ try
meck:new(fabric, [passthrough]),
- meck:expect(
- fabric,
- purge_docs,
- fun(_, _, _) ->
+ meck:expect(fabric, purge_docs, fun(_, _, _) ->
+ {accepted, [
{accepted, [
- {accepted, [
- {1,
- <<57, 27, 64, 134, 152, 18, 73, 243, 40, 1, 141,
214, 135, 104, 79,
- 188>>}
- ]}
+ {1, <<187, 82, 160, 135, 14, 97, 52, 47, 28, 172, 13, 249,
96, 182, 127, 97>>}
]}
- end
- ),
- {ok, Status, _, ResultBody} = test_request:post(
- Url ++ "/_purge/",
- [?CONTENT_JSON, ?AUTH],
- IdsRevs
- ),
- ResultJson = ?JSON_DECODE(ResultBody),
- meck:unload(fabric),
- ?assert(Status =:= 202),
- ?assertEqual(
- {[
- {<<"purge_seq">>, null},
- {<<"purged">>,
- {[
- {<<"doc1">>, [Rev1]}
- ]}}
- ]},
- ResultJson
- )
- end).
-
-test_partial_purge_request(Url) ->
- ?_test(begin
- {ok, _, _, Body} = create_doc(Url, "doc1"),
- {Json} = ?JSON_DECODE(Body),
- Rev1 = couch_util:get_value(<<"rev">>, Json, undefined),
-
- NewDoc =
- "{\"new_edits\": false, \"docs\": [{\"_id\": \"doc1\",\n"
- " \"_revisions\": {\"start\": 1, \"ids\": [\"12345\",
\"67890\"]},\n"
- " \"content\": \"updated\", \"_rev\": \"" ++ ?b2l(Rev1)
++ "\"}]}",
- {ok, _, _, _} = test_request:post(
- Url ++ "/_bulk_docs/",
- [?CONTENT_JSON, ?AUTH],
- NewDoc
- ),
-
- IdsRevsEJson = {[{<<"doc1">>, [Rev1]}]},
- IdsRevs = binary_to_list(?JSON_ENCODE(IdsRevsEJson)),
- {ok, Status, _, ResultBody} = test_request:post(
- Url ++ "/_purge/",
- [?CONTENT_JSON, ?AUTH],
- IdsRevs
- ),
- ResultJson = ?JSON_DECODE(ResultBody),
- ?assert(Status =:= 201 orelse Status =:= 202),
- ?assertEqual(
- {[
- {<<"purge_seq">>, null},
- {<<"purged">>,
- {[
- {<<"doc1">>, [Rev1]}
- ]}}
- ]},
- ResultJson
- ),
- {ok, Status2, _, ResultBody2} = test_request:get(
- Url ++
- "/doc1/",
- [?AUTH]
- ),
- {Json2} = ?JSON_DECODE(ResultBody2),
- Content = couch_util:get_value(<<"content">>, Json2, undefined),
- ?assertEqual(<<"updated">>, Content),
- ?assert(Status2 =:= 200)
- end).
-
-test_mixed_purge_request(Url) ->
- ?_test(begin
- {ok, _, _, Body} = create_doc(Url, "doc1"),
- {Json} = ?JSON_DECODE(Body),
- Rev1 = couch_util:get_value(<<"rev">>, Json, undefined),
-
- NewDoc =
- "{\"new_edits\": false, \"docs\": [{\"_id\": \"doc1\",\n"
- " \"_revisions\": {\"start\": 1, \"ids\": [\"12345\",
\"67890\"]},\n"
- " \"content\": \"updated\", \"_rev\": \"" ++ ?b2l(Rev1)
++ "\"}]}",
- {ok, _, _, _} = test_request:post(
- Url ++ "/_bulk_docs/",
- [?CONTENT_JSON, ?AUTH],
- NewDoc
- ),
-
- {ok, _, _, _Body2} = create_doc(Url, "doc2", "content2"),
- {ok, _, _, Body3} = create_doc(Url, "doc3", "content3"),
- {Json3} = ?JSON_DECODE(Body3),
- Rev3 = couch_util:get_value(<<"rev">>, Json3, undefined),
-
- IdsRevsEJson =
- {[
- % partial purge
- {<<"doc1">>, [Rev1]},
- % correct format, but invalid rev
- {<<"doc2">>, [Rev3, Rev1]},
- % correct format and rev
- {<<"doc3">>, [Rev3]}
- ]},
- IdsRevs = binary_to_list(?JSON_ENCODE(IdsRevsEJson)),
- {ok, Status, _, Body4} = test_request:post(
- Url ++ "/_purge/",
- [?CONTENT_JSON, ?AUTH],
- IdsRevs
- ),
- ResultJson = ?JSON_DECODE(Body4),
- ?assert(Status =:= 201 orelse Status =:= 202),
- ?assertEqual(
- {[
- {<<"purge_seq">>, null},
- {<<"purged">>,
- {[
- {<<"doc1">>, [Rev1]},
- {<<"doc2">>, []},
- {<<"doc3">>, [Rev3]}
- ]}}
- ]},
- ResultJson
- ),
- {ok, Status2, _, Body5} = test_request:get(
- Url ++
- "/doc1/",
- [?AUTH]
- ),
- {Json5} = ?JSON_DECODE(Body5),
- Content = couch_util:get_value(<<"content">>, Json5, undefined),
- ?assertEqual(<<"updated">>, Content),
- ?assert(Status2 =:= 200)
- end).
-
-test_overmany_ids_or_revs_purge_request(Url) ->
- ?_test(begin
- {ok, _, _, Body} = create_doc(Url, "doc1"),
- {Json} = ?JSON_DECODE(Body),
- Rev1 = couch_util:get_value(<<"rev">>, Json, undefined),
-
- NewDoc =
- "{\"new_edits\": false, \"docs\": [{\"_id\": \"doc1\",\n"
- " \"_revisions\": {\"start\": 1, \"ids\": [\"12345\",
\"67890\"]},\n"
- " \"content\": \"updated\", \"_rev\": \"" ++ ?b2l(Rev1)
++ "\"}]}",
- {ok, _, _, _} = test_request:post(
- Url ++ "/_bulk_docs/",
- [?CONTENT_JSON, ?AUTH],
- NewDoc
- ),
-
- {ok, _, _, _Body2} = create_doc(Url, "doc2", "content2"),
- {ok, _, _, Body3} = create_doc(Url, "doc3", "content3"),
- {Json3} = ?JSON_DECODE(Body3),
- Rev3 = couch_util:get_value(<<"rev">>, Json3, undefined),
-
- IdsRevsEJson =
- {[
- % partial purge
- {<<"doc1">>, [Rev1]},
- % correct format, but invalid rev
- {<<"doc2">>, [Rev3, Rev1]},
- % correct format and rev
- {<<"doc3">>, [Rev3]}
- ]},
- IdsRevs = binary_to_list(?JSON_ENCODE(IdsRevsEJson)),
-
- % Ids larger than expected
- config:set("purge", "max_document_id_number", "1"),
- {ok, Status, _, Body4} = test_request:post(
- Url ++ "/_purge/",
- [?CONTENT_JSON, ?AUTH],
- IdsRevs
- ),
- config:delete("purge", "max_document_id_number"),
- ResultJson = ?JSON_DECODE(Body4),
- ?assertEqual(400, Status),
- ?assertMatch(
- {[
- {<<"error">>, <<"bad_request">>},
- {<<"reason">>, <<"Exceeded maximum number of documents.">>}
- ]},
- ResultJson
- ),
-
- % Revs larger than expected
- config:set("purge", "max_revisions_number", "1"),
- {ok, Status2, _, Body5} = test_request:post(
- Url ++ "/_purge/",
- [?CONTENT_JSON, ?AUTH],
- IdsRevs
- ),
- config:delete("purge", "max_revisions_number"),
- ResultJson2 = ?JSON_DECODE(Body5),
- ?assertEqual(400, Status2),
- ?assertMatch(
- {[
- {<<"error">>, <<"bad_request">>},
- {<<"reason">>, <<"Exceeded maximum number of revisions.">>}
- ]},
- ResultJson2
- )
- end).
-
-test_exceed_limits_on_purge_infos(Url) ->
- ?_test(begin
- {ok, Status1, _, _} = test_request:put(
- Url ++ "/_purged_infos_limit/",
- [?CONTENT_JSON, ?AUTH],
- "2"
- ),
- ?assert(Status1 =:= 200),
-
- {ok, _, _, Body} = create_doc(Url, "doc1"),
- {Json} = ?JSON_DECODE(Body),
- Rev1 = couch_util:get_value(<<"rev">>, Json, undefined),
- {ok, _, _, Body2} = create_doc(Url, "doc2"),
- {Json2} = ?JSON_DECODE(Body2),
- Rev2 = couch_util:get_value(<<"rev">>, Json2, undefined),
- {ok, _, _, Body3} = create_doc(Url, "doc3"),
- {Json3} = ?JSON_DECODE(Body3),
- Rev3 = couch_util:get_value(<<"rev">>, Json3, undefined),
-
- IdsRevsEJson =
- {[
- {<<"doc1">>, [Rev1]},
- {<<"doc2">>, [Rev2]},
- {<<"doc3">>, [Rev3]}
- ]},
- IdsRevs = binary_to_list(?JSON_ENCODE(IdsRevsEJson)),
-
- {ok, Status2, _, ResultBody} = test_request:post(
- Url ++ "/_purge/",
- [?CONTENT_JSON, ?AUTH],
- IdsRevs
- ),
-
- ResultJson = ?JSON_DECODE(ResultBody),
- ?assert(Status2 =:= 201 orelse Status2 =:= 202),
- ?assertEqual(
- {[
- {<<"purge_seq">>, null},
- {<<"purged">>,
- {[
- {<<"doc1">>, [Rev1]},
- {<<"doc2">>, [Rev2]},
- {<<"doc3">>, [Rev3]}
- ]}}
- ]},
- ResultJson
- )
- end).
-
-should_error_set_purged_docs_limit_to0(Url) ->
- ?_test(begin
- {ok, Status, _, _} = test_request:put(
- Url ++ "/_purged_infos_limit/",
- [?CONTENT_JSON, ?AUTH],
- "0"
- ),
- ?assert(Status =:= 400)
- end).
-
-test_timeout_set_purged_infos_limit(Url) ->
- ?_test(begin
- meck:new(fabric, [passthrough]),
- meck:expect(fabric, set_purge_infos_limit, fun(_, _, _) ->
- {error, timeout}
+ ]}
end),
- {ok, Status, _, ResultBody} = test_request:put(
- Url ++
- "/_purged_infos_limit/",
- [?CONTENT_JSON, ?AUTH],
- "2"
- ),
- meck:unload(fabric),
- ResultJson = ?JSON_DECODE(ResultBody),
- ?assert(Status =:= 500),
- ?assertMatch(
- {[
- {<<"error">>, <<"error">>},
- {<<"reason">>, <<"timeout">>}
- ]},
- ResultJson
- )
- end).
+ {_, IdsRevs} = get_id_rev_map(DbUrl, "doc1"),
+ {Status, Response} = req(post, url(DbUrl, "_purge"), IdsRevs),
+ ?assertMatch(#{<<"purge_seq">> := null, <<"purged">> := IdsRevs},
Response),
+ ?assert(Status =:= 202)
+ after
+ meck:unload(fabric)
+ end.
+
+t_partial_purge_request(DbUrl) ->
+ IdsRevs = create_and_update_doc(DbUrl, "doc1"),
+ {Status1, Response1} = req(post, url(DbUrl, "_purge"), IdsRevs),
+ ?assertMatch(#{<<"purge_seq">> := null, <<"purged">> := IdsRevs},
Response1),
+ ?assert(Status1 =:= 201 orelse Status1 =:= 202),
+ {Status2, #{<<"content">> := Content}} = req(get, url(DbUrl, "doc1")),
+ ?assertEqual(<<"updated">>, Content),
+ ?assert(Status2 =:= 200).
+
+t_mixed_purge_request(DbUrl) ->
+ Doc1IdRevs = create_and_update_doc(DbUrl, "doc1"),
+ [Rev1] = maps:get(?l2b("doc1"), Doc1IdRevs),
+ get_id_rev_map(DbUrl, "doc2"),
+ {Rev3, _} = get_id_rev_map(DbUrl, "doc3"),
+
+ IdsRevs = #{
+ % partial purge
+ <<"doc1">> => [Rev1],
+ % correct format, but invalid rev
+ <<"doc2">> => [Rev1, Rev3],
+ % correct format and rev
+ <<"doc3">> => [Rev3]
+ },
+
+ {Status1, Response} = req(post, url(DbUrl, "_purge"), IdsRevs),
+ ?assertMatch(
+ #{
+ <<"purge_seq">> := null,
+ <<"purged">> := #{<<"doc1">> := [Rev1], <<"doc2">> := [],
<<"doc3">> := [Rev3]}
+ },
+ Response
+ ),
+ ?assert(Status1 =:= 201 orelse Status1 =:= 202),
+
+ {Status2, #{<<"content">> := Content}} = req(get, url(DbUrl, "doc1")),
+ ?assertEqual(<<"updated">>, Content),
+ ?assert(Status2 =:= 200).
+
+t_over_many_ids_or_revs_purge_request(DbUrl) ->
+ Doc1IdRevs = create_and_update_doc(DbUrl, "doc1"),
+ [Rev1] = maps:get(?l2b("doc1"), Doc1IdRevs),
+ get_id_rev_map(DbUrl, "doc2"),
+ {Rev3, _} = get_id_rev_map(DbUrl, "doc3"),
+
+ IdsRevs = #{
+ % partial purge
+ <<"doc1">> => [Rev1],
+ % correct format, but invalid rev
+ <<"doc2">> => [Rev1, Rev3],
+ % correct format and rev
+ <<"doc3">> => [Rev3]
+ },
+
+ % Ids larger than expected
+ config:set("purge", "max_document_id_number", "1", _Persist = false),
+ try
+ {Status1, #{<<"reason">> := Error1}} = req(post, url(DbUrl, "_purge"),
IdsRevs),
+ ?assertEqual(<<"Exceeded maximum number of documents.">>, Error1),
+ ?assertEqual(400, Status1)
+ after
+ config:delete("purge", "max_document_id_number", _Persist)
+ end,
+
+ % Revs larger than expected
+ config:set("purge", "max_revisions_number", "1", _Persist),
+ try
+ {Status2, #{<<"reason">> := Error2}} = req(post, url(DbUrl, "_purge"),
IdsRevs),
+ ?assertEqual(<<"Exceeded maximum number of revisions.">>, Error2),
+ ?assertEqual(400, Status2)
+ after
+ config:delete("purge", "max_revisions_number", _Persist)
+ end.
+
+t_purged_infos_limit_only_get_put_allowed(DbUrl) ->
+ {Status, Response} = req(post, url(DbUrl, "_purged_infos_limit"), "2"),
+ ?assertMatch(#{<<"reason">> := <<"Only GET,PUT allowed">>}, Response),
+ ?assert(Status =:= 405).
+
+t_exceed_limits_on_purge_infos(DbUrl) ->
+ {200, _} = req(put, url(DbUrl, "_purged_infos_limit"), "2"),
+ {201, Response1} = create_docs(DbUrl, docs(3)),
+ IdsRevs = ids_revs(Response1),
+
+ {Status2, Response2} = req(post, url(DbUrl, "_purge"), IdsRevs),
+ ?assertMatch(#{<<"purge_seq">> := null, <<"purged">> := IdsRevs},
Response2),
+ ?assert(Status2 =:= 201 orelse Status2 =:= 202).
+
+t_should_error_set_purged_docs_limit_to_0(DbUrl) ->
+ {Status, Res} = req(put, url(DbUrl, "_purged_infos_limit"), "0"),
+ ?assertMatch(
+ #{
+ <<"error">> := <<"bad_request">>,
+ <<"reason">> := <<"`purged_infos_limit` must be positive integer">>
+ },
+ Res
+ ),
+ ?assert(Status =:= 400).
+
+t_timeout_set_purged_infos_limit(DbUrl) ->
+ try
+ meck:new(fabric, [passthrough]),
+ meck:expect(fabric, set_purge_infos_limit, fun(_, _, _) -> {error,
timeout} end),
+ {Status, #{<<"reason">> := Error}} = req(put, url(DbUrl,
"_purged_infos_limit"), "2"),
+ ?assertEqual(<<"timeout">>, Error),
+ ?assert(Status =:= 500)
+ after
+ meck:unload(fabric)
+ end.
+
+t_purged_infos_only_get_allowed(DbUrl) ->
+ {Status, Response} = req(post, url(DbUrl, "_purged_infos")),
+ ?assertMatch(#{<<"reason">> := <<"Only GET allowed">>}, Response),
+ ?assert(Status =:= 405).
+
+t_empty_purged_infos(DbUrl) ->
+ {Status, Response} = req(get, url(DbUrl, "_purged_infos")),
+ ?assertMatch(#{<<"purged_infos">> := []}, Response),
+ ?assert(Status =:= 200).
+
+t_purged_infos_after_purge_request(DbUrl) ->
+ PurgedDocsNum = 3,
+ {Status1, Response1} = create_docs(DbUrl, docs(PurgedDocsNum)),
+ ?assert(Status1 =:= 201 orelse Status1 =:= 202),
+ IdsRevs = ids_revs(Response1),
+
+ {Status2, Response2} = req(post, url(DbUrl, "_purge"), IdsRevs),
+ ?assert(Status2 =:= 201 orelse Status2 =:= 202),
+ ?assertMatch(#{<<"purge_seq">> := null, <<"purged">> := IdsRevs},
Response2),
+
+ {Status3, Response3} = req(get, url(DbUrl, "_purged_infos")),
+ ?assert(Status3 =:= 200),
+ ?assertMatch(#{<<"purged_infos">> := [_ | _]}, Response3),
+ Info = maps:get(<<"purged_infos">>, Response3),
+ ?assertEqual(3, length(Info)).
+
+t_purged_infos_after_multiple_purge_requests(DbUrl) ->
+ {Rev1, Doc1IdRevs} = get_id_rev_map(DbUrl, "doc1"),
+ {Rev2, Doc2IdRevs} = get_id_rev_map(DbUrl, "doc2"),
+ {201, #{<<"rev">> := Rev1New}} =
+ req(put, url(DbUrl, "doc1?rev=" ++ ?b2l(Rev1)), #{<<"val">> =>
<<"updated">>}),
+ req(post, url(DbUrl, "_purge"), Doc1IdRevs),
+ req(post, url(DbUrl, "_purge"), Doc2IdRevs),
+ req(post, url(DbUrl, "_purge"), #{<<"doc1">> => [Rev1New]}),
+ {Status, #{<<"purged_infos">> := IdRevsList}} = req(get, url(DbUrl,
"_purged_infos")),
+ ?assert(lists:member(#{<<"id">> => <<"doc1">>, <<"revs">> => [Rev1]},
IdRevsList)),
+ ?assert(lists:member(#{<<"id">> => <<"doc1">>, <<"revs">> => [Rev1New]},
IdRevsList)),
+ ?assert(lists:member(#{<<"id">> => <<"doc2">>, <<"revs">> => [Rev2]},
IdRevsList)),
+ ?assert(Status =:= 200).
+
+%%%%%%%%%%%%%%%%%%%% Utility Functions %%%%%%%%%%%%%%%%%%%%
+url(Url, Path) ->
+ Url ++ "/" ++ Path.
+
+create_db(Url) ->
+ case req(put, Url) of
+ {201, #{}} -> ok;
+ Error -> error({failed_to_create_test_db, Error})
+ end.
+
+delete_db(Url) ->
+ case req(delete, Url) of
+ {200, #{}} -> ok;
+ Error -> error({failed_to_delete_test_db, Error})
+ end.
+
+create_doc(Url, Id) ->
+ req(put, url(Url, Id), #{<<"val">> => ?l2b(Id)}).
+
+create_docs(Url, Docs) ->
+ req(post, url(Url, "_bulk_docs"), #{<<"docs">> => Docs}).
+
+docs(Counter) ->
+ lists:foldl(
+ fun(I, Acc) ->
+ Id = ?l2b(integer_to_list(I)),
+ Doc = #{<<"_id">> => Id, <<"val">> => I},
+ [Doc | Acc]
+ end,
+ [],
+ lists:seq(1, Counter)
+ ).
+
+ids_revs(Response) ->
+ IdsRevs = lists:map(
+ fun(DocResp) ->
+ Id = maps:get(<<"id">>, DocResp),
+ Rev = maps:get(<<"rev">>, DocResp),
+ {Id, [Rev]}
+ end,
+ Response
+ ),
+ maps:from_list(IdsRevs).
+
+get_id_rev_map(Url, Id) ->
+ {_, #{<<"rev">> := Rev}} = create_doc(Url, Id),
+ {Rev, #{?l2b(Id) => [Rev]}}.
+
+create_and_update_doc(Url, Id) ->
+ {Rev, IdRevs} = get_id_rev_map(Url, Id),
+ NewDoc = new_doc(Rev),
+ {201, _} = req(post, url(Url, "_bulk_docs"), NewDoc),
+ IdRevs.
+
+new_doc(Rev) ->
+ #{
+ <<"new_edits">> => false,
+ <<"docs">> => [
+ #{
+ <<"_id">> => <<"doc1">>,
+ <<"_revisions">> => #{
+ <<"start">> => 1,
+ <<"ids">> => [<<"12345">>, <<"67890">>]
+ },
+ <<"content">> => <<"updated">>,
+ <<"_rev">> => Rev
+ }
+ ]
+ }.
+
+req(Method, Url) ->
+ Headers = [?CONTENT_JSON, ?AUTH],
+ {ok, Code, _, Res} = test_request:request(Method, Url, Headers),
+ {Code, jiffy:decode(Res, [return_maps])}.
+
+req(Method, Url, #{} = Body) ->
+ req(Method, Url, jiffy:encode(Body));
+req(Method, Url, Body) ->
+ Headers = [?CONTENT_JSON, ?AUTH],
+ {ok, Code, _, Res} = test_request:request(Method, Url, Headers, Body),
+ {Code, jiffy:decode(Res, [return_maps])}.
diff --git a/src/docs/src/api/database/misc.rst
b/src/docs/src/api/database/misc.rst
index 288b04871..d2e974cdc 100644
--- a/src/docs/src/api/database/misc.rst
+++ b/src/docs/src/api/database/misc.rst
@@ -175,6 +175,56 @@ following behavior:
record of the document will be deleted. The document will no longer be found
in searches.
+.. _api/db/_purged_infos:
+
+==============================
+``/{db}/_purged_infos``
+==============================
+
+.. http:get:: /{db}/_purged_infos
+ :synopsis: Returns a history list of purged document IDs and revisions
+
+ Get a list of purged document IDs and revisions stored in the database.
+
+ :param db: Database name
+ :<header Accept: - :mimetype:`application/json`
+ - :mimetype:`text/plain`
+ :>header Content-Type: - :mimetype:`application/json`
+ - :mimetype:`text/plain; charset=utf-8`
+ :code 200: Request completed successfully
+ :code 400: Invalid database name
+
+ **Request**:
+
+ .. code-block:: http
+
+ GET /db/_purged_infos HTTP/1.1
+ Accept: application/json
+ Host: localhost:5984
+
+ **Response**:
+
+ .. code-block:: http
+
+ HTTP/1.1 200 OK
+ Cache-Control: must-revalidate
+ Content-Length: 75
+ Content-Type: application/json
+ Date: Thu, 24 Aug 2023 20:56:06 GMT
+ Server: CouchDB (Erlang/OTP)
+
+ {
+ "purged_infos": [
+ {
+ "id": "doc_id",
+ "revs": [
+ "1-85cfcb946ba8fea03ba81ec38a7a9998",
+ "2-c6548393a891f2cec9c7755832ff9d6f"
+ ]
+ }
+ ]
+ }
+
.. _api/db/_purged_infos_limit:
==============================
diff --git a/src/fabric/src/fabric.erl b/src/fabric/src/fabric.erl
index dfd433d38..c36c8f4df 100644
--- a/src/fabric/src/fabric.erl
+++ b/src/fabric/src/fabric.erl
@@ -31,6 +31,7 @@
get_all_security/1, get_all_security/2,
get_purge_infos_limit/1,
set_purge_infos_limit/3,
+ get_purged_infos/1,
compact/1, compact/2,
get_partition_info/2
]).
@@ -211,6 +212,12 @@ get_purge_infos_limit(DbName) ->
catch couch_db:close(Db)
end.
+%% @doc returns purged requests history for the given database
+-spec get_purged_infos(dbname()) ->
+ {ok, [{purged_infos, json_obj()}]} | {error, Reason :: term()}.
+get_purged_infos(Db) ->
+ fabric_db_purged_infos:go(dbname(Db)).
+
get_security(DbName) ->
get_security(DbName, [?ADMIN_CTX]).
diff --git a/src/fabric/src/fabric_db_purged_infos.erl
b/src/fabric/src/fabric_db_purged_infos.erl
new file mode 100644
index 000000000..a85897bcb
--- /dev/null
+++ b/src/fabric/src/fabric_db_purged_infos.erl
@@ -0,0 +1,79 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(fabric_db_purged_infos).
+
+-export([go/1]).
+
+-include_lib("fabric/include/fabric.hrl").
+-include_lib("mem3/include/mem3.hrl").
+
+-record(pacc, {
+ counters,
+ replies,
+ ring_opts
+}).
+
+go(DbName) ->
+ Shards = mem3:shards(DbName),
+ Workers = fabric_util:submit_jobs(Shards, get_purged_infos, []),
+ RexiMon = fabric_util:create_monitors(Shards),
+ Fun = fun handle_message/3,
+ Acc0 = #pacc{
+ counters = fabric_dict:init(Workers, nil),
+ replies = [],
+ ring_opts = [{any, Shards}]
+ },
+ try
+ case fabric_util:recv(Workers, #shard.ref, Fun, Acc0) of
+ {ok, Res} ->
+ {ok, Res};
+ {timeout, {WorkersDict, _}} ->
+ DefunctWorkers = fabric_util:remove_done_workers(WorkersDict,
nil),
+ fabric_util:log_timeout(DefunctWorkers, "get_purged_infos"),
+ {error, timeout};
+ {error, Error} ->
+ throw(Error)
+ end
+ after
+ rexi_monitor:stop(RexiMon)
+ end.
+
+handle_message({rexi_DOWN, _, {_, NodeRef}, _}, _Shard, #pacc{} = Acc) ->
+ #pacc{counters = Counters, ring_opts = RingOpts} = Acc,
+ case fabric_util:remove_down_workers(Counters, NodeRef, RingOpts) of
+ {ok, NewCounters} ->
+ {ok, Acc#pacc{counters = NewCounters}};
+ error ->
+ {error, {nodedown, <<"progress not possible">>}}
+ end;
+handle_message({rexi_EXIT, Reason}, Shard, #pacc{} = Acc) ->
+ #pacc{counters = Counters, ring_opts = RingOpts} = Acc,
+ NewCounters = fabric_dict:erase(Shard, Counters),
+ case fabric_ring:is_progress_possible(NewCounters, RingOpts) of
+ true ->
+ {ok, Acc#pacc{counters = NewCounters}};
+ false ->
+ {error, Reason}
+ end;
+handle_message({ok, Info}, #shard{} = Shard, #pacc{} = Acc) ->
+ #pacc{counters = Counters, replies = Replies} = Acc,
+ Replies1 = [Info | Replies],
+ Counters1 = fabric_dict:erase(Shard, Counters),
+ case fabric_dict:size(Counters1) =:= 0 of
+ true ->
+ {stop, lists:flatten(Replies1)};
+ false ->
+ {ok, Acc#pacc{counters = Counters1, replies = Replies1}}
+ end;
+handle_message(_, _, #pacc{} = Acc) ->
+ {ok, Acc}.
diff --git a/src/fabric/src/fabric_rpc.erl b/src/fabric/src/fabric_rpc.erl
index fa6ea5116..d01f1f5a7 100644
--- a/src/fabric/src/fabric_rpc.erl
+++ b/src/fabric/src/fabric_rpc.erl
@@ -39,7 +39,7 @@
]).
-export([get_all_security/2, open_shard/2]).
-export([compact/1, compact/2]).
--export([get_purge_seq/2, purge_docs/3, set_purge_infos_limit/3]).
+-export([get_purge_seq/2, get_purged_infos/1, purge_docs/3,
set_purge_infos_limit/3]).
-export([
get_db_info/2,
@@ -299,6 +299,10 @@ update_docs(DbName, Docs0, Options) ->
Docs2 = make_att_readers(Docs1),
with_db(DbName, Options, {couch_db, update_docs, [Docs2, Options, Type]}).
+get_purged_infos(DbName) ->
+ FoldFun = fun({_Seq, _UUID, Id, Revs}, Acc) -> {ok, [{Id, Revs} | Acc]}
end,
+ with_db(DbName, [], {couch_db, fold_purge_infos, [0, FoldFun, []]}).
+
get_purge_seq(DbName, Options) ->
with_db(DbName, Options, {couch_db, get_purge_seq, []}).
