[couchdb] 01/01: Merge pull request #1370 from apache/COUCHDB-3326-clustered-purge-pr5-implementation
This is an automated email from the ASF dual-hosted git repository. jiangphcn pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/couchdb.git commit f82b156e20ec5fd82be191dbf6044fca0b5f0386 Merge: 2d3f935 73ac8d1 Author: Peng Hui Jiang AuthorDate: Wed Aug 22 01:31:35 2018 +0800 Merge pull request #1370 from apache/COUCHDB-3326-clustered-purge-pr5-implementation [5/5] Clustered Purge Implementation src/chttpd/src/chttpd_db.erl | 58 ++- src/chttpd/test/chttpd_purge_tests.erl | 320 src/couch/priv/stats_descriptions.cfg | 12 + src/couch/src/couch_bt_engine.erl | 251 +++-- src/couch/src/couch_bt_engine.hrl | 4 +- src/couch/src/couch_bt_engine_compactor.erl| 122 - src/couch/src/couch_bt_engine_header.erl | 35 +- src/couch/src/couch_db.erl | 157 +- src/couch/src/couch_db_engine.erl | 167 -- src/couch/src/couch_db_plugin.erl | 6 + src/couch/src/couch_db_updater.erl | 185 --- src/couch/src/couch_httpd_db.erl | 23 +- src/couch/test/couch_bt_engine_upgrade_tests.erl | 220 src/couch/test/fixtures/db_with_1_purge_req.couch | Bin 0 -> 12470 bytes .../fixtures/db_with_1_purge_req_for_2_docs.couch | Bin 0 -> 16557 bytes src/couch/test/fixtures/db_with_2_purge_req.couch | Bin 0 -> 16566 bytes src/couch/test/fixtures/db_without_purge_req.couch | Bin 0 -> 61644 bytes src/couch_index/src/couch_index_epi.erl| 5 +- .../src/couch_index_plugin_couch_db.erl} | 26 +- src/couch_index/src/couch_index_updater.erl| 50 +- src/couch_mrview/src/couch_mrview_cleanup.erl | 16 +- src/couch_mrview/src/couch_mrview_index.erl| 115 - src/couch_mrview/src/couch_mrview_test_util.erl| 5 + src/couch_mrview/src/couch_mrview_updater.erl | 14 +- src/couch_mrview/src/couch_mrview_util.erl | 39 +- .../test/couch_mrview_purge_docs_fabric_tests.erl | 276 ++ .../test/couch_mrview_purge_docs_tests.erl | 506 ++ src/couch_pse_tests/src/cpse_test_compaction.erl | 143 +- .../src/cpse_test_fold_purge_infos.erl | 166 ++ .../src/cpse_test_get_set_props.erl| 3 +- .../src/cpse_test_purge_bad_checkpoints.erl| 80 +++ src/couch_pse_tests/src/cpse_test_purge_docs.erl | 506 ++ .../src/cpse_test_purge_replication.erl| 202 src/couch_pse_tests/src/cpse_test_purge_seqs.erl | 124 + src/couch_pse_tests/src/cpse_util.erl | 183 ++- src/fabric/src/fabric.erl | 27 +- src/fabric/src/fabric_db_info.erl | 29 +- src/fabric/src/fabric_db_meta.erl | 26 +- src/fabric/src/fabric_doc_open.erl | 73 ++- src/fabric/src/fabric_doc_open_revs.erl| 262 -- src/fabric/src/fabric_doc_purge.erl| 572 + src/fabric/src/fabric_rpc.erl | 128 - src/fabric/test/fabric_rpc_purge_tests.erl | 285 ++ src/mem3/src/mem3_epi.erl | 3 +- .../src/mem3_plugin_couch_db.erl} | 21 +- src/mem3/src/mem3_rep.erl | 206 +++- src/mem3/src/mem3_rpc.erl | 71 ++- test/javascript/tests/erlang_views.js | 5 +- test/javascript/tests/purge.js | 27 +- 49 files changed, 5260 insertions(+), 494 deletions(-)
[couchdb] branch master updated (2d3f935 -> f82b156)
This is an automated email from the ASF dual-hosted git repository.
jiangphcn pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/couchdb.git.
from 2d3f935 Merge pull request #1369 from
apache/COUCHDB-3326-clustered-purge-pr4-on-compact-plugin
add febb0fc [01/10] Clustered Purge: Define new purge API
add eca282e [02/10] Clustered Purge: Update single node APIs
add bccc39d [03/10] Clustered Purge: Update couch_bt_engine
add 8a9112f [04/10] Clustered Purge: Update couch_pse_tests
add 48378a9 [05/10] Clustered Purge: Add upgrade tests
add 3be455b [06/10] Clustered Purge: Update mrview indexes
add c625933 [07/10] Clustered Purge: Internal replication
add 0234679 [08/10] Clustered Purge: Update read-repair
add 932dd93 [09/10] Clustered Purge: Fabric API
add 73ac8d1 [10/10] Clustered Purge: Clustered HTTP API
new f82b156 Merge pull request #1370 from
apache/COUCHDB-3326-clustered-purge-pr5-implementation
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
src/chttpd/src/chttpd_db.erl | 58 ++-
src/chttpd/test/chttpd_purge_tests.erl | 320
src/couch/priv/stats_descriptions.cfg | 12 +
src/couch/src/couch_bt_engine.erl | 251 +++--
src/couch/src/couch_bt_engine.hrl | 4 +-
src/couch/src/couch_bt_engine_compactor.erl| 122 -
src/couch/src/couch_bt_engine_header.erl | 35 +-
src/couch/src/couch_db.erl | 157 +-
src/couch/src/couch_db_engine.erl | 167 --
src/couch/src/couch_db_plugin.erl | 6 +
src/couch/src/couch_db_updater.erl | 185 ---
src/couch/src/couch_httpd_db.erl | 23 +-
src/couch/test/couch_bt_engine_upgrade_tests.erl | 220
.../{test.couch => db_with_1_purge_req.couch} | Bin 16482 -> 12470 bytes
couch => db_with_1_purge_req_for_2_docs.couch} | Bin 16482 -> 16557 bytes
.../{test.couch => db_with_2_purge_req.couch} | Bin 16482 -> 16566 bytes
src/couch/test/fixtures/db_without_purge_req.couch | Bin 0 -> 61644 bytes
src/couch_index/src/couch_index_epi.erl| 5 +-
.../src/couch_index_plugin_couch_db.erl} | 15 +-
src/couch_index/src/couch_index_updater.erl| 50 +-
src/couch_mrview/src/couch_mrview_cleanup.erl | 16 +-
src/couch_mrview/src/couch_mrview_index.erl| 115 -
src/couch_mrview/src/couch_mrview_test_util.erl| 5 +
src/couch_mrview/src/couch_mrview_updater.erl | 14 +-
src/couch_mrview/src/couch_mrview_util.erl | 39 +-
.../test/couch_mrview_purge_docs_fabric_tests.erl | 276 ++
.../test/couch_mrview_purge_docs_tests.erl | 506 ++
src/couch_pse_tests/src/cpse_test_compaction.erl | 143 +-
.../src/cpse_test_fold_purge_infos.erl | 166 ++
.../src/cpse_test_get_set_props.erl| 3 +-
.../src/cpse_test_purge_bad_checkpoints.erl| 80 +++
src/couch_pse_tests/src/cpse_test_purge_docs.erl | 506 ++
.../src/cpse_test_purge_replication.erl| 202
src/couch_pse_tests/src/cpse_test_purge_seqs.erl | 124 +
src/couch_pse_tests/src/cpse_util.erl | 183 ++-
src/fabric/src/fabric.erl | 27 +-
src/fabric/src/fabric_db_info.erl | 29 +-
src/fabric/src/fabric_db_meta.erl | 26 +-
src/fabric/src/fabric_doc_open.erl | 73 ++-
src/fabric/src/fabric_doc_open_revs.erl| 262 --
src/fabric/src/fabric_doc_purge.erl| 572 +
src/fabric/src/fabric_rpc.erl | 128 -
src/fabric/test/fabric_rpc_purge_tests.erl | 285 ++
src/mem3/src/mem3_epi.erl | 3 +-
.../src/mem3_plugin_couch_db.erl} | 8 +-
src/mem3/src/mem3_rep.erl | 206 +++-
src/mem3/src/mem3_rpc.erl | 71 ++-
test/javascript/tests/erlang_views.js | 5 +-
test/javascript/tests/purge.js | 27 +-
49 files changed, 5248 insertions(+), 482 deletions(-)
create mode 100644 src/chttpd/test/chttpd_purge_tests.erl
create mode 100644 src/couch/test/couch_bt_engine_upgrade_tests.erl
copy src/couch/test/fixtures/{test.couch => db_with_1_purge_req.couch} (66%)
copy src/couch/test/fixtures/{test.couch =>
db_with_1_purge_req_for_2_docs.couch} (83%)
copy src/couch/test/fixtures/{test.couch => db_with_2_purge_req.couch} (86%)
create mode 100644 src/couch/test/fixtures/db_without_purge_req.couch
copy
[couchdb] 07/10: [07/10] Clustered Purge: Internal replication
This is an automated email from the ASF dual-hosted git repository.
jiangphcn pushed a commit to branch
COUCHDB-3326-clustered-purge-pr5-implementation
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit c625933fc3c69d300da3d6dbb2ffb5d3c8a26499
Author: Paul J. Davis
AuthorDate: Tue Apr 24 12:27:14 2018 -0500
[07/10] Clustered Purge: Internal replication
This commit implements the internal replication of purge requests. This
part of the anit-entropy process is important for ensuring that shard
copies continue to be eventually consistent even if updates happen to
shards independently due to a network split or other event that prevents
the successful purge request to a given copy.
The main addition to internal replication is that we both pull and push
purge requests between the source and target shards. The push direction
is obvious given that internal replication is in the push direction
already. Pull isn't quite as obvious but is required so that we don't
push an update that was already purged on the target.
Of note is that internal replication also has to maintain _local doc
checkpoints to prevent compaction removing old purge requests or else
shard copies could end up missing purge requests which would prevent the
shard copies from ever reaching a consistent state.
COUCHDB-3326
Co-authored-by: Mayya Sharipova
Co-authored-by: jiangphcn
---
.../src/cpse_test_purge_replication.erl| 202
src/couch_pse_tests/src/cpse_util.erl | 32 +++-
src/mem3/src/mem3_epi.erl | 3 +-
.../src/{mem3_epi.erl => mem3_plugin_couch_db.erl} | 39 +---
src/mem3/src/mem3_rep.erl | 206 +++--
src/mem3/src/mem3_rpc.erl | 71 ++-
6 files changed, 499 insertions(+), 54 deletions(-)
diff --git a/src/couch_pse_tests/src/cpse_test_purge_replication.erl
b/src/couch_pse_tests/src/cpse_test_purge_replication.erl
new file mode 100644
index 000..fb09eeb
--- /dev/null
+++ b/src/couch_pse_tests/src/cpse_test_purge_replication.erl
@@ -0,0 +1,202 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(cpse_test_purge_replication).
+-compile(export_all).
+-compile(nowarn_export_all).
+
+
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("mem3/include/mem3.hrl").
+
+
+setup_all() ->
+cpse_util:setup_all([mem3, fabric, couch_replicator]).
+
+
+setup_each() ->
+{ok, Src} = cpse_util:create_db(),
+{ok, Tgt} = cpse_util:create_db(),
+{couch_db:name(Src), couch_db:name(Tgt)}.
+
+
+teardown_each({SrcDb, TgtDb}) ->
+ok = couch_server:delete(SrcDb, []),
+ok = couch_server:delete(TgtDb, []).
+
+
+cpse_purge_http_replication({Source, Target}) ->
+{ok, Rev1} = cpse_util:save_doc(Source, {[{'_id', foo}, {vsn, 1}]}),
+
+cpse_util:assert_db_props(?MODULE, ?LINE, Source, [
+{doc_count, 1},
+{del_doc_count, 0},
+{update_seq, 1},
+{changes, 1},
+{purge_seq, 0},
+{purge_infos, []}
+]),
+
+RepObject = {[
+{<<"source">>, Source},
+{<<"target">>, Target}
+]},
+
+{ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER),
+{ok, Doc1} = cpse_util:open_doc(Target, foo),
+
+cpse_util:assert_db_props(?MODULE, ?LINE, Target, [
+{doc_count, 1},
+{del_doc_count, 0},
+{update_seq, 1},
+{changes, 1},
+{purge_seq, 0},
+{purge_infos, []}
+]),
+
+PurgeInfos = [
+{cpse_util:uuid(), <<"foo">>, [Rev1]}
+],
+
+{ok, [{ok, PRevs}]} = cpse_util:purge(Source, PurgeInfos),
+?assertEqual([Rev1], PRevs),
+
+cpse_util:assert_db_props(?MODULE, ?LINE, Source, [
+{doc_count, 0},
+{del_doc_count, 0},
+{update_seq, 2},
+{changes, 0},
+{purge_seq, 1},
+{purge_infos, PurgeInfos}
+]),
+
+% Show that a purge on the source is
+% not replicated to the target
+{ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER),
+{ok, Doc2} = cpse_util:open_doc(Target, foo),
+[Rev2] = Doc2#doc_info.revs,
+?assertEqual(Rev1, Rev2#rev_info.rev),
+?assertEqual(Doc1, Doc2),
+
+cpse_util:assert_db_props(?MODULE, ?LINE, Target, [
+{doc_count, 1},
+{del_doc_count, 0},
+{update_seq, 1},
+
[couchdb] 03/10: [03/10] Clustered Purge: Update couch_bt_engine
This is an automated email from the ASF dual-hosted git repository.
jiangphcn pushed a commit to branch
COUCHDB-3326-clustered-purge-pr5-implementation
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit bccc39d4e61e087c8d4f144cf87e635ad8a5c495
Author: Paul J. Davis
AuthorDate: Tue Apr 24 12:25:43 2018 -0500
[03/10] Clustered Purge: Update couch_bt_engine
This commit updates the couch_bt_engine storage engine implementation to
satisfy the newly defined single-node purge APIs. This is accomplished
by storing two new database btrees.
The purge_seq_tree orders purge requests by their purge_seq. This tree
is used to satisfy the fold_purge_infos API for database components to
enumerate the list of purge requests in a defined order.
The second index is the purge_tree which orders purge requests by their
UUID to make for an efficient lookup when filtering replicated purge
requests.
COUCHDB-3326
Co-authored-by: Mayya Sharipova
Co-authored-by: jiangphcn
---
src/couch/src/couch_bt_engine.erl | 251
src/couch/src/couch_bt_engine.hrl | 4 +-
src/couch/src/couch_bt_engine_compactor.erl | 122 --
src/couch/src/couch_bt_engine_header.erl| 35 ++--
4 files changed, 344 insertions(+), 68 deletions(-)
diff --git a/src/couch/src/couch_bt_engine.erl
b/src/couch/src/couch_bt_engine.erl
index c5df11b..6d858ed 100644
--- a/src/couch/src/couch_bt_engine.erl
+++ b/src/couch/src/couch_bt_engine.erl
@@ -35,8 +35,9 @@
get_disk_version/1,
get_doc_count/1,
get_epochs/1,
-get_last_purged/1,
get_purge_seq/1,
+get_oldest_purge_seq/1,
+get_purge_infos_limit/1,
get_revs_limit/1,
get_security/1,
get_size_info/1,
@@ -44,15 +45,18 @@
get_uuid/1,
set_revs_limit/2,
+set_purge_infos_limit/2,
set_security/2,
open_docs/2,
open_local_docs/2,
read_doc_body/2,
+load_purge_infos/2,
serialize_doc/2,
write_doc_body/2,
-write_doc_infos/4,
+write_doc_infos/3,
+purge_docs/3,
commit_data/1,
@@ -63,6 +67,7 @@
fold_docs/4,
fold_local_docs/4,
fold_changes/5,
+fold_purge_infos/5,
count_changes_since/2,
start_compaction/4,
@@ -85,7 +90,13 @@
seq_tree_reduce/2,
local_tree_split/1,
-local_tree_join/2
+local_tree_join/2,
+
+purge_tree_split/1,
+purge_tree_join/2,
+purge_tree_reduce/2,
+purge_seq_tree_split/1,
+purge_seq_tree_join/2
]).
@@ -217,18 +228,24 @@ get_epochs(#st{header = Header}) ->
couch_bt_engine_header:get(Header, epochs).
-get_last_purged(#st{header = Header} = St) ->
-case couch_bt_engine_header:get(Header, purged_docs) of
-nil ->
-[];
-Pointer ->
-{ok, PurgeInfo} = couch_file:pread_term(St#st.fd, Pointer),
-PurgeInfo
-end.
+get_purge_seq(#st{purge_seq_tree = PurgeSeqTree}) ->
+Fun = fun({PurgeSeq, _, _, _}, _Reds, _Acc) ->
+{stop, PurgeSeq}
+end,
+{ok, _, PurgeSeq} = couch_btree:fold(PurgeSeqTree, Fun, 0, [{dir, rev}]),
+PurgeSeq.
+
+
+get_oldest_purge_seq(#st{purge_seq_tree = PurgeSeqTree}) ->
+Fun = fun({PurgeSeq, _, _, _}, _Reds, _Acc) ->
+{stop, PurgeSeq}
+end,
+{ok, _, PurgeSeq} = couch_btree:fold(PurgeSeqTree, Fun, 0, []),
+PurgeSeq.
-get_purge_seq(#st{header = Header}) ->
-couch_bt_engine_header:get(Header, purge_seq).
+get_purge_infos_limit(#st{header = Header}) ->
+couch_bt_engine_header:get(Header, purge_infos_limit).
get_revs_limit(#st{header = Header}) ->
@@ -284,6 +301,16 @@ set_revs_limit(#st{header = Header} = St, RevsLimit) ->
{ok, increment_update_seq(NewSt)}.
+set_purge_infos_limit(#st{header = Header} = St, PurgeInfosLimit) ->
+NewSt = St#st{
+header = couch_bt_engine_header:set(Header, [
+{purge_infos_limit, PurgeInfosLimit}
+]),
+needs_commit = true
+},
+{ok, increment_update_seq(NewSt)}.
+
+
set_security(#st{header = Header} = St, NewSecurity) ->
Options = [{compression, St#st.compression}],
{ok, Ptr, _} = couch_file:append_term(St#st.fd, NewSecurity, Options),
@@ -320,6 +347,14 @@ read_doc_body(#st{} = St, #doc{} = Doc) ->
}.
+load_purge_infos(St, UUIDs) ->
+Results = couch_btree:lookup(St#st.purge_tree, UUIDs),
+lists:map(fun
+({ok, Info}) -> Info;
+(not_found) -> not_found
+end, Results).
+
+
serialize_doc(#st{} = St, #doc{} = Doc) ->
Compress = fun(Term) ->
case couch_compress:is_compressed(Term, St#st.compression) of
@@ -351,7 +386,7 @@ write_doc_body(St, #doc{} = Doc) ->
{ok, Doc#doc{body = Ptr}, Written}.
-write_doc_infos(#st{} = St, Pairs, LocalDocs, PurgedIdRevs) ->
+write_doc_infos(#st{} = St, Pairs, LocalDocs) ->
#st{
id_tree = IdTree,
seq_tree = SeqTree,
@@ -391,23 +426,9
[couchdb] 05/10: [05/10] Clustered Purge: Add upgrade tests
This is an automated email from the ASF dual-hosted git repository.
jiangphcn pushed a commit to branch
COUCHDB-3326-clustered-purge-pr5-implementation
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 48378a97153006e75b9909064c67ce1391f266cf
Author: jiangph
AuthorDate: Tue Jul 10 14:58:49 2018 +0800
[05/10] Clustered Purge: Add upgrade tests
These test that we can successfully upgrade old databases that have
various configurations of purge requests in the legacy format.
COUCHDB-3326
Co-authored-by: Mayya Sharipova
Co-authored-by: jiangphcn
---
src/couch/test/couch_bt_engine_upgrade_tests.erl | 220 +
src/couch/test/fixtures/db_with_1_purge_req.couch | Bin 0 -> 12470 bytes
.../fixtures/db_with_1_purge_req_for_2_docs.couch | Bin 0 -> 16557 bytes
src/couch/test/fixtures/db_with_2_purge_req.couch | Bin 0 -> 16566 bytes
src/couch/test/fixtures/db_without_purge_req.couch | Bin 0 -> 61644 bytes
5 files changed, 220 insertions(+)
diff --git a/src/couch/test/couch_bt_engine_upgrade_tests.erl
b/src/couch/test/couch_bt_engine_upgrade_tests.erl
new file mode 100644
index 000..1d2a86d
--- /dev/null
+++ b/src/couch/test/couch_bt_engine_upgrade_tests.erl
@@ -0,0 +1,220 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_bt_engine_upgrade_tests).
+
+-include_lib("couch/include/couch_eunit.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+
+setup() ->
+Ctx = test_util:start_couch(),
+DbDir = config:get("couchdb", "database_dir"),
+DbFileNames = [
+"db_without_purge_req.couch",
+"db_with_1_purge_req.couch",
+"db_with_2_purge_req.couch",
+"db_with_1_purge_req_for_2_docs.couch"
+],
+NewPaths = lists:map(fun(DbFileName) ->
+OldDbFilePath = filename:join([?FIXTURESDIR, DbFileName]),
+NewDbFilePath = filename:join([DbDir, DbFileName]),
+ok = filelib:ensure_dir(NewDbFilePath),
+file:delete(NewDbFilePath),
+{ok, _} = file:copy(OldDbFilePath, NewDbFilePath),
+NewDbFilePath
+end, DbFileNames),
+{Ctx, NewPaths}.
+
+
+teardown({Ctx, Paths}) ->
+test_util:stop_couch(Ctx),
+lists:foreach(fun(Path) ->
+file:delete(Path)
+end, Paths).
+
+
+upgrade_test_() ->
+{
+"Couch Bt Engine Upgrade tests",
+{
+setup,
+fun setup/0,
+fun teardown/1,
+[
+t_upgrade_without_purge_req(),
+t_upgrade_with_1_purge_req(),
+t_upgrade_with_N_purge_req(),
+t_upgrade_with_1_purge_req_for_2_docs()
+]
+}
+}.
+
+
+t_upgrade_without_purge_req() ->
+?_test(begin
+% There are three documents in the fixture
+% db with zero purge entries
+DbName = <<"db_without_purge_req">>,
+
+{ok, UpgradedPurged} = couch_util:with_db(DbName, fun(Db) ->
+?assertEqual(0, couch_db:get_purge_seq(Db)),
+couch_db:fold_purge_infos(Db, 0, fun fold_fun/2, [])
+end),
+?assertEqual([], UpgradedPurged),
+
+{ok, Rev} = save_doc(
+DbName, {[{<<"_id">>, <<"doc4">>}, {<<"v">>, 1}]}
+),
+{ok, _} = save_doc(DbName, {[{<<"_id">>, <<"doc5">>}, {<<"v">>, 2}]}),
+
+couch_util:with_db(DbName, fun(Db) ->
+?assertEqual({ok, 5}, couch_db:get_doc_count(Db)),
+?assertEqual(0, couch_db:get_purge_seq(Db))
+end),
+
+PurgeReqs = [
+{couch_uuids:random(), <<"doc4">>, [Rev]}
+],
+
+{ok, [{ok, PRevs}]} = couch_util:with_db(DbName, fun(Db) ->
+couch_db:purge_docs(Db, PurgeReqs)
+end),
+?assertEqual(PRevs, [Rev]),
+
+couch_util:with_db(DbName, fun(Db) ->
+?assertEqual({ok, 4}, couch_db:get_doc_count(Db)),
+?assertEqual(1, couch_db:get_purge_seq(Db))
+end)
+end).
+
+
+t_upgrade_with_1_purge_req() ->
+?_test(begin
+% There are two documents in the fixture database
+% with a single purge entry
+DbName = <<"db_with_1_purge_req">>,
+
+{ok, UpgradedPurged} = couch_util:with_db(DbName, fun(Db) ->
+?assertEqual(1, couch_db:get_purge_seq(Db)),
+couch_db:fold_purge_infos(Db, 0, fun fold_fun/2, [])
+end),
+?assertEqual([{1, <<"doc1">>}], UpgradedPurged),
+
+{ok, Rev} = save_doc(
+DbName,
[couchdb] 08/10: [08/10] Clustered Purge: Update read-repair
This is an automated email from the ASF dual-hosted git repository.
jiangphcn pushed a commit to branch
COUCHDB-3326-clustered-purge-pr5-implementation
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 0234679661383fd71dd7d15a51d4454d138f0e90
Author: Paul J. Davis
AuthorDate: Wed May 30 17:26:02 2018 -0500
[08/10] Clustered Purge: Update read-repair
Read-repair needs to know which nodes have requested an update to a
local doc so that it can determine if the update is applied. The basic
idea here is that we may have gotten an update from a remote node that
has yet to apply a purge request. If the local node were to apply this
update it would effectively undo a succesful purge request.
COUCHDB-3326
Co-authored-by: Mayya Sharipova
Co-authored-by: jiangphcn
---
src/fabric/src/fabric_doc_open.erl | 73 +++-
src/fabric/src/fabric_doc_open_revs.erl| 262 +-
src/fabric/src/fabric_rpc.erl | 128 -
src/fabric/test/fabric_rpc_purge_tests.erl | 285 +
4 files changed, 692 insertions(+), 56 deletions(-)
diff --git a/src/fabric/src/fabric_doc_open.erl
b/src/fabric/src/fabric_doc_open.erl
index 93f73a8..0a85346 100644
--- a/src/fabric/src/fabric_doc_open.erl
+++ b/src/fabric/src/fabric_doc_open.erl
@@ -25,6 +25,7 @@
r,
state,
replies,
+node_revs = [],
q_reply
}).
@@ -83,7 +84,13 @@ handle_message({rexi_EXIT, _Reason}, Worker, Acc) ->
end;
handle_message(Reply, Worker, Acc) ->
NewReplies = fabric_util:update_counter(Reply, 1, Acc#acc.replies),
-NewAcc = Acc#acc{replies = NewReplies},
+NewNodeRevs = case Reply of
+{ok, #doc{revs = {Pos, [Rev | _]}}} ->
+[{Worker#shard.node, [{Pos, Rev}]} | Acc#acc.node_revs];
+_ ->
+Acc#acc.node_revs
+end,
+NewAcc = Acc#acc{replies = NewReplies, node_revs = NewNodeRevs},
case is_r_met(Acc#acc.workers, NewReplies, Acc#acc.r) of
{true, QuorumReply} ->
fabric_util:cleanup(lists:delete(Worker, Acc#acc.workers)),
@@ -122,14 +129,14 @@ is_r_met(Workers, Replies, R) ->
no_more_workers
end.
-read_repair(#acc{dbname=DbName, replies=Replies}) ->
+read_repair(#acc{dbname=DbName, replies=Replies, node_revs=NodeRevs}) ->
Docs = [Doc || {_, {{ok, #doc{}=Doc}, _}} <- Replies],
case Docs of
% omit local docs from read repair
[#doc{id = <>} | _] ->
choose_reply(Docs);
[#doc{id=Id} | _] ->
-Opts = [replicated_changes, ?ADMIN_CTX],
+Opts = [?ADMIN_CTX, {read_repair, NodeRevs}],
Res = fabric:update_docs(DbName, Docs, Opts),
case Res of
{ok, []} ->
@@ -205,6 +212,7 @@ open_doc_test_() ->
t_handle_message_down(),
t_handle_message_exit(),
t_handle_message_reply(),
+t_store_node_revs(),
t_read_repair(),
t_handle_response_quorum_met(),
t_get_doc_info()
@@ -397,6 +405,65 @@ t_handle_message_reply() ->
end).
+t_store_node_revs() ->
+W1 = #shard{node = w1, ref = erlang:make_ref()},
+W2 = #shard{node = w2, ref = erlang:make_ref()},
+W3 = #shard{node = w3, ref = erlang:make_ref()},
+Foo1 = {ok, #doc{id = <<"bar">>, revs = {1, [<<"foo">>]}}},
+Foo2 = {ok, #doc{id = <<"bar">>, revs = {2, [<<"foo2">>, <<"foo">>]}}},
+NFM = {not_found, missing},
+
+InitAcc = #acc{workers = [W1, W2, W3], replies = [], r = 2},
+
+?_test(begin
+meck:expect(rexi, kill, fun(_, _) -> ok end),
+
+% Simple case
+{ok, #acc{node_revs = NodeRevs1}} = handle_message(Foo1, W1, InitAcc),
+?assertEqual([{w1, [{1, <<"foo">>}]}], NodeRevs1),
+
+% Make sure we only hold the head rev
+{ok, #acc{node_revs = NodeRevs2}} = handle_message(Foo2, W1, InitAcc),
+?assertEqual([{w1, [{2, <<"foo2">>}]}], NodeRevs2),
+
+% Make sure we don't capture anything on error
+{ok, #acc{node_revs = NodeRevs3}} = handle_message(NFM, W1, InitAcc),
+?assertEqual([], NodeRevs3),
+
+% Make sure we accumulate node revs
+Acc1 = InitAcc#acc{node_revs = [{w1, [{1, <<"foo">>}]}]},
+{ok, #acc{node_revs = NodeRevs4}} = handle_message(Foo2, W2, Acc1),
+?assertEqual(
+[{w2, [{2, <<"foo2">>}]}, {w1, [{1, <<"foo">>}]}],
+NodeRevs4
+),
+
+% Make sure rexi_DOWN doesn't modify node_revs
+Down = {rexi_DOWN, nil, {nil, w1}, nil},
+{ok, #acc{node_revs = NodeRevs5}} = handle_message(Down, W2, Acc1),
+?assertEqual([{w1, [{1, <<"foo">>}]}], NodeRevs5),
+
+% Make sure rexi_EXIT doesn't modify node_revs
+Exit = {rexi_EXIT, reason},
+{ok, #acc{node_revs = NodeRevs6}} = handle_message(Exit, W2, Acc1),
+?assertEqual([{w1, [{1, <<"foo">>}]}], NodeRevs6),
+
+% Make sure an error
[couchdb] 09/10: [09/10] Clustered Purge: Fabric API
This is an automated email from the ASF dual-hosted git repository.
jiangphcn pushed a commit to branch
COUCHDB-3326-clustered-purge-pr5-implementation
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 932dd93b0e6edaa5ba2eb3775925301e950657ee
Author: Paul J. Davis
AuthorDate: Tue Apr 24 12:27:43 2018 -0500
[09/10] Clustered Purge: Fabric API
This commit implements the clustered API for performing purge requests.
This change should be a fairly straightforward change for anyone already
familiar with the general implementation of a fabric coordinator given
that the purge API is fairly simple.
COUCHDB-3326
Co-authored-by: Mayya Sharipova
Co-authored-by: jiangphcn
---
src/fabric/src/fabric.erl | 27 +-
src/fabric/src/fabric_db_info.erl | 29 +-
src/fabric/src/fabric_db_meta.erl | 26 +-
src/fabric/src/fabric_doc_purge.erl | 572
4 files changed, 638 insertions(+), 16 deletions(-)
diff --git a/src/fabric/src/fabric.erl b/src/fabric/src/fabric.erl
index 5ad9b45..cf9ad94 100644
--- a/src/fabric/src/fabric.erl
+++ b/src/fabric/src/fabric.erl
@@ -21,12 +21,13 @@
delete_db/2, get_db_info/1, get_doc_count/1, set_revs_limit/3,
set_security/2, set_security/3, get_revs_limit/1, get_security/1,
get_security/2, get_all_security/1, get_all_security/2,
+get_purge_infos_limit/1, set_purge_infos_limit/3,
compact/1, compact/2]).
% Documents
-export([open_doc/3, open_revs/4, get_doc_info/3, get_full_doc_info/3,
get_missing_revs/2, get_missing_revs/3, update_doc/3, update_docs/3,
-purge_docs/2, att_receiver/2]).
+purge_docs/3, att_receiver/2]).
% Views
-export([all_docs/4, all_docs/5, changes/4, query_view/3, query_view/4,
@@ -137,6 +138,18 @@ set_security(DbName, SecObj) ->
set_security(DbName, SecObj, Options) ->
fabric_db_meta:set_security(dbname(DbName), SecObj, opts(Options)).
+%% @doc sets the upper bound for the number of stored purge requests
+-spec set_purge_infos_limit(dbname(), pos_integer(), [option()]) -> ok.
+set_purge_infos_limit(DbName, Limit, Options)
+when is_integer(Limit), Limit > 0 ->
+fabric_db_meta:set_purge_infos_limit(dbname(DbName), Limit, opts(Options)).
+
+%% @doc retrieves the upper bound for the number of stored purge requests
+-spec get_purge_infos_limit(dbname()) -> pos_integer() | no_return().
+get_purge_infos_limit(DbName) ->
+{ok, Db} = fabric_util:get_db(dbname(DbName), [?ADMIN_CTX]),
+try couch_db:get_purge_infos_limit(Db) after catch couch_db:close(Db) end.
+
get_security(DbName) ->
get_security(DbName, [?ADMIN_CTX]).
@@ -271,8 +284,16 @@ update_docs(DbName, Docs, Options) ->
{aborted, PreCommitFailures}
end.
-purge_docs(_DbName, _IdsRevs) ->
-not_implemented.
+
+%% @doc purge revisions for a list '{Id, Revs}'
+%% returns {ok, {PurgeSeq, Results}}
+-spec purge_docs(dbname(), [{docid(), [revision()]}], [option()]) ->
+{ok, [{Health, [revision()]}] | {error, any()}} when
+Health :: ok | accepted.
+purge_docs(DbName, IdsRevs, Options) when is_list(IdsRevs) ->
+IdsRevs2 = [idrevs(IdRs) || IdRs <- IdsRevs],
+fabric_doc_purge:go(dbname(DbName), IdsRevs2, opts(Options)).
+
%% @doc spawns a process to upload attachment data and
%% returns a fabric attachment receiver context tuple
diff --git a/src/fabric/src/fabric_db_info.erl
b/src/fabric/src/fabric_db_info.erl
index 98e8e52..97a31c2 100644
--- a/src/fabric/src/fabric_db_info.erl
+++ b/src/fabric/src/fabric_db_info.erl
@@ -23,10 +23,12 @@ go(DbName) ->
RexiMon = fabric_util:create_monitors(Shards),
Fun = fun handle_message/3,
{ok, ClusterInfo} = get_cluster_info(Shards),
-Acc0 = {fabric_dict:init(Workers, nil), [{cluster, ClusterInfo}]},
+Acc0 = {fabric_dict:init(Workers, nil), [], [{cluster, ClusterInfo}]},
try
case fabric_util:recv(Workers, #shard.ref, Fun, Acc0) of
-{ok, Acc} -> {ok, Acc};
+
+{ok, Acc} ->
+{ok, Acc};
{timeout, {WorkersDict, _}} ->
DefunctWorkers = fabric_util:remove_done_workers(
WorkersDict,
@@ -37,44 +39,49 @@ go(DbName) ->
"get_db_info"
),
{error, timeout};
-{error, Error} -> throw(Error)
+{error, Error} ->
+throw(Error)
end
after
rexi_monitor:stop(RexiMon)
end.
-handle_message({rexi_DOWN, _, {_,NodeRef},_}, _Shard, {Counters, Acc}) ->
+handle_message({rexi_DOWN,
+_, {_,NodeRef},_}, _Shard, {Counters, PseqAcc, Acc}) ->
case fabric_util:remove_down_workers(Counters, NodeRef) of
{ok, NewCounters} ->
-{ok, {NewCounters, Acc}};
+{ok, {NewCounters, PseqAcc, Acc}};
error ->
{error, {nodedown, <<"progress not possible">>}}
end;
-handle_message({rexi_EXIT, Reason}, Shard,
[couchdb] 01/10: [01/10] Clustered Purge: Define new purge API
This is an automated email from the ASF dual-hosted git repository.
jiangphcn pushed a commit to branch
COUCHDB-3326-clustered-purge-pr5-implementation
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit febb0fc8b5abe2b8b3f62c8806ab799919ab897e
Author: Paul J. Davis
AuthorDate: Tue Apr 24 12:23:16 2018 -0500
[01/10] Clustered Purge: Define new purge API
This is the first of a series of commits to implement the new clustered
purge API. Currently purge is a single-node only API that allows for
removing document revisions (and by extension entire documents)
completely from a database. However, given our anti-entropy measures
this API is extremely difficult to use in a cluster and requires
significant operator intervention.
Along with the operator intervention, this API is inherently unsafe with
regards to accidentally triggering the rebuild of secondary indices. As
such this patch set is aimed at creating a cluster aware API that is
both easier to use and less likely to cause application downtime while
secondary indices are rebuilt.
There are four major areas that will be covered by this patch set:
1. Single node APIs and behavior changes
2. Cluster aware APIs
3. Anti-entropy updates
4. Cluster HTTP implementation
This patch set is split up into a series of commits to aid in the review
by other commiters that will hopefully allow for a logical and intuitive
progression of implementation rather than landing as a single opaque
commit covering a huge swath of the code base.
COUCHDB-3326
Co-authored-by: Mayya Sharipova
Co-authored-by: jiangphcn
---
src/couch/src/couch_db_engine.erl | 167 ++
1 file changed, 134 insertions(+), 33 deletions(-)
diff --git a/src/couch/src/couch_db_engine.erl
b/src/couch/src/couch_db_engine.erl
index 2fe0b0d..ea30dbc 100644
--- a/src/couch/src/couch_db_engine.erl
+++ b/src/couch/src/couch_db_engine.erl
@@ -22,6 +22,8 @@
-type rev() :: {non_neg_integer(), binary()}.
-type revs() :: [rev()].
-type json() :: any().
+-type uuid() :: binary().
+-type purge_seq() :: non_neg_integer().
-type doc_pair() :: {
#full_doc_info{} | not_found,
@@ -39,7 +41,7 @@
sync
].
--type purge_info() :: [{docid(), revs()}].
+-type purge_info() :: {purge_seq(), uuid(), docid(), revs()}.
-type epochs() :: [{Node::atom(), UpdateSeq::non_neg_integer()}].
-type size_info() :: [{Name::atom(), Size::non_neg_integer()}].
@@ -62,6 +64,13 @@
{dir, fwd | rev}
].
+-type purge_fold_options() :: [
+{start_key, Key::any()} |
+{end_key, Key::any()} |
+{end_key_gt, Key::any()} |
+{dir, fwd | rev}
+].
+
-type db_handle() :: any().
-type doc_fold_fun() :: fun((#full_doc_info{}, UserAcc::any()) ->
@@ -76,6 +85,10 @@
{ok, NewUserAcc::any()} |
{stop, NewUserAcc::any()}).
+-type purge_fold_fun() :: fun((purge_info(), UserAcc::any()) ->
+{ok, NewUserAcc::any()} |
+{stop, NewUserAcc::any()}).
+
% This is called by couch_server to determine which
% engine should be used for the given database. DbPath
@@ -206,13 +219,18 @@
-callback get_epochs(DbHandle::db_handle()) -> Epochs::epochs().
-% Get the last purge request performed.
--callback get_last_purged(DbHandle::db_handle()) -> LastPurged::purge_info().
+% Get the current purge sequence known to the engine. This
+% value should be updated during calls to purge_docs.
+-callback get_purge_seq(DbHandle::db_handle()) -> purge_seq().
+
+% Get the oldest purge sequence known to the engine
+-callback get_oldest_purge_seq(DbHandle::db_handle()) -> purge_seq().
-% Get the current purge sequence. This should be incremented
-% for every purge operation.
--callback get_purge_seq(DbHandle::db_handle()) -> PurgeSeq::non_neg_integer().
+
+% Get the purged infos limit. This should just return the last
+% value that was passed to set_purged_docs_limit/2.
+-callback get_purge_infos_limit(DbHandle::db_handle()) -> pos_integer().
% Get the revision limit. This should just return the last
@@ -261,6 +279,11 @@
-callback set_revs_limit(DbHandle::db_handle(), RevsLimit::pos_integer()) ->
{ok, NewDbHandle::db_handle()}.
+
+-callback set_purge_infos_limit(DbHandle::db_handle(), Limit::pos_integer()) ->
+{ok, NewDbHandle::db_handle()}.
+
+
-callback set_security(DbHandle::db_handle(), SecProps::any()) ->
{ok, NewDbHandle::db_handle()}.
@@ -301,6 +324,15 @@
doc().
+% This function will be called from many contexts concurrently.
+% If the storage engine has a purge_info() record for any of the
+% provided UUIDs, those purge_info() records should be returned. The
+% resulting list should have the same length as the input list of
+% UUIDs.
+-callback load_purge_infos(DbHandle::db_handle(), [uuid()]) ->
+[purge_info() |
[couchdb] 10/10: [10/10] Clustered Purge: Clustered HTTP API
This is an automated email from the ASF dual-hosted git repository.
jiangphcn pushed a commit to branch
COUCHDB-3326-clustered-purge-pr5-implementation
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 73ac8d1c11557ec1ffa537ac1c376fe55fb7a543
Author: Paul J. Davis
AuthorDate: Tue Apr 24 12:27:58 2018 -0500
[10/10] Clustered Purge: Clustered HTTP API
The HTTP API for clustered purge is fairly straightforward. It is
designed to match the general shape of the single node API. The only
major caveat here is that the purge sequence is now hardcoded as null
since the purge sequence would now otherwise be an opaque blob similar
to the update_seq blobs.
Its important to note that there is as yet no API invented for
traversing the history of purge requests in any shape or form as that
would mostly invalidate the entire purpose of using purge to remove any
trace of a document from a database at the HTTP level. Although there
will still be traces in individual shard files until all database
components have processed the purge and compaction has run (while
allowing for up to purge_infos_limit requests to remain available in
perpetuity).
COUCHDB-3326
Co-authored-by: Mayya Sharipova
Co-authored-by: jiangphcn
---
src/chttpd/src/chttpd_db.erl | 58 --
src/chttpd/test/chttpd_purge_tests.erl | 320 +
test/javascript/tests/erlang_views.js | 5 +-
test/javascript/tests/purge.js | 27 +--
4 files changed, 378 insertions(+), 32 deletions(-)
diff --git a/src/chttpd/src/chttpd_db.erl b/src/chttpd/src/chttpd_db.erl
index 7761007..d3655c3 100644
--- a/src/chttpd/src/chttpd_db.erl
+++ b/src/chttpd/src/chttpd_db.erl
@@ -495,24 +495,33 @@ db_req(#httpd{path_parts=[_, <<"_bulk_get">>]}=Req, _Db)
->
db_req(#httpd{method='POST',path_parts=[_,<<"_purge">>]}=Req, Db) ->
+couch_stats:increment_counter([couchdb, httpd, purge_requests]),
chttpd:validate_ctype(Req, "application/json"),
+W = chttpd:qs_value(Req, "w", integer_to_list(mem3:quorum(Db))),
+Options = [{user_ctx, Req#httpd.user_ctx}, {w, W}],
{IdsRevs} = chttpd:json_body_obj(Req),
IdsRevs2 = [{Id, couch_doc:parse_revs(Revs)} || {Id, Revs} <- IdsRevs],
-case fabric:purge_docs(Db, IdsRevs2) of
-{ok, PurgeSeq, PurgedIdsRevs} ->
-PurgedIdsRevs2 = [{Id, couch_doc:revs_to_strs(Revs)} || {Id, Revs}
-<- PurgedIdsRevs],
-send_json(Req, 200, {[
-{<<"purge_seq">>, PurgeSeq},
-{<<"purged">>, {PurgedIdsRevs2}}
-]});
-Error ->
-throw(Error)
-end;
+MaxIds = config:get_integer("purge", "max_document_id_number", 100),
+case length(IdsRevs2) =< MaxIds of
+false -> throw({bad_request, "Exceeded maximum number of documents."});
+true -> ok
+end,
+RevsLen = lists:foldl(fun({_Id, Revs}, Acc) ->
+length(Revs) + Acc
+end, 0, IdsRevs2),
+MaxRevs = config:get_integer("purge", "max_revisions_number", 1000),
+case RevsLen =< MaxRevs of
+false -> throw({bad_request, "Exceeded maximum number of revisions."});
+true -> ok
+end,
+{ok, Results} = fabric:purge_docs(Db, IdsRevs2, Options),
+{Code, Json} = purge_results_to_json(IdsRevs2, Results),
+send_json(Req, Code, {[{<<"purge_seq">>, null}, {<<"purged">>, {Json}}]});
db_req(#httpd{path_parts=[_,<<"_purge">>]}=Req, _Db) ->
send_method_not_allowed(Req, "POST");
+
db_req(#httpd{method='GET',path_parts=[_,OP]}=Req, Db) when ?IS_ALL_DOCS(OP) ->
case chttpd:qs_json_value(Req, "keys", nil) of
Keys when is_list(Keys) ->
@@ -622,6 +631,19 @@
db_req(#httpd{method='GET',path_parts=[_,<<"_revs_limit">>]}=Req, Db) ->
db_req(#httpd{path_parts=[_,<<"_revs_limit">>]}=Req, _Db) ->
send_method_not_allowed(Req, "PUT,GET");
+db_req(#httpd{method='PUT',path_parts=[_,<<"_purged_infos_limit">>]}=Req, Db)
->
+Options = [{user_ctx, Req#httpd.user_ctx}],
+case chttpd:json_body(Req) of
+Limit when is_integer(Limit), Limit > 0 ->
+ok = fabric:set_purge_infos_limit(Db, Limit, Options),
+send_json(Req, {[{<<"ok">>, true}]});
+_->
+throw({bad_request, "`purge_infos_limit` must be positive
integer"})
+end;
+
+db_req(#httpd{method='GET',path_parts=[_,<<"_purged_infos_limit">>]}=Req, Db)
->
+send_json(Req, fabric:get_purge_infos_limit(Db));
+
% Special case to enable using an unencoded slash in the URL of design docs,
% as slashes in document IDs must otherwise be URL encoded.
db_req(#httpd{method='GET', mochi_req=MochiReq, path_parts=[_DbName,
<<"_design/", _/binary>> | _]}=Req, _Db) ->
@@ -992,6 +1014,20 @@ update_doc_result_to_json(DocId, Error) ->
{_Code, ErrorStr, Reason} = chttpd:error_info(Error),
{[{id, DocId}, {error, ErrorStr}, {reason, Reason}]}.
+purge_results_to_json([], []) ->
+{201, []};
[couchdb] 06/10: [06/10] Clustered Purge: Update mrview indexes
This is an automated email from the ASF dual-hosted git repository.
jiangphcn pushed a commit to branch
COUCHDB-3326-clustered-purge-pr5-implementation
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 3be455b4816858a092b43e5a2ea8a5e6a564e4d8
Author: Paul J. Davis
AuthorDate: Tue Apr 24 12:26:42 2018 -0500
[06/10] Clustered Purge: Update mrview indexes
This commit updates the mrview secondary index to properly process the
new history of purge requests as well as to store the _local purge
checkpoint doc.
The importance of the _local checkpoint doc is to ensure that compaction
of a database does not remove any purge requests that have not yet been
processed by this secondary index.
COUCHDB-3326
Co-authored-by: Mayya Sharipova
Co-authored-by: jiangphcn
---
src/couch_index/src/couch_index_epi.erl| 5 +-
...dex_epi.erl => couch_index_plugin_couch_db.erl} | 39 +-
src/couch_index/src/couch_index_updater.erl| 50 +-
src/couch_mrview/src/couch_mrview_cleanup.erl | 16 +-
src/couch_mrview/src/couch_mrview_index.erl| 115 -
src/couch_mrview/src/couch_mrview_test_util.erl| 5 +
src/couch_mrview/src/couch_mrview_updater.erl | 14 +-
src/couch_mrview/src/couch_mrview_util.erl | 39 +-
.../test/couch_mrview_purge_docs_fabric_tests.erl | 276 +++
.../test/couch_mrview_purge_docs_tests.erl | 506 +
10 files changed, 1004 insertions(+), 61 deletions(-)
diff --git a/src/couch_index/src/couch_index_epi.erl
b/src/couch_index/src/couch_index_epi.erl
index 946a590..1c4eb95 100644
--- a/src/couch_index/src/couch_index_epi.erl
+++ b/src/couch_index/src/couch_index_epi.erl
@@ -28,8 +28,9 @@ app() ->
couch_index.
providers() ->
-[].
-
+[
+{couch_db, couch_index_plugin_couch_db}
+].
services() ->
[
diff --git a/src/couch_index/src/couch_index_epi.erl
b/src/couch_index/src/couch_index_plugin_couch_db.erl
similarity index 52%
copy from src/couch_index/src/couch_index_epi.erl
copy to src/couch_index/src/couch_index_plugin_couch_db.erl
index 946a590..0af22e3 100644
--- a/src/couch_index/src/couch_index_epi.erl
+++ b/src/couch_index/src/couch_index_plugin_couch_db.erl
@@ -2,7 +2,7 @@
% use this file except in compliance with the License. You may obtain a copy of
% the License at
%
-% http://www.apache.org/licenses/LICENSE-2.0
+% http://www.apache.org/licenses/LICENSE-2.0
%
% Unless required by applicable law or agreed to in writing, software
% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
@@ -10,40 +10,17 @@
% License for the specific language governing permissions and limitations under
% the License.
--module(couch_index_epi).
-
--behaviour(couch_epi_plugin).
+-module(couch_index_plugin_couch_db).
-export([
-app/0,
-providers/0,
-services/0,
-data_subscriptions/0,
-data_providers/0,
-processes/0,
-notify/3
+is_valid_purge_client/2,
+on_compact/2
]).
-app() ->
-couch_index.
-
-providers() ->
-[].
-
-
-services() ->
-[
-{couch_index, couch_index_plugin}
-].
-
-data_subscriptions() ->
-[].
-data_providers() ->
-[].
+is_valid_purge_client(DbName, Props) ->
+couch_mrview_index:verify_index_exists(DbName, Props).
-processes() ->
-[].
-notify(_Key, _Old, _New) ->
-ok.
+on_compact(DbName, DDocs) ->
+couch_mrview_index:ensure_local_purge_docs(DbName, DDocs).
diff --git a/src/couch_index/src/couch_index_updater.erl
b/src/couch_index/src/couch_index_updater.erl
index 5ab9ea8..7864bde 100644
--- a/src/couch_index/src/couch_index_updater.erl
+++ b/src/couch_index/src/couch_index_updater.erl
@@ -141,12 +141,10 @@ update(Idx, Mod, IdxState) ->
DbUpdateSeq = couch_db:get_update_seq(Db),
DbCommittedSeq = couch_db:get_committed_update_seq(Db),
-PurgedIdxState = case purge_index(Db, Mod, IdxState) of
-{ok, IdxState0} -> IdxState0;
-reset -> exit({reset, self()})
-end,
-
-NumChanges = couch_db:count_changes_since(Db, CurrSeq),
+NumUpdateChanges = couch_db:count_changes_since(Db, CurrSeq),
+NumPurgeChanges = count_pending_purged_docs_since(Db, Mod, IdxState),
+TotalChanges = NumUpdateChanges + NumPurgeChanges,
+{ok, PurgedIdxState} = purge_index(Db, Mod, IdxState),
GetSeq = fun
(#full_doc_info{update_seq=Seq}) -> Seq;
@@ -185,8 +183,13 @@ update(Idx, Mod, IdxState) ->
{ok, {NewSt, true}}
end
end,
+{ok, InitIdxState} = Mod:start_update(
+Idx,
+PurgedIdxState,
+TotalChanges,
+NumPurgeChanges
+),
-{ok, InitIdxState} = Mod:start_update(Idx, PurgedIdxState, NumChanges),
Acc0 = {InitIdxState, true},
{ok, Acc} =
[couchdb] 04/10: [04/10] Clustered Purge: Update couch_pse_tests
This is an automated email from the ASF dual-hosted git repository.
jiangphcn pushed a commit to branch
COUCHDB-3326-clustered-purge-pr5-implementation
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 8a9112fdb891dd420ebe1ce41cf72b214ec76d56
Author: Paul J. Davis
AuthorDate: Wed May 30 16:13:08 2018 -0500
[04/10] Clustered Purge: Update couch_pse_tests
This updates the couch_pse_tests to account for the new purge APIs as
well as introduces a bunch of new tests for covering the new APIs.
COUCHDB-3326
Co-authored-by: Mayya Sharipova
Co-authored-by: jiangphcn
---
src/couch_pse_tests/src/cpse_test_compaction.erl | 143 +-
.../src/cpse_test_fold_purge_infos.erl | 166 +++
.../src/cpse_test_get_set_props.erl| 3 +-
.../src/cpse_test_purge_bad_checkpoints.erl| 80
src/couch_pse_tests/src/cpse_test_purge_docs.erl | 506 +
src/couch_pse_tests/src/cpse_test_purge_seqs.erl | 124 +
src/couch_pse_tests/src/cpse_util.erl | 151 +-
7 files changed, 1057 insertions(+), 116 deletions(-)
diff --git a/src/couch_pse_tests/src/cpse_test_compaction.erl
b/src/couch_pse_tests/src/cpse_test_compaction.erl
index 11bf106..d006111 100644
--- a/src/couch_pse_tests/src/cpse_test_compaction.erl
+++ b/src/couch_pse_tests/src/cpse_test_compaction.erl
@@ -97,10 +97,8 @@ cpse_compact_with_everything(Db1) ->
BarRev = cpse_util:prev_rev(BarFDI),
Actions3 = [
-{batch, [
-{purge, {<<"foo">>, FooRev#rev_info.rev}},
-{purge, {<<"bar">>, BarRev#rev_info.rev}}
-]}
+{purge, {<<"foo">>, FooRev#rev_info.rev}},
+{purge, {<<"bar">>, BarRev#rev_info.rev}}
],
{ok, Db4} = cpse_util:apply_actions(Db3, Actions3),
@@ -110,10 +108,9 @@ cpse_compact_with_everything(Db1) ->
{<<"foo">>, [FooRev#rev_info.rev]}
],
-?assertEqual(
-PurgedIdRevs,
-lists:sort(couch_db_engine:get_last_purged(Db4))
-),
+{ok, PIdRevs4} = couch_db_engine:fold_purge_infos(
+Db4, 0, fun fold_fun/2, [], []),
+?assertEqual(PurgedIdRevs, PIdRevs4),
{ok, Db5} = try
[Att0, Att1, Att2, Att3, Att4] = cpse_util:prep_atts(Db4, [
@@ -181,6 +178,132 @@ cpse_recompact_updates(Db1) ->
?assertEqual(nodiff, Diff).
+cpse_purge_during_compact(Db1) ->
+Actions1 = lists:map(fun(Seq) ->
+{create, {docid(Seq), {[{<<"int">>, Seq}]}}}
+end, lists:seq(1, 1000)),
+Actions2 = [
+{create, {<<"foo">>, {[]}}},
+{create, {<<"bar">>, {[]}}},
+{create, {<<"baz">>, {[]}}}
+],
+{ok, Db2} = cpse_util:apply_batch(Db1, Actions1 ++ Actions2),
+Actions3 = [
+{conflict, {<<"bar">>, {[{<<"vsn">>, 2}]}}}
+],
+{ok, Db3} = cpse_util:apply_actions(Db2, Actions3),
+
+{ok, Pid} = couch_db:start_compact(Db3),
+catch erlang:suspend_process(Pid),
+
+[BarFDI, BazFDI] = couch_db_engine:open_docs(Db3, [<<"bar">>, <<"baz">>]),
+BarRev = cpse_util:prev_rev(BarFDI),
+BazRev = cpse_util:prev_rev(BazFDI),
+Actions4 = [
+{purge, {<<"bar">>, BarRev#rev_info.rev}},
+{purge, {<<"baz">>, BazRev#rev_info.rev}}
+],
+
+{ok, Db4} = cpse_util:apply_actions(Db3, Actions4),
+Term1 = cpse_util:db_as_term(Db4),
+
+catch erlang:resume_process(Pid),
+cpse_util:compact(Db4),
+
+{ok, Db5} = couch_db:reopen(Db4),
+Term2 = cpse_util:db_as_term(Db5),
+
+Diff = cpse_util:term_diff(Term1, Term2),
+?assertEqual(nodiff, Diff).
+
+
+cpse_multiple_purge_during_compact(Db1) ->
+Actions1 = lists:map(fun(Seq) ->
+{create, {docid(Seq), {[{<<"int">>, Seq}]}}}
+end, lists:seq(1, 1000)),
+Actions2 = [
+{create, {<<"foo">>, {[]}}},
+{create, {<<"bar">>, {[]}}},
+{create, {<<"baz">>, {[]}}}
+],
+{ok, Db2} = cpse_util:apply_batch(Db1, Actions1 ++ Actions2),
+
+Actions3 = [
+{conflict, {<<"bar">>, {[{<<"vsn">>, 2}]}}}
+],
+{ok, Db3} = cpse_util:apply_actions(Db2, Actions3),
+
+
+{ok, Pid} = couch_db:start_compact(Db3),
+catch erlang:suspend_process(Pid),
+
+[BarFDI, BazFDI] = couch_db_engine:open_docs(Db3, [<<"bar">>, <<"baz">>]),
+BarRev = cpse_util:prev_rev(BarFDI),
+Actions4 = [
+{purge, {<<"bar">>, BarRev#rev_info.rev}}
+],
+{ok, Db4} = cpse_util:apply_actions(Db3, Actions4),
+
+BazRev = cpse_util:prev_rev(BazFDI),
+Actions5 = [
+{purge, {<<"baz">>, BazRev#rev_info.rev}}
+],
+
+{ok, Db5} = cpse_util:apply_actions(Db4, Actions5),
+Term1 = cpse_util:db_as_term(Db5),
+
+catch erlang:resume_process(Pid),
+cpse_util:compact(Db5),
+
+{ok, Db6} = couch_db:reopen(Db5),
+Term2 = cpse_util:db_as_term(Db6),
+
+Diff = cpse_util:term_diff(Term1, Term2),
+?assertEqual(nodiff, Diff).
+
+
+cpse_compact_purged_docs_limit(Db1) ->
+NumDocs = 1200,
+
[couchdb] branch COUCHDB-3326-clustered-purge-pr5-implementation updated (bf809a8 -> 73ac8d1)
This is an automated email from the ASF dual-hosted git repository. jiangphcn pushed a change to branch COUCHDB-3326-clustered-purge-pr5-implementation in repository https://gitbox.apache.org/repos/asf/couchdb.git. omit bf809a8 [10/10] Clustered Purge: Clustered HTTP API omit 110dcdc [09/10] Clustered Purge: Fabric API omit 4dcfb50 [08/10] Clustered Purge: Update read-repair omit 973a599 [07/10] Clustered Purge: Internal replication omit a947b5c [06/10] Clustered Purge: Update mrview indexes omit 3279c38 [05/10] Clustered Purge: Add upgrade tests omit a192aef [04/10] Clustered Purge: Update couch_pse_tests omit ac8d0f6 [03/10] Clustered Purge: Update couch_bt_engine omit 4d695f1 [02/10] Clustered Purge: Update single node APIs omit 0254305 [01/10] Clustered Purge: Define new purge API omit 830240b Create new on_compact trigger omit 4c87d78 Enhance PSE tests with setup/teardown functions omit c998aa4 Update to use new couch_pse_tests app omit 0d85abe Update PSE test definitions for new util module omit 3c94bc4 Rename PSE test modules omit db28f4c Move PSE tests to their own app omit f45c036 Rewrite the PSE test suite to use couch_server omit da43788 Simplify logic in mem3_rep add fc10fb5 Merge pull request #1366 from apache/COUCHDB-3326-clustered-purge-pr1-misc-cleanup add 391d7b5 Simplify logic in mem3_rep add 8819a27 Merge pull request #1570 from apache/COUCHDB-3326-clustered-purge-pr2-simplify-mem3-rep add 9b7c2d1 Rewrite the PSE test suite to use couch_server add 5230ca0 Move PSE tests to their own app add d033063 Rename PSE test modules add bbc74b9 Update PSE test definitions for new util module add 776349c Update to use new couch_pse_tests app add e0c7ad3 Enhance PSE tests with setup/teardown functions add c02f6d2 Merge pull request #1368 from apache/COUCHDB-3326-clustered-purge-pr3-refactor-pse-tests add ab12e0b Create new on_compact trigger add 2d3f935 Merge pull request #1369 from apache/COUCHDB-3326-clustered-purge-pr4-on-compact-plugin new febb0fc [01/10] Clustered Purge: Define new purge API new eca282e [02/10] Clustered Purge: Update single node APIs new bccc39d [03/10] Clustered Purge: Update couch_bt_engine new 8a9112f [04/10] Clustered Purge: Update couch_pse_tests new 48378a9 [05/10] Clustered Purge: Add upgrade tests new 3be455b [06/10] Clustered Purge: Update mrview indexes new c625933 [07/10] Clustered Purge: Internal replication new 0234679 [08/10] Clustered Purge: Update read-repair new 932dd93 [09/10] Clustered Purge: Fabric API new 73ac8d1 [10/10] Clustered Purge: Clustered HTTP API This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (bf809a8) \ N -- N -- N refs/heads/COUCHDB-3326-clustered-purge-pr5-implementation (73ac8d1) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. The 10 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes:
[couchdb] branch master updated (c02f6d2 -> 2d3f935)
This is an automated email from the ASF dual-hosted git repository. jiangphcn pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/couchdb.git. from c02f6d2 Merge pull request #1368 from apache/COUCHDB-3326-clustered-purge-pr3-refactor-pse-tests add ab12e0b Create new on_compact trigger new 2d3f935 Merge pull request #1369 from apache/COUCHDB-3326-clustered-purge-pr4-on-compact-plugin The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: src/couch/src/couch_bt_engine_compactor.erl | 2 ++ src/couch/src/couch_db_engine.erl | 31 - src/couch/src/couch_db_plugin.erl | 5 + 3 files changed, 37 insertions(+), 1 deletion(-)
[couchdb] 01/01: Merge pull request #1369 from apache/COUCHDB-3326-clustered-purge-pr4-on-compact-plugin
This is an automated email from the ASF dual-hosted git repository. jiangphcn pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/couchdb.git commit 2d3f935eb87224906ab7b1f273b15fa736cee6b4 Merge: c02f6d2 ab12e0b Author: Peng Hui Jiang AuthorDate: Wed Aug 22 00:53:20 2018 +0800 Merge pull request #1369 from apache/COUCHDB-3326-clustered-purge-pr4-on-compact-plugin [4/5] Clustered Purge: Add on_compact EPI hook src/couch/src/couch_bt_engine_compactor.erl | 2 ++ src/couch/src/couch_db_engine.erl | 31 - src/couch/src/couch_db_plugin.erl | 5 + 3 files changed, 37 insertions(+), 1 deletion(-)
[couchdb-documentation] branch master updated (77ffbae -> 95efadb)
This is an automated email from the ASF dual-hosted git repository. jiangphcn pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/couchdb-documentation.git. from 77ffbae Merge pull request #311 from cloudant/node-restart add cd0db0a Documentation on clustered purge new 95efadb Merge pull request #285 from cloudant/COUCHDB-3326-clustered-purge The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: images/purge-checkpoint-docs.png | Bin 0 -> 69408 bytes images/rev-tree1.png | Bin 0 -> 13910 bytes images/rev-tree2.png | Bin 0 -> 19104 bytes images/rev-tree3.png | Bin 0 -> 10439 bytes src/api/database/misc.rst| 249 +++ src/cluster/index.rst| 1 + src/cluster/purging.rst | 173 +++ 7 files changed, 376 insertions(+), 47 deletions(-) create mode 100644 images/purge-checkpoint-docs.png create mode 100644 images/rev-tree1.png create mode 100644 images/rev-tree2.png create mode 100644 images/rev-tree3.png create mode 100644 src/cluster/purging.rst
[couchdb-documentation] 01/01: Merge pull request #285 from cloudant/COUCHDB-3326-clustered-purge
This is an automated email from the ASF dual-hosted git repository. jiangphcn pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/couchdb-documentation.git commit 95efadbea6eeeb5df5823c7460a3a94b75a4a412 Merge: 77ffbae cd0db0a Author: Peng Hui Jiang AuthorDate: Wed Aug 22 00:38:39 2018 +0800 Merge pull request #285 from cloudant/COUCHDB-3326-clustered-purge Couchdb 3326 clustered purge images/purge-checkpoint-docs.png | Bin 0 -> 69408 bytes images/rev-tree1.png | Bin 0 -> 13910 bytes images/rev-tree2.png | Bin 0 -> 19104 bytes images/rev-tree3.png | Bin 0 -> 10439 bytes src/api/database/misc.rst| 249 +++ src/cluster/index.rst| 1 + src/cluster/purging.rst | 173 +++ 7 files changed, 376 insertions(+), 47 deletions(-)
[couchdb] branch COUCHDB-3326-clustered-purge-pr4-on-compact-plugin updated (830240b -> ab12e0b)
This is an automated email from the ASF dual-hosted git repository. jiangphcn pushed a change to branch COUCHDB-3326-clustered-purge-pr4-on-compact-plugin in repository https://gitbox.apache.org/repos/asf/couchdb.git. omit 830240b Create new on_compact trigger omit 4c87d78 Enhance PSE tests with setup/teardown functions omit c998aa4 Update to use new couch_pse_tests app omit 0d85abe Update PSE test definitions for new util module omit 3c94bc4 Rename PSE test modules omit db28f4c Move PSE tests to their own app omit f45c036 Rewrite the PSE test suite to use couch_server omit da43788 Simplify logic in mem3_rep add fc10fb5 Merge pull request #1366 from apache/COUCHDB-3326-clustered-purge-pr1-misc-cleanup add 391d7b5 Simplify logic in mem3_rep add 8819a27 Merge pull request #1570 from apache/COUCHDB-3326-clustered-purge-pr2-simplify-mem3-rep add 9b7c2d1 Rewrite the PSE test suite to use couch_server add 5230ca0 Move PSE tests to their own app add d033063 Rename PSE test modules add bbc74b9 Update PSE test definitions for new util module add 776349c Update to use new couch_pse_tests app add e0c7ad3 Enhance PSE tests with setup/teardown functions add c02f6d2 Merge pull request #1368 from apache/COUCHDB-3326-clustered-purge-pr3-refactor-pse-tests new ab12e0b Create new on_compact trigger This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (830240b) \ N -- N -- N refs/heads/COUCHDB-3326-clustered-purge-pr4-on-compact-plugin (ab12e0b) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes:
[couchdb] 01/01: Create new on_compact trigger
This is an automated email from the ASF dual-hosted git repository.
jiangphcn pushed a commit to branch
COUCHDB-3326-clustered-purge-pr4-on-compact-plugin
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit ab12e0be80f37bbf9b77f0977393c89696a0b948
Author: Paul J. Davis
AuthorDate: Wed May 30 14:34:44 2018 -0500
Create new on_compact trigger
This trigger allows any storage engine that makes use of compaction to
notify that compaction is starting. This is preparatory work for
clustered indexes so that existing indexes are allowed to ensure they
have a clustered purge local doc before compaction runs.
COUCHDB-3326
Co-Authored-By: jiangphcn
---
src/couch/src/couch_bt_engine_compactor.erl | 2 ++
src/couch/src/couch_db_engine.erl | 31 -
src/couch/src/couch_db_plugin.erl | 5 +
3 files changed, 37 insertions(+), 1 deletion(-)
diff --git a/src/couch/src/couch_bt_engine_compactor.erl
b/src/couch/src/couch_bt_engine_compactor.erl
index 4e52064..2c5b78e 100644
--- a/src/couch/src/couch_bt_engine_compactor.erl
+++ b/src/couch/src/couch_bt_engine_compactor.erl
@@ -44,6 +44,8 @@ start(#st{} = St, DbName, Options, Parent) ->
} = St,
couch_log:debug("Compaction process spawned for db \"~s\"", [DbName]),
+couch_db_engine:trigger_on_compact(DbName),
+
{ok, NewSt, DName, DFd, MFd, Retry} =
open_compaction_files(Header, FilePath, Options),
erlang:monitor(process, MFd),
diff --git a/src/couch/src/couch_db_engine.erl
b/src/couch/src/couch_db_engine.erl
index 4974201..2fe0b0d 100644
--- a/src/couch/src/couch_db_engine.erl
+++ b/src/couch/src/couch_db_engine.erl
@@ -627,7 +627,8 @@
count_changes_since/2,
start_compaction/1,
-finish_compaction/2
+finish_compaction/2,
+trigger_on_compact/1
]).
@@ -891,3 +892,31 @@ finish_compaction(Db, CompactInfo) ->
end,
ok = gen_server:call(couch_server, {db_updated, NewDb}, infinity),
{ok, NewDb}.
+
+
+trigger_on_compact(DbName) ->
+{ok, DDocs} = get_ddocs(DbName),
+couch_db_plugin:on_compact(DbName, DDocs).
+
+
+get_ddocs(<<"shards/", _/binary>> = DbName) ->
+{_, Ref} = spawn_monitor(fun() ->
+exit(fabric:design_docs(mem3:dbname(DbName)))
+end),
+receive
+{'DOWN', Ref, _, _, {ok, JsonDDocs}} ->
+{ok, lists:map(fun(JsonDDoc) ->
+couch_doc:from_json_obj(JsonDDoc)
+end, JsonDDocs)};
+{'DOWN', Ref, _, _, Else} ->
+Else
+end;
+get_ddocs(DbName) ->
+couch_util:with_db(DbName, fun(Db) ->
+FoldFun = fun(FDI, Acc) ->
+Doc = couch_db:open_doc_int(Db, FDI, []),
+{ok, [Doc | Acc]}
+end,
+{ok, Docs} = couch_db:fold_design_docs(Db, FoldFun, [], []),
+{ok, lists:reverse(Docs)}
+end).
diff --git a/src/couch/src/couch_db_plugin.erl
b/src/couch/src/couch_db_plugin.erl
index 740b812..8163256 100644
--- a/src/couch/src/couch_db_plugin.erl
+++ b/src/couch/src/couch_db_plugin.erl
@@ -18,6 +18,7 @@
after_doc_read/2,
validate_docid/1,
check_is_admin/1,
+on_compact/2,
on_delete/2
]).
@@ -56,6 +57,10 @@ check_is_admin(Db) ->
%% callbacks return true only if it specifically allow the given Id
couch_epi:any(Handle, ?SERVICE_ID, check_is_admin, [Db], []).
+on_compact(DbName, DDocs) ->
+Handle = couch_epi:get_handle(?SERVICE_ID),
+couch_epi:apply(Handle, ?SERVICE_ID, on_compact, [DbName, DDocs], []).
+
on_delete(DbName, Options) ->
Handle = couch_epi:get_handle(?SERVICE_ID),
couch_epi:apply(Handle, ?SERVICE_ID, on_delete, [DbName, Options], []).
[couchdb] branch master updated (8819a27 -> c02f6d2)
This is an automated email from the ASF dual-hosted git repository.
jiangphcn pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/couchdb.git.
from 8819a27 Merge pull request #1570 from
apache/COUCHDB-3326-clustered-purge-pr2-simplify-mem3-rep
add 9b7c2d1 Rewrite the PSE test suite to use couch_server
add 5230ca0 Move PSE tests to their own app
add d033063 Rename PSE test modules
add bbc74b9 Update PSE test definitions for new util module
add 776349c Update to use new couch_pse_tests app
add e0c7ad3 Enhance PSE tests with setup/teardown functions
new c02f6d2 Merge pull request #1368 from
apache/COUCHDB-3326-clustered-purge-pr3-refactor-pse-tests
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
rebar.config.script| 1 +
src/couch/src/test_engine_compaction.erl | 185 ---
src/couch/src/test_engine_fold_changes.erl | 190 ---
src/couch/src/test_engine_get_set_props.erl| 70 ---
src/couch/src/test_engine_open_close_delete.erl| 81 ---
src/couch/src/test_engine_purge_docs.erl | 158 --
src/couch/src/test_engine_read_write_docs.erl | 317 ---
src/couch/src/test_engine_util.erl | 608 -
src/couch/test/couch_bt_engine_tests.erl | 2 +-
.../src/couch_pse_tests.app.src} | 11 +-
src/couch_pse_tests/src/cpse_gather.erl| 95
.../src/cpse_test_attachments.erl} | 50 +-
src/couch_pse_tests/src/cpse_test_compaction.erl | 191 +++
src/couch_pse_tests/src/cpse_test_fold_changes.erl | 198 +++
.../src/cpse_test_fold_docs.erl} | 167 +++---
.../src/cpse_test_get_set_props.erl| 94
.../src/cpse_test_open_close_delete.erl| 76 +++
src/couch_pse_tests/src/cpse_test_purge_docs.erl | 159 ++
.../src/cpse_test_read_write_docs.erl | 318 +++
.../src/cpse_test_ref_counting.erl}| 57 +-
src/couch_pse_tests/src/cpse_util.erl | 515 +
21 files changed, 1803 insertions(+), 1740 deletions(-)
delete mode 100644 src/couch/src/test_engine_compaction.erl
delete mode 100644 src/couch/src/test_engine_fold_changes.erl
delete mode 100644 src/couch/src/test_engine_get_set_props.erl
delete mode 100644 src/couch/src/test_engine_open_close_delete.erl
delete mode 100644 src/couch/src/test_engine_purge_docs.erl
delete mode 100644 src/couch/src/test_engine_read_write_docs.erl
delete mode 100644 src/couch/src/test_engine_util.erl
copy src/{couch_log/src/couch_log.app.src =>
couch_pse_tests/src/couch_pse_tests.app.src} (76%)
create mode 100644 src/couch_pse_tests/src/cpse_gather.erl
rename src/{couch/src/test_engine_attachments.erl =>
couch_pse_tests/src/cpse_test_attachments.erl} (63%)
create mode 100644 src/couch_pse_tests/src/cpse_test_compaction.erl
create mode 100644 src/couch_pse_tests/src/cpse_test_fold_changes.erl
rename src/{couch/src/test_engine_fold_docs.erl =>
couch_pse_tests/src/cpse_test_fold_docs.erl} (62%)
create mode 100644 src/couch_pse_tests/src/cpse_test_get_set_props.erl
create mode 100644 src/couch_pse_tests/src/cpse_test_open_close_delete.erl
create mode 100644 src/couch_pse_tests/src/cpse_test_purge_docs.erl
create mode 100644 src/couch_pse_tests/src/cpse_test_read_write_docs.erl
rename src/{couch/src/test_engine_ref_counting.erl =>
couch_pse_tests/src/cpse_test_ref_counting.erl} (62%)
create mode 100644 src/couch_pse_tests/src/cpse_util.erl
[couchdb] 01/01: Merge pull request #1368 from apache/COUCHDB-3326-clustered-purge-pr3-refactor-pse-tests
This is an automated email from the ASF dual-hosted git repository. jiangphcn pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/couchdb.git commit c02f6d294d962d18c22d8179f43ee07add5437eb Merge: 8819a27 e0c7ad3 Author: Peng Hui Jiang AuthorDate: Wed Aug 22 00:04:43 2018 +0800 Merge pull request #1368 from apache/COUCHDB-3326-clustered-purge-pr3-refactor-pse-tests [3/5] Clustered Purge - Rewrite pluggable storage engine tests rebar.config.script| 1 + src/couch/src/test_engine_compaction.erl | 185 --- src/couch/src/test_engine_fold_changes.erl | 190 --- src/couch/src/test_engine_get_set_props.erl| 70 --- src/couch/src/test_engine_open_close_delete.erl| 81 --- src/couch/src/test_engine_purge_docs.erl | 158 -- src/couch/src/test_engine_read_write_docs.erl | 317 --- src/couch/src/test_engine_util.erl | 608 - src/couch/test/couch_bt_engine_tests.erl | 2 +- .../src/couch_pse_tests.app.src} | 16 +- src/couch_pse_tests/src/cpse_gather.erl| 95 .../src/cpse_test_attachments.erl} | 50 +- src/couch_pse_tests/src/cpse_test_compaction.erl | 191 +++ src/couch_pse_tests/src/cpse_test_fold_changes.erl | 198 +++ .../src/cpse_test_fold_docs.erl} | 167 +++--- .../src/cpse_test_get_set_props.erl| 94 .../src/cpse_test_open_close_delete.erl| 76 +++ src/couch_pse_tests/src/cpse_test_purge_docs.erl | 159 ++ .../src/cpse_test_read_write_docs.erl | 318 +++ .../src/cpse_test_ref_counting.erl}| 57 +- src/couch_pse_tests/src/cpse_util.erl | 515 + 21 files changed, 1805 insertions(+), 1743 deletions(-)
[couchdb] 06/06: Enhance PSE tests with setup/teardown functions
This is an automated email from the ASF dual-hosted git repository.
jiangphcn pushed a commit to branch
COUCHDB-3326-clustered-purge-pr3-refactor-pse-tests
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit e0c7ad386803dea20573201f4e6468bdae8294e1
Author: Paul J. Davis
AuthorDate: Wed May 23 15:31:08 2018 -0500
Enhance PSE tests with setup/teardown functions
---
src/couch_pse_tests/src/cpse_gather.erl| 95 ++
src/couch_pse_tests/src/cpse_test_attachments.erl | 15 ++--
src/couch_pse_tests/src/cpse_test_compaction.erl | 26 +++---
src/couch_pse_tests/src/cpse_test_fold_changes.erl | 54 ++--
src/couch_pse_tests/src/cpse_test_fold_docs.erl| 90 ++--
.../src/cpse_test_get_set_props.erl| 28 ---
.../src/cpse_test_open_close_delete.erl| 34
src/couch_pse_tests/src/cpse_test_purge_docs.erl | 23 +++---
.../src/cpse_test_read_write_docs.erl | 49 ---
src/couch_pse_tests/src/cpse_test_ref_counting.erl | 19 +++--
src/couch_pse_tests/src/cpse_util.erl | 52
11 files changed, 290 insertions(+), 195 deletions(-)
diff --git a/src/couch_pse_tests/src/cpse_gather.erl
b/src/couch_pse_tests/src/cpse_gather.erl
new file mode 100644
index 000..7804d41
--- /dev/null
+++ b/src/couch_pse_tests/src/cpse_gather.erl
@@ -0,0 +1,95 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(cpse_gather).
+
+
+-export([
+module/1
+]).
+
+
+module(ModName) ->
+Exports = ModName:module_info(exports),
+
+SetupMod = get_setup_all(ModName, Exports),
+TeardownMod = get_teardown_all(ModName, Exports),
+SetupTest = get_fun(ModName, setup_each, 0, Exports),
+TeardownTest = get_fun(ModName, teardown_each, 1, Exports),
+
+RevTests = lists:foldl(fun({Fun, Arity}, Acc) ->
+case {atom_to_list(Fun), Arity} of
+{[$c, $p, $s, $e, $_ | _], Arity} when Arity == 0; Arity == 1 ->
+TestFun = make_test_fun(ModName, Fun, Arity),
+[TestFun | Acc];
+_ ->
+Acc
+end
+end, [], Exports),
+Tests = lists:reverse(RevTests),
+
+{
+setup,
+spawn,
+SetupMod,
+TeardownMod,
+[
+{
+foreach,
+SetupTest,
+TeardownTest,
+Tests
+}
+]
+}.
+
+
+get_setup_all(ModName, Exports) ->
+case lists:member({setup_all, 0}, Exports) of
+true -> fun ModName:setup_all/0;
+false -> fun cpse_util:setup_all/0
+end.
+
+
+get_teardown_all(ModName, Exports) ->
+case lists:member({teardown_all, 1}, Exports) of
+true -> fun ModName:teardown_all/1;
+false -> fun cpse_util:teardown_all/1
+end.
+
+
+get_fun(ModName, FunName, Arity, Exports) ->
+case lists:member({FunName, Arity}, Exports) of
+true -> fun ModName:FunName/Arity;
+false when Arity == 0 -> fun() -> ok end;
+false when Arity == 1 -> fun(_) -> ok end
+end.
+
+
+make_test_fun(Module, Fun, Arity) ->
+Name = atom_to_list(Fun),
+case Arity of
+0 ->
+fun(_) ->
+{timeout, 60, {Name, fun() ->
+process_flag(trap_exit, true),
+Module:Fun()
+end}}
+end;
+1 ->
+fun(Arg) ->
+{timeout, 60, {Name, fun() ->
+process_flag(trap_exit, true),
+Module:Fun(Arg)
+end}}
+end
+end.
diff --git a/src/couch_pse_tests/src/cpse_test_attachments.erl
b/src/couch_pse_tests/src/cpse_test_attachments.erl
index 7372ba4..61ada38 100644
--- a/src/couch_pse_tests/src/cpse_test_attachments.erl
+++ b/src/couch_pse_tests/src/cpse_test_attachments.erl
@@ -18,9 +18,16 @@
-include_lib("couch/include/couch_db.hrl").
-cet_write_attachment() ->
-{ok, Db1} = cpse_util:create_db(),
+setup_each() ->
+{ok, Db} = cpse_util:create_db(),
+Db.
+
+teardown_each(Db) ->
+ok = couch_server:delete(couch_db:name(Db), []).
+
+
+cpse_write_attachment(Db1) ->
AttBin = crypto:strong_rand_bytes(32768),
try
@@ -72,9 +79,7 @@ cet_write_attachment() ->
% attachments streams when restarting (for instance if
% we ever have something that stores attachemnts in
% an external object store)
-cet_inactive_stream() ->
-{ok, Db1} =
[couchdb] 02/06: Move PSE tests to their own app
This is an automated email from the ASF dual-hosted git repository.
jiangphcn pushed a commit to branch
COUCHDB-3326-clustered-purge-pr3-refactor-pse-tests
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 5230ca02c4a75c637ec914f9a4d6216cad64c7e3
Author: Paul J. Davis
AuthorDate: Fri May 4 08:55:33 2018 -0500
Move PSE tests to their own app
---
src/couch_pse_tests/src/couch_pse_tests.app.src | 20
.../src/test_engine_attachments.erl | 0
.../src/test_engine_compaction.erl | 0
.../src/test_engine_fold_changes.erl | 0
.../src/test_engine_fold_docs.erl| 0
.../src/test_engine_get_set_props.erl| 0
.../src/test_engine_open_close_delete.erl| 0
.../src/test_engine_purge_docs.erl | 0
.../src/test_engine_read_write_docs.erl | 0
.../src/test_engine_ref_counting.erl | 0
.../src/test_engine_util.erl | 0
11 files changed, 20 insertions(+)
diff --git a/src/couch_pse_tests/src/couch_pse_tests.app.src
b/src/couch_pse_tests/src/couch_pse_tests.app.src
new file mode 100644
index 000..83f3875
--- /dev/null
+++ b/src/couch_pse_tests/src/couch_pse_tests.app.src
@@ -0,0 +1,20 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+{application, couch_pse_tests, [
+{description, "Apache CouchDB Pluggable Storage Engine Test Suite"},
+{vsn, git},
+{applications, [
+kernel,
+stdlib
+]}
+]}.
diff --git a/src/couch/src/test_engine_attachments.erl
b/src/couch_pse_tests/src/test_engine_attachments.erl
similarity index 100%
rename from src/couch/src/test_engine_attachments.erl
rename to src/couch_pse_tests/src/test_engine_attachments.erl
diff --git a/src/couch/src/test_engine_compaction.erl
b/src/couch_pse_tests/src/test_engine_compaction.erl
similarity index 100%
rename from src/couch/src/test_engine_compaction.erl
rename to src/couch_pse_tests/src/test_engine_compaction.erl
diff --git a/src/couch/src/test_engine_fold_changes.erl
b/src/couch_pse_tests/src/test_engine_fold_changes.erl
similarity index 100%
rename from src/couch/src/test_engine_fold_changes.erl
rename to src/couch_pse_tests/src/test_engine_fold_changes.erl
diff --git a/src/couch/src/test_engine_fold_docs.erl
b/src/couch_pse_tests/src/test_engine_fold_docs.erl
similarity index 100%
rename from src/couch/src/test_engine_fold_docs.erl
rename to src/couch_pse_tests/src/test_engine_fold_docs.erl
diff --git a/src/couch/src/test_engine_get_set_props.erl
b/src/couch_pse_tests/src/test_engine_get_set_props.erl
similarity index 100%
rename from src/couch/src/test_engine_get_set_props.erl
rename to src/couch_pse_tests/src/test_engine_get_set_props.erl
diff --git a/src/couch/src/test_engine_open_close_delete.erl
b/src/couch_pse_tests/src/test_engine_open_close_delete.erl
similarity index 100%
rename from src/couch/src/test_engine_open_close_delete.erl
rename to src/couch_pse_tests/src/test_engine_open_close_delete.erl
diff --git a/src/couch/src/test_engine_purge_docs.erl
b/src/couch_pse_tests/src/test_engine_purge_docs.erl
similarity index 100%
rename from src/couch/src/test_engine_purge_docs.erl
rename to src/couch_pse_tests/src/test_engine_purge_docs.erl
diff --git a/src/couch/src/test_engine_read_write_docs.erl
b/src/couch_pse_tests/src/test_engine_read_write_docs.erl
similarity index 100%
rename from src/couch/src/test_engine_read_write_docs.erl
rename to src/couch_pse_tests/src/test_engine_read_write_docs.erl
diff --git a/src/couch/src/test_engine_ref_counting.erl
b/src/couch_pse_tests/src/test_engine_ref_counting.erl
similarity index 100%
rename from src/couch/src/test_engine_ref_counting.erl
rename to src/couch_pse_tests/src/test_engine_ref_counting.erl
diff --git a/src/couch/src/test_engine_util.erl
b/src/couch_pse_tests/src/test_engine_util.erl
similarity index 100%
rename from src/couch/src/test_engine_util.erl
rename to src/couch_pse_tests/src/test_engine_util.erl
[couchdb] 04/06: Update PSE test definitions for new util module
This is an automated email from the ASF dual-hosted git repository.
jiangphcn pushed a commit to branch
COUCHDB-3326-clustered-purge-pr3-refactor-pse-tests
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit bbc74b9787b1635cde7f28ff068b72413621ae2d
Author: Paul J. Davis
AuthorDate: Wed May 23 14:28:32 2018 -0500
Update PSE test definitions for new util module
---
src/couch_pse_tests/src/cpse_test_attachments.erl | 16 ++---
src/couch_pse_tests/src/cpse_test_compaction.erl | 74 +++---
src/couch_pse_tests/src/cpse_test_fold_changes.erl | 36 +--
src/couch_pse_tests/src/cpse_test_fold_docs.erl| 4 +-
.../src/cpse_test_get_set_props.erl| 12 ++--
.../src/cpse_test_open_close_delete.erl| 32 +-
src/couch_pse_tests/src/cpse_test_purge_docs.erl | 36 +--
.../src/cpse_test_read_write_docs.erl | 64 +--
src/couch_pse_tests/src/cpse_test_ref_counting.erl | 6 +-
9 files changed, 140 insertions(+), 140 deletions(-)
diff --git a/src/couch_pse_tests/src/cpse_test_attachments.erl
b/src/couch_pse_tests/src/cpse_test_attachments.erl
index d8fd2b8..7372ba4 100644
--- a/src/couch_pse_tests/src/cpse_test_attachments.erl
+++ b/src/couch_pse_tests/src/cpse_test_attachments.erl
@@ -19,12 +19,12 @@
cet_write_attachment() ->
-{ok, Db1} = test_engine_util:create_db(),
+{ok, Db1} = cpse_util:create_db(),
AttBin = crypto:strong_rand_bytes(32768),
try
-[Att0] = test_engine_util:prep_atts(Db1, [
+[Att0] = cpse_util:prep_atts(Db1, [
{<<"ohai.txt">>, AttBin}
]),
@@ -32,9 +32,9 @@ cet_write_attachment() ->
?assertEqual(true, couch_db_engine:is_active_stream(Db1, Stream)),
Actions = [{create, {<<"first">>, {[]}, [Att0]}}],
-{ok, Db2} = test_engine_util:apply_actions(Db1, Actions),
+{ok, Db2} = cpse_util:apply_actions(Db1, Actions),
{ok, _} = couch_db:ensure_full_commit(Db2),
-test_engine_util:shutdown_db(Db2),
+cpse_util:shutdown_db(Db2),
{ok, Db3} = couch_db:reopen(Db2),
@@ -44,7 +44,7 @@ cet_write_attachment() ->
rev = {RevPos, PrevRevId},
deleted = Deleted,
body_sp = DocPtr
-} = test_engine_util:prev_rev(FDI),
+} = cpse_util:prev_rev(FDI),
Doc0 = #doc{
id = <<"foo">>,
@@ -73,19 +73,19 @@ cet_write_attachment() ->
% we ever have something that stores attachemnts in
% an external object store)
cet_inactive_stream() ->
-{ok, Db1} = test_engine_util:create_db(),
+{ok, Db1} = cpse_util:create_db(),
AttBin = crypto:strong_rand_bytes(32768),
try
-[Att0] = test_engine_util:prep_atts(Db1, [
+[Att0] = cpse_util:prep_atts(Db1, [
{<<"ohai.txt">>, AttBin}
]),
{stream, Stream} = couch_att:fetch(data, Att0),
?assertEqual(true, couch_db_engine:is_active_stream(Db1, Stream)),
-test_engine_util:shutdown_db(Db1),
+cpse_util:shutdown_db(Db1),
{ok, Db2} = couch_db:reopen(Db1),
?assertEqual(false, couch_db_engine:is_active_stream(Db2, Stream))
diff --git a/src/couch_pse_tests/src/cpse_test_compaction.erl
b/src/couch_pse_tests/src/cpse_test_compaction.erl
index 0c81707..122feba 100644
--- a/src/couch_pse_tests/src/cpse_test_compaction.erl
+++ b/src/couch_pse_tests/src/cpse_test_compaction.erl
@@ -19,50 +19,50 @@
cet_compact_empty() ->
-{ok, Db1} = test_engine_util:create_db(),
-Term1 = test_engine_util:db_as_term(Db1),
+{ok, Db1} = cpse_util:create_db(),
+Term1 = cpse_util:db_as_term(Db1),
-test_engine_util:compact(Db1),
+cpse_util:compact(Db1),
{ok, Db2} = couch_db:reopen(Db1),
-Term2 = test_engine_util:db_as_term(Db2),
+Term2 = cpse_util:db_as_term(Db2),
-Diff = test_engine_util:term_diff(Term1, Term2),
+Diff = cpse_util:term_diff(Term1, Term2),
?assertEqual(nodiff, Diff).
cet_compact_doc() ->
-{ok, Db1} = test_engine_util:create_db(),
+{ok, Db1} = cpse_util:create_db(),
Actions = [{create, {<<"foo">>, {[]}}}],
-{ok, Db2} = test_engine_util:apply_actions(Db1, Actions),
-Term1 = test_engine_util:db_as_term(Db2),
+{ok, Db2} = cpse_util:apply_actions(Db1, Actions),
+Term1 = cpse_util:db_as_term(Db2),
-test_engine_util:compact(Db2),
+cpse_util:compact(Db2),
{ok, Db3} = couch_db:reopen(Db2),
-Term2 = test_engine_util:db_as_term(Db3),
+Term2 = cpse_util:db_as_term(Db3),
-Diff = test_engine_util:term_diff(Term1, Term2),
+Diff = cpse_util:term_diff(Term1, Term2),
?assertEqual(nodiff, Diff).
cet_compact_local_doc() ->
-{ok, Db1} = test_engine_util:create_db(),
+{ok, Db1} = cpse_util:create_db(),
Actions = [{create, {<<"_local/foo">>, {[]}}}],
-{ok, Db2} = test_engine_util:apply_actions(Db1, Actions),
-Term1 =
[couchdb] 05/06: Update to use new couch_pse_tests app
This is an automated email from the ASF dual-hosted git repository. jiangphcn pushed a commit to branch COUCHDB-3326-clustered-purge-pr3-refactor-pse-tests in repository https://gitbox.apache.org/repos/asf/couchdb.git commit 776349cd5054c319e8775cee5da1f7836818dcc5 Author: Paul J. Davis AuthorDate: Wed May 23 14:31:26 2018 -0500 Update to use new couch_pse_tests app --- rebar.config.script | 1 + src/couch/test/couch_bt_engine_tests.erl | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/rebar.config.script b/rebar.config.script index b4de2d9..22dde7c 100644 --- a/rebar.config.script +++ b/rebar.config.script @@ -31,6 +31,7 @@ SubDirs = [ "src/couch_mrview", "src/couch_replicator", "src/couch_plugins", +"src/couch_pse_tests", "src/couch_event", "src/couch_stats", "src/couch_peruser", diff --git a/src/couch/test/couch_bt_engine_tests.erl b/src/couch/test/couch_bt_engine_tests.erl index c4055d0..3e3ecbf 100644 --- a/src/couch/test/couch_bt_engine_tests.erl +++ b/src/couch/test/couch_bt_engine_tests.erl @@ -17,4 +17,4 @@ couch_bt_engine_test_()-> -test_engine_util:create_tests(couch, couch_bt_engine, "couch"). +cpse_util:create_tests(couch, couch_bt_engine, "couch").
[couchdb] branch COUCHDB-3326-clustered-purge-pr3-refactor-pse-tests updated (4c87d78 -> e0c7ad3)
This is an automated email from the ASF dual-hosted git repository. jiangphcn pushed a change to branch COUCHDB-3326-clustered-purge-pr3-refactor-pse-tests in repository https://gitbox.apache.org/repos/asf/couchdb.git. omit 4c87d78 Enhance PSE tests with setup/teardown functions omit c998aa4 Update to use new couch_pse_tests app omit 0d85abe Update PSE test definitions for new util module omit 3c94bc4 Rename PSE test modules omit db28f4c Move PSE tests to their own app omit f45c036 Rewrite the PSE test suite to use couch_server omit da43788 Simplify logic in mem3_rep add fc10fb5 Merge pull request #1366 from apache/COUCHDB-3326-clustered-purge-pr1-misc-cleanup add 391d7b5 Simplify logic in mem3_rep add 8819a27 Merge pull request #1570 from apache/COUCHDB-3326-clustered-purge-pr2-simplify-mem3-rep new 9b7c2d1 Rewrite the PSE test suite to use couch_server new 5230ca0 Move PSE tests to their own app new d033063 Rename PSE test modules new bbc74b9 Update PSE test definitions for new util module new 776349c Update to use new couch_pse_tests app new e0c7ad3 Enhance PSE tests with setup/teardown functions This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (4c87d78) \ N -- N -- N refs/heads/COUCHDB-3326-clustered-purge-pr3-refactor-pse-tests (e0c7ad3) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. The 6 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes:
[couchdb] 01/06: Rewrite the PSE test suite to use couch_server
This is an automated email from the ASF dual-hosted git repository.
jiangphcn pushed a commit to branch
COUCHDB-3326-clustered-purge-pr3-refactor-pse-tests
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 9b7c2d1bfd011b671e578fed61bf0da69a938675
Author: Paul J. Davis
AuthorDate: Thu Apr 26 11:41:01 2018 -0500
Rewrite the PSE test suite to use couch_server
It turns out that if any storage engine has to open itself during
a callback it would end up violating the guarantee of a single writer.
This change in the test suite changes things to use couch_server so that
storage engines are now free to do as they want reopening themselves.
---
src/couch/src/test_engine_attachments.erl | 35 +--
src/couch/src/test_engine_compaction.erl| 154 +-
src/couch/src/test_engine_fold_changes.erl | 134 +
src/couch/src/test_engine_fold_docs.erl | 99 +++
src/couch/src/test_engine_get_set_props.erl | 84 +++---
src/couch/src/test_engine_open_close_delete.erl | 75 +++--
src/couch/src/test_engine_purge_docs.erl| 140 -
src/couch/src/test_engine_read_write_docs.erl | 300 ++-
src/couch/src/test_engine_ref_counting.erl | 42 +--
src/couch/src/test_engine_util.erl | 370 ++--
src/couch/test/couch_bt_engine_tests.erl| 2 +-
11 files changed, 713 insertions(+), 722 deletions(-)
diff --git a/src/couch/src/test_engine_attachments.erl
b/src/couch/src/test_engine_attachments.erl
index 691d4bd..9763ef5 100644
--- a/src/couch/src/test_engine_attachments.erl
+++ b/src/couch/src/test_engine_attachments.erl
@@ -19,25 +19,26 @@
cet_write_attachment() ->
-{ok, Engine, DbPath, St1} = test_engine_util:init_engine(dbpath),
+{ok, Db1} = test_engine_util:create_db(),
AttBin = crypto:strong_rand_bytes(32768),
try
-[Att0] = test_engine_util:prep_atts(Engine, St1, [
+[Att0] = test_engine_util:prep_atts(Db1, [
{<<"ohai.txt">>, AttBin}
]),
{stream, Stream} = couch_att:fetch(data, Att0),
-?assertEqual(true, Engine:is_active_stream(St1, Stream)),
+?assertEqual(true, couch_db_engine:is_active_stream(Db1, Stream)),
-Actions = [{create, {<<"first">>, [], [Att0]}}],
-{ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions),
-{ok, St3} = Engine:commit_data(St2),
-Engine:terminate(normal, St3),
+Actions = [{create, {<<"first">>, {[]}, [Att0]}}],
+{ok, Db2} = test_engine_util:apply_actions(Db1, Actions),
+{ok, _} = couch_db:ensure_full_commit(Db2),
+test_engine_util:shutdown_db(Db2),
-{ok, St4} = Engine:init(DbPath, []),
-[FDI] = Engine:open_docs(St4, [<<"first">>]),
+{ok, Db3} = couch_db:reopen(Db2),
+
+[FDI] = couch_db_engine:open_docs(Db3, [<<"first">>]),
#rev_info{
rev = {RevPos, PrevRevId},
@@ -52,12 +53,12 @@ cet_write_attachment() ->
body = DocPtr
},
-Doc1 = Engine:read_doc_body(St4, Doc0),
+Doc1 = couch_db_engine:read_doc_body(Db3, Doc0),
Atts1 = if not is_binary(Doc1#doc.atts) -> Doc1#doc.atts; true ->
couch_compress:decompress(Doc1#doc.atts)
end,
-StreamSrc = fun(Sp) -> Engine:open_read_stream(St4, Sp) end,
+StreamSrc = fun(Sp) -> couch_db_engine:open_read_stream(Db3, Sp) end,
[Att1] = [couch_att:from_disk_term(StreamSrc, T) || T <- Atts1],
ReadBin = couch_att:to_binary(Att1),
?assertEqual(AttBin, ReadBin)
@@ -72,22 +73,22 @@ cet_write_attachment() ->
% we ever have something that stores attachemnts in
% an external object store)
cet_inactive_stream() ->
-{ok, Engine, DbPath, St1} = test_engine_util:init_engine(dbpath),
+{ok, Db1} = test_engine_util:create_db(),
AttBin = crypto:strong_rand_bytes(32768),
try
-[Att0] = test_engine_util:prep_atts(Engine, St1, [
+[Att0] = test_engine_util:prep_atts(Db1, [
{<<"ohai.txt">>, AttBin}
]),
{stream, Stream} = couch_att:fetch(data, Att0),
-?assertEqual(true, Engine:is_active_stream(St1, Stream)),
+?assertEqual(true, couch_db_engine:is_active_stream(Db1, Stream)),
-Engine:terminate(normal, St1),
-{ok, St2} = Engine:init(DbPath, []),
+test_engine_util:shutdown_db(Db1),
+{ok, Db2} = couch_db:reopen(Db1),
-?assertEqual(false, Engine:is_active_stream(St2, Stream))
+?assertEqual(false, couch_db_engine:is_active_stream(Db2, Stream))
catch throw:not_supported ->
ok
end.
diff --git a/src/couch/src/test_engine_compaction.erl
b/src/couch/src/test_engine_compaction.erl
index 09a1e4e..44c5357 100644
--- a/src/couch/src/test_engine_compaction.erl
+++ b/src/couch/src/test_engine_compaction.erl
@@ -19,66 +19,75 @@
[couchdb] 03/06: Rename PSE test modules
This is an automated email from the ASF dual-hosted git repository.
jiangphcn pushed a commit to branch
COUCHDB-3326-clustered-purge-pr3-refactor-pse-tests
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit d0330635008067b642bd56a553958810425e5461
Author: Paul J. Davis
AuthorDate: Wed May 23 14:23:07 2018 -0500
Rename PSE test modules
---
...e_attachments.erl => cpse_test_attachments.erl} | 2 +-
...ine_compaction.erl => cpse_test_compaction.erl} | 2 +-
...fold_changes.erl => cpse_test_fold_changes.erl} | 2 +-
...ngine_fold_docs.erl => cpse_test_fold_docs.erl} | 2 +-
...t_set_props.erl => cpse_test_get_set_props.erl} | 2 +-
..._delete.erl => cpse_test_open_close_delete.erl} | 2 +-
...ine_purge_docs.erl => cpse_test_purge_docs.erl} | 2 +-
...rite_docs.erl => cpse_test_read_write_docs.erl} | 2 +-
...ref_counting.erl => cpse_test_ref_counting.erl} | 2 +-
.../src/{test_engine_util.erl => cpse_util.erl}| 51 ++
10 files changed, 22 insertions(+), 47 deletions(-)
diff --git a/src/couch_pse_tests/src/test_engine_attachments.erl
b/src/couch_pse_tests/src/cpse_test_attachments.erl
similarity index 98%
rename from src/couch_pse_tests/src/test_engine_attachments.erl
rename to src/couch_pse_tests/src/cpse_test_attachments.erl
index 9763ef5..d8fd2b8 100644
--- a/src/couch_pse_tests/src/test_engine_attachments.erl
+++ b/src/couch_pse_tests/src/cpse_test_attachments.erl
@@ -10,7 +10,7 @@
% License for the specific language governing permissions and limitations under
% the License.
--module(test_engine_attachments).
+-module(cpse_test_attachments).
-compile(export_all).
diff --git a/src/couch_pse_tests/src/test_engine_compaction.erl
b/src/couch_pse_tests/src/cpse_test_compaction.erl
similarity index 99%
rename from src/couch_pse_tests/src/test_engine_compaction.erl
rename to src/couch_pse_tests/src/cpse_test_compaction.erl
index 44c5357..0c81707 100644
--- a/src/couch_pse_tests/src/test_engine_compaction.erl
+++ b/src/couch_pse_tests/src/cpse_test_compaction.erl
@@ -10,7 +10,7 @@
% License for the specific language governing permissions and limitations under
% the License.
--module(test_engine_compaction).
+-module(cpse_test_compaction).
-compile(export_all).
diff --git a/src/couch_pse_tests/src/test_engine_fold_changes.erl
b/src/couch_pse_tests/src/cpse_test_fold_changes.erl
similarity index 99%
rename from src/couch_pse_tests/src/test_engine_fold_changes.erl
rename to src/couch_pse_tests/src/cpse_test_fold_changes.erl
index 4ca09a8..5ed45c1 100644
--- a/src/couch_pse_tests/src/test_engine_fold_changes.erl
+++ b/src/couch_pse_tests/src/cpse_test_fold_changes.erl
@@ -10,7 +10,7 @@
% License for the specific language governing permissions and limitations under
% the License.
--module(test_engine_fold_changes).
+-module(cpse_test_fold_changes).
-compile(export_all).
diff --git a/src/couch_pse_tests/src/test_engine_fold_docs.erl
b/src/couch_pse_tests/src/cpse_test_fold_docs.erl
similarity index 99%
rename from src/couch_pse_tests/src/test_engine_fold_docs.erl
rename to src/couch_pse_tests/src/cpse_test_fold_docs.erl
index 3ed068f..fbea7da 100644
--- a/src/couch_pse_tests/src/test_engine_fold_docs.erl
+++ b/src/couch_pse_tests/src/cpse_test_fold_docs.erl
@@ -10,7 +10,7 @@
% License for the specific language governing permissions and limitations under
% the License.
--module(test_engine_fold_docs).
+-module(cpse_test_fold_docs).
-compile(export_all).
diff --git a/src/couch_pse_tests/src/test_engine_get_set_props.erl
b/src/couch_pse_tests/src/cpse_test_get_set_props.erl
similarity index 98%
rename from src/couch_pse_tests/src/test_engine_get_set_props.erl
rename to src/couch_pse_tests/src/cpse_test_get_set_props.erl
index 764fe39..678f0cb 100644
--- a/src/couch_pse_tests/src/test_engine_get_set_props.erl
+++ b/src/couch_pse_tests/src/cpse_test_get_set_props.erl
@@ -10,7 +10,7 @@
% License for the specific language governing permissions and limitations under
% the License.
--module(test_engine_get_set_props).
+-module(cpse_test_get_set_props).
-compile(export_all).
diff --git a/src/couch_pse_tests/src/test_engine_open_close_delete.erl
b/src/couch_pse_tests/src/cpse_test_open_close_delete.erl
similarity index 98%
rename from src/couch_pse_tests/src/test_engine_open_close_delete.erl
rename to src/couch_pse_tests/src/cpse_test_open_close_delete.erl
index ce0187e..3fb023b 100644
--- a/src/couch_pse_tests/src/test_engine_open_close_delete.erl
+++ b/src/couch_pse_tests/src/cpse_test_open_close_delete.erl
@@ -10,7 +10,7 @@
% License for the specific language governing permissions and limitations under
% the License.
--module(test_engine_open_close_delete).
+-module(cpse_test_open_close_delete).
-compile(export_all).
diff --git a/src/couch_pse_tests/src/test_engine_purge_docs.erl
b/src/couch_pse_tests/src/cpse_test_purge_docs.erl
similarity index 99%
rename from
[couchdb] 01/01: Merge pull request #1570 from apache/COUCHDB-3326-clustered-purge-pr2-simplify-mem3-rep
This is an automated email from the ASF dual-hosted git repository. jiangphcn pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/couchdb.git commit 8819a27864e4f4dbe0f4544986749b51d4a1d7eb Merge: fc10fb5 391d7b5 Author: Peng Hui Jiang AuthorDate: Tue Aug 21 23:28:15 2018 +0800 Merge pull request #1570 from apache/COUCHDB-3326-clustered-purge-pr2-simplify-mem3-rep Simplify logic in mem3_rep src/mem3/src/mem3_rep.erl | 19 +-- 1 file changed, 9 insertions(+), 10 deletions(-)
[couchdb] branch COUCHDB-3326-clustered-purge-pr2-simplify-mem3-rep updated (169e84c -> 391d7b5)
This is an automated email from the ASF dual-hosted git repository. davisp pushed a change to branch COUCHDB-3326-clustered-purge-pr2-simplify-mem3-rep in repository https://gitbox.apache.org/repos/asf/couchdb.git. omit 169e84c Merge branch 'master' into COUCHDB-3326-clustered-purge-pr2-simplify-mem3-rep omit da43788 Simplify logic in mem3_rep add 391d7b5 Simplify logic in mem3_rep This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (169e84c) \ N -- N -- N refs/heads/COUCHDB-3326-clustered-purge-pr2-simplify-mem3-rep (391d7b5) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes:
[couchdb] 01/01: Merge branch 'master' into COUCHDB-3326-clustered-purge-pr2-simplify-mem3-rep
This is an automated email from the ASF dual-hosted git repository. davisp pushed a commit to branch COUCHDB-3326-clustered-purge-pr2-simplify-mem3-rep in repository https://gitbox.apache.org/repos/asf/couchdb.git commit 169e84c9bb58fcf9e10ceaece6c5f6974511ae62 Merge: da43788 fc10fb5 Author: Paul J. Davis AuthorDate: Tue Aug 21 09:38:48 2018 -0500 Merge branch 'master' into COUCHDB-3326-clustered-purge-pr2-simplify-mem3-rep
[couchdb] branch COUCHDB-3326-clustered-purge-pr2-simplify-mem3-rep updated (da43788 -> 169e84c)
This is an automated email from the ASF dual-hosted git repository. davisp pushed a change to branch COUCHDB-3326-clustered-purge-pr2-simplify-mem3-rep in repository https://gitbox.apache.org/repos/asf/couchdb.git. from da43788 Simplify logic in mem3_rep add fc10fb5 Merge pull request #1366 from apache/COUCHDB-3326-clustered-purge-pr1-misc-cleanup new 169e84c Merge branch 'master' into COUCHDB-3326-clustered-purge-pr2-simplify-mem3-rep The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes:
[couchdb] branch revert-1367-COUCHDB-3326-clustered-purge-pr2-simplify-mem3-rep created (now 2390092)
This is an automated email from the ASF dual-hosted git repository. jiangphcn pushed a change to branch revert-1367-COUCHDB-3326-clustered-purge-pr2-simplify-mem3-rep in repository https://gitbox.apache.org/repos/asf/couchdb.git. at 2390092 Revert "[2/5] Clustered Purge - Simplify mem3_rep" This branch includes the following new commits: new 2390092 Revert "[2/5] Clustered Purge - Simplify mem3_rep" The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
[couchdb] 01/01: Revert "[2/5] Clustered Purge - Simplify mem3_rep"
This is an automated email from the ASF dual-hosted git repository.
jiangphcn pushed a commit to branch
revert-1367-COUCHDB-3326-clustered-purge-pr2-simplify-mem3-rep
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 23900925f3fccb6a22c7f31643a3b999739aa63e
Author: Peng Hui Jiang
AuthorDate: Tue Aug 21 22:28:04 2018 +0800
Revert "[2/5] Clustered Purge - Simplify mem3_rep"
---
src/mem3/src/mem3_rep.erl | 19 ++-
1 file changed, 10 insertions(+), 9 deletions(-)
diff --git a/src/mem3/src/mem3_rep.erl b/src/mem3/src/mem3_rep.erl
index 670f990..8d996d6 100644
--- a/src/mem3/src/mem3_rep.erl
+++ b/src/mem3/src/mem3_rep.erl
@@ -73,11 +73,12 @@ go(#shard{} = Source, #shard{} = Target, Opts) ->
go(Acc).
-go(#acc{source=Source, batch_count=BC}=Acc) ->
+go(#acc{source=Source, batch_count=BC}=Acc0) ->
case couch_db:open(Source#shard.name, [?ADMIN_CTX]) of
{ok, Db} ->
+Acc = Acc0#acc{db=Db},
Resp = try
-repl(Acc#acc{db = Db})
+repl(Db, Acc)
catch error:{not_found, no_db_file} ->
{error, missing_target}
after
@@ -169,9 +170,9 @@ find_source_seq_int(#doc{body={Props}}, SrcNode0, TgtNode0,
TgtUUID, TgtSeq) ->
end.
-repl(#acc{db = Db} = Acc0) ->
+repl(Db, Acc0) ->
erlang:put(io_priority, {internal_repl, couch_db:name(Db)}),
-#acc{seq=Seq} = Acc1 = calculate_start_seq(Acc0),
+#acc{seq=Seq} = Acc1 = calculate_start_seq(Acc0#acc{source = Db}),
case Seq >= couch_db:get_update_seq(Db) of
true ->
{ok, 0};
@@ -185,7 +186,7 @@ repl(#acc{db = Db} = Acc0) ->
calculate_start_seq(Acc) ->
#acc{
-db = Db,
+source = Db,
target = #shard{node=Node, name=Name}
} = Acc,
%% Give the target our UUID and ask it to return the checkpoint doc
@@ -221,7 +222,7 @@ calculate_start_seq(Acc) ->
compare_epochs(Acc) ->
#acc{
-db = Db,
+source = Db,
target = #shard{node=Node, name=Name}
} = Acc,
UUID = couch_db:get_uuid(Db),
@@ -302,13 +303,13 @@ chunk_revs([{Id, R, A}|Revs], {Count, Chunk}, Chunks,
Limit) ->
).
-open_docs(#acc{db=Db, infos=Infos}, Missing) ->
+open_docs(#acc{source=Source, infos=Infos}, Missing) ->
lists:flatmap(fun({Id, Revs, _}) ->
FDI = lists:keyfind(Id, #full_doc_info.id, Infos),
#full_doc_info{rev_tree=RevTree} = FDI,
{FoundRevs, _} = couch_key_tree:get_key_leafs(RevTree, Revs),
lists:map(fun({#leaf{deleted=IsDel, ptr=SummaryPtr}, FoundRevPath}) ->
-couch_db:make_doc(Db, Id, IsDel, SummaryPtr, FoundRevPath)
+couch_db:make_doc(Source, Id, IsDel, SummaryPtr, FoundRevPath)
end, FoundRevs)
end, Missing).
@@ -324,7 +325,7 @@ save_on_target(Node, Name, Docs) ->
update_locals(Acc) ->
-#acc{seq=Seq, db=Db, target=Target, localid=Id, history=History} = Acc,
+#acc{seq=Seq, source=Db, target=Target, localid=Id, history=History} = Acc,
#shard{name=Name, node=Node} = Target,
NewEntry = [
{<<"source_node">>, atom_to_binary(node(), utf8)},
[couchdb] 01/01: Merge pull request #1367 from apache/COUCHDB-3326-clustered-purge-pr2-simplify-mem3-rep
This is an automated email from the ASF dual-hosted git repository. jiangphcn pushed a commit to branch COUCHDB-3326-clustered-purge-pr1-misc-cleanup in repository https://gitbox.apache.org/repos/asf/couchdb.git commit 046ef3b3131b04465ab8ff276e46fd663b28fe36 Merge: aea2833 da43788 Author: Peng Hui Jiang AuthorDate: Tue Aug 21 22:17:34 2018 +0800 Merge pull request #1367 from apache/COUCHDB-3326-clustered-purge-pr2-simplify-mem3-rep [2/5] Clustered Purge - Simplify mem3_rep src/mem3/src/mem3_rep.erl | 19 +-- 1 file changed, 9 insertions(+), 10 deletions(-)
[couchdb] branch master updated (b847adb -> fc10fb5)
This is an automated email from the ASF dual-hosted git repository. jiangphcn pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/couchdb.git. from b847adb Merge pull request #1543 from cloudant/implement-node-restart add 3852a00 Fix typos in couch_db_engine.erl add da2e22a Fix bug during purge add 151b9b0 Fix default security object handling add bb27d5e Fix race on couch_db:reopen/1 add aea2833 Update fabric_doc_open eunit tests new fc10fb5 Merge pull request #1366 from apache/COUCHDB-3326-clustered-purge-pr1-misc-cleanup The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: src/couch/src/couch_bt_engine.erl | 2 +- src/couch/src/couch_db.erl | 7 +- src/couch/src/couch_db_engine.erl | 10 +- src/couch/src/couch_db_updater.erl | 2 +- src/fabric/src/fabric_doc_open.erl | 631 ++--- 5 files changed, 322 insertions(+), 330 deletions(-)
[couchdb] branch COUCHDB-3326-clustered-purge-pr1-misc-cleanup updated (aea2833 -> 046ef3b)
This is an automated email from the ASF dual-hosted git repository. jiangphcn pushed a change to branch COUCHDB-3326-clustered-purge-pr1-misc-cleanup in repository https://gitbox.apache.org/repos/asf/couchdb.git. from aea2833 Update fabric_doc_open eunit tests add da43788 Simplify logic in mem3_rep new 046ef3b Merge pull request #1367 from apache/COUCHDB-3326-clustered-purge-pr2-simplify-mem3-rep The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: src/mem3/src/mem3_rep.erl | 19 +-- 1 file changed, 9 insertions(+), 10 deletions(-)
[couchdb] 01/01: Merge pull request #1366 from apache/COUCHDB-3326-clustered-purge-pr1-misc-cleanup
This is an automated email from the ASF dual-hosted git repository. jiangphcn pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/couchdb.git commit fc10fb522a2f0623cd5c3a3d1a3427c35066bd29 Merge: b847adb aea2833 Author: Peng Hui Jiang AuthorDate: Tue Aug 21 22:16:50 2018 +0800 Merge pull request #1366 from apache/COUCHDB-3326-clustered-purge-pr1-misc-cleanup [1/5] Clustered Purge - Misc Cleanup src/couch/src/couch_bt_engine.erl | 2 +- src/couch/src/couch_db.erl | 7 +- src/couch/src/couch_db_engine.erl | 10 +- src/couch/src/couch_db_updater.erl | 2 +- src/fabric/src/fabric_doc_open.erl | 631 ++--- 5 files changed, 322 insertions(+), 330 deletions(-)
[couchdb-nano] branch master updated: removed spatial function - fixes issue #103 (#117)
This is an automated email from the ASF dual-hosted git repository.
glynnbird pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/couchdb-nano.git
The following commit(s) were added to refs/heads/master by this push:
new f69ab4f removed spatial function - fixes issue #103 (#117)
f69ab4f is described below
commit f69ab4f13fccd1f1a4d675a18cd901e8208972c5
Author: Glynn Bird
AuthorDate: Tue Aug 21 12:41:02 2018 +0100
removed spatial function - fixes issue #103 (#117)
---
lib/nano.d.ts| 11 ---
lib/nano.js | 6 --
tests/unit/design/spatial.js | 28
3 files changed, 45 deletions(-)
diff --git a/lib/nano.d.ts b/lib/nano.d.ts
index e00befd..b6dd3ae 100644
--- a/lib/nano.d.ts
+++ b/lib/nano.d.ts
@@ -235,17 +235,6 @@ declare namespace nano {
params: DocumentSearchParams,
callback?: Callback>
): Request;
-spatial(
- ddoc: string,
- viewname: string,
- callback?: Callback
-): Promise;
-spatial(
- ddoc: string,
- viewname: string,
- params: any,
- callback?: Callback
-): Promise;
//
http://docs.couchdb.org/en/latest/api/ddoc/views.html#get--db-_design-ddoc-_view-view
//
http://docs.couchdb.org/en/latest/api/ddoc/views.html#post--db-_design-ddoc-_view-view
view(
diff --git a/lib/nano.js b/lib/nano.js
index 8809f90..572c9d9 100644
--- a/lib/nano.js
+++ b/lib/nano.js
@@ -667,11 +667,6 @@ module.exports = exports = function dbScope (cfg) {
return view(ddoc, viewName, {type: 'view', stream: true}, qs, callback)
}
-// geocouch
-function viewSpatial (ddoc, viewName, qs, callback) {
- return view(ddoc, viewName, {type: 'spatial'}, qs, callback)
-}
-
// cloudant
function viewSearch (ddoc, viewName, qs, callback) {
return view(ddoc, viewName, {type: 'search'}, qs, callback)
@@ -912,7 +907,6 @@ module.exports = exports = function dbScope (cfg) {
updateWithHandler: updateWithHandler,
search: viewSearch,
searchAsStream: viewSearchAsStream,
- spatial: viewSpatial,
view: viewDocs,
viewAsStream: viewDocsAsStream,
find: find,
diff --git a/tests/unit/design/spatial.js b/tests/unit/design/spatial.js
deleted file mode 100644
index da02d1b..000
--- a/tests/unit/design/spatial.js
+++ /dev/null
@@ -1,28 +0,0 @@
-// Licensed under the Apache License, Version 2.0 (the 'License'); you may not
-// use this file except in compliance with the License. You may obtain a copy
of
-// the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an 'AS IS' BASIS, WITHOUT
-// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-// License for the specific language governing permissions and limitations
under
-// the License.
-
-'use strict'
-
-const geoDesign = require('../../helpers/unit').unit([
- 'view',
- 'spatial'
-])
-
-geoDesign('people', 'byArea', {x: '1'}, {
- headers: {
-accept: 'application/json',
-'content-type': 'application/json'
- },
- method: 'GET',
- qs: {x: '1'},
- uri: '/mock/_design/people/_spatial/byArea'
-})
[couchdb-nano] branch issue103 deleted (was c35166d)
This is an automated email from the ASF dual-hosted git repository. glynnbird pushed a change to branch issue103 in repository https://gitbox.apache.org/repos/asf/couchdb-nano.git. was c35166d removed spatial function - fixes issue #103 The revisions that were on this branch are still contained in other references; therefore, this change does not discard any commits from the repository.
[couchdb] 01/05: implement partitioned views
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch user-partitioned-dbs-6
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 4369b7d81b30d41357a2e9f38d84cd24959872d4
Author: Robert Newson
AuthorDate: Tue Aug 7 15:44:33 2018 +0100
implement partitioned views
Co-authored-by: Robert Newson
Co-authored-by: Paul J. Davis
---
src/couch/src/couch_btree.erl | 14 +++
src/couch/src/couch_ejson_compare.erl | 4 ++
src/couch_mrview/src/couch_mrview.erl | 2 +
src/couch_mrview/src/couch_mrview_updater.erl | 14 ++-
src/couch_mrview/src/couch_mrview_util.erl| 53 ---
src/fabric/src/fabric_view.erl| 18 +++--
6 files changed, 95 insertions(+), 10 deletions(-)
diff --git a/src/couch/src/couch_btree.erl b/src/couch/src/couch_btree.erl
index ea224b1..d11d7e6 100644
--- a/src/couch/src/couch_btree.erl
+++ b/src/couch/src/couch_btree.erl
@@ -133,6 +133,20 @@ make_group_fun(Bt, exact) ->
end;
make_group_fun(Bt, GroupLevel) when is_integer(GroupLevel), GroupLevel > 0 ->
fun
+({{p, _Partition, Key1}, _}, {{p, _Partition, Key2}, _}) ->
+SL1 = lists:sublist(Key1, GroupLevel),
+SL2 = lists:sublist(Key2, GroupLevel),
+case less(Bt, {SL1, nil}, {SL2, nil}) of
+false ->
+case less(Bt, {SL2, nil}, {SL1, nil}) of
+false ->
+true;
+_ ->
+false
+end;
+_ ->
+false
+end;
({[_|_] = Key1, _}, {[_|_] = Key2, _}) ->
SL1 = lists:sublist(Key1, GroupLevel),
SL2 = lists:sublist(Key2, GroupLevel),
diff --git a/src/couch/src/couch_ejson_compare.erl
b/src/couch/src/couch_ejson_compare.erl
index 81adbb8..ca36c86 100644
--- a/src/couch/src/couch_ejson_compare.erl
+++ b/src/couch/src/couch_ejson_compare.erl
@@ -22,6 +22,10 @@ init() ->
Dir = code:priv_dir(couch),
ok = erlang:load_nif(filename:join(Dir, ?MODULE), NumScheds).
+% partitioned row comparison
+less({p, PA, A}, {p, PB, B}) ->
+less([PA, A], [PB, B]);
+
less(A, B) ->
try
less_nif(A, B)
diff --git a/src/couch_mrview/src/couch_mrview.erl
b/src/couch_mrview/src/couch_mrview.erl
index db467f0..09945f5 100644
--- a/src/couch_mrview/src/couch_mrview.erl
+++ b/src/couch_mrview/src/couch_mrview.erl
@@ -614,6 +614,8 @@ red_fold(Db, {NthRed, _Lang, View}=RedView, Args, Callback,
UAcc) ->
end, Acc, OptList),
finish_fold(Acc2, []).
+red_fold({p, _Partition, Key}, Red, Acc) ->
+red_fold(Key, Red, Acc);
red_fold(_Key, _Red, #mracc{skip=N}=Acc) when N > 0 ->
{ok, Acc#mracc{skip=N-1, last_go=ok}};
red_fold(Key, Red, #mracc{meta_sent=false}=Acc) ->
diff --git a/src/couch_mrview/src/couch_mrview_updater.erl
b/src/couch_mrview/src/couch_mrview_updater.erl
index 214f487..2b69eee 100644
--- a/src/couch_mrview/src/couch_mrview_updater.erl
+++ b/src/couch_mrview/src/couch_mrview_updater.erl
@@ -311,9 +311,11 @@ write_kvs(State, UpdateSeq, ViewKVs, DocIdKeys, Seqs,
Log0) ->
#mrst{
id_btree=IdBtree,
log_btree=LogBtree,
-first_build=FirstBuild
+first_build=FirstBuild,
+design_opts=DesignOpts
} = State,
+Partitioned = couch_util:get_value(<<"partitioned">>, DesignOpts, false),
Revs = dict:from_list(dict:fetch_keys(Log0)),
Log = dict:fold(fun({Id, _Rev}, DIKeys, Acc) ->
@@ -328,8 +330,9 @@ write_kvs(State, UpdateSeq, ViewKVs, DocIdKeys, Seqs, Log0)
->
_ -> update_log(LogBtree, Log, Revs, Seqs, FirstBuild)
end,
-UpdateView = fun(#mrview{id_num=ViewId}=View, {ViewId, {KVs, SKVs}}) ->
+UpdateView = fun(#mrview{id_num=ViewId}=View, {ViewId, {KVs0, SKVs}}) ->
#mrview{seq_indexed=SIndexed, keyseq_indexed=KSIndexed} = View,
+KVs = if Partitioned -> inject_partition(KVs0); true -> KVs0 end,
ToRem = couch_util:dict_find(ViewId, ToRemByView, []),
{ok, VBtree2} = couch_btree:add_remove(View#mrview.btree, KVs, ToRem),
NewUpdateSeq = case VBtree2 =/= View#mrview.btree of
@@ -378,6 +381,13 @@ write_kvs(State, UpdateSeq, ViewKVs, DocIdKeys, Seqs,
Log0) ->
log_btree=LogBtree2
}.
+inject_partition(KVs) ->
+[{{{p, partition(DocId), Key}, DocId}, Value} || {{Key, DocId}, Value} <-
KVs].
+
+partition(DocId) ->
+[Partition, _Rest] = binary:split(DocId, <<":">>),
+Partition.
+
update_id_btree(Btree, DocIdKeys, true) ->
ToAdd = [{Id, DIKeys} || {Id, DIKeys} <- DocIdKeys, DIKeys /= []],
couch_btree:query_modify(Btree, [], ToAdd, []);
diff --git a/src/couch_mrview/src/couch_mrview_util.erl
b/src/couch_mrview/src/couch_mrview_util.erl
index 592bfb5..574aac7 100644
--- a/src/couch_mrview/src/couch_mrview_util.erl
+++
[couchdb] 02/05: mem3 tests for partitioned databases
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch user-partitioned-dbs-6
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 0ff8646e06fb38e2a63368b0bba83ae56fa1905c
Author: Garren Smith
AuthorDate: Tue Aug 7 12:15:14 2018 +0200
mem3 tests for partitioned databases
---
src/mem3/src/mem3.erl| 22 ++
src/mem3/src/mem3_shards.erl | 68 +++-
src/mem3/src/mem3_util.erl | 59 +-
3 files changed, 147 insertions(+), 2 deletions(-)
diff --git a/src/mem3/src/mem3.erl b/src/mem3/src/mem3.erl
index 99c0863..aecca2f 100644
--- a/src/mem3/src/mem3.erl
+++ b/src/mem3/src/mem3.erl
@@ -389,4 +389,26 @@ allowed_nodes_test_() ->
]
}]}.
+is_partitioned_false_shards_db_test() ->
+meck:expect(config, get, fun (_, _, Default) -> Default end),
+?assertEqual(is_partitioned(<<"_dbs">>), false),
+meck:unload().
+
+is_partitioned_false_nodes_db_test() ->
+meck:expect(config, get, fun (_, _, Default) -> Default end),
+?assertEqual(is_partitioned(<<"_nodes">>), false),
+meck:unload().
+
+is_partitioned_true_partitioned_db_test() ->
+Shard = #shard{
+opts = [{partitioned, true}]
+},
+?assertEqual(is_partitioned([Shard]), true).
+
+is_partitioned_false_partitioned_db_test() ->
+Shard = #shard{
+opts = []
+},
+?assertEqual(is_partitioned([Shard]), false).
+
-endif.
diff --git a/src/mem3/src/mem3_shards.erl b/src/mem3/src/mem3_shards.erl
index 1cd7be6..cfbc32e 100644
--- a/src/mem3/src/mem3_shards.erl
+++ b/src/mem3/src/mem3_shards.erl
@@ -559,7 +559,9 @@ mem3_shards_test_() ->
t_writer_does_not_delete_other_writers_for_same_shard(),
t_spawn_writer_in_load_shards_from_db(),
t_cache_insert_takes_new_update(),
-t_cache_insert_ignores_stale_update_and_kills_worker()
+t_cache_insert_ignores_stale_update_and_kills_worker(),
+t_load_shards_from_disk_returns_correct_shard_for_partition(),
+t_for_docid_returns_correct_shard_for_partition()
]
}.
@@ -703,6 +705,70 @@ t_cache_insert_ignores_stale_update_and_kills_worker() ->
?assertEqual([], ets:tab2list(?OPENERS))
end).
+t_load_shards_from_disk_returns_correct_shard_for_partition() ->
+?_test(begin
+Shards = [
+#ordered_shard{
+name = <<"shards/8000-9fff/db1.1533630706">>,
+node = 'node1@127.0.0.1',
+dbname = <<"db1">>,
+range = [2147483648,2684354559],
+ref = undefined,
+order = 1,
+opts = [{partitioned,true}]
+}
+],
+DbName = <<"db1">>,
+DocId = <<"foo:123">>,
+Doc = #doc{body = {[]}},
+meck:expect(couch_db, open_doc, 3, {ok, Doc}),
+meck:expect(couch_db, get_update_seq, 1, 1),
+meck:expect(mem3_util, build_ordered_shards, 2, Shards),
+meck:expect(mem3_util, ensure_exists, 1, {ok, <<"shard-name">>}),
+meck:expect(couch_db, close, 1, ok),
+
+[Shard] = load_shards_from_disk(DbName, DocId),
+
+meck:validate(couch_db),
+meck:validate(mem3_util),
+
+ShardName = Shard#ordered_shard.name,
+?assertEqual(ShardName, <<"shards/8000-9fff/db1.1533630706">>)
+end).
+
+t_for_docid_returns_correct_shard_for_partition() ->
+?_test(begin
+Shards = [
+#ordered_shard{
+name = <<"shards/6000-7fff/db1.1533630706">>,
+node = 'node1@127.0.0.1',
+dbname = <<"db1">>,
+range = [1610612736,2147483647],
+ref = undefined,
+order = 1,
+opts = [{partitioned,true}]
+},
+#ordered_shard{
+name = <<"shards/8000-9fff/db1.1533630706">>,
+node = 'node1@127.0.0.1',
+dbname = <<"db1">>,
+range = [2147483648,2684354559],
+ref = undefined,
+order = 1,
+opts = [{partitioned,true}]
+}
+],
+DbName = <<"db1">>,
+DocId = <<"foo:123">>,
+
+true = ets:insert(?SHARDS, Shards),
+
+[Shard] = for_docid(DbName, DocId, [ordered]),
+
+ShardName = Shard#ordered_shard.name,
+?assertEqual(ShardName,
<<"shards/8000-9fff/db1.1533630706">>)
+end).
+
mock_state(UpdateSeq) ->
#st{
diff --git a/src/mem3/src/mem3_util.erl b/src/mem3/src/mem3_util.erl
index 9620e98..7b8dd16 100644
--- a/src/mem3/src/mem3_util.erl
+++ b/src/mem3/src/mem3_util.erl
@@ -35,7 +35,6 @@ hash(Item) when is_binary(Item) ->
hash(Item) ->
[couchdb] 04/05: optimize _all_docs requests that are bounded within a single partition
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch user-partitioned-dbs-6
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 48412df84c16ee4fd6b10ac9d672912e055eaf68
Author: Robert Newson
AuthorDate: Mon Aug 13 22:34:59 2018 +0100
optimize _all_docs requests that are bounded within a single partition
---
src/fabric/src/fabric_view_all_docs.erl | 24 +++-
1 file changed, 23 insertions(+), 1 deletion(-)
diff --git a/src/fabric/src/fabric_view_all_docs.erl
b/src/fabric/src/fabric_view_all_docs.erl
index d515ab8..83c3790 100644
--- a/src/fabric/src/fabric_view_all_docs.erl
+++ b/src/fabric/src/fabric_view_all_docs.erl
@@ -21,7 +21,7 @@
-include_lib("couch_mrview/include/couch_mrview.hrl").
go(DbName, Options, #mrargs{keys=undefined} = QueryArgs, Callback, Acc) ->
-Shards = mem3:shards(DbName),
+Shards = shards(DbName, QueryArgs),
Workers0 = fabric_util:submit_jobs(
Shards, fabric_rpc, all_docs, [Options, QueryArgs]),
RexiMon = fabric_util:create_monitors(Workers0),
@@ -136,6 +136,28 @@ go(DbName, _Options, Workers, QueryArgs, Callback, Acc0) ->
{ok, Resp}
end.
+shards(DbName, Args) ->
+case couch_mrview_util:get_extra(Args, partitioned) of
+true ->
+StartKey = partition(Args#mrargs.start_key),
+EndKey = partition(Args#mrargs.end_key),
+case {StartKey, EndKey} of
+{Same, Same} when Same =/= undefined ->
+mem3:shards(DbName, <>);
+{_, _} ->
+mem3:shards(DbName)
+end;
+_ ->
+mem3:shards(DbName)
+end.
+
+partition(undefined) ->
+undefined;
+partition(null) ->
+null;
+partition(Key) when is_binary(Key) ->
+hd(binary:split(Key, <<":">>)).
+
handle_message({rexi_DOWN, _, {_, NodeRef}, _}, _, State) ->
fabric_view:check_down_shards(State, NodeRef);
[couchdb] 03/05: Implement _all_docs and _find support
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch user-partitioned-dbs-6
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 45c1df0f92920aab77448b59f17553b37ad5c22f
Author: Robert Newson
AuthorDate: Thu Aug 9 13:43:17 2018 +0100
Implement _all_docs and _find support
Co-authored-by: Garren Smith
Co-authored-by: Robert Newson
---
src/chttpd/src/chttpd_db.erl | 9 ++--
src/couch_mrview/src/couch_mrview.erl | 7 +--
src/couch_mrview/src/couch_mrview_updater.erl | 4 +-
src/couch_mrview/src/couch_mrview_util.erl| 75 ++-
src/fabric/src/fabric.erl | 7 ++-
src/fabric/src/fabric_view.erl| 2 +-
src/fabric/src/fabric_view_all_docs.erl | 2 +-
src/mango/src/mango_cursor_view.erl | 23 ++--
src/mango/src/mango_opts.erl | 6 +++
src/mem3/src/mem3.erl | 5 +-
10 files changed, 112 insertions(+), 28 deletions(-)
diff --git a/src/chttpd/src/chttpd_db.erl b/src/chttpd/src/chttpd_db.erl
index ddcad97..06b3296 100644
--- a/src/chttpd/src/chttpd_db.erl
+++ b/src/chttpd/src/chttpd_db.erl
@@ -684,12 +684,15 @@ multi_all_docs_view(Req, Db, OP, Queries) ->
all_docs_view(Req, Db, Keys, OP) ->
Args0 = couch_mrview_http:parse_params(Req, Keys),
Args1 = Args0#mrargs{view_type=map},
-Args2 = couch_mrview_util:validate_args(Args1),
-Args3 = set_namespace(OP, Args2),
+DbPartitioned = mem3:is_partitioned(couch_db:name(Db)),
+Args2 = couch_mrview_util:set_extra(Args1, partitioned, DbPartitioned),
+Args3 = couch_mrview_util:set_extra(Args2, style, all_docs),
+Args4 = couch_mrview_util:validate_args(Args3),
+Args5 = set_namespace(OP, Args4),
Options = [{user_ctx, Req#httpd.user_ctx}],
Max = chttpd:chunked_response_buffer_size(),
VAcc = #vacc{db=Db, req=Req, threshold=Max},
-{ok, Resp} = fabric:all_docs(Db, Options, fun couch_mrview_http:view_cb/2,
VAcc, Args3),
+{ok, Resp} = fabric:all_docs(Db, Options, fun couch_mrview_http:view_cb/2,
VAcc, Args5),
{ok, Resp#vacc.resp}.
db_doc_req(#httpd{method='DELETE'}=Req, Db, DocId) ->
diff --git a/src/couch_mrview/src/couch_mrview.erl
b/src/couch_mrview/src/couch_mrview.erl
index 09945f5..f5963e7 100644
--- a/src/couch_mrview/src/couch_mrview.erl
+++ b/src/couch_mrview/src/couch_mrview.erl
@@ -228,12 +228,13 @@ query_all_docs(Db, Args0, Callback, Acc) ->
couch_index_util:hexsig(couch_hash:md5_hash(term_to_binary(Info)))
end),
Args1 = Args0#mrargs{view_type=map},
-Args2 = couch_mrview_util:validate_args(Args1),
-{ok, Acc1} = case Args2#mrargs.preflight_fun of
+Args2 = couch_mrview_util:set_extra(Args1, style, all_docs),
+Args3 = couch_mrview_util:validate_args(Args2),
+{ok, Acc1} = case Args3#mrargs.preflight_fun of
PFFun when is_function(PFFun, 2) -> PFFun(Sig, Acc);
_ -> {ok, Acc}
end,
-all_docs_fold(Db, Args2, Callback, Acc1).
+all_docs_fold(Db, Args3, Callback, Acc1).
query_view(Db, DDoc, VName) ->
diff --git a/src/couch_mrview/src/couch_mrview_updater.erl
b/src/couch_mrview/src/couch_mrview_updater.erl
index 2b69eee..0c1a17c 100644
--- a/src/couch_mrview/src/couch_mrview_updater.erl
+++ b/src/couch_mrview/src/couch_mrview_updater.erl
@@ -315,7 +315,9 @@ write_kvs(State, UpdateSeq, ViewKVs, DocIdKeys, Seqs, Log0)
->
design_opts=DesignOpts
} = State,
-Partitioned = couch_util:get_value(<<"partitioned">>, DesignOpts, false),
+DbPartitioned = mem3:is_partitioned(State#mrst.db_name),
+Partitioned = couch_util:get_value(<<"partitioned">>, DesignOpts,
DbPartitioned),
+
Revs = dict:from_list(dict:fetch_keys(Log0)),
Log = dict:fold(fun({Id, _Rev}, DIKeys, Acc) ->
diff --git a/src/couch_mrview/src/couch_mrview_util.erl
b/src/couch_mrview/src/couch_mrview_util.erl
index 574aac7..02e695d 100644
--- a/src/couch_mrview/src/couch_mrview_util.erl
+++ b/src/couch_mrview/src/couch_mrview_util.erl
@@ -39,8 +39,11 @@
-define(GET_VIEW_RETRY_COUNT, 1).
-define(GET_VIEW_RETRY_DELAY, 50).
-define(LOWEST_KEY, null).
--define(HIGHEST_KEY, {[{<<239, 191, 176>>, null}]}). % is {"\ufff0": null}
-
+-define(HIGHEST_KEY, {<<255, 255, 255, 255>>}).
+-define(PARTITION_START(P), <>).
+-define(PARTITION_END(P), <>).
+-define(LOWEST(A, B), (if A < B -> A; true -> B end)).
+-define(HIGHEST(A, B), (if A > B -> A; true -> B end)).
-include_lib("couch/include/couch_db.hrl").
-include_lib("couch_mrview/include/couch_mrview.hrl").
@@ -583,20 +586,38 @@ validate_args(Args) ->
_ -> mrverror(<<"Invalid value for `sorted`.">>)
end,
-case {get_extra(Args, partitioned, false), get_extra(Args, partition)} of
-{true, undefined} ->
+Style = get_extra(Args, style, normal),
+Partitioned = get_extra(Args, partitioned, false),
+Partition = get_extra(Args, partition),
[couchdb] 05/05: Add test for all_docs partition optimisation
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch user-partitioned-dbs-6
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 298fa182dcc97a71daf6cad7d47ced6604f99dc5
Author: Garren Smith
AuthorDate: Tue Aug 14 15:43:01 2018 +0200
Add test for all_docs partition optimisation
Adds tests to validate the all_docs optimisations works for partitions
---
src/fabric/src/fabric_view_all_docs.erl | 86 +
1 file changed, 86 insertions(+)
diff --git a/src/fabric/src/fabric_view_all_docs.erl
b/src/fabric/src/fabric_view_all_docs.erl
index 83c3790..b12bcde 100644
--- a/src/fabric/src/fabric_view_all_docs.erl
+++ b/src/fabric/src/fabric_view_all_docs.erl
@@ -316,3 +316,89 @@ cancel_read_pids(Pids) ->
{empty, _} ->
ok
end.
+
+-ifdef(TEST).
+
+-include_lib("eunit/include/eunit.hrl").
+
+shards_for_partition_gets_partitioned_shards_test() ->
+DbName = <<"db">>,
+Args = #mrargs{
+start_key = <<"pk:id">>,
+end_key = <<"pk:idZ">>,
+extra = [{partitioned, true}]
+},
+meck:expect(mem3, shards, fun(<<"db">>, <<"pk:foo">>) -> [] end),
+shards(DbName, Args),
+meck:validate(mem3),
+meck:unload(mem3).
+
+shards_for_no_partition_gets_all_shards_test() ->
+DbName = <<"db">>,
+Args = #mrargs{
+start_key = <<"pk:id">>,
+end_key = <<"pk:idZ">>,
+extra = [{partitioned, false}]
+},
+meck:expect(mem3, shards, fun(<<"db">>) -> [] end),
+shards(DbName, Args),
+meck:validate(mem3),
+meck:unload(mem3).
+
+shards_for_different_partitions_gets_all_shards_test() ->
+DbName = <<"db">>,
+Args = #mrargs{
+start_key = <<"pk1:id">>,
+end_key = <<"pk2:idZ">>,
+extra = [{partitioned, true}]
+},
+meck:expect(mem3, shards, fun(<<"db">>) -> [] end),
+shards(DbName, Args),
+meck:validate(mem3),
+meck:unload(mem3).
+
+shards_for_no_startkey_all_shards_test() ->
+DbName = <<"db">>,
+Args = #mrargs{
+end_key = <<"pk:idZ">>,
+extra = [{partitioned, true}]
+},
+meck:expect(mem3, shards, fun(<<"db">>) -> [] end),
+shards(DbName, Args),
+meck:validate(mem3),
+meck:unload(mem3).
+
+shards_for_no_endkey_all_shards_test() ->
+DbName = <<"db">>,
+Args = #mrargs{
+start_key = <<"pk:idZ">>,
+extra = [{partitioned, true}]
+},
+meck:expect(mem3, shards, fun(<<"db">>) -> [] end),
+shards(DbName, Args),
+meck:validate(mem3),
+meck:unload(mem3).
+
+shards_for_no_keys_all_shards_test() ->
+DbName = <<"db">>,
+Args = #mrargs{
+extra = [{partitioned, true}]
+},
+meck:expect(mem3, shards, fun(<<"db">>) -> [] end),
+shards(DbName, Args),
+meck:validate(mem3),
+meck:unload(mem3).
+
+shards_for_non_binary_keys_all_shards_test() ->
+DbName = <<"db">>,
+Args = #mrargs{
+start_key = null,
+end_key = null,
+extra = [{partitioned, true}]
+},
+meck:expect(mem3, shards, fun(<<"db">>) -> [] end),
+shards(DbName, Args),
+meck:validate(mem3),
+meck:unload(mem3).
+
+-endif.
