This is an automated email from the ASF dual-hosted git repository. davisp pushed a commit to branch prototype/views in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 81521e1e621dd5ca17a27d9c7d3893e9eb4224e7 Author: Paul J. Davis <[email protected]> AuthorDate: Tue Jul 23 15:12:31 2019 -0500 Add total row count support --- src/couch_views/include/couch_views.hrl | 16 ++++---- src/couch_views/src/couch_views_fdb.erl | 50 ++++++++++++++++++++--- src/couch_views/src/couch_views_reader.erl | 9 ++-- src/couch_views/test/couch_views_indexer_test.erl | 4 +- 4 files changed, 59 insertions(+), 20 deletions(-) diff --git a/src/couch_views/include/couch_views.hrl b/src/couch_views/include/couch_views.hrl index 4fcc57e..525f62f 100644 --- a/src/couch_views/include/couch_views.hrl +++ b/src/couch_views/include/couch_views.hrl @@ -11,15 +11,15 @@ % the License. % indexing --define(VIEW_UPDATE_SEQ, 1). --define(VIEW_ID_RANGE, 2). --define(VIEW_MAP_RANGE, 3). --define(VIEW_BUILDS, 4). --define(VIEW_STATUS, 5). --define(VIEW_WATCH, 6). +-define(VIEW_UPDATE_SEQ, 0). +-define(VIEW_ID_INFO, 1). +-define(VIEW_ID_RANGE, 2). +-define(VIEW_MAP_RANGE, 3). --define(VIEW_ROW_KEY, 0). --define(VIEW_ROW_VALUE, 1). +-define(VIEW_ROW_COUNT, 0). + +-define(VIEW_ROW_KEY, 0). +-define(VIEW_ROW_VALUE, 1). % jobs api -define(INDEX_JOB_TYPE, <<"views">>). diff --git a/src/couch_views/src/couch_views_fdb.erl b/src/couch_views/src/couch_views_fdb.erl index 09a9802..a0e4bd1 100644 --- a/src/couch_views/src/couch_views_fdb.erl +++ b/src/couch_views/src/couch_views_fdb.erl @@ -16,6 +16,8 @@ get_update_seq/2, set_update_seq/3, + get_row_count/3, + fold_map_idx/6, write_doc/4 @@ -59,6 +61,18 @@ set_update_seq(TxDb, Sig, Seq) -> ok = erlfdb:set(Tx, seq_key(DbPrefix, Sig), Seq). +get_row_count(TxDb, #mrst{sig = Sig}, ViewId) -> + #{ + tx := Tx, + db_prefix := DbPrefix + } = TxDb, + + case erlfdb:wait(erlfdb:get(Tx, row_count_key(DbPrefix, Sig, ViewId))) of + not_found -> 0; % Can this happen? + CountBin -> ?bin2uint(CountBin) + end. + + fold_map_idx(TxDb, Sig, ViewId, Options, Callback, Acc0) -> #{ db_prefix := DbPrefix @@ -107,8 +121,9 @@ write_doc(TxDb, Sig, _ViewIds, #{deleted := true} = Doc) -> ExistingViewKeys = get_view_keys(TxDb, Sig, DocId), clear_id_idx(TxDb, Sig, DocId), - lists:foreach(fun({ViewId, ViewKeys}) -> - clear_map_idx(TxDb, Sig, ViewId, DocId, ViewKeys) + lists:foreach(fun({ViewId, TotalKeys, UniqueKeys}) -> + clear_map_idx(TxDb, Sig, ViewId, DocId, UniqueKeys), + update_row_count(TxDb, Sig, ViewId, -TotalKeys) end, ExistingViewKeys); write_doc(TxDb, Sig, ViewIds, Doc) -> @@ -122,8 +137,17 @@ write_doc(TxDb, Sig, ViewIds, Doc) -> clear_id_idx(TxDb, Sig, DocId), lists:foreach(fun({ViewId, NewRows}) -> - ExistingKeys = fabric2_util:get_value(ViewId, ExistingViewKeys, []), update_id_idx(TxDb, Sig, ViewId, DocId, NewRows), + + ExistingKeys = case lists:keyfind(ViewId, 1, ExistingViewKeys) of + {ViewId, TotalRows, EKeys} -> + Change = length(NewRows) - TotalRows, + update_row_count(TxDb, Sig, ViewId, Change), + EKeys; + false -> + update_row_count(TxDb, Sig, ViewId, length(NewRows)), + [] + end, update_map_idx(TxDb, Sig, ViewId, DocId, ExistingKeys, NewRows) end, lists:zip(ViewIds, Results)). @@ -251,7 +275,7 @@ update_id_idx(TxDb, Sig, ViewId, DocId, NewRows) -> couch_log:error("Updating ID index: ~p ~p ~p ~p", [ViewId, DocId, NewRows, Unique]), Key = id_idx_key(DbPrefix, Sig, DocId, ViewId), - Val = couch_views_encoding:encode(Unique), + Val = couch_views_encoding:encode([length(NewRows), Unique]), ok = erlfdb:set(Tx, Key, Val). @@ -289,16 +313,30 @@ get_view_keys(TxDb, Sig, DocId) -> lists:map(fun({K, V}) -> {?DB_VIEWS, Sig, ?VIEW_ID_RANGE, DocId, ViewId} = erlfdb_tuple:unpack(K, DbPrefix), - ViewKeys = couch_views_encoding:decode(V), - {ViewId, ViewKeys} + [TotalKeys, UniqueKeys] = couch_views_encoding:decode(V), + {ViewId, TotalKeys, UniqueKeys} end, erlfdb:get_range(Tx, Start, End, [])). +update_row_count(TxDb, Sig, ViewId, Increment) -> + #{ + tx := Tx, + db_prefix := DbPrefix + } = TxDb, + Key = row_count_key(DbPrefix, Sig, ViewId), + erlfdb:add(Tx, Key, Increment). + + seq_key(DbPrefix, Sig) -> Key = {?DB_VIEWS, Sig, ?VIEW_UPDATE_SEQ}, erlfdb_tuple:pack(Key, DbPrefix). +row_count_key(DbPrefix, Sig, ViewId) -> + Key = {?DB_VIEWS, Sig, ?VIEW_ID_INFO, ViewId, ?VIEW_ROW_COUNT}, + erlfdb_tuple:pack(Key, DbPrefix). + + id_idx_key(DbPrefix, Sig, DocId, ViewId) -> Key = {?DB_VIEWS, Sig, ?VIEW_ID_RANGE, DocId, ViewId}, erlfdb_tuple:pack(Key, DbPrefix). diff --git a/src/couch_views/src/couch_views_reader.erl b/src/couch_views/src/couch_views_reader.erl index ce5097ba..8bbe1fc 100644 --- a/src/couch_views/src/couch_views_reader.erl +++ b/src/couch_views/src/couch_views_reader.erl @@ -34,11 +34,12 @@ read(Db, Mrst, ViewName, UserCallback, UserAcc0, Args) -> Fun = fun handle_row/4, try - % Need to add total_rows support - Meta = {meta, [{total_rows, null}, {offset, null}]}, - UserAcc1 = maybe_stop(UserCallback(Meta, UserAcc0)), - fabric2_fdb:transactional(Db, fun(TxDb) -> + TotalRows = couch_views_fdb:get_row_count(TxDb, Mrst, ViewId), + + Meta = {meta, [{total, TotalRows}, {offset, null}]}, + UserAcc1 = maybe_stop(UserCallback(Meta, UserAcc0)), + Acc0 = #{ db => TxDb, skip => Args#mrargs.skip, diff --git a/src/couch_views/test/couch_views_indexer_test.erl b/src/couch_views/test/couch_views_indexer_test.erl index e6dfdc4..fa0d99e 100644 --- a/src/couch_views/test/couch_views_indexer_test.erl +++ b/src/couch_views/test/couch_views_indexer_test.erl @@ -157,7 +157,7 @@ updated_docs_are_reindexed(Db) -> DbName = fabric2_db:name(Db), {ok, Mrst} = couch_views_util:ddoc_to_mrst(DbName, DDoc), Sig = Mrst#mrst.sig, - Expect = [{0, [1]}, {1, []}], + Expect = [{0, 1, [1]}, {1, 0, []}], fabric2_fdb:transactional(Db, fun(TxDb) -> ?assertEqual( Expect, @@ -214,7 +214,7 @@ updated_docs_without_changes_are_reindexed(Db) -> DbName = fabric2_db:name(Db), {ok, Mrst} = couch_views_util:ddoc_to_mrst(DbName, DDoc), Sig = Mrst#mrst.sig, - Expect = [{0, [0]}, {1, []}], + Expect = [{0, 1, [0]}, {1, 0, []}], fabric2_fdb:transactional(Db, fun(TxDb) -> ?assertEqual( Expect,
