[GitHub] [couchdb] nickva commented on a change in pull request #2540: Use multi-transactional iterators
nickva commented on a change in pull request #2540: Use multi-transactional
iterators
URL: https://github.com/apache/couchdb/pull/2540#discussion_r379628925
##
File path: src/fabric/test/fabric2_changes_fold_tests.erl
##
@@ -21,28 +21,56 @@
-define(DOC_COUNT, 25).
+-define(PDICT_ERROR_IN_FOLD_RANGE, '$fabric2_error_in_fold_range').
+-define(PDICT_ERROR_IN_USER_FUN, '$fabric2_error_throw_in_user_fun').
+
changes_fold_test_() ->
{
"Test changes fold operations",
{
setup,
-fun setup/0,
-fun cleanup/1,
-with([
-?TDEF(fold_changes_basic),
-?TDEF(fold_changes_since_now),
-?TDEF(fold_changes_since_seq),
-?TDEF(fold_changes_basic_rev),
-?TDEF(fold_changes_since_now_rev),
-?TDEF(fold_changes_since_seq_rev)
-])
+fun setup_all/0,
+fun teardown_all/1,
+{
+foreach,
+fun setup/0,
+fun cleanup/1,
+[
+?TDEF_FE(fold_changes_basic),
+?TDEF_FE(fold_changes_since_now),
+?TDEF_FE(fold_changes_since_seq),
+?TDEF_FE(fold_changes_basic_rev),
+?TDEF_FE(fold_changes_since_now_rev),
+?TDEF_FE(fold_changes_since_seq_rev),
+?TDEF_FE(fold_changes_basic_tx_too_long),
+?TDEF_FE(fold_changes_reverse_tx_too_long),
+?TDEF_FE(fold_changes_tx_too_long_with_single_row_emits),
+?TDEF_FE(fold_changes_since_seq_tx_too_long),
+?TDEF_FE(fold_changes_not_progressing)
+]
+}
}
}.
-setup() ->
+setup_all() ->
Ctx = test_util:start_couch([fabric]),
+meck:new(erlfdb, [passthrough]),
+Ctx.
+
+
+teardown_all(Ctx) ->
+meck:unload(),
+test_util:stop_couch(Ctx).
+
+
+setup() ->
+meck:reset([erlfdb]),
Review comment:
It just rests call counts. I never ended up using call counts. I'll remove
the line
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] [couchdb] nickva commented on a change in pull request #2540: Use multi-transactional iterators
nickva commented on a change in pull request #2540: Use multi-transactional
iterators
URL: https://github.com/apache/couchdb/pull/2540#discussion_r377920781
##
File path: src/fabric/src/fabric2_fdb.erl
##
@@ -1361,21 +1426,66 @@ get_fold_opts(RangePrefix, Options) ->
B when is_boolean(B) -> [{snapshot, B}]
end,
-OutOpts = [{reverse, Reverse}]
-++ Limit
+BaseOpts = [{reverse, Reverse}]
++ TargetBytes
++ StreamingMode
++ Snapshot,
-{StartKey3, EndKey3, Skip, OutOpts}.
+BaseAcc#fold_acc{
+start_key = StartKey3,
+end_key = EndKey3,
+skip = Skip,
+limit = Limit,
+base_opts = BaseOpts,
+user_fun = UserCallback,
+user_acc = UserAcc
+}.
-fold_range_cb(KV, {skip, 0, Callback, Acc}) ->
-NewAcc = Callback(KV, Acc),
-{skip, 0, Callback, NewAcc};
+fold_range_cb({K, V}, #fold_acc{} = Acc) ->
+#fold_acc{
+skip = Skip,
+limit = Limit,
+user_fun = UserFun,
+user_acc = UserAcc,
+base_opts = Opts
+} = Acc,
+Acc1 = case Skip =:= 0 of
+true ->
+UserAcc1 = UserFun({K, V}, UserAcc),
+Acc#fold_acc{limit = Limit - 1, user_acc = UserAcc1};
+false ->
+Acc#fold_acc{skip = Skip - 1, limit = Limit - 1}
+end,
+Acc2 = case fabric2_util:get_value(reverse, Opts, false) of
+true -> Acc1#fold_acc{end_key = K};
+false ->Acc1#fold_acc{start_key = K}
+end,
+put(?PDICT_FOLD_ACC_STATE, Acc2),
+Acc2.
-fold_range_cb(_KV, {skip, N, Callback, Acc}) when is_integer(N), N > 0 ->
-{skip, N - 1, Callback, Acc}.
+
+restart_fold(Tx, #fold_acc{} = Acc) ->
+erase(?PDICT_CHECKED_MD_IS_CURRENT),
+ok = erlfdb:reset(Tx),
+case get(?PDICT_FOLD_ACC_STATE) of
+#fold_acc{db = Db0} = Acc1 ->
+erase(?PDICT_FOLD_ACC_STATE),
+Db = check_db_instance(Db0),
+Acc1#fold_acc{db = Db};
+undefined ->
+#fold_acc{
+db = Db,
+start_key = Start,
+end_key = End,
+limit = Limit,
+user_fun = UserFun
+} = Acc,
+Db = check_db_instance(Acc#fold_acc.db),
+Args = [Db, Start, End, Limit, UserFun],
+couch_log:error("fold_range not progressing ~p ~p:~p ~p ~p", Args),
Review comment:
Good idea. I was thinking we'd set PDICT_FOLD_ACC_STATE to `initialized` in
the first call to fold_range then detect in the restart_fold
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] [couchdb] nickva commented on a change in pull request #2540: Use multi-transactional iterators
nickva commented on a change in pull request #2540: Use multi-transactional iterators URL: https://github.com/apache/couchdb/pull/2540#discussion_r377881851 ## File path: src/fabric/src/fabric2_db.erl ## @@ -1001,6 +1002,11 @@ maybe_set_user_ctx(Db, Options) -> end. +set_tx_options(Db, Options) -> Review comment: It was mostly for tests to avoid setting up mocks. But good call on not mixing it together. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] [couchdb] nickva commented on a change in pull request #2540: Use multi-transactional iterators
nickva commented on a change in pull request #2540: Use multi-transactional
iterators
URL: https://github.com/apache/couchdb/pull/2540#discussion_r377880280
##
File path: src/fabric/src/fabric2_fdb.erl
##
@@ -815,25 +846,52 @@ get_last_change(#{} = Db) ->
end.
-fold_range(#{} = Db, RangePrefix, Callback, Acc, Options) ->
+fold_range(#{} = Db, RangePrefix, UserFun, UserAcc, Options) ->
#{
tx := Tx
-} = ensure_current(Db),
-fold_range({tx, Tx}, RangePrefix, Callback, Acc, Options);
-
-fold_range({tx, Tx}, RangePrefix, UserCallback, UserAcc, Options) ->
-case fabric2_util:get_value(limit, Options) of
-0 ->
-% FoundationDB treats a limit of 0 as unlimited
-% so we have to guard for that here.
-UserAcc;
-_ ->
-{Start, End, Skip, FoldOpts} = get_fold_opts(RangePrefix, Options),
-Callback = fun fold_range_cb/2,
-Acc = {skip, Skip, UserCallback, UserAcc},
-{skip, _, UserCallback, OutAcc} =
-erlfdb:fold_range(Tx, Start, End, Callback, Acc, FoldOpts),
-OutAcc
+} = CurrDb = ensure_current(Db),
+% FoundationDB treats a limit 0 of as unlimited so we guard against it
+case fabric2_util:get_value(limit, Options) of 0 -> UserAcc; _ ->
+FAcc = get_fold_acc(CurrDb, RangePrefix, UserFun, UserAcc, Options),
+try
+fold_range(Tx, FAcc)
+after
+erase(?PDICT_FOLD_ACC_STATE)
+end
+end;
+
+fold_range({tx, Tx}, RangePrefix, UserFun, UserAcc, Options) ->
Review comment:
Agree
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] [couchdb] nickva commented on a change in pull request #2540: Use multi-transactional iterators
nickva commented on a change in pull request #2540: Use multi-transactional iterators URL: https://github.com/apache/couchdb/pull/2540#discussion_r377880172 ## File path: src/fabric/include/fabric2.hrl ## @@ -57,7 +57,10 @@ -define(PDICT_TX_ID_KEY, '$fabric_tx_id'). -define(PDICT_TX_RES_KEY, '$fabric_tx_result'). -define(PDICT_ON_COMMIT_FUN, '$fabric_on_commit_fun'). +-define(PDICT_FOLD_ACC_STATE, '$fabric_fold_acc_state'). + -define(COMMIT_UNKNOWN_RESULT, 1021). +-define(TRANSACTION_TOO_OLD, 1007). Review comment: Good idea This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
