This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
The following commit(s) were added to refs/heads/main by this push:
new 833e4c35d Handle missing VDU better in In QuickJS scanner plugin
833e4c35d is described below
commit 833e4c35d09ae06cc8240e3236fd77a9426042c3
Author: Nick Vatamaniuc <[email protected]>
AuthorDate: Thu May 16 20:08:57 2024 -0400
Handle missing VDU better in In QuickJS scanner plugin
Previously we tried to run the VDU query prompt even if there was not VDU.
That
would have yielded a match anyway as both result in an error, but it added
noise to the logs.
Handle the missing VDU like we handle the missing filters in
filter_doc_validate.
---
.../src/couch_quickjs_scanner_plugin.erl | 5 +-
.../test/couch_quickjs_scanner_plugin_tests.erl | 298 +++++++++++++++++----
2 files changed, 243 insertions(+), 60 deletions(-)
diff --git a/src/couch_quickjs/src/couch_quickjs_scanner_plugin.erl
b/src/couch_quickjs/src/couch_quickjs_scanner_plugin.erl
index a30b7b687..c68313d43 100644
--- a/src/couch_quickjs/src/couch_quickjs_scanner_plugin.erl
+++ b/src/couch_quickjs/src/couch_quickjs_scanner_plugin.erl
@@ -100,7 +100,7 @@ ddoc(#st{sid = SId} = St, DbName, #doc{id = DDocId} = DDoc)
->
?INFO(#{sid => SId, db => DbName, ddoc => DDocId}),
#st{ddoc_cnt = Cnt} = St,
St1 = St#st{ddoc_cnt = Cnt + 1},
- #doc{body = {Props = [_ | _]}} = DDoc,
+ #doc{body = {Props}} = DDoc,
case couch_util:get_value(<<"language">>, Props, <<"javascript">>) of
<<"javascript">> -> {ok, process_ddoc(St1, DbName, DDoc)};
_ -> {ok, St1}
@@ -401,6 +401,9 @@ vdu_load(#st{qjs_proc = Qjs, sm_proc = Sm}, VDU) ->
false -> throw({validate, {vdu, QjsRes, SmRes}})
end.
+vdu_doc_validate(#st{}, _DDocId, undefined, _Doc) ->
+ % No VDU
+ ok;
vdu_doc_validate(#st{} = St, DDocId, _VDU, Doc) ->
#st{qjs_proc = Qjs, sm_proc = Sm} = St,
QjsRes = vdu_doc(Qjs, DDocId, Doc),
diff --git a/src/couch_quickjs/test/couch_quickjs_scanner_plugin_tests.erl
b/src/couch_quickjs/test/couch_quickjs_scanner_plugin_tests.erl
index 8118bf614..de6bfe806 100644
--- a/src/couch_quickjs/test/couch_quickjs_scanner_plugin_tests.erl
+++ b/src/couch_quickjs/test/couch_quickjs_scanner_plugin_tests.erl
@@ -21,7 +21,12 @@ couch_quickjs_scanner_plugin_test_() ->
fun setup/0,
fun teardown/1,
[
- ?TDEF_FE(t_basic, 10)
+ ?TDEF_FE(t_vdu_filter_map, 10),
+ ?TDEF_FE(t_filter_map, 10),
+ ?TDEF_FE(t_map_only, 10),
+ ?TDEF_FE(t_vdu_only, 10),
+ ?TDEF_FE(t_filter_only, 10),
+ ?TDEF_FE(t_empty_ddoc, 10)
]
}.
@@ -47,63 +52,6 @@ setup() ->
ok = add_doc(DbName, ?DOC3, #{a => z}),
ok = add_doc(DbName, ?DOC4, #{a => w}),
ok = add_doc(DbName, ?DOC5, #{a => u}),
- ok = add_doc(DbName, ?DDOC1, #{
- views => #{
- v => #{
- map => <<
- "function(doc) {\n"
- " if(doc.a == 'x') {\n"
- " r = doc.a.search(/(x+)/); emit(r, RegExp.$1)\n"
- " } else {\n"
- " emit(doc.a, doc.a);\n"
- " }\n"
- "}"
- >>,
- reduce => <<
- "function(ks, vs, rereduce) {\n"
- " v0 = vs[0];\n"
- " if (!rereduce) {\n"
- " k0 = ks[0];\n"
- " if (k0 == 'y') {\n"
- " k0.search(/(y+)/);\n"
- " return RegExp.$1;\n"
- " };\n"
- " return v0;\n"
- " } else {\n"
- " if (v0 == 'u') {\n"
- " v0.search(/(u+)/);\n"
- " return RegExp.$1;\n"
- " };\n"
- " return v0;\n"
- " }\n"
- "}"
- >>
- }
- },
- filters => #{
- f => <<
- "function(doc, req) {\n"
- " if(doc.a == 'z') {\n"
- " doc.a.search(/(z+)/);\n"
- " if(RegExp.$1 == 'z') {return true} else {return false};\n"
- " } else {\n"
- " return true;\n"
- " }\n"
- "}"
- >>
- },
- validate_doc_update => <<
- "function(newdoc, olddoc, userctx, sec){\n"
- " if(newdoc.a == 'w') {\n"
- " newdoc.a.search(/(w+)/);\n"
- " if(RegExp.$1 == 'w') {return true} else
{throw('forbidden')}\n"
- " } else {\n"
- " return true\n"
- " }\n"
- "}"
- >>
- }),
- couch_scanner:reset_checkpoints(),
config:set(atom_to_list(?PLUGIN), "max_batch_items", "1", false),
{Ctx, DbName}.
@@ -117,7 +65,8 @@ teardown({Ctx, DbName}) ->
test_util:stop_couch(Ctx),
meck:unload().
-t_basic({_, DbName}) ->
+t_vdu_filter_map({_, DbName}) ->
+ ok = add_doc(DbName, ?DDOC1, ddoc_vdu(ddoc_filter(ddoc_view(#{})))),
meck:reset(couch_scanner_server),
meck:reset(?PLUGIN),
config:set("couch_scanner_plugins", atom_to_list(?PLUGIN), "true", false),
@@ -140,12 +89,177 @@ t_basic({_, DbName}) ->
?assert(num_calls(doc, 3) >= 5),
DbClosingCount = num_calls(db_closing, 2),
?assertEqual(DbOpenedCount, DbClosingCount),
+ ?assertEqual(0, couch_stats:sample([couchdb, query_server,
process_error_exits])),
+ ?assertEqual(0, couch_stats:sample([couchdb, query_server,
process_errors])),
+ ?assertEqual(0, couch_stats:sample([couchdb, query_server,
process_exits])),
% start, complete and each of the 5 docs should fail = 7 total
?assertEqual(7, log_calls(warning));
false ->
ok
end.
+t_filter_map({_, DbName}) ->
+ ok = add_doc(DbName, ?DDOC1, ddoc_filter(ddoc_view(#{}))),
+ meck:reset(couch_scanner_server),
+ meck:reset(?PLUGIN),
+ config:set("couch_scanner_plugins", atom_to_list(?PLUGIN), "true", false),
+ wait_exit(10000),
+ ?assertEqual(1, num_calls(start, 2)),
+ case couch_server:with_spidermonkey() of
+ true ->
+ ?assertEqual(1, num_calls(complete, 1)),
+ ?assertEqual(2, num_calls(checkpoint, 1)),
+ ?assertEqual(1, num_calls(db, ['_', DbName])),
+ ?assertEqual(1, num_calls(ddoc, ['_', DbName, '_'])),
+ ?assert(num_calls(shards, 2) >= 1),
+ DbOpenedCount = num_calls(db_opened, 2),
+ ?assert(DbOpenedCount >= 2),
+ ?assertEqual(1, num_calls(doc_id, ['_', ?DOC1, '_'])),
+ ?assertEqual(1, num_calls(doc_id, ['_', ?DOC2, '_'])),
+ ?assertEqual(1, num_calls(doc_id, ['_', ?DOC3, '_'])),
+ ?assertEqual(1, num_calls(doc_id, ['_', ?DOC4, '_'])),
+ ?assertEqual(1, num_calls(doc_id, ['_', ?DOC5, '_'])),
+ ?assert(num_calls(doc, 3) >= 5),
+ DbClosingCount = num_calls(db_closing, 2),
+ ?assertEqual(DbOpenedCount, DbClosingCount),
+ ?assertEqual(0, couch_stats:sample([couchdb, query_server,
process_error_exits])),
+ ?assertEqual(0, couch_stats:sample([couchdb, query_server,
process_errors])),
+ ?assertEqual(0, couch_stats:sample([couchdb, query_server,
process_exits])),
+ % start, complete and each of the 4 docs should fail = 6 total
+ ?assertEqual(6, log_calls(warning));
+ false ->
+ ok
+ end.
+
+t_map_only({_, DbName}) ->
+ ok = add_doc(DbName, ?DDOC1, ddoc_view(#{})),
+ meck:reset(couch_scanner_server),
+ meck:reset(?PLUGIN),
+ config:set("couch_scanner_plugins", atom_to_list(?PLUGIN), "true", false),
+ wait_exit(10000),
+ ?assertEqual(1, num_calls(start, 2)),
+ case couch_server:with_spidermonkey() of
+ true ->
+ ?assertEqual(1, num_calls(complete, 1)),
+ ?assertEqual(2, num_calls(checkpoint, 1)),
+ ?assertEqual(1, num_calls(db, ['_', DbName])),
+ ?assertEqual(1, num_calls(ddoc, ['_', DbName, '_'])),
+ ?assert(num_calls(shards, 2) >= 1),
+ DbOpenedCount = num_calls(db_opened, 2),
+ ?assert(DbOpenedCount >= 2),
+ ?assertEqual(1, num_calls(doc_id, ['_', ?DOC1, '_'])),
+ ?assertEqual(1, num_calls(doc_id, ['_', ?DOC2, '_'])),
+ ?assertEqual(1, num_calls(doc_id, ['_', ?DOC3, '_'])),
+ ?assertEqual(1, num_calls(doc_id, ['_', ?DOC4, '_'])),
+ ?assertEqual(1, num_calls(doc_id, ['_', ?DOC5, '_'])),
+ ?assert(num_calls(doc, 3) >= 5),
+ DbClosingCount = num_calls(db_closing, 2),
+ ?assertEqual(DbOpenedCount, DbClosingCount),
+ ?assertEqual(0, couch_stats:sample([couchdb, query_server,
process_error_exits])),
+ ?assertEqual(0, couch_stats:sample([couchdb, query_server,
process_errors])),
+ ?assertEqual(0, couch_stats:sample([couchdb, query_server,
process_exits])),
+ % start, complete and each of the 3 docs should fail = 5 total
+ ?assertEqual(5, log_calls(warning));
+ false ->
+ ok
+ end.
+
+t_vdu_only({_, DbName}) ->
+ ok = add_doc(DbName, ?DDOC1, ddoc_vdu(#{})),
+ meck:reset(couch_scanner_server),
+ meck:reset(?PLUGIN),
+ config:set("couch_scanner_plugins", atom_to_list(?PLUGIN), "true", false),
+ wait_exit(10000),
+ ?assertEqual(1, num_calls(start, 2)),
+ case couch_server:with_spidermonkey() of
+ true ->
+ ?assertEqual(1, num_calls(complete, 1)),
+ ?assertEqual(2, num_calls(checkpoint, 1)),
+ ?assertEqual(1, num_calls(db, ['_', DbName])),
+ ?assertEqual(1, num_calls(ddoc, ['_', DbName, '_'])),
+ ?assert(num_calls(shards, 2) >= 1),
+ DbOpenedCount = num_calls(db_opened, 2),
+ ?assert(DbOpenedCount >= 2),
+ ?assertEqual(1, num_calls(doc_id, ['_', ?DOC1, '_'])),
+ ?assertEqual(1, num_calls(doc_id, ['_', ?DOC2, '_'])),
+ ?assertEqual(1, num_calls(doc_id, ['_', ?DOC3, '_'])),
+ ?assertEqual(1, num_calls(doc_id, ['_', ?DOC4, '_'])),
+ ?assertEqual(1, num_calls(doc_id, ['_', ?DOC5, '_'])),
+ ?assert(num_calls(doc, 3) >= 5),
+ DbClosingCount = num_calls(db_closing, 2),
+ ?assertEqual(DbOpenedCount, DbClosingCount),
+ ?assertEqual(0, couch_stats:sample([couchdb, query_server,
process_error_exits])),
+ ?assertEqual(0, couch_stats:sample([couchdb, query_server,
process_errors])),
+ ?assertEqual(0, couch_stats:sample([couchdb, query_server,
process_exits])),
+ % start, complete and 1 vdu only = 3 total
+ ?assertEqual(3, log_calls(warning));
+ false ->
+ ok
+ end.
+
+t_filter_only({_, DbName}) ->
+ ok = add_doc(DbName, ?DDOC1, ddoc_filter(#{})),
+ meck:reset(couch_scanner_server),
+ meck:reset(?PLUGIN),
+ config:set("couch_scanner_plugins", atom_to_list(?PLUGIN), "true", false),
+ wait_exit(10000),
+ ?assertEqual(1, num_calls(start, 2)),
+ case couch_server:with_spidermonkey() of
+ true ->
+ ?assertEqual(1, num_calls(complete, 1)),
+ ?assertEqual(2, num_calls(checkpoint, 1)),
+ ?assertEqual(1, num_calls(db, ['_', DbName])),
+ ?assertEqual(1, num_calls(ddoc, ['_', DbName, '_'])),
+ ?assert(num_calls(shards, 2) >= 1),
+ DbOpenedCount = num_calls(db_opened, 2),
+ ?assert(DbOpenedCount >= 2),
+ ?assertEqual(1, num_calls(doc_id, ['_', ?DOC1, '_'])),
+ ?assertEqual(1, num_calls(doc_id, ['_', ?DOC2, '_'])),
+ ?assertEqual(1, num_calls(doc_id, ['_', ?DOC3, '_'])),
+ ?assertEqual(1, num_calls(doc_id, ['_', ?DOC4, '_'])),
+ ?assertEqual(1, num_calls(doc_id, ['_', ?DOC5, '_'])),
+ ?assert(num_calls(doc, 3) >= 5),
+ DbClosingCount = num_calls(db_closing, 2),
+ ?assertEqual(DbOpenedCount, DbClosingCount),
+ ?assertEqual(0, couch_stats:sample([couchdb, query_server,
process_error_exits])),
+ ?assertEqual(0, couch_stats:sample([couchdb, query_server,
process_errors])),
+ ?assertEqual(0, couch_stats:sample([couchdb, query_server,
process_exits])),
+ % start, complete and one filter only = 3 total
+ ?assertEqual(3, log_calls(warning));
+ false ->
+ ok
+ end.
+
+t_empty_ddoc({_, DbName}) ->
+ ok = add_doc(DbName, ?DDOC1, #{}),
+ meck:reset(couch_scanner_server),
+ meck:reset(?PLUGIN),
+ config:set("couch_scanner_plugins", atom_to_list(?PLUGIN), "true", false),
+ wait_exit(10000),
+ ?assertEqual(1, num_calls(start, 2)),
+ case couch_server:with_spidermonkey() of
+ true ->
+ ?assertEqual(1, num_calls(complete, 1)),
+ ?assertEqual(2, num_calls(checkpoint, 1)),
+ ?assertEqual(1, num_calls(db, ['_', DbName])),
+ ?assertEqual(1, num_calls(ddoc, ['_', DbName, '_'])),
+ ?assert(num_calls(shards, 2) >= 1),
+ ?assertEqual(0, num_calls(db_opened, 2)),
+ ?assertEqual(0, num_calls(doc_id, ['_', ?DOC1, '_'])),
+ ?assertEqual(0, num_calls(doc_id, ['_', ?DOC2, '_'])),
+ ?assertEqual(0, num_calls(doc_id, ['_', ?DOC3, '_'])),
+ ?assertEqual(0, num_calls(doc_id, ['_', ?DOC4, '_'])),
+ ?assertEqual(0, num_calls(doc_id, ['_', ?DOC5, '_'])),
+ ?assertEqual(0, num_calls(db_closing, 2)),
+ ?assertEqual(0, couch_stats:sample([couchdb, query_server,
process_error_exits])),
+ ?assertEqual(0, couch_stats:sample([couchdb, query_server,
process_errors])),
+ ?assertEqual(0, couch_stats:sample([couchdb, query_server,
process_exits])),
+ % start and complete only = 2 total
+ ?assertEqual(2, log_calls(warning));
+ false ->
+ ok
+ end.
+
config_delete_section(Section) ->
[config:delete(K, V, false) || {K, V} <- config:get(Section)].
@@ -165,3 +279,69 @@ log_calls(Level) ->
wait_exit(MSec) ->
meck:wait(couch_scanner_server, handle_info, [{'EXIT', '_', '_'}, '_'],
MSec).
+
+ddoc_vdu(Doc) ->
+ Doc#{
+ validate_doc_update => <<
+ "function(newdoc, olddoc, userctx, sec){\n"
+ " if(newdoc.a == 'w') {\n"
+ " newdoc.a.search(/(w+)/);\n"
+ " if(RegExp.$1 == 'w') {return true} else
{throw('forbidden')}\n"
+ " } else {\n"
+ " return true\n"
+ " }\n"
+ "}"
+ >>
+ }.
+
+ddoc_filter(Doc) ->
+ Doc#{
+ filters => #{
+ f => <<
+ "function(doc, req) {\n"
+ " if(doc.a == 'z') {\n"
+ " doc.a.search(/(z+)/);\n"
+ " if(RegExp.$1 == 'z') {return true} else {return false};\n"
+ " } else {\n"
+ " return true;\n"
+ " }\n"
+ "}"
+ >>
+ }
+ }.
+
+ddoc_view(Doc) ->
+ Doc#{
+ views => #{
+ v => #{
+ map => <<
+ "function(doc) {\n"
+ " if(doc.a == 'x') {\n"
+ " r = doc.a.search(/(x+)/); emit(r, RegExp.$1)\n"
+ " } else {\n"
+ " emit(doc.a, doc.a);\n"
+ " }\n"
+ "}"
+ >>,
+ reduce => <<
+ "function(ks, vs, rereduce) {\n"
+ " v0 = vs[0];\n"
+ " if (!rereduce) {\n"
+ " k0 = ks[0];\n"
+ " if (k0 == 'y') {\n"
+ " k0.search(/(y+)/);\n"
+ " return RegExp.$1;\n"
+ " };\n"
+ " return v0;\n"
+ " } else {\n"
+ " if (v0 == 'u') {\n"
+ " v0.search(/(u+)/);\n"
+ " return RegExp.$1;\n"
+ " };\n"
+ " return v0;\n"
+ " }\n"
+ "}"
+ >>
+ }
+ }
+ }.
