This is an automated email from the ASF dual-hosted git repository.
nickva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
The following commit(s) were added to refs/heads/main by this push:
new b54baba66 Tighten options and term decoding
b54baba66 is described below
commit b54baba6607b7605f036998db1877079c99e6f6d
Author: Nick Vatamaniuc <[email protected]>
AuthorDate: Sat Nov 14 11:47:22 2026 -0500
Tighten options and term decoding
Improve replicator options and term decoding. Add some limits and for
options,
since we already have an accepted allow-list that only takes string,
booleans
and numbers just use a plain regex for it.
---
src/couch/src/couch_util.erl | 22 ++++
src/couch/test/eunit/couch_util_tests.erl | 10 ++
.../src/couch_replicator_parse.erl | 126 +++++++++++++++++----
src/dreyfus/src/dreyfus_bookmark.erl | 2 +-
src/fabric/src/fabric_view_changes.erl | 2 +-
src/mango/src/mango_json_bookmark.erl | 2 +-
6 files changed, 139 insertions(+), 25 deletions(-)
diff --git a/src/couch/src/couch_util.erl b/src/couch/src/couch_util.erl
index 6ac8a47ea..b25b1458c 100644
--- a/src/couch/src/couch_util.erl
+++ b/src/couch/src/couch_util.erl
@@ -48,6 +48,7 @@
-export([new_set/0, set_from_list/1]).
-export([hibernate_after/1]).
-export([gunzip_with_limit/1, gunzip_with_limit/2]).
+-export([binary_to_term_safe/1, binary_to_term_safe/2]).
-include_lib("couch/include/couch_db.hrl").
@@ -68,6 +69,9 @@
-define(DEFAULT_HIBERNATE_AFTER, 5000).
+% Max uncompressed size param terms
+-define(TERM_MAX_SIZE, 1048576).
+
priv_dir() ->
case code:priv_dir(couch) of
{error, bad_name} ->
@@ -826,6 +830,24 @@ hibernate_after(Module) when is_atom(Module) ->
[{hibernate_after, Timeout}]
end.
+binary_to_term_safe(Bin) ->
+ binary_to_term_safe(Bin, ?TERM_MAX_SIZE).
+
+binary_to_term_safe(Bin, MaxSize) when is_integer(MaxSize), MaxSize >= 0 ->
+ case ext_term_size(Bin) of
+ Size when Size > MaxSize -> error({term_too_large, Size});
+ _ -> binary_to_term(Bin, [safe])
+ end.
+
+ext_term_size(<<131, 80, Size:32, _/binary>> = Bin) ->
+ % See https://www.erlang.org/doc/apps/erts/erl_ext_dist.html
+ max(Size, byte_size(Bin));
+ext_term_size(<<Bin/binary>>) ->
+ byte_size(Bin);
+ext_term_size(_NotBinary) ->
+ % we'll get a badarg in binary_to_term for it
+ 0.
+
gunzip_with_limit(Data) ->
Limit = chttpd_util:get_chttpd_config_integer("max_http_request_size",
4294967296),
gunzip_with_limit(Data, Limit).
diff --git a/src/couch/test/eunit/couch_util_tests.erl
b/src/couch/test/eunit/couch_util_tests.erl
index ac8b5119c..fcafdaaa3 100644
--- a/src/couch/test/eunit/couch_util_tests.erl
+++ b/src/couch/test/eunit/couch_util_tests.erl
@@ -316,3 +316,13 @@ url_strip_password_test() ->
Cases
),
?assertEqual(undefined, couch_util:url_strip_password(undefined)).
+
+binary_to_term_safe_too_large_test() ->
+ % Not compressed
+ Bin = term_to_binary(binary:copy(<<0>>, 1000)),
+ ?assertEqual(1000, byte_size(couch_util:binary_to_term_safe(Bin,
byte_size(Bin)))),
+ ?assertError({term_too_large, _}, couch_util:binary_to_term_safe(Bin,
byte_size(Bin) - 1)),
+ % Compressed
+ Compressed = term_to_binary(binary:copy(<<0>>, 100000), [compressed]),
+ ?assertEqual(100000, byte_size(couch_util:binary_to_term_safe(Compressed,
100005))),
+ ?assertError({term_too_large, 100005},
couch_util:binary_to_term_safe(Compressed, 100004)).
diff --git a/src/couch_replicator/src/couch_replicator_parse.erl
b/src/couch_replicator/src/couch_replicator_parse.erl
index 985ca911b..23486b3e6 100644
--- a/src/couch_replicator/src/couch_replicator_parse.erl
+++ b/src/couch_replicator/src/couch_replicator_parse.erl
@@ -38,6 +38,9 @@
-define(VALID_IBROWSE_OPTS, [
prefer_ipv6
]).
+% {K, V} option tuples. K is binary and V = true | false | 0..9
+-define(OPT_RE,
"\\{\\s*([a-z][a-z0-9_]*)\\s*,\\s*(true|false|[0-9]{1,9})\\s*\\}").
+-define(OPT_MAX, 1024).
-define(VALID_PROTOCOLS, #{
endpoint => [http, https],
proxy => [http, https, socks5]
@@ -297,34 +300,45 @@ cfg_atoms(Cfg, Default) ->
end.
cfg_sock_opts() ->
- CfgTerm = cfg("socket_options"),
- parse_opts(CfgTerm, ?DEFAULT_SOCK_OPTS, ?VALID_SOCK_OPTS).
+ cfg_opts("socket_options", ?DEFAULT_SOCK_OPTS, ?VALID_SOCK_OPTS).
cfg_ibrowse_opts() ->
- CfgTerm = cfg("ibrowse_options"),
- parse_opts(CfgTerm, ?DEFAULT_IBROWSE_OPTS, ?VALID_IBROWSE_OPTS).
+ cfg_opts("ibrowse_options", ?DEFAULT_IBROWSE_OPTS, ?VALID_IBROWSE_OPTS).
cfg(Var) ->
config:get("replicator", Var).
-parse_opts(undefined, Defaults, _) ->
- Defaults;
-parse_opts(Term, _Defaults, ValidOpts) ->
- SocketOptions =
- case couch_util:parse_term(Term) of
- {ok, Opts} -> Opts;
- {error, _Error} -> []
- end,
- Fun = fun({K, _}) -> lists:member(K, ValidOpts) end,
- lists:filtermap(Fun, SocketOptions).
+cfg_opts(Var, Defaults, ValidOpts) ->
+ case cfg(Var) of
+ undefined -> Defaults;
+ Val -> parse_opts(list_to_binary(Val), ValidOpts)
+ end.
-sock_opts(CfgTerm) ->
+sock_opts(Val) ->
ValidOpts = cfg_atoms("valid_socket_options", ?VALID_SOCK_OPTS),
- parse_opts(CfgTerm, ?DEFAULT_SOCK_OPTS, ValidOpts).
+ parse_opts(Val, ValidOpts).
-ibrowse_opts(CfgTerm) ->
+ibrowse_opts(Val) ->
ValidOpts = cfg_atoms("valid_ibrowse_options", ?VALID_IBROWSE_OPTS),
- parse_opts(CfgTerm, ?DEFAULT_IBROWSE_OPTS, ValidOpts).
+ parse_opts(Val, ValidOpts).
+
+parse_opts(Val, ValidOpts) when is_binary(Val), byte_size(Val) =< ?OPT_MAX ->
+ Names = #{atom_to_binary(Name) => Name || Name <- ValidOpts},
+ case re:run(Val, ?OPT_RE, [global, {capture, all_but_first, binary}]) of
+ {match, Pairs} ->
+ [{map_get(K, Names), opt_val(V)} || [K, V] <- Pairs, is_map_key(K,
Names)];
+ nomatch ->
+ []
+ end;
+parse_opts(_, _ValidOpts) ->
+ [].
+
+opt_val(~"true") ->
+ true;
+opt_val(~"false") ->
+ false;
+opt_val(<<Digits/binary>>) ->
+ binary_to_integer(Digits).
-spec convert_options([_]) -> [_].
convert_options([]) ->
@@ -384,9 +398,9 @@ convert_options([{<<"connection_timeout">>, V} | R]) ->
[{connection_timeout, couch_util:to_integer(V)} | convert_options(R)];
convert_options([{<<"retries_per_request">>, V} | R]) ->
[{retries, couch_util:to_integer(V)} | convert_options(R)];
-convert_options([{<<"socket_options">>, V} | R]) ->
+convert_options([{<<"socket_options">>, V} | R]) when is_binary(V) ->
[{socket_options, sock_opts(V)} | convert_options(R)];
-convert_options([{<<"ibrowse_options">>, V} | R]) ->
+convert_options([{<<"ibrowse_options">>, V} | R]) when is_binary(V) ->
[{ibrowse_options, ibrowse_opts(V)} | convert_options(R)];
convert_options([{<<"since_seq">>, V} | R]) ->
[{since_seq, V} | convert_options(R)];
@@ -642,7 +656,10 @@ local_replication_endpoint_error_test_() ->
?TDEF_FE(t_parse_proxy_invalid_protocol),
?TDEF_FE(t_parse_sock_opts),
?TDEF_FE(t_parse_ibrowse_opts),
- ?TDEF_FE(t_parse_opts_invalid)
+ ?TDEF_FE(t_parse_opts_invalid),
+ ?TDEF_FE(t_parse_opts),
+ ?TDEF_FE(t_parse_opts_max_size),
+ ?TDEF_FE(t_parse_opts_no_new_atoms)
]
}.
@@ -850,6 +867,71 @@ t_parse_ibrowse_opts(_) ->
).
t_parse_opts_invalid(_) ->
- ?assertEqual([], parse_opts(<<"<}garbage]][">>, [], [])).
+ Ignored = [
+ ~"",
+ ~"randomjunk",
+ ~"<}potato]][",
+ ~"keepalive",
+ ~"[keepalive, nodelay]",
+ ~"[{keepalive}]",
+ ~"[{keepalive, true, true}]",
+ ~"[{'keepalive', true}]",
+ ~"[{keepalive, 'true'}]",
+ ~"[{keepalive, \"true\"}]",
+ ~"[{recbuf, -1}]",
+ ~"[{recbuf, 1.5}]",
+ ~"[{recbuf, 16#ff}]",
+ ~"[{recbuf, 1_000}]",
+ ~"[{recbuf, [1]}]",
+ ~"[{recbuf, 1234567890}]"
+ ],
+ [?assertEqual([], sock_opts(Val), Val) || Val <- Ignored].
+
+t_parse_opts(_) ->
+ ?assertEqual([], sock_opts(~"[]")),
+ ?assertEqual(
+ [{keepalive, true}, {nodelay, false}],
+ sock_opts(~"[{keepalive, true}, {nodelay, false}]")
+ ),
+ ?assertEqual(
+ [{keepalive, false}, {recbuf, 65536}],
+ sock_opts(~"[{keepalive, false},{ recbuf,65536 } ]")
+ ),
+ % no binaries
+ ?assertEqual([], sock_opts(~"[{recbuf, <<1>>}]")).
+
+t_parse_opts_max_size(_) ->
+ Opts = ~"[{nodelay, true}, {prefer_ipv6, true}]",
+ Pad = fun(Sz) -> <<Opts/binary, (binary:copy(~" ", Sz -
byte_size(Opts)))/binary>> end,
+ ?assertEqual([{nodelay, true}], sock_opts(Pad(?OPT_MAX))),
+ ?assertEqual([{prefer_ipv6, true}], ibrowse_opts(Pad(?OPT_MAX))),
+ ?assertEqual([], sock_opts(Pad(?OPT_MAX + 1))),
+ ?assertEqual([], ibrowse_opts(Pad(?OPT_MAX + 1))).
+
+t_parse_opts_no_new_atoms(_) ->
+ [N1, N2, N3] = [new_name() || _ <- lists:seq(1, 3)],
+ F = fun(Fmt, Args) -> iolist_to_binary(io_lib:format(Fmt, Args)) end,
+ ?assertEqual([], sock_opts(F("[~s, '~s']", [N1, N2]))),
+ ?assertEqual([], sock_opts(F("[{~s, 1}, {'~s', true}]", [N1, N2]))),
+ ?assertEqual([], ibrowse_opts(F("[{~s, 1}, {'~s', true}]", [N1, N2]))),
+ ?assertEqual([], sock_opts(F("[{keepalive, ~s}, {recbuf, ['~s']}]", [N3,
N3]))),
+ RepDoc =
+ {[
+ {~"source", ~"http://a"},
+ {~"target", ~"http://b"},
+ {~"socket_options", F("[{nodelay, true}, {~s, 1}]", [N1])},
+ {~"ibrowse_options", F("[~s]", [N2])}
+ ]},
+ #rep{options = Options} = parse_rep_doc_without_id(RepDoc),
+ ?assertEqual([{nodelay, true}], get_value(socket_options, Options)),
+ ?assertEqual([], get_value(ibrowse_options, Options)),
+ % We never created those new atoms
+ ?assertError(badarg, binary_to_existing_atom(N1)),
+ ?assertError(badarg, binary_to_existing_atom(N2)),
+ ?assertError(badarg, binary_to_existing_atom(N3)).
+
+new_name() ->
+ Uniq = integer_to_binary(erlang:unique_integer([positive])),
+ <<"anatom_", Uniq/binary>>.
-endif.
diff --git a/src/dreyfus/src/dreyfus_bookmark.erl
b/src/dreyfus/src/dreyfus_bookmark.erl
index 268a6b845..145dda0b7 100644
--- a/src/dreyfus/src/dreyfus_bookmark.erl
+++ b/src/dreyfus/src/dreyfus_bookmark.erl
@@ -67,7 +67,7 @@ unpack(DbName, Packed) when is_binary(Packed) ->
).
unpack_entries(Packed) ->
- case binary_to_term(couch_util:decodeBase64Url(Packed), [safe]) of
+ case couch_util:binary_to_term_safe(couch_util:decodeBase64Url(Packed)) of
Entries when is_list(Entries) ->
lists:foreach(fun validate_entry/1, Entries),
Entries;
diff --git a/src/fabric/src/fabric_view_changes.erl
b/src/fabric/src/fabric_view_changes.erl
index f97475b8a..f79f6219c 100644
--- a/src/fabric/src/fabric_view_changes.erl
+++ b/src/fabric/src/fabric_view_changes.erl
@@ -489,7 +489,7 @@ unpack_seq_term(Opaque) ->
% Looks like a valid term_to_binary at least
SeqList =
try
- binary_to_term(Bin, [safe])
+ couch_util:binary_to_term_safe(Bin)
catch
error:badarg -> error(?BADSEQ)
end,
diff --git a/src/mango/src/mango_json_bookmark.erl
b/src/mango/src/mango_json_bookmark.erl
index 8446e0c8a..9ddf194f3 100644
--- a/src/mango/src/mango_json_bookmark.erl
+++ b/src/mango/src/mango_json_bookmark.erl
@@ -52,7 +52,7 @@ unpack(nil) ->
nil;
unpack(Packed) ->
try
- Bookmark = binary_to_term(couch_util:decodeBase64Url(Packed), [safe]),
+ Bookmark =
couch_util:binary_to_term_safe(couch_util:decodeBase64Url(Packed)),
verify(Bookmark)
catch
_:_ ->