This is an automated email from the ASF dual-hosted git repository.
nickva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
The following commit(s) were added to refs/heads/main by this push:
new afb135c26 Strip more replicator creds
afb135c26 is described below
commit afb135c26f6346f3ac484e6a7a02506a10762f19
Author: Nick Vatamaniuc <[email protected]>
AuthorDate: Tue Oct 6 12:30:41 2026 -0400
Strip more replicator creds
Fix non-owner readers seeing credentials in a shared _replicator db. Also,
while at it, fix stripping of creds for logging and API endpoints returning
replication stats
---
src/couch/src/couch_util.erl | 6 +-
src/couch/test/eunit/couch_util_tests.erl | 30 +++
src/couch_replicator/src/couch_replicator_docs.erl | 207 ++++++++++++++++++---
.../src/couch_replicator_scheduler_job.erl | 118 +++++++++++-
4 files changed, 335 insertions(+), 26 deletions(-)
diff --git a/src/couch/src/couch_util.erl b/src/couch/src/couch_util.erl
index 7a82888f2..6ac8a47ea 100644
--- a/src/couch/src/couch_util.erl
+++ b/src/couch/src/couch_util.erl
@@ -517,10 +517,12 @@ reorder_results(Keys, SortedResults, Default) ->
[maps:get(Key, Map, Default) || Key <- Keys].
url_strip_password(Url) when is_list(Url) ->
+ % \1: any scheme any case
+ % \2: username, possibly empty
re:replace(
Url,
- "(http|https|socks5)://([^:]+):[^@]+@(.*)$",
- "\\1://\\2:*****@\\3",
+ "^(\\s*[a-zA-Z][a-zA-Z0-9+.-]*://)([^:/?#]*):[^/?#]*@",
+ "\\1\\2:*****@",
[{return, list}]
);
url_strip_password(Other) ->
diff --git a/src/couch/test/eunit/couch_util_tests.erl
b/src/couch/test/eunit/couch_util_tests.erl
index ff40a6fdd..ac8b5119c 100644
--- a/src/couch/test/eunit/couch_util_tests.erl
+++ b/src/couch/test/eunit/couch_util_tests.erl
@@ -286,3 +286,33 @@ wait_hibernate(Pid) ->
end
end,
test_util:wait(WaitFun).
+
+url_strip_password_test() ->
+ Cases = [
+ {"http://h/db", "http://h/db"},
+ {"http://h:15984/db", "http://h:15984/db"},
+ {"http://u@h/db", "http://u@h/db"},
+ {"http://u:p@h/db", "http://u:*****@h/db"},
+ {"https://u:p@h:5984/db?a=b", "https://u:*****@h:5984/db?a=b"},
+ {"socks5://u:p@h:1081", "socks5://u:*****@h:1081"},
+ {"HTTPS://u:p@h/db", "HTTPS://u:*****@h/db"},
+ {"https://:p@h/db", "https://:*****@h/db"},
+ {"https://u:p@ss@h/db", "https://u:*****@h/db"},
+ {"https://u:p%40ss@h/db", "https://u:*****@h/db"},
+ {"http://u:p@[::1]:25984/db", "http://u:*****@[::1]:25984/db"},
+ {"http://u:p@h/d@b", "http://u:*****@h/d@b"},
+ {"http://h/d:x@b", "http://h/d:x@b"},
+ {"http://h:35984?x=a@b", "http://h:35984?x=a@b"},
+ {"https://[email protected]:p@h/db", "https://[email protected]:*****@h/db"},
+ {"https://[email protected]:p@ss@h/db", "https://[email protected]:*****@h/db"},
+ {"https://[email protected]@h/db", "https://[email protected]@h/db"},
+ {"http://u@h:15984/db", "http://u@h:15984/db"},
+ {"http://[::1]:25984/d@b", "http://[::1]:25984/d@b"}
+ ],
+ lists:foreach(
+ fun({Url, Expect}) ->
+ ?assertEqual(Expect, couch_util:url_strip_password(Url))
+ end,
+ Cases
+ ),
+ ?assertEqual(undefined, couch_util:url_strip_password(undefined)).
diff --git a/src/couch_replicator/src/couch_replicator_docs.erl
b/src/couch_replicator/src/couch_replicator_docs.erl
index 8f134aaab..1df692080 100644
--- a/src/couch_replicator/src/couch_replicator_docs.erl
+++ b/src/couch_replicator/src/couch_replicator_docs.erl
@@ -295,10 +295,7 @@ after_doc_read(#doc{body = {Body}} = Doc, Db) ->
% replicator app: this is another user reading it. Strip
% the creds and make up a fake "rev" since we're returning
% a technically non-existent document body
- Source = strip_credentials(get_value(<<"source">>, Body)),
- Target = strip_credentials(get_value(<<"target">>, Body)),
- NewBody0 = ?replace(Body, <<"source">>, Source),
- NewBody = ?replace(NewBody0, <<"target">>, Target),
+ NewBody = strip_body_credentials(Body),
#doc{revs = {Pos, [_ | Revs]}} = Doc,
NewDoc = Doc#doc{body = {NewBody}, revs = {Pos - 1, Revs}},
NewRevId = couch_db:new_revid(NewDoc),
@@ -306,25 +303,61 @@ after_doc_read(#doc{body = {Body}} = Doc, Db) ->
end
end.
+-define(CREDS, [~"source", ~"target", ~"proxy", ~"source_proxy",
~"target_proxy"]).
+
+strip_body_credentials(Body) ->
+ lists:map(
+ fun({K, V}) ->
+ case lists:member(K, ?CREDS) of
+ true -> {K, strip_credentials(V)};
+ false -> {K, V}
+ end
+ end,
+ Body
+ ).
+
-spec strip_credentials
- (undefined) -> undefined;
- (binary()) -> binary();
- ({[_]}) -> {[_]}.
-strip_credentials(undefined) ->
- undefined;
+ (binary()) -> binary() | null;
+ ({[_]}) -> {[_]};
+ (number() | boolean() | null | list()) -> null.
strip_credentials(Url) when is_binary(Url) ->
- re:replace(
- Url,
- "http(s)?://(?:[^:]+):[^@]+@(.*)$",
- "http\\1://\\2",
- [{return, binary}]
- );
-strip_credentials({Props0}) ->
+ strip_url_userinfo(Url);
+strip_credentials({Props0}) when is_list(Props0) ->
Props1 = lists:keydelete(<<"headers">>, 1, Props0),
% Strip "auth" just like headers, for replication plugins it can be a place
% to stash credential that are not necessarily in headers
Props2 = lists:keydelete(<<"auth">>, 1, Props1),
- {Props2}.
+ % Object form endpoints may have creds in "url"
+ Props3 =
+ case lists:keyfind(~"url", 1, Props2) of
+ {~"url", Url} -> ?replace(Props2, ~"url", strip_credentials(Url));
+ false -> Props2
+ end,
+ {Props3};
+strip_credentials(_Other) ->
+ % Failed or broken replication docs. Just hide the whole thing as might
+ % still contain mistyped creds.
+ null.
+
+% Remove userinfo from url. Match the parser with the replicator. If we can't
+% parse remove everything between scheme and last "@"
+%
+strip_url_userinfo(Url) when is_binary(Url) ->
+ case uri_string:parse(Url) of
+ #{userinfo := _} = UriMap ->
+ case uri_string:recompose(maps:remove(userinfo, UriMap)) of
+ <<_/binary>> = NoCreds -> NoCreds;
+ {error, _, _} -> null
+ end;
+ #{} ->
+ Url;
+ {error, _, _} ->
+ % \1 is scheme in any case, maybe with leading spaces. Use dotall
+ % to match even newlines in case passwords have those too somehow
+ re:replace(Url, "^(\\s*[a-zA-Z][a-zA-Z0-9+.-]*://).*@", "\\1", [
+ dotall, {return, binary}
+ ])
+ end.
error_reason({shutdown, Error}) ->
error_reason(Error);
@@ -356,10 +389,6 @@ check_strip_credentials_test() ->
[
?assertEqual(Expected, strip_credentials(Body))
|| {Expected, Body} <- [
- {
- undefined,
- undefined
- },
{
<<"https://remote_server/database">>,
<<"https://foo:bar@remote_server/database">>
@@ -379,10 +408,95 @@ check_strip_credentials_test() ->
{
{[{<<"_id">>, <<"foo">>}]},
{[{<<"_id">>, <<"foo">>}, {<<"auth">>, <<"pluginsecret">>}]}
+ },
+ {
+ {[{~"url", ~"https://remote_server/database"}]},
+ {[{~"url", ~"https://foo:bar@remote_server/database"}]}
+ },
+ {
+ {[{~"url", ~"http://remote_server/database"}]},
+ {[
+ {~"url", ~"http://foo:bar@remote_server/database"},
+ {~"headers", ~"bar"},
+ {~"auth", ~"pluginsecret"}
+ ]}
+ },
+ {
+ {[{~"url", ~"https://remote_server/database"}]},
+ {[{~"url", ~"https://foo:p%40ss%3Aw@remote_server/database"}]}
+ },
+ {
+ ~"HTTPS://remote_server/database",
+ ~"HTTPS://foo:bar@remote_server/database"
+ },
+ {
+ ~"https://remote_server/database",
+ ~"https://:bar@remote_server/database"
+ },
+ {
+ ~"https://remote_server/database",
+ ~"https://foo@remote_server/database"
+ },
+ {
+ ~"https://remote_server/database",
+ ~"https://foo:b@r@remote_server/database"
+ },
+ {
+ ~"https://remote_server/database",
+ ~"https://foo:b/r@remote_server/database"
+ },
+ {
+ ~"http://[::1]:5984/db?a=b",
+ ~"http://foo:bar@[::1]:5984/db?a=b"
+ },
+ {
+ ~"http://remote_server/d@b",
+ ~"http://foo:bar@remote_server/d@b"
+ },
+ {
+ ~"http://remote_server/database",
+ ~"http://remote_server/database"
+ },
+ {
+ ~"localdb",
+ ~"localdb"
+ },
+ {
+ null,
+ [~"https://foo:bar@remote_server/database"]
+ },
+ {
+ null,
+ 42
}
]
].
+check_strip_body_credentials_test() ->
+ Body = [
+ {~"_id", ~"foo"},
+ {~"source", {[{~"url", ~"https://u:p1@s/db"}]}},
+ {~"target", ~"https://u:p2@t/db"},
+ {~"proxy", ~"http://u:p3@proxy:3128"},
+ {~"source_proxy", ~"socks5://u:p4@sproxy:1080"},
+ {~"target_proxy", ~"http://u:p5@tproxy:3128"},
+ {~"other", ~"https://u:p6@x/db"}
+ ],
+ ?assertEqual(
+ [
+ {~"_id", ~"foo"},
+ {~"source", {[{~"url", ~"https://s/db"}]}},
+ {~"target", ~"https://t/db"},
+ {~"proxy", ~"http://proxy:3128"},
+ {~"source_proxy", ~"socks5://sproxy:1080"},
+ {~"target_proxy", ~"http://tproxy:3128"},
+ {~"other", ~"https://u:p6@x/db"}
+ ],
+ strip_body_credentials(Body)
+ ),
+ % Missing fields are not added
+ ?assertEqual([{~"_id", ~"foo"}], strip_body_credentials([{~"_id",
~"foo"}])).
+
setup() ->
TmpDbName = ?tempdb(),
DbName = <<TmpDbName/binary, "/_replicator">>,
@@ -450,7 +564,9 @@ replicator_can_update_docs_test_() ->
?TDEF_FE(t_after_doc_read_as_replicator),
?TDEF_FE(t_after_doc_read_internal_replicator),
?TDEF_FE(t_after_doc_read_matching_owner),
- ?TDEF_FE(t_after_doc_read_not_matching_owner)
+ ?TDEF_FE(t_after_doc_read_not_matching_owner),
+ ?TDEF_FE(t_after_doc_read_not_matching_owner_object_endpoints),
+ ?TDEF_FE(t_after_doc_read_not_matching_owner_proxies)
]
}
}.
@@ -605,6 +721,53 @@ t_after_doc_read_not_matching_owner(DbName) ->
},
?assertEqual(StrippedMap, Map1).
+t_after_doc_read_not_matching_owner_object_endpoints(DbName) ->
+ DocId = ~"doc1",
+ Map = #{
+ ~"source" => #{~"url" => ~"https://user1:pass1@localhost:5984/db1"},
+ ~"target" => #{
+ ~"url" => ~"https://user2:pass2@localhost:5984/db2",
+ ~"headers" => #{~"Authorization" => ~"Bearer secret"},
+ ~"auth" => #{
+ ~"basic" => #{~"username" => ~"u", ~"password" => ~"p"}
+ }
+ },
+ ?OWNER => ~"o1"
+ },
+ {ok, _} = write_doc(DbName, DocId, Map),
+ Ctx = {user_ctx, #user_ctx{name = ~"o2", roles = [~"tomato", ~"potato"]}},
+ Map1 = read_doc(DbName, DocId, Ctx),
+ StrippedMap = #{
+ ~"source" => #{~"url" => ~"https://localhost:5984/db1"},
+ ~"target" => #{~"url" => ~"https://localhost:5984/db2"},
+ ?OWNER => ~"o1"
+ },
+ ?assertEqual(StrippedMap, Map1).
+
+t_after_doc_read_not_matching_owner_proxies(DbName) ->
+ DocId = ~"doc1",
+ Map = #{
+ ~"source" => ~"https://localhost:5984/db1",
+ ~"target" => ~"https://localhost:5984/db2",
+ ~"source_proxy" => ~"http://user1:pass1@proxy1:3128",
+ ~"target_proxy" => ~"socks5://user2:pass2@proxy2:1080",
+ ?OWNER => ~"o1"
+ },
+ {ok, _} = write_doc(DbName, DocId, Map),
+ Ctx = {user_ctx, #user_ctx{name = ~"o2", roles = [~"tomato", ~"potato"]}},
+ Map1 = read_doc(DbName, DocId, Ctx),
+ StrippedMap = #{
+ ~"source" => ~"https://localhost:5984/db1",
+ ~"target" => ~"https://localhost:5984/db2",
+ ~"source_proxy" => ~"http://proxy1:3128",
+ ~"target_proxy" => ~"socks5://proxy2:1080",
+ ?OWNER => ~"o1"
+ },
+ ?assertEqual(StrippedMap, Map1),
+ % Owner still sees everything
+ OwnerCtx = {user_ctx, #user_ctx{name = ~"o1", roles = []}},
+ ?assertEqual(Map, read_doc(DbName, DocId, OwnerCtx)).
+
ejson_from_map(#{} = Map) ->
?JSON_DECODE(?JSON_ENCODE(Map)).
diff --git a/src/couch_replicator/src/couch_replicator_scheduler_job.erl
b/src/couch_replicator/src/couch_replicator_scheduler_job.erl
index 55a28ad7e..f795917bc 100644
--- a/src/couch_replicator/src/couch_replicator_scheduler_job.erl
+++ b/src/couch_replicator/src/couch_replicator_scheduler_job.erl
@@ -544,10 +544,48 @@ headers_strip_creds([{Key, Value0} | Rest], Acc) ->
end,
headers_strip_creds(Rest, [{Key, Value} | Acc]).
-httpdb_strip_creds(#httpdb{url = Url, headers = Headers} = HttpDb) ->
+% normalize_basic_auth moves basic user:pass combo to an auth props object
+auth_props_strip_creds(Props) ->
+ lists:map(
+ fun
+ ({~"username", User}) when is_binary(User) -> {~"username", User};
+ ({Key, {Nested}}) when is_list(Nested) -> {Key,
{auth_props_strip_creds(Nested)}};
+ ({Key, _}) -> {Key, ~"*****"};
+ (_) -> ~"*****"
+ end,
+ Props
+ ).
+
+% ibrowse opts can have cred too for proxies and ssl options
+ibrowse_options_strip_creds(Options) ->
+ lists:map(
+ fun
+ ({proxy_password, _}) ->
+ {proxy_password, "*****"};
+ ({socks5_password, _}) ->
+ {socks5_password, "*****"};
+ ({ssl_options, SslOpts}) when is_list(SslOpts) ->
+ {ssl_options, lists:keyreplace(password, 1, SslOpts,
{password, "*****"})};
+ (Opt) ->
+ Opt
+ end,
+ Options
+ ).
+
+httpdb_strip_creds(#httpdb{} = HttpDb) ->
+ #httpdb{
+ url = Url,
+ headers = Headers,
+ proxy_url = ProxyUrl,
+ auth_props = AuthProps,
+ ibrowse_options = IbrowseOptions
+ } = HttpDb,
HttpDb#httpdb{
url = couch_util:url_strip_password(Url),
- headers = headers_strip_creds(Headers, [])
+ headers = headers_strip_creds(Headers, []),
+ proxy_url = couch_util:url_strip_password(ProxyUrl),
+ auth_props = auth_props_strip_creds(AuthProps),
+ ibrowse_options = ibrowse_options_strip_creds(IbrowseOptions)
};
httpdb_strip_creds(LocalDb) ->
LocalDb.
@@ -1274,4 +1312,80 @@ t_scheduler_job_format_status(_) ->
?assertEqual(<<"4">>, maps:get(current_through_seq, State)),
?assertEqual(<<"5">>, maps:get(highest_seq_done, State)).
+httpdb_rec_strip_creds_test_() ->
+ {
+ foreach,
+ fun meck_config/0,
+ fun(_) -> meck:unload() end,
+ [
+ ?TDEF_FE(t_httpdb_strip_creds),
+ ?TDEF_FE(t_httpdb_strip_creds_socks5)
+ ]
+ }.
+
+t_httpdb_strip_creds(_) ->
+ Endpoint =
+ {[
+ {~"url", ~"http://u:urlpass@h1/d1"},
+ {~"headers", {[{~"Authorization", ~"Bearer hidethebear"}]}}
+ ]},
+ Proxy = ~"http://pu:psecret@myproxy:8080",
+ HttpDb = couch_replicator_parse:parse_rep_db(Endpoint, Proxy, []),
+ HttpDb1 = httpdb_strip_creds(HttpDb),
+ % Doesn't show up if we dump the data structure to the log
+ ToStr = lists:flatten(io_lib:format("~p", [HttpDb1])),
+ [
+ ?assertEqual(nomatch, string:find(ToStr, S))
+ || S <- ["urlpass", "hidethebear", "psecret"]
+ ],
+ ?assertEqual("http://pu:*****@myproxy:8080", HttpDb1#httpdb.proxy_url),
+ ?assertEqual({"u", "*****"},
couch_replicator_utils:get_basic_auth_creds(HttpDb1)).
+
+t_httpdb_strip_creds_socks5(_) ->
+ % The proxy password can gets stuck into the ibrowse options so we check
+ % it's cleaned up there too
+ Proxy = ~"socks5://pu:psecret@myproxy:2080",
+ HttpDb = couch_replicator_parse:parse_rep_db(~"http://h1/d1", Proxy, []),
+ HttpDb1 = httpdb_strip_creds(HttpDb),
+ ToStr = lists:flatten(io_lib:format("~p", [HttpDb1])),
+ ?assertEqual(nomatch, string:find(ToStr, "psecret")),
+ ?assertEqual("*****", couch_util:get_value(socks5_password,
HttpDb1#httpdb.ibrowse_options)).
+
+httpdb_strip_creds_basic_auth_test() ->
+ % normalize_basic_auth/1 moves the creds from the url/headers to auth_props
+ % see if that gets cleaned up
+ Header = {"Authorization", "Basic " ++
base64:encode_to_string("user2:secret2")},
+ Basic = {[{~"username", ~"user3"}, {~"password", ~"secret3"}]},
+ lists:foreach(
+ fun({User, Pass, HttpDb0}) ->
+ HttpDb = couch_replicator_utils:normalize_basic_auth(HttpDb0),
+ ?assertEqual({User, Pass},
couch_replicator_utils:get_basic_auth_creds(HttpDb)),
+ HttpDb1 = httpdb_strip_creds(HttpDb),
+ ToStr = lists:flatten(io_lib:format("~p", [HttpDb1])),
+ ?assertEqual(nomatch, string:find(ToStr, Pass))
+ end,
+ [
+ {"user1", "secret1", #httpdb{url = "http://user1:secret1@h/db/"}},
+ {"user2", "secret2", #httpdb{url = "http://h/db/", headers =
[Header]}},
+ {"user3", "secret3", #httpdb{url = "http://h/db/", auth_props =
[{~"basic", Basic}]}}
+ ]
+ ).
+
+httpdb_strip_creds_auth_plugins_test() ->
+ % We clean up various plugins as well
+ IAM = {[{~"iam", {[{~"api_key", ~"secret4"}]}}]},
+ Other = {[{~"token", ~"secret5"}]},
+ AuthProps = [{~"ibm", IAM}, {~"other", Other}],
+ HttpDb1 = httpdb_strip_creds(#httpdb{url = "http://h/db/", auth_props =
AuthProps}),
+ ToStr = lists:flatten(io_lib:format("~p", [HttpDb1])),
+ ?assertEqual(nomatch, string:find(ToStr, "secret4")),
+ ?assertEqual(nomatch, string:find(ToStr, "secret5")),
+ ?assertEqual(
+ [
+ {~"ibm", {[{~"iam", {[{~"api_key", ~"*****"}]}}]}},
+ {~"other", {[{~"token", ~"*****"}]}}
+ ],
+ HttpDb1#httpdb.auth_props
+ ).
+
-endif.