This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
The following commit(s) were added to refs/heads/main by this push:
new 1860ebb Improve basic auth credentials handling in replicator
1860ebb is described below
commit 1860ebbf2fa1731a62f3c9b107b2e52811489c1e
Author: Nick Vatamaniuc <[email protected]>
AuthorDate: Thu Jun 3 14:09:43 2021 -0400
Improve basic auth credentials handling in replicator
This is a port of commit ecd266b0e87f44e1080cabdb4c28e4758f5a4406 from 3.x
to
main. Including the same commit message from there for completeness and then
towards the end, there is a description of changes required to port the PR
to
main.
Previously, there were two ways to pass in basic auth credentials for
endpoints
-- using URL's userinfo part, and encoding them in an `"Authorization":
"basic
..."` header. Neither one is ideal for these reasons:
* Passwords in userinfo don't allow using ":", "@" and other characters.
However, even after switching to always unquoting them like we did
recently
[1], it could break authentication for usernames or passwords previously
containing "+" or "%HH" patterns, as "+" might now be decoded to a " ".
* Base64 encoded headers need an extra step to encode them. Also, quite
often
these encoded headers are confused as being "encrypted" and shared in a
clear channel.
To improve this, revert the recent commit to unquote URL userinfo parts to
restore backwards compatibility, and introduce a way to pass in basic auth
credentials in the "auth" object. The "auth" object was already added a
while
back to allow authentication plugins to store their credentials in it. The
format is:
```
"source": {
"url": "https://host/db";,
"auth": {
"basic": {
"username":"myuser",
"password":"mypassword"
}
}
}
```
{"auth" : "basic" : {...}} object is checked first, and if credentials are
provided, they will be used. If they are not then userinfo and basic auth
header will be parsed.
Internally, there was a good amount duplication related to parsing
credentials
from userinfo and headers in replication ID generation logic and in the auth
session plugin. As a cleanup, consolidate that logic in the
`couch_replicator_parse` module.
The commit is quite different from the 3.x one for these two reasons:
* `main` uses two types of replication endpoint "objects": `#httpdb`
records
and `HttpDb` maps. In most cases it uses maps which can be serialized and
deserialized to and from json. But in lower level, connection handling
code
in couch_replicator_httpc, it uses `#httpdb` records. This explain the
need
to still handle both representations. Auth session plugin, for instance,
uses the lower level #httpdb records while replicator ID handling code
uses
the map based one.
* `main` has all the parsing of replication documents and `_replicate`
request
bodies in a separate `couch_replicator_parse`. So, most of the code which
handles normalizing basic auth creds is there instead of
`couch_replicator_docs` or `couch_replicator_utils` like it is in 3.x
---
.../src/couch_replicator_auth_session.erl | 238 ++++----------
.../src/couch_replicator_httpc.erl | 8 +-
src/couch_replicator/src/couch_replicator_ids.erl | 114 ++++---
src/couch_replicator/src/couch_replicator_job.erl | 4 +-
.../src/couch_replicator_parse.erl | 355 ++++++++++++++++++++-
.../src/couch_replicator_utils.erl | 121 +++----
6 files changed, 521 insertions(+), 319 deletions(-)
diff --git a/src/couch_replicator/src/couch_replicator_auth_session.erl
b/src/couch_replicator/src/couch_replicator_auth_session.erl
index 6ca30c8..b2bf317 100644
--- a/src/couch_replicator/src/couch_replicator_auth_session.erl
+++ b/src/couch_replicator/src/couch_replicator_auth_session.erl
@@ -79,7 +79,6 @@
-type headers() :: [{string(), string()}].
-type code() :: non_neg_integer().
--type creds() :: {string() | undefined, string() | undefined}.
-type time_sec() :: non_neg_integer().
-type age() :: time_sec() | undefined.
@@ -246,59 +245,15 @@ init_state(#httpdb{} = HttpDb) ->
-spec extract_creds(#httpdb{}) ->
{ok, string(), string(), #httpdb{}} | {error, term()}.
-extract_creds(#httpdb{url = Url, headers = Headers} = HttpDb) ->
- {{HeadersUser, HeadersPass}, HeadersNoCreds} =
- couch_replicator_utils:remove_basic_auth_from_headers(Headers),
- case extract_creds_from_url(Url) of
- {ok, UrlUser, UrlPass, UrlNoCreds} ->
- case pick_creds({UrlUser, UrlPass}, {HeadersUser, HeadersPass}) of
- {ok, User, Pass} ->
- HttpDb1 = HttpDb#httpdb{
- url = UrlNoCreds,
- headers = HeadersNoCreds
- },
- {ok, User, Pass, HttpDb1};
- {error, Error} ->
- {error, Error}
- end;
- {error, Error} ->
- {error, Error}
- end.
-
-
-% Credentials could be specified in the url and/or in the headers.
-% * If no credentials specified return error.
-% * If specified in url but not in headers, pick url creds.
-% * Otherwise pick headers creds.
-%
--spec pick_creds(creds(), creds()) ->
- {ok, string(), string()} | {error, missing_credentials}.
-pick_creds({undefined, _}, {undefined, _}) ->
- {error, missing_credentials};
-pick_creds({UrlUser, UrlPass}, {undefined, _}) ->
- {ok, UrlUser, UrlPass};
-pick_creds({_, _}, {HeadersUser, HeadersPass}) ->
- {ok, HeadersUser, HeadersPass}.
-
-
--spec extract_creds_from_url(string()) ->
- {ok, string() | undefined, string() | undefined, string()} |
- {error, term()}.
-extract_creds_from_url(Url) ->
- case ibrowse_lib:parse_url(Url) of
- {error, Error} ->
- {error, Error};
- #url{username = undefined, password = undefined} ->
- {ok, undefined, undefined, Url};
- #url{protocol = Proto, username = User, password = Pass} ->
- % Excise user and pass parts from the url. Try to keep the host,
- % port and path as they were in the original.
- Prefix = lists:concat([Proto, "://", User, ":", Pass, "@"]),
- Suffix = lists:sublist(Url, length(Prefix) + 1, length(Url) + 1),
- NoCreds = lists:concat([Proto, "://", Suffix]),
- User1 = chttpd:unquote(User),
- Pass1 = chttpd:unquote(Pass),
- {ok, User1, Pass1, NoCreds}
+extract_creds(#httpdb{} = HttpDb) ->
+ case couch_replicator_utils:get_basic_auth_creds(HttpDb) of
+ {undefined, undefined} ->
+ % Return error. Session plugin should ignore this replication
+ % endpoint as there are no valid creds which can be used
+ {error, missing_credentials};
+ {User, Pass} when is_list(User), is_list(Pass) ->
+ HttpDb1 = remove_basic_auth_creds(HttpDb),
+ {ok, User, Pass, HttpDb1}
end.
@@ -567,6 +522,11 @@ b64creds(User, Pass) ->
base64:encode_to_string(User ++ ":" ++ Pass).
+remove_basic_auth_creds(#httpdb{auth_props = Props} = HttpDb) ->
+ Props1 = maps:remove(<<"basic">>, Props),
+ HttpDb#httpdb{auth_props = Props1}.
+
+
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
@@ -582,116 +542,15 @@ get_session_url_test_() ->
]].
-extract_creds_success_test_() ->
- DefaultHeaders = (#httpdb{})#httpdb.headers,
- [?_assertEqual({ok, User, Pass, HttpDb2}, extract_creds(HttpDb1)) ||
- {HttpDb1, {User, Pass, HttpDb2}} <- [
- {
- #httpdb{url = "http://u:[email protected]/db"},
- {"u", "p", #httpdb{url = "http://x.y/db"}}
- },
- {
- #httpdb{url = "http://u%40:p%[email protected]/db"},
- {"u@", "p@", #httpdb{url = "http://x.y/db"}}
- },
- {
- #httpdb{url = "http://u%40u:p%[email protected]/db"},
- {"u@u", "p@p", #httpdb{url = "http://x.y/db"}}
- },
- {
- #httpdb{url = "http://u%40%401:p%40%[email protected]/db"},
- {"u@@1", "p@@1", #httpdb{url = "http://x.y/db"}}
- },
- {
- #httpdb{url = "http://u%40%2540:p%40%[email protected]/db"},
- {"u@%40", "p@%40", #httpdb{url = "http://x.y/db"}}
- },
- {
- #httpdb{url = "http://u:p@h:80/db"},
- {"u", "p", #httpdb{url = "http://h:80/db"}}
- },
- {
- #httpdb{url = "http://u%3A:p%3A@h:80/db"},
- {"u:", "p:", #httpdb{url = "http://h:80/db"}}
- },
- {
- #httpdb{url = "https://u:p@h/db"},
- {"u", "p", #httpdb{url = "https://h/db"}}
- },
- {
- #httpdb{url = "https://u%2F:p%2F@h/db"},
- {"u/", "p/", #httpdb{url = "https://h/db"}}
- },
- {
- #httpdb{url = "http://u:[email protected]:5984/db"},
- {"u", "p", #httpdb{url = "http://127.0.0.1:5984/db"}}
- },
- {
- #httpdb{url = "http://u:p@[2001:db8:a1b:12f9::1]/db"},
- {"u", "p", #httpdb{url = "http://[2001:db8:a1b:12f9::1]/db"}}
- },
- {
- #httpdb{url = "http://u:p@[2001:db8:a1b:12f9::1]:81/db"},
- {"u", "p", #httpdb{url = "http://[2001:db8:a1b:12f9::1]:81/db"}}
- },
- {
- #httpdb{url =
"http://u:p%3A%2F%5B%5D%40@[2001:db8:a1b:12f9::1]:81/db"},
- {"u", "p:/[]@", #httpdb{url =
"http://[2001:db8:a1b:12f9::1]:81/db"}}
- },
- {
- #httpdb{url = "http://u:[email protected]/db/other?query=Z&query=w"},
- {"u", "p", #httpdb{url = "http://x.y/db/other?query=Z&query=w"}}
- },
- {
- #httpdb{url = "http://u:p%[email protected]/db/other?query=Z&query=w"},
- {"u", "p?", #httpdb{url = "http://x.y/db/other?query=Z&query=w"}}
- },
- {
- #httpdb{
- url = "http://h/db";,
- headers = DefaultHeaders ++ [
- {"Authorization", "Basic " ++ b64creds("u", "p")}
- ]
- },
- {"u", "p", #httpdb{url = "http://h/db"}}
- },
- {
- #httpdb{
- url = "http://h/db";,
- headers = DefaultHeaders ++ [
- {"Authorization", "Basic " ++ b64creds("u", "p@")}
- ]
- },
- {"u", "p@", #httpdb{url = "http://h/db"}}
- },
- {
- #httpdb{
- url = "http://h/db";,
- headers = DefaultHeaders ++ [
- {"Authorization", "Basic " ++ b64creds("u", "p@%40")}
- ]
- },
- {"u", "p@%40", #httpdb{url = "http://h/db"}}
- },
- {
- #httpdb{
- url = "http://h/db";,
- headers = DefaultHeaders ++ [
- {"aUthoriZation", "bASIC " ++ b64creds("U", "p")}
- ]
- },
- {"U", "p", #httpdb{url = "http://h/db"}}
- },
- {
- #httpdb{
- url = "http://u1:p1@h/db";,
- headers = DefaultHeaders ++ [
- {"Authorization", "Basic " ++ b64creds("u2", "p2")}
- ]
- },
- {"u2", "p2", #httpdb{url = "http://h/db"}}
+extract_creds_success_test() ->
+ HttpDb = #httpdb{auth_props = #{
+ <<"basic">> => #{
+ <<"username">> => <<"u2">>,
+ <<"password">> => <<"p2">>
}
- ]].
+ }},
+ ?assertEqual({ok, "u2", "p2", #httpdb{}}, extract_creds(HttpDb)),
+ ?assertEqual({error, missing_credentials}, extract_creds(#httpdb{})).
cookie_update_test_() ->
@@ -808,7 +667,7 @@ t_process_ok_no_cookie() ->
t_init_state_fails_on_401() ->
?_test(begin
mock_http_401_response(),
- {error, Error} = init_state(#httpdb{url = "http://u:p@h"}),
+ {error, Error} = init_state(httpdb("http://u:p@h";)),
SessionUrl = "http://h/_session";,
?assertEqual({session_request_unauthorized, SessionUrl, "u"}, Error)
end).
@@ -818,32 +677,42 @@ t_init_state_401_with_require_valid_user() ->
?_test(begin
mock_http_401_response_with_require_valid_user(),
?assertMatch({ok, #httpdb{}, #state{cookie = "Cookie"}},
- init_state(#httpdb{url = "http://u:p@h"}))
+ init_state(httpdb("http://u:p@h";)))
end).
t_init_state_404() ->
?_test(begin
mock_http_404_response(),
- ?assertEqual(ignore, init_state(#httpdb{url = "http://u:p@h"}))
+ ?assertEqual(ignore, init_state(httpdb("http://u:p@h";)))
end).
t_init_state_no_creds() ->
?_test(begin
- ?_assertEqual(ignore, init_state(#httpdb{url = "http://h"}))
+ ?_assertEqual(ignore, init_state(httpdb("http://h";)))
end).
t_init_state_http_error() ->
?_test(begin
mock_http_error_response(),
- {error, Error} = init_state(#httpdb{url = "http://u:p@h"}),
+ {error, Error} = init_state(httpdb("http://u:p@h";)),
SessionUrl = "http://h/_session";,
?assertEqual({session_request_failed, SessionUrl, "u", x}, Error)
end).
+httpdb(Url) when is_list(Url) ->
+ FakeDoc = #{
+ <<"source">> => list_to_binary(Url),
+ <<"target">> => <<"http://somehost";>>
+ },
+ {ok, Rep} = couch_replicator_parse:parse_rep(FakeDoc, null),
+ HttpDb = maps:get(<<"source">>, Rep),
+ couch_replicator_api_wrap:db_from_json(HttpDb).
+
+
setup_all() ->
meck:expect(couch_replicator_httpc_pool, get_worker, 1, {ok, worker}),
meck:expect(couch_replicator_httpc_pool, release_worker_sync, 2, ok),
@@ -898,14 +767,6 @@ mock_http_error_response() ->
meck:expect(ibrowse, send_req_direct, 7, {error, x}).
-extract_creds_error_test_() ->
- [?_assertMatch({error, Error}, extract_creds(HttpDb)) ||
- {HttpDb, Error} <- [
- {#httpdb{url = "some_junk"}, invalid_uri},
- {#httpdb{url = "http://h/db"}, missing_credentials}
- ]].
-
-
parse_max_age_test_() ->
[?_assertEqual(R, parse_max_age(mochiweb_headers:make([{"Max-Age", A}])))
|| {A, R} <- [
@@ -922,4 +783,29 @@ parse_max_age_test_() ->
].
+remove_basic_auth_creds_test() ->
+ Check = fun(Props) ->
+ HttpDb = remove_basic_auth_creds(#httpdb{auth_props = Props}),
+ HttpDb#httpdb.auth_props
+ end,
+
+ ?assertEqual(#{}, Check(#{})),
+
+ ?assertEqual(#{<<"other">> => #{}}, Check(#{<<"other">> => #{}})),
+
+ ?assertEqual(#{}, Check(#{
+ <<"basic">> => #{
+ <<"username">> => <<"u">>,
+ <<"password">> => <<"p">>
+ }
+ })),
+
+ ?assertEqual(#{<<"other">> => #{}}, Check(#{
+ <<"basic">> => #{
+ <<"username">> => <<"u">>,
+ <<"password">> => <<"p">>
+ },
+ <<"other">> => #{}
+ })).
+
-endif.
diff --git a/src/couch_replicator/src/couch_replicator_httpc.erl
b/src/couch_replicator/src/couch_replicator_httpc.erl
index 28b0f38..59a79cf 100644
--- a/src/couch_replicator/src/couch_replicator_httpc.erl
+++ b/src/couch_replicator/src/couch_replicator_httpc.erl
@@ -119,7 +119,13 @@ send_ibrowse_req(#httpdb{headers = BaseHeaders} = HttpDb0,
Params) ->
end
end,
{ok, Worker} =
couch_replicator_httpc_pool:get_worker(HttpDb#httpdb.httpc_pool),
- IbrowseOptions = [
+ BasicAuthOpts = case couch_replicator_utils:get_basic_auth_creds(HttpDb) of
+ {undefined, undefined} ->
+ [];
+ {User, Pass} when is_list(User), is_list(Pass) ->
+ [{basic_auth, {User, Pass}}]
+ end,
+ IbrowseOptions = BasicAuthOpts ++ [
{response_format, binary}, {inactivity_timeout, HttpDb#httpdb.timeout}
|
lists:ukeymerge(1, get_value(ibrowse_options, Params, []),
HttpDb#httpdb.ibrowse_options)
diff --git a/src/couch_replicator/src/couch_replicator_ids.erl
b/src/couch_replicator/src/couch_replicator_ids.erl
index 44b9e47..b781404 100644
--- a/src/couch_replicator/src/couch_replicator_ids.erl
+++ b/src/couch_replicator/src/couch_replicator_ids.erl
@@ -174,22 +174,12 @@ get_rep_endpoint(#{<<"url">> := Url0, <<"headers">> :=
Headers0}) ->
get_v4_endpoint(#{} = HttpDb) ->
{remote, Url, Headers} = get_rep_endpoint(HttpDb),
- {{UserFromHeaders, _}, HeadersWithoutBasicAuth} =
- couch_replicator_utils:remove_basic_auth_from_headers(Headers),
- {UserFromUrl, Host, NonDefaultPort, Path} = get_v4_url_info(Url),
- User = pick_defined_value([UserFromUrl, UserFromHeaders]),
+ {User, _} = couch_replicator_utils:get_basic_auth_creds(HttpDb),
+ {Host, NonDefaultPort, Path} = get_v4_url_info(Url),
OAuth = undefined, % Keep this to ensure checkpoints don't change
- {remote, User, Host, NonDefaultPort, Path, HeadersWithoutBasicAuth, OAuth}.
+ {remote, User, Host, NonDefaultPort, Path, Headers, OAuth}.
-pick_defined_value(Values) ->
- case [V || V <- Values, V /= undefined] of
- [] ->
- undefined;
- DefinedValues ->
- hd(DefinedValues)
- end.
-
get_v4_url_info(Url) when is_binary(Url) ->
get_v4_url_info(binary_to_list(Url));
@@ -198,16 +188,16 @@ get_v4_url_info(Url) ->
{error, invalid_uri} ->
% Tolerate errors here to avoid a bad user document
% crashing the replicator
- {undefined, Url, undefined, undefined};
+ {Url, undefined, undefined};
#url{
protocol = Schema,
- username = User,
+
host = Host,
port = Port,
path = Path
} ->
NonDefaultPort = get_non_default_port(Schema, Port),
- {User, Host, NonDefaultPort, Path}
+ {Host, NonDefaultPort, Path}
end.
@@ -239,82 +229,82 @@ replication_id_convert_test_() ->
http_v4_endpoint_test_() ->
[?_assertMatch({remote, User, Host, Port, Path, HeadersNoAuth, undefined},
- get_v4_endpoint(#{<<"url">> => Url, <<"headers">> => Headers})) ||
- {{User, Host, Port, Path, HeadersNoAuth}, {Url, Headers}} <- [
+ get_v4_endpoint(HttpDb)) ||
+ {{User, Host, Port, Path, HeadersNoAuth}, HttpDb} <- [
{
{undefined, "host", default, "/", []},
- {<<"http://host";>>, #{}}
+ httpdb("http://host";)
},
{
{undefined, "host", default, "/", []},
- {<<"https://host";>>, #{}}
+ httpdb("https://host";)
},
{
{undefined, "host", default, "/", []},
- {<<"http://host:5984";>>, #{}}
+ httpdb("http://host:5984";)
},
{
{undefined, "host", 1, "/", []},
- {<<"http://host:1";>>, #{}}
+ httpdb("http://host:1";)
},
{
{undefined, "host", 2, "/", []},
- {<<"https://host:2";>>, #{}}
+ httpdb("https://host:2";)
},
{
{undefined, "host", default, "/", [{"h", "v"}]},
- {<<"http://host";>>, #{<<"h">> => <<"v">>}}
+ httpdb("http://host";, undefined, undefined, #{"h" => "v"})
},
{
{undefined, "host", default, "/a/b", []},
- {<<"http://host/a/b";>>, #{}}
- },
- {
- {"user", "host", default, "/", []},
- {<<"http://user:pass@host";>>, #{}}
- },
- {
- {"user", "host", 3, "/", []},
- {<<"http://user:pass@host:3";>>, #{}}
+ httpdb("http://host/a/b";)
},
{
{"user", "host", default, "/", []},
- {<<"http://user:newpass@host";>>, #{}}
+ httpdb("http://host";, "user", "pass")
},
{
{"user", "host", default, "/", []},
- {<<"http://host";>>, basic_auth(<<"user">>, <<"pass">>)}
- },
- {
- {"user", "host", default, "/", []},
- {<<"http://host";>>, basic_auth(<<"user">>, <<"newpass">>)}
- },
- {
- {"user1", "host", default, "/", []},
- {<<"http://user1:pass1@host";>>, basic_auth(<<"user2">>,
- <<"pass2">>)}
- },
- {
- {"user", "host", default, "/", [{"h", "v"}]},
- {<<"http://host";>>, maps:merge(#{<<"h">> => <<"v">>},
- basic_auth(<<"user">>, <<"pass">>))}
+ httpdb("http://host";, "user", "newpass")
},
{
- {undefined, "random_junk", undefined, undefined},
- {<<"random_junk">>, #{}}
- },
- {
- {undefined, "host", default, "/", []},
- {<<"http://host";>>, #{<<"Authorization">> =>
- <<"Basic bad">>}}
+ {"user2", "host", default, "/", [{"h", "v"}]},
+ httpdb("http://host";, "user2", "pass2", #{"h" => "v"})
}
]
].
-basic_auth(User, Pass) ->
- B64Auth = base64:encode(<<User/binary, ":", Pass/binary>>),
- #{<<"Authorization">> => <<"Basic ", B64Auth/binary>>}.
+httpdb(Url) ->
+ #{
+ <<"url">> => list_to_binary(Url),
+ <<"auth_props">> => #{},
+ <<"headers">> => #{}
+ }.
+
+
+httpdb(Url, User, Pass) ->
+ #{
+ <<"url">> => list_to_binary(Url),
+ <<"auth_props">> => #{
+ <<"basic">> => #{
+ <<"username">> => list_to_binary(User),
+ <<"password">> => list_to_binary(Pass)
+ }
+ },
+ <<"headers">> => #{}
+ }.
+
+
+httpdb(Url, User, Pass, #{} = Headers) ->
+ HttpDb1 = case {User, Pass} of
+ {undefined, undefined} -> httpdb(Url);
+ {User, Pass} -> httpdb(Url, User, Pass)
+ end,
+ Headers1 = maps:fold(fun(K, V, Acc) ->
+ Acc#{list_to_binary(K) => list_to_binary(V)}
+ end, #{}, Headers),
+ HttpDb1#{<<"headers">> => Headers1}.
version4_matches_couchdb3_test_() ->
@@ -352,4 +342,10 @@ id_matches_couchdb3(_) ->
?assertEqual(BaseId3x, BaseId).
+auth_props(User, Pass) when is_list(User), is_list(Pass) ->
+ [{<<"basic">>, {[
+ {<<"username">>, list_to_binary(User)},
+ {<<"password">>, list_to_binary(Pass)}
+ ]}}].
+
-endif.
diff --git a/src/couch_replicator/src/couch_replicator_job.erl
b/src/couch_replicator/src/couch_replicator_job.erl
index 951471a..381fe57 100644
--- a/src/couch_replicator/src/couch_replicator_job.erl
+++ b/src/couch_replicator/src/couch_replicator_job.erl
@@ -1727,8 +1727,8 @@ t_format_status(_) ->
},
Format = format_status(opts_ignored, [pdict, State]),
FmtOptions = proplists:get_value(options, Format),
- ?assertEqual("http://u:*****@h1/d1/";, proplists:get_value(source, Format)),
- ?assertEqual("http://u:*****@h2/d2/";, proplists:get_value(target, Format)),
+ ?assertEqual("http://h1/d1/";, proplists:get_value(source, Format)),
+ ?assertEqual("http://h2/d2/";, proplists:get_value(target, Format)),
?assertEqual(<<"base+ext">>, proplists:get_value(rep_id, Format)),
?assertEqual(true, maps:get(<<"create_target">>, FmtOptions)),
?assertEqual(<<"mydoc">>, proplists:get_value(doc_id, Format)),
diff --git a/src/couch_replicator/src/couch_replicator_parse.erl
b/src/couch_replicator/src/couch_replicator_parse.erl
index ac25bee..38d50d4 100644
--- a/src/couch_replicator/src/couch_replicator_parse.erl
+++ b/src/couch_replicator/src/couch_replicator_parse.erl
@@ -205,7 +205,7 @@ parse_rep_db(#{} = Endpoint, #{} = ProxyParams, #{} =
Options) ->
}, ProxyParams),
SslParams = ssl_params(Url),
- #{
+ HttpDb = #{
<<"url">> => Url,
<<"auth_props">> => AuthProps,
<<"headers">> => Headers,
@@ -214,7 +214,8 @@ parse_rep_db(#{} = Endpoint, #{} = ProxyParams, #{} =
Options) ->
<<"http_connections">> => maps:get(<<"http_connections">>, Options),
<<"retries">> => maps:get(<<"retries_per_request">>, Options),
<<"proxy_url">> => ProxyUrl
- };
+ },
+ normalize_basic_auth(HttpDb);
parse_rep_db(<<"http://";, _/binary>> = Url, Proxy, Options) ->
parse_rep_db(#{<<"url">> => Url}, Proxy, Options);
@@ -475,6 +476,117 @@ ssl_verify_options(false) ->
}.
+-spec set_basic_auth_creds(string(), string(), map()) -> map().
+set_basic_auth_creds(undefined, undefined, #{}= HttpDb) ->
+ HttpDb;
+set_basic_auth_creds(User, Pass, #{} = HttpDb)
+ when is_list(User), is_list(Pass) ->
+ #{<<"auth_props">> := AuthProps} = HttpDb,
+ UserPass = #{
+ <<"username">> => list_to_binary(User),
+ <<"password">> => list_to_binary(Pass)
+ },
+ AuthProps1 = AuthProps#{<<"basic">> => UserPass},
+ HttpDb#{<<"auth_props">> := AuthProps1}.
+
+
+-spec extract_creds_from_url(binary()) ->
+ {ok, {string() | undefined, string() | undefined}, string()} |
+ {error, term()}.
+extract_creds_from_url(Url0) ->
+ Url = binary_to_list(Url0),
+ case ibrowse_lib:parse_url(Url) of
+ {error, Error} ->
+ {error, Error};
+ #url{username = undefined, password = undefined} ->
+ {ok, {undefined, undefined}, Url0};
+ #url{protocol = Proto, username = User, password = Pass} ->
+ % Excise user and pass parts from the url. Try to keep the host,
+ % port and path as they were in the original.
+ Prefix = lists:concat([Proto, "://", User, ":", Pass, "@"]),
+ Suffix = lists:sublist(Url, length(Prefix) + 1, length(Url) + 1),
+ NoCreds = lists:concat([Proto, "://", Suffix]),
+ {ok, {User, Pass}, list_to_binary(NoCreds)}
+ end.
+
+
+% Normalize basic auth credentials so they are set only in the auth props
+% object. If multiple basic auth credentials are provided, the resulting
+% credentials are picked in the following order.
+% 1) {"auth": "basic": {"username":.., "password": ...} ...}
+% 2) URL userinfo part
+% 3) "Authentication" : "basic $base64" headers
+%
+-spec normalize_basic_auth(map()) -> map().
+normalize_basic_auth(#{} = HttpDb) ->
+ #{
+ <<"url">> := Url,
+ <<"headers">> := Headers
+ } = HttpDb,
+ {HeaderCreds, HeadersNoCreds} = remove_basic_auth_from_headers(Headers),
+ {UrlCreds, UrlWithoutCreds} = case extract_creds_from_url(Url) of
+ {ok, Creds = {_, _}, UrlNoCreds} ->
+ {Creds, UrlNoCreds};
+ {error, _Error} ->
+ % Don't crash replicator if user provided an invalid
+ % userinfo part
+ {undefined, undefined}
+ end,
+ AuthCreds = {_, _} = couch_replicator_utils:get_basic_auth_creds(HttpDb),
+ HttpDb1 = HttpDb#{
+ <<"url">> := UrlWithoutCreds,
+ <<"headers">> := HeadersNoCreds
+ },
+ {User, Pass} = case {AuthCreds, UrlCreds, HeaderCreds} of
+ {{U, P}, {_, _}, {_, _}} when is_list(U), is_list(P) -> {U, P};
+ {{_, _}, {U, P}, {_, _}} when is_list(U), is_list(P) -> {U, P};
+ {{_, _}, {_, _}, {U, P}} -> {U, P}
+ end,
+ set_basic_auth_creds(User, Pass, HttpDb1).
+
+
+remove_basic_auth_from_headers(#{} = HeadersMap) ->
+ % Headers are passed in a map however mochiweb_headers expects them to be
+ % lists so we transform them to lists first, then back to maps
+ Headers = maps:fold(fun(K, V, Acc) ->
+ [{binary_to_list(K), binary_to_list(V)} | Acc]
+ end, [], HeadersMap),
+ Headers1 = mochiweb_headers:make(Headers),
+ case mochiweb_headers:get_value("Authorization", Headers1) of
+ undefined ->
+ {{undefined, undefined}, HeadersMap};
+ Auth ->
+ {Basic, B64} = lists:splitwith(fun(X) -> X =/= $\s end, Auth),
+ BasicLower = string:to_lower(Basic),
+ Result = maybe_remove_basic_auth(BasicLower, B64, Headers1),
+ {{User, Pass}, Headers2} = Result,
+ HeadersMapResult = lists:foldl(fun({K, V}, Acc) ->
+ Acc#{list_to_binary(K) => list_to_binary(V)}
+ end, #{}, Headers2),
+ {{User, Pass}, HeadersMapResult}
+ end.
+
+
+maybe_remove_basic_auth("basic", " " ++ Base64, Headers) ->
+ Headers1 = mochiweb_headers:delete_any("Authorization", Headers),
+ {decode_basic_creds(Base64), mochiweb_headers:to_list(Headers1)};
+maybe_remove_basic_auth(_, _, Headers) ->
+ {{undefined, undefined}, mochiweb_headers:to_list(Headers)}.
+
+
+decode_basic_creds(Base64) ->
+ try re:split(base64:decode(Base64), ":", [{return, list}, {parts, 2}]) of
+ [User, Pass] ->
+ {User, Pass};
+ _ ->
+ {undefined, undefined}
+ catch
+ % Tolerate invalid B64 values here to avoid crashing replicator
+ error:function_clause ->
+ {undefined, undefined}
+ end.
+
+
-ifdef(TEST).
-include_lib("couch/include/couch_eunit.hrl").
@@ -565,4 +677,243 @@ t_error_on_local_endpoint(_) ->
?assertThrow({bad_rep_doc, Expect}, parse_rep_doc(RepDoc)).
+remove_basic_auth_from_headers_test_() ->
+ B64 = list_to_binary(b64creds("user", "pass")),
+ [?_assertEqual({{User, Pass}, NoAuthHeaders},
+ remove_basic_auth_from_headers(Headers)) ||
+ {{User, Pass, NoAuthHeaders}, Headers} <- [
+ {
+ {undefined, undefined, #{}},
+ #{}
+ },
+ {
+ {undefined, undefined, #{<<"h">> => <<"v">>}},
+ #{<<"h">> => <<"v">>}
+ },
+ {
+ {undefined, undefined, #{<<"Authorization">> => <<"junk">>}},
+ #{<<"Authorization">> => <<"junk">>}
+ },
+ {
+ {undefined, undefined, #{}},
+ #{<<"Authorization">> => <<"basic X">>}
+ },
+ {
+ {"user", "pass", #{}},
+ #{<<"Authorization">> => <<"Basic ", B64/binary>>}
+ },
+ {
+ {"user", "pass", #{}},
+ #{<<"AuThorization">> => <<"Basic ", B64/binary>>}
+ },
+ {
+ {"user", "pass", #{}},
+ #{<<"Authorization">> => <<"bAsIc ", B64/binary>>}
+ },
+ {
+ {"user", "pass", #{<<"h">> => <<"v">>}},
+ #{
+ <<"Authorization">> => <<"Basic ", B64/binary>>,
+ <<"h">> => <<"v">>
+ }
+ }
+ ]
+ ].
+
+
+b64creds(User, Pass) ->
+ base64:encode_to_string(User ++ ":" ++ Pass).
+
+
+set_basic_auth_creds_test() ->
+ Check = fun(User, Pass, Props) ->
+ HttpDb = #{<<"auth_props">> => Props},
+ HttpDb1 = set_basic_auth_creds(User, Pass, HttpDb),
+ maps:get(<<"auth_props">>, HttpDb1)
+ end,
+
+ ?assertEqual(#{}, Check(undefined, undefined, #{})),
+
+ ?assertEqual(#{<<"other">> => #{}}, Check(undefined, undefined,
+ #{<<"other">> => #{}})),
+
+ ?assertEqual(#{
+ <<"basic">> => #{
+ <<"username">> => <<"u">>,
+ <<"password">> => <<"p">>
+ }
+ }, Check("u", "p", #{})),
+
+ ?assertEqual(#{
+ <<"other">> => #{},
+ <<"basic">> => #{
+ <<"username">> => <<"u">>,
+ <<"password">> => <<"p">>
+ }
+ }, Check("u", "p", #{<<"other">> => #{}})).
+
+
+normalize_basic_creds_test_() ->
+ DefaultHeaders = couch_replicator_utils:default_headers_map(),
+ [?_assertEqual(Expect, normalize_basic_auth(Input)) || {Input, Expect} <- [
+ {
+ #{
+ <<"url">> => <<"http://u:[email protected]/db";>>,
+ <<"auth_props">> => #{},
+ <<"headers">> => DefaultHeaders
+ },
+ #{
+ <<"url">> => <<"http://x.y/db";>>,
+ <<"auth_props">> => auth_props("u", "p"),
+ <<"headers">> => DefaultHeaders
+ }
+ },
+ {
+ #{
+ <<"url">> => <<"http://u:p@h:80/db";>>,
+ <<"auth_props">> => #{},
+ <<"headers">> => DefaultHeaders
+ },
+ #{
+ <<"url">> => <<"http://h:80/db";>>,
+ <<"auth_props">> => auth_props("u", "p"),
+ <<"headers">> => DefaultHeaders
+ }
+ },
+ {
+ #{
+ <<"url">> => <<"https://u:p@h/db";>>,
+ <<"auth_props">> => #{},
+ <<"headers">> => DefaultHeaders
+ },
+ #{
+ <<"url">> => <<"https://h/db";>>,
+ <<"auth_props">> => auth_props("u", "p"),
+ <<"headers">> => DefaultHeaders
+ }
+ },
+ {
+ #{
+ <<"url">> => <<"http://u:p@[2001:db8:a1b:12f9::1]/db";>>,
+ <<"auth_props">> => #{},
+ <<"headers">> => DefaultHeaders
+ },
+ #{
+ <<"url">> => <<"http://[2001:db8:a1b:12f9::1]/db";>>,
+ <<"auth_props">> => auth_props("u", "p"),
+ <<"headers">> => DefaultHeaders
+ }
+ },
+ {
+ #{
+ <<"url">> => <<"http://h/db";>>,
+ <<"auth_props">> => #{},
+ <<"headers">> => maps:merge(DefaultHeaders, #{
+ <<"authorization">> => basic_b64("u", "p")
+ })
+ },
+ #{
+ <<"url">> => <<"http://h/db";>>,
+ <<"auth_props">> => auth_props("u", "p"),
+ <<"headers">> => DefaultHeaders
+ }
+ },
+ {
+ #{
+ <<"url">> => <<"http://h/db";>>,
+ <<"auth_props">> => #{},
+ <<"headers">> => maps:merge(DefaultHeaders, #{
+ <<"authorization">> => basic_b64("u", "p@")
+ })
+ },
+ #{
+ <<"url">> => <<"http://h/db";>>,
+ <<"auth_props">> => auth_props("u", "p@"),
+ <<"headers">> => DefaultHeaders
+ }
+ },
+ {
+ #{
+ <<"url">> => <<"http://h/db";>>,
+ <<"auth_props">> => #{},
+ <<"headers">> => maps:merge(DefaultHeaders, #{
+ <<"authorization">> => basic_b64("u", "p@%40")
+ })
+ },
+ #{
+ <<"url">> => <<"http://h/db";>>,
+ <<"auth_props">> => auth_props("u", "p@%40"),
+ <<"headers">> => DefaultHeaders
+ }
+ },
+ {
+ #{
+ <<"url">> => <<"http://h/db";>>,
+ <<"auth_props">> => #{},
+ <<"headers">> => maps:merge(DefaultHeaders, #{
+ <<"aUthoriZation">> => basic_b64("U", "p")
+ })
+ },
+ #{
+ <<"url">> => <<"http://h/db";>>,
+ <<"auth_props">> => auth_props("U", "p"),
+ <<"headers">> => DefaultHeaders
+ }
+ },
+ {
+ #{
+ <<"url">> => <<"http://u1:p1@h/db";>>,
+ <<"auth_props">> => #{},
+ <<"headers">> => maps:merge(DefaultHeaders, #{
+ <<"Authorization">> => basic_b64("u2", "p2")
+ })
+ },
+ #{
+ <<"url">> => <<"http://h/db";>>,
+ <<"auth_props">> => auth_props("u1", "p1"),
+ <<"headers">> => DefaultHeaders
+ }
+ },
+ {
+ #{
+ <<"url">> => <<"http://u1:p1@h/db";>>,
+ <<"auth_props">> => auth_props("u2", "p2"),
+ <<"headers">> => DefaultHeaders
+ },
+ #{
+ <<"url">> => <<"http://h/db";>>,
+ <<"auth_props">> => auth_props("u2", "p2"),
+ <<"headers">> => DefaultHeaders
+ }
+ },
+ {
+ #{
+ <<"url">> => <<"http://u1:p1@h/db";>>,
+ <<"auth_props">> => auth_props("u2", "p2"),
+ <<"headers">> => maps:merge(DefaultHeaders, #{
+ <<"Authorization">> => basic_b64("u3", "p3")
+ })
+ },
+ #{
+ <<"url">> => <<"http://h/db";>>,
+ <<"auth_props">> => auth_props("u2", "p2"),
+ <<"headers">> => DefaultHeaders
+ }
+ }
+ ]].
+
+
+basic_b64(User, Pass) when is_list(User), is_list(Pass) ->
+ B64Creds = list_to_binary(b64creds(User, Pass)),
+ <<"basic ", B64Creds/binary>>.
+
+
+auth_props(User, Pass) when is_list(User), is_list(Pass) ->
+ #{
+ <<"basic">> => #{
+ <<"username">> => list_to_binary(User),
+ <<"password">> => list_to_binary(Pass)
+ }
+ }.
+
-endif.
diff --git a/src/couch_replicator/src/couch_replicator_utils.erl
b/src/couch_replicator/src/couch_replicator_utils.erl
index 523de5f..c60cf56 100644
--- a/src/couch_replicator/src/couch_replicator_utils.erl
+++ b/src/couch_replicator/src/couch_replicator_utils.erl
@@ -18,12 +18,12 @@
iso8601/1,
rfc1123_local/0,
rfc1123_local/1,
- remove_basic_auth_from_headers/1,
normalize_rep/1,
compare_reps/2,
default_headers_map/0,
parse_replication_states/1,
parse_int_param/5,
+ get_basic_auth_creds/1,
proplist_options/1
]).
@@ -71,37 +71,6 @@ rfc1123_local(Sec) ->
list_to_binary(httpd_util:rfc1123_date(Local)).
-remove_basic_auth_from_headers(Headers) ->
- Headers1 = mochiweb_headers:make(Headers),
- case mochiweb_headers:get_value("Authorization", Headers1) of
- undefined ->
- {{undefined, undefined}, Headers};
- Auth ->
- {Basic, Base64} = lists:splitwith(fun(X) -> X =/= $\s end, Auth),
- maybe_remove_basic_auth(string:to_lower(Basic), Base64, Headers1)
- end.
-
-
-maybe_remove_basic_auth("basic", " " ++ Base64, Headers) ->
- Headers1 = mochiweb_headers:delete_any("Authorization", Headers),
- {decode_basic_creds(Base64), mochiweb_headers:to_list(Headers1)};
-maybe_remove_basic_auth(_, _, Headers) ->
- {{undefined, undefined}, mochiweb_headers:to_list(Headers)}.
-
-
-decode_basic_creds(Base64) ->
- try re:split(base64:decode(Base64), ":", [{return, list}, {parts, 2}]) of
- [User, Pass] ->
- {User, Pass};
- _ ->
- {undefined, undefined}
- catch
- % Tolerate invalid B64 values here to avoid crashing replicator
- error:function_clause ->
- {undefined, undefined}
- end.
-
-
-spec compare_reps(#{} | null, #{} | null) -> boolean().
compare_reps(Rep1, Rep2) ->
NormRep1 = normalize_rep(Rep1),
@@ -199,56 +168,30 @@ unix_sec_to_timestamp(Sec) when is_integer(Sec) ->
{MegaSecPart, SecPart, 0}.
--ifdef(TEST).
+-spec get_basic_auth_creds(#httpdb{} | map()) ->
+ {string(), string()} | {undefined, undefined}.
+get_basic_auth_creds(#httpdb{auth_props = AuthProps}) ->
+ get_basic_auth_creds(#{<<"auth_props">> => AuthProps});
+
+get_basic_auth_creds(#{<<"auth_props">> := Props}) ->
+ case Props of
+ #{<<"basic">> := Basic} ->
+ User = maps:get(<<"username">>, Basic, undefined),
+ Pass = maps:get(<<"password">>, Basic, undefined),
+ case {User, Pass} of
+ _ when is_binary(User), is_binary(Pass) ->
+ {binary_to_list(User), binary_to_list(Pass)};
+ _Other ->
+ {undefined, undefined}
+ end;
+ _Other ->
+ {undefined, undefined}
+ end.
--include_lib("eunit/include/eunit.hrl").
-remove_basic_auth_from_headers_test_() ->
- [?_assertMatch({{User, Pass}, NoAuthHeaders},
- remove_basic_auth_from_headers(Headers)) ||
- {{User, Pass, NoAuthHeaders}, Headers} <- [
- {
- {undefined, undefined, []},
- []
- },
- {
- {undefined, undefined, [{"h", "v"}]},
- [{"h", "v"}]
- },
- {
- {undefined, undefined, [{"Authorization", "junk"}]},
- [{"Authorization", "junk"}]
- },
- {
- {undefined, undefined, []},
- [{"Authorization", "basic X"}]
- },
- {
- {"user", "pass", []},
- [{"Authorization", "Basic " ++ b64creds("user", "pass")}]
- },
- {
- {"user", "pass", []},
- [{"AuThorization", "Basic " ++ b64creds("user", "pass")}]
- },
- {
- {"user", "pass", []},
- [{"Authorization", "bAsIc " ++ b64creds("user", "pass")}]
- },
- {
- {"user", "pass", [{"h", "v"}]},
- [
- {"Authorization", "Basic " ++ b64creds("user", "pass")},
- {"h", "v"}
- ]
- }
- ]
- ].
-
-
-b64creds(User, Pass) ->
- base64:encode_to_string(User ++ ":" ++ Pass).
+-ifdef(TEST).
+-include_lib("eunit/include/eunit.hrl").
normalize_rep_test_() ->
{
@@ -299,4 +242,24 @@ normalize_endpoint() ->
?assertEqual(<<"local">>, normalize_endpoint(<<"local">>)).
+get_basic_auth_creds_from_httpdb_test() ->
+ Check = fun(Props) ->
+ get_basic_auth_creds(#{<<"auth_props">> => Props})
+ end,
+
+ ?assertEqual({undefined, undefined}, Check(#{})),
+
+ ?assertEqual({undefined, undefined}, Check(#{a => b})),
+
+ ?assertEqual({undefined, undefined}, Check(#{<<"other">> => <<"x">>})),
+
+ ?assertEqual({undefined, undefined}, Check(#{<<"basic">> => #{}})),
+
+ UserPass1 = #{<<"username">> => <<"u">>, <<"password">> => <<"p">>},
+ ?assertEqual({"u", "p"}, Check(#{<<"basic">> => UserPass1})),
+
+ UserPass2 = #{<<"username">> => <<"u">>, <<"password">> => null},
+ ?assertEqual({undefined, undefined}, Check(#{<<"basic">> => UserPass2})).
+
+
-endif.
