This is an automated email from the ASF dual-hosted git repository. rnewson pushed a commit to branch remove-content-md5-header in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 141eb3d19cca2552fff7dc453a65fefebc00b8ce Author: Robert Newson <[email protected]> AuthorDate: Thu May 4 10:39:52 2023 +0100 WIP remove Content-MD5 header support Part of a series of changes to expunge MD5 entirely. --- src/chttpd/src/chttpd_db.erl | 34 +----- src/couch/src/couch_att.erl | 10 +- src/couch/src/couch_db.erl | 23 +--- src/couch/src/couch_httpd_db.erl | 31 +----- src/couch/test/eunit/couchdb_attachments_tests.erl | 122 +-------------------- test/elixir/test/attachments_test.exs | 23 ---- 6 files changed, 7 insertions(+), 236 deletions(-) diff --git a/src/chttpd/src/chttpd_db.erl b/src/chttpd/src/chttpd_db.erl index f2bf447b7..e2de301b2 100644 --- a/src/chttpd/src/chttpd_db.erl +++ b/src/chttpd/src/chttpd_db.erl @@ -1728,18 +1728,7 @@ db_attachment_req(#httpd{method = 'GET', mochi_req = MochiReq} = Req, Db, DocId, {identity, Ranges} when is_list(Ranges) andalso length(Ranges) < 10 -> send_ranges_multipart(Req, Type, Len, Att, Ranges); _ -> - Headers1 = - Headers ++ - if - Enc =:= identity orelse ReqAcceptsAttEnc =:= true -> - [ - {"Content-MD5", - base64:encode(couch_att:fetch(md5, Att))} - ]; - true -> - [] - end, - {ok, Resp} = start_response_length(Req, 200, Headers1, Len), + {ok, Resp} = start_response_length(Req, 200, Headers, Len), AttFun(Att, fun(Seg, _) -> send(Resp, Seg) end, {ok, Resp}) end end @@ -1802,7 +1791,6 @@ db_attachment_req(#httpd{method = Method, user_ctx = Ctx} = Req, Db, DocId, File {type, MimeType}, {data, Data}, {att_len, ContentLen}, - {md5, get_md5_header(Req)}, {encoding, Encoding} ]) ] @@ -1944,26 +1932,6 @@ parse_ranges([{From, To} | Rest], Len, Acc) -> make_content_range(From, To, Len) -> ?l2b(io_lib:format("bytes ~B-~B/~B", [From, To, Len])). -get_md5_header(Req) -> - ContentMD5 = couch_httpd:header_value(Req, "Content-MD5"), - Length = couch_httpd:body_length(Req), - Trailer = couch_httpd:header_value(Req, "Trailer"), - case {ContentMD5, Length, Trailer} of - _ when is_list(ContentMD5) orelse is_binary(ContentMD5) -> - base64:decode(ContentMD5); - {_, chunked, undefined} -> - <<>>; - {_, chunked, _} -> - case re:run(Trailer, "\\bContent-MD5\\b", [caseless]) of - {match, _} -> - md5_in_footer; - _ -> - <<>> - end; - _ -> - <<>> - end. - parse_doc_query(Req) -> lists:foldl(fun parse_doc_query/2, #doc_query_args{}, chttpd:qs(Req)). diff --git a/src/couch/src/couch_att.erl b/src/couch/src/couch_att.erl index b3b2f23eb..5ca3927e7 100644 --- a/src/couch/src/couch_att.erl +++ b/src/couch/src/couch_att.erl @@ -567,14 +567,8 @@ flush_data(Db, Fun, Att) when is_function(Fun) -> % Called with Length == 0 on the last time. % WriterFun returns NewState. fun - ({0, Footers}, _Total) -> - F = mochiweb_headers:from_binary(Footers), - case mochiweb_headers:get_value("Content-MD5", F) of - undefined -> - ok; - Md5 -> - {md5, base64:decode(Md5)} - end; + ({0, _Footers}, _Total) -> + ok; ({Length, Chunk}, Total0) -> Total = Total0 + Length, validate_attachment_size(AttName, Total, MaxAttSize), diff --git a/src/couch/src/couch_db.erl b/src/couch/src/couch_db.erl index d89d3e3f3..66e9a6db0 100644 --- a/src/couch/src/couch_db.erl +++ b/src/couch/src/couch_db.erl @@ -1599,16 +1599,6 @@ compressible_att_type(MimeType) -> [T || T <- TypeExpList, T /= []] ). -% From RFC 2616 3.6.1 - Chunked Transfer Coding -% -% In other words, the origin server is willing to accept -% the possibility that the trailer fields might be silently -% discarded along the path to the client. -% -% I take this to mean that if "Trailers: Content-MD5\r\n" -% is present in the request, but there is no Content-MD5 -% trailer, we're free to ignore this inconsistency and -% pretend that no Content-MD5 exists. with_stream(Db, Att, Fun) -> [InMd5, Type, Enc] = couch_att:fetch([md5, type, encoding], Att), BufferSize = config:get_integer( @@ -1631,19 +1621,10 @@ with_stream(Db, Att, Fun) -> [{buffer_size, BufferSize}] end, {ok, OutputStream} = open_write_stream(Db, Options), - ReqMd5 = - case Fun(OutputStream) of - {md5, FooterMd5} -> - case InMd5 of - md5_in_footer -> FooterMd5; - _ -> InMd5 - end; - _ -> - InMd5 - end, + Fun(OutputStream), {StreamEngine, Len, IdentityLen, Md5, IdentityMd5} = couch_stream:close(OutputStream), - couch_util:check_md5(IdentityMd5, ReqMd5), + couch_util:check_md5(IdentityMd5, InMd5), {AttLen, DiskLen, NewEnc} = case Enc of identity -> diff --git a/src/couch/src/couch_httpd_db.erl b/src/couch/src/couch_httpd_db.erl index f0786536d..e045510cc 100644 --- a/src/couch/src/couch_httpd_db.erl +++ b/src/couch/src/couch_httpd_db.erl @@ -1073,15 +1073,7 @@ db_attachment_req(#httpd{method = 'GET', mochi_req = MochiReq} = Req, Db, DocId, -> send_ranges_multipart(Req, Type, Len, Att, Ranges); _ -> - Headers1 = - Headers ++ - if - Enc =:= identity orelse ReqAcceptsAttEnc =:= true -> - [{"Content-MD5", base64:encode(Md5)}]; - true -> - [] - end, - {ok, Resp} = start_response_length(Req, 200, Headers1, Len), + {ok, Resp} = start_response_length(Req, 200, Headers, Len), AttFun(Att, fun(Seg, _) -> send(Resp, Seg) end, {ok, Resp}) end end @@ -1174,7 +1166,6 @@ db_attachment_req( {type, MimeType}, {data, Data}, {att_len, AttLen}, - {md5, get_md5_header(Req)}, {encoding, Encoding} ]) ] @@ -1250,26 +1241,6 @@ parse_ranges([{From, none} | Rest], Len, Acc) -> parse_ranges([{From, To} | Rest], Len, Acc) -> parse_ranges(Rest, Len, [{From, To}] ++ Acc). -get_md5_header(Req) -> - ContentMD5 = couch_httpd:header_value(Req, "Content-MD5"), - Length = couch_httpd:body_length(Req), - Trailer = couch_httpd:header_value(Req, "Trailer"), - case {ContentMD5, Length, Trailer} of - _ when is_list(ContentMD5) orelse is_binary(ContentMD5) -> - base64:decode(ContentMD5); - {_, chunked, undefined} -> - <<>>; - {_, chunked, _} -> - case re:run(Trailer, "\\bContent-MD5\\b", [caseless]) of - {match, _} -> - md5_in_footer; - _ -> - <<>> - end; - _ -> - <<>> - end. - parse_doc_query(Req) -> lists:foldl( fun({Key, Value}, Args) -> diff --git a/src/couch/test/eunit/couchdb_attachments_tests.erl b/src/couch/test/eunit/couchdb_attachments_tests.erl index 376553985..ef42b9f15 100644 --- a/src/couch/test/eunit/couchdb_attachments_tests.erl +++ b/src/couch/test/eunit/couchdb_attachments_tests.erl @@ -102,13 +102,7 @@ attachments_md5_tests() -> fun teardown/1, [ fun should_upload_attachment_without_md5/1, - fun should_upload_attachment_by_chunks_without_md5/1, - fun should_upload_attachment_with_valid_md5_header/1, - fun should_upload_attachment_by_chunks_with_valid_md5_header/1, - fun should_upload_attachment_by_chunks_with_valid_md5_trailer/1, - fun should_reject_attachment_with_invalid_md5/1, - fun should_reject_chunked_attachment_with_invalid_md5/1, - fun should_reject_chunked_attachment_with_invalid_md5_trailer/1 + fun should_upload_attachment_by_chunks_without_md5/1 ] } }. @@ -207,120 +201,6 @@ should_upload_attachment_by_chunks_without_md5({Host, DbName}) -> ?assertEqual(true, get_json(Json, [<<"ok">>])) end). -should_upload_attachment_with_valid_md5_header({Host, DbName}) -> - ?_test(begin - AttUrl = string:join(["", DbName, ?docid(), "readme.txt"], "/"), - Body = "We all live in a yellow submarine!", - Headers = [ - {"Content-Length", "34"}, - {"Content-Type", "text/plain"}, - {"Content-MD5", ?b2l(base64:encode(couch_hash:md5_hash(Body)))}, - {"Host", Host} - ], - {ok, Code, Json} = request("PUT", AttUrl, Headers, Body), - ?assertEqual(201, Code), - ?assertEqual(true, get_json(Json, [<<"ok">>])) - end). - -should_upload_attachment_by_chunks_with_valid_md5_header({Host, DbName}) -> - ?_test(begin - AttUrl = string:join(["", DbName, ?docid(), "readme.txt"], "/"), - AttData = <<"We all live in a yellow submarine!">>, - <<Part1:21/binary, Part2:13/binary>> = AttData, - Body = [chunked_body([Part1, Part2]), "\r\n"], - Headers = [ - {"Content-Type", "text/plain"}, - {"Content-MD5", ?b2l(base64:encode(couch_hash:md5_hash(AttData)))}, - {"Host", Host}, - {"Transfer-Encoding", "chunked"} - ], - {ok, Code, Json} = request("PUT", AttUrl, Headers, Body), - ?assertEqual(201, Code), - ?assertEqual(true, get_json(Json, [<<"ok">>])) - end). - -should_upload_attachment_by_chunks_with_valid_md5_trailer({Host, DbName}) -> - ?_test(begin - AttUrl = string:join(["", DbName, ?docid(), "readme.txt"], "/"), - AttData = <<"We all live in a yellow submarine!">>, - <<Part1:21/binary, Part2:13/binary>> = AttData, - Body = [ - chunked_body([Part1, Part2]), - "Content-MD5: ", - base64:encode(couch_hash:md5_hash(AttData)), - "\r\n\r\n" - ], - Headers = [ - {"Content-Type", "text/plain"}, - {"Host", Host}, - {"Trailer", "Content-MD5"}, - {"Transfer-Encoding", "chunked"} - ], - {ok, Code, Json} = request("PUT", AttUrl, Headers, Body), - ?assertEqual(201, Code), - ?assertEqual(true, get_json(Json, [<<"ok">>])) - end). - -should_reject_attachment_with_invalid_md5({Host, DbName}) -> - ?_test(begin - AttUrl = string:join(["", DbName, ?docid(), "readme.txt"], "/"), - Body = "We all live in a yellow submarine!", - Headers = [ - {"Content-Length", "34"}, - {"Content-Type", "text/plain"}, - {"Content-MD5", ?b2l(base64:encode(<<"foobar!">>))}, - {"Host", Host} - ], - {ok, Code, Json} = request("PUT", AttUrl, Headers, Body), - ?assertEqual(400, Code), - ?assertEqual( - <<"content_md5_mismatch">>, - get_json(Json, [<<"error">>]) - ) - end). - -should_reject_chunked_attachment_with_invalid_md5({Host, DbName}) -> - ?_test(begin - AttUrl = string:join(["", DbName, ?docid(), "readme.txt"], "/"), - AttData = <<"We all live in a yellow submarine!">>, - <<Part1:21/binary, Part2:13/binary>> = AttData, - Body = [chunked_body([Part1, Part2]), "\r\n"], - Headers = [ - {"Content-Type", "text/plain"}, - {"Content-MD5", ?b2l(base64:encode(<<"foobar!">>))}, - {"Host", Host}, - {"Transfer-Encoding", "chunked"} - ], - {ok, Code, Json} = request("PUT", AttUrl, Headers, Body), - ?assertEqual(400, Code), - ?assertEqual( - <<"content_md5_mismatch">>, - get_json(Json, [<<"error">>]) - ) - end). - -should_reject_chunked_attachment_with_invalid_md5_trailer({Host, DbName}) -> - ?_test(begin - AttUrl = string:join(["", DbName, ?docid(), "readme.txt"], "/"), - AttData = <<"We all live in a yellow submarine!">>, - <<Part1:21/binary, Part2:13/binary>> = AttData, - Body = [ - chunked_body([Part1, Part2]), - "Content-MD5: ", - base64:encode(<<"foobar!">>), - "\r\n\r\n" - ], - Headers = [ - {"Content-Type", "text/plain"}, - {"Host", Host}, - {"Trailer", "Content-MD5"}, - {"Transfer-Encoding", "chunked"} - ], - {ok, Code, Json} = request("PUT", AttUrl, Headers, Body), - ?assertEqual(400, Code), - ?assertEqual(<<"content_md5_mismatch">>, get_json(Json, [<<"error">>])) - end). - should_get_att_without_accept_gzip_encoding(_, {Data, {_, _, AttUrl}}) -> ?_test(begin {ok, Code, Headers, Body} = test_request:get(AttUrl), diff --git a/test/elixir/test/attachments_test.exs b/test/elixir/test/attachments_test.exs index f1dd3ef61..13a5c84f9 100644 --- a/test/elixir/test/attachments_test.exs +++ b/test/elixir/test/attachments_test.exs @@ -430,29 +430,6 @@ defmodule AttachmentsTest do assert resp.body["error"] == "missing_stub" end - @tag :with_db - test "md5 header for attachments", context do - db_name = context[:db_name] - md5 = "MntvB0NYESObxH4VRDUycw==" - - bin_data = "foo bar" - - resp = - Couch.put( - "/#{db_name}/bin_doc8/attachment.txt", - body: bin_data, - headers: ["Content-Type": "application/octet-stream", "Content-MD5": md5], - query: %{w: 3} - ) - - assert resp.status_code in [201, 202] - assert resp.body["ok"] - - resp = Couch.get("/#{db_name}/bin_doc8/attachment.txt") - assert resp.status_code == 200 - assert md5 == resp.headers["Content-MD5"] - end - @tag :with_db test "attachment via multipart/form-data", context do db_name = context[:db_name]
