Repository: couchdb-chttpd Updated Branches: refs/heads/2992-limit-doc-size f7affd0d2 -> 1ac365dd8
Move max_document_size logic to update only COUCHDB-2992 Project: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/repo Commit: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/commit/03fdcde5 Tree: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/tree/03fdcde5 Diff: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/diff/03fdcde5 Branch: refs/heads/2992-limit-doc-size Commit: 03fdcde571585f17c2077a3a0313bc8703e32197 Parents: f7affd0 Author: Tony Sun <tony....@cloudant.com> Authored: Tue Apr 19 18:41:32 2016 -0700 Committer: Tony Sun <tony....@cloudant.com> Committed: Tue Apr 19 18:41:32 2016 -0700 ---------------------------------------------------------------------- include/chttpd.hrl | 3 ++ src/chttpd.erl | 9 ++--- src/chttpd_db.erl | 65 ++++++++++++++++++++--------------- src/chttpd_external.erl | 4 +-- test/chttpd_db_doc_size_test.erl | 6 ++-- 5 files changed, 50 insertions(+), 37 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/blob/03fdcde5/include/chttpd.hrl ---------------------------------------------------------------------- diff --git a/include/chttpd.hrl b/include/chttpd.hrl index a7f9aaa..d26a2ee 100644 --- a/include/chttpd.hrl +++ b/include/chttpd.hrl @@ -26,3 +26,6 @@ (C >= $a andalso C =< $f) orelse (C >= $A andalso C =< $F) )). + +% For any PUT/POST request, max size will be 4 GB (semi-unlimited) +-define(DEFAULT_RECV_BODY, 4294967296). http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/blob/03fdcde5/src/chttpd.erl ---------------------------------------------------------------------- diff --git a/src/chttpd.erl b/src/chttpd.erl index 1f8c160..ef1c140 100644 --- a/src/chttpd.erl +++ b/src/chttpd.erl @@ -599,12 +599,9 @@ body_length(#httpd{mochi_req=MochiReq}) -> body(#httpd{mochi_req=MochiReq, req_body=ReqBody}) -> case ReqBody of undefined -> - % Maximum size of document PUT request body (4GB) - MaxSize = list_to_integer( - config:get("couchdb", "max_document_size", "4294967296")), Begin = os:timestamp(), try - MochiReq:recv_body(MaxSize) + MochiReq:recv_body(?DEFAULT_RECV_BODY) after T = timer:now_diff(os:timestamp(), Begin) div 1000, put(body_time, T) @@ -858,9 +855,9 @@ error_info({error, {database_name_too_long, DbName}}) -> <<"At least one path segment of `", DbName/binary, "` is too long.">>}; error_info({missing_stub, Reason}) -> {412, <<"missing_stub">>, Reason}; -error_info({single_doc_too_large, MaxSize}) -> +error_info({request_entity_too_large, MaxSize}) -> SizeBin = list_to_binary(integer_to_list(MaxSize)), - {413, <<"too_large">>, <<"Document exceeded single_max_doc_size:", + {413, <<"too_large">>, <<"Document exceeded max_document_size:", SizeBin/binary>>}; error_info(request_entity_too_large) -> {413, <<"too_large">>, <<"the request entity is too large">>}; http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/blob/03fdcde5/src/chttpd_db.erl ---------------------------------------------------------------------- diff --git a/src/chttpd_db.erl b/src/chttpd_db.erl index 2a3fe1b..fdfd03d 100644 --- a/src/chttpd_db.erl +++ b/src/chttpd_db.erl @@ -13,6 +13,7 @@ -module(chttpd_db). -include_lib("couch/include/couch_db.hrl"). -include_lib("couch_mrview/include/couch_mrview.hrl"). +-include_lib("chttpd/include/chttpd.hrl"). -export([handle_request/1, handle_compact_req/2, handle_design_req/2, db_req/2, couch_doc_open/4,handle_changes_req/2, @@ -309,7 +310,7 @@ db_req(#httpd{method='POST', path_parts=[DbName], user_ctx=Ctx}=Req, Db) -> Options = [{user_ctx,Ctx}, {w,W}], Body = chttpd:json_body(Req), - ok = verify_doc_size(Body), + ok = maybe_verify_body_size(Body), Doc = couch_doc:from_json_obj(Body), Doc2 = case Doc#doc.id of @@ -377,7 +378,7 @@ db_req(#httpd{method='POST',path_parts=[_,<<"_bulk_docs">>], user_ctx=Ctx}=Req, DocsArray0 -> DocsArray0 end, - {ExceedErrs, DocsArray1} = remove_exceed_docs(DocsArray), + {ExceedErrs, DocsArray1} = maybe_remove_exceed_docs(DocsArray), couch_stats:update_histogram([couchdb, httpd, bulk_docs], length(DocsArray)), W = case couch_util:get_value(<<"w">>, JsonProps) of Value when is_integer(Value) -> @@ -769,7 +770,7 @@ db_doc_req(#httpd{method='PUT', user_ctx=Ctx}=Req, Db, DocId) -> "ok" -> % batch Body = chttpd:json_body(Req), - ok = verify_doc_size(Body), + ok = maybe_verify_body_size(Body), Doc = couch_doc_from_req(Req, DocId, Body), spawn(fun() -> @@ -787,7 +788,7 @@ db_doc_req(#httpd{method='PUT', user_ctx=Ctx}=Req, Db, DocId) -> _Normal -> % normal Body = chttpd:json_body(Req), - ok = verify_doc_size(Body), + ok = maybe_verify_body_size(Body), Doc = couch_doc_from_req(Req, DocId, Body), send_updated_doc(Req, Db, DocId, Doc, RespHeaders, UpdateType) end @@ -1634,33 +1635,43 @@ bulk_get_open_doc_revs1(Db, Props, _, {DocId, Revs, Options}) -> {DocId, Results, Options} end. -remove_exceed_docs(DocArray0) -> - MaxSize = list_to_integer(config:get("couchdb", - "single_max_doc_size", "1048576")), - {ExceedDocs, DocsArray} = lists:splitwith(fun (Doc) -> - exceed_doc_size(Doc, MaxSize) - end, DocArray0), - ExceedErrs = lists:map (fun ({Doc}) -> - DocId = case couch_util:get_value(<<"_id">>, Doc) of - undefined -> couch_uuids:new(); - Id0 -> Id0 - end, - Reason = lists:concat(["Document exceeded single_max_doc_size:", - " of ", MaxSize, " bytes"]), - {[{id, DocId}, {error, <<"too_large">>}, - {reason, ?l2b(Reason)}]} - end, ExceedDocs), - {ExceedErrs, DocsArray}. +maybe_remove_exceed_docs(DocArray0) -> + case config:get_boolean("couchdb", "use_max_document_size", false) of + true -> + Max = config:get_integer("couchdb", "max_document_size", + 16777216), + {ExceedDocs, DocsArray} = lists:splitwith(fun (Doc) -> + exceed_doc_size(Doc, Max) + end, DocArray0), + ExceedErrs = lists:map (fun ({Doc}) -> + DocId = case couch_util:get_value(<<"_id">>, Doc) of + undefined -> <<"no id generated since update failed">>; + Id0 -> Id0 + end, + Reason = lists:concat(["Document exceeded max_document_size:", + " of ", Max, " bytes"]), + {[{id, DocId}, {error, <<"request_entity_too_large">>}, + {reason, ?l2b(Reason)}]} + end, ExceedDocs), + {ExceedErrs, DocsArray}; + false -> + {[], DocArray0} + end. exceed_doc_size(JsonBody, MaxSize) -> size(term_to_binary(JsonBody)) > MaxSize. -verify_doc_size(JsonBody) -> - SMaxSize = list_to_integer(config:get("couchdb", - "single_max_doc_size", "1048576")), - case exceed_doc_size(JsonBody, SMaxSize) of - true -> throw({single_doc_too_large, SMaxSize}); - false -> ok +maybe_verify_body_size(JsonBody) -> + case config:get_boolean("couchdb", "use_max_document_size", false) of + true -> + Max = config:get_integer("couchdb", "max_document_size", + 16777216), + case exceed_doc_size(JsonBody, Max) of + true -> throw({request_entity_too_large, Max}); + false -> ok + end; + false -> + ok end. parse_field(<<"id">>, undefined) -> http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/blob/03fdcde5/src/chttpd_external.erl ---------------------------------------------------------------------- diff --git a/src/chttpd_external.erl b/src/chttpd_external.erl index f9bebed..e8efff2 100644 --- a/src/chttpd_external.erl +++ b/src/chttpd_external.erl @@ -20,6 +20,7 @@ -import(chttpd,[send_error/4]). -include_lib("couch/include/couch_db.hrl"). +-include_lib("chttpd/include/chttpd.hrl"). % handle_external_req/2 % for the old type of config usage: @@ -93,8 +94,7 @@ json_req_obj_field(<<"headers">>, #httpd{mochi_req=Req}, _Db, _DocId) -> Hlist = mochiweb_headers:to_list(Headers), to_json_terms(Hlist); json_req_obj_field(<<"body">>, #httpd{req_body=undefined, mochi_req=Req}, _Db, _DocId) -> - MaxSize = config:get_integer("couchdb", "max_document_size", 4294967296), - Req:recv_body(MaxSize); + Req:recv_body(?DEFAULT_RECV_BODY); json_req_obj_field(<<"body">>, #httpd{req_body=Body}, _Db, _DocId) -> Body; json_req_obj_field(<<"peer">>, #httpd{mochi_req=Req}, _Db, _DocId) -> http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/blob/03fdcde5/test/chttpd_db_doc_size_test.erl ---------------------------------------------------------------------- diff --git a/test/chttpd_db_doc_size_test.erl b/test/chttpd_db_doc_size_test.erl index 02ae3cd..ce29c17 100644 --- a/test/chttpd_db_doc_size_test.erl +++ b/test/chttpd_db_doc_size_test.erl @@ -22,7 +22,8 @@ setup() -> ok = config:set("admins", ?USER, ?PASS, _Persist=false), - ok = config:set("couchdb", "single_max_doc_size", "50"), + ok = config:set("couchdb", "max_document_size", "50"), + ok = config:set("couchdb", "use_max_document_size", "true"), TmpDb = ?tempdb(), Addr = config:get("chttpd", "bind_address", "127.0.0.1"), Port = mochiweb_socket_server:get(chttpd, port), @@ -33,7 +34,8 @@ setup() -> teardown(Url) -> delete_db(Url), ok = config:delete("admins", ?USER, _Persist=false), - ok = config:delete("couchdb", "single_max_doc_size"). + ok = config:delete("couchdb", "max_document_size"), + ok = config:delete("couchdb", "use_max_document_size"). create_db(Url) -> {ok, Status, _, _} = test_request:put(Url, [?CONTENT_JSON, ?AUTH], "{}"),
