Hi, Currently the webdav module writes the request body to a temporary file and moves it to destination filepath once upload is complete.
The patch below introduces dav_request_buffering. Default is "on". If "off", the request body will be written directly to destination filepath (unless destination file already exists, in which case 409 is returned). Rationale: We work with a system where http clients can upload large files (>100 GB, think raw uncompressed HD footage). We want to use nginx + the dav module for this. However, other clients want to access the uploaded files as soon as possible (even if the upload is still running). Thus we implemented "dav_request_buffering off". To make sure a client can't overwrite arbitrary files we protect the location with the secure_link module, i.e. the system has previously handed out a secure upload url (with unique filename to avoid collisions) to the client. Implementation details: The name "dav_request_buffering" is supposed to be analogous with "proxy_request_buffering" which hopefully make sense even though the implementation is very different. We extend ngx_http_request_s with request_body_temp_name, which the dav module sets when dav_request_buffering is off, and then ngx_create_temp_file writes directly to this filepath. request_body_in_clean_file is not set in this case. In other words, we instruct nginx core to write the "temporary" file to the final destination directly. Any chance seeing this patch/feature merged? Thanks, Tommy # HG changeset patch # User Tommy Lindgren <[email protected]> # Date 1496048257 -7200 # Mon May 29 10:57:37 2017 +0200 # Node ID 4f4a483ca56229b7690f5d305bf056027005c7f2 # Parent ed1101bbf19f1edf300665b6398cd66d45b71577 Dav: introduced dav_request_buffering directive By default the dav module writes the request body to a temporary file and moves it to destination filepath when upload is complete. If dav_request_buffering is "off", the request body will be written directly to destination filepath (unless destination file already exists, in which case 409 is returned). diff --git a/src/core/ngx_file.c b/src/core/ngx_file.c --- a/src/core/ngx_file.c +++ b/src/core/ngx_file.c @@ -150,6 +150,22 @@ ngx_create_temp_file(ngx_file_t *file, n ngx_pool_cleanup_t *cln; ngx_pool_cleanup_file_t *clnf; + if (file->name.data != NULL) { + file->fd = ngx_open_tempfile(file->name.data, 1, access); + + ngx_log_debug1(NGX_LOG_DEBUG_CORE, file->log, 0, + "forced temp fd:%d", file->fd); + + if (file->fd == NGX_INVALID_FILE) { + ngx_log_error(NGX_LOG_ERR, file->log, ngx_errno, + ngx_open_tempfile_n " \"%s\" failed", + file->name.data); + return NGX_ERROR; + } + + return NGX_OK; + } + if (file->name.len) { name = file->name; levels = 0; diff --git a/src/http/modules/ngx_http_dav_module.c b/src/http/modules/ngx_http_dav_module.c --- a/src/http/modules/ngx_http_dav_module.c +++ b/src/http/modules/ngx_http_dav_module.c @@ -23,6 +23,7 @@ typedef struct { ngx_uint_t access; ngx_uint_t min_delete_depth; ngx_flag_t create_full_put_path; + ngx_flag_t dav_request_buffering; } ngx_http_dav_loc_conf_t; @@ -104,6 +105,13 @@ static ngx_command_t ngx_http_dav_comma offsetof(ngx_http_dav_loc_conf_t, access), NULL }, + { ngx_string("dav_request_buffering"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG, + ngx_conf_set_flag_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_dav_loc_conf_t, dav_request_buffering), + NULL }, + ngx_null_command }; @@ -142,8 +150,12 @@ ngx_module_t ngx_http_dav_module = { static ngx_int_t ngx_http_dav_handler(ngx_http_request_t *r) { + size_t root; + ngx_str_t path; ngx_int_t rc; ngx_http_dav_loc_conf_t *dlcf; + ngx_file_info_t fi; + ngx_err_t err; dlcf = ngx_http_get_module_loc_conf(r, ngx_http_dav_module); @@ -169,10 +181,39 @@ ngx_http_dav_handler(ngx_http_request_t r->request_body_in_file_only = 1; r->request_body_in_persistent_file = 1; - r->request_body_in_clean_file = 1; r->request_body_file_group_access = 1; r->request_body_file_log_level = 0; + if (dlcf->dav_request_buffering) { + r->request_body_in_clean_file = 1; + } else { + if (ngx_http_map_uri_to_path(r, &path, &root, 0) == NULL) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + + if (ngx_file_info(path.data, &fi) != NGX_FILE_ERROR) { + return NGX_HTTP_CONFLICT; + } + + if (dlcf->create_full_put_path) { + err = ngx_create_full_path(path.data, ngx_dir_access(dlcf->access)); + if (err) { + ngx_log_error(NGX_LOG_CRIT, r->connection->log, err, + ngx_create_dir_n " \"%s\" failed", path.data); + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + } + + r->request_body_temp_name.len = path.len; + r->request_body_temp_name.data = ngx_pnalloc(r->pool, path.len + 1); + + if (r->request_body_temp_name.data == NULL) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + + ngx_memcpy(r->request_body_temp_name.data, path.data, path.len); + } + rc = ngx_http_read_client_request_body(r, ngx_http_dav_put_handler); if (rc >= NGX_HTTP_SPECIAL_RESPONSE) { @@ -213,6 +254,16 @@ ngx_http_dav_put_handler(ngx_http_reques ngx_ext_rename_file_t ext; ngx_http_dav_loc_conf_t *dlcf; + dlcf = ngx_http_get_module_loc_conf(r, ngx_http_dav_module); + + if (!dlcf->dav_request_buffering) { + r->headers_out.content_length_n = 0; + r->headers_out.status = NGX_HTTP_CREATED; + r->header_only = 1; + ngx_http_finalize_request(r, ngx_http_send_header(r)); + return; + } + if (r->request_body == NULL || r->request_body->temp_file == NULL) { ngx_http_finalize_request(r, NGX_HTTP_INTERNAL_SERVER_ERROR); return; @@ -251,8 +302,6 @@ ngx_http_dav_put_handler(ngx_http_reques } } - dlcf = ngx_http_get_module_loc_conf(r, ngx_http_dav_module); - ext.access = dlcf->access; ext.path_access = dlcf->access; ext.time = -1; @@ -1115,6 +1164,7 @@ ngx_http_dav_create_loc_conf(ngx_conf_t conf->min_delete_depth = NGX_CONF_UNSET_UINT; conf->access = NGX_CONF_UNSET_UINT; conf->create_full_put_path = NGX_CONF_UNSET; + conf->dav_request_buffering = NGX_CONF_UNSET; return conf; } @@ -1137,6 +1187,9 @@ ngx_http_dav_merge_loc_conf(ngx_conf_t * ngx_conf_merge_value(conf->create_full_put_path, prev->create_full_put_path, 0); + ngx_conf_merge_value(conf->dav_request_buffering, + prev->dav_request_buffering, 1); + return NGX_CONF_OK; } diff --git a/src/http/ngx_http_request.h b/src/http/ngx_http_request.h --- a/src/http/ngx_http_request.h +++ b/src/http/ngx_http_request.h @@ -481,6 +481,7 @@ struct ngx_http_request_s { unsigned request_body_file_group_access:1; unsigned request_body_file_log_level:3; unsigned request_body_no_buffering:1; + ngx_str_t request_body_temp_name; unsigned subrequest_in_memory:1; unsigned waited:1; diff --git a/src/http/ngx_http_request_body.c b/src/http/ngx_http_request_body.c --- a/src/http/ngx_http_request_body.c +++ b/src/http/ngx_http_request_body.c @@ -450,6 +450,7 @@ ngx_http_write_request_body(ngx_http_req tf->file.fd = NGX_INVALID_FILE; tf->file.log = r->connection->log; tf->path = clcf->client_body_temp_path; + tf->file.name = r->request_body_temp_name; tf->pool = r->pool; tf->warn = "a client request body is buffered to a temporary file"; tf->log_level = r->request_body_file_log_level; _______________________________________________ nginx-devel mailing list [email protected] http://mailman.nginx.org/mailman/listinfo/nginx-devel
