# HG changeset patch # User hucongcong <hucon...@foxmail.com> # Date 1509038506 -28800 # Fri Oct 27 01:21:46 2017 +0800 # Node ID 798418b3f9c21811debf6cbeaea3eb554c886905 # Parent a429a3b21a3950d69781e109111831e7c5f33566 Range filter: support multiple ranges.
Nginx will return 416 when the ranges (first-byte-pos) are not listed in ascending order or when one byte-range-spec contains another. And besides, nginx will coalesce any of the ranges that overlap, or that are separated by a gap that is smaller than the NGX_HTTP_RANGE_MULTIPART_GAP macro. diff -r a429a3b21a39 -r 798418b3f9c2 src/http/modules/ngx_http_range_filter_module.c --- a/src/http/modules/ngx_http_range_filter_module.c Fri Oct 27 00:38:56 2017 +0800 +++ b/src/http/modules/ngx_http_range_filter_module.c Fri Oct 27 01:21:46 2017 +0800 @@ -10,6 +10,9 @@ #include <ngx_http.h> +#define NGX_HTTP_RANGE_MULTIPART_GAP 80 + + /* * the single part format: * @@ -55,6 +58,7 @@ typedef struct { typedef struct { off_t offset; ngx_str_t boundary_header; + ngx_uint_t index; /* start with 1 */ ngx_array_t ranges; } ngx_http_range_filter_ctx_t; @@ -66,12 +70,14 @@ static ngx_int_t ngx_http_range_singlepa static ngx_int_t ngx_http_range_multipart_header(ngx_http_request_t *r, ngx_http_range_filter_ctx_t *ctx); static ngx_int_t ngx_http_range_not_satisfiable(ngx_http_request_t *r); -static ngx_int_t ngx_http_range_test_overlapped(ngx_http_request_t *r, - ngx_http_range_filter_ctx_t *ctx, ngx_chain_t *in); static ngx_int_t ngx_http_range_singlepart_body(ngx_http_request_t *r, ngx_http_range_filter_ctx_t *ctx, ngx_chain_t *in); static ngx_int_t ngx_http_range_multipart_body(ngx_http_request_t *r, ngx_http_range_filter_ctx_t *ctx, ngx_chain_t *in); +static ngx_int_t ngx_http_range_link_boundary_header(ngx_http_request_t *r, + ngx_http_range_filter_ctx_t *ctx, ngx_chain_t ***lll); +static ngx_int_t ngx_http_range_link_last_boundary(ngx_http_request_t *r, + ngx_http_range_filter_ctx_t *ctx, ngx_chain_t **ll); static ngx_int_t ngx_http_range_header_filter_init(ngx_conf_t *cf); static ngx_int_t ngx_http_range_body_filter_init(ngx_conf_t *cf); @@ -270,9 +276,9 @@ ngx_http_range_parse(ngx_http_request_t ngx_uint_t ranges) { u_char *p; - off_t start, end, size, content_length, cutoff, - cutlim; - ngx_uint_t suffix; + off_t start, end, size, content_length, + suffix_length, cutoff, cutlim; + ngx_uint_t i, suffix; ngx_http_range_t *range; ngx_http_range_filter_ctx_t *mctx; @@ -280,6 +286,7 @@ ngx_http_range_parse(ngx_http_request_t mctx = ngx_http_get_module_ctx(r->main, ngx_http_range_body_filter_module); if (mctx) { + ctx->boundary_header = mctx->boundary_header; ctx->ranges = mctx->ranges; return NGX_OK; } @@ -293,6 +300,8 @@ ngx_http_range_parse(ngx_http_request_t p = r->headers_in.range->value.data + 6; size = 0; + range = NULL; + suffix_length = 0; content_length = r->headers_out.content_length_n; cutoff = NGX_MAX_OFF_T_VALUE / 10; @@ -374,6 +383,94 @@ ngx_http_range_parse(ngx_http_request_t return NGX_DECLINED; } + if (suffix) { + + if (end - start > suffix_length) { + suffix_length = end - start; + } + + } else { + + if (ctx->ranges.nelts) { + + if (start < range->start) { + ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0, + "descending order ranges %O-%O, %O-%O", + range->start, range->end - 1, + start, end - 1); + + return NGX_HTTP_RANGE_NOT_SATISFIABLE; + } + + if (end <= range->end) { + ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0, + "inclusive ranges %O-%O, %O-%O", + range->start, range->end - 1, + start, end - 1); + + return NGX_HTTP_RANGE_NOT_SATISFIABLE; + } + + if (start < range->end + NGX_HTTP_RANGE_MULTIPART_GAP) { + size -= range->end - range->start; + start = range->start; + ctx->ranges.nelts--; + } + } + + range = ngx_array_push(&ctx->ranges); + if (range == NULL) { + return NGX_ERROR; + } + + range->start = start; + range->end = end; + + if (size > NGX_MAX_OFF_T_VALUE - (end - start)) { + return NGX_HTTP_RANGE_NOT_SATISFIABLE; + } + + size += end - start; + } + + } else if (start == 0) { + return NGX_DECLINED; + } + + if (*p++ != ',') { + break; + } + } + + if (suffix_length) { + start = content_length - suffix_length; + end = content_length; + + i = 0; + range = ctx->ranges.elts; + + while (ctx->ranges.nelts) { + + i = ctx->ranges.nelts - 1; + if (start >= range[i].start) { + break; + } + + size -= range[i].end - range[i].start; + ctx->ranges.nelts--; + } + + if (ctx->ranges.nelts + && start < range[i].end + NGX_HTTP_RANGE_MULTIPART_GAP) + { + size -= range[i].end - range[i].start; + start = range[i].start; + ctx->ranges.nelts--; + i--; + } + + if (ctx->ranges.nelts == 0 || end > range[i].end) { + range = ngx_array_push(&ctx->ranges); if (range == NULL) { return NGX_ERROR; @@ -387,13 +484,6 @@ ngx_http_range_parse(ngx_http_request_t } size += end - start; - - } else if (start == 0) { - return NGX_DECLINED; - } - - if (*p++ != ',') { - break; } } @@ -470,6 +560,10 @@ ngx_http_range_multipart_header(ngx_http ngx_http_range_t *range; ngx_atomic_uint_t boundary; + if (r != r->main) { + return ngx_http_next_header_filter(r); + } + size = sizeof(CRLF "--") - 1 + NGX_ATOMIC_T_LEN + sizeof(CRLF "Content-Type: ") - 1 + r->headers_out.content_type.len @@ -571,10 +665,11 @@ ngx_http_range_multipart_header(ngx_http - range[i].content_range.data; len += ctx->boundary_header.len + range[i].content_range.len - + (range[i].end - range[i].start); + + (range[i].end - range[i].start); } r->headers_out.content_length_n = len; + r->headers_out.content_offset = range[0].start; if (r->headers_out.content_length) { r->headers_out.content_length->hash = 0; @@ -640,63 +735,15 @@ ngx_http_range_body_filter(ngx_http_requ return ngx_http_range_singlepart_body(r, ctx, in); } - /* - * multipart ranges are supported only if whole body is in a single buffer - */ - if (ngx_buf_special(in->buf)) { return ngx_http_next_body_filter(r, in); } - if (ngx_http_range_test_overlapped(r, ctx, in) != NGX_OK) { - return NGX_ERROR; - } - return ngx_http_range_multipart_body(r, ctx, in); } static ngx_int_t -ngx_http_range_test_overlapped(ngx_http_request_t *r, - ngx_http_range_filter_ctx_t *ctx, ngx_chain_t *in) -{ - off_t start, last; - ngx_buf_t *buf; - ngx_uint_t i; - ngx_http_range_t *range; - - if (ctx->offset) { - goto overlapped; - } - - buf = in->buf; - - if (!buf->last_buf) { - start = ctx->offset; - last = ctx->offset + ngx_buf_size(buf); - - range = ctx->ranges.elts; - for (i = 0; i < ctx->ranges.nelts; i++) { - if (start > range[i].start || last < range[i].end) { - goto overlapped; - } - } - } - - ctx->offset = ngx_buf_size(buf); - - return NGX_OK; - -overlapped: - - ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0, - "range in overlapped buffers"); - - return NGX_ERROR; -} - - -static ngx_int_t ngx_http_range_singlepart_body(ngx_http_request_t *r, ngx_http_range_filter_ctx_t *ctx, ngx_chain_t *in) { @@ -787,96 +834,225 @@ static ngx_int_t ngx_http_range_multipart_body(ngx_http_request_t *r, ngx_http_range_filter_ctx_t *ctx, ngx_chain_t *in) { - ngx_buf_t *b, *buf; - ngx_uint_t i; - ngx_chain_t *out, *hcl, *rcl, *dcl, **ll; - ngx_http_range_t *range; + off_t start, last, back; + ngx_buf_t *buf, *b; + ngx_uint_t i, finished; + ngx_chain_t *out, *cl, *ncl, **ll; + ngx_http_range_t *range, *tail; - ll = &out; - buf = in->buf; range = ctx->ranges.elts; - for (i = 0; i < ctx->ranges.nelts; i++) { + if (!ctx->index) { + for (i = 0; i < ctx->ranges.nelts; i++) { + if (ctx->offset < range[i].end) { + ctx->index = i + 1; + break; + } + } + } + + tail = range + ctx->ranges.nelts - 1; + range += ctx->index - 1; + + out = NULL; + ll = &out; + finished = 0; - /* - * The boundary header of the range: - * CRLF - * "--0123456789" CRLF - * "Content-Type: image/jpeg" CRLF - * "Content-Range: bytes " - */ + for (cl = in; cl; cl = cl->next) { + + buf = cl->buf; + + start = ctx->offset; + last = ctx->offset + ngx_buf_size(buf); - b = ngx_calloc_buf(r->pool); - if (b == NULL) { - return NGX_ERROR; + ctx->offset = last; + + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http range multipart body buf: %O-%O", start, last); + + if (ngx_buf_special(buf)) { + *ll = cl; + ll = &cl->next; + continue; } - b->memory = 1; - b->pos = ctx->boundary_header.data; - b->last = ctx->boundary_header.data + ctx->boundary_header.len; + if (range->end <= start || range->start >= last) { + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http range multipart body skip"); - hcl = ngx_alloc_chain_link(r->pool); - if (hcl == NULL) { - return NGX_ERROR; + if (buf->in_file) { + buf->file_pos = buf->file_last; + } + + buf->pos = buf->last; + buf->sync = 1; + + continue; } - hcl->buf = b; + if (range->start >= start) { + if (ngx_http_range_link_boundary_header(r, ctx, &ll) != NGX_OK) { + return NGX_ERROR; + } - /* "SSSS-EEEE/TTTT" CRLF CRLF */ + if (buf->in_file) { + buf->file_pos += range->start - start; + } - b = ngx_calloc_buf(r->pool); - if (b == NULL) { - return NGX_ERROR; + if (ngx_buf_in_memory(buf)) { + buf->pos += (size_t) (range->start - start); + } } - b->temporary = 1; - b->pos = range[i].content_range.data; - b->last = range[i].content_range.data + range[i].content_range.len; + if (range->end <= last) { + + if (range < tail && range[1].start < last) { + + b = ngx_alloc_buf(r->pool); + if (b == NULL) { + return NGX_ERROR; + } + + ncl = ngx_alloc_chain_link(r->pool); + if (ncl == NULL) { + return NGX_ERROR; + } - rcl = ngx_alloc_chain_link(r->pool); - if (rcl == NULL) { - return NGX_ERROR; - } + ncl->buf = b; + ncl->next = cl->next; + cl->next = ncl; + + ngx_memcpy(b, buf, sizeof(ngx_buf_t)); + buf->last_in_chain = 0; + buf->last_buf = 0; + + back = last - range->end; + ctx->offset -= back; + + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http range multipart body reuse buf: %O-%O", + ctx->offset, ctx->offset + back); - rcl->buf = b; + if (b->in_file) { + b->file_pos = b->file_last - back; + } + if (ngx_buf_in_memory(b)) { + b->pos = b->last - back; + } + } + + if (buf->in_file) { + buf->file_last -= last - range->end; + } - /* the range data */ + if (ngx_buf_in_memory(buf)) { + buf->last -= (size_t) (last - range->end); + } - b = ngx_calloc_buf(r->pool); - if (b == NULL) { - return NGX_ERROR; + if (range == tail) { + buf->last_buf = (r == r->main) ? 1 : 0; + buf->last_in_chain = 1; + *ll = cl; + ll = &cl->next; + + finished = 1; + break; + } + + range++; + ctx->index++; } - b->in_file = buf->in_file; - b->temporary = buf->temporary; - b->memory = buf->memory; - b->mmap = buf->mmap; - b->file = buf->file; + *ll = cl; + ll = &cl->next; + } + + if (out == NULL) { + return NGX_OK; + } + + *ll = NULL; + + if (finished + && ngx_http_range_link_last_boundary(r, ctx, ll) != NGX_OK) + { + return NGX_ERROR; + } + + return ngx_http_next_body_filter(r, out); +} + - if (buf->in_file) { - b->file_pos = buf->file_pos + range[i].start; - b->file_last = buf->file_pos + range[i].end; - } +static ngx_int_t +ngx_http_range_link_boundary_header(ngx_http_request_t *r, + ngx_http_range_filter_ctx_t *ctx, ngx_chain_t ***lll) +{ + ngx_buf_t *b; + ngx_chain_t *hcl, *rcl; + ngx_http_range_t *range; + + /* + * The boundary header of the range: + * CRLF + * "--0123456789" CRLF + * "Content-Type: image/jpeg" CRLF + * "Content-Range: bytes " + */ + + b = ngx_calloc_buf(r->pool); + if (b == NULL) { + return NGX_ERROR; + } - if (ngx_buf_in_memory(buf)) { - b->pos = buf->pos + (size_t) range[i].start; - b->last = buf->pos + (size_t) range[i].end; - } + b->memory = 1; + b->pos = ctx->boundary_header.data; + b->last = ctx->boundary_header.data + ctx->boundary_header.len; + + hcl = ngx_alloc_chain_link(r->pool); + if (hcl == NULL) { + return NGX_ERROR; + } + + hcl->buf = b; + + + /* "SSSS-EEEE/TTTT" CRLF CRLF */ + + b = ngx_calloc_buf(r->pool); + if (b == NULL) { + return NGX_ERROR; + } - dcl = ngx_alloc_chain_link(r->pool); - if (dcl == NULL) { - return NGX_ERROR; - } + range = ctx->ranges.elts; + b->temporary = 1; + b->pos = range[ctx->index - 1].content_range.data; + b->last = range[ctx->index - 1].content_range.data + + range[ctx->index - 1].content_range.len; + + rcl = ngx_alloc_chain_link(r->pool); + if (rcl == NULL) { + return NGX_ERROR; + } + + rcl->buf = b; - dcl->buf = b; + **lll = hcl; + hcl->next = rcl; + *lll = &rcl->next; + + return NGX_OK; +} - *ll = hcl; - hcl->next = rcl; - rcl->next = dcl; - ll = &dcl->next; - } + +static ngx_int_t +ngx_http_range_link_last_boundary(ngx_http_request_t *r, + ngx_http_range_filter_ctx_t *ctx, ngx_chain_t **ll) +{ + ngx_buf_t *b; + ngx_chain_t *hcl; /* the last boundary CRLF "--0123456789--" CRLF */ @@ -886,7 +1062,8 @@ ngx_http_range_multipart_body(ngx_http_r } b->temporary = 1; - b->last_buf = 1; + b->last_in_chain = 1; + b->last_buf = (r == r->main) ? 1 : 0; b->pos = ngx_pnalloc(r->pool, sizeof(CRLF "--") - 1 + NGX_ATOMIC_T_LEN + sizeof("--" CRLF) - 1); @@ -909,7 +1086,7 @@ ngx_http_range_multipart_body(ngx_http_r *ll = hcl; - return ngx_http_next_body_filter(r, out); + return NGX_OK; } _______________________________________________ nginx-devel mailing list nginx-devel@nginx.org http://mailman.nginx.org/mailman/listinfo/nginx-devel