This patch imports the HTTP implementation from ffmpeg. Incomplete list of new features and fixes:
* Support for HTTP seeking (intial ffmpeg commit e05655f) * Custom user agent (initial ffmpeg commit 2bb1c71) * Support for reading ICY metadata (initial ffmpeg commit a92fbe1) * Support for reading and setting cookies (initial ffmpeg commit 0b80a12) * Support for setting initial and ending offsets (initial ffmpeg commit d52882fa) * Improve options descriptions (initial ffmpeg commit 255ec768) Note that the commit ID specifies when the feature was originally introduced in ffmpeg, but doesn't account for subsequent changes and fixes. Bug-Debian: https://bugs.debian.org/740421 Bug-Debian: https://bugs.debian.org/739936 --- doc/protocols.texi | 66 ++++++++++++++ libavformat/http.c | 248 ++++++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 301 insertions(+), 13 deletions(-) diff --git a/doc/protocols.texi b/doc/protocols.texi index 1a9f575..cc3b4b6 100644 --- a/doc/protocols.texi +++ b/doc/protocols.texi @@ -89,6 +89,72 @@ m3u8 files. HTTP (Hyper Text Transfer Protocol). +This protocol accepts the following options. + +@table @option +@item seekable +Control seekability of connection. If set to 1 the resource is +supposed to be seekable, if set to 0 it is assumed not to be seekable, +if set to -1 it will try to autodetect if it is seekable. Default +value is -1. + +@item chunked_post +If set to 1 use chunked transfer-encoding for posts, default is 1. + +@item headers +Set custom HTTP headers, can override built in default headers. The +value must be a string encoding the headers. + +@item content_type +Force a content type. + +@item user-agent +Override User-Agent header. If not specified the protocol will use a +string describing the libavformat build. + +@item multiple_requests +Use persistent connections if set to 1. By default it is 0. + +@item post_data +Set custom HTTP post data. + +@item timeout +Set timeout of socket I/O operations used by the underlying low level +operation. By default it is set to -1, which means that the timeout is +not specified. + +@item mime_type +Set MIME type. + +@item icy +If set to 1 request ICY (SHOUTcast) metadata from the server. If the server +supports this, the metadata has to be retrieved by the application by reading +the @option{icy_metadata_headers} and @option{icy_metadata_packet} options. +The default is 0. + +@item icy_metadata_headers +If the server supports ICY metadata, this contains the ICY specific HTTP reply +headers, separated with newline characters. + +@item icy_metadata_packet +If the server supports ICY metadata, and @option{icy} was set to 1, this +contains the last non-empty metadata packet sent by the server. + +@item cookies +Set the cookies to be sent in future requests. The format of each cookie is the +same as the value of a Set-Cookie HTTP response field. Multiple cookies can be +delimited by a newline character. +@end table + +@subsection HTTP Cookies + +Some HTTP requests will be denied unless cookie values are passed in with the +request. The @option{cookies} option allows these cookies to be specified. At +the very least, each cookie must specify a value along with a path and domain. +HTTP requests that match both the domain and path will automatically include the +cookie value in the HTTP Cookie header field. Multiple cookies can be delimited +by a newline. + @section mmst MMS (Microsoft Media Server) protocol over TCP. diff --git a/libavformat/http.c b/libavformat/http.c index 96f56f8..f9ce2c3 100644 --- a/libavformat/http.c +++ b/libavformat/http.c @@ -50,18 +50,30 @@ typedef struct { int line_count; int http_code; int64_t chunksize; /**< Used if "Transfer-Encoding: chunked" otherwise -1. */ - int64_t off, filesize; + char *content_type; + char *user_agent; + int64_t off, filesize, req_end_offset; + int icy_data_read; ///< how much data was read since last ICY metadata packet + int icy_metaint; ///< after how many bytes of read data a new metadata packet will be found char *location; HTTPAuthState auth_state; HTTPAuthState proxy_auth_state; char *headers; int willclose; /**< Set if the server correctly handles Connection: close and will close the connection after feeding us the content. */ + int seekable; /**< Control seekability, 0 = disable, 1 = enable, -1 = probe. */ int chunked_post; int end_chunked_post; /**< A flag which indicates if the end of chunked encoding has been sent. */ int end_header; /**< A flag which indicates we have finished to read POST reply. */ int multiple_requests; /**< A flag which indicates if we use persistent connections. */ uint8_t *post_data; int post_datalen; + int is_akamai; + int is_mediagateway; + char *mime_type; + char *cookies; ///< holds newline (\n) delimited Set-Cookie header field values (without the "Set-Cookie: " field name) + int icy; + char *icy_metadata_headers; + char *icy_metadata_packet; #if CONFIG_ZLIB int compressed; z_stream inflate_stream; @@ -74,16 +86,27 @@ typedef struct { #define OFFSET(x) offsetof(HTTPContext, x) #define D AV_OPT_FLAG_DECODING_PARAM #define E AV_OPT_FLAG_ENCODING_PARAM +#define DEFAULT_USER_AGENT "Lavf/" AV_STRINGIFY(LIBAVFORMAT_VERSION) static const AVOption options[] = { +{"seekable", "control seekability of connection", OFFSET(seekable), AV_OPT_TYPE_INT, {.i64 = -1}, -1, 1, D }, {"chunked_post", "use chunked transfer-encoding for posts", OFFSET(chunked_post), AV_OPT_TYPE_INT, {.i64 = 1}, 0, 1, E }, -{"headers", "custom HTTP headers, can override built in default headers", OFFSET(headers), AV_OPT_TYPE_STRING, { 0 }, 0, 0, D|E }, +{"headers", "set custom HTTP headers, can override built in default headers", OFFSET(headers), AV_OPT_TYPE_STRING, { 0 }, 0, 0, D|E }, +{"content_type", "force a content type", OFFSET(content_type), AV_OPT_TYPE_STRING, { 0 }, 0, 0, D|E }, +{"user-agent", "override User-Agent header", OFFSET(user_agent), AV_OPT_TYPE_STRING, {.str = DEFAULT_USER_AGENT}, 0, 0, D }, {"multiple_requests", "use persistent connections", OFFSET(multiple_requests), AV_OPT_TYPE_INT, {.i64 = 0}, 0, 1, D|E }, -{"post_data", "custom HTTP post data", OFFSET(post_data), AV_OPT_TYPE_BINARY, .flags = D|E }, +{"post_data", "set custom HTTP post data", OFFSET(post_data), AV_OPT_TYPE_BINARY, .flags = D|E }, +{"mime_type", "set MIME type", OFFSET(mime_type), AV_OPT_TYPE_STRING, {0}, 0, 0, 0 }, +{"cookies", "set cookies to be sent in applicable future requests, use newline delimited Set-Cookie HTTP field value syntax", OFFSET(cookies), AV_OPT_TYPE_STRING, {0}, 0, 0, D }, +{"icy", "request ICY metadata", OFFSET(icy), AV_OPT_TYPE_INT, {.i64 = 0}, 0, 1, D }, +{"icy_metadata_headers", "return ICY metadata headers", OFFSET(icy_metadata_headers), AV_OPT_TYPE_STRING, {0}, 0, 0, 0 }, +{"icy_metadata_packet", "return current ICY metadata packet", OFFSET(icy_metadata_packet), AV_OPT_TYPE_STRING, {0}, 0, 0, 0 }, {"auth_type", "HTTP authentication type", OFFSET(auth_state.auth_type), AV_OPT_TYPE_INT, {.i64 = HTTP_AUTH_NONE}, HTTP_AUTH_NONE, HTTP_AUTH_BASIC, D|E, "auth_type" }, {"none", "No auth method set, autodetect", 0, AV_OPT_TYPE_CONST, {.i64 = HTTP_AUTH_NONE}, 0, 0, D|E, "auth_type" }, {"basic", "HTTP basic authentication", 0, AV_OPT_TYPE_CONST, {.i64 = HTTP_AUTH_BASIC}, 0, 0, D|E, "auth_type" }, {"send_expect_100", "Force sending an Expect: 100-continue header for POST", OFFSET(send_expect_100), AV_OPT_TYPE_INT, {.i64 = 0}, 0, 1, E }, {"location", "The actual location of the data received", OFFSET(location), AV_OPT_TYPE_STRING, { 0 }, 0, 0, D|E }, +{"offset", "initial byte offset", OFFSET(off), AV_OPT_TYPE_INT64, {.i64 = 0}, 0, INT64_MAX, D }, +{"end_offset", "try to limit the request to bytes preceding this offset", OFFSET(req_end_offset), AV_OPT_TYPE_INT64, {.i64 = 0}, 0, INT64_MAX, D }, {NULL} }; #define HTTP_CLASS(flavor)\ @@ -219,6 +242,7 @@ int ff_http_do_new_request(URLContext *h, const char *uri) int ret; s->off = 0; + s->icy_data_read = 0; av_free(s->location); s->location = av_strdup(uri); if (!s->location) @@ -236,7 +260,10 @@ static int http_open(URLContext *h, const char *uri, int flags, HTTPContext *s = h->priv_data; int ret; - h->is_streamed = 1; + if (s->seekable == 1) + h->is_streamed = 0; + else + h->is_streamed = 1; s->filesize = -1; s->location = av_strdup(uri); @@ -361,8 +388,9 @@ static int process_line(URLContext *h, char *line, int line_count, if ((slash = strchr(p, '/')) && strlen(slash) > 0) s->filesize = strtoll(slash+1, NULL, 10); } - h->is_streamed = 0; /* we _can_ in fact seek */ - } else if (!av_strcasecmp(tag, "Accept-Ranges") && !strncmp(p, "bytes", 5)) { + if (s->seekable == -1 && (!s->is_akamai || s->filesize != 2147483647)) + h->is_streamed = 0; /* we _can_ in fact seek */ + } else if (!av_strcasecmp(tag, "Accept-Ranges") && !strncmp(p, "bytes", 5) && s->seekable == -1) { h->is_streamed = 0; } else if (!av_strcasecmp (tag, "Transfer-Encoding") && !av_strncasecmp(p, "chunked", 7)) { s->filesize = -1; @@ -376,6 +404,38 @@ static int process_line(URLContext *h, char *line, int line_count, } else if (!av_strcasecmp (tag, "Connection")) { if (!strcmp(p, "close")) s->willclose = 1; + } else if (!av_strcasecmp (tag, "Server")) { + if (!av_strcasecmp (p, "AkamaiGHost")) { + s->is_akamai = 1; + } else if (!av_strncasecmp (p, "MediaGateway", 12)) { + s->is_mediagateway = 1; + } + } else if (!av_strcasecmp (tag, "Content-Type")) { + av_free(s->mime_type); s->mime_type = av_strdup(p); + } else if (!av_strcasecmp (tag, "Set-Cookie")) { + if (!s->cookies) { + if (!(s->cookies = av_strdup(p))) + return AVERROR(ENOMEM); + } else { + char *tmp = s->cookies; + size_t str_size = strlen(tmp) + strlen(p) + 2; + if (!(s->cookies = av_malloc(str_size))) { + s->cookies = tmp; + return AVERROR(ENOMEM); + } + snprintf(s->cookies, str_size, "%s\n%s", tmp, p); + av_free(tmp); + } + } else if (!av_strcasecmp (tag, "Icy-MetaInt")) { + s->icy_metaint = strtoll(p, NULL, 10); + } else if (!av_strncasecmp(tag, "Icy-", 4)) { + // Concat all Icy- header lines + char *buf = av_asprintf("%s%s: %s\n", + s->icy_metadata_headers ? s->icy_metadata_headers : "", tag, p); + if (!buf) + return AVERROR(ENOMEM); + av_freep(&s->icy_metadata_headers); + s->icy_metadata_headers = buf; } else if (!av_strcasecmp (tag, "Content-Encoding")) { if (!av_strncasecmp(p, "gzip", 4) || !av_strncasecmp(p, "deflate", 7)) { #if CONFIG_ZLIB @@ -399,13 +459,112 @@ static int process_line(URLContext *h, char *line, int line_count, // the header at all if this is the case). } else { av_log(h, AV_LOG_WARNING, "Unknown content coding: %s\n", p); - return AVERROR(ENOSYS); } } } return 1; } +/** + * Create a string containing cookie values for use as a HTTP cookie header + * field value for a particular path and domain from the cookie values stored in + * the HTTP protocol context. The cookie string is stored in *cookies. + * + * @return a negative value if an error condition occurred, 0 otherwise + */ +static int get_cookies(HTTPContext *s, char **cookies, const char *path, + const char *domain) +{ + // cookie strings will look like Set-Cookie header field values. Multiple + // Set-Cookie fields will result in multiple values delimited by a newline + int ret = 0; + char *next, *cookie, *set_cookies = av_strdup(s->cookies), *cset_cookies = set_cookies; + + if (!set_cookies) return AVERROR(EINVAL); + + *cookies = NULL; + while ((cookie = av_strtok(set_cookies, "\n", &next))) { + int domain_offset = 0; + char *param, *next_param, *cdomain = NULL, *cpath = NULL, *cvalue = NULL; + set_cookies = NULL; + + while ((param = av_strtok(cookie, "; ", &next_param))) { + cookie = NULL; + if (!av_strncasecmp("path=", param, 5)) { + av_free(cpath); + cpath = av_strdup(¶m[5]); + } else if (!av_strncasecmp("domain=", param, 7)) { + // if the cookie specifies a sub-domain, skip the leading dot thereby + // supporting URLs that point to sub-domains and the master domain + int leading_dot = (param[7] == '.'); + av_free(cdomain); + cdomain = av_strdup(¶m[7+leading_dot]); + } else if (!av_strncasecmp("secure", param, 6) || + !av_strncasecmp("comment", param, 7) || + !av_strncasecmp("max-age", param, 7) || + !av_strncasecmp("version", param, 7)) { + // ignore Comment, Max-Age, Secure and Version + } else { + av_free(cvalue); + cvalue = av_strdup(param); + } + } + if (!cdomain) + cdomain = av_strdup(domain); + + // ensure all of the necessary values are valid + if (!cdomain || !cpath || !cvalue) { + av_log(s, AV_LOG_WARNING, + "Invalid cookie found, no value, path or domain specified\n"); + goto done_cookie; + } + + // check if the request path matches the cookie path + if (av_strncasecmp(path, cpath, strlen(cpath))) + goto done_cookie; + + // the domain should be at least the size of our cookie domain + domain_offset = strlen(domain) - strlen(cdomain); + if (domain_offset < 0) + goto done_cookie; + + // match the cookie domain + if (av_strcasecmp(&domain[domain_offset], cdomain)) + goto done_cookie; + + // cookie parameters match, so copy the value + if (!*cookies) { + if (!(*cookies = av_strdup(cvalue))) { + ret = AVERROR(ENOMEM); + goto done_cookie; + } + } else { + char *tmp = *cookies; + size_t str_size = strlen(cvalue) + strlen(*cookies) + 3; + if (!(*cookies = av_malloc(str_size))) { + ret = AVERROR(ENOMEM); + goto done_cookie; + } + snprintf(*cookies, str_size, "%s; %s", tmp, cvalue); + av_free(tmp); + } + +done_cookie: + av_free(cdomain); + av_free(cpath); + av_free(cvalue); + if (ret < 0) { + if (*cookies) av_freep(cookies); + av_free(cset_cookies); + return ret; + } + } + + av_free(cset_cookies); + + return 0; +} + static inline int has_header(const char *str, const char *header) { /* header + 2 to skip over CRLF prefix. (make sure you have one!) */ @@ -436,6 +595,9 @@ static int http_read_header(URLContext *h, int *new_location) s->line_count++; } + if (s->seekable == -1 && s->is_mediagateway && s->filesize == 2000000000) + h->is_streamed = 1; /* we can in fact _not_ seek */ + return err; } @@ -445,7 +607,7 @@ static int http_connect(URLContext *h, const char *path, const char *local_path, { HTTPContext *s = h->priv_data; int post, err; - char headers[1024] = ""; + char headers[4096] = ""; char *authstr = NULL, *proxyauthstr = NULL; int64_t off = s->off; int len = 0; @@ -482,14 +644,23 @@ static int http_connect(URLContext *h, const char *path, const char *local_path, /* set default headers if needed */ if (!has_header(s->headers, "\r\nUser-Agent: ")) - len += av_strlcatf(headers + len, sizeof(headers) - len, - "User-Agent: %s\r\n", LIBAVFORMAT_IDENT); + len += av_strlcatf(headers + len, sizeof(headers) - len, + "User-Agent: %s\r\n", s->user_agent); if (!has_header(s->headers, "\r\nAccept: ")) len += av_strlcpy(headers + len, "Accept: */*\r\n", sizeof(headers) - len); - if (!has_header(s->headers, "\r\nRange: ") && !post) + // Note: we send this on purpose even when s->off is 0 when we're probing, + // since it allows us to detect more reliably if a (non-conforming) + // server supports seeking by analysing the reply headers. + if (!has_header(s->headers, "\r\nRange: ") && !post && (s->off > 0 || s->req_end_offset || s->seekable == -1)) { len += av_strlcatf(headers + len, sizeof(headers) - len, - "Range: bytes=%"PRId64"-\r\n", s->off); + "Range: bytes=%"PRId64"-", s->off); + if (s->req_end_offset) + len += av_strlcatf(headers + len, sizeof(headers) - len, + "%"PRId64, s->req_end_offset - 1); + len += av_strlcpy(headers + len, "\r\n", + sizeof(headers) - len); + } if (send_expect_100 && !has_header(s->headers, "\r\nExpect: ")) len += av_strlcatf(headers + len, sizeof(headers) - len, "Expect: 100-continue\r\n"); @@ -510,6 +681,21 @@ static int http_connect(URLContext *h, const char *path, const char *local_path, if (!has_header(s->headers, "\r\nContent-Length: ") && s->post_data) len += av_strlcatf(headers + len, sizeof(headers) - len, "Content-Length: %d\r\n", s->post_datalen); + if (!has_header(s->headers, "\r\nContent-Type: ") && s->content_type) + len += av_strlcatf(headers + len, sizeof(headers) - len, + "Content-Type: %s\r\n", s->content_type); + if (!has_header(s->headers, "\r\nCookie: ") && s->cookies) { + char *cookies = NULL; + if (!get_cookies(s, &cookies, path, hoststr)) { + len += av_strlcatf(headers + len, sizeof(headers) - len, + "Cookie: %s\r\n", cookies); + av_free(cookies); + } + } + if (!has_header(s->headers, "\r\nIcy-MetaData: ") && s->icy) { + len += av_strlcatf(headers + len, sizeof(headers) - len, + "Icy-MetaData: %d\r\n", 1); + } /* now add in custom headers */ if (s->headers) @@ -531,6 +717,9 @@ static int http_connect(URLContext *h, const char *path, const char *local_path, av_freep(&authstr); av_freep(&proxyauthstr); + + av_log(h, AV_LOG_DEBUG, "request: %s\n", s->buffer); + if ((err = ffurl_write(s->hd, s->buffer, strlen(s->buffer))) < 0) return err; @@ -543,6 +732,7 @@ static int http_connect(URLContext *h, const char *path, const char *local_path, s->buf_end = s->buffer; s->line_count = 0; s->off = 0; + s->icy_data_read = 0; s->filesize = -1; s->willclose = 0; s->end_chunked_post = 0; @@ -582,6 +772,7 @@ static int http_buf_read(URLContext *h, uint8_t *buf, int size) } if (len > 0) { s->off += len; + s->icy_data_read += len; if (s->chunksize > 0) s->chunksize -= len; } @@ -655,6 +846,32 @@ static int http_read(URLContext *h, uint8_t *buf, int size) } size = FFMIN(size, s->chunksize); } + if (s->icy_metaint > 0) { + int remaining = s->icy_metaint - s->icy_data_read; /* until next metadata packet */ + if (!remaining) { + // The metadata packet is variable sized. It has a 1 byte header + // which sets the length of the packet (divided by 16). If it's 0, + // the metadata doesn't change. After the packet, icy_metaint bytes + // of normal data follow. + int ch = http_getc(s); + if (ch < 0) + return ch; + if (ch > 0) { + char data[255 * 16 + 1]; + int n; + int ret; + ch *= 16; + for (n = 0; n < ch; n++) + data[n] = http_getc(s); + data[ch + 1] = 0; + if ((ret = av_opt_set(s, "icy_metadata_packet", data, 0)) < 0) + return ret; + } + s->icy_data_read = 0; + remaining = s->icy_metaint; + } + size = FFMIN(size, remaining); + } #if CONFIG_ZLIB if (s->compressed) return http_buf_read_compressed(h, buf, size); @@ -737,6 +954,8 @@ static int64_t http_seek(URLContext *h, int64_t off, int whence) if (whence == AVSEEK_SIZE) return s->filesize; + else if ((whence == SEEK_CUR && off == 0) || (whence == SEEK_SET && off == s->off)) + return s->off; else if ((s->filesize == -1 && whence == SEEK_END) || h->is_streamed) return -1; @@ -824,7 +1043,10 @@ static int http_proxy_open(URLContext *h, const char *uri, int flags) char *authstr; int new_loc; - h->is_streamed = 1; + if (s->seekable == 1) + h->is_streamed = 0; + else + h->is_streamed = 1; av_url_split(NULL, 0, auth, sizeof(auth), hostname, sizeof(hostname), &port, pathbuf, sizeof(pathbuf), uri); -- 1.9.0 _______________________________________________ libav-devel mailing list [email protected] https://lists.libav.org/mailman/listinfo/libav-devel
