PR #24810 opened by Niklas Haas (haasn) URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24810 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24810.patch
Feature parity with http.c Supersedes #23934. >From e65bff58e41efe443f35192073edb03d50da24d0 Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Wed, 30 Sep 2026 12:52:48 +0200 Subject: [PATCH 1/8] avformat/libcurl: keep track of last HTTP status Update whenever a new HTTP status is received, and cleared explicitly on a seek (which conceptually starts a new request lifecycle). Signed-off-by: Niklas Haas <[email protected]> --- libavformat/libcurl.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libavformat/libcurl.c b/libavformat/libcurl.c index 983b207722..9e02861381 100644 --- a/libavformat/libcurl.c +++ b/libavformat/libcurl.c @@ -185,6 +185,7 @@ struct CurlContext { int aborted; /* transfer should stop (open was interrupted) */ int status; /* current stream status (AVERROR code) */ CURLcode curl_status; /* corresponding libcurl status code */ + int http_status; /* HTTP response code of last request, or 0 */ int64_t icy_metaint; /* in-band metadata interval, 0 if none */ }; @@ -536,6 +537,7 @@ static size_t header_callback(char *ptr, size_t size, size_t nitems, void *userd c->hdr_content_length = cl >= 0 ? cl : -1; pthread_mutex_lock(&c->mutex); + c->http_status = status; if (status >= 200 && status < 300) { int64_t content_start = status == 206 ? c->hdr_content_start : 0; int64_t content_end = status == 206 ? c->hdr_content_end : -1; @@ -820,6 +822,7 @@ static void execute_command(CurlLoop *loop, CurlCmd *cmd) c->paused = 0; c->status = 0; c->curl_status = 0; + c->http_status = 0; pthread_mutex_unlock(&c->mutex); c->request_start = cmd->pos; c->request_received = 0; -- 2.52.0 >From 01235be5d0ce6f335a799948f8333c7ae347aaca Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Thu, 9 Jul 2026 19:29:08 +0200 Subject: [PATCH 2/8] avformat/libcurl: add -reconnect_on_http_error option Parity with http.c. This piggy-backs off the previously added mechanism for triggering a retry by just setting the stream status to AVERROR(EAGAIN) and letting the caller deal with it. Sponsored-by: nxtedition AB Signed-off-by: Niklas Haas <[email protected]> --- doc/protocols.texi | 4 ++++ libavformat/libcurl.c | 26 ++++++++++++++++++++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/doc/protocols.texi b/doc/protocols.texi index 606ce07f3f..5a959b1f6e 100644 --- a/doc/protocols.texi +++ b/doc/protocols.texi @@ -1174,6 +1174,10 @@ connection may be closed and recreated for each subsequent initial request. Maximum number of retries after a recoverable error on a seekable transfer. Default is @code{5}. +@item reconnect_on_http_error +A comma separated list of HTTP status codes to reconnect on. The list can +include specific status codes (e.g. '503') or the strings '4xx' / '5xx'. + @item short_seek_size Set the threshold, in bytes, for when a readahead should be preferred over a seek and new HTTP request. This is useful, for example, to make sure the same connection diff --git a/libavformat/libcurl.c b/libavformat/libcurl.c index 9e02861381..70760ce9f4 100644 --- a/libavformat/libcurl.c +++ b/libavformat/libcurl.c @@ -143,6 +143,7 @@ struct CurlContext { char *icy_metadata_headers; /* "Icy-*" reply headers (output) */ char *icy_metadata_packet; /* last in-band block (output) */ AVDictionary *metadata; /* ICY metadata (output) */ + char *reconnect_on_http_error; /* URL thread bookkeeping, not touched by loop thread */ int64_t logical_pos; /* next byte url_read() will return, caller side */ @@ -233,9 +234,29 @@ static int curlmcode_to_curlcode(CURLMcode code) } } -static int is_recoverable(CURLcode code) +static int http_error_is_recoverable(CurlContext *c, long http_status) +{ + if (!c->reconnect_on_http_error || !http_status) + return 0; + + const char *status_group = NULL; + if (http_status >= 400 && http_status < 500) + status_group = "4xx"; + else if (http_status >= 500 && http_status < 600) + status_group = "5xx"; + if (status_group && av_match_list(status_group, c->reconnect_on_http_error, ',') > 0) + return 1; + + char http_code[4]; + snprintf(http_code, sizeof(http_code), "%ld", http_status); + return av_match_list(http_code, c->reconnect_on_http_error, ',') > 0; +} + +static int is_recoverable(CurlContext *c, CURLcode code, int http_status) { switch (code) { + case CURLE_OK: + return http_error_is_recoverable(c, http_status); case CURLE_RECV_ERROR: case CURLE_SEND_ERROR: case CURLE_PARTIAL_FILE: @@ -1489,7 +1510,7 @@ static int libcurl_read(URLContext *h, unsigned char *buf, int size) } if (c->status) { - if (c->seekable && is_recoverable(c->curl_status)) + if (c->seekable && is_recoverable(c, c->curl_status, c->http_status)) return retry_request_locked(h, nonblock); if (c->status == AVERROR_EOF && c->icy_block_len >= 0) av_log(h, AV_LOG_WARNING, @@ -1643,6 +1664,7 @@ static const AVOption options[] = { { "3", "HTTP/3, fall back to earlier versions", 0, AV_OPT_TYPE_CONST, { .i64 = CURL_HTTP_VERSION_3 }, 0, 0, D, .unit = "http_version" }, { "3only", "HTTP/3 only", 0, AV_OPT_TYPE_CONST, { .i64 = CURL_HTTP_VERSION_3ONLY }, 0, 0, D, .unit = "http_version" }, { "short_seek_size", "threshold to favor readahead over seek", OFFSET(short_seek_size), AV_OPT_TYPE_INT64, { .i64 = 0 }, 0, INT64_MAX, D }, + { "reconnect_on_http_error", "list of http status codes to reconnect on", OFFSET(reconnect_on_http_error), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, D }, { NULL } }; -- 2.52.0 >From a8679e49b93494660f9298f003b4abed08ce9bb9 Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Thu, 9 Jul 2026 19:44:20 +0200 Subject: [PATCH 3/8] avformat/libcurl: also continue sending requests on unknown size Treat files with unknown size as having infinite size, at least until we see a 416 Range Not Satisfiable. Sponsored-by: nxtedition AB Signed-off-by: Niklas Haas <[email protected]> --- libavformat/libcurl.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/libavformat/libcurl.c b/libavformat/libcurl.c index 70760ce9f4..011d8462c8 100644 --- a/libavformat/libcurl.c +++ b/libavformat/libcurl.c @@ -766,7 +766,9 @@ static void on_done(CurlContext *c, CURLcode code) int64_t file_end = c->content_size > 0 ? c->content_size - 1 : -1; if (c->end_off > 0) file_end = FFMIN(file_end, c->end_off - 1); - if (c->seekable && c->request_end >= 0 && c->request_end < file_end) { + if (c->seekable && c->request_end >= 0 && + (file_end < 0 || c->request_end < file_end)) + { c->is_initial = 0; start_request(c); return; -- 2.52.0 >From 4a144f45519ff3f9323fad3f841d652e6e88f5f2 Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Wed, 30 Sep 2026 13:04:44 +0200 Subject: [PATCH 4/8] avformat/libcurl: only print relevant status on retry It's a bit confusing to print e.g. "<error message> (No error)" when there is a nonzero status without a corresponding curl error code. Signed-off-by: Niklas Haas <[email protected]> --- libavformat/libcurl.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/libavformat/libcurl.c b/libavformat/libcurl.c index 011d8462c8..7eb09bc635 100644 --- a/libavformat/libcurl.c +++ b/libavformat/libcurl.c @@ -1331,10 +1331,16 @@ static int retry_request_locked(URLContext *h, int nonblock) c->retry_time = av_gettime_relative() + delay; c->retry_count++; + char err[256]; + if (code == CURLE_OK) { + snprintf(err, sizeof(err), "%s", av_err2str(status)); + } else { + snprintf(err, sizeof(err), "%s (%s)", av_err2str(status), + curl_easy_strerror(code)); + } + av_log(h, AV_LOG_WARNING, "Retrying (#%d) from %"PRId64" in %.3fs " - "after: %s (%s)\n", c->retry_count, c->logical_pos, - delay * 1e-6, av_err2str(status), - curl_easy_strerror(code)); + "after: %s\n", c->retry_count, c->logical_pos, delay * 1e-6, err); } const int64_t sleep_us = c->retry_time - av_gettime_relative(); -- 2.52.0 >From 82ead32ba2923b9019068e755f51df7f58701be8 Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Thu, 9 Jul 2026 19:48:37 +0200 Subject: [PATCH 5/8] avformat/libcurl: don't tear down connection on HTTP error This otherwise defeats the purpose of keep-alive / -multiple_requests, especially when hitting a temporary error (429, 416, 503, etc). The CURL_WRITEFUNC_ERROR is used to interrupt curl mid-transfer, which is only really needed for c->aborted anyways. Sponsored-by: nxtedition AB Signed-off-by: Niklas Haas <[email protected]> --- libavformat/libcurl.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libavformat/libcurl.c b/libavformat/libcurl.c index 7eb09bc635..8bbf3e64e3 100644 --- a/libavformat/libcurl.c +++ b/libavformat/libcurl.c @@ -293,12 +293,12 @@ static size_t write_callback(char *ptr, size_t size, size_t nmemb, void *userdat pthread_mutex_lock(&c->mutex); - if (c->aborted || !c->stream_ok) { + if (c->aborted) { pthread_mutex_unlock(&c->mutex); return CURL_WRITEFUNC_ERROR; } - if (c->seek_queued) { + if (c->seek_queued || !c->stream_ok) { pthread_mutex_unlock(&c->mutex); return bytes; /* discard */ } -- 2.52.0 >From 8f3d2209791e23317a8974058f2401f601ad192b Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Thu, 9 Jul 2026 19:38:55 +0200 Subject: [PATCH 6/8] avformat/libcurl: add -respect_retry_after option Feature parity with http.c. This overrides the built-in retry delay if set. We still clamp to the maximum retry delay of 1 minute as a safety precaution against servers sending very large values stalling the demuxer thread indefinitely. A 1 minute range is still plenty enough to implement rate limiting. Sponsored-by: nxtedition AB Signed-off-by: Niklas Haas <[email protected]> --- doc/protocols.texi | 5 +++++ libavformat/libcurl.c | 16 +++++++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/doc/protocols.texi b/doc/protocols.texi index 5a959b1f6e..061fa3871c 100644 --- a/doc/protocols.texi +++ b/doc/protocols.texi @@ -1178,6 +1178,11 @@ Default is @code{5}. A comma separated list of HTTP status codes to reconnect on. The list can include specific status codes (e.g. '503') or the strings '4xx' / '5xx'. +@item respect_retry_after +If enabled, and a Retry-After header is encountered, its requested reconnection +delay will be honored, rather than using exponential backoff. Useful for 429 and +503 errors. Default enabled. + @item short_seek_size Set the threshold, in bytes, for when a readahead should be preferred over a seek and new HTTP request. This is useful, for example, to make sure the same connection diff --git a/libavformat/libcurl.c b/libavformat/libcurl.c index 8bbf3e64e3..3be56f63fb 100644 --- a/libavformat/libcurl.c +++ b/libavformat/libcurl.c @@ -144,6 +144,7 @@ struct CurlContext { char *icy_metadata_packet; /* last in-band block (output) */ AVDictionary *metadata; /* ICY metadata (output) */ char *reconnect_on_http_error; + int respect_retry_after; /* URL thread bookkeeping, not touched by loop thread */ int64_t logical_pos; /* next byte url_read() will return, caller side */ @@ -172,11 +173,12 @@ struct CurlContext { AVDictionary *hdr_icy; /* "Icy-*" headers of this block */ int64_t hdr_icy_metaint; /* in-band metadata interval, or -1 */ - /* Probe result. Set by the loop thread, read by url_open() once probed. */ + /* Probe result. Set by the loop thread, read by URL thread once probed. */ int probed; int stream_ok; int seekable; int64_t content_size; + int64_t retry_after; /* raw value returned by server, or 0 */ /* Shared transfer state, guarded by mutex. */ pthread_mutex_t mutex; @@ -634,6 +636,10 @@ static size_t header_callback(char *ptr, size_t size, size_t nitems, void *userd c->loop->num_errors++; c->stream_ok = 0; update_status_locked(c, ff_http_averror(status, AVERROR(EIO)), CURLE_OK); + + curl_off_t retry_after = 0; + curl_easy_getinfo(c->easy, CURLINFO_RETRY_AFTER, &retry_after); + c->retry_after = retry_after; } c->probed = 1; pthread_cond_broadcast(&c->cond); @@ -682,6 +688,7 @@ static void start_request(CurlContext *c) c->loop->num_requests++; c->request_received = 0; c->request_end = -1; + c->retry_after = 0; c->active = 1; CURLMcode res = curl_multi_add_handle(c->loop->multi, c->easy); if (res != CURLM_OK) { @@ -1307,6 +1314,12 @@ static int wait_for_probe(CurlContext *c) /* Scales by the recurrence relationship x := 2x + 1, i.e. 2^n - 1 */ static int64_t retry_delay(CurlContext *c) { + int64_t retry_after = c->retry_after; + if (c->respect_retry_after && retry_after > 0) { + retry_after = FFMIN(retry_after, CURL_RETRY_MAX_US / 1000000); + return retry_after * 1000000; + } + int64_t factor = INT64_MAX >> FFMAX(63 - c->retry_count, 0); if (factor >= CURL_RETRY_MAX_US / CURL_RETRY_BASE_US) return CURL_RETRY_MAX_US; @@ -1673,6 +1686,7 @@ static const AVOption options[] = { { "3only", "HTTP/3 only", 0, AV_OPT_TYPE_CONST, { .i64 = CURL_HTTP_VERSION_3ONLY }, 0, 0, D, .unit = "http_version" }, { "short_seek_size", "threshold to favor readahead over seek", OFFSET(short_seek_size), AV_OPT_TYPE_INT64, { .i64 = 0 }, 0, INT64_MAX, D }, { "reconnect_on_http_error", "list of http status codes to reconnect on", OFFSET(reconnect_on_http_error), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, D }, + { "respect_retry_after", "respect the Retry-After header when retrying connections", OFFSET(respect_retry_after), AV_OPT_TYPE_BOOL, { .i64 = 1 }, 0, 1, D }, { NULL } }; -- 2.52.0 >From 60890fab4bfc39f64230aad295cca06274d4200c Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Tue, 21 Jul 2026 12:08:20 +0200 Subject: [PATCH 7/8] avformat/libcurl: retry 429 and 503 errors by default These are explicitly considered temporary failures that should be retried, and a conforming server would advertise a Retry-After header that we respect. Sponsored-by: nxtedition AB Signed-off-by: Niklas Haas <[email protected]> --- doc/protocols.texi | 1 + libavformat/libcurl.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/protocols.texi b/doc/protocols.texi index 061fa3871c..6ac6c186c7 100644 --- a/doc/protocols.texi +++ b/doc/protocols.texi @@ -1177,6 +1177,7 @@ Default is @code{5}. @item reconnect_on_http_error A comma separated list of HTTP status codes to reconnect on. The list can include specific status codes (e.g. '503') or the strings '4xx' / '5xx'. +Defaults to @code{429,503}. @item respect_retry_after If enabled, and a Retry-After header is encountered, its requested reconnection diff --git a/libavformat/libcurl.c b/libavformat/libcurl.c index 3be56f63fb..f0ecd59487 100644 --- a/libavformat/libcurl.c +++ b/libavformat/libcurl.c @@ -1685,7 +1685,7 @@ static const AVOption options[] = { { "3", "HTTP/3, fall back to earlier versions", 0, AV_OPT_TYPE_CONST, { .i64 = CURL_HTTP_VERSION_3 }, 0, 0, D, .unit = "http_version" }, { "3only", "HTTP/3 only", 0, AV_OPT_TYPE_CONST, { .i64 = CURL_HTTP_VERSION_3ONLY }, 0, 0, D, .unit = "http_version" }, { "short_seek_size", "threshold to favor readahead over seek", OFFSET(short_seek_size), AV_OPT_TYPE_INT64, { .i64 = 0 }, 0, INT64_MAX, D }, - { "reconnect_on_http_error", "list of http status codes to reconnect on", OFFSET(reconnect_on_http_error), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, D }, + { "reconnect_on_http_error", "list of http status codes to reconnect on", OFFSET(reconnect_on_http_error), AV_OPT_TYPE_STRING, { .str = "429,503" }, 0, 0, D }, { "respect_retry_after", "respect the Retry-After header when retrying connections", OFFSET(respect_retry_after), AV_OPT_TYPE_BOOL, { .i64 = 1 }, 0, 1, D }, { NULL } }; -- 2.52.0 >From 67c311697989204014f8047860c20bffefbd0753 Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Wed, 30 Sep 2026 14:20:36 +0200 Subject: [PATCH 8/8] avformat/libcurl: add -reconnect_streamed Parity with http.c. This will just result in a non-range request, which is analogous to the situation in http.c. It's up to the server to send meaningful data in this case. It's worth pointing out that we need to seek back to 0 in this case, since otherwise the content range validation will error out. Sponsored-by: nxtedition AB Signed-off-by: Niklas Haas <[email protected]> --- doc/protocols.texi | 6 ++++++ libavformat/libcurl.c | 12 ++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/doc/protocols.texi b/doc/protocols.texi index 6ac6c186c7..7f9536fa33 100644 --- a/doc/protocols.texi +++ b/doc/protocols.texi @@ -1179,6 +1179,12 @@ A comma separated list of HTTP status codes to reconnect on. The list can include specific status codes (e.g. '503') or the strings '4xx' / '5xx'. Defaults to @code{429,503}. +@item reconnect_streamed +If set then even streamed/non seekable streams will be reconnected on errors. +Note that this can result in nonsensical/corrupt output if the server does +not actually resume the stream at the same position after reconnecting. +Use with caution. Default is @code{0}. + @item respect_retry_after If enabled, and a Retry-After header is encountered, its requested reconnection delay will be honored, rather than using exponential backoff. Useful for 429 and diff --git a/libavformat/libcurl.c b/libavformat/libcurl.c index f0ecd59487..75d77b5280 100644 --- a/libavformat/libcurl.c +++ b/libavformat/libcurl.c @@ -28,6 +28,7 @@ #include <stdlib.h> #include <string.h> +#include "libavutil/avassert.h" #include "libavutil/avstring.h" #include "libavutil/bprint.h" #include "libavutil/dict.h" @@ -144,6 +145,7 @@ struct CurlContext { char *icy_metadata_packet; /* last in-band block (output) */ AVDictionary *metadata; /* ICY metadata (output) */ char *reconnect_on_http_error; + int reconnect_streamed; int respect_retry_after; /* URL thread bookkeeping, not touched by loop thread */ @@ -256,6 +258,9 @@ static int http_error_is_recoverable(CurlContext *c, long http_status) static int is_recoverable(CurlContext *c, CURLcode code, int http_status) { + if (!c->seekable && !c->reconnect_streamed) + return 0; + switch (code) { case CURLE_OK: return http_error_is_recoverable(c, http_status); @@ -1364,7 +1369,9 @@ static int retry_request_locked(URLContext *h, int nonblock) * the reset of c->state is observable, before the next libcurl_read() * call, otherwise this might hit the exact same retry path a second * time. */ - int ret = curl_dispatch(c->loop, CMD_SEEK, c, c->logical_pos, 1); + av_assert0(c->seekable || c->reconnect_streamed); + const int64_t pos = c->seekable ? c->logical_pos : 0; + int ret = curl_dispatch(c->loop, CMD_SEEK, c, pos, 1); if (ret < 0) return ret; c->retry_time = 0; @@ -1531,7 +1538,7 @@ static int libcurl_read(URLContext *h, unsigned char *buf, int size) } if (c->status) { - if (c->seekable && is_recoverable(c, c->curl_status, c->http_status)) + if (is_recoverable(c, c->curl_status, c->http_status)) return retry_request_locked(h, nonblock); if (c->status == AVERROR_EOF && c->icy_block_len >= 0) av_log(h, AV_LOG_WARNING, @@ -1686,6 +1693,7 @@ static const AVOption options[] = { { "3only", "HTTP/3 only", 0, AV_OPT_TYPE_CONST, { .i64 = CURL_HTTP_VERSION_3ONLY }, 0, 0, D, .unit = "http_version" }, { "short_seek_size", "threshold to favor readahead over seek", OFFSET(short_seek_size), AV_OPT_TYPE_INT64, { .i64 = 0 }, 0, INT64_MAX, D }, { "reconnect_on_http_error", "list of http status codes to reconnect on", OFFSET(reconnect_on_http_error), AV_OPT_TYPE_STRING, { .str = "429,503" }, 0, 0, D }, + { "reconnect_streamed", "auto reconnect streamed / non seekable streams", OFFSET(reconnect_streamed), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, D }, { "respect_retry_after", "respect the Retry-After header when retrying connections", OFFSET(respect_retry_after), AV_OPT_TYPE_BOOL, { .i64 = 1 }, 0, 1, D }, { NULL } }; -- 2.52.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
