PR #23624 opened by Niklas Haas (haasn) URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23624 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23624.patch
Reproduce: ``` $ make && ./ffmpeg_g -re -i fallbeatcaptiontest.mp4 -c copy -f null -t 10 - [aist#0:1/aac @ 0x2b944f80] Resumed reading at pts 0.512 with rate 1.050 after a lag of 0.488s [vist#0:0/h264 @ 0x2b945480] Resumed reading at pts 1.000 with rate 1.050 after a lag of 0.306s [aist#0:1/aac @ 0x2b944f80] Resumed reading at pts 0.981 with rate 1.050 after a lag of 0.802s [aist#0:1/aac @ 0x2b944f80] Resumed reading at pts 0.981 with rate 1.050 after a lag of 1.120s [vist#0:0/h264 @ 0x2b945480] Resumed reading at pts 2.000 with rate 1.050 after a lag of 0.610s [vist#0:0/h264 @ 0x2b945480] Resumed reading at pts 2.000 with rate 1.050 after a lag of 0.915s [aist#0:1/aac @ 0x2b944f80] Resumed reading at pts 1.963 with rate 1.050 after a lag of 1.430s [aist#0:1/aac @ 0x2b944f80] Resumed reading at pts 1.963 with rate 1.050 after a lag of 1.747s [vist#0:0/h264 @ 0x2b945480] Resumed reading at pts 3.000 with rate 1.050 after a lag of 1.219s [vist#0:0/h264 @ 0x2b945480] Resumed reading at pts 3.000 with rate 1.050 after a lag of 1.523s [aist#0:1/aac @ 0x2b944f80] Resumed reading at pts 2.965 with rate 1.050 after a lag of 2.067s [aist#0:1/aac @ 0x2b944f80] Resumed reading at pts 2.965 with rate 1.050 after a lag of 2.384s [vist#0:0/h264 @ 0x2b945480] Resumed reading at pts 4.000 with rate 1.050 after a lag of 1.839s [vist#0:0/h264 @ 0x2b945480] Resumed reading at pts 4.000 with rate 1.050 after a lag of 2.143s [aist#0:1/aac @ 0x2b944f80] Resumed reading at pts 3.968 with rate 1.050 after a lag of 2.685s [aist#0:1/aac @ 0x2b944f80] Resumed reading at pts 3.968 with rate 1.050 after a lag of 3.002s [vist#0:0/h264 @ 0x2b945480] Resumed reading at pts 5.000 with rate 1.050 after a lag of 2.459s [vist#0:0/h264 @ 0x2b945480] Resumed reading at pts 5.000 with rate 1.050 after a lag of 2.763s [aist#0:1/aac @ 0x2b944f80] Resumed reading at pts 4.971 with rate 1.050 after a lag of 3.302s [aist#0:1/aac @ 0x2b944f80] Resumed reading at pts 4.971 with rate 1.050 after a lag of 3.619s [vist#0:0/h264 @ 0x2b945480] Resumed reading at pts 6.000 with rate 1.050 after a lag of 3.078s [vist#0:0/h264 @ 0x2b945480] Resumed reading at pts 6.000 with rate 1.050 after a lag of 3.383s [aist#0:1/aac @ 0x2b944f80] Resumed reading at pts 5.973 with rate 1.050 after a lag of 3.919s [aist#0:1/aac @ 0x2b944f80] Resumed reading at pts 5.973 with rate 1.050 after a lag of 4.236s [vist#0:0/h264 @ 0x2b945480] Resumed reading at pts 7.000 with rate 1.050 after a lag of 3.699s [vist#0:0/h264 @ 0x2b945480] Resumed reading at pts 7.000 with rate 1.050 after a lag of 4.003s [aist#0:1/aac @ 0x2b944f80] Resumed reading at pts 6.976 with rate 1.050 after a lag of 4.537s [aist#0:1/aac @ 0x2b944f80] Resumed reading at pts 6.976 with rate 1.050 after a lag of 4.854s [vist#0:0/h264 @ 0x2b945480] Resumed reading at pts 8.000 with rate 1.050 after a lag of 4.318s [vist#0:0/h264 @ 0x2b945480] Resumed reading at pts 8.000 with rate 1.050 after a lag of 4.623s [aist#0:1/aac @ 0x2b944f80] Resumed reading at pts 7.979 with rate 1.050 after a lag of 5.154s [aist#0:1/aac @ 0x2b944f80] Resumed reading at pts 7.979 with rate 1.050 after a lag of 5.471s [vist#0:0/h264 @ 0x2b945480] Resumed reading at pts 9.000 with rate 1.050 after a lag of 4.939s [vist#0:0/h264 @ 0x2b945480] Resumed reading at pts 9.000 with rate 1.050 after a lag of 5.243s [aist#0:1/aac @ 0x2b944f80] Resumed reading at pts 8.981 with rate 1.050 after a lag of 5.771s [aist#0:1/aac @ 0x2b944f80] Resumed reading at pts 8.981 with rate 1.050 after a lag of 6.088s [vist#0:0/h264 @ 0x2b945480] Resumed reading at pts 10.000 with rate 1.050 after a lag of 5.559s [vist#0:0/h264 @ 0x2b945480] Resumed reading at pts 10.000 with rate 1.050 after a lag of 5.863s frame= 300 fps= 19 q=-1.0 Lsize=N/A time=00:00:10.03 bitrate=N/A speed=0.645x elapsed=0:00:15.56 ``` I'm not sure if I am allowed to share the sample file here. >From 5c648050c7198c6b829a9709820ccf5f066dc7dc Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Sun, 28 Jun 2026 14:27:28 +0200 Subject: [PATCH 1/3] fftools/ffmpeg_demux: skip finished/unstarted streams in readrate_sleep() This shouldn't affect the actual behavior, as the initialization of ds->dts (implicitly zero'd) and the previous calculation of stream_ts_offset guarantees that the `if (pts <= stream_ts_offset) continue;` branch fires. Mainly a minor clarification of the code for the upcoming refactor. Signed-off-by: Niklas Haas <[email protected]> --- fftools/ffmpeg_demux.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fftools/ffmpeg_demux.c b/fftools/ffmpeg_demux.c index 2d2dd5a82d..9287a31774 100644 --- a/fftools/ffmpeg_demux.c +++ b/fftools/ffmpeg_demux.c @@ -521,10 +521,10 @@ static void readrate_sleep(Demuxer *d) InputStream *ist = f->streams[i]; DemuxStream *ds = ds_from_ist(ist); int64_t stream_ts_offset, pts, now, wc_elapsed, elapsed, lag, max_pts, limit_pts; + if (ds->discard || ds->finished || ds->first_dts == AV_NOPTS_VALUE) + continue; - if (ds->discard) continue; - - stream_ts_offset = FFMAX(ds->first_dts != AV_NOPTS_VALUE ? ds->first_dts : 0, file_start); + stream_ts_offset = FFMAX(ds->first_dts, file_start); pts = av_rescale(ds->dts, 1000000, AV_TIME_BASE); now = av_gettime_relative(); wc_elapsed = now - d->wallclock_start; -- 2.52.0 >From 6c42209ae8b95d5cbc3bd4a7993db8c30122c1c5 Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Sun, 28 Jun 2026 14:43:14 +0200 Subject: [PATCH 2/3] fftools/ffmpeg_demux: remove unused variable This is a dead assignment except on a single branch, so just define it locally. Signed-off-by: Niklas Haas <[email protected]> --- fftools/ffmpeg_demux.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/fftools/ffmpeg_demux.c b/fftools/ffmpeg_demux.c index 9287a31774..5dbd14dc0a 100644 --- a/fftools/ffmpeg_demux.c +++ b/fftools/ffmpeg_demux.c @@ -520,7 +520,7 @@ static void readrate_sleep(Demuxer *d) for (int i = 0; i < f->nb_streams; i++) { InputStream *ist = f->streams[i]; DemuxStream *ds = ds_from_ist(ist); - int64_t stream_ts_offset, pts, now, wc_elapsed, elapsed, lag, max_pts, limit_pts; + int64_t stream_ts_offset, pts, now, wc_elapsed, lag, max_pts, limit_pts; if (ds->discard || ds->finished || ds->first_dts == AV_NOPTS_VALUE) continue; @@ -544,10 +544,9 @@ static void readrate_sleep(Demuxer *d) if (ds->lag && !lag) ds->lag = ds->resume_wc = ds->resume_pts = 0; if (ds->resume_wc) { - elapsed = now - ds->resume_wc; + int64_t elapsed = now - ds->resume_wc; limit_pts = ds->resume_pts + (int64_t)(elapsed * d->readrate_catchup); } else { - elapsed = wc_elapsed; limit_pts = max_pts; } -- 2.52.0 >From 9652c9cbb781eda2e1272ee8e4aac0869def53fd Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Sun, 28 Jun 2026 14:48:19 +0200 Subject: [PATCH 3/3] fftools/ffmpeg_demux: only throttle readrate on the slowest stream If streams are badly interleaved, then the readrate logic can end up accumulating an ever-growing lag. Rather than looping over each stream and sleeping for each stream individually based on the local DTS and lag logic, pull the sleep out of the loop and only sleep once based on the furthest-behind stream (i.e. the stream contributing the lowest sleep duration). To reproduce: $ ./ffmpeg -re -i fallbeatcaptiontest.mp4 -c copy -f null -t 10 - Before this commit, this would run at ~0.7x and accumulate an infinitely growing lag in one stream. After this commit, both streams run at ~1x as expected, after an initial burst period due to the bad (1s granularity) interleaving. Signed-off-by: Niklas Haas <[email protected]> --- fftools/ffmpeg_demux.c | 78 +++++++++++++++++++++++++----------------- 1 file changed, 46 insertions(+), 32 deletions(-) diff --git a/fftools/ffmpeg_demux.c b/fftools/ffmpeg_demux.c index 5dbd14dc0a..38bd580bfc 100644 --- a/fftools/ffmpeg_demux.c +++ b/fftools/ffmpeg_demux.c @@ -99,12 +99,6 @@ typedef struct DemuxStream { uint64_t nb_packets; // combined size of all the packets read uint64_t data_size; - // latest wallclock time at which packet reading resumed after a stall - used for readrate - int64_t resume_wc; - // timestamp of first packet sent after the latest stall - used for readrate - int64_t resume_pts; - // measure of how far behind packet reading is against spceified readrate - int64_t lag; } DemuxStream; typedef struct DemuxStreamGroup { @@ -147,6 +141,13 @@ typedef struct Demuxer { double readrate_initial_burst; float readrate_catchup; + // latest wallclock time at which packet reading resumed after a stall - used for readrate + int64_t resume_wc; + // relative timestamp of first packet sent after the latest stall - used for readrate + int64_t resume_progress; + // measure of how far behind packet reading is against spceified readrate + int64_t lag; + Scheduler *sch; AVPacket *pkt_heartbeat; @@ -517,42 +518,55 @@ static void readrate_sleep(Demuxer *d) int64_t initial_burst = AV_TIME_BASE * d->readrate_initial_burst; int resume_warn = 0; + DemuxStream *slowest = NULL; + int64_t progress = INT64_MAX; + for (int i = 0; i < f->nb_streams; i++) { InputStream *ist = f->streams[i]; DemuxStream *ds = ds_from_ist(ist); - int64_t stream_ts_offset, pts, now, wc_elapsed, lag, max_pts, limit_pts; + int64_t stream_ts_offset, pts, pts_diff; if (ds->discard || ds->finished || ds->first_dts == AV_NOPTS_VALUE) continue; stream_ts_offset = FFMAX(ds->first_dts, file_start); pts = av_rescale(ds->dts, 1000000, AV_TIME_BASE); - now = av_gettime_relative(); - wc_elapsed = now - d->wallclock_start; - - if (pts <= stream_ts_offset + initial_burst) continue; - - max_pts = stream_ts_offset + initial_burst + (int64_t)(wc_elapsed * d->readrate); - lag = FFMAX(max_pts - pts, 0); - if ( (!ds->lag && lag > 0.3 * AV_TIME_BASE) || ( lag > ds->lag + 0.3 * AV_TIME_BASE) ) { - ds->lag = lag; - ds->resume_wc = now; - ds->resume_pts = pts; - av_log_once(ds, AV_LOG_WARNING, AV_LOG_DEBUG, &resume_warn, - "Resumed reading at pts %0.3f with rate %0.3f after a lag of %0.3fs\n", - (float)pts/AV_TIME_BASE, d->readrate_catchup, (float)lag/AV_TIME_BASE); + pts_diff = pts - stream_ts_offset; + if (pts_diff < progress) { + progress = pts_diff; + slowest = ds; } - if (ds->lag && !lag) - ds->lag = ds->resume_wc = ds->resume_pts = 0; - if (ds->resume_wc) { - int64_t elapsed = now - ds->resume_wc; - limit_pts = ds->resume_pts + (int64_t)(elapsed * d->readrate_catchup); - } else { - limit_pts = max_pts; - } - - if (pts > limit_pts) - av_usleep(pts - limit_pts); } + + if (!slowest || progress <= initial_burst) + return; + + int64_t now = av_gettime_relative(); + int64_t wc_elapsed = now - d->wallclock_start; + int64_t max_prog = initial_burst + (int64_t)(wc_elapsed * d->readrate); + int64_t lag = FFMAX(max_prog - progress, 0); + int64_t limit; + + if ( (!d->lag && lag > 0.3 * AV_TIME_BASE) || ( lag > d->lag + 0.3 * AV_TIME_BASE) ) { + d->lag = lag; + d->resume_wc = now; + d->resume_progress = progress; + + int64_t pts = FFMAX(slowest->first_dts, file_start) + progress; + av_log_once(slowest, AV_LOG_WARNING, AV_LOG_DEBUG, &resume_warn, + "Resumed reading at pts %0.3f with rate %0.3f after a lag of %0.3fs\n", + (float)pts/AV_TIME_BASE, d->readrate_catchup, (float)lag/AV_TIME_BASE); + } + if (d->lag && !lag) + d->lag = d->resume_wc = d->resume_progress = 0; + if (d->resume_wc) { + int64_t elapsed = now - d->resume_wc; + limit = d->resume_progress + (int64_t)(elapsed * d->readrate_catchup); + } else { + limit = max_prog; + } + + if (progress > limit) + av_usleep(progress - limit); } static int do_send(Demuxer *d, DemuxStream *ds, AVPacket *pkt, unsigned flags, -- 2.52.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
