PR #23263 opened by birkedal URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23263 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23263.patch
An initial implementation of MoQ for FFmpeg using fragmented mp4 (fMP4). ### How to test MoQ locally 1. Clone https://github.com/moq-dev/moq 2. run `just relay` to run the moq-relay locally. 3. run `just web` to run the player locally, listening for streams to the local relay. 4. Build FFmpeg with support for MoQ `--enable-libmoq`, details in muxers.texi. 5. `ffmpeg -i INPUT -c copy -f moq http://localhost:4443/test/stream` 6. Check that it plays fine in the web player. ### Additional features that could make sense to implement and test 1. Support for MPEG-TS over MoQ 2. Configurable fragment periods, especially for audio 3. Untested with other media types like data tracks (scte35, time codes etc.) >From d61a2a1a041efd55612c2feab733c2f3fc371849 Mon Sep 17 00:00:00 2001 From: Ole Andre Birkedal <[email protected]> Date: Thu, 28 May 2026 15:27:15 +0200 Subject: [PATCH] Media over QUIC (MoQ) support for FFmpeg using fragmented mp4 --- MAINTAINERS | 1 + configure | 14 ++ doc/general_contents.texi | 8 + doc/muxers.texi | 57 +++++ libavformat/Makefile | 1 + libavformat/allformats.c | 1 + libavformat/moqenc.c | 455 ++++++++++++++++++++++++++++++++++++++ 7 files changed, 537 insertions(+) create mode 100644 libavformat/moqenc.c diff --git a/MAINTAINERS b/MAINTAINERS index 3bf41bd8cb..bb6d332356 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -494,6 +494,7 @@ Muxers/Demuxers: webm dash (matroskaenc.c) Vignesh Venkatasubramanian webvtt* Matthew J Heaney westwood.c Mike Melanson + moqenc.c [2] Ole Andre Birkedal whip.c [2] Jack Lau wtv.c Peter Ross diff --git a/configure b/configure index 1abf53b678..205e4cfc71 100755 --- a/configure +++ b/configure @@ -249,6 +249,7 @@ External library support: --enable-liblcevc-dec enable LCEVC decoding via liblcevc-dec [no] --enable-liblensfun enable lensfun lens correction [no] --enable-libmodplug enable ModPlug via libmodplug [no] + --enable-libmoq enable Media over QUIC output via libmoq [no] --enable-libmp3lame enable MP3 encoding via libmp3lame [no] --enable-libmpeghdec enable MPEG-H 3DA decoding via libmpeghdec [no] --enable-liboapv enable APV encoding via liboapv [no] @@ -2104,6 +2105,7 @@ EXTERNAL_LIBRARY_LIST=" liblc3 liblcevc_dec libmodplug + libmoq libmp3lame libmysofa liboapv @@ -3970,6 +3972,8 @@ mov_demuxer_select="iso_media riffdec" mov_demuxer_suggest="iamfdec zlib" mov_muxer_select="cbs_apv_lavf cbs_av1_lavf iso_media iso_writer riffenc rtpenc_chain vp9_superframe_bsf aac_adtstoasc_bsf ac3_parser" mov_muxer_suggest="iamfenc" +moq_muxer_deps="libmoq" +moq_muxer_select="mov_muxer" mp3_demuxer_select="mpegaudio_parser" mp3_muxer_select="mpegaudioheader" mp4_muxer_select="mov_muxer" @@ -7395,6 +7399,16 @@ if enabled libmfx; then fi enabled libmodplug && require_pkg_config libmodplug libmodplug libmodplug/modplug.h ModPlug_Load +if enabled libmoq; then + case $target_os in + darwin*) + require libmoq moq.h moq_session_connect -lmoq -framework CoreFoundation -framework Security + ;; + *) + require libmoq moq.h moq_session_connect -lmoq -lpthread -ldl -lm + ;; + esac +fi enabled libmp3lame && require "libmp3lame >= 3.98.3" lame/lame.h lame_set_VBR_quality -lmp3lame $libm_extralibs enabled libmpeghdec && require_pkg_config libmpeghdec "mpeghdec >= 3.0.0" mpeghdec/mpeghdecoder.h mpeghdecoder_init enabled libmysofa && { check_pkg_config libmysofa libmysofa mysofa.h mysofa_neighborhood_init_withstepdefine || diff --git a/doc/general_contents.texi b/doc/general_contents.texi index 5fed093642..d9043d84bd 100644 --- a/doc/general_contents.texi +++ b/doc/general_contents.texi @@ -199,6 +199,14 @@ Go to @url{http://www.webmproject.org/} and follow the instructions for installing the library. Then pass @code{--enable-libvpx} to configure to enable it. +@section libmoq + +FFmpeg can make use of the libmoq library for publishing live media streams +via Media over QUIC (MoQ). + +Go to @url{https://github.com/moq-dev/moq/tree/main/rs/libmoq} and follow the instructions for +building the library. Then pass @code{--enable-libmoq} to configure to enable it. + @section ModPlug FFmpeg can make use of this library, originating in Modplug-XMMS, to read from MOD-like music files. diff --git a/doc/muxers.texi b/doc/muxers.texi index 26199ad836..3f3fa8502c 100644 --- a/doc/muxers.texi +++ b/doc/muxers.texi @@ -2992,6 +2992,63 @@ assistants. This muxer accepts a single @samp{adpcm_yamaha} audio stream. +@section moq + +Media over QUIC (MoQ) muxer that publishes live media streams to a MoQ relay +using the libmoq library. MoQ is a low-latency media delivery protocol built +on top of QUIC and WebTransport, designed for real-time broadcasting with +sub-second latency. + +Each input stream is independently packaged into fragmented MP4 (fMP4) and +published as a separate MoQ track. Video streams produce one fragment per frame, +while audio streams are fragmented at 100ms intervals. + +To build with support for MoQ you need to configure FFmpeg with +@code{--enable-libmoq}. To get @{libmoq} you need to build it using +these instructions @url{https://github.com/moq-dev/moq/tree/main/rs/libmoq}. + +You could either use the generated @code{pkg-config} file to find the +library or set include and library paths explicitly: + +@example +./configure --enable-libmoq --extra-cflags=-I./moq/target/include/ + --extra-ldflags=-L./moq/target/release/ +@end example + +The output URL specifies the relay address with the broadcast path appended as +the URL path: + +@example +ffmpeg -i INPUT -f moq https://relay.example.com:4443/live/stream1 +@end example + +Here @code{https://relay.example.com:4443} is the relay and +@code{live/stream1} is the broadcast path that subscribers use to find +the stream. The broadcast path can alternatively be specified with the +@option{broadcast_path} option: + +@example +ffmpeg -i INPUT -f moq -broadcast_path live/stream1 https://relay.example.com:4443 +@end example + +Both forms above are equivalent. + +The default port is 443 for @code{https} or 4443 for @code{http} if not +specified explicitly. + +@subsection Options + +This muxer supports the following options: + +@table @option + +@item broadcast_path @var{string} +Set the broadcast path (e.g. @code{live/stream1}). When specified, this +overrides any path component in the output URL. Subscribers use this path to +find and consume the stream from the relay. + +@end table + @section mp3 The MP3 muxer writes a raw MP3 stream with the following optional features: diff --git a/libavformat/Makefile b/libavformat/Makefile index 33525369a4..c7395910e4 100644 --- a/libavformat/Makefile +++ b/libavformat/Makefile @@ -393,6 +393,7 @@ OBJS-$(CONFIG_MOV_MUXER) += movenc.o \ movenchint.o mov_chan.o rtp.o \ movenccenc.o movenc_ttml.o rawutils.o \ apv.o dovi_isom.o evc.o +OBJS-$(CONFIG_MOQ_MUXER) += moqenc.o OBJS-$(CONFIG_MP2_MUXER) += rawenc.o OBJS-$(CONFIG_MP3_DEMUXER) += mp3dec.o replaygain.o OBJS-$(CONFIG_MP3_MUXER) += mp3enc.o rawenc.o id3v2enc.o diff --git a/libavformat/allformats.c b/libavformat/allformats.c index af7eea5e5c..027d2287e3 100644 --- a/libavformat/allformats.c +++ b/libavformat/allformats.c @@ -289,6 +289,7 @@ extern const FFInputFormat ff_mods_demuxer; extern const FFInputFormat ff_moflex_demuxer; extern const FFInputFormat ff_mov_demuxer; extern const FFOutputFormat ff_mov_muxer; +extern const FFOutputFormat ff_moq_muxer; extern const FFOutputFormat ff_mp2_muxer; extern const FFInputFormat ff_mp3_demuxer; extern const FFOutputFormat ff_mp3_muxer; diff --git a/libavformat/moqenc.c b/libavformat/moqenc.c new file mode 100644 index 0000000000..e6433dfb76 --- /dev/null +++ b/libavformat/moqenc.c @@ -0,0 +1,455 @@ +/* + * Media over QUIC (MoQ) muxer + * Copyright (c) 2026 Ole Andre Birkedal + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "config.h" +#include "config_components.h" + +#include <moq.h> + +#include "libavutil/avstring.h" +#include "libavutil/mem.h" +#include "libavutil/opt.h" + +#include "avformat.h" +#include "mux.h" + +typedef struct MOQOutputStream { + AVFormatContext *ctx; + int ctx_inited; + + uint32_t track_id; + + uint8_t *init_data; + int init_data_size; + + int64_t last_input_dts; +} MOQOutputStream; + +typedef struct MOQContext { + const AVClass *class; + + /* libmoq handles */ + uint32_t session; + uint32_t origin; + uint32_t broadcast; + + char *broadcast_path; + + /* Parsed from output URL */ + char parsed_relay_url[1024]; + char parsed_broadcast_path[1024]; + + MOQOutputStream *streams; + int tracks_created; +} MOQContext; + +static void moq_session_status_callback(void *user_data, int32_t code) +{ + AVFormatContext *s = (AVFormatContext *)user_data; + if (code < 0) + av_log(s, AV_LOG_ERROR, "MoQ session error: %d\n", code); + else + av_log(s, AV_LOG_DEBUG, "MoQ session status: %d\n", code); +} + +static int moq_init_stream(AVFormatContext *s, MOQOutputStream *os, int stream_idx) +{ + AVFormatContext *ctx; + AVStream *in_st, *out_st; + AVDictionary *opts = NULL; + int ret; + + ctx = avformat_alloc_context(); + if (!ctx) + return AVERROR(ENOMEM); + + os->ctx = ctx; + + ctx->oformat = av_guess_format("mp4", NULL, NULL); + if (!ctx->oformat) { + av_log(s, AV_LOG_ERROR, "Could not find mp4 muxer\n"); + return AVERROR_MUXER_NOT_FOUND; + } + + ctx->interrupt_callback = s->interrupt_callback; + ctx->flags = s->flags; + ctx->avoid_negative_ts = AVFMT_AVOID_NEG_TS_DISABLED; + ctx->max_interleave_delta = 0; + ctx->flush_packets = 1; + + in_st = s->streams[stream_idx]; + out_st = avformat_new_stream(ctx, NULL); + if (!out_st) + return AVERROR(ENOMEM); + + ret = avcodec_parameters_copy(out_st->codecpar, in_st->codecpar); + if (ret < 0) + return ret; + + out_st->time_base = in_st->time_base; + out_st->sample_aspect_ratio = in_st->sample_aspect_ratio; + out_st->avg_frame_rate = in_st->avg_frame_rate; + out_st->codecpar->initial_padding = 0; + + ret = avio_open_dyn_buf(&ctx->pb); + if (ret < 0) + return ret; + + av_dict_set(&opts, "movflags", + "empty_moov+default_base_moof+omit_tfhd_offset+skip_trailer", + AV_DICT_APPEND); + if (in_st->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) { + av_dict_set(&opts, "movflags", "+frag_every_frame", AV_DICT_APPEND); + } else { + av_dict_set_int(&opts, "frag_duration", 100000, 0); + } + av_dict_set(&opts, "avoid_negative_ts", "disabled", 0); + av_dict_set(&opts, "use_editlist", "0", 0); + av_dict_set(&opts, "write_tmcd", "0", 0); + + ret = avformat_write_header(ctx, &opts); + av_dict_free(&opts); + if (ret < 0) { + av_log(s, AV_LOG_ERROR, "Stream %d: could not write fMP4 header: %s\n", + stream_idx, av_err2str(ret)); + return ret; + } + + os->init_data_size = avio_close_dyn_buf(ctx->pb, &os->init_data); + ctx->pb = NULL; + + if (os->init_data_size <= 0) { + av_log(s, AV_LOG_ERROR, "Stream %d: failed to extract init segment\n", stream_idx); + return AVERROR(EINVAL); + } + + ret = avio_open_dyn_buf(&ctx->pb); + if (ret < 0) + return ret; + + os->ctx_inited = 1; + return 0; +} + +static int moq_create_track(AVFormatContext *s, MOQOutputStream *os, int stream_idx) +{ + MOQContext *moq = s->priv_data; + static const char format[] = "fmp4"; + int ret; + + if (!os->init_data) + return AVERROR(EINVAL); + + // We currently just support fmp4 here, MPEG-TS could also be supported + ret = moq_publish_media_ordered(moq->broadcast, format, sizeof(format) - 1, os->init_data, os->init_data_size); + if (ret < 0) { + av_log(s, AV_LOG_ERROR, "Stream %d: failed to create MoQ track: %d\n", + stream_idx, ret); + return AVERROR_EXTERNAL; + } + + os->track_id = (uint32_t)ret; + + return 0; +} + +static int moq_parse_url(AVFormatContext *s) +{ + MOQContext *moq = s->priv_data; + const char *url = s->url; + char proto[10], authorization[256], hostname[256], path[1024]; + int port; + + if (!url || !*url) { + av_log(s, AV_LOG_ERROR, "No URL provided\n"); + return AVERROR(EINVAL); + } + + av_url_split(proto, sizeof(proto), + authorization, sizeof(authorization), + hostname, sizeof(hostname), + &port, + path, sizeof(path), + url); + + if (!*hostname) { + av_log(s, AV_LOG_ERROR, "Invalid URL format: %s\n", url); + av_log(s, AV_LOG_ERROR, "Expected format: http://relay:port/broadcast/path\n";); + return AVERROR(EINVAL); + } + + if (port > 0) { + snprintf(moq->parsed_relay_url, sizeof(moq->parsed_relay_url), + "%s://%s:%d", proto, hostname, port); + } else { + int default_port = (strcmp(proto, "https") == 0) ? 443 : 4443; + snprintf(moq->parsed_relay_url, sizeof(moq->parsed_relay_url), + "%s://%s:%d", proto, hostname, default_port); + } + + if (moq->broadcast_path) { + av_strlcpy(moq->parsed_broadcast_path, moq->broadcast_path, + sizeof(moq->parsed_broadcast_path)); + } else if (*path == '/') { + av_strlcpy(moq->parsed_broadcast_path, path + 1, sizeof(moq->parsed_broadcast_path)); + } else { + av_strlcpy(moq->parsed_broadcast_path, path, sizeof(moq->parsed_broadcast_path)); + } + + if (!*moq->parsed_broadcast_path) { + av_log(s, AV_LOG_ERROR, "No broadcast path specified in URL or via -broadcast_path\n"); + av_log(s, AV_LOG_ERROR, "Expected format: http://relay:port/broadcast/path\n";); + return AVERROR(EINVAL); + } + + av_log(s, AV_LOG_INFO, "Relay URL: %s\n", moq->parsed_relay_url); + av_log(s, AV_LOG_INFO, "Broadcast path: %s\n", moq->parsed_broadcast_path); + + return 0; +} + +static int moq_connect_relay(AVFormatContext *s) +{ + MOQContext *moq = s->priv_data; + const char *relay_url = moq->parsed_relay_url; + const char *broadcast_path = moq->parsed_broadcast_path; + int ret; + + av_log(s, AV_LOG_INFO, "Connecting to MoQ relay: %s\n", relay_url); + + ret = moq_origin_create(); + if (ret < 0) { + av_log(s, AV_LOG_ERROR, "Failed to create MoQ origin: %d\n", ret); + return AVERROR_EXTERNAL; + } + moq->origin = (uint32_t)ret; + + ret = moq_session_connect(relay_url, strlen(relay_url), moq->origin, 0, moq_session_status_callback, s); + if (ret < 0) { + av_log(s, AV_LOG_ERROR, "Failed to connect to MoQ relay: %d\n", ret); + moq_origin_close(moq->origin); + return AVERROR_EXTERNAL; + } + moq->session = (uint32_t)ret; + + ret = moq_publish_create(); + if (ret < 0) { + av_log(s, AV_LOG_ERROR, "Failed to create broadcast: %d\n", ret); + moq_session_close(moq->session); + moq_origin_close(moq->origin); + return AVERROR_EXTERNAL; + } + moq->broadcast = (uint32_t)ret; + + ret = moq_origin_publish(moq->origin, broadcast_path, strlen(broadcast_path), moq->broadcast); + if (ret < 0) { + av_log(s, AV_LOG_ERROR, "Failed to publish broadcast: %d\n", ret); + moq_publish_close(moq->broadcast); + moq_session_close(moq->session); + moq_origin_close(moq->origin); + return AVERROR_EXTERNAL; + } + + av_log(s, AV_LOG_INFO, "Connected to MoQ relay, broadcasting to: %s\n", broadcast_path); + return 0; +} + +static int moq_write_header(AVFormatContext *s) +{ + MOQContext *moq = s->priv_data; + int ret; + + moq->streams = av_calloc(s->nb_streams, sizeof(*moq->streams)); + if (!moq->streams) + return AVERROR(ENOMEM); + + for (unsigned int i = 0; i < s->nb_streams; i++) { + MOQOutputStream *os = &moq->streams[i]; + os->last_input_dts = AV_NOPTS_VALUE; + } + + ret = moq_parse_url(s); + if (ret < 0) + return ret; + + ret = moq_connect_relay(s); + if (ret < 0) + return ret; + + for (unsigned int i = 0; i < s->nb_streams; i++) { + ret = moq_init_stream(s, &moq->streams[i], i); + if (ret < 0) + return ret; + } + + for (unsigned int i = 0; i < s->nb_streams; i++) { + ret = moq_create_track(s, &moq->streams[i], i); + if (ret < 0) + return ret; + } + + moq->tracks_created = 1; + + return 0; +} + +static int moq_write_packet(AVFormatContext *s, AVPacket *pkt) +{ + MOQContext *moq = s->priv_data; + MOQOutputStream *os; + uint8_t *buf; + int size, ret; + + if (!moq->tracks_created) + return AVERROR(EINVAL); + + if (pkt->stream_index >= s->nb_streams) + return AVERROR(EINVAL); + + os = &moq->streams[pkt->stream_index]; + if (!os->ctx_inited) + return AVERROR(EINVAL); + + if (!pkt->duration && os->last_input_dts != AV_NOPTS_VALUE) + pkt->duration = pkt->dts - os->last_input_dts; + os->last_input_dts = pkt->dts; + + ret = ff_write_chained(os->ctx, 0, pkt, s, 0); + if (ret < 0) { + av_log(s, AV_LOG_ERROR, "Stream %d: ff_write_chained failed: %s\n", + pkt->stream_index, av_err2str(ret)); + return ret; + } + + if (avio_tell(os->ctx->pb) > 0) { + size = avio_close_dyn_buf(os->ctx->pb, &buf); + if (size > 0) { + ret = moq_publish_media_frame(os->track_id, buf, size, 0); + if (ret < 0) + av_log(s, AV_LOG_WARNING, "Stream %d: moq_publish_media_frame failed: %d (%d bytes, key=%d)\n", + pkt->stream_index, ret, size, + (pkt->flags & AV_PKT_FLAG_KEY) ? 1 : 0); + av_free(buf); + } + + ret = avio_open_dyn_buf(&os->ctx->pb); + if (ret < 0) + return ret; + } + + return 0; +} + +static int moq_write_trailer(AVFormatContext *s) +{ + MOQContext *moq = s->priv_data; + uint8_t *buf; + int size; + + av_log(s, AV_LOG_INFO, "MoQ muxer closing\n"); + + for (unsigned int i = 0; i < s->nb_streams; i++) { + MOQOutputStream *os = &moq->streams[i]; + + if (!os->ctx) + continue; + + av_write_frame(os->ctx, NULL); + av_write_trailer(os->ctx); + + if (os->ctx->pb) { + size = avio_close_dyn_buf(os->ctx->pb, &buf); + if (size > 0 && os->track_id > 0) + moq_publish_media_frame(os->track_id, buf, size, 0); + av_free(buf); + os->ctx->pb = NULL; + } + + avformat_free_context(os->ctx); + os->ctx = NULL; + } + + for (unsigned int i = 0; i < s->nb_streams; i++) { + if (moq->streams[i].track_id > 0) + moq_publish_media_close(moq->streams[i].track_id); + } + + if (moq->broadcast > 0) + moq_publish_close(moq->broadcast); + if (moq->session > 0) + moq_session_close(moq->session); + if (moq->origin > 0) + moq_origin_close(moq->origin); + + av_log(s, AV_LOG_INFO, "MoQ muxer closed\n"); + return 0; +} + +static void moq_deinit(AVFormatContext *s) +{ + MOQContext *moq = s->priv_data; + uint8_t *buf; + + if (moq->streams) { + for (unsigned int i = 0; i < s->nb_streams; i++) { + MOQOutputStream *os = &moq->streams[i]; + if (os->ctx) { + if (os->ctx->pb) { + avio_close_dyn_buf(os->ctx->pb, &buf); + av_free(buf); + } + avformat_free_context(os->ctx); + } + av_freep(&os->init_data); + } + av_freep(&moq->streams); + } +} + +#define OFFSET(x) offsetof(MOQContext, x) +#define ENC AV_OPT_FLAG_ENCODING_PARAM + +static const AVOption options[] = { + { "broadcast_path", "Broadcast path (overrides path parsed from output URL)", OFFSET(broadcast_path), AV_OPT_TYPE_STRING, {.str = NULL}, 0, 0, ENC }, + { NULL } +}; + +static const AVClass moq_muxer_class = { + .class_name = "moq muxer", + .item_name = av_default_item_name, + .option = options, + .version = LIBAVUTIL_VERSION_INT, +}; + +const FFOutputFormat ff_moq_muxer = { + .p.name = "moq", + .p.long_name = NULL_IF_CONFIG_SMALL("Media over QUIC (MoQ)"), + .p.extensions = "", + .priv_data_size = sizeof(MOQContext), + .p.audio_codec = AV_CODEC_ID_AAC, + .p.video_codec = AV_CODEC_ID_H264, + .p.flags = AVFMT_GLOBALHEADER | AVFMT_NOFILE | AVFMT_TS_NONSTRICT, + .p.priv_class = &moq_muxer_class, + .write_header = moq_write_header, + .write_packet = moq_write_packet, + .write_trailer = moq_write_trailer, + .deinit = moq_deinit, +}; -- 2.52.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
