PR #23588 opened by James Almer (jamrial) URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23588 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23588.patch
This new API attempts to solve the limitation of the current BSF API where filters can support only packets from one input and similar export packets meant for one output. Examples of scenarios that require it are splitting off certain embedded payloads within a packet, or the inverse, merging packets from separate streams into a single stream packet. It is heavily based on the avfilter graph API, adapted and simplified for packet needs. No existing filter is ported to it currently as they don't benefit from it, but it will be done eventually so they may be used inside a graph. A graph parser is also not included for now, but will once scheduler support is added to ffmpeg.c One filter using the new API is introduced, and custom support for it is added to ffmpeg.c while scheduler work is ongoing. The old API is not deprecated as it's much simpler for the common scenario of one in one out. The BSF packet list however will be deprecated and removed once all filters have been ported and a graph parser is introduced. >From 501952bab5d907cedba9b14b6e907614785450d4 Mon Sep 17 00:00:00 2001 From: James Almer <[email protected]> Date: Wed, 24 Jun 2026 08:24:01 -0300 Subject: [PATCH 1/3] avcodec: add a graph based bitstream filtering API Signed-off-by: James Almer <[email protected]> --- configure | 7 +- libavcodec/Makefile | 2 + libavcodec/bitstream_filters.c | 7 + libavcodec/bitstreamfilter.c | 883 +++++++++++++++++++++++++++++++++ libavcodec/bsf.c | 8 +- libavcodec/bsf.h | 383 ++++++++++++++ libavcodec/bsf/Makefile | 6 + libavcodec/bsf/filters.h | 204 ++++++++ libavcodec/bsf/sink.c | 148 ++++++ libavcodec/bsf/source.c | 224 +++++++++ libavcodec/bsf_internal.h | 193 +++++++ libavcodec/bsfgraph.c | 374 ++++++++++++++ 12 files changed, 2433 insertions(+), 6 deletions(-) create mode 100644 libavcodec/bitstreamfilter.c create mode 100644 libavcodec/bsf/filters.h create mode 100644 libavcodec/bsf/sink.c create mode 100644 libavcodec/bsf/source.c create mode 100644 libavcodec/bsfgraph.c diff --git a/configure b/configure index 2bf25e9755..e12cf3e6b5 100755 --- a/configure +++ b/configure @@ -4567,7 +4567,7 @@ find_things_extern(){ pattern=$2 file=$source_path/$3 out=${4:-$thing} - sed -n "s/^[^#]*extern.*$pattern *ff_\([^ ]*\)_$thing;/\1_$out/p" "$file" + sed -n "s/^[^#]*extern const.*$pattern *ff_\([^ ]*\)_$thing;/\1_$out/p" "$file" } find_filters_extern(){ @@ -8975,6 +8975,11 @@ print_enabled_components(){ printf " &ff_%s,\n" $c >> $TMPH done fi + if [ "$name" = "bitstream_filters" ]; then + for c in source_bsf sink_bsf; do + printf " &ff_%s,\n" $c >> $TMPH + done + fi echo " NULL };" >> $TMPH cp_if_changed $TMPH $file } diff --git a/libavcodec/Makefile b/libavcodec/Makefile index 95ba308504..5f00dd8762 100644 --- a/libavcodec/Makefile +++ b/libavcodec/Makefile @@ -34,8 +34,10 @@ OBJS = ac3_parser.o \ avdct.o \ packet.o \ bitstream.o \ + bitstreamfilter.o \ bitstream_filters.o \ bsf.o \ + bsfgraph.o \ codec_desc.o \ codec_par.o \ d3d11va.o \ diff --git a/libavcodec/bitstream_filters.c b/libavcodec/bitstream_filters.c index 9c9443e5b1..a9f3218f0f 100644 --- a/libavcodec/bitstream_filters.c +++ b/libavcodec/bitstream_filters.c @@ -49,6 +49,7 @@ extern const FFBitStreamFilter ff_hapqa_extract_bsf; extern const FFBitStreamFilter ff_hevc_metadata_bsf; extern const FFBitStreamFilter ff_hevc_mp4toannexb_bsf; extern const FFBitStreamFilter ff_imx_dump_header_bsf; +extern const FFBitStreamFilter ff_lcevc_merge_bsf; extern const FFBitStreamFilter ff_lcevc_metadata_bsf; extern const FFBitStreamFilter ff_media100_to_mjpegb_bsf; extern const FFBitStreamFilter ff_mjpeg2jpeg_bsf; @@ -76,6 +77,12 @@ extern const FFBitStreamFilter ff_vp9_superframe_split_bsf; extern const FFBitStreamFilter ff_vvc_metadata_bsf; extern const FFBitStreamFilter ff_vvc_mp4toannexb_bsf; +/* these filters are part of public or internal API, + * they are formatted to not be found by the grep + * as they are manually added */ +extern const FFBitStreamFilter ff_source_bsf; +extern const FFBitStreamFilter ff_sink_bsf; + #include "libavcodec/bsf_list.c" const AVBitStreamFilter *av_bsf_iterate(void **opaque) diff --git a/libavcodec/bitstreamfilter.c b/libavcodec/bitstreamfilter.c new file mode 100644 index 0000000000..82cc0bb1a1 --- /dev/null +++ b/libavcodec/bitstreamfilter.c @@ -0,0 +1,883 @@ +/* + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "config.h" + +#include <string.h> + +#include "libavutil/avassert.h" +#include "libavutil/mem.h" +#include "libavutil/opt.h" + +#include "bsf.h" +#include "bsf_internal.h" + +static const char *default_bsf_name(void *filter_ctx) +{ + AVBitStreamFilterContext *ctx = filter_ctx; + return ctx->name ? ctx->name : ctx->filter->name; +} + +static void *bsf_child_next(void *obj, void *prev) +{ + AVBitStreamFilterContext *ctx = obj; + if (!prev && ctx->filter && ctx->filter->priv_class && ctx->priv_data) + return ctx->priv_data; + return NULL; +} + +static const AVClass bitstreamfilter_class = { + .class_name = "AVBitStreamFilterContext", + .item_name = default_bsf_name, + .version = LIBAVUTIL_VERSION_INT, + .category = AV_CLASS_CATEGORY_BITSTREAM_FILTER, + .child_next = bsf_child_next, + .child_class_iterate = ff_bsf_child_class_iterate, + .state_flags_offset = offsetof(FFBitStreamFilterContext, state_flags), +}; + +AVBitStreamFilterContext *ff_bsf_alloc(const AVBitStreamFilter *filter, const char *inst_name) +{ + FFBitStreamFilterContext *ctx; + AVBitStreamFilterContext *ret; + const FFBitStreamFilter *const fi = ff_bsf(filter); + int preinited = 0; + + if (!filter) + return NULL; + + ctx = av_mallocz(sizeof(*ctx)); + if (!ctx) + return NULL; + ret = &ctx->p; + + ret->av_class = &bitstreamfilter_class; + ret->filter = filter; + ret->name = inst_name ? av_strdup(inst_name) : NULL; + if (fi->priv_data_size) { + ret->priv_data = av_mallocz(fi->priv_data_size); + if (!ret->priv_data) + goto err; + } + if (fi->preinit) { + if (fi->preinit(ret) < 0) + goto err; + preinited = 1; + } + + av_opt_set_defaults(ret); + if (filter->priv_class) { + *(const AVClass**)ret->priv_data = filter->priv_class; + av_opt_set_defaults(ret->priv_data); + } + + ret->nb_inputs = fi->nb_inputs; + if (ret->nb_inputs ) { + ret->input_pads = av_memdup(fi->inputs, ret->nb_inputs * sizeof(*fi->inputs)); + if (!ret->input_pads) + goto err; + ret->inputs = av_calloc(ret->nb_inputs, sizeof(*ret->inputs)); + if (!ret->inputs) + goto err; + } + + ret->nb_outputs = fi->nb_outputs; + if (ret->nb_outputs) { + ret->output_pads = av_memdup(fi->outputs, ret->nb_outputs * sizeof(*fi->outputs)); + if (!ret->output_pads) + goto err; + ret->outputs = av_calloc(ret->nb_outputs, sizeof(*ret->outputs)); + if (!ret->outputs) + goto err; + } + + return ret; + +err: + if (preinited) + fi->uninit(ret); + av_freep(&ret->name); + av_freep(&ret->inputs); + av_freep(&ret->input_pads); + ret->nb_inputs = 0; + av_freep(&ret->outputs); + av_freep(&ret->output_pads); + ret->nb_outputs = 0; + av_freep(&ret->priv_data); + av_free(ret); + return NULL; +} + +int av_bsf_link(AVBitStreamFilterContext *src, unsigned srcpad, + AVBitStreamFilterContext *dst, unsigned dstpad) +{ + BitStreamFilterLinkInternal *li; + AVBitStreamFilterLink *link; + + av_assert0(src->graph); + av_assert0(dst->graph); + av_assert0(src->graph == dst->graph); + + if (src->nb_outputs <= srcpad || dst->nb_inputs <= dstpad || + src->outputs[srcpad] || dst->inputs[dstpad]) + return AVERROR(EINVAL); + + if (!(ffbsfctx(src)->state_flags & AV_CLASS_STATE_INITIALIZED) || + !(ffbsfctx(dst)->state_flags & AV_CLASS_STATE_INITIALIZED)) { + av_log(src, AV_LOG_ERROR, "Filters must be initialized before linking.\n"); + return AVERROR(EINVAL); + } + + if (src->output_pads[srcpad].codec_ids && dst->input_pads[dstpad].codec_ids) { + int i; + for (i = 0; src->output_pads[srcpad].codec_ids[i] != AV_CODEC_ID_NONE; i++) + for (int j = 0; dst->input_pads[dstpad].codec_ids[j] != AV_CODEC_ID_NONE; j++) + if (src->output_pads[srcpad].codec_ids[i] == dst->input_pads[dstpad].codec_ids[j]) + break; + + if (src->output_pads->codec_ids[i] == AV_CODEC_ID_NONE) { + av_log(src, AV_LOG_ERROR, "No common codec id between source and dest pads\n"); + av_log(src, AV_LOG_ERROR, "Supported input pad codecs are: "); + for (i = 0; dst->input_pads->codec_ids[i] != AV_CODEC_ID_NONE; i++) { + enum AVCodecID codec_id = dst->input_pads->codec_ids[i]; + av_log(src, AV_LOG_ERROR, "%s (%d) ", + avcodec_get_name(codec_id), codec_id); + } + av_log(src, AV_LOG_ERROR, "\n"); + av_log(src, AV_LOG_ERROR, "Supported output pad codecs are: "); + for (i = 0; src->output_pads->codec_ids[i] != AV_CODEC_ID_NONE; i++) { + enum AVCodecID codec_id = src->output_pads->codec_ids[i]; + av_log(src, AV_LOG_ERROR, "%s (%d) ", + avcodec_get_name(codec_id), codec_id); + } + av_log(src, AV_LOG_ERROR, "\n"); + + return AVERROR(EINVAL); + } + } + + li = av_mallocz(sizeof(*li)); + if (!li) + return AVERROR(ENOMEM); + link = &li->l.pub; + + src->outputs[srcpad] = dst->inputs[dstpad] = link; + + link->src = src; + link->dst = dst; + link->srcpad = &src->output_pads[srcpad]; + link->dstpad = &dst->input_pads[dstpad]; + li->l.graph = src->graph; + li->fifo = av_container_fifo_alloc_avpacket(0); + if (!li->fifo) + return AVERROR(ENOMEM); + + return 0; +} + +static void link_free(AVBitStreamFilterLink **link) +{ + BitStreamFilterLinkInternal *li; + + if (!*link) + return; + li = ff_link_internal(*link); + + avcodec_parameters_free(&li->l.par); + av_container_fifo_free(&li->fifo); + + av_freep(link); +} + +static void free_link(AVBitStreamFilterLink *link) +{ + if (!link) + return; + + if (link->src) + link->src->outputs[link->srcpad - link->src->output_pads] = NULL; + if (link->dst) + link->dst->inputs[link->dstpad - link->dst->input_pads] = NULL; + + link_free(&link); +} + +void ff_bsf_free(AVBitStreamFilterContext *ctx) +{ + int i; + + if (!ctx) + return; + + if (ctx->graph) + ff_bsf_graph_remove_filter(ctx->graph, ctx); + + if (ff_bsf(ctx->filter)->uninit) + ff_bsf(ctx->filter)->uninit(ctx); + + for (i = 0; i < ctx->nb_inputs; i++) { + free_link(ctx->inputs[i]); + if (ctx->input_pads[i].flags & FF_BSF_PAD_FLAG_FREE_NAME) + av_freep(&ctx->input_pads[i].name); + } + for (i = 0; i < ctx->nb_outputs; i++) { + free_link(ctx->outputs[i]); + if (ctx->output_pads[i].flags & FF_BSF_PAD_FLAG_FREE_NAME) + av_freep(&ctx->output_pads[i].name); + } + + if (ctx->filter->priv_class) + av_opt_free(ctx->priv_data); + + av_freep(&ctx->name); + av_freep(&ctx->input_pads); + av_freep(&ctx->output_pads); + av_freep(&ctx->inputs); + av_freep(&ctx->outputs); + av_freep(&ctx->priv_data); + av_opt_free(ctx); + av_free(ctx); +} + +int ff_bsf_opt_parse(void *logctx, const AVClass *priv_class, + AVDictionary **options, const char *args) +{ + const AVOption *o = NULL; + int ret; + int offset= -1; + + if (!args) + return 0; + + while (*args) { + char *parsed_key, *value; + const char *key; + const char *shorthand = NULL; + int additional_flags = 0; + + if (priv_class && (o = av_opt_next(&priv_class, o))) { + if (o->type == AV_OPT_TYPE_CONST || o->offset == offset) + continue; + offset = o->offset; + shorthand = o->name; + } + + ret = av_opt_get_key_value(&args, "=", ":", + shorthand ? AV_OPT_FLAG_IMPLICIT_KEY : 0, + &parsed_key, &value); + if (ret < 0) { + if (ret == AVERROR(EINVAL)) + av_log(logctx, AV_LOG_ERROR, "No option name near '%s'\n", args); + else + av_log(logctx, AV_LOG_ERROR, "Unable to parse '%s': %s\n", args, + av_err2str(ret)); + return ret; + } + if (*args) + args++; + if (parsed_key) { + key = parsed_key; + additional_flags = AV_DICT_DONT_STRDUP_KEY; + priv_class = NULL; /* reject all remaining shorthand */ + } else { + key = shorthand; + } + + av_log(logctx, AV_LOG_DEBUG, "Setting '%s' to value '%s'\n", key, value); + + av_dict_set(options, key, value, + additional_flags | AV_DICT_DONT_STRDUP_VAL | AV_DICT_MULTIKEY); + } + + return 0; +} + +int av_bsf_init_dict(AVBitStreamFilterContext *ctx, AVDictionary **options) +{ + FFBitStreamFilterContext *ctxi = ffbsfctx(ctx); + int ret = 0; + + if (ctxi->state_flags & AV_CLASS_STATE_INITIALIZED) { + av_log(ctx, AV_LOG_ERROR, "Filter already initialized\n"); + return AVERROR(EINVAL); + } + + ret = av_opt_set_dict2(ctx, options, AV_OPT_SEARCH_CHILDREN); + if (ret < 0) { + av_log(ctx, AV_LOG_ERROR, "Error applying generic filter options.\n"); + return ret; + } + + if (ff_bsf(ctx->filter)->init2) + ret = ff_bsf(ctx->filter)->init2(ctx); + if (ret < 0) + return ret; + + ctxi->state_flags |= AV_CLASS_STATE_INITIALIZED; + + return 0; +} + +int av_bsf_init_str(AVBitStreamFilterContext *ctx, const char *args) +{ + AVDictionary *options = NULL; + const AVDictionaryEntry *e; + int ret = 0; + + if (args && *args) { + ret = ff_bsf_opt_parse(ctx, ctx->filter->priv_class, &options, args); + if (ret < 0) + goto fail; + } + + ret = av_bsf_init_dict(ctx, &options); + if (ret < 0) + goto fail; + + if ((e = av_dict_iterate(options, NULL))) { + av_log(ctx, AV_LOG_ERROR, "No such option: %s.\n", e->key); + ret = AVERROR_OPTION_NOT_FOUND; + goto fail; + } + +fail: + av_dict_free(&options); + + return ret; +} + +const char *av_bsf_pad_get_name(const AVBitStreamFilterPad *pads, int pad_idx) +{ + return pads[pad_idx].name; +} + +const enum AVCodecID *av_bsf_pad_get_codec_ids(const AVBitStreamFilterPad *pads, int pad_idx) +{ + return pads[pad_idx].codec_ids; +} + +static void update_link_current_pts(BitStreamFilterLinkInternal *li, int64_t pts) +{ + if (pts == AV_NOPTS_VALUE) + return; + li->l.current_pts = pts; + li->l.current_pts_us = av_rescale_q(pts, li->l.time_base, AV_TIME_BASE_Q); + if (li->l.graph && li->age_index >= 0) + ff_bsf_graph_update_heap(li->l.graph, li); +} + +void ff_bsf_set_ready(AVBitStreamFilterContext *filter, unsigned priority) +{ + FFBitStreamFilterContext *ctxi = ffbsfctx(filter); + ctxi->ready = FFMAX(ctxi->ready, priority); +} + +/** + * Clear packet_blocked_in on all outputs. + * This is necessary whenever something changes on input. + */ +static void filter_unblock(AVBitStreamFilterContext *filter) +{ + unsigned i; + + for (i = 0; i < filter->nb_outputs; i++) { + BitStreamFilterLinkInternal * const li = ff_link_internal(filter->outputs[i]); + li->packet_blocked_in = 0; + } +} + +void ff_bsf_link_set_in_status(AVBitStreamFilterLink *link, int status, int64_t pts) +{ + BitStreamFilterLinkInternal * const li = ff_link_internal(link); + + if (li->status_in == status) + return; + av_assert0(!li->status_in); + li->status_in = status; + li->status_in_pts = pts; + li->packet_wanted_out = 0; + li->packet_blocked_in = 0; + filter_unblock(link->dst); + ff_bsf_set_ready(link->dst, 200); +} + +/** + * Set the status field of a link from the destination filter. + * The pts should probably be left unset (AV_NOPTS_VALUE). + */ +static void link_set_out_status(AVBitStreamFilterLink *link, int status, int64_t pts) +{ + BitStreamFilterLinkInternal * const li = ff_link_internal(link); + + av_assert0(!li->packet_wanted_out); + av_assert0(!li->status_out); + li->status_out = status; + if (pts != AV_NOPTS_VALUE) + update_link_current_pts(li, pts); + filter_unblock(link->dst); + ff_bsf_set_ready(link->src, 200); +} + +int ff_bsf_config_links(AVBitStreamFilterContext *filter) +{ + int (*config_link)(AVBitStreamFilterLink *); + unsigned i; + int ret; + + for (i = 0; i < filter->nb_inputs; i ++) { + AVBitStreamFilterLink *link = filter->inputs[i]; + AVBitStreamFilterLink *inlink; + BitStreamFilterLinkInternal *li = ff_link_internal(link); + + if (!link) continue; + if (!link->src || !link->dst) { + av_log(filter, AV_LOG_ERROR, + "Not all input and output are properly linked (%d).\n", i); + return AVERROR(EINVAL); + } + + inlink = link->src->nb_inputs ? link->src->inputs[0] : NULL; + li->l.current_pts = + li->l.current_pts_us = AV_NOPTS_VALUE; + + switch (li->init_state) { + case AVLINK_INIT: + continue; + case AVLINK_STARTINIT: + av_log(filter, AV_LOG_INFO, "circular filter chain detected\n"); + return 0; + case AVLINK_UNINIT: + li->init_state = AVLINK_STARTINIT; + + if ((ret = ff_bsf_config_links(link->src)) < 0) + return ret; + + if (!(config_link = link->srcpad->config_props)) { + if (link->src->nb_inputs != 1) { + av_log(link->src, AV_LOG_ERROR, "Source filters and filters " + "with more than one input " + "must set config_props() " + "callbacks on all outputs\n"); + return AVERROR(EINVAL); + } + } + + if (inlink) { + av_assert0(!li->l.par && ff_bsf_link(inlink)->par); + li->l.par = avcodec_parameters_alloc(); + if (!li->l.par) + return AVERROR(ENOMEM); + ret = avcodec_parameters_copy(li->l.par, ff_bsf_link(inlink)->par); + if (ret < 0) + return ret; + } else { + av_assert0(!li->l.par); + li->l.par = avcodec_parameters_alloc(); + if (!li->l.par) + return AVERROR(ENOMEM); + } + + if (config_link && (ret = config_link(link)) < 0) { + av_log(link->src, AV_LOG_ERROR, + "Failed to configure output pad on %s\n", + link->src->name); + return ret; + } + + switch (li->l.par->codec_type) { + case AVMEDIA_TYPE_VIDEO: + if (!li->l.time_base.num && !li->l.time_base.den) + li->l.time_base = inlink ? ff_bsf_link(inlink)->time_base : AV_TIME_BASE_Q; + break; + + case AVMEDIA_TYPE_AUDIO: + if (inlink) { + if (!li->l.time_base.num && !li->l.time_base.den) + li->l.time_base = ff_bsf_link(inlink)->time_base; + } + + if (!li->l.time_base.num && !li->l.time_base.den) + li->l.time_base = (AVRational) {1, li->l.par->sample_rate}; + break; + } + + if (link->dstpad->codec_ids) { + int j; + for (j = 0; link->dstpad->codec_ids[j] != AV_CODEC_ID_NONE; j++) + if (li->l.par->codec_id == link->dstpad->codec_ids[j]) + break; + if (link->dstpad->codec_ids[j] == AV_CODEC_ID_NONE) { + av_log(link->dst, AV_LOG_ERROR, "Unsupported input codec id %s (%d). Supported input pad codecs are: ", + avcodec_get_name(li->l.par->codec_id), li->l.par->codec_id); + for (j = 0; link->dstpad->codec_ids[j] != AV_CODEC_ID_NONE; j++) { + enum AVCodecID codec_id = link->dstpad->codec_ids[j]; + av_log(link->dst, AV_LOG_ERROR, "%s (%d) ", + avcodec_get_name(codec_id), codec_id); + } + av_log(link->dst, AV_LOG_ERROR, "\n"); + return AVERROR(EINVAL); + } + } + + if ((config_link = link->dstpad->config_props)) + if ((ret = config_link(link)) < 0) { + av_log(link->dst, AV_LOG_ERROR, + "Failed to configure input pad on %s\n", + link->dst->name); + return ret; + } + + li->init_state = AVLINK_INIT; + } + } + + return 0; +} + +int ff_bsf_request_packet(AVBitStreamFilterLink *link) +{ + BitStreamFilterLinkInternal * const li = ff_link_internal(link); + + av_assert1(!fffilter(link->dst->filter)->activate); + if (li->status_out) + return li->status_out; + if (li->status_in) { + if (av_container_fifo_can_read(li->fifo)) { + av_assert1(!li->packet_wanted_out); + av_assert1(fffilterctx(link->dst)->ready >= 300); + return 0; + } else { + /* Acknowledge status change. Filters using ff_bsf_request_packet() will + handle the change automatically. Filters can also check the + status directly but none do yet. */ + link_set_out_status(link, li->status_in, li->status_in_pts); + return li->status_out; + } + } + li->packet_wanted_out = 1; + ff_bsf_set_ready(link->src, 100); + return 0; +} + +static int64_t guess_status_pts(AVBitStreamFilterContext *ctx, int status, AVRational link_time_base) +{ + unsigned i; + int64_t r = INT64_MAX; + + for (i = 0; i < ctx->nb_inputs; i++) { + BitStreamFilterLinkInternal * const li = ff_link_internal(ctx->inputs[i]); + if (li->status_out == status) + r = FFMIN(r, av_rescale_q(li->l.current_pts, li->l.time_base, link_time_base)); + } + if (r < INT64_MAX) + return r; + av_log(ctx, AV_LOG_WARNING, "EOF timestamp not reliable\n"); + for (i = 0; i < ctx->nb_inputs; i++) { + BitStreamFilterLinkInternal * const li = ff_link_internal(ctx->inputs[i]); + r = FFMIN(r, av_rescale_q(li->status_in_pts, li->l.time_base, link_time_base)); + } + if (r < INT64_MAX) + return r; + return AV_NOPTS_VALUE; +} + +static int request_packet_to_filter(AVBitStreamFilterLink *link) +{ + BitStreamFilterLinkInternal * const li = ff_link_internal(link); + int ret = -1; + + /* Assume the filter is blocked, let the method clear it if not */ + li->packet_blocked_in = 1; + if (link->srcpad->request_packet) + ret = link->srcpad->request_packet(link); + else if (link->src->inputs[0]) + ret = ff_bsf_request_packet(link->src->inputs[0]); + if (ret < 0) { + if (ret != AVERROR(EAGAIN) && ret != li->status_in) + ff_bsf_link_set_in_status(link, ret, guess_status_pts(link->src, ret, li->l.time_base)); + if (ret == AVERROR_EOF) + ret = 0; + } + return ret; +} + +static int default_filter_packet(AVBitStreamFilterLink *link, AVPacket *pkt) +{ + return ff_bsf_filter_packet(link->dst->outputs[0], pkt); +} + +static int filter_packet_framed(AVBitStreamFilterLink *link, AVPacket *pkt) +{ + BitStreamFilterLink *l = ff_bsf_link(link); + int (*filter)(AVBitStreamFilterLink *, AVPacket *); + AVBitStreamFilterPad *dst = link->dstpad; + int ret; + + if (!(filter = dst->filter)) + filter = default_filter_packet; + + if (dst->flags & FF_BSF_PAD_FLAG_NEEDS_WRITABLE) { + ret = av_packet_make_writable(pkt); + if (ret < 0) + goto fail; + } + + ret = filter(link, pkt); + l->packet_count_out++; + return ret; + +fail: + av_packet_free(&pkt); + return ret; +} + +int ff_bsf_filter_packet(AVBitStreamFilterLink *link, AVPacket *pkt) +{ + BitStreamFilterLinkInternal * const li = ff_link_internal(link); + int ret; + + li->packet_blocked_in = li->packet_wanted_out = 0; + li->l.packet_count_in++; + filter_unblock(link->dst); + ret = av_container_fifo_write(li->fifo, pkt, 0); + av_packet_free(&pkt); + if (ret < 0) + return ret; + ff_bsf_set_ready(link->dst, 300); + return 0; +} + +static int filter_packet_to_filter(AVBitStreamFilterLink *link) +{ + BitStreamFilterLinkInternal * const li = ff_link_internal(link); + AVPacket *pkt = NULL; + AVBitStreamFilterContext *dst = link->dst; + int ret; + + av_assert1(av_container_fifo_can_read(li->fifo)); + ret = ff_bsf_inlink_consume_packet(link, &pkt); + av_assert1(ret); + if (ret < 0) { + av_assert1(!pkt); + return ret; + } + + /* The filter will soon have received a new packet, that may allow it to + produce one or more: unblock its outputs. */ + filter_unblock(dst); + /* AVBitStreamFilterPad.filter_packet() expect packet_count_out to have the value + before the packet; filter_packet_framed() will re-increment it. */ + li->l.packet_count_out--; + ret = filter_packet_framed(link, pkt); + if (ret < 0 && ret != li->status_out) { + link_set_out_status(link, ret, AV_NOPTS_VALUE); + } else { + /* Run once again, to see if several packets were available, or if + the input status has also changed, or any other reason. */ + ff_bsf_set_ready(dst, 300); + } + return ret; +} + +static int forward_status_change(AVBitStreamFilterContext *filter, BitStreamFilterLinkInternal *li_in) +{ + AVBitStreamFilterLink *in = &li_in->l.pub; + unsigned out = 0, progress = 0; + int ret; + + av_assert0(!li_in->status_out); + if (!filter->nb_outputs) { + /* not necessary with the current API and sinks */ + return 0; + } + while (!li_in->status_out) { + BitStreamFilterLinkInternal *li_out = ff_link_internal(filter->outputs[out]); + + if (!li_out->status_in) { + progress++; + ret = request_packet_to_filter(filter->outputs[out]); + if (ret < 0) + return ret; + } + if (++out == filter->nb_outputs) { + if (!progress) { + /* Every output already closed: input no longer interesting. */ + link_set_out_status(in, li_in->status_in, li_in->status_in_pts); + return 0; + } + progress = 0; + out = 0; + } + } + ff_bsf_set_ready(filter, 200); + return 0; +} + +static int filter_activate_default(AVBitStreamFilterContext *ctx) +{ + int i, nb_eofs = 0; + + for (i = 0; i < ctx->nb_outputs; i++) + nb_eofs += ff_bsf_outlink_get_status(ctx->outputs[i]) == AVERROR_EOF; + if (ctx->nb_outputs && nb_eofs == ctx->nb_outputs) { + for (int j = 0; j < ctx->nb_inputs; j++) + ff_bsf_inlink_set_status(ctx->inputs[j], AVERROR_EOF); + return 0; + } + + for (i = 0; i < ctx->nb_inputs; i++) { + BitStreamFilterLinkInternal *li = ff_link_internal(ctx->inputs[i]); + if (av_container_fifo_can_read(li->fifo)) { + return filter_packet_to_filter(ctx->inputs[i]); + } + } + for (i = 0; i < ctx->nb_inputs; i++) { + BitStreamFilterLinkInternal * const li = ff_link_internal(ctx->inputs[i]); + if (li->status_in && !li->status_out) { + av_assert1(!av_container_fifo_can_read(li->fifo)); + return forward_status_change(ctx, li); + } + } + for (i = 0; i < ctx->nb_outputs; i++) { + BitStreamFilterLinkInternal * const li = ff_link_internal(ctx->outputs[i]); + if (li->packet_wanted_out && + !li->packet_blocked_in) { + return request_packet_to_filter(ctx->outputs[i]); + } + } + for (i = 0; i < ctx->nb_outputs; i++) { + BitStreamFilterLinkInternal * const li = ff_link_internal(ctx->outputs[i]); + if (li->packet_wanted_out) + return request_packet_to_filter(ctx->outputs[i]); + } + if (!ctx->nb_outputs) { + ff_bsf_inlink_request_packet(ctx->inputs[0]); + return 0; + } + return FFERROR_BSF_NOT_READY; +} + +int ff_bsf_activate(AVBitStreamFilterContext *ctx) +{ + FFBitStreamFilterContext *ctxi = ffbsfctx(ctx); + const FFBitStreamFilter *const fi = ff_bsf(ctx->filter); + int ret; + + ctxi->ready = 0; + ret = fi->activate ? fi->activate(ctx) : filter_activate_default(ctx); + if (ret == FFERROR_BSF_NOT_READY) + ret = 0; + return ret; +} + +int ff_bsf_inlink_acknowledge_status(AVBitStreamFilterLink *link, int *rstatus, int64_t *rpts) +{ + BitStreamFilterLinkInternal * const li = ff_link_internal(link); + *rpts = li->l.current_pts; + if (av_container_fifo_can_read(li->fifo)) + return *rstatus = 0; + if (li->status_out) + return *rstatus = li->status_out; + if (!li->status_in) + return *rstatus = 0; + *rstatus = li->status_out = li->status_in; + update_link_current_pts(li, li->status_in_pts); + *rpts = li->l.current_pts; + return 1; +} + +size_t ff_bsf_inlink_queued_packets(AVBitStreamFilterLink *link) +{ + BitStreamFilterLinkInternal * const li = ff_link_internal(link); + return av_container_fifo_can_read(li->fifo); +} + +int ff_bsf_inlink_check_available_packet(AVBitStreamFilterLink *link) +{ + BitStreamFilterLinkInternal * const li = ff_link_internal(link); + return av_container_fifo_can_read(li->fifo) > 0; +} + +static void consume_update(BitStreamFilterLinkInternal *li, const AVPacket *pkt) +{ + update_link_current_pts(li, pkt->pts); + li->l.packet_count_out++; +} + +int ff_bsf_inlink_consume_packet(AVBitStreamFilterLink *link, AVPacket **rpkt) +{ + BitStreamFilterLinkInternal * const li = ff_link_internal(link); + AVPacket *pkt; + + *rpkt = NULL; + if (!av_container_fifo_can_read(li->fifo)) + return 0; + + pkt = av_packet_alloc(); + if (!pkt) + return AVERROR(ENOMEM); + + av_container_fifo_read(li->fifo, pkt, 0); + consume_update(li, pkt); + *rpkt = pkt; + return 1; +} + +void ff_bsf_inlink_request_packet(AVBitStreamFilterLink *link) +{ + BitStreamFilterLinkInternal *li = ff_link_internal(link); + av_assert1(!li->status_in); + av_assert1(!li->status_out); + li->packet_wanted_out = 1; + ff_bsf_set_ready(link->src, 100); +} + +void ff_bsf_inlink_set_status(AVBitStreamFilterLink *link, int status) +{ + BitStreamFilterLinkInternal * const li = ff_link_internal(link); + if (li->status_out) + return; + li->packet_wanted_out = 0; + li->packet_blocked_in = 0; + link_set_out_status(link, status, AV_NOPTS_VALUE); + while (av_container_fifo_can_read(li->fifo)) { + AVPacket pkt; + av_container_fifo_read(li->fifo, &pkt, 0); + av_packet_unref(&pkt); + } + if (!li->status_in) + li->status_in = status; +} + +int ff_bsf_outlink_get_status(AVBitStreamFilterLink *link) +{ + BitStreamFilterLinkInternal * const li = ff_link_internal(link); + return li->status_in; +} + +int ff_bsf_outlink_packet_wanted(AVBitStreamFilterLink *link) +{ + BitStreamFilterLinkInternal * const li = ff_link_internal(link); + return li->packet_wanted_out; +} + +const AVBitStreamFilterPad ff_default_bsf_pad[1] = { + { + .name = "default", + } +}; diff --git a/libavcodec/bsf.c b/libavcodec/bsf.c index 1e710f7d4a..74d767ee7c 100644 --- a/libavcodec/bsf.c +++ b/libavcodec/bsf.c @@ -33,11 +33,6 @@ #include "codec_par.h" #include "packet_internal.h" -static av_always_inline const FFBitStreamFilter *ff_bsf(const AVBitStreamFilter *bsf) -{ - return (const FFBitStreamFilter*)bsf; -} - typedef struct FFBSFContext { AVBSFContext pub; AVPacket *buffer_pkt; @@ -107,6 +102,9 @@ int av_bsf_alloc(const AVBitStreamFilter *filter, AVBSFContext **pctx) FFBSFContext *bsfi; int ret; + if (!ff_bsf(filter)->filter) + return AVERROR(ENOTSUP); + bsfi = av_mallocz(sizeof(*bsfi)); if (!bsfi) return AVERROR(ENOMEM); diff --git a/libavcodec/bsf.h b/libavcodec/bsf.h index a09c69f242..33c2530ab8 100644 --- a/libavcodec/bsf.h +++ b/libavcodec/bsf.h @@ -326,6 +326,389 @@ int av_bsf_list_parse_str(const char *str, AVBSFContext **bsf); int av_bsf_get_null_filter(AVBSFContext **bsf); /** + * @defgroup lavc_bsfgraph Bitstream filter graph + * Graph-based API for bitstream filters. + * @{ + */ + +typedef struct AVBitStreamFilterLink AVBitStreamFilterLink; +typedef struct AVBitStreamFilterPad AVBitStreamFilterPad; + +/** An instance of a filter */ +typedef struct AVBitStreamFilterContext { + /** + * A class for logging and AVOptions + */ + const AVClass *av_class; + + /** + * The bitstream filter this context is an instance of. + */ + const struct AVBitStreamFilter *filter; + + /** + * name of this filter instance + */ + char *name; + + AVBitStreamFilterPad *input_pads; ///< array of input pads + AVBitStreamFilterLink **inputs; ///< array of pointers to input links + unsigned nb_inputs; ///< number of input pads + + AVBitStreamFilterPad *output_pads; ///< array of output pads + AVBitStreamFilterLink **outputs; ///< array of pointers to output links + unsigned nb_outputs; ///< number of output pads + + /** + * Opaque filter-specific private data. If filter->priv_class is non-NULL, + * this is an AVOptions-enabled struct. + */ + void *priv_data; + + /** + * filtergraph this filter belongs to + */ + struct AVBitStreamFilterGraph *graph; +} AVBitStreamFilterContext; + +/** + * The number of the filter inputs is not determined just by the filter's static + * inputs. The filter might add additional inputs during initialization depending + * on the options supplied to it. + */ +#define AV_BSF_FLAG_DYNAMIC_INPUTS (1 << 0) +/** + * The number of the filter outputs is not determined just by the filter's static + * outputs. The filter might add additional outputs during initialization depending + * on the options supplied to it. + */ +#define AV_BSF_FLAG_DYNAMIC_OUTPUTS (1 << 1) +/** + * The filter is a "metadata" filter - it does not modify the packet data in any + * way. It may only affect the metadata (i.e. those fields copied by + * av_packet_copy_props()). + * + * More precisely, this means that the data of any packet output by the filter + * must be exactly equal to some packet that is received on one of its inputs. + * Furthermore, all packets produced on a given output must correspond to packet + * received on the same input and their order must be unchanged. + * Note that the filter may still drop or duplicate the frames. + */ +#define AV_BSF_FLAG_METADATA_ONLY (1 << 2) + +/** + * A link between two filters. This contains pointers to the source and + * destination filters between which this link exists, and the indexes of + * the pads involved. In addition, this link also contains the parameters + * which have been negotiated and agreed upon between the filter, such as + * image dimensions, format, etc. + * + * Applications must not normally access the link structure directly. + * Use the buffersrc and buffersink API instead. + * In the future, access to the header may be reserved for filters + * implementation. + */ +struct AVBitStreamFilterLink { + AVBitStreamFilterContext *src; ///< source filter + AVBitStreamFilterPad *srcpad; ///< output pad on the source filter + + AVBitStreamFilterContext *dst; ///< dest filter + AVBitStreamFilterPad *dstpad; ///< input pad on the dest filter +}; + +/** + * Get the name of an AVBitStreamFilterPad. + * + * @param pads an array of AVBitStreamFilterPads + * @param pad_idx index of the pad in the array; it is the caller's + * responsibility to ensure the index is valid + * + * @return name of the pad_idx'th pad in pads + */ +const char *av_bsf_pad_get_name(const AVBitStreamFilterPad *pads, int pad_idx); + +/** + * Get the codec ids supported by an AVBitStreamFilterPad. + * + * @param pads an array of AVBitStreamFilterPads + * @param pad_idx index of the pad in the array; it is the caller's + * responsibility to ensure the index is valid + * + * @return an array of AVCodecID terminated by AV_CODEC_ID_NONE, or NULL + * if the pad has no codec id constrains. + */ +const enum AVCodecID *av_bsf_pad_get_codec_ids(const AVBitStreamFilterPad *pads, int pad_idx); + +/** + * Link two filters together. + * + * @param src the source filter + * @param srcpad index of the output pad on the source filter + * @param dst the destination filter + * @param dstpad index of the input pad on the destination filter + * @return zero on success + */ +int av_bsf_link(AVBitStreamFilterContext *src, unsigned srcpad, + AVBitStreamFilterContext *dst, unsigned dstpad); + +/** + * Initialize a filter with the supplied parameters. + * + * @param ctx uninitialized filter context to initialize + * @param args Options to initialize the filter with. This must be a + * ':'-separated list of options in the 'key=value' form. + * May be NULL if the options have been set directly using the + * AVOptions API or there are no options that need to be set. + * @return 0 on success, a negative AVERROR on failure + */ +int av_bsf_init_str(AVBitStreamFilterContext *ctx, const char *args); + +/** + * Initialize a filter with the supplied dictionary of options. + * + * @param ctx uninitialized filter context to initialize + * @param options An AVDictionary filled with options for this filter. On + * return this parameter will be destroyed and replaced with + * a dict containing options that were not found. This dictionary + * must be freed by the caller. + * May be NULL, then this function is equivalent to + * av_bsf_init_str() with the second parameter set to NULL. + * @return 0 on success, a negative AVERROR on failure + * + * @note This function and av_bsf_init_str() do essentially the same thing, + * the difference is in manner in which the options are passed. It is up to the + * calling code to choose whichever is more preferable. The two functions also + * behave differently when some of the provided options are not declared as + * supported by the filter. In such a case, av_bsf_init_str() will fail, but + * this function will leave those extra options in the options AVDictionary and + * continue as usual. + */ +int av_bsf_init_dict(AVBitStreamFilterContext *ctx, AVDictionary **options); + +typedef struct AVBitStreamFilterGraph { + const AVClass *av_class; + + AVBitStreamFilterContext **filters; + + unsigned nb_filters; +} AVBitStreamFilterGraph; + +/** + * Allocate a filter graph. + * + * @return the allocated filter graph on success or NULL. + */ +AVBitStreamFilterGraph *av_bsf_graph_alloc(void); + +/** + * Create a new filter instance in a filter graph. + * + * @param graph graph in which the new filter will be used + * @param filter the filter to create an instance of + * @param name Name to give to the new instance (will be copied to + * AVBitStreamFilterContext.name). This may be used by the caller to + * identify different filters, libavcodec itself assigns no semantics + * to this parameter. May be NULL. + * + * @return the context of the newly created filter instance (note that it is + * also retrievable directly through AVBitStreamFilterGraph.filters or with + * av_bsf_graph_get_filter()) on success or NULL on failure. + */ +AVBitStreamFilterContext *av_bsf_graph_alloc_filter(AVBitStreamFilterGraph *graph, + const AVBitStreamFilter *filter, + const char *name); + +/** + * Get a filter instance identified by instance name from graph. + * + * @param graph filter graph to search through. + * @param name filter instance name (should be unique in the graph). + * @return the pointer to the found filter instance or NULL if it + * cannot be found. + */ +AVBitStreamFilterContext *av_bsf_graph_get_filter(AVBitStreamFilterGraph *graph, const char *name); + +/** + * A convenience wrapper that allocates and initializes a filter in a single + * step. The filter instance is created from the filter filt and inited with the + * parameter args. opaque is currently ignored. + * + * In case of success put in *filt_ctx the pointer to the created + * filter instance, otherwise set *filt_ctx to NULL. + * + * @param name the instance name to give to the created filter instance + * @param graph_ctx the filter graph + * @return a negative AVERROR error code in case of failure, a non + * negative value otherwise + * + * @warning Since the filter is initialized after this function successfully + * returns, you MUST NOT set any further options on it. If you need to + * do that, call ::av_bsf_graph_alloc_filter(), followed by setting + * the options, followed by ::av_bsf_init_dict() instead of this + * function. + */ +int av_bsf_graph_create_filter(AVBitStreamFilterContext **filt_ctx, const AVBitStreamFilter *filt, + const char *name, AVDictionary **options, AVBitStreamFilterGraph *graph_ctx); + +/** + * Check validity and configure all the links and formats in the graph. + * + * @param graphctx the filter graph + * @param log_ctx context used for logging + * @return >= 0 in case of success, a negative AVERROR code otherwise + */ +int av_bsf_graph_config(AVBitStreamFilterGraph *graphctx, void *log_ctx); + +/** + * Request a packet on the oldest sink link. + * + * If the request returns AVERROR_EOF, try the next. + * + * Note that this function is not meant to be the sole scheduling mechanism + * of a filtergraph, only a convenience function to help drain a filtergraph + * in a balanced way under normal circumstances. + * + * Also note that AVERROR_EOF does not mean that packets did not arrive on + * some of the sinks during the process. + * When there are multiple sink links, in case the requested link + * returns an EOF, this may cause a filter to flush pending packets + * which are sent to another sink link, although unrequested. + * + * @return the return value of ff_request_packet(), + * or AVERROR_EOF if all links returned AVERROR_EOF + */ +int av_bsf_graph_request_oldest(AVBitStreamFilterGraph *graph); + +/** + * Free a graph, destroy its links, and set *graph to NULL. + * If *graph is NULL, do nothing. + */ +void av_bsf_graph_free(AVBitStreamFilterGraph **graph); + +/** + * @defgroup lavc_bsfgraph_source Packet source API + * @{ + */ + +enum { + /** + * Immediately push the packet to the output. + */ + AV_BSF_SOURCE_FLAG_PUSH = 1 << 0, + + /** + * Keep a reference to the packet. + */ + AV_BSF_SOURCE_FLAG_KEEP_REF = 1 << 1, +}; + +/** + * Initialize the source filter with the provided parameters. + * This function may be called multiple times, the later calls override the + * previous ones. Some of the parameters may also be set through AVOptions, then + * whatever method is used last takes precedence. + * + * @param ctx an instance of the source filter + * @param param the stream parameters. The packet later passed to this filter + * must conform to those parameters. All the allocated fields in + * param remain owned by the caller, libavcodec will make internal + * copies or references when necessary. + * @return 0 on success, a negative AVERROR code on failure. + */ +int av_bsf_source_parameters_set(AVBitStreamFilterContext *ctx, const AVCodecParameters *par); + +/** + * Add a packet to the buffer source. + * + * By default, this function will take ownership of the reference(s) and reset + * the packet. This can be controlled using the flags. + * + * If this function returns an error, the input packet is not touched. + * + * @param buffer_src pointer to a source filter context + * @param packet a packet, or NULL to mark EOF + * @param flags a combination of AV_BSF_FLAG_* + * @return >= 0 in case of success, a negative AVERROR code + * in case of failure + */ +av_warn_unused_result +int av_bsf_source_add_packet(AVBitStreamFilterContext *ctx, AVPacket *pkt, int flags); + +/** + * Returns 0 or a negative AVERROR code. Currently, this will only ever + * return AVERROR(EOF), to indicate that the buffer source has been closed, + * either as a result of av_bsf_source_close(), or because the downstream + * filter is no longer accepting new data. + */ +int av_bsf_source_get_status(AVBitStreamFilterContext *ctx); + +/** + * Get the number of failed requests. + * + * A failed request is when the request_packet method is called while no + * packet is present in the buffer. + * The number is reset when a packet is added. + */ +unsigned av_bsf_source_get_nb_failed_requests(AVBitStreamFilterContext *src); + +/** + * Close the source after EOF. + * + * This is similar to passing NULL to av_bsf_source_add_packet() + * except it takes the timestamp of the EOF, i.e. the timestamp of the end + * of the last packet. + */ +int av_bsf_source_close(AVBitStreamFilterContext *ctx, int64_t pts, unsigned flags); + +/** + * @} + */ + +/** + * @defgroup lavc_bsfgraph_sink Packet sink API + * @{ + * + * The sink filter is there to connect filter graphs to applications + * They have a single input, connected to the graph, and no output. + * Packets must be extracted using av_bsf_sink_get_packet(). + */ + +enum { + /** + * Tell av_buffersink_get_buffer_ref() to read video/samples buffer + * reference, but not remove it from the buffer. This is useful if you + * need only to read a video/samples buffer, without to fetch it. + */ + AV_BSF_SINK_FLAG_PEEK = 1 << 0, + + /** + * Tell av_bsf_sink_get_packet() not to request a packet from its input. + * If a packet is already buffered, it is read (and removed from the buffer), + * but if no packet is present, return AVERROR(EAGAIN). + */ + AV_BSF_SINK_FLAG_NO_REQUEST = 1 << 1, +}; + +/** + * Get a packet with filtered data from sink and put it in packet. + * + * @param ctx pointer to a buffersink or abuffersink filter context. + * @param packet pointer to an allocated packet that will be filled with data. + * The data must be freed using av_packet_unref() / av_packet_free() + * @param flags a combination of AV_BSF_SINK_FLAG_* flags + * + * @return >= 0 in for success, a negative AVERROR code for failure. + */ +int av_bsf_sink_get_packet(AVBitStreamFilterContext *ctx, AVPacket *pkt, int flags); + +AVRational av_bsf_sink_get_time_base(const AVBitStreamFilterContext *ctx); +const AVCodecParameters *av_bsf_sink_get_parameters(const AVBitStreamFilterContext *ctx); + +/** + * @} + * + * @} + * * @} */ diff --git a/libavcodec/bsf/Makefile b/libavcodec/bsf/Makefile index 981bb8276b..bd91eda83c 100644 --- a/libavcodec/bsf/Makefile +++ b/libavcodec/bsf/Makefile @@ -1,6 +1,11 @@ clean:: $(RM) $(CLEANSUFFIXES:%=libavcodec/bsf/%) +OBJS += \ + bsf/packetsync.o \ + bsf/source.o \ + bsf/sink.o + OBJS-$(CONFIG_AAC_ADTSTOASC_BSF) += bsf/aac_adtstoasc.o OBJS-$(CONFIG_AHX_TO_MP2_BSF) += bsf/ahx_to_mp2.o OBJS-$(CONFIG_APV_METADATA_BSF) += bsf/apv_metadata.o @@ -26,6 +31,7 @@ OBJS-$(CONFIG_DOVI_RPU_BSF) += bsf/dovi_rpu.o OBJS-$(CONFIG_DOVI_SPLIT_BSF) += bsf/dovi_split.o OBJS-$(CONFIG_HEVC_MP4TOANNEXB_BSF) += bsf/hevc_mp4toannexb.o OBJS-$(CONFIG_IMX_DUMP_HEADER_BSF) += bsf/imx_dump_header.o +OBJS-$(CONFIG_LCEVC_MERGE_BSF) += bsf/lcevc_merge.o OBJS-$(CONFIG_LCEVC_METADATA_BSF) += bsf/lcevc_metadata.o OBJS-$(CONFIG_MEDIA100_TO_MJPEGB_BSF) += bsf/media100_to_mjpegb.o OBJS-$(CONFIG_MJPEG2JPEG_BSF) += bsf/mjpeg2jpeg.o diff --git a/libavcodec/bsf/filters.h b/libavcodec/bsf/filters.h new file mode 100644 index 0000000000..4b744e6390 --- /dev/null +++ b/libavcodec/bsf/filters.h @@ -0,0 +1,204 @@ +/* + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef AVCODEC_BSF_FILTERS_H +#define AVCODEC_BSF_FILTERS_H + +#include "libavutil/log.h" +#include "libavutil/rational.h" + +#include "libavcodec/bsf.h" +#include "libavcodec/codec_par.h" +#include "libavcodec/packet.h" + +/** + * Special return code when activate() did not do anything. + */ +#define FFERROR_BSF_NOT_READY FFERRTAG('N','R','D','Y') +#define FFERROR_SOURCE_EMPTY FFERRTAG('M','P','T','Y') + +struct AVBitStreamFilterPad { + /** + * Pad name. The name is unique among inputs and among outputs, but an + * input may have the same name as an output. This may be NULL if this + * pad has no need to ever be referenced by name. + */ + const char *name; + + /** + * A list of codec ids supported by the pad, terminated by + * AV_CODEC_ID_NONE. + * May be NULL, in that case the pad works with any codec id. + */ + const enum AVCodecID *codec_ids; + + /** + * The filter expects writable packets from its input link, + * duplicating data buffers if needed. + * + * input pads only. + */ +#define FF_BSF_PAD_FLAG_NEEDS_WRITABLE (1 << 0) + + /** + * The pad's name is allocated and should be freed generically. + */ +#define FF_BSF_PAD_FLAG_FREE_NAME (1 << 1) + + /** + * A combination of FF_BSF_PAD_FLAG_* flags. + */ + int flags; + + /** + * Filtering callback. This is where a filter receives a packet with + * audio/video data and should do its processing. + * + * Input pads only. + * + * @return >= 0 on success, a negative AVERROR on error. This function + * must ensure that packet is properly unreferenced on error if it + * hasn't been passed on to another filter. + */ + int (*filter)(AVBitStreamFilterLink *link, AVPacket *pkt); + + /** + * Packet request callback. A call to this should result in some progress + * towards producing output over the given link. This should return zero + * on success, and another value on error. + * + * Output pads only. + */ + int (*request_packet)(AVBitStreamFilterLink *link); + + /** + * Link configuration callback. + * + * For output pads, this should set the link properties such as + * width/height. + * + * For input pads, this should check the properties of the link, and update + * the filter's internal state as necessary. + * + * For both input and output filters, this should return zero on success, + * and another value on error. + */ + int (*config_props)(AVBitStreamFilterLink *link); +}; + +typedef struct BitStreamFilterLink { + AVBitStreamFilterLink pub; + + /** + * Graph the filter belongs to. + */ + struct AVBitStreamFilterGraph *graph; + + + AVCodecParameters *par; + + AVRational time_base; + + /** + * Current timestamp of the link, as defined by the most recent + * packet(s), in link time_base units. + */ + int64_t current_pts; + + /** + * Current timestamp of the link, as defined by the most recent + * packet(s), in AV_TIME_BASE units. + */ + int64_t current_pts_us; + + /** + * Number of past packets sent through the link. + */ + int64_t packet_count_in, packet_count_out; +} BitStreamFilterLink; + +static inline BitStreamFilterLink* ff_bsf_link(AVBitStreamFilterLink *link) +{ + return (BitStreamFilterLink*)link; +} + +#define BSFILTER_INOUTPADS(inout, array) \ + .inout = array, \ + .nb_ ## inout = FF_ARRAY_ELEMS(array) +#define BSFILTER_INPUTS(array) BSFILTER_INOUTPADS(inputs, (array)) +#define BSFILTER_OUTPUTS(array) BSFILTER_INOUTPADS(outputs, (array)) + +extern const AVBitStreamFilterPad ff_default_bsf_pad[1]; + +#define BSF_DEFINE_CLASS_EXT(name, desc, options) \ + static const AVClass name##_class = { \ + .class_name = desc, \ + .item_name = av_default_item_name, \ + .option = options, \ + .version = LIBAVUTIL_VERSION_INT, \ + .category = AV_CLASS_CATEGORY_BITSTREAM_FILTER, \ + } +#define BSF_DEFINE_CLASS(fname) \ + BSF_DEFINE_CLASS_EXT(fname, #fname, fname##_options) + +/** + * Mark a filter ready and schedule it for activation. + * + * This is automatically done when something happens to the filter (queued + * packet, status change, request on output). + * Filters implementing the activate callback can call it directly to + * perform one more round of processing later. + * It is also useful for filters reacting to external or asynchronous + * events. + */ +void ff_bsf_set_ready(AVBitStreamFilterContext *filter, unsigned priority); + +/** + * Send a packet of data to the next filter. + * + * @param link the output link over which the data is being sent + * @param pkt a reference to the buffer of data being sent. The + * receiving filter will free this reference when it no longer + * needs it or pass it on to the next filter. + * + * @return >= 0 on success, a negative AVERROR on error. The receiving filter + * is responsible for unreferencing pkt in case of error. + */ +int ff_bsf_filter_packet(AVBitStreamFilterLink *link, AVPacket *pkt); + +int ff_bsf_request_packet(AVBitStreamFilterLink *link); + +int ff_bsf_inlink_consume_packet(AVBitStreamFilterLink *link, AVPacket **pkt); + +void ff_bsf_inlink_request_packet(AVBitStreamFilterLink *link); + +int ff_bsf_inlink_acknowledge_status(AVBitStreamFilterLink *link, int *rstatus, int64_t *rpts); + +size_t ff_bsf_inlink_queued_packets(AVBitStreamFilterLink *link); + +int ff_bsf_inlink_check_available_packet(AVBitStreamFilterLink *link); + +void ff_bsf_inlink_set_status(AVBitStreamFilterLink *link, int status); + +void ff_bsf_link_set_in_status(AVBitStreamFilterLink *link, int status, int64_t pts); + +int ff_bsf_outlink_get_status(AVBitStreamFilterLink *link); + +int ff_bsf_outlink_packet_wanted(AVBitStreamFilterLink *link); + +#endif /* AVCODEC_BSF_FILTERS_H */ diff --git a/libavcodec/bsf/sink.c b/libavcodec/bsf/sink.c new file mode 100644 index 0000000000..3774a1d014 --- /dev/null +++ b/libavcodec/bsf/sink.c @@ -0,0 +1,148 @@ +/* + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/** + * @file + * Packet sink. Heavily based on libavfilter/buffersink.c + */ + +#include "libavutil/avassert.h" +#include "libavutil/opt.h" + +#include "libavcodec/bsf.h" +#include "libavcodec/bsf_internal.h" +#include "libavcodec/packet.h" +#include "libavcodec/packet_internal.h" + +typedef struct BufferSinkContext { + const AVClass *class; + unsigned warning_limit; + + AVPacket *peeked_pkt; +} BufferSinkContext; + +static int return_or_keep_packet(BufferSinkContext *buf, AVPacket *out, AVPacket *in, int flags) +{ + if ((flags & AV_BSF_SINK_FLAG_PEEK)) { + buf->peeked_pkt = in; + return av_packet_ref(out, in); + } else { + buf->peeked_pkt = NULL; + av_packet_move_ref(out, in); + av_packet_free(&in); + return 0; + } +} + +int attribute_align_arg av_bsf_sink_get_packet(AVBitStreamFilterContext *ctx, AVPacket *pkt, int flags) +{ + BufferSinkContext *buf = ctx->priv_data; + AVBitStreamFilterLink *inlink = ctx->inputs[0]; + BitStreamFilterLinkInternal *li = ff_link_internal(inlink); + int status, ret; + AVPacket *cur_pkt; + int64_t pts; + int buffersrc_empty = 0; + + if (buf->peeked_pkt) + return return_or_keep_packet(buf, pkt, buf->peeked_pkt, flags); + + while (1) { + ret = ff_bsf_inlink_consume_packet(inlink, &cur_pkt); + if (ret < 0) { + return ret; + } else if (ret) { + return return_or_keep_packet(buf, pkt, cur_pkt, flags); + } else if (ff_bsf_inlink_acknowledge_status(inlink, &status, &pts)) { + return status; + } else if ((flags & AV_BSF_SINK_FLAG_NO_REQUEST)) { + return AVERROR(EAGAIN); + } else if (li->packet_wanted_out) { + ret = ff_bsf_graph_run_once(ctx->graph); + if (ret == FFERROR_SOURCE_EMPTY) { + buffersrc_empty = 1; + } else if (ret == AVERROR(EAGAIN)) { + if (buffersrc_empty) + return ret; + ff_bsf_inlink_request_packet(inlink); + } else if (ret < 0) { + return ret; + } + } else { + ff_bsf_inlink_request_packet(inlink); + } + } +} + +static int init(AVBitStreamFilterContext *ctx) +{ + BufferSinkContext *s = ctx->priv_data; + + s->warning_limit = 100; + + return 0; +} + +static void uninit(AVBitStreamFilterContext *ctx) +{ + BufferSinkContext *buf = ctx->priv_data; + + av_packet_free(&buf->peeked_pkt); +} + +static int activate(AVBitStreamFilterContext *ctx) +{ + BufferSinkContext *buf = ctx->priv_data; + BitStreamFilterLinkInternal * const li = ff_link_internal(ctx->inputs[0]); + + if (buf->warning_limit && + av_container_fifo_can_read(li->fifo) >= buf->warning_limit) { + av_log(ctx, AV_LOG_WARNING, + "%d buffers queued in %s, something may be wrong.\n", + buf->warning_limit, + (char *)av_x_if_null(ctx->name, ctx->filter->name)); + buf->warning_limit *= 10; + } + + /* The packet is queued, the rest is up to av_bsf_sink_get_packet */ + return 0; +} + +AVRational av_bsf_sink_get_time_base(const AVBitStreamFilterContext *ctx) +{ + av_assert0(ff_bsf(ctx->filter)->activate == activate); + return ff_bsf_link(ctx->inputs[0])->time_base; +} + +const AVCodecParameters *av_bsf_sink_get_parameters(const AVBitStreamFilterContext *ctx) +{ + av_assert0(ff_bsf(ctx->filter)->activate == activate); + return ff_bsf_link(ctx->inputs[0])->par; +} + +BSF_DEFINE_CLASS_EXT(sink, "sink", NULL); + +const FFBitStreamFilter ff_sink_bsf = { + .p.name = "sink", + .p.priv_class = &sink_class, + .priv_data_size = sizeof(BufferSinkContext), + .init2 = init, + .uninit = uninit, + .activate = activate, + BSFILTER_INPUTS(ff_default_bsf_pad), +}; diff --git a/libavcodec/bsf/source.c b/libavcodec/bsf/source.c new file mode 100644 index 0000000000..8e1c2445af --- /dev/null +++ b/libavcodec/bsf/source.c @@ -0,0 +1,224 @@ +/* + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/** + * @file + * Packet source. Heavily based on libavfilter/buffersrc.c + */ + +#include <float.h> + +#include "libavutil/container_fifo.h" +#include "libavutil/opt.h" + +#include "libavcodec/bsf.h" +#include "libavcodec/bsf_internal.h" +#include "libavcodec/packet.h" +#include "libavcodec/packet_internal.h" + +typedef struct SourceContext { + const AVClass *class; + AVCodecParameters *par; + AVRational time_base; ///< time_base to set in the output link + + unsigned nb_failed_requests; + unsigned warning_limit; + + int eof; + int64_t last_pts; + int link_delta, prev_delta; +} SourceContext; + +int av_bsf_source_parameters_set(AVBitStreamFilterContext *ctx, const AVCodecParameters *par) +{ + SourceContext *s = ctx->priv_data; + + return avcodec_parameters_copy(s->par, par); +} + +static int push_packet(AVBitStreamFilterGraph *graph) +{ + int ret; + + while (1) { + ret = ff_bsf_graph_run_once(graph); + if (ret == AVERROR(EAGAIN)) + break; + if (ret < 0 && ret != FFERROR_SOURCE_EMPTY) + return ret; + } + return 0; +} + +int attribute_align_arg av_bsf_source_add_packet(AVBitStreamFilterContext *ctx, AVPacket *pkt, int flags) +{ + SourceContext *s = ctx->priv_data; + AVPacket *copy; + int ret; + + s->nb_failed_requests = 0; + + if (!pkt || AVPACKET_IS_EMPTY(pkt)) + return av_bsf_source_close(ctx, s->last_pts, flags); + if (s->eof) + return AVERROR_EOF; + + s->last_pts = pkt->pts + pkt->duration; + + copy = av_packet_alloc(); + if (!copy) + return AVERROR(ENOMEM); + + if ((flags & AV_BSF_SOURCE_FLAG_KEEP_REF)) { + ret = av_packet_ref(copy, pkt); + if (ret < 0) + return ret; + } else + av_packet_move_ref(copy, pkt); + + ret = ff_bsf_filter_packet(ctx->outputs[0], copy); + if (ret < 0) + return ret; + + if ((flags & AV_BSF_SOURCE_FLAG_PUSH)) { + ret = push_packet(ctx->graph); + if (ret < 0) + return ret; + } + + BitStreamFilterLinkInternal *const li = ff_link_internal(ctx->outputs[0]); + if (s->warning_limit && + av_container_fifo_can_read(li->fifo) >= s->warning_limit) { + av_log(s, AV_LOG_WARNING, + "%d buffers queued in %s, something may be wrong.\n", + s->warning_limit, + (char *)av_x_if_null(ctx->name, ctx->filter->name)); + s->warning_limit *= 10; + } + + return 0; +} + +int av_bsf_source_close(AVBitStreamFilterContext *ctx, int64_t pts, unsigned flags) +{ + SourceContext *s = ctx->priv_data; + + s->eof = 1; + ff_bsf_link_set_in_status(ctx->outputs[0], AVERROR_EOF, pts); + return 0; +} + +int av_bsf_source_get_status(AVBitStreamFilterContext *ctx) +{ + SourceContext *s = ctx->priv_data; + + if (!s->eof && ff_bsf_outlink_get_status(ctx->outputs[0])) + s->eof = 1; + + return s->eof ? AVERROR(EOF) : 0; +} + +static av_cold int preinit(AVBitStreamFilterContext *ctx) +{ + SourceContext *c = ctx->priv_data; + + c->par = avcodec_parameters_alloc(); + if (!c->par) + return AVERROR(ENOMEM); + + return 0; +} + +static av_cold int init(AVBitStreamFilterContext *ctx) +{ + SourceContext *c = ctx->priv_data; + + if (av_q2d(c->time_base) <= 0) { + av_log(ctx, AV_LOG_ERROR, "Invalid time base %d/%d\n", c->time_base.num, c->time_base.den); + return AVERROR(EINVAL); + } + + c->warning_limit = 100; + return 0; +} + +unsigned av_bsf_source_get_nb_failed_requests(AVBitStreamFilterContext *buffer_src) +{ + return ((SourceContext *)buffer_src->priv_data)->nb_failed_requests; +} + +#define OFFSET(x) offsetof(SourceContext, x) +#define FLAGS (AV_OPT_FLAG_BSF_PARAM|AV_OPT_FLAG_AUDIO_PARAM|AV_OPT_FLAG_VIDEO_PARAM) +static const AVOption buffer_options[] = { + { "time_base", NULL, OFFSET(time_base), AV_OPT_TYPE_RATIONAL, { .dbl = 0 }, 0, DBL_MAX, FLAGS }, + { NULL }, +}; + +BSF_DEFINE_CLASS(buffer); + +static av_cold void uninit(AVBitStreamFilterContext *ctx) +{ + SourceContext *s = ctx->priv_data; + avcodec_parameters_free(&s->par); +} + +static int config_props(AVBitStreamFilterLink *link) +{ + BitStreamFilterLink *l = ff_bsf_link(link); + SourceContext *c = link->src->priv_data; + + l->time_base = c->time_base; + return avcodec_parameters_copy(l->par, c->par); +} + +static int activate(AVBitStreamFilterContext *ctx) +{ + AVBitStreamFilterLink *outlink = ctx->outputs[0]; + SourceContext *c = ctx->priv_data; + + if (!c->eof && ff_bsf_outlink_get_status(outlink)) { + c->eof = 1; + return 0; + } + + if (c->eof) { + ff_bsf_link_set_in_status(outlink, AVERROR_EOF, c->last_pts); + return 0; + } + c->nb_failed_requests++; + return FFERROR_SOURCE_EMPTY; +} + +static const AVBitStreamFilterPad source_outputs[] = { + { + .name = "default", + .config_props = config_props, + }, +}; + +const FFBitStreamFilter ff_source_bsf = { + .p.name = "source", + .p.priv_class = &buffer_class, + .priv_data_size = sizeof(SourceContext), + .activate = activate, + .preinit = preinit, + .init2 = init, + .uninit = uninit, + + BSFILTER_OUTPUTS(source_outputs), +}; diff --git a/libavcodec/bsf_internal.h b/libavcodec/bsf_internal.h index 922b03c01b..4303ef996e 100644 --- a/libavcodec/bsf_internal.h +++ b/libavcodec/bsf_internal.h @@ -19,10 +19,12 @@ #ifndef AVCODEC_BSF_INTERNAL_H #define AVCODEC_BSF_INTERNAL_H +#include "libavutil/container_fifo.h" #include "libavutil/log.h" #include "bsf.h" #include "packet.h" +#include "bsf/filters.h" typedef struct FFBitStreamFilter { /** @@ -35,8 +37,53 @@ typedef struct FFBitStreamFilter { int (*filter)(AVBSFContext *ctx, AVPacket *pkt); void (*close)(AVBSFContext *ctx); void (*flush)(AVBSFContext *ctx); + + // Graph based API + + /** + * List of static inputs. + */ + const struct AVBitStreamFilterPad *inputs; + + /** + * List of static outputs. + */ + const struct AVBitStreamFilterPad *outputs; + + /** + * The number of entries in the list of inputs. + */ + uint8_t nb_inputs; + + /** + * The number of entries in the list of outputs. + */ + uint8_t nb_outputs; + + int (*preinit)(AVBitStreamFilterContext *ctx); + int (*init2)(AVBitStreamFilterContext *ctx); + void (*uninit)(AVBitStreamFilterContext *ctx); + + /** + * Filter activation function. + * + * Called when any processing is needed from the filter, instead of any + * filter_packet and request_packet on pads. + * + * The function must examine inlinks and outlinks and perform a single + * step of processing. If there is nothing to do, the function must do + * nothing and not return an error. If more steps are or may be + * possible, it must use ff_filter_set_ready() to schedule another + * activation. + */ + int (*activate)(AVBitStreamFilterContext *ctx); } FFBitStreamFilter; +static av_always_inline const FFBitStreamFilter *ff_bsf(const AVBitStreamFilter *bsf) +{ + return (const FFBitStreamFilter*)bsf; +} + /** * Called by the bitstream filters to get the next packet for filtering. * The filter is responsible for either freeing the packet or passing it to the @@ -57,4 +104,150 @@ int ff_bsf_get_packet_ref(AVBSFContext *ctx, AVPacket *pkt); const AVClass *ff_bsf_child_class_iterate(void **opaque); + +// Graph based API + +typedef struct BitStreamFilterLinkInternal { + BitStreamFilterLink l; + + /** + * Queue of packets waiting to be filtered. + */ + AVContainerFifo *fifo; + + /** + * If set, the source filter can not generate a packet as is. + * The goal is to avoid repeatedly calling the request_packet() method on + * the same link. + */ + int packet_blocked_in; + + /** + * Link input status. + * If not zero, all attempts of filter_packet will fail with the + * corresponding code. + */ + int status_in; + + /** + * Timestamp of the input status change. + */ + int64_t status_in_pts; + + /** + * Link output status. + * If not zero, all attempts of request_packet will fail with the + * corresponding code. + */ + int status_out; + + /** + * True if a packet is currently wanted on the output of this filter. + * Set when ff_request_packet() is called by the output, + * cleared when a packet is filtered. + */ + int packet_wanted_out; + + /** + * Index in the age array. + */ + int age_index; + + /** stage of the initialization of the link properties (dimensions, etc) */ + enum { + AVLINK_UNINIT = 0, ///< not started + AVLINK_STARTINIT, ///< started, but incomplete + AVLINK_INIT ///< complete + } init_state; +} BitStreamFilterLinkInternal; + +static inline BitStreamFilterLinkInternal *ff_link_internal(AVBitStreamFilterLink *link) +{ + return (BitStreamFilterLinkInternal*)link; +} + +/** + * Parse filter options into a dictionary. + * + * @param logctx context for logging + * @param priv_class a filter's private class for shorthand options or NULL + * @param options dictionary to store parsed options in + * @param args options string to parse + * + * @return a non-negative number on success, a negative error code on failure + */ +int ff_bsf_opt_parse(void *logctx, const AVClass *priv_class, + AVDictionary **options, const char *args); + +int ff_bsf_config_links(AVBitStreamFilterContext *filter); + +/** + * Run one round of processing on a filter graph. + */ +int ff_bsf_graph_run_once(AVBitStreamFilterGraph *graph); + +int ff_bsf_activate(AVBitStreamFilterContext *ctx); + +void ff_bsf_graph_update_heap(AVBitStreamFilterGraph *graph, BitStreamFilterLinkInternal *li); + +typedef struct FFBitStreamFilterContext { + /** + * The public AVBitStreamFilterContext. See bsf.h for it. + */ + AVBitStreamFilterContext p; + + // AV_CLASS_STATE_FLAG_* + unsigned state_flags; + + /** + * Ready status of the filter. + * A non-0 value means that the filter needs activating; + * a higher value suggests a more urgent activation. + */ + unsigned ready; +} FFBitStreamFilterContext; + +static inline FFBitStreamFilterContext *ffbsfctx(AVBitStreamFilterContext *ctx) +{ + return (FFBitStreamFilterContext*)ctx; +} + +/** + * Free a filter context. This will also remove the filter from its + * filtergraph's list of filters. + * + * @param filter the filter to free + */ +void ff_bsf_free(AVBitStreamFilterContext *filter); + +typedef struct FFBitStreamFilterGraph { + /** + * The public AVBitStreamFilterGraph. See bsf.h for it. + */ + AVBitStreamFilterGraph p; + + struct BitStreamFilterLinkInternal **sink_links; + int sink_links_count; +} FFBitStreamFilterGraph; + +static inline FFBitStreamFilterGraph *ffbsffiltergraph(AVBitStreamFilterGraph *graph) +{ + return (FFBitStreamFilterGraph*)graph; +} + +/** + * Allocate a new filter context and return it. + * + * @param filter what filter to create an instance of + * @param inst_name name to give to the new filter context + * + * @return newly created filter context or NULL on failure + */ +AVBitStreamFilterContext *ff_bsf_alloc(const AVBitStreamFilter *filter, const char *inst_name); + +/** + * Remove a filter from a graph; + */ +void ff_bsf_graph_remove_filter(AVBitStreamFilterGraph *graph, AVBitStreamFilterContext *filter); + #endif /* AVCODEC_BSF_INTERNAL_H */ diff --git a/libavcodec/bsfgraph.c b/libavcodec/bsfgraph.c new file mode 100644 index 0000000000..d9a1d9dd6f --- /dev/null +++ b/libavcodec/bsfgraph.c @@ -0,0 +1,374 @@ +/* + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "config.h" + +#include <string.h> + +#include "libavutil/avassert.h" +#include "libavutil/error.h" +#include "libavutil/mem.h" +#include "libavutil/opt.h" + +#include "bsf.h" +#include "bsf_internal.h" + +AVBitStreamFilterGraph *av_bsf_graph_alloc(void) +{ + FFBitStreamFilterGraph *graph = av_mallocz(sizeof(*graph)); + AVBitStreamFilterGraph *ret; + + if (!graph) + return NULL; + + ret = &graph->p; + + return ret; +} + +void ff_bsf_graph_remove_filter(AVBitStreamFilterGraph *graph, AVBitStreamFilterContext *filter) +{ + int i, j; + for (i = 0; i < graph->nb_filters; i++) { + if (graph->filters[i] == filter) { + FFSWAP(AVBitStreamFilterContext*, graph->filters[i], + graph->filters[graph->nb_filters - 1]); + graph->nb_filters--; + filter->graph = NULL; + for (j = 0; j<filter->nb_outputs; j++) + if (filter->outputs[j]) + ff_bsf_link(filter->outputs[j])->graph = NULL; + + return; + } + } +} + +AVBitStreamFilterContext *av_bsf_graph_get_filter(AVBitStreamFilterGraph *graph, const char *name) +{ + int i; + + for (i = 0; i < graph->nb_filters; i++) + if (graph->filters[i]->name && !strcmp(name, graph->filters[i]->name)) + return graph->filters[i]; + + return NULL; +} + +void av_bsf_graph_free(AVBitStreamFilterGraph **graphp) +{ + AVBitStreamFilterGraph *graph = *graphp; + FFBitStreamFilterGraph *graphi = ffbsffiltergraph(graph); + + if (!graph) + return; + + while (graph->nb_filters) + ff_bsf_free(graph->filters[0]); + + av_freep(&graphi->sink_links); + + av_opt_free(graph); + + av_freep(&graph->filters); + av_freep(graphp); +} + +int av_bsf_graph_create_filter(AVBitStreamFilterContext **filt_ctx, const AVBitStreamFilter *filt, + const char *name, AVDictionary **options, AVBitStreamFilterGraph *graph_ctx) +{ + int ret; + + *filt_ctx = av_bsf_graph_alloc_filter(graph_ctx, filt, name); + if (!*filt_ctx) + return AVERROR(ENOMEM); + + ret = av_bsf_init_dict(*filt_ctx, options); + if (ret < 0) + goto fail; + + return 0; + +fail: + ff_bsf_free(*filt_ctx); + *filt_ctx = NULL; + return ret; +} + +AVBitStreamFilterContext *av_bsf_graph_alloc_filter(AVBitStreamFilterGraph *graph, + const AVBitStreamFilter *filter, + const char *name) +{ + AVBitStreamFilterContext **filters, *s; + + if (!ff_bsf(filter)->activate && !ff_bsf(filter)->nb_inputs && !ff_bsf(filter)->nb_outputs) + return NULL; + + filters = av_realloc_array(graph->filters, graph->nb_filters + 1, sizeof(*filters)); + if (!filters) + return NULL; + graph->filters = filters; + + s = ff_bsf_alloc(filter, name); + if (!s) + return NULL; + + graph->filters[graph->nb_filters++] = s; + + s->graph = graph; + + return s; +} + +/** + * Check for the validity of graph. + * + * A graph is considered valid if all its input and output pads are + * connected. + * + * @return >= 0 in case of success, a negative value otherwise + */ +static int graph_check_validity(AVBitStreamFilterGraph *graph, void *log_ctx) +{ + AVBitStreamFilterContext *filt; + int i, j; + + for (i = 0; i < graph->nb_filters; i++) { + const AVBitStreamFilterPad *pad; + filt = graph->filters[i]; + + for (j = 0; j < filt->nb_inputs; j++) { + if (!filt->inputs[j] || !filt->inputs[j]->src) { + pad = &filt->input_pads[j]; + av_log(log_ctx, AV_LOG_ERROR, + "Input pad \"%s\" of the filter instance \"%s\" of %s not connected to any source\n", + pad->name, filt->name, filt->filter->name); + return AVERROR(EINVAL); + } + } + + for (j = 0; j < filt->nb_outputs; j++) { + if (!filt->outputs[j] || !filt->outputs[j]->dst) { + pad = &filt->output_pads[j]; + av_log(log_ctx, AV_LOG_ERROR, + "Output pad \"%s\" of the filter instance \"%s\" of %s not connected to any destination\n", + pad->name, filt->name, filt->filter->name); + return AVERROR(EINVAL); + } + } + } + + return 0; +} + +/** + * Configure all the links of graphctx. + * + * @return >= 0 in case of success, a negative value otherwise + */ +static int graph_config_links(AVBitStreamFilterGraph *graph, void *log_ctx) +{ + AVBitStreamFilterContext *filt; + int i, ret; + + for (i = 0; i < graph->nb_filters; i++) { + filt = graph->filters[i]; + + if (!filt->nb_outputs) { + if ((ret = ff_bsf_config_links(filt))) + return ret; + } + } + + return 0; +} + +static int graph_config_pointers(AVBitStreamFilterGraph *graph, void *log_ctx) +{ + unsigned i, j; + int sink_links_count = 0, n = 0; + AVBitStreamFilterContext *f; + BitStreamFilterLinkInternal **sinks; + + for (i = 0; i < graph->nb_filters; i++) { + f = graph->filters[i]; + for (j = 0; j < f->nb_inputs; j++) { + ff_link_internal(f->inputs[j])->age_index = -1; + } + for (j = 0; j < f->nb_outputs; j++) { + ff_link_internal(f->outputs[j])->age_index = -1; + } + if (!f->nb_outputs) { + if (f->nb_inputs > INT_MAX - sink_links_count) + return AVERROR(EINVAL); + sink_links_count += f->nb_inputs; + } + } + sinks = av_calloc(sink_links_count, sizeof(*sinks)); + if (!sinks) + return AVERROR(ENOMEM); + for (i = 0; i < graph->nb_filters; i++) { + f = graph->filters[i]; + if (!f->nb_outputs) { + for (j = 0; j < f->nb_inputs; j++) { + sinks[n] = ff_link_internal(f->inputs[j]); + sinks[n]->age_index = n; + n++; + } + } + } + av_assert0(n == sink_links_count); + ffbsffiltergraph(graph)->sink_links = sinks; + ffbsffiltergraph(graph)->sink_links_count = sink_links_count; + return 0; +} + +int av_bsf_graph_config(AVBitStreamFilterGraph *graphctx, void *log_ctx) +{ + int ret; + + if ((ret = graph_check_validity(graphctx, log_ctx))) + return ret; + if ((ret = graph_config_links(graphctx, log_ctx))) + return ret; + if ((ret = graph_config_pointers(graphctx, log_ctx))) + return ret; + + return 0; +} + +static void heap_bubble_up(FFBitStreamFilterGraph *graph, + BitStreamFilterLinkInternal *li, int index) +{ + BitStreamFilterLinkInternal **links = graph->sink_links; + + av_assert0(index >= 0); + + while (index) { + int parent = (index - 1) >> 1; + if (links[parent]->l.current_pts_us >= li->l.current_pts_us) + break; + links[index] = links[parent]; + links[index]->age_index = index; + index = parent; + } + links[index] = li; + li->age_index = index; +} + +static void heap_bubble_down(FFBitStreamFilterGraph *graph, + BitStreamFilterLinkInternal *li, int index) +{ + BitStreamFilterLinkInternal **links = graph->sink_links; + + av_assert0(index >= 0); + + while (1) { + int child = 2 * index + 1; + if (child >= graph->sink_links_count) + break; + if (child + 1 < graph->sink_links_count && + links[child + 1]->l.current_pts_us < links[child]->l.current_pts_us) + child++; + if (li->l.current_pts_us < links[child]->l.current_pts_us) + break; + links[index] = links[child]; + links[index]->age_index = index; + index = child; + } + links[index] = li; + li->age_index = index; +} + +void ff_bsf_graph_update_heap(AVBitStreamFilterGraph *graph, BitStreamFilterLinkInternal *li) +{ + FFBitStreamFilterGraph *graphi = ffbsffiltergraph(graph); + + heap_bubble_up (graphi, li, li->age_index); + heap_bubble_down(graphi, li, li->age_index); +} + +int av_bsf_graph_request_oldest(AVBitStreamFilterGraph *graph) +{ + FFBitStreamFilterGraph *graphi = ffbsffiltergraph(graph); + BitStreamFilterLinkInternal *oldesti = graphi->sink_links[0]; + AVBitStreamFilterLink *oldest = &oldesti->l.pub; + int64_t packet_count; + int r; + + while (graphi->sink_links_count) { + oldesti = graphi->sink_links[0]; + oldest = &oldesti->l.pub; + if (ff_bsf(oldest->dst->filter)->activate) { + r = av_bsf_sink_get_packet(oldest->dst, NULL, AV_BSF_SINK_FLAG_PEEK); + if (r != AVERROR_EOF) + return r; + } else { + r = ff_bsf_request_packet(oldest); + } + if (r != AVERROR_EOF) + break; + av_log(oldest->dst, AV_LOG_DEBUG, "EOF on sink link %s:%s.\n", + oldest->dst->name, + oldest->dstpad->name); + /* EOF: remove the link from the heap */ + if (oldesti->age_index < --graphi->sink_links_count) + heap_bubble_down(graphi, graphi->sink_links[graphi->sink_links_count], + oldesti->age_index); + oldesti->age_index = -1; + } + if (!graphi->sink_links_count) + return AVERROR_EOF; + av_assert1(!ff_bsf(oldest->dst->filter)->activate); + av_assert1(oldesti->age_index >= 0); + packet_count = oldesti->l.packet_count_out; + while (packet_count == oldesti->l.packet_count_out) { + r = ff_bsf_graph_run_once(graph); + if (r == FFERROR_SOURCE_EMPTY) + r = 0; + if (r == AVERROR(EAGAIN) && + !oldesti->packet_wanted_out && !oldesti->packet_blocked_in && + !oldesti->status_in) + (void)ff_bsf_request_packet(oldest); + else if (r < 0) + return r; + } + return 0; +} + +int ff_bsf_graph_run_once(AVBitStreamFilterGraph *graph) +{ + FFBitStreamFilterContext *ctxi; + unsigned i; + + av_assert0(graph->nb_filters); + ctxi = ffbsfctx(graph->filters[0]); + for (i = 1; i < graph->nb_filters; i++) { + FFBitStreamFilterContext *ctxi_other = ffbsfctx(graph->filters[i]); + + if (ctxi_other->ready > ctxi->ready) + ctxi = ctxi_other; + } + + if (!ctxi->ready) + return AVERROR(EAGAIN); + + ctxi->ready = 0; + + return ff_bsf_activate(&ctxi->p); +} -- 2.52.0 >From 29f1b075b970dd9281795d4297f206b7db7954df Mon Sep 17 00:00:00 2001 From: James Almer <[email protected]> Date: Wed, 24 Jun 2026 08:26:37 -0300 Subject: [PATCH 2/3] avcodec/bsf: add an LCEVC stream merging bsf Signed-off-by: James Almer <[email protected]> --- libavcodec/bsf/lcevc_merge.c | 185 ++++++++++++++++ libavcodec/bsf/packetsync.c | 416 +++++++++++++++++++++++++++++++++++ libavcodec/bsf/packetsync.h | 345 +++++++++++++++++++++++++++++ 3 files changed, 946 insertions(+) create mode 100644 libavcodec/bsf/lcevc_merge.c create mode 100644 libavcodec/bsf/packetsync.c create mode 100644 libavcodec/bsf/packetsync.h diff --git a/libavcodec/bsf/lcevc_merge.c b/libavcodec/bsf/lcevc_merge.c new file mode 100644 index 0000000000..483e2e164d --- /dev/null +++ b/libavcodec/bsf/lcevc_merge.c @@ -0,0 +1,185 @@ +/* + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "libavutil/opt.h" + +#include "libavcodec/bsf.h" +#include "libavcodec/bsf_internal.h" +#include "libavcodec/bsf/packetsync.h" +#include "libavcodec/bytestream.h" +#include "libavcodec/h2645_parse.h" +#include "libavcodec/packet.h" +#include "libavcodec/packet_internal.h" + +typedef struct LCEVCMergeContext { + AVClass *class; + FFPacketSync ps; + int nal_length_size; +} LCEVCMergeContext; + +static AVPacket *lcevc_merge_packets(AVBitStreamFilterContext *ctx, + AVPacket *base_pkt, const AVPacket *lcevc_pkt) +{ + LCEVCMergeContext *lcevc = ctx->priv_data; + AVPacket *out = av_packet_clone(base_pkt); + + if (!out) + return base_pkt; + + if (lcevc->nal_length_size && lcevc_pkt->size > lcevc->nal_length_size) { + GetByteContext bc; + size_t size = 0; + + bytestream2_init(&bc, lcevc_pkt->data, lcevc_pkt->size); + do { + int i = 0; + int nal_size = get_nalsize(lcevc->nal_length_size, + bc.buffer, bytestream2_get_bytes_left(&bc), &i, ctx); + if (nal_size < 0) + return base_pkt; + if (nal_size > INT_MAX - 4) + return base_pkt; + size += nal_size + 4; + nal_size += i; + if (nal_size > bytestream2_get_bytes_left(&bc)) + return base_pkt; + bytestream2_skip(&bc, nal_size); + } while (bytestream2_get_bytes_left(&bc) > 0); + + uint8_t *buf = av_packet_new_side_data(out, AV_PKT_DATA_LCEVC, size); + if (!buf) + return base_pkt; + + PutByteContext pc; + bytestream2_init_writer(&pc, buf, size); + bytestream2_init(&bc, lcevc_pkt->data, lcevc_pkt->size); + do { + int i = 0; + int nal_size = get_nalsize(lcevc->nal_length_size, + bc.buffer, bytestream2_get_bytes_left(&bc), &i, ctx); + bytestream2_skipu(&bc, i); + bytestream2_put_be32u(&pc, 1); // start code + bytestream2_put_bufferu(&pc, bc.buffer, nal_size); + bytestream2_skipu(&bc, nal_size); + } while (bytestream2_get_bytes_left(&bc) > 0); + } else { + uint8_t *buf = av_packet_new_side_data(out, AV_PKT_DATA_LCEVC, lcevc_pkt->size); + if (!buf) + return base_pkt; + memcpy(buf, lcevc_pkt->data, lcevc_pkt->size); + } + + av_packet_free(&base_pkt); + return out; +} + +static int lcevc_merge(FFPacketSync *fs) +{ + AVBitStreamFilterContext *ctx = fs->parent; + AVPacket *base, *enhancement, *out = NULL; + int ret; + + ret = ff_packetsync_dualinput_get(fs, &base, &enhancement); + if (ret < 0) + return ret; + if (!enhancement) + return ff_bsf_filter_packet(ctx->outputs[0], base); + out = lcevc_merge_packets(ctx, base, enhancement); + return ff_bsf_filter_packet(ctx->outputs[0], out); +} + +static int config_output(AVBitStreamFilterLink *outlink) +{ + BitStreamFilterLink *l = ff_bsf_link(outlink); + AVBitStreamFilterContext *ctx = outlink->src; + LCEVCMergeContext *lcevc = ctx->priv_data; + int ret; + + ret = ff_packetsync_init_dualinput(&lcevc->ps, ctx); + if (ret < 0) + return ret; + + ret = ff_packetsync_configure(&lcevc->ps); + l->time_base = lcevc->ps.time_base; + + return ret; +} + +static av_cold int init(AVBitStreamFilterContext *ctx) +{ + LCEVCMergeContext *lcevc = ctx->priv_data; + + lcevc->ps.on_event = lcevc_merge; + return 0; +} + +static av_cold void uninit(AVBitStreamFilterContext *ctx) +{ + LCEVCMergeContext *lcevc = ctx->priv_data; + + ff_packetsync_uninit(&lcevc->ps); +} + +#define OFFSET(x) offsetof(LCEVCMergeContext, x) +#define FLAGS (AV_OPT_FLAG_VIDEO_PARAM|AV_OPT_FLAG_BSF_PARAM) +static const AVOption lcevc_merge_options[] = { + { "nal_length_size", "Size of LCEVC NAL length. 0 if Annex B", + OFFSET(nal_length_size), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 4, FLAGS }, + { NULL } +}; + +PACKETSYNC_DEFINE_CLASS(lcevc_merge, LCEVCMergeContext, ps); + +static int activate(AVBitStreamFilterContext *ctx) +{ + LCEVCMergeContext *lcevc = ctx->priv_data; + return ff_packetsync_activate(&lcevc->ps); +} + +static const enum AVCodecID enhancement_codec_ids[] = { + AV_CODEC_ID_LCEVC, AV_CODEC_ID_NONE, +}; + +static const AVBitStreamFilterPad lcevc_merge_inputs[] = { + { + .name = "base", + }, + { + .name = "enhancement", + .codec_ids = enhancement_codec_ids, + }, +}; + +static const AVBitStreamFilterPad lcevc_merge_outputs[] = { + { + .name = "default", + .config_props = config_output, + }, +}; + +const FFBitStreamFilter ff_lcevc_merge_bsf = { + .p.name = "lcevc_merge", + .p.priv_class = &lcevc_merge_class, + .priv_data_size = sizeof(LCEVCMergeContext), + .preinit = lcevc_merge_packetsync_preinit, + .init2 = init, + .uninit = uninit, + .activate = activate, + BSFILTER_INPUTS(lcevc_merge_inputs), + BSFILTER_OUTPUTS(lcevc_merge_outputs), +}; diff --git a/libavcodec/bsf/packetsync.c b/libavcodec/bsf/packetsync.c new file mode 100644 index 0000000000..39431c02b2 --- /dev/null +++ b/libavcodec/bsf/packetsync.c @@ -0,0 +1,416 @@ +/* + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with FFmpeg; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/** + * @file + * Packet sync API. Heavily based on libavfilter/framesync.c by Nicolas George + */ + +#include "libavutil/avassert.h" +#include "libavutil/mem.h" +#include "libavutil/opt.h" + +#include "libavcodec/bsf.h" +#include "filters.h" +#include "packetsync.h" + +#define OFFSET(member) offsetof(FFPacketSync, member) +#define FLAGS (AV_OPT_FLAG_BSF_PARAM|AV_OPT_FLAG_AUDIO_PARAM|AV_OPT_FLAG_VIDEO_PARAM) + +static const char *packetsync_name(void *ptr) +{ + return "packetsync"; +} + +static const AVOption packetsync_options[] = { + { "eof_action", "Action to take when encountering EOF from secondary input ", + OFFSET(opt_eof_action), AV_OPT_TYPE_INT, { .i64 = EOF_ACTION_PASS }, + EOF_ACTION_ENDALL, EOF_ACTION_PASS, .flags = FLAGS, .unit = "eof_action" }, + { "endall", "End both streams.", 0, AV_OPT_TYPE_CONST, { .i64 = EOF_ACTION_ENDALL }, .flags = FLAGS, .unit = "eof_action" }, + { "pass", "Pass through the main input.", 0, AV_OPT_TYPE_CONST, { .i64 = EOF_ACTION_PASS }, .flags = FLAGS, .unit = "eof_action" }, + { "ts_sync_mode", "How strictly to sync streams based on secondary input timestamps", + OFFSET(opt_ts_sync_mode), AV_OPT_TYPE_INT, { .i64 = TS_DEFAULT }, + TS_DEFAULT, TS_NEAREST, .flags = FLAGS, .unit = "ts_sync_mode" }, + { "default", "Packet from secondary input with the nearest lower or equal timestamp to the primary input packet", + 0, AV_OPT_TYPE_CONST, { .i64 = TS_DEFAULT }, .flags = FLAGS, .unit = "ts_sync_mode" }, + { "nearest", "Packet from secondary input with the absolute nearest timestamp to the primary input packet", + 0, AV_OPT_TYPE_CONST, { .i64 = TS_NEAREST }, .flags = FLAGS, .unit = "ts_sync_mode" }, + { NULL } +}; +const AVClass ff_packetsync_class = { + .version = LIBAVUTIL_VERSION_INT, + .class_name = "packetsync", + .item_name = packetsync_name, + .category = AV_CLASS_CATEGORY_BITSTREAM_FILTER, + .option = packetsync_options, + .parent_log_context_offset = OFFSET(parent), +}; + +const AVClass *ff_packetsync_child_class_iterate(void **iter) +{ + const AVClass *c = *iter ? NULL : &ff_packetsync_class; + *iter = (void *)(uintptr_t)c; + return c; +} + +enum { + STATE_BOF, + STATE_RUN, + STATE_EOF, +}; + +static int consume_from_fifos(FFPacketSync *fs); + +void ff_packetsync_preinit(FFPacketSync *fs) +{ + if (fs->class) + return; + fs->class = &ff_packetsync_class; + av_opt_set_defaults(fs); +} + +int ff_packetsync_init(FFPacketSync *fs, AVBitStreamFilterContext *parent, unsigned nb_in) +{ + /* For filters with several outputs, we will not be able to assume which + output is relevant for ff_outlink_packet_wanted() and + ff_bsf_link_set_in_status(). To be designed when needed. */ + av_assert0(parent->nb_outputs == 1); + + ff_packetsync_preinit(fs); + fs->parent = parent; + fs->nb_in = nb_in; + + fs->in = av_calloc(nb_in, sizeof(*fs->in)); + if (!fs->in) { + fs->nb_in = 0; + return AVERROR(ENOMEM); + } + + return 0; +} + +static void packetsync_eof(FFPacketSync *fs, int64_t pts) +{ + fs->eof = 1; + fs->pkt_ready = 0; + ff_bsf_link_set_in_status(fs->parent->outputs[0], AVERROR_EOF, pts); +} + +static void packetsync_sync_level_update(FFPacketSync *fs, int64_t eof_pts) +{ + unsigned i, level = 0; + + for (i = 0; i < fs->nb_in; i++) + if (fs->in[i].state != STATE_EOF) + level = FFMAX(level, fs->in[i].sync); + av_assert0(level <= fs->sync_level); + if (level < fs->sync_level) + av_log(fs, AV_LOG_VERBOSE, "Sync level %u\n", level); + if (fs->opt_ts_sync_mode > TS_DEFAULT) { + for (i = 0; i < fs->nb_in; i++) { + if (fs->in[i].sync < level) + fs->in[i].ts_mode = fs->opt_ts_sync_mode; + else + fs->in[i].ts_mode = TS_DEFAULT; + } + } + if (level) + fs->sync_level = level; + else + packetsync_eof(fs, eof_pts); +} + +int ff_packetsync_configure(FFPacketSync *fs) +{ + unsigned i; + + for (i = 1; i < fs->nb_in; i++) { + fs->in[i].after = EXT_NULL; + fs->in[i].sync = 0; + } + if (fs->opt_eof_action == EOF_ACTION_ENDALL) { + for (i = 0; i < fs->nb_in; i++) + fs->in[i].after = EXT_STOP; + } + + if (!fs->time_base.num) { + for (i = 0; i < fs->nb_in; i++) { + if (fs->in[i].sync) { + if (fs->time_base.num) { + fs->time_base = av_gcd_q(fs->time_base, fs->in[i].time_base, + AV_TIME_BASE / 2, AV_TIME_BASE_Q); + } else { + fs->time_base = fs->in[i].time_base; + } + } + } + if (!fs->time_base.num) { + av_log(fs, AV_LOG_ERROR, "Impossible to set time base\n"); + return AVERROR(EINVAL); + } + av_log(fs, AV_LOG_VERBOSE, "Selected %d/%d time base\n", + fs->time_base.num, fs->time_base.den); + } + + for (i = 0; i < fs->nb_in; i++) + fs->in[i].pts = fs->in[i].pts_next = AV_NOPTS_VALUE; + fs->sync_level = UINT_MAX; + packetsync_sync_level_update(fs, AV_NOPTS_VALUE); + + return 0; +} + +static int packetsync_advance(FFPacketSync *fs) +{ + unsigned i; + int64_t pts; + int ret; + + while (!(fs->pkt_ready || fs->eof)) { + ret = consume_from_fifos(fs); + if (ret <= 0) + return ret; + + pts = INT64_MAX; + for (i = 0; i < fs->nb_in; i++) + if (fs->in[i].have_next && fs->in[i].pts_next < pts) + pts = fs->in[i].pts_next; + if (pts == INT64_MAX) { + packetsync_eof(fs, AV_NOPTS_VALUE); + break; + } + for (i = 0; i < fs->nb_in; i++) { + if (fs->in[i].pts_next == pts || + (fs->in[i].ts_mode == TS_NEAREST && + fs->in[i].have_next && + fs->in[i].pts_next != INT64_MAX && fs->in[i].pts != AV_NOPTS_VALUE && + fs->in[i].pts_next - pts < pts - fs->in[i].pts)) { + av_packet_free(&fs->in[i].pkt); + fs->in[i].pkt = fs->in[i].pkt_next; + fs->in[i].pts = fs->in[i].pts_next; + fs->in[i].pkt_next = NULL; + fs->in[i].pts_next = AV_NOPTS_VALUE; + fs->in[i].have_next = 0; + fs->in[i].state = fs->in[i].pkt ? STATE_RUN : STATE_EOF; + if (fs->in[i].sync == fs->sync_level && fs->in[i].pkt) + fs->pkt_ready = 1; + if (fs->in[i].state == STATE_EOF && + fs->in[i].after == EXT_STOP) + packetsync_eof(fs, AV_NOPTS_VALUE); + } + } + if (fs->pkt_ready) + for (i = 0; i < fs->nb_in; i++) + if ((fs->in[i].state == STATE_BOF && + fs->in[i].before == EXT_STOP)) + fs->pkt_ready = 0; + fs->pts = pts; + } + return 0; +} + +static int64_t packetsync_pts_extrapolate(FFPacketSync *fs, unsigned in, + int64_t pts) +{ + return pts + 1; +} + +static void packetsync_inject_packet(FFPacketSync *fs, unsigned in, AVPacket *pkt) +{ + int64_t pts; + + av_assert0(!fs->in[in].have_next); + av_assert0(pkt); + pts = av_rescale_q_rnd(pkt->pts, fs->in[in].time_base, fs->time_base, AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX); + pkt->pts = pts; + fs->in[in].pkt_next = pkt; + fs->in[in].pts_next = pts; + fs->in[in].have_next = 1; +} + +static void packetsync_inject_status(FFPacketSync *fs, unsigned in, int status, int64_t eof_pts) +{ + av_assert0(!fs->in[in].have_next); + fs->in[in].sync = 0; + packetsync_sync_level_update(fs, status == AVERROR_EOF ? eof_pts : AV_NOPTS_VALUE); + fs->in[in].pkt_next = NULL; + fs->in[in].pts_next = fs->in[in].state != STATE_RUN + ? INT64_MAX : packetsync_pts_extrapolate(fs, in, fs->in[in].pts); + fs->in[in].have_next = 1; +} + +int ff_packetsync_get_packet(FFPacketSync *fs, unsigned in, AVPacket **rpkt, + unsigned get) +{ + AVPacket *pkt; + unsigned need_copy = 0, i; + int64_t pts_next; + + if (!fs->in[in].pkt) { + *rpkt = NULL; + return 0; + } + pkt = fs->in[in].pkt; + if (get) { + /* Find out if we need to copy the packet: is there another sync + stream, and do we know if its current packet will outlast this one? */ + pts_next = fs->in[in].have_next ? fs->in[in].pts_next : INT64_MAX; + for (i = 0; i < fs->nb_in && !need_copy; i++) + if (i != in && fs->in[i].sync && + (!fs->in[i].have_next || fs->in[i].pts_next < pts_next)) + need_copy = 1; + if (need_copy) { + if (!(pkt = av_packet_clone(pkt))) + return AVERROR(ENOMEM); + } else { + fs->in[in].pkt = NULL; + } + fs->pkt_ready = 0; + } + *rpkt = pkt; + return 0; +} + +void ff_packetsync_uninit(FFPacketSync *fs) +{ + unsigned i; + + for (i = 0; i < fs->nb_in; i++) { + av_packet_free(&fs->in[i].pkt); + av_packet_free(&fs->in[i].pkt_next); + } + + av_freep(&fs->in); +} + +static int consume_from_fifos(FFPacketSync *fs) +{ + AVBitStreamFilterContext *ctx = fs->parent; + AVPacket *pkt = NULL; + int64_t pts; + unsigned i, nb_active, nb_miss; + int ret, status; + + nb_active = nb_miss = 0; + for (i = 0; i < fs->nb_in; i++) { + if (fs->in[i].have_next || fs->in[i].state == STATE_EOF) + continue; + nb_active++; + ret = ff_bsf_inlink_consume_packet(ctx->inputs[i], &pkt); + if (ret < 0) + return ret; + if (ret) { + av_assert0(pkt); + packetsync_inject_packet(fs, i, pkt); + } else { + ret = ff_bsf_inlink_acknowledge_status(ctx->inputs[i], &status, &pts); + if (ret > 0) { + packetsync_inject_status(fs, i, status, pts); + } else if (!ret) { + nb_miss++; + } + } + } + if (nb_miss) { + if (nb_miss == nb_active && !ff_bsf_outlink_packet_wanted(ctx->outputs[0])) + return FFERROR_BSF_NOT_READY; + for (i = 0; i < fs->nb_in; i++) + if (!fs->in[i].have_next && fs->in[i].state != STATE_EOF) + ff_bsf_inlink_request_packet(ctx->inputs[i]); + return 0; + } + return 1; +} + +int ff_packetsync_activate(FFPacketSync *fs) +{ + AVBitStreamFilterContext *ctx = fs->parent; + int ret; + + ret = ff_bsf_outlink_get_status(ctx->outputs[0]); + if (ret) { + unsigned i; + for (i = 0; i < ctx->nb_inputs; i++) + ff_bsf_inlink_set_status(ctx->inputs[i], ret); + return 0; + } + + ret = packetsync_advance(fs); + if (ret < 0) + return ret; + if (fs->eof || !fs->pkt_ready) + return 0; + ret = fs->on_event(fs); + if (ret < 0) + return ret; + fs->pkt_ready = 0; + + return 0; +} + +int ff_packetsync_init_dualinput(FFPacketSync *fs, AVBitStreamFilterContext *parent) +{ + int ret; + + ret = ff_packetsync_init(fs, parent, 2); + if (ret < 0) + return ret; + fs->in[0].time_base = ff_bsf_link(parent->inputs[0])->time_base; + fs->in[1].time_base = ff_bsf_link(parent->inputs[1])->time_base; + fs->in[0].sync = 2; + fs->in[0].before = EXT_STOP; + fs->in[0].after = EXT_STOP; + fs->in[1].sync = 1; + fs->in[1].before = EXT_NULL; + fs->in[1].after = EXT_NULL; + return 0; +} + +int ff_packetsync_dualinput_get(FFPacketSync *fs, AVPacket **f0, AVPacket **f1) +{ + AVBitStreamFilterContext *ctx = fs->parent; + AVPacket *mainpic = NULL, *secondpic = NULL; + int ret; + + if ((ret = ff_packetsync_get_packet(fs, 0, &mainpic, 1)) < 0 || + (ret = ff_packetsync_get_packet(fs, 1, &secondpic, 0)) < 0) { + av_packet_free(&mainpic); + return ret; + } + av_assert0(mainpic); + mainpic->pts = av_rescale_q(fs->pts, fs->time_base, ff_bsf_link(ctx->outputs[0])->time_base); + *f0 = mainpic; + *f1 = secondpic; + return 0; +} + +int ff_packetsync_dualinput_get_writable(FFPacketSync *fs, AVPacket **f0, AVPacket **f1) +{ + int ret; + + ret = ff_packetsync_dualinput_get(fs, f0, f1); + if (ret < 0) + return ret; + ret = av_packet_make_writable(*f0); + if (ret < 0) { + av_packet_free(f0); + *f1 = NULL; + return ret; + } + return 0; +} diff --git a/libavcodec/bsf/packetsync.h b/libavcodec/bsf/packetsync.h new file mode 100644 index 0000000000..03f7d01c82 --- /dev/null +++ b/libavcodec/bsf/packetsync.h @@ -0,0 +1,345 @@ +/* + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with FFmpeg; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef AVCODEC_BSF_PACKETSYNC_H +#define AVCODEC_BSF_PACKETSYNC_H + +#include <stdint.h> + +#include "libavutil/log.h" + +#include "libavcodec/bsf.h" +#include "libavcodec/packet.h" + +enum EOFAction { + EOF_ACTION_ENDALL, + EOF_ACTION_PASS +}; + +/** + * This API is intended as a helper for filters that have several video + * input and need to combine them somehow. If the inputs have different or + * variable frame rate, getting the input packets to match requires a rather + * complex logic and a few user-tunable options. + * + * In this API, when a set of synchronized input packets is ready to be + * processed is called a packet event. packet event can be generated in + * response to input packets on any or all inputs and the handling of + * situations where some stream extend beyond the beginning or the end of + * others can be configured. + * + * The basic working of this API is the following: set the on_event + * callback, then call ff_packetsync_activate() from the filter's activate + * callback. + */ + +/** + * Stream extrapolation mode + * + * Describe how the packets of a stream are extrapolated before the first one + * and after EOF to keep sync with possibly longer other streams. + */ +enum FFPacketSyncExtMode { + + /** + * Completely stop all streams with this one. + */ + EXT_STOP, + + /** + * Ignore this stream and continue processing the other ones. + */ + EXT_NULL, +}; + +/** + * Timestamp synchronization mode + * + * Describe how the packets of a stream are synchronized based on timestamp + * distance. + */ +enum FFPacketTSSyncMode { + + /** + * Sync to packets from secondary input with the nearest, lower or equal + * timestamp to the packet event one. + */ + TS_DEFAULT, + + /** + * Sync to packets from secondary input with the absolute nearest timestamp + * to the packet event one. + */ + TS_NEAREST, +}; + +/** + * Input stream structure + */ +typedef struct FFPacketSyncIn { + + /** + * Extrapolation mode for timestamps before the first packet + */ + enum FFPacketSyncExtMode before; + + /** + * Extrapolation mode for timestamps after the last packet + */ + enum FFPacketSyncExtMode after; + + /** + * Time base for the incoming packets + */ + AVRational time_base; + + /** + * Current packet, may be NULL before the first one or after EOF + */ + AVPacket *pkt; + + /** + * Next packet, for internal use + */ + AVPacket *pkt_next; + + /** + * PTS of the current packet + */ + int64_t pts; + + /** + * PTS of the next packet, for internal use + */ + int64_t pts_next; + + /** + * Boolean flagging the next packet, for internal use + */ + uint8_t have_next; + + /** + * State: before first, in stream or after EOF, for internal use + */ + uint8_t state; + + /** + * Synchronization level: packets on input at the highest sync level will + * generate output packet events. + * + * For example, if inputs #0 and #1 have sync level 2 and input #2 has + * sync level 1, then a packet on either input #0 or #1 will generate a + * packet event, but not a packet on input #2 until both inputs #0 and #1 + * have reached EOF. + * + * If sync is 0, no packet event will be generated. + */ + unsigned sync; + + enum FFPacketTSSyncMode ts_mode; +} FFPacketSyncIn; + +/** + * Packet sync structure. + */ +typedef struct FFPacketSync { + const AVClass *class; + + /** + * Parent filter context. + */ + AVBitStreamFilterContext *parent; + + /** + * Number of input streams + */ + unsigned nb_in; + + /** + * Time base for the output events + */ + AVRational time_base; + + /** + * Timestamp of the current event + */ + int64_t pts; + + /** + * Callback called when a packet event is ready + */ + int (*on_event)(struct FFPacketSync *fs); + + /** + * Opaque pointer, not used by the API + */ + void *opaque; + + /** + * Index of the input that requires a request + */ + unsigned in_request; + + /** + * Synchronization level: only inputs with the same sync level are sync + * sources. + */ + unsigned sync_level; + + /** + * Flag indicating that a packet event is ready + */ + uint8_t pkt_ready; + + /** + * Flag indicating that output has reached EOF. + */ + uint8_t eof; + + /** + * Pointer to array of inputs. + */ + FFPacketSyncIn *in; + + int opt_eof_action; + int opt_ts_sync_mode; + +} FFPacketSync; + +/** + * Pre-initialize a packet sync structure. + * + * It sets the class pointer and inits the options to their default values. + * The entire structure is expected to be already set to 0. + * This step is optional, but necessary to use the options. + */ +void ff_packetsync_preinit(FFPacketSync *fs); + +/** + * Initialize a packet sync structure. + * + * The entire structure is expected to be already set to 0 or preinited. + * + * @param fs packet sync structure to initialize + * @param parent parent AVBitStreamFilterContext object + * @param nb_in number of inputs + * @return >= 0 for success or a negative error code + */ +int ff_packetsync_init(FFPacketSync *fs, AVBitStreamFilterContext *parent, unsigned nb_in); + +/** + * Configure a packet sync structure. + * + * Must be called after all options are set but before all use. + * + * @return >= 0 for success or a negative error code + */ +int ff_packetsync_configure(FFPacketSync *fs); + +/** + * Free all memory currently allocated. + */ +void ff_packetsync_uninit(FFPacketSync *fs); + +/** + * Get the current packet in an input. + * + * @param fs packet sync structure + * @param in index of the input + * @param rpacket used to return the current packet (or NULL) + * @param get if not zero, the calling code needs to get ownership of + * the returned packet; the current packet will either be + * duplicated or removed from the packetsync structure + */ +int ff_packetsync_get_packet(FFPacketSync *fs, unsigned in, AVPacket **rpacket, + unsigned get); + +/** + * Examine the packets in the filter's input and try to produce output. + * + * This function can be the complete implementation of the activate + * method of a filter using packetsync. + */ +int ff_packetsync_activate(FFPacketSync *fs); + +/** + * Initialize a packet sync structure for dualinput. + * + * Compared to generic packetsync, dualinput assumes the first input is the + * main one and the filtering is performed on it. The first input will be + * the only one with sync set and generic timeline support will just pass it + * unchanged when disabled. + * + * Equivalent to ff_packetsync_init(fs, parent, 2) then setting the time + * base, sync and ext modes on the inputs. + */ +int ff_packetsync_init_dualinput(FFPacketSync *fs, AVBitStreamFilterContext *parent); + +/** + * @param f0 used to return the main packet + * @param f1 used to return the second packet, or NULL if disabled + * @return >=0 for success or AVERROR code + * @note The packet returned in f0 belongs to the caller (get = 1 in + * ff_packetsync_get_packet()) while the packet returned in f1 is still owned + * by the packetsync structure. + */ +int ff_packetsync_dualinput_get(FFPacketSync *fs, AVPacket **f0, AVPacket **f1); + +/** + * Same as ff_packetsync_dualinput_get(), but make sure that f0 is writable. + */ +int ff_packetsync_dualinput_get_writable(FFPacketSync *fs, AVPacket **f0, AVPacket **f1); + +const AVClass *ff_packetsync_child_class_iterate(void **iter); +extern const AVClass ff_packetsync_class; + +#define PACKETSYNC_DEFINE_PURE_CLASS(name, desc, func_prefix, options) \ +static const AVClass name##_class = { \ + .class_name = desc, \ + .item_name = av_default_item_name, \ + .option = options, \ + .version = LIBAVUTIL_VERSION_INT, \ + .category = AV_CLASS_CATEGORY_BITSTREAM_FILTER, \ + .child_class_iterate = ff_packetsync_child_class_iterate, \ + .child_next = func_prefix##_child_next, \ +} + +/* A filter that uses the *_child_next-function from this macro + * is required to initialize the FFPacketSync structure in AVBitStreamFilter.preinit + * via the *_packetsync_preinit function defined alongside it. */ +#define PACKETSYNC_AUXILIARY_FUNCS(func_prefix, context, field) \ +static int func_prefix##_packetsync_preinit(AVBitStreamFilterContext *ctx) \ +{ \ + context *s = ctx->priv_data; \ + ff_packetsync_preinit(&s->field); \ + return 0; \ +} \ +static void *func_prefix##_child_next(void *obj, void *prev) \ +{ \ + context *s = obj; \ + return prev ? NULL : &s->field; \ +} + +#define PACKETSYNC_DEFINE_CLASS_EXT(name, context, field, options) \ +PACKETSYNC_AUXILIARY_FUNCS(name, context, field) \ +PACKETSYNC_DEFINE_PURE_CLASS(name, #name, name, options) + +#define PACKETSYNC_DEFINE_CLASS(name, context, field) \ +PACKETSYNC_DEFINE_CLASS_EXT(name, context, field, name##_options) + +#endif /* AVCODEC_BSF_PACKETSYNC_H */ -- 2.52.0 >From df5794eb94193eae7ad9ce6d34936a74f3d1515f Mon Sep 17 00:00:00 2001 From: James Almer <[email protected]> Date: Wed, 24 Jun 2026 08:28:52 -0300 Subject: [PATCH 3/3] fftools/ffmpeg_demux: merge LCEVC enhancement payloads into the base stream This is a temporary implementation until support for bitstream filter graphs is generically added to the scheduler. Signed-off-by: James Almer <[email protected]> --- fftools/ffmpeg_demux.c | 304 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 273 insertions(+), 31 deletions(-) diff --git a/fftools/ffmpeg_demux.c b/fftools/ffmpeg_demux.c index 2d2dd5a82d..c3f7f3d4ee 100644 --- a/fftools/ffmpeg_demux.c +++ b/fftools/ffmpeg_demux.c @@ -41,6 +41,19 @@ #include "libavformat/avformat.h" +typedef struct DemuxStreamGroup { + InputStreamGroup istg; + + // name used for logging + char log_name[32]; + + // Temporary fields for LCEVC merging + AVBitStreamFilterGraph *graph; + AVBitStreamFilterContext *lcevc[2]; + AVBitStreamFilterContext *sink; + int graph_enabled; +} DemuxStreamGroup; + typedef struct DemuxStream { InputStream ist; @@ -95,6 +108,9 @@ typedef struct DemuxStream { AVBSFContext *bsf; + DemuxStreamGroup **dsg; + int nb_dsg; + /* number of packets successfully read for this stream */ uint64_t nb_packets; // combined size of all the packets read @@ -107,13 +123,6 @@ typedef struct DemuxStream { int64_t lag; } DemuxStream; -typedef struct DemuxStreamGroup { - InputStreamGroup istg; - - // name used for logging - char log_name[32]; -} DemuxStreamGroup; - typedef struct Demuxer { InputFile f; @@ -163,6 +172,11 @@ typedef struct DemuxThreadContext { AVPacket *pkt_bsf; } DemuxThreadContext; +static DemuxStreamGroup *dsg_from_istg(InputStreamGroup *istg) +{ + return (DemuxStreamGroup*)istg; +} + static DemuxStream *ds_from_ist(InputStream *ist) { return (DemuxStream*)ist; @@ -585,34 +599,69 @@ static int do_send(Demuxer *d, DemuxStream *ds, AVPacket *pkt, unsigned flags, return 0; } -static int demux_send(Demuxer *d, DemuxThreadContext *dt, DemuxStream *ds, - AVPacket *pkt, unsigned flags) +static int do_bsf_graph(Demuxer *d, DemuxThreadContext *dt, DemuxStreamGroup *dsg, + DemuxStream *ds, AVPacket *pkt) { - InputFile *f = &d->f; - int ret; + const AVStreamGroup *stg = dsg->istg.stg; + const AVStreamGroupLayeredVideo *lcevc = stg->params.layered_video; + const int enhancement = ds->ist.index == stg->streams[lcevc->el_index]->index; + AVBitStreamFilterContext *source = dsg->lcevc[enhancement]; + int ret, flags = AV_BSF_SOURCE_FLAG_PUSH; - // pkt can be NULL only when flushing BSFs - av_assert0(ds->bsf || pkt); + if (enhancement) + flags |= AV_BSF_SOURCE_FLAG_KEEP_REF; - // send heartbeat for sub2video streams - if (d->pkt_heartbeat && pkt && pkt->pts != AV_NOPTS_VALUE) { - for (int i = 0; i < f->nb_streams; i++) { - DemuxStream *ds1 = ds_from_ist(f->streams[i]); + ret = av_bsf_source_add_packet(source, pkt, flags); + if (ret < 0) { + if (pkt) + av_packet_unref(pkt); + av_log(dsg, AV_LOG_ERROR, "Error submitting a packet for filtering: %s\n", + av_err2str(ret)); + return ret; + } - if (ds1->finished || !ds1->have_sub2video) - continue; - - d->pkt_heartbeat->pts = pkt->pts; - d->pkt_heartbeat->time_base = pkt->time_base; - d->pkt_heartbeat->opaque = (void*)(intptr_t)PKT_OPAQUE_SUB_HEARTBEAT; - - ret = do_send(d, ds1, d->pkt_heartbeat, 0, "heartbeat"); - if (ret < 0) + if (pkt && enhancement) { + if (ds->discard) + av_packet_unref(pkt); + else { + ret = do_send(d, ds, pkt, 0, "filtered"); + if (ret < 0) { + av_packet_unref(pkt); return ret; + } + } + return 0; + } + + while (1) { + ret = av_bsf_sink_get_packet(dsg->sink, dt->pkt_bsf, 0); + if (ret == AVERROR(EAGAIN)) + return 0; + else if (ret < 0) { + if (ret != AVERROR_EOF) + av_log(dsg, AV_LOG_ERROR, + "Error applying bitstream filters to a packet: %s\n", + av_err2str(ret)); + return ret; + } + + dt->pkt_bsf->time_base = av_bsf_sink_get_time_base(dsg->sink); + + ret = do_send(d, ds, dt->pkt_bsf, 0, "filtered"); + if (ret < 0) { + av_packet_unref(dt->pkt_bsf); + return ret; } } - if (ds->bsf) { + return 0; +} + +static int do_bsf(Demuxer *d, DemuxThreadContext *dt, DemuxStream *ds, + AVPacket *pkt) +{ + int ret; + if (pkt) av_packet_rescale_ts(pkt, pkt->time_base, ds->bsf->time_base_in); @@ -645,7 +694,59 @@ static int demux_send(Demuxer *d, DemuxThreadContext *dt, DemuxStream *ds, return ret; } } - } else { + + return 0; +} + +static int demux_send(Demuxer *d, DemuxThreadContext *dt, DemuxStream *ds, + AVPacket *pkt, unsigned flags) +{ + InputFile *f = &d->f; + DemuxStreamGroup *dsg = NULL; + int ret; + + for (int i = 0; i < ds->nb_dsg; i++) { + const InputStreamGroup *istg = &ds->dsg[i]->istg; + + if (istg->stg->type != AV_STREAM_GROUP_PARAMS_LCEVC) + continue; + dsg = ds->dsg[i]; + break; + } + + // pkt can be NULL only when flushing BSFs + av_assert0(ds->bsf || (dsg && dsg->graph) || pkt); + + // send heartbeat for sub2video streams + if (d->pkt_heartbeat && pkt && pkt->pts != AV_NOPTS_VALUE) { + for (int i = 0; i < f->nb_streams; i++) { + DemuxStream *ds1 = ds_from_ist(f->streams[i]); + + if (ds1->finished || !ds1->have_sub2video) + continue; + + d->pkt_heartbeat->pts = pkt->pts; + d->pkt_heartbeat->time_base = pkt->time_base; + d->pkt_heartbeat->opaque = (void*)(intptr_t)PKT_OPAQUE_SUB_HEARTBEAT; + + ret = do_send(d, ds1, d->pkt_heartbeat, 0, "heartbeat"); + if (ret < 0) + return ret; + } + } + + if (dsg && dsg->graph_enabled) { + ret = do_bsf_graph(d, dt, dsg, ds, pkt); + if (ret < 0) + return ret; + } + if (ds->bsf) { + ret = do_bsf(d, dt, ds, pkt); + if (ret < 0) + return ret; + } else if (ds->discard && pkt) { + av_packet_unref(pkt); + } else if (!dsg || !dsg->graph_enabled) { ret = do_send(d, ds, pkt, flags, "demuxed"); if (ret < 0) return ret; @@ -661,8 +762,18 @@ static int demux_bsf_flush(Demuxer *d, DemuxThreadContext *dt) for (unsigned i = 0; i < f->nb_streams; i++) { DemuxStream *ds = ds_from_ist(f->streams[i]); + DemuxStreamGroup *dsg = NULL; - if (!ds->bsf) + for (int j = 0; j < ds->nb_dsg; j++) { + const InputStreamGroup *istg = &ds->dsg[j]->istg; + + if (istg->stg->type != AV_STREAM_GROUP_PARAMS_LCEVC) + continue; + dsg = ds->dsg[j]; + break; + } + + if (!ds->bsf && (!dsg || !dsg->graph_enabled)) continue; ret = demux_send(d, dt, ds, NULL, 0); @@ -673,7 +784,8 @@ static int demux_bsf_flush(Demuxer *d, DemuxThreadContext *dt) return ret; } - av_bsf_flush(ds->bsf); + if (ds->bsf) + av_bsf_flush(ds->bsf); } return 0; @@ -796,7 +908,7 @@ static int input_thread(void *arg) dynamically in stream : we ignore them */ ds = dt.pkt_demux->stream_index < f->nb_streams ? ds_from_ist(f->streams[dt.pkt_demux->stream_index]) : NULL; - if (!ds || ds->discard || ds->finished) { + if (!ds || ds->finished) { report_new_stream(d, dt.pkt_demux); av_packet_unref(dt.pkt_demux); continue; @@ -902,10 +1014,15 @@ static void ist_free(InputStream **pist) static void istg_free(InputStreamGroup **pistg) { InputStreamGroup *istg = *pistg; + DemuxStreamGroup *dsg; if (!istg) return; + dsg = dsg_from_istg(istg); + + av_bsf_graph_free(&dsg->graph); + av_freep(pistg); } @@ -971,6 +1088,53 @@ int ist_use(InputStream *ist, int decoding_needed, ds->decoding_needed |= decoding_needed; ds->streamcopy_needed |= !decoding_needed; + for (int i = 0; i < ds->nb_dsg; i++) { + DemuxStreamGroup *dsg = ds->dsg[i]; + const InputStreamGroup *istg = &dsg->istg; + + if (!dsg->graph || istg->stg->type != AV_STREAM_GROUP_PARAMS_LCEVC) + continue; + const AVStreamGroupLayeredVideo *lcevc = istg->stg->params.layered_video; + if (ist->st->index == istg->stg->streams[lcevc->el_index]->index) + break; + + AVBitStreamFilterContext *lcevc_merge = av_bsf_graph_get_filter(dsg->graph, "lcevc_merge"); + + for (int j = 0; j < 2; j++) { + ret = av_bsf_init_dict(dsg->lcevc[j], NULL); + if (ret < 0) + return ret; + } + + ret = av_bsf_init_dict(lcevc_merge, NULL); + if (ret < 0) + return ret; + ret = av_bsf_init_dict(dsg->sink, NULL); + if (ret < 0) + return ret; + + ret = av_bsf_link(dsg->lcevc[0], 0, lcevc_merge, 0); + if (ret < 0) + return ret; + ret = av_bsf_link(dsg->lcevc[1], 0, lcevc_merge, 1); + if (ret < 0) + return ret; + ret = av_bsf_link(lcevc_merge, 0, dsg->sink, 0); + if (ret < 0) + return ret; + + ret = av_bsf_graph_config(dsg->graph, d); + if (ret < 0) + return ret; + + InputStream *lcevc_ist = d->f.streams[istg->stg->streams[lcevc->el_index]->index]; + lcevc_ist->st->discard = 0; + + dsg->graph_enabled = 1; + + break; + } + if (decoding_needed && ds->sch_idx_dec < 0) { int is_audio = ist->st->codecpar->codec_type == AVMEDIA_TYPE_AUDIO; int is_unreliable = !!(d->f.ctx->iformat->flags & AVFMT_NOTIMESTAMPS); @@ -1860,8 +2024,68 @@ fail: return ret; } +static int istg_parse_lcevc(const OptionsContext *o, Demuxer *d, DemuxStreamGroup *dsg) +{ + InputFile *f = &d->f; + const AVStreamGroup *stg = dsg->istg.stg; + const AVStreamGroupLayeredVideo *lcevc = stg->params.layered_video; + const AVBitStreamFilter *filter, *lcevc_filter = av_bsf_get_by_name("lcevc_merge"); + const InputStream *lcevc_ist = f->streams[stg->streams[lcevc->el_index]->index]; + const InputStream *base_ist = f->streams[stg->streams[!lcevc->el_index]->index]; + int ret; + + if (!lcevc_filter || stg->nb_streams != 2) + return 0; + + dsg->graph = av_bsf_graph_alloc(); + if(!dsg->graph) + return AVERROR(ENOMEM); + + filter = av_bsf_get_by_name("source"); + if (!filter) + return AVERROR_BUG; + + dsg->lcevc[0] = av_bsf_graph_alloc_filter(dsg->graph, filter, "lcevc_merge_base"); + if (!dsg->lcevc[0]) + return AVERROR(ENOMEM); + av_opt_set_q(dsg->lcevc[0]->priv_data, "time_base", base_ist->st->time_base, 0); + ret = av_bsf_source_parameters_set(dsg->lcevc[0], base_ist->par); + if (ret < 0) + return ret; + + dsg->lcevc[1] = av_bsf_graph_alloc_filter(dsg->graph, filter, "lcevc_merge_enhancement"); + if (!dsg->lcevc[1]) + return AVERROR(ENOMEM); + av_opt_set_q(dsg->lcevc[1]->priv_data, "time_base", lcevc_ist->st->time_base, 0); + ret = av_bsf_source_parameters_set(dsg->lcevc[1], lcevc_ist->par); + if (ret < 0) + return ret; + + AVBitStreamFilterContext *lcevc_merge = av_bsf_graph_alloc_filter(dsg->graph, lcevc_filter, "lcevc_merge"); + if (!lcevc_merge) + return AVERROR(ENOMEM); + + filter = av_bsf_get_by_name("sink"); + if (!filter) + return AVERROR_BUG; + dsg->sink = av_bsf_graph_alloc_filter(dsg->graph, filter, "lcevc_merge_sink"); + if (!dsg->sink) + return AVERROR(ENOMEM); + + if (lcevc_ist->par->extradata_size > 4) { + int is_lvcc = !!lcevc_ist->par->extradata[0]; + + if (is_lvcc) + av_opt_set_int(lcevc_merge->priv_data, "nal_length_size", + (lcevc_ist->par->extradata[4] >> 6) + 1, 0); + } + + return 0; +} + static int istg_add(const OptionsContext *o, Demuxer *d, AVStreamGroup *stg) { + InputFile *f = &d->f; DemuxStreamGroup *dsg; InputStreamGroup *istg; int ret; @@ -1872,12 +2096,30 @@ static int istg_add(const OptionsContext *o, Demuxer *d, AVStreamGroup *stg) istg = &dsg->istg; + switch (stg->type) { + case AV_STREAM_GROUP_PARAMS_LCEVC: + for (int i = 0; i < istg->stg->nb_streams; i++) { + DemuxStream *ds = ds_from_ist(f->streams[istg->stg->streams[i]->index]); + ret = av_dynarray_add_nofree(&ds->dsg, &ds->nb_dsg, dsg); + if (ret < 0) + return ret; + } + break; + default: + break; + } + switch (stg->type) { case AV_STREAM_GROUP_PARAMS_TILE_GRID: ret = istg_parse_tile_grid(o, d, istg); if (ret < 0) return ret; break; + case AV_STREAM_GROUP_PARAMS_LCEVC: + ret = istg_parse_lcevc(o, d, dsg); + if (ret < 0) + return ret; + break; default: break; } -- 2.52.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
