PR #25254 opened by James Almer (jamrial) URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/25254 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/25254.patch
Used by formats like HEIC, where the Alpha channel for the derived picture is stored as the luma channel from a separate stream. >From fe829b037623b4edd6bc57be764411308a6ef7d6 Mon Sep 17 00:00:00 2001 From: James Almer <[email protected]> Date: Mon, 19 Oct 2026 11:19:57 -0300 Subject: [PATCH 1/6] avformat: add an split Color and Alpha channel Stream Group Signed-off-by: James Almer <[email protected]> --- libavformat/avformat.c | 2 ++ libavformat/avformat.h | 1 + libavformat/options.c | 3 +++ 3 files changed, 6 insertions(+) diff --git a/libavformat/avformat.c b/libavformat/avformat.c index e8ebb89097b..aca8c4834fd 100644 --- a/libavformat/avformat.c +++ b/libavformat/avformat.c @@ -123,6 +123,7 @@ FF_ENABLE_DEPRECATION_WARNINGS case AV_STREAM_GROUP_PARAMS_LCEVC: case AV_STREAM_GROUP_PARAMS_DOLBY_VISION: case AV_STREAM_GROUP_PARAMS_GAIN_MAP: + case AV_STREAM_GROUP_PARAMS_ALPHA: av_opt_free(stg->params.layered_video); av_freep(&stg->params.layered_video); break; @@ -290,6 +291,7 @@ const char *avformat_stream_group_name(enum AVStreamGroupParamsType type) case AV_STREAM_GROUP_PARAMS_TREF: return "Track Reference"; case AV_STREAM_GROUP_PARAMS_DOLBY_VISION: return "Dolby Vision (Split base and enhancement layer)"; case AV_STREAM_GROUP_PARAMS_GAIN_MAP: return "Gain Map (Split base rendition and gain map)"; + case AV_STREAM_GROUP_PARAMS_ALPHA: return "Alpha (Split color and alpha channels)"; } return NULL; } diff --git a/libavformat/avformat.h b/libavformat/avformat.h index fc46ae6b8d0..8a21c6bbfcf 100644 --- a/libavformat/avformat.h +++ b/libavformat/avformat.h @@ -1165,6 +1165,7 @@ enum AVStreamGroupParamsType { AV_STREAM_GROUP_PARAMS_TREF, AV_STREAM_GROUP_PARAMS_DOLBY_VISION, AV_STREAM_GROUP_PARAMS_GAIN_MAP, + AV_STREAM_GROUP_PARAMS_ALPHA, }; struct AVIAMFAudioElement; diff --git a/libavformat/options.c b/libavformat/options.c index df7b6434d8a..e6ab0a7e38f 100644 --- a/libavformat/options.c +++ b/libavformat/options.c @@ -404,6 +404,7 @@ static void *stream_group_child_next(void *obj, void *prev) case AV_STREAM_GROUP_PARAMS_LCEVC: case AV_STREAM_GROUP_PARAMS_DOLBY_VISION: case AV_STREAM_GROUP_PARAMS_GAIN_MAP: + case AV_STREAM_GROUP_PARAMS_ALPHA: return stg->params.layered_video; default: break; @@ -438,6 +439,7 @@ static const AVClass *stream_group_child_iterate(void **opaque) case AV_STREAM_GROUP_PARAMS_LCEVC: case AV_STREAM_GROUP_PARAMS_DOLBY_VISION: case AV_STREAM_GROUP_PARAMS_GAIN_MAP: + case AV_STREAM_GROUP_PARAMS_ALPHA: ret = &layered_video_class; break; default: @@ -518,6 +520,7 @@ AVStreamGroup *avformat_stream_group_create(AVFormatContext *s, case AV_STREAM_GROUP_PARAMS_LCEVC: case AV_STREAM_GROUP_PARAMS_DOLBY_VISION: case AV_STREAM_GROUP_PARAMS_GAIN_MAP: + case AV_STREAM_GROUP_PARAMS_ALPHA: stg->params.layered_video = av_mallocz(sizeof(*stg->params.layered_video)); if (!stg->params.layered_video) goto fail; -- 2.54.0 >From 1f3aba927cbf93a71760ce2c4daac3c099bf0e8c Mon Sep 17 00:00:00 2001 From: James Almer <[email protected]> Date: Mon, 19 Oct 2026 11:20:16 -0300 Subject: [PATCH 2/6] avformat/dump: print Alpha channel Stream Group information And merge all the Layered Video based group types into a single block. Signed-off-by: James Almer <[email protected]> --- libavformat/dump.c | 77 ++++++++++++++++++++++------------------------ 1 file changed, 36 insertions(+), 41 deletions(-) diff --git a/libavformat/dump.c b/libavformat/dump.c index c2f9fc6f7bb..6bd6b3462c3 100644 --- a/libavformat/dump.c +++ b/libavformat/dump.c @@ -853,16 +853,33 @@ static void dump_stream_group(const AVFormatContext *ic, uint8_t *printed, } break; } + case AV_STREAM_GROUP_PARAMS_ALPHA: case AV_STREAM_GROUP_PARAMS_DOLBY_VISION: + case AV_STREAM_GROUP_PARAMS_GAIN_MAP: case AV_STREAM_GROUP_PARAMS_LCEVC: { const AVStreamGroupLayeredVideo *layered_video = stg->params.layered_video; AVCodecContext *avctx = avcodec_alloc_context3(NULL); const char *ptr = NULL; - av_log(NULL, AV_LOG_INFO, " %s:", stg->type == AV_STREAM_GROUP_PARAMS_DOLBY_VISION ? - "Dolby Vision" : "LCEVC"); + switch (stg->type) { + case AV_STREAM_GROUP_PARAMS_ALPHA: + av_log(NULL, AV_LOG_INFO, " Alpha:"); + break; + case AV_STREAM_GROUP_PARAMS_DOLBY_VISION: + av_log(NULL, AV_LOG_INFO, " Dolby Vision:"); + break; + case AV_STREAM_GROUP_PARAMS_GAIN_MAP: + av_log(NULL, AV_LOG_INFO, " Gain Map:"); + break; + case AV_STREAM_GROUP_PARAMS_LCEVC: + av_log(NULL, AV_LOG_INFO, " LCEVC:"); + break; + } if (avctx && stg->nb_entries == 2 && - stg->entries[!layered_video->el_index]->type == AV_STREAM_GROUP_ENTRY_STREAM) { - const AVStream *st = stg->entries[!layered_video->el_index]->stream; + (stg->entries[ layered_video->el_index]->type == AV_STREAM_GROUP_ENTRY_STREAM || + stg->entries[!layered_video->el_index]->type == AV_STREAM_GROUP_ENTRY_STREAM)) { + const AVStream *st = stg->entries[!layered_video->el_index]->type == AV_STREAM_GROUP_ENTRY_STREAM ? + stg->entries[!layered_video->el_index]->stream : + stg->entries[ layered_video->el_index]->stream; if (!avcodec_parameters_to_context(avctx, st->codecpar)) { avctx->width = layered_video->width; avctx->height = layered_video->height; @@ -874,6 +891,9 @@ static void dump_stream_group(const AVFormatContext *ic, uint8_t *printed, avcodec_string(buf, sizeof(buf), avctx, is_output); ptr = av_stristr(buf, " "); } + } else { + buf[0] = 0; + av_strlcatf(buf, sizeof(buf), " %dx%d", layered_video->width, layered_video->height); } avcodec_free_context(&avctx); if (ptr) @@ -881,43 +901,6 @@ static void dump_stream_group(const AVFormatContext *ic, uint8_t *printed, dump_disposition(stg->disposition, log_level); av_log(NULL, log_level, "\n"); dump_metadata(NULL, stg->metadata, extra_indent, log_level); - dump_sidedata(NULL, stg->coded_side_data, stg->nb_coded_side_data, - layered_video->width, layered_video->height, (AVRational) {0,1}, - extra_indent, log_level); - for (int i = 0; i < stg->nb_entries; i++) { - const AVStreamGroupEntry *entry = stg->entries[i]; - if (entry->type != AV_STREAM_GROUP_ENTRY_STREAM) - continue; - const AVStream *st = entry->stream; - dump_stream_format(ic, st->index, i, index, is_output, AV_LOG_VERBOSE); - printed[st->index] = 1; - } - break; - } - case AV_STREAM_GROUP_PARAMS_TREF: { - av_log(NULL, log_level, " Track Reference:\n"); - for (int i = 0; i < stg->nb_entries; i++) { - const AVStreamGroupEntry *entry = stg->entries[i]; - if (entry->type != AV_STREAM_GROUP_ENTRY_STREAM) - continue; - const AVStream *st = entry->stream; - dump_stream_format(ic, st->index, i, index, is_output, log_level); - printed[st->index] = 1; - } - break; - } - case AV_STREAM_GROUP_PARAMS_GAIN_MAP: { - const AVStreamGroupLayeredVideo *layered_video = stg->params.layered_video; - const char *ptr = NULL; - av_log(NULL, AV_LOG_INFO, " Gain Map:"); - buf[0] = 0; - av_strlcatf(buf, sizeof(buf), " %dx%d", layered_video->width, layered_video->height); - ptr = av_stristr(buf, " "); - if (ptr) - av_log(NULL, log_level, "%s", ptr); - dump_disposition(stg->disposition, log_level); - av_log(NULL, log_level, "\n"); - dump_metadata(NULL, stg->metadata, extra_indent, log_level); dump_sidedata(NULL, stg->coded_side_data, stg->nb_coded_side_data, layered_video->width, layered_video->height, (AVRational) {0,1}, extra_indent, log_level); @@ -933,6 +916,18 @@ static void dump_stream_group(const AVFormatContext *ic, uint8_t *printed, } break; } + case AV_STREAM_GROUP_PARAMS_TREF: { + av_log(NULL, log_level, " Track Reference:\n"); + for (int i = 0; i < stg->nb_entries; i++) { + const AVStreamGroupEntry *entry = stg->entries[i]; + if (entry->type != AV_STREAM_GROUP_ENTRY_STREAM) + continue; + const AVStream *st = entry->stream; + dump_stream_format(ic, st->index, i, index, is_output, log_level); + printed[st->index] = 1; + } + break; + } default: break; } -- 2.54.0 >From 02dd21aa01c550079f51219fba536e2a46a41942 Mon Sep 17 00:00:00 2001 From: James Almer <[email protected]> Date: Mon, 19 Oct 2026 11:24:37 -0300 Subject: [PATCH 3/6] avformat/mov: fix parsing auxl derived layered video stream groups if they don't reference a tile grid mov_parse_heif_layered_video() needs to be called even if nb_heif_group is 0, otherwise auxl derived groups might be skipped. Signed-off-by: James Almer <[email protected]> --- libavformat/mov.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/libavformat/mov.c b/libavformat/mov.c index bb4c112f55c..d24185608d1 100644 --- a/libavformat/mov.c +++ b/libavformat/mov.c @@ -11986,15 +11986,13 @@ static int mov_parse_heif_items(AVFormatContext *s) mov_build_index(mov, st); } - if (mov->nb_heif_group) { - err = mov_parse_tiles(s); - if (err < 0) - return err; + err = mov_parse_tiles(s); + if (err < 0) + return err; - err = mov_parse_heif_layered_video(s); - if (err < 0) - return err; - } + err = mov_parse_heif_layered_video(s); + if (err < 0) + return err; return 0; } -- 2.54.0 >From f69b15b7ec553a0b19a9e15d5d5f3a7636ad01a7 Mon Sep 17 00:00:00 2001 From: James Almer <[email protected]> Date: Mon, 19 Oct 2026 12:38:46 -0300 Subject: [PATCH 4/6] avformat/mov: export Alpha type Stream Groups Signed-off-by: James Almer <[email protected]> --- libavformat/isom.h | 1 + libavformat/mov.c | 86 ++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 77 insertions(+), 10 deletions(-) diff --git a/libavformat/isom.h b/libavformat/isom.h index 2af2d73df26..18c2fe41b9a 100644 --- a/libavformat/isom.h +++ b/libavformat/isom.h @@ -315,6 +315,7 @@ typedef struct HEIFItem { int vflip; int type; int is_gainmap; + int is_alpha; int is_idat_relative; AVExifMetadata ifd; uint8_t *icc_profile; diff --git a/libavformat/mov.c b/libavformat/mov.c index d24185608d1..a6b1590f4eb 100644 --- a/libavformat/mov.c +++ b/libavformat/mov.c @@ -9450,9 +9450,13 @@ static int mov_read_auxc(MOVContext *c, AVIOContext *pb, MOVAtom atom) c->cur_item_id, aux_type.str); item = get_heif_item(c, c->cur_item_id); - if (item && (av_stristr(aux_type.str, "hdrgainmap"))) - item->is_gainmap = 1; - + if (item) { + if ((av_stristr(aux_type.str, "hdrgainmap"))) + item->is_gainmap = 1; + else if (strstr(aux_type.str, "urn:mpeg:hevc:2015:auxid:1") || + strstr(aux_type.str, "urn:mpeg:mpegB:cicp:systems:auxiliary:alpha")) + item->is_alpha = 1; + } return 0; } @@ -11755,9 +11759,66 @@ fail: return err; } +static int mov_parse_heif_layered_video_auxl_alpha(AVFormatContext *s, + HEIFItem *base, HEIFItem *alpha) +{ + AVStreamGroupLayeredVideo *layered_video; + AVStreamGroup *stg; + int err; + + if ((!alpha->st && !alpha->stg) || + (!base->st && !base->stg)) { + av_log(s, AV_LOG_WARNING, "HEIF alpha id %d or its base doesn't reference " + "a stream or group\n", + alpha->item_id); + return 0; + } + + stg = avformat_stream_group_create(s, AV_STREAM_GROUP_PARAMS_ALPHA, NULL); + if (!stg) + return AVERROR(ENOMEM); + + stg->id = alpha->item_id; + layered_video = stg->params.layered_video; + + layered_video->width = base->width; + layered_video->height = base->height; + layered_video->el_index = 1; + + av_assert0(!base->st != !base->stg); + if (base->st) + err = avformat_stream_group_add_stream(stg, base->st); + else + err = avformat_stream_group_add_stream_group(stg, base->stg); + if (err < 0) { + if (err == AVERROR(EEXIST)) + err = AVERROR_INVALIDDATA; + return err; + } + + av_assert0(!alpha->st != !alpha->stg); + if (alpha->st) + err = avformat_stream_group_add_stream(stg, alpha->st); + else + err = avformat_stream_group_add_stream_group(stg, alpha->stg); + if (err < 0) { + if (err == AVERROR(EEXIST)) + err = AVERROR_INVALIDDATA; + return err; + } + + // Apply all references from the base image to the Alpha group + err = mov_apply_heif_item_references(s, base, &stg->coded_side_data, + &stg->nb_coded_side_data); + if (err < 0) + return err; + + return 0; +} + // Parse Apple style Gain Map metadata from Exif values // See https://developer.apple.com/documentation/appkit/applying-apple-hdr-effect-to-your-photos -static int mov_parse_heif_layered_video_auxl(AVFormatContext *s, +static int mov_parse_heif_layered_video_auxl_gmap(AVFormatContext *s, HEIFItem *base, HEIFItem *gain_map) { AVStreamGroupLayeredVideo *layered_video; @@ -11918,13 +11979,18 @@ static int mov_parse_heif_layered_video(AVFormatContext *s) av_assert0(ref); switch(item->iref_to_list[j].type) { case MKTAG('a','u','x','l'): - if (!item->is_gainmap) + if (item->is_alpha) { + err = mov_parse_heif_layered_video_auxl_alpha(s, ref, item); + if (err < 0) + return err; break; - - err = mov_parse_heif_layered_video_auxl(s, ref, item); - if (err < 0) - return err; - break; + } + if (item->is_gainmap) { + err = mov_parse_heif_layered_video_auxl_gmap(s, ref, item); + if (err < 0) + return err; + break; + } } } } -- 2.54.0 >From 6e39eb6ff9dfa2937e38bc17a75afc3894626fac Mon Sep 17 00:00:00 2001 From: James Almer <[email protected]> Date: Mon, 19 Oct 2026 19:05:29 -0300 Subject: [PATCH 5/6] avformat/mov: simplify setting Exif side data Signed-off-by: James Almer <[email protected]> --- libavformat/mov.c | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/libavformat/mov.c b/libavformat/mov.c index a6b1590f4eb..3f8c3f93302 100644 --- a/libavformat/mov.c +++ b/libavformat/mov.c @@ -11113,28 +11113,22 @@ static int set_exif_from_item(AVFormatContext *s, AV_PKT_DATA_EXIF); const AVExifMetadata *ifd = &item->ifd; AVBufferRef *buf = NULL; - int64_t offset; int err; if (sd) return 0; - err = av_exif_write(s, ifd, &buf, AV_EXIF_T_OFF); + err = av_exif_write(s, ifd, &buf, AV_EXIF_TIFF_HEADER); if (err < 0) return err; - offset = AV_RB32(buf->data) + 4; - if (offset >= buf->size) { - av_buffer_unref(&buf); - return AVERROR_INVALIDDATA; - } sd = av_packet_side_data_new(coded_side_data, nb_coded_side_data, - AV_PKT_DATA_EXIF, buf->size - offset, 0); + AV_PKT_DATA_EXIF, buf->size, 0); if (!sd) { av_buffer_unref(&buf); return AVERROR(ENOMEM); } - memcpy(sd->data, buf->data + offset, buf->size - offset); + memcpy(sd->data, buf->data, buf->size); av_buffer_unref(&buf); -- 2.54.0 >From aec3cd5a34909d795eb9552c0cb6947fade6f8ef Mon Sep 17 00:00:00 2001 From: James Almer <[email protected]> Date: Tue, 20 Oct 2026 10:40:22 -0300 Subject: [PATCH 6/6] fftools/ffmpeg: automatically apply Alpha layer on Alpha Stream Group input Signed-off-by: James Almer <[email protected]> --- fftools/ffmpeg_demux.c | 62 +++++++++++++++++++++++++++++++++++++++ fftools/ffmpeg_mux_init.c | 12 ++++++++ 2 files changed, 74 insertions(+) diff --git a/fftools/ffmpeg_demux.c b/fftools/ffmpeg_demux.c index f460d38b305..5409dcdc6c0 100644 --- a/fftools/ffmpeg_demux.c +++ b/fftools/ffmpeg_demux.c @@ -1942,6 +1942,63 @@ static DemuxStreamGroup *demux_stream_group_alloc(Demuxer *d, AVStreamGroup *stg return dsg; } +static int istg_parse_alpha(const OptionsContext *o, Demuxer *d, InputStreamGroup *istg) +{ + InputFile *f = &d->f; + AVStreamGroup *stg = istg->stg; + const AVStreamGroupLayeredVideo *layered_video = stg->params.layered_video; + OutputFilterOptions opts; + AVBPrint bp; + char *graph_str; + int ret; + + if (stg->nb_entries != 2) { + av_log(istg, AV_LOG_FATAL, "A demuxer exported an invalid Alpha stream group. " + "This is a bug, please report it.\n"); + return AVERROR_BUG; + } + + memset(&opts, 0, sizeof(opts)); + + av_bprint_init(&bp, 0, AV_BPRINT_SIZE_UNLIMITED); + if (stg->entries[!layered_video->el_index]->type == AV_STREAM_GROUP_ENTRY_STREAM) + av_bprintf(&bp, "[%d:g:%d:%d]", f->index, stg->index, !layered_video->el_index); + else + av_bprintf(&bp, "[%d:g:%d]", f->index, + stg->entries[!layered_video->el_index]->stream_group->index); + if (stg->entries[layered_video->el_index]->type == AV_STREAM_GROUP_ENTRY_STREAM) + av_bprintf(&bp, "[%d:g:%d:%d]", f->index, stg->index, layered_video->el_index); + else + av_bprintf(&bp, "[%d:g:%d]", f->index, + stg->entries[layered_video->el_index]->stream_group->index); + av_bprintf(&bp, "alphamerge[%d:g:%d]", f->index, stg->index); + ret = av_bprint_finalize(&bp, &graph_str); + if (ret < 0) + return ret; + + for (int i = 0; i < stg->nb_coded_side_data; i++) { + const AVPacketSideData *sd = &stg->coded_side_data[i]; + + ret = av_packet_side_data_to_frame(&opts.side_data, &opts.nb_side_data, sd, 0); + if (ret < 0 && ret != AVERROR(EINVAL)) + goto fail; + } + + ret = fg_create(NULL, &graph_str, d->sch, &opts); + if (ret < 0) + goto fail; + + istg->fg = filtergraphs[nb_filtergraphs-1]; + istg->fg->is_internal = 1; + + ret = 0; +fail: + if (ret < 0) + av_freep(&graph_str); + + return ret; +} + static int istg_parse_tile_grid(const OptionsContext *o, Demuxer *d, InputStreamGroup *istg) { InputFile *f = &d->f; @@ -2114,6 +2171,11 @@ static int istg_add(const OptionsContext *o, Demuxer *d, AVStreamGroup *stg) } switch (stg->type) { + case AV_STREAM_GROUP_PARAMS_ALPHA: + ret = istg_parse_alpha(o, d, istg); + if (ret < 0) + return ret; + break; case AV_STREAM_GROUP_PARAMS_TILE_GRID: ret = istg_parse_tile_grid(o, d, istg); if (ret < 0) diff --git a/fftools/ffmpeg_mux_init.c b/fftools/ffmpeg_mux_init.c index 27d64923888..8985e24cf86 100644 --- a/fftools/ffmpeg_mux_init.c +++ b/fftools/ffmpeg_mux_init.c @@ -1707,6 +1707,18 @@ static int map_auto_video(Muxer *mux, const OptionsContext *o) + 5000000*!!(istg->stg->disposition & AV_DISPOSITION_DEFAULT); break; } + case AV_STREAM_GROUP_PARAMS_ALPHA: { + const AVStreamGroupLayeredVideo *layered_video = istg->stg->params.layered_video; + for (int j = 0; j < istg->stg->nb_entries; j++) { + int disposition = istg->stg->entries[j]->type == AV_STREAM_GROUP_ENTRY_STREAM + ? istg->stg->entries[j]->stream->disposition + : istg->stg->entries[j]->stream_group->disposition; + + score += 5000000*!!(disposition & AV_DISPOSITION_DEFAULT) + 1; + } + score += layered_video->width * (int64_t)layered_video->height; + break; + } default: continue; } -- 2.54.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
