PR #24879 opened by Niklas Haas (haasn) URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24879 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24879.patch
And one small fix for the jpeg mpf muxer. >From 08c4c1a8f2c2003bbeda69092b323d1f14f073cf Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Fri, 2 Oct 2026 16:31:57 +0200 Subject: [PATCH 1/7] avformat/dump: add AV_STREAM_GROUP_PARAMS_GAIN_MAP These were introduced by caa5824750, but the corresponding printout was never added; leading to them being printed with a bare label. Signed-off-by: Niklas Haas <[email protected]> --- libavformat/dump.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/libavformat/dump.c b/libavformat/dump.c index 6a1ad91516..c82f076fba 100644 --- a/libavformat/dump.c +++ b/libavformat/dump.c @@ -844,6 +844,15 @@ static void dump_stream_group(const AVFormatContext *ic, uint8_t *printed, } break; } + case AV_STREAM_GROUP_PARAMS_GAIN_MAP: { + av_log(NULL, AV_LOG_INFO, " Gain Map:\n"); + for (int i = 0; i < stg->nb_streams; i++) { + const AVStream *st = stg->streams[i]; + dump_stream_format(ic, st->index, i, index, is_output, AV_LOG_INFO); + printed[st->index] = 1; + } + break; + } default: break; } -- 2.52.0 >From a09ade8cd34394d35e86e2c7a89022d93d1e90d9 Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Fri, 2 Oct 2026 16:34:04 +0200 Subject: [PATCH 2/7] fftools/ffprobe: print gain map parameters Signed-off-by: Niklas Haas <[email protected]> --- fftools/ffprobe.c | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/fftools/ffprobe.c b/fftools/ffprobe.c index 3051fe43b5..b1a3002c65 100644 --- a/fftools/ffprobe.c +++ b/fftools/ffprobe.c @@ -42,6 +42,7 @@ #include "libavutil/downmix_info.h" #include "libavutil/display.h" #include "libavutil/film_grain_params.h" +#include "libavutil/gain_map.h" #include "libavutil/hdr_dynamic_metadata.h" #include "libavutil/iamf.h" #include "libavutil/mastering_display_metadata.h" @@ -966,6 +967,35 @@ static void print_ambient_viewing_environment(AVTextFormatContext *tfc, print_q("ambient_light_y", env->ambient_light_y, '/'); } +static void print_gain_map_params(AVTextFormatContext *tfc, + const AVGainMapParams *p) +{ + const char *const channel_names[] = { "R", "G", "B" }; + + print_int("version", p->version); + print_int("use_base_color_space", p->use_base_color_space); + print_q("base_hdr_headroom", p->base_hdr_headroom, '/'); + print_q("alternate_hdr_headroom", p->alternate_hdr_headroom, '/'); + if (!(p->nb_channels == 1 || p->nb_channels == 3)) + return; + + avtext_print_section_header(tfc, NULL, SECTION_ID_FRAME_SIDE_DATA_COMPONENT_LIST); + for (int c = 0; c < p->nb_channels; c++) { + const struct AVGainMapChannel *ch = &p->channels[c]; + const char *comp_name = p->nb_channels == 1 ? "RGB" : channel_names[c]; + avtext_print_section_header(tfc, comp_name, SECTION_ID_FRAME_SIDE_DATA_COMPONENT); + print_q("gain_map_min", ch->gain_map_min, '/'); + print_q("gain_map_max", ch->gain_map_max, '/'); + print_q("gamma", ch->gamma, '/'); + print_q("base_offset", ch->base_offset, '/'); + print_q("alternate_offset", ch->alternate_offset, '/'); + // SECTION_ID_FRAME_SIDE_DATA_COMPONENT + avtext_print_section_footer(tfc); + } + // SECTION_ID_FRAME_SIDE_DATA_COMPONENT_LIST + avtext_print_section_footer(tfc); +} + static void print_film_grain_params(AVTextFormatContext *tfc, const AVFilmGrainParams *fgp) { @@ -1490,6 +1520,9 @@ static void print_frame_side_data(AVTextFormatContext *tfc, } else if (sd->type == AV_FRAME_DATA_FILM_GRAIN_PARAMS) { AVFilmGrainParams *fgp = (AVFilmGrainParams *)sd->data; print_film_grain_params(tfc, fgp); + } else if (sd->type == AV_FRAME_DATA_GAIN_MAP_PARAMS && + sd->size >= sizeof(AVGainMapParams)) { + print_gain_map_params(tfc, (const AVGainMapParams *)sd->data); } else if (sd->type == AV_FRAME_DATA_VIEW_ID) { print_int("view_id", *(int*)sd->data); } else if (sd->type == AV_FRAME_DATA_EXIF) { -- 2.52.0 >From 148d66bf54ae5bcb404e5030b53ce15a15eb88c3 Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Fri, 2 Oct 2026 17:20:38 +0200 Subject: [PATCH 3/7] avcodec: implement ISO 21496 gain map parsing The parsing logic for this already existed in libavcodec, but was unused until now. This commit adds the APP2 extraction logic and the necessary scaffolding to make this metadata survive the decoder. Since the base layer of a JPEG Ultra HDR image also contains a stub copy of the same gain map params struct, we ignore that one here. (It's only useful for the demuxer, not the decoder) Signed-off-by: Niklas Haas <[email protected]> --- libavcodec/Makefile | 2 +- libavcodec/mjpegdec.c | 48 +++++++++++++++++++++++++++++++++++++++++++ libavcodec/mjpegdec.h | 4 ++++ 3 files changed, 53 insertions(+), 1 deletion(-) diff --git a/libavcodec/Makefile b/libavcodec/Makefile index 5ee0c00bc6..84d3d63008 100644 --- a/libavcodec/Makefile +++ b/libavcodec/Makefile @@ -534,7 +534,7 @@ OBJS-$(CONFIG_METASOUND_DECODER) += metasound.o twinvq.o OBJS-$(CONFIG_MICRODVD_DECODER) += microdvddec.o ass.o OBJS-$(CONFIG_MIMIC_DECODER) += mimic.o OBJS-$(CONFIG_MISC4_DECODER) += misc4.o -OBJS-$(CONFIG_MJPEG_DECODER) += mjpegdec.o mjpegdec_common.o +OBJS-$(CONFIG_MJPEG_DECODER) += mjpegdec.o mjpegdec_common.o gain_map.o OBJS-$(CONFIG_MJPEG_QSV_DECODER) += qsvdec.o OBJS-$(CONFIG_MJPEG_ENCODER) += mjpegenc.o mjpegenc_common.o \ mjpegenc_huffman.o gain_map.o diff --git a/libavcodec/mjpegdec.c b/libavcodec/mjpegdec.c index 3dc8662abb..40f56cb4b6 100644 --- a/libavcodec/mjpegdec.c +++ b/libavcodec/mjpegdec.c @@ -44,6 +44,7 @@ #include "decode.h" #include "exif.h" #include "exif_internal.h" +#include "gain_map.h" #include "hwaccel_internal.h" #include "hwconfig.h" #include "idctdsp.h" @@ -2137,6 +2138,38 @@ static int mjpeg_decode_app(MJpegDecodeContext *s, int start_code) av_log(s->avctx, AV_LOG_WARNING, "Read more ICC markers than are supposed to be coded\n"); } + if (start_code == APP2 && id == AV_RB32("urn:")) { + bytestream2_seek(&s->gB, -4, SEEK_CUR); /* include `urn:` in the string */ + len += 4; + + if (len >= sizeof(AV_ISO21496_IDENTIFIER) && + !memcmp(s->gB.buffer, AV_ISO21496_IDENTIFIER, sizeof(AV_ISO21496_IDENTIFIER))) + { + bytestream2_skipu(&s->gB, sizeof(AV_ISO21496_IDENTIFIER)); + len -= sizeof(AV_ISO21496_IDENTIFIER); + + /* Discard gain map version stubs from base layer */ + if (len <= sizeof(uint16_t[2])) + goto out; + else if (s->gain_map) { + av_log(s->avctx, AV_LOG_WARNING, "Multiple ISO/IEC 21496 gain map headers\n"); + goto out; + } + + s->gain_map = av_gain_map_params_alloc(&s->gain_map_size); + if (!s->gain_map) + return AVERROR(ENOMEM); + + ret = ff_gain_map_params_from_iso21496(s->gain_map, s->gB.buffer, len); + if (ret < 0) { + av_log(s->avctx, AV_LOG_WARNING, "Failed to parse ISO/IEC 21496 gain map header\n"); + av_freep(&s->gain_map); + s->gain_map_size = 0; + goto out; + } + } + } + out: /* slow but needed for extreme adobe jpegs */ if (len < 0) @@ -2410,6 +2443,7 @@ int ff_mjpeg_decode_frame_from_buf(AVCodecContext *avctx, AVFrame *frame, av_exif_free(&s->exif_metadata); av_freep(&s->stereo3d); + av_freep(&s->gain_map); s->adobe_transform = -1; if (s->iccnum != 0) @@ -2842,6 +2876,18 @@ the_end: av_freep(&s->stereo3d); } + if (s->gain_map) { + AVFrameSideData *sd; + ret = ff_frame_new_side_data(avctx, frame, AV_FRAME_DATA_GAIN_MAP_PARAMS, s->gain_map_size, &sd); + if (ret < 0) + return ret; + + if (sd) + memcpy(sd->data, s->gain_map, s->gain_map_size); + av_freep(&s->gain_map); + s->gain_map_size = 0; + } + if (s->iccnum != 0 && s->iccnum == s->iccread) { AVFrameSideData *sd; size_t offset = 0; @@ -2915,6 +2961,8 @@ av_cold int ff_mjpeg_decode_end(AVCodecContext *avctx) av_freep(&s->stereo3d); av_freep(&s->ljpeg_buffer); s->ljpeg_buffer_size = 0; + av_freep(&s->gain_map); + s->gain_map_size = 0; for (i = 0; i < 3; i++) { for (j = 0; j < 4; j++) diff --git a/libavcodec/mjpegdec.h b/libavcodec/mjpegdec.h index 5ba96eaa3e..ae61db3728 100644 --- a/libavcodec/mjpegdec.h +++ b/libavcodec/mjpegdec.h @@ -29,6 +29,7 @@ #ifndef AVCODEC_MJPEGDEC_H #define AVCODEC_MJPEGDEC_H +#include "libavutil/gain_map.h" #include "libavutil/log.h" #include "libavutil/mem_internal.h" #include "libavutil/pixdesc.h" @@ -153,6 +154,9 @@ typedef struct MJpegDecodeContext { AVStereo3D *stereo3d; ///!< stereoscopic information (cached, since it is read before frame allocation) + AVGainMapParams *gain_map; + size_t gain_map_size; + const AVPixFmtDescriptor *pix_desc; ICCEntry *iccentries; -- 2.52.0 >From ca335126812923760b64ce6d57468fd11cdd0278 Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Fri, 2 Oct 2026 20:41:55 +0200 Subject: [PATCH 4/7] avformat/jpegmpf: split common definitions off from jpegmpfenc Also replaces one instance of magic numbers by an enum. Signed-off-by: Niklas Haas <[email protected]> --- libavformat/jpegmpf.h | 62 ++++++++++++++++++++++++++++++++++++++++ libavformat/jpegmpfenc.c | 33 +++------------------ 2 files changed, 66 insertions(+), 29 deletions(-) create mode 100644 libavformat/jpegmpf.h diff --git a/libavformat/jpegmpf.h b/libavformat/jpegmpf.h new file mode 100644 index 0000000000..69a6ab5a18 --- /dev/null +++ b/libavformat/jpegmpf.h @@ -0,0 +1,62 @@ +/* + * JPEG Multi-Picture Format common definitions + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/** + * @file + * Shared definitions for JPEG Multi-Picture Format (CIPA DC-007). + */ + +#ifndef AVFORMAT_JPEGMPF_H +#define AVFORMAT_JPEGMPF_H + +/* Individual Image Attribute bits, from CIPA DC-007 Figure 8 */ +enum MPFAttribute { + MPF_DEPENDENT_PARENT = 0x80000000, + MPF_DEPENDENT_CHILD = 0x40000000, + MPF_FORMAT_JPEG = 0x00000000, +}; + +/* MP Type Codes, from CIPA DC-007 Table 4 */ +enum MPFType { + MPF_TYPE_UNDEFINED = 0x000000, + MPF_TYPE_PRIMARY = 0x030000, /* Baseline MP Primary Image */ + MPF_TYPE_GAIN_MAP = 0x050000, +}; + +/* MP Index IFD tags, from CIPA DC-007 Table 3 */ +enum MPFTag { + MPF_TAG_MPFVersion = 0xB000, + MPF_TAG_NumberOfImages = 0xB001, + MPF_TAG_MPEntry = 0xB002, +}; + +/* APP2 extension segments */ +typedef enum MPFExt { + MPF_EXT_XMP, /* GContainer directory, First Individual Image only */ + MPF_EXT_INDEX, /* MP Index IFD, First Individual Image only */ + MPF_EXT_ISO21496, /* ISO 21496-1 version fields (in the base image) */ + MPF_EXT_NB, +} MPFExt; + +/* Namespace URIs of the GContainer XMP extension */ +#define XMP_CONTAINER_NS "http://ns.google.com/photos/1.0/container/"; +#define XMP_ITEM_NS "http://ns.google.com/photos/1.0/container/item/"; + +#endif /* AVFORMAT_JPEGMPF_H */ diff --git a/libavformat/jpegmpfenc.c b/libavformat/jpegmpfenc.c index 44d2a3b697..2d3d12295e 100644 --- a/libavformat/jpegmpfenc.c +++ b/libavformat/jpegmpfenc.c @@ -40,26 +40,9 @@ #include "avformat.h" #include "avio_internal.h" +#include "jpegmpf.h" #include "mux.h" -/* Namespace URIs of the GContainer XMP extension */ -#define XMP_CONTAINER_NS "http://ns.google.com/photos/1.0/container/"; -#define XMP_ITEM_NS "http://ns.google.com/photos/1.0/container/item/"; - -/* Individual Image Attribute bits, from CIPA DC-007 Figure 8 */ -enum MPFAttribute { - MPF_DEPENDENT_PARENT = 0x80000000, - MPF_DEPENDENT_CHILD = 0x40000000, - MPF_FORMAT_JPEG = 0x00000000, -}; - -/* MP Index IFD tags, from CIPA DC-007 Table 3 */ -enum MPFTag { - MPF_TAG_MPFVersion = 0xB000, - MPF_TAG_NumberOfImages = 0xB001, - MPF_TAG_MPEntry = 0xB002, -}; - typedef enum MPFRole { MPF_ROLE_UNKNOWN = 0, MPF_ROLE_PRIMARY, @@ -67,20 +50,12 @@ typedef enum MPFRole { MPF_ROLE_NB, } MPFRole; -/* APP2 extension segments */ -typedef enum MPFExt { - MPF_EXT_XMP, /* GContainer directory, First Individual Image only */ - MPF_EXT_INDEX, /* MP Index IFD, First Individual Image only */ - MPF_EXT_ISO21496, /* ISO 21496-1 version fields (in the base image) */ - MPF_EXT_NB, -} MPFExt; - static const struct { const char *name; - uint32_t type; /* from CIPA DC-007 Table 4 */ + enum MPFType type; } mpf_roles[MPF_ROLE_NB] = { - [MPF_ROLE_PRIMARY] = { "primary image", 0x030000 }, - [MPF_ROLE_GAIN_MAP] = { "gain map image", 0x050000 }, + [MPF_ROLE_PRIMARY] = { "primary image", MPF_TYPE_PRIMARY }, + [MPF_ROLE_GAIN_MAP] = { "gain map image", MPF_TYPE_GAIN_MAP }, }; typedef struct MPFImage MPFImage; -- 2.52.0 >From 918c86df313035e6e6186bec6fa0b54c92830640 Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Sun, 4 Oct 2026 14:36:51 +0200 Subject: [PATCH 5/7] avformat/jpegmpfenc: skip marker padding bytes JPEG segments may be preceded by an arbitrary number of 0xFF bytes. We don't need a separate length check here because the bytestream2 API already returns 0 when the end of the segment is reached prematurely, which would fail the following length check. Signed-off-by: Niklas Haas <[email protected]> --- libavformat/jpegmpfenc.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libavformat/jpegmpfenc.c b/libavformat/jpegmpfenc.c index 2d3d12295e..3f005bb302 100644 --- a/libavformat/jpegmpfenc.c +++ b/libavformat/jpegmpfenc.c @@ -260,6 +260,8 @@ static int mpf_write_packet(AVFormatContext *ctx, AVPacket *pkt) } int marker = bytestream2_get_byte(&gb); + while (marker == 0xFF) /* fill bytes */ + marker = bytestream2_get_byte(&gb); if (marker == SOS || marker == TEM || (marker >= RST0 && marker <= EOI)) { if (!img->splice_pos) img->splice_pos = pos; -- 2.52.0 >From 7799d9ed4b3ec8feed1b1dde01cc7f94f44e8773 Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Sun, 4 Oct 2026 18:34:38 +0200 Subject: [PATCH 6/7] avformat/jpegmpfdec: add JPEG Multi-Picture Format demuxer This format is only auto-probed when we can be certain that the image is a JPEG MPF file; as evidenced either by the extension (.mpo), or by the presence of an MPF APP2 marker or an ISO21496 gain map stub. This does require parsing possibly arbitrarily far into the file, since the MPF header can be after a > 2 KB EXIF header; and this matches files produced in the wild (by e.g. Apple's jpeg ultra hdr encoder). The way this is handled now relies on the probe logic continuing to feed more bytes to the demuxer while the probe score is still tied; but since image2 claims any .jpg for itself quite liberally, we need to at least match it until we can be certain that the file is an MPF file (or not). A few additional notes: - libultrahdr produces quite spec-incompliant files which we unfortunately have to work around - since the MPF format table can be abritrarily far from the header, we also need to store possibly a lot of data and re-return it to the caller; I have followed the convention established by e.g. jpegxl_anim_dec.c of just keeping a spare AVPacket for bytes already read during header parsing Signed-off-by: Niklas Haas <[email protected]> --- doc/demuxers.texi | 13 ++ libavformat/Makefile | 1 + libavformat/allformats.c | 1 + libavformat/jpegmpf.h | 27 ++- libavformat/jpegmpfdec.c | 466 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 505 insertions(+), 3 deletions(-) create mode 100644 libavformat/jpegmpfdec.c diff --git a/doc/demuxers.texi b/doc/demuxers.texi index 7610055193..2d9520abed 100644 --- a/doc/demuxers.texi +++ b/doc/demuxers.texi @@ -746,6 +746,19 @@ ffmpeg -framerate 10 -pattern_type glob -i "*.png" out.mkv @end example @end itemize +@section jpeg_mpf + +JPEG Multi-Picture Format demuxer. + +Exposes each JPEG image stored in a CIPA DC-007 MP file as a separate stream, +with the primary image as the first stream. If the file contains an ISO 21496-1 +gain map, as is the case for JPEG Ultra HDR, the primary image and the gain map +are linked by a @samp{gain_map} stream group, and the gain map parameters are +exported as frame side data by the decoder. + +Images of any other type (e.g. large thumbnails or multi-view images) are +exposed as unrelated streams. + @section ktx Khronos Texture 1.0 demuxer for ASTC textures. diff --git a/libavformat/Makefile b/libavformat/Makefile index 24066050a5..5a02f56ea2 100644 --- a/libavformat/Makefile +++ b/libavformat/Makefile @@ -347,6 +347,7 @@ OBJS-$(CONFIG_IVF_MUXER) += ivfenc.o OBJS-$(CONFIG_IVR_DEMUXER) += rmdec.o rm.o rmsipr.o OBJS-$(CONFIG_JACOSUB_DEMUXER) += jacosubdec.o subtitles.o OBJS-$(CONFIG_JACOSUB_MUXER) += jacosubenc.o rawenc.o +OBJS-$(CONFIG_JPEG_MPF_DEMUXER) += jpegmpfdec.o OBJS-$(CONFIG_JPEG_MPF_MUXER) += jpegmpfenc.o OBJS-$(CONFIG_JPEGXL_ANIM_DEMUXER) += jpegxl_anim_dec.o OBJS-$(CONFIG_JV_DEMUXER) += jvdec.o diff --git a/libavformat/allformats.c b/libavformat/allformats.c index a79b1e16c0..68da203116 100644 --- a/libavformat/allformats.c +++ b/libavformat/allformats.c @@ -252,6 +252,7 @@ extern const FFInputFormat ff_ivr_demuxer; extern const FFInputFormat ff_jacosub_demuxer; extern const FFOutputFormat ff_jacosub_muxer; extern const FFInputFormat ff_jv_demuxer; +extern const FFInputFormat ff_jpeg_mpf_demuxer; extern const FFOutputFormat ff_jpeg_mpf_muxer; extern const FFInputFormat ff_jpegxl_anim_demuxer; extern const FFInputFormat ff_ktx_demuxer; diff --git a/libavformat/jpegmpf.h b/libavformat/jpegmpf.h index 69a6ab5a18..0cf1a6522e 100644 --- a/libavformat/jpegmpf.h +++ b/libavformat/jpegmpf.h @@ -30,16 +30,37 @@ enum MPFAttribute { MPF_DEPENDENT_PARENT = 0x80000000, MPF_DEPENDENT_CHILD = 0x40000000, + MPF_REPRESENTATIVE = 0x20000000, + MPF_ATTR_FORMAT_MASK = 0x07000000, + MPF_ATTR_TYPE_MASK = 0x00FFFFFF, MPF_FORMAT_JPEG = 0x00000000, }; /* MP Type Codes, from CIPA DC-007 Table 4 */ enum MPFType { - MPF_TYPE_UNDEFINED = 0x000000, - MPF_TYPE_PRIMARY = 0x030000, /* Baseline MP Primary Image */ - MPF_TYPE_GAIN_MAP = 0x050000, + MPF_TYPE_UNDEFINED = 0x000000, + MPF_TYPE_LARGE_THUMBNAIL = 0x010000, /* + class (1-5) */ + MPF_TYPE_MULTI_FRAME = 0x020000, /* + panorama / disparity / multi-angle */ + MPF_TYPE_PRIMARY = 0x030000, /* Baseline MP Primary Image */ + MPF_TYPE_ORIGINAL = 0x040000, /* Original preservation image */ + MPF_TYPE_GAIN_MAP = 0x050000, + + MPF_TYPE_INFO_MASK = 0xFF0000, /* see CIPA DC-007 Figure 9 */ }; +/* Returns whether images of this type only exist as dependent images */ +static inline int mpf_type_is_child(enum MPFType type) +{ + switch (type & MPF_TYPE_INFO_MASK) { + case MPF_TYPE_LARGE_THUMBNAIL: + case MPF_TYPE_ORIGINAL: + case MPF_TYPE_GAIN_MAP: + return 1; + default: + return 0; + } +} + /* MP Index IFD tags, from CIPA DC-007 Table 3 */ enum MPFTag { MPF_TAG_MPFVersion = 0xB000, diff --git a/libavformat/jpegmpfdec.c b/libavformat/jpegmpfdec.c new file mode 100644 index 0000000000..2385f097b0 --- /dev/null +++ b/libavformat/jpegmpfdec.c @@ -0,0 +1,466 @@ +/* + * JPEG Multi-Picture Format demuxer + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/** + * @file + * Demuxer for the JPEG Multi-Picture Format file format. + */ + +#include <stdlib.h> +#include <string.h> + +#include "libavutil/intreadwrite.h" +#include "libavutil/gain_map.h" +#include "libavutil/mem.h" + +#include "libavcodec/bytestream.h" +#include "libavcodec/exif.h" +#include "libavcodec/mjpeg.h" + +#include "avformat.h" +#include "demux.h" +#include "jpegmpf.h" +#include "internal.h" + +typedef struct MPFDemuxImage { + AVStream *st; + int64_t start; + int64_t size; + + uint32_t type; + int index; /* of the corresponding MP Entry */ + int child; /* is a dependent image */ +} MPFDemuxImage; + +typedef struct MPFDemuxContext { + MPFDemuxImage *images; /* sorted by position in the file */ + int nb_images; + + int cur_image; /* index of next image to return */ + int has_gain_map; /* the primary image carries ISO 21496-1 metadata */ + int passthrough; /* not an MPF file */ + + /* image data parsed during mpf_read_header() */ + AVPacket *initial; + int64_t file_start; +} MPFDemuxContext; + +#define CHECK_IDENT(strc) (len >= sizeof(strc) && !memcmp(gb->buffer, strc, sizeof(strc))) + +static int mpf_probe(const AVProbeData *p) +{ + GetByteContext gbc, *gb = &gbc; + bytestream2_init(gb, p->buf, p->buf_size); + if (bytestream2_get_be16(gb) != (0xFF00 | SOI)) + return 0; + + while (bytestream2_get_bytes_left(gb) >= 4) { + if (bytestream2_get_byte(gb) != 0xFF) + return 0; /* not a JPEG marker */ + + int marker = bytestream2_get_byte(gb); + while (marker == 0xFF) /* fill bytes */ + marker = bytestream2_get_byte(gb); + if (marker == SOS || marker == TEM || (marker >= RST0 && marker <= EOI)) + return 0; /* start of entropy coded data */ + + const int len = bytestream2_get_be16(gb) - 2; + if (len < 0) + return 0; + if (len > bytestream2_get_bytes_left(gb)) + break; + + switch (marker) { + case APP2: + /* Marker segments unique to MPF files */ + if (CHECK_IDENT("MPF") || CHECK_IDENT(AV_ISO21496_IDENTIFIER)) + return AVPROBE_SCORE_EXTENSION + 10; + break; + } + + bytestream2_skip(gb, len); + } + + /* If we ran out of probe bytes, we don't know yet whether this is an + * MPF file or not, so just return a score that ties with image2 */ + if (av_match_ext(p->filename, "mpo")) /* unique to MPF files */ + return AVPROBE_SCORE_EXTENSION + 5; + if (av_match_ext(p->filename, "jpg,jpeg")) + return AVPROBE_SCORE_EXTENSION; + + return 0; +} + +/* returns the offset of the first byte read */ +static int grow_header(AVFormatContext *ctx, int size) +{ + MPFDemuxContext *s = ctx->priv_data; + const int pos = s->initial->size; + int ret = av_append_packet(ctx->pb, s->initial, size); + if (ret < 0) + return ret; + if (ret < size) + return AVERROR_INVALIDDATA; + return pos; +} + +static int parse_mp_index(AVFormatContext *ctx, const uint8_t *data, int len, + int64_t tiff_base) +{ + MPFDemuxContext *s = ctx->priv_data; + AVExifMetadata ifd = { 0 }; + AVExifEntry *num = NULL, *entries = NULL; + int ret; + + if (len < 2) + return AVERROR_INVALIDDATA; + const int le = AV_RB16(data) == AV_RB16("II"); + + ret = av_exif_parse_buffer(ctx, data, len, &ifd, AV_EXIF_TIFF_HEADER); + if (ret < 0) + return ret; + + if ((ret = av_exif_get_entry(ctx, &ifd, MPF_TAG_NumberOfImages, 0, &num)) < 0 || + (ret = av_exif_get_entry(ctx, &ifd, MPF_TAG_MPEntry, 0, &entries)) < 0) + goto fail; + + if (!num || num->type != AV_TIFF_LONG || num->count != 1 || + !entries || entries->type != AV_TIFF_UNDEFINED) { + av_log(ctx, AV_LOG_ERROR, "Missing or malformed MP Index IFD\n"); + ret = AVERROR_INVALIDDATA; + goto fail; + } + + /* The spec asserts MPEntry count = 16 * NumberOfImages */ + const int entry_size = sizeof(uint32_t[3]) + sizeof(uint16_t[2]); + const uint64_t nb_images = num->value.uint[0]; + if (!nb_images || nb_images != entries->count / entry_size) { + av_log(ctx, AV_LOG_ERROR, "Invalid number of images: %"PRIu64"\n", + nb_images); + ret = AVERROR_INVALIDDATA; + goto fail; + } + + s->images = av_calloc(nb_images, sizeof(*s->images)); + if (!s->images) { + ret = AVERROR(ENOMEM); + goto fail; + } + + GetByteContext gb; + bytestream2_init(&gb, entries->value.ubytes, entries->count); + + const int explode = ctx->error_recognition & AV_EF_EXPLODE; + for (int i = 0; i < nb_images; i++) { + const uint32_t attr = le ? bytestream2_get_le32u(&gb) : bytestream2_get_be32u(&gb); + const uint32_t size = le ? bytestream2_get_le32u(&gb) : bytestream2_get_be32u(&gb); + const uint32_t offset = le ? bytestream2_get_le32u(&gb) : bytestream2_get_be32u(&gb); + bytestream2_skipu(&gb, sizeof(uint16_t[2])); /* dependent images */ + + if ((attr & MPF_ATTR_FORMAT_MASK) != MPF_FORMAT_JPEG) { + const int fatal = !i || explode; + av_log(ctx, fatal ? AV_LOG_ERROR : AV_LOG_WARNING, "Image %d: " + "unsupported format 0x%"PRIx32"\n", i, + attr & MPF_ATTR_FORMAT_MASK); + if (fatal) { + ret = AVERROR_INVALIDDATA; + goto fail; + } + continue; + } else if (i && !offset) { + av_log(ctx, explode ? AV_LOG_ERROR : AV_LOG_WARNING, + "Image %d: missing data offset\n", i); + if (explode) { + ret = AVERROR_INVALIDDATA; + goto fail; + } + continue; + } + + /* offset of the First Individual Image is always zero */ + MPFDemuxImage *img = &s->images[s->nb_images++]; + img->index = i; + img->start = i ? tiff_base + offset : s->file_start; + img->size = size; + img->type = attr & MPF_ATTR_TYPE_MASK; + img->child = !!(attr & MPF_DEPENDENT_CHILD); + } + + ret = 0; +fail: + av_exif_free(&ifd); + return ret; +} + +static int parse_app2(AVFormatContext *ctx, int offset, int len) +{ + MPFDemuxContext *s = ctx->priv_data; + GetByteContext gbc, *gb = &gbc; + bytestream2_init(gb, s->initial->data + offset, len); + + if (CHECK_IDENT(AV_ISO21496_IDENTIFIER)) { + s->has_gain_map = 1; + } else if (CHECK_IDENT("MPF")) { + if (s->images) { + const int explode = ctx->error_recognition & AV_EF_EXPLODE; + av_log(ctx, explode ? AV_LOG_ERROR : AV_LOG_WARNING, + "Duplicate MP Index IFD\n"); + return explode ? AVERROR_INVALIDDATA : 0; + } + + /* MP offsets are relative to the start of the TIFF header */ + const int skip = sizeof("MPF"); + return parse_mp_index(ctx, gb->buffer + skip, len - skip, + s->file_start + offset + skip); + } + + return 0; +} + +static int cmp_image_start(const void *a, const void *b) +{ + const MPFDemuxImage *ia = a, *ib = b; + if (ia->start != ib->start) + return FFDIFFSIGN(ia->start, ib->start); + return FFDIFFSIGN(ia->index, ib->index); +} + +static int add_gain_map_group(AVFormatContext *ctx, const MPFDemuxImage *gain) +{ + MPFDemuxContext *s = ctx->priv_data; + const MPFDemuxImage *base = &s->images[0]; + + if (gain->index == base->index) { + av_log(ctx, AV_LOG_ERROR, "The first image cannot be a gain map\n"); + return AVERROR_INVALIDDATA; + } + + AVStreamGroup *stg = avformat_stream_group_create(ctx, AV_STREAM_GROUP_PARAMS_GAIN_MAP, NULL); + if (!stg) + return AVERROR(ENOMEM); + + int ret; + if ((ret = avformat_stream_group_add_stream(stg, base->st)) < 0 || + (ret = avformat_stream_group_add_stream(stg, gain->st)) < 0) + return ret; + + stg->params.layered_video->el_index = stg->nb_streams - 1; + return 0; +} + +static int mpf_read_header(AVFormatContext *ctx) +{ + MPFDemuxContext *s = ctx->priv_data; + int ret; + + s->initial = av_packet_alloc(); + if (!s->initial) + return AVERROR(ENOMEM); + + s->file_start = avio_tell(ctx->pb); + if ((ret = grow_header(ctx, 2)) < 0) + return ret; + if (AV_RB16(s->initial->data) != (0xFF00 | SOI)) + return AVERROR_INVALIDDATA; + + /* Walk the JPEG marker sections up until the start of entropy coded data */ + for (;;) { + const int pos = s->initial->size; + if ((ret = grow_header(ctx, 1)) < 0) + return ret; + uint8_t marker = s->initial->data[ret]; + if (marker != 0xFF) { + av_log(ctx, AV_LOG_ERROR, "Malformed JPEG marker at 0x%x\n", pos); + return AVERROR_INVALIDDATA; + } + + while (marker == 0xFF) { + if ((ret = grow_header(ctx, 1)) < 0) + return ret; + marker = s->initial->data[ret]; + } + + if (marker == SOS || marker == TEM || (marker >= RST0 && marker <= EOI)) + break; /* start of entropy coded data */ + + if ((ret = grow_header(ctx, 2)) < 0) + return ret; + const int len = AV_RB16(s->initial->data + ret) - 2; + if (len < 0) { + av_log(ctx, AV_LOG_ERROR, "Invalid marker segment at 0x%x\n", pos); + return AVERROR_INVALIDDATA; + } + + if ((ret = grow_header(ctx, len)) < 0) + return ret; + if (marker == APP2 && (ret = parse_app2(ctx, ret, len)) < 0) + return ret; + } + + if (!s->images) { + /* Not an MPF file, just return as-is to the underlying decoder */ + const int64_t file_size = avio_size(ctx->pb); + s->passthrough = file_size <= s->file_start; + s->nb_images = 1; + s->images = av_malloc(sizeof(*s->images)); + if (!s->images) + return AVERROR(ENOMEM); + s->images[0] = (MPFDemuxImage){ + .type = MPF_TYPE_PRIMARY, + .start = s->file_start, + .size = s->passthrough ? s->initial->size + : file_size - s->file_start, + }; + } + + /* Compatibility hack for libultrahdr, which produces invalid MPF files + * whose gain maps have no MP type. Needed to support a large number + * of JPEG Ultra HDR images in the wild */ + if (ctx->strict_std_compliance < FF_COMPLIANCE_STRICT && + s->has_gain_map && s->nb_images == 2 && + s->images[1].type == MPF_TYPE_UNDEFINED) + { + s->images[1].type = MPF_TYPE_GAIN_MAP; + } + + for (int i = 0; i < s->nb_images; i++) { + MPFDemuxImage *img = &s->images[i]; + AVStream *st = img->st = avformat_new_stream(ctx, NULL); + if (!st) + return AVERROR(ENOMEM); + + st->id = img->index; + st->codecpar->codec_type = AVMEDIA_TYPE_VIDEO; + st->codecpar->codec_id = AV_CODEC_ID_MJPEG; + if (!i) + st->disposition |= AV_DISPOSITION_DEFAULT; + if (img->child || mpf_type_is_child(img->type)) + st->disposition |= AV_DISPOSITION_DEPENDENT; + avpriv_set_pts_info(st, 64, 1, 25); + if (s->passthrough) /* we don't know the image size */ + ffstream(st)->need_parsing = AVSTREAM_PARSE_FULL; + + switch (img->type & MPF_TYPE_INFO_MASK) { + case MPF_TYPE_GAIN_MAP: + ret = add_gain_map_group(ctx, img); + if (ret < 0) + return ret; + break; + case MPF_TYPE_LARGE_THUMBNAIL: + case MPF_TYPE_MULTI_FRAME: + case MPF_TYPE_PRIMARY: + case MPF_TYPE_ORIGINAL: + /* no special handling */ + break; + case MPF_TYPE_UNDEFINED: + default: + av_log(ctx, img->child ? AV_LOG_VERBOSE : AV_LOG_WARNING, + "Image %d: unhandled MP type 0x%06"PRIx32", treating as an " + "%s image\n", img->index, img->type, + img->child ? "unspecified dependent" : "unrelated"); + break; + } + } + + /* Sort the images by file offset for mpf_read_packet() */ + qsort(s->images, s->nb_images, sizeof(*s->images), cmp_image_start); + + for (int i = 0; i < s->nb_images - 1; i++) { + MPFDemuxImage *img = &s->images[i], *next = &s->images[i + 1]; + if (next->start < img->start + img->size) { + const int explode = ctx->error_recognition & AV_EF_EXPLODE; + av_log(ctx, explode ? AV_LOG_ERROR : AV_LOG_WARNING, + "Image %d overlaps image %d%s\n", img->index, next->index, + explode ? "" : ", truncating"); + if (explode) + return AVERROR_INVALIDDATA; + img->size = next->start - img->start; + } + } + + return 0; +} + +static int mpf_read_packet(AVFormatContext *ctx, AVPacket *pkt) +{ + MPFDemuxContext *s = ctx->priv_data; + AVIOContext *pb = ctx->pb; + int ret; + + if (s->cur_image >= s->nb_images) { + if (!s->passthrough) + return AVERROR_EOF; + + if ((ret = av_get_packet(pb, pkt, 4096)) < 0) + return ret; + pkt->flags &= ~AV_PKT_FLAG_CORRUPT; /* short read at EOF */ + pkt->stream_index = 0; + return 0; + } + + const MPFDemuxImage *img = &s->images[s->cur_image++]; + if (img->size > INT_MAX - AV_INPUT_BUFFER_PADDING_SIZE) + return AVERROR(ERANGE); + + if (s->initial) { + av_packet_move_ref(pkt, s->initial); + av_packet_free(&s->initial); + av_shrink_packet(pkt, FFMIN(pkt->size, img->size)); + } else { + int64_t pos = avio_seek(pb, img->start, SEEK_SET); + if (pos < 0) { + av_log(ctx, AV_LOG_ERROR, "Failed seeking to image %d at offset " + "%"PRId64"\n", img->st->id, img->start); + return pos; + } + } + + if (pkt->size < img->size) { + ret = av_append_packet(pb, pkt, img->size - pkt->size); + if (ret < 0 && !pkt->size) + return ret; + } + + pkt->stream_index = img->st->index; + pkt->pts = pkt->dts = 0; + pkt->duration = 1; + pkt->flags |= AV_PKT_FLAG_KEY; + return 0; +} + +static int mpf_read_close(AVFormatContext *ctx) +{ + MPFDemuxContext *s = ctx->priv_data; + av_freep(&s->images); + av_packet_free(&s->initial); + return 0; +} + +const FFInputFormat ff_jpeg_mpf_demuxer = { + .p.name = "jpeg_mpf", + .p.long_name = NULL_IF_CONFIG_SMALL("JPEG Multi-Picture Format"), + .p.extensions = "jpg,jpeg,mpo", + .priv_data_size = sizeof(MPFDemuxContext), + .flags_internal = FF_INFMT_FLAG_INIT_CLEANUP, + .read_probe = mpf_probe, + .read_header = mpf_read_header, + .read_packet = mpf_read_packet, + .read_close = mpf_read_close, +}; -- 2.52.0 >From 0a36907e5236e8e3032c9ba4b5b9af792cb751a1 Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Sun, 4 Oct 2026 18:50:55 +0200 Subject: [PATCH 7/7] tests/fate/image: add roundtrip test for JPEG Ultra HDR Signed-off-by: Niklas Haas <[email protected]> --- tests/fate/image.mak | 15 ++++++---- tests/filtergraphs/jpeg_mpf_gainmap | 2 +- tests/ref/fate/jpeg-mpf-gainmap | 46 ++++++++++++++++++++++++++++- 3 files changed, 55 insertions(+), 8 deletions(-) diff --git a/tests/fate/image.mak b/tests/fate/image.mak index c78baa8e08..b0b0f8c296 100644 --- a/tests/fate/image.mak +++ b/tests/fate/image.mak @@ -628,14 +628,17 @@ FATE_IMAGE_TRANSCODE += $(FATE_IMAGE_TRANSCODE-yes) FATE_JPEG_MPF-$(call ALLYES, LAVFI_INDEV TESTSRC2_FILTER SCALE_FILTER FORMAT_FILTER \ SPLIT_FILTER SETPARAMS_FILTER GAINMAP_FILTER MJPEG_ENCODER JPEG_MPF_MUXER \ - FILE_PROTOCOL) += fate-jpeg-mpf-gainmap + JPEG_MPF_DEMUXER MJPEG_DECODER RAWVIDEO_ENCODER FRAMECRC_MUXER PIPE_PROTOCOL \ + FILE_PROTOCOL FFPROBE) += fate-jpeg-mpf-gainmap fate-jpeg-mpf-gainmap: tests/data/filtergraphs/jpeg_mpf_gainmap -fate-jpeg-mpf-gainmap: CMD = md5 -bitexact -sws_flags +accurate_rnd+bitexact \ - -/filter_complex $(TARGET_PATH)/tests/data/filtergraphs/jpeg_mpf_gainmap \ - -map "[base]" -map "[gain]" -stream_group "type=gain_map:st=0:st=1:el_index=1" \ - -pix_fmt yuvj444p -c:v mjpeg -dct int -f jpeg_mpf +fate-jpeg-mpf-gainmap: CMD = transcode lavfi testsrc2=s=64x48:r=1:d=1 jpeg_mpf \ + "-sws_flags +accurate_rnd+bitexact \ + -/filter_complex $(TARGET_PATH)/tests/data/filtergraphs/jpeg_mpf_gainmap \ + -map [base] -map [gain] -stream_group type=gain_map:st=0:st=1:el_index=1 \ + -pix_fmt yuvj444p -c:v mjpeg -dct int" "-map 0" \ + "-show_entries stream_group=index,type,nb_streams:stream_group_stream=index:frame=stream_index:frame_side_data_list" -FATE_FFMPEG += $(FATE_JPEG_MPF-yes) +FATE_FFMPEG_FFPROBE += $(FATE_JPEG_MPF-yes) fate-jpeg-mpf: $(FATE_JPEG_MPF-yes) FATE_IMG2_MUXER-$(call ALLYES, LAVFI_INDEV COLOR_FILTER FORMAT_FILTER \ diff --git a/tests/filtergraphs/jpeg_mpf_gainmap b/tests/filtergraphs/jpeg_mpf_gainmap index 8f778501cc..d2f0007de2 100644 --- a/tests/filtergraphs/jpeg_mpf_gainmap +++ b/tests/filtergraphs/jpeg_mpf_gainmap @@ -1,4 +1,4 @@ -testsrc2=s=64x48:r=1:d=1,setparams=color_trc=smpte2084:color_primaries=bt2020,scale,split [a][hdr]; +[0:v] setparams=color_trc=smpte2084:color_primaries=bt2020,scale,split [a][hdr]; [a] setparams=color_trc=iec61966-2-1:color_primaries=bt709,split [b][sdr]; [b] scale [base]; [sdr][hdr] gainmap=mode=maxrgb,scale [gain] diff --git a/tests/ref/fate/jpeg-mpf-gainmap b/tests/ref/fate/jpeg-mpf-gainmap index c90fdd1fe1..3ae5864287 100644 --- a/tests/ref/fate/jpeg-mpf-gainmap +++ b/tests/ref/fate/jpeg-mpf-gainmap @@ -1 +1,45 @@ -7cab095419286aec7affcf4eb697f4c9 +7cab095419286aec7affcf4eb697f4c9 *tests/data/fate/jpeg-mpf-gainmap.jpeg_mpf +5653 tests/data/fate/jpeg-mpf-gainmap.jpeg_mpf +#tb 0: 1/25 +#media_type 0: video +#codec_id 0: rawvideo +#dimensions 0: 64x48 +#sar 0: 1/1 +#tb 1: 1/25 +#media_type 1: video +#codec_id 1: rawvideo +#dimensions 1: 64x48 +#sar 1: 1/1 +0, 0, 0, 1, 9216, 0xdfa842b3 +1, 0, 0, 1, 9216, 0x662fcc6a +[FRAME] +stream_index=0 +[/FRAME] +[FRAME] +stream_index=1 +[SIDE_DATA] +side_data_type=Gain map parameters +version=0 +use_base_color_space=1 +base_hdr_headroom=0/1 +alternate_hdr_headroom=5895489/1048576 +[COMPONENT] +gain_map_min=-3392315/4194304 +gain_map_max=13278945/2097152 +gamma=691797/1048576 +base_offset=1/64 +alternate_offset=1/64 +[/COMPONENT] +[/SIDE_DATA] +[/FRAME] +[STREAM_GROUP] +index=0 +nb_streams=2 +type=Gain Map (Split base rendition and gain map) +[STREAM] +index=0 +[/STREAM] +[STREAM] +index=1 +[/STREAM] +[/STREAM_GROUP] -- 2.52.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
