PR #21134 opened by Niklas Haas (haasn) URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/21134 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/21134.patch
In #21104, the idea was raised that swscale/format.c should contain only a single format list, like the legacy format table. In the process of implementing it, I realized that a lot of the information ended up redundant with AVPixFmtDescriptor, so we should prefer parsing it from there when possible. Unfortunately, the way that struct is laid out for packed formats makes it exceptionally difficult to use; because the number of special cases/exceptions means we need to basically have a separate test for each "irregular" format that tells the logic how to interpret the fields for that format in particular. As a compromise, retain a list of "irregular" formats for which we hard-code the details we care about; while handling all other formats automatically. >From 67b9e23c34364c1462c4364dc4b9a582aeeefdae Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Fri, 5 Dec 2025 18:28:10 +0100 Subject: [PATCH 1/8] swscale/format: exclude U32 from sws_pixel_type() This function is supposed to give us representable pixel types; but U32 is not representable (due only to the AVRational range limit). --- libswscale/format.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libswscale/format.c b/libswscale/format.c index cbb5cb35fe..f9a156e29e 100644 --- a/libswscale/format.c +++ b/libswscale/format.c @@ -616,12 +616,13 @@ static SwsPixelType fmt_pixel_type(enum AVPixelFormat fmt) if (desc->flags & AV_PIX_FMT_FLAG_FLOAT) { switch (bits) { case 32: return SWS_PIXEL_F32; + /* TODO: no support for 16-bit float yet */ } } else { switch (bits) { case 8: return SWS_PIXEL_U8; case 16: return SWS_PIXEL_U16; - case 32: return SWS_PIXEL_U32; + /* TODO: AVRational cannot represent UINT32_MAX */ } } -- 2.49.1 >From e8027da9983ec6f43ac03cee80df22268645ca68 Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Fri, 5 Dec 2025 18:32:10 +0100 Subject: [PATCH 2/8] swscale/format: check SwsPixelType in fmt_read_write() This is the only function that actually has the ability to return an error, so just move the pixel type assignment here and add a check to ensure a valid pixel type is found. --- libswscale/format.c | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/libswscale/format.c b/libswscale/format.c index f9a156e29e..a1afa64f4d 100644 --- a/libswscale/format.c +++ b/libswscale/format.c @@ -788,12 +788,16 @@ static SwsConst fmt_clear(enum AVPixelFormat fmt) } static int fmt_read_write(enum AVPixelFormat fmt, SwsReadWriteOp *rw_op, - SwsPackOp *pack_op) + SwsPackOp *pack_op, SwsPixelType *pixel_type) { const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(fmt); if (!desc) return AVERROR(EINVAL); + *pixel_type = fmt_pixel_type(fmt); + if (!*pixel_type) + return AVERROR(ENOTSUP); + switch (fmt) { case AV_PIX_FMT_NONE: case AV_PIX_FMT_NB: @@ -1024,12 +1028,13 @@ static SwsPixelType get_packed_type(SwsPackOp pack) int ff_sws_decode_pixfmt(SwsOpList *ops, enum AVPixelFormat fmt) { const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(fmt); - SwsPixelType pixel_type = fmt_pixel_type(fmt); - SwsPixelType raw_type = pixel_type; + SwsPixelType pixel_type; SwsReadWriteOp rw_op; SwsPackOp unpack; - RET(fmt_read_write(fmt, &rw_op, &unpack)); + RET(fmt_read_write(fmt, &rw_op, &unpack, &pixel_type)); + + SwsPixelType raw_type = pixel_type; if (unpack.pattern[0]) raw_type = get_packed_type(unpack); @@ -1085,12 +1090,13 @@ int ff_sws_decode_pixfmt(SwsOpList *ops, enum AVPixelFormat fmt) int ff_sws_encode_pixfmt(SwsOpList *ops, enum AVPixelFormat fmt) { const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(fmt); - SwsPixelType pixel_type = fmt_pixel_type(fmt); - SwsPixelType raw_type = pixel_type; + SwsPixelType pixel_type; SwsReadWriteOp rw_op; SwsPackOp pack; - RET(fmt_read_write(fmt, &rw_op, &pack)); + RET(fmt_read_write(fmt, &rw_op, &pack, &pixel_type)); + + SwsPixelType raw_type = pixel_type; if (pack.pattern[0]) raw_type = get_packed_type(pack); -- 2.49.1 >From e1e1e04c08d67c5faed1f47b9d3494bb234d40bf Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Fri, 5 Dec 2025 18:45:22 +0100 Subject: [PATCH 3/8] swscale/format: derive fmt_swizzle() from AVPixFmtDescriptor when possible Unfortunately, this is exceptionally difficult to handle in the general case, when packed/bitstream formats come into play - the actual interpretation of the offset, shift etc. are so difficult to deal with in a general case that I think it's simpler to continue falling back to a static table of variants for these exceptions. They are fortunately small in number. --- libswscale/format.c | 108 ++++++++++++++++++++++++-------------------- 1 file changed, 59 insertions(+), 49 deletions(-) diff --git a/libswscale/format.c b/libswscale/format.c index a1afa64f4d..7be3685c6c 100644 --- a/libswscale/format.c +++ b/libswscale/format.c @@ -629,41 +629,86 @@ static SwsPixelType fmt_pixel_type(enum AVPixelFormat fmt) return SWS_PIXEL_NONE; } +/* A regular format is defined as any format that contains only a single + * component per elementary data type (i.e. no sub-byte pack/unpack needed), + * and whose components map 1:1 onto elementary data units */ +static int is_regular_fmt(enum AVPixelFormat fmt) +{ + const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(fmt); + if (desc->flags & (AV_PIX_FMT_FLAG_PAL | AV_PIX_FMT_FLAG_BAYER)) + return 0; /* no 1:1 correspondence between components and data units */ + if (desc->flags & (AV_PIX_FMT_FLAG_BITSTREAM)) + return 0; /* bitstream formats are packed by definition */ + if ((desc->flags & AV_PIX_FMT_FLAG_PLANAR) || desc->nb_components == 1) + return 1; /* planar formats are regular by definition */ + + const int step = desc->comp[0].step; + int total_bits = 0; + + for (int i = 0; i < desc->nb_components; i++) { + if (desc->comp[i].shift || desc->comp[i].step != step) + return 0; /* irregular/packed format */ + total_bits += desc->comp[i].depth; + } + + /* Exclude formats with missing components like RGB0, 0RGB, etc. */ + return total_bits == step * 8; +} + +struct comp { + int index; + int plane; + int offset; +}; + +/* Compare by (plane, offset) */ +static int cmp_comp(const void *a, const void *b) { + const struct comp *ca = a; + const struct comp *cb = b; + if (ca->plane != cb->plane) + return ca->plane - cb->plane; + return ca->offset - cb->offset; +} + static SwsSwizzleOp fmt_swizzle(enum AVPixelFormat fmt) { const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(fmt); - if (desc->nb_components == 2) /* YA formats */ - return (SwsSwizzleOp) {{ .x = 0, 3, 1, 2 }}; + if (desc->nb_components == 2) { + /* YA formats */ + return SWS_SWIZZLE(0, 3, 1, 2); + } else if (is_regular_fmt(fmt)) { + /* Sort by increasing component order */ + struct comp sorted[4] = { {0}, {1}, {2}, {3} }; + for (int i = 0; i < desc->nb_components; i++) { + sorted[i].plane = desc->comp[i].plane; + sorted[i].offset = desc->comp[i].offset; + } + + qsort(sorted, desc->nb_components, sizeof(struct comp), cmp_comp); + + SwsSwizzleOp swiz = SWS_SWIZZLE(0, 1, 2, 3); + for (int i = 0; i < desc->nb_components; i++) + swiz.in[i] = sorted[i].index; + return swiz; + } switch (fmt) { - case AV_PIX_FMT_ARGB: case AV_PIX_FMT_0RGB: - case AV_PIX_FMT_AYUV64LE: - case AV_PIX_FMT_AYUV64BE: - case AV_PIX_FMT_AYUV: case AV_PIX_FMT_X2RGB10LE: case AV_PIX_FMT_X2RGB10BE: return (SwsSwizzleOp) {{ .x = 3, 0, 1, 2 }}; - case AV_PIX_FMT_BGR24: case AV_PIX_FMT_BGR8: case AV_PIX_FMT_BGR4: case AV_PIX_FMT_BGR4_BYTE: - case AV_PIX_FMT_BGRA: case AV_PIX_FMT_BGR565BE: case AV_PIX_FMT_BGR565LE: case AV_PIX_FMT_BGR555BE: case AV_PIX_FMT_BGR555LE: case AV_PIX_FMT_BGR444BE: case AV_PIX_FMT_BGR444LE: - case AV_PIX_FMT_BGR48BE: - case AV_PIX_FMT_BGR48LE: - case AV_PIX_FMT_BGRA64BE: - case AV_PIX_FMT_BGRA64LE: case AV_PIX_FMT_BGR0: - case AV_PIX_FMT_VUYA: case AV_PIX_FMT_VUYX: return (SwsSwizzleOp) {{ .x = 2, 1, 0, 3 }}; - case AV_PIX_FMT_ABGR: case AV_PIX_FMT_0BGR: case AV_PIX_FMT_X2BGR10LE: case AV_PIX_FMT_X2BGR10BE: @@ -671,7 +716,6 @@ static SwsSwizzleOp fmt_swizzle(enum AVPixelFormat fmt) case AV_PIX_FMT_XV30BE: case AV_PIX_FMT_XV30LE: return (SwsSwizzleOp) {{ .x = 3, 2, 0, 1 }}; - case AV_PIX_FMT_VYU444: case AV_PIX_FMT_V30XBE: case AV_PIX_FMT_V30XLE: return (SwsSwizzleOp) {{ .x = 2, 0, 1, 3 }}; @@ -679,41 +723,7 @@ static SwsSwizzleOp fmt_swizzle(enum AVPixelFormat fmt) case AV_PIX_FMT_XV36LE: case AV_PIX_FMT_XV48BE: case AV_PIX_FMT_XV48LE: - case AV_PIX_FMT_UYVA: return (SwsSwizzleOp) {{ .x = 1, 0, 2, 3 }}; - case AV_PIX_FMT_GBRP: - case AV_PIX_FMT_GBRP9BE: - case AV_PIX_FMT_GBRP9LE: - case AV_PIX_FMT_GBRP10BE: - case AV_PIX_FMT_GBRP10LE: - case AV_PIX_FMT_GBRP12BE: - case AV_PIX_FMT_GBRP12LE: - case AV_PIX_FMT_GBRP14BE: - case AV_PIX_FMT_GBRP14LE: - case AV_PIX_FMT_GBRP16BE: - case AV_PIX_FMT_GBRP16LE: - case AV_PIX_FMT_GBRPF16BE: - case AV_PIX_FMT_GBRPF16LE: - case AV_PIX_FMT_GBRAP: - case AV_PIX_FMT_GBRAP10LE: - case AV_PIX_FMT_GBRAP10BE: - case AV_PIX_FMT_GBRAP12LE: - case AV_PIX_FMT_GBRAP12BE: - case AV_PIX_FMT_GBRAP14LE: - case AV_PIX_FMT_GBRAP14BE: - case AV_PIX_FMT_GBRAP16LE: - case AV_PIX_FMT_GBRAP16BE: - case AV_PIX_FMT_GBRPF32BE: - case AV_PIX_FMT_GBRPF32LE: - case AV_PIX_FMT_GBRAPF16BE: - case AV_PIX_FMT_GBRAPF16LE: - case AV_PIX_FMT_GBRAPF32BE: - case AV_PIX_FMT_GBRAPF32LE: - case AV_PIX_FMT_GBRP10MSBBE: - case AV_PIX_FMT_GBRP10MSBLE: - case AV_PIX_FMT_GBRP12MSBBE: - case AV_PIX_FMT_GBRP12MSBLE: - return (SwsSwizzleOp) {{ .x = 1, 2, 0, 3 }}; default: return (SwsSwizzleOp) {{ .x = 0, 1, 2, 3 }}; } -- 2.49.1 >From 10b31055c8520da4c09b837adca6ea3347664984 Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Fri, 5 Dec 2025 18:49:54 +0100 Subject: [PATCH 4/8] swscale/format: derive fmt_shift() from AVPixFmtDescriptor XV36 is the odd one out, being a byte-shifted packed format whose components don't actually cross any byte boundaries. --- libswscale/format.c | 39 +++++++++++---------------------------- 1 file changed, 11 insertions(+), 28 deletions(-) diff --git a/libswscale/format.c b/libswscale/format.c index 7be3685c6c..076990d48d 100644 --- a/libswscale/format.c +++ b/libswscale/format.c @@ -742,36 +742,19 @@ static SwsSwizzleOp swizzle_inv(SwsSwizzleOp swiz) { /* Shift factor for MSB aligned formats */ static int fmt_shift(enum AVPixelFormat fmt) { - switch (fmt) { - case AV_PIX_FMT_P010BE: - case AV_PIX_FMT_P010LE: - case AV_PIX_FMT_P210BE: - case AV_PIX_FMT_P210LE: - case AV_PIX_FMT_Y210BE: - case AV_PIX_FMT_Y210LE: - case AV_PIX_FMT_YUV444P10MSBBE: - case AV_PIX_FMT_YUV444P10MSBLE: - case AV_PIX_FMT_GBRP10MSBBE: - case AV_PIX_FMT_GBRP10MSBLE: - return 6; - case AV_PIX_FMT_P012BE: - case AV_PIX_FMT_P012LE: - case AV_PIX_FMT_P212BE: - case AV_PIX_FMT_P212LE: - case AV_PIX_FMT_P412BE: - case AV_PIX_FMT_P412LE: - case AV_PIX_FMT_XV36BE: - case AV_PIX_FMT_XV36LE: - case AV_PIX_FMT_XYZ12BE: - case AV_PIX_FMT_XYZ12LE: - case AV_PIX_FMT_YUV444P12MSBBE: - case AV_PIX_FMT_YUV444P12MSBLE: - case AV_PIX_FMT_GBRP12MSBBE: - case AV_PIX_FMT_GBRP12MSBLE: - return 4; + if (is_regular_fmt(fmt)) { + const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(fmt); + return desc->comp[0].shift; } - return 0; + switch (fmt) { + case AV_PIX_FMT_XV36BE: + case AV_PIX_FMT_XV36LE: + return 4; + default: + /* Sub-byte irregular formats are always handled by SWS_OP_(UN)PACK */ + return 0; + } } /** -- 2.49.1 >From 91f88d1765cb9c929b43c80872f5b37570bd4bb6 Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Fri, 5 Dec 2025 19:12:11 +0100 Subject: [PATCH 5/8] swscale/format: explicitly test for unsupported subsampled formats This includes semiplanar formats. Note that the first check typically subsumes the second check, but I decided to keep both for clarity. --- libswscale/format.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/libswscale/format.c b/libswscale/format.c index 076990d48d..ffdde2af01 100644 --- a/libswscale/format.c +++ b/libswscale/format.c @@ -787,6 +787,15 @@ static int fmt_read_write(enum AVPixelFormat fmt, SwsReadWriteOp *rw_op, if (!desc) return AVERROR(EINVAL); + /* No support for subsampled formats at the moment */ + if (desc->log2_chroma_w || desc->log2_chroma_h) + return AVERROR(ENOTSUP); + + /* No support for semi-planar formats at the moment */ + if (desc->flags & AV_PIX_FMT_FLAG_PLANAR && + av_pix_fmt_count_planes(fmt) < desc->nb_components) + return AVERROR(ENOTSUP); + *pixel_type = fmt_pixel_type(fmt); if (!*pixel_type) return AVERROR(ENOTSUP); -- 2.49.1 >From 677fda4e4ae289f5e28891d3f5f4f344f8ce4228 Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Fri, 5 Dec 2025 19:17:30 +0100 Subject: [PATCH 6/8] swscale/format: derive fmt_read_write() for regular formats --- libswscale/format.c | 129 ++++---------------------------------------- 1 file changed, 9 insertions(+), 120 deletions(-) diff --git a/libswscale/format.c b/libswscale/format.c index ffdde2af01..013ff040b4 100644 --- a/libswscale/format.c +++ b/libswscale/format.c @@ -800,11 +800,16 @@ static int fmt_read_write(enum AVPixelFormat fmt, SwsReadWriteOp *rw_op, if (!*pixel_type) return AVERROR(ENOTSUP); - switch (fmt) { - case AV_PIX_FMT_NONE: - case AV_PIX_FMT_NB: - break; + if (is_regular_fmt(fmt)) { + *pack_op = (SwsPackOp) {0}; + *rw_op = (SwsReadWriteOp) { + .elems = desc->nb_components, + .packed = desc->nb_components > 1 && !(desc->flags & AV_PIX_FMT_FLAG_PLANAR), + }; + return 0; + } + switch (fmt) { /* Packed bitstream formats */ case AV_PIX_FMT_MONOWHITE: case AV_PIX_FMT_MONOBLACK: @@ -888,122 +893,6 @@ static int fmt_read_write(enum AVPixelFormat fmt, SwsReadWriteOp *rw_op, *pack_op = (SwsPackOp) {0}; *rw_op = (SwsReadWriteOp) { .elems = 4, .packed = true }; return 0; - /* Unpacked byte-aligned 4:4:4 formats */ - case AV_PIX_FMT_YUV444P: - case AV_PIX_FMT_YUVJ444P: - case AV_PIX_FMT_YUV444P9BE: - case AV_PIX_FMT_YUV444P9LE: - case AV_PIX_FMT_YUV444P10BE: - case AV_PIX_FMT_YUV444P10LE: - case AV_PIX_FMT_YUV444P12BE: - case AV_PIX_FMT_YUV444P12LE: - case AV_PIX_FMT_YUV444P14BE: - case AV_PIX_FMT_YUV444P14LE: - case AV_PIX_FMT_YUV444P16BE: - case AV_PIX_FMT_YUV444P16LE: - case AV_PIX_FMT_YUV444P10MSBBE: - case AV_PIX_FMT_YUV444P10MSBLE: - case AV_PIX_FMT_YUV444P12MSBBE: - case AV_PIX_FMT_YUV444P12MSBLE: - case AV_PIX_FMT_YUVA444P: - case AV_PIX_FMT_YUVA444P9BE: - case AV_PIX_FMT_YUVA444P9LE: - case AV_PIX_FMT_YUVA444P10BE: - case AV_PIX_FMT_YUVA444P10LE: - case AV_PIX_FMT_YUVA444P12BE: - case AV_PIX_FMT_YUVA444P12LE: - case AV_PIX_FMT_YUVA444P16BE: - case AV_PIX_FMT_YUVA444P16LE: - case AV_PIX_FMT_AYUV: - case AV_PIX_FMT_UYVA: - case AV_PIX_FMT_VYU444: - case AV_PIX_FMT_AYUV64BE: - case AV_PIX_FMT_AYUV64LE: - case AV_PIX_FMT_VUYA: - case AV_PIX_FMT_RGB24: - case AV_PIX_FMT_BGR24: - case AV_PIX_FMT_RGB48BE: - case AV_PIX_FMT_RGB48LE: - case AV_PIX_FMT_BGR48BE: - case AV_PIX_FMT_BGR48LE: - //case AV_PIX_FMT_RGB96BE: TODO: AVRational can't fit 2^32-1 - //case AV_PIX_FMT_RGB96LE: - //case AV_PIX_FMT_RGBF16BE: TODO: no support for float16 currently - //case AV_PIX_FMT_RGBF16LE: - case AV_PIX_FMT_RGBF32BE: - case AV_PIX_FMT_RGBF32LE: - case AV_PIX_FMT_ARGB: - case AV_PIX_FMT_RGBA: - case AV_PIX_FMT_ABGR: - case AV_PIX_FMT_BGRA: - case AV_PIX_FMT_RGBA64BE: - case AV_PIX_FMT_RGBA64LE: - case AV_PIX_FMT_BGRA64BE: - case AV_PIX_FMT_BGRA64LE: - //case AV_PIX_FMT_RGBA128BE: TODO: AVRational can't fit 2^32-1 - //case AV_PIX_FMT_RGBA128LE: - case AV_PIX_FMT_RGBAF32BE: - case AV_PIX_FMT_RGBAF32LE: - case AV_PIX_FMT_GBRP: - case AV_PIX_FMT_GBRP9BE: - case AV_PIX_FMT_GBRP9LE: - case AV_PIX_FMT_GBRP10BE: - case AV_PIX_FMT_GBRP10LE: - case AV_PIX_FMT_GBRP12BE: - case AV_PIX_FMT_GBRP12LE: - case AV_PIX_FMT_GBRP14BE: - case AV_PIX_FMT_GBRP14LE: - case AV_PIX_FMT_GBRP16BE: - case AV_PIX_FMT_GBRP16LE: - //case AV_PIX_FMT_GBRPF16BE: TODO - //case AV_PIX_FMT_GBRPF16LE: - case AV_PIX_FMT_GBRP10MSBBE: - case AV_PIX_FMT_GBRP10MSBLE: - case AV_PIX_FMT_GBRP12MSBBE: - case AV_PIX_FMT_GBRP12MSBLE: - case AV_PIX_FMT_GBRPF32BE: - case AV_PIX_FMT_GBRPF32LE: - case AV_PIX_FMT_GBRAP: - case AV_PIX_FMT_GBRAP10BE: - case AV_PIX_FMT_GBRAP10LE: - case AV_PIX_FMT_GBRAP12BE: - case AV_PIX_FMT_GBRAP12LE: - case AV_PIX_FMT_GBRAP14BE: - case AV_PIX_FMT_GBRAP14LE: - case AV_PIX_FMT_GBRAP16BE: - case AV_PIX_FMT_GBRAP16LE: - //case AV_PIX_FMT_GBRAPF16BE: TODO - //case AV_PIX_FMT_GBRAPF16LE: - case AV_PIX_FMT_GBRAPF32BE: - case AV_PIX_FMT_GBRAPF32LE: - case AV_PIX_FMT_GRAY8: - case AV_PIX_FMT_GRAY9BE: - case AV_PIX_FMT_GRAY9LE: - case AV_PIX_FMT_GRAY10BE: - case AV_PIX_FMT_GRAY10LE: - case AV_PIX_FMT_GRAY12BE: - case AV_PIX_FMT_GRAY12LE: - case AV_PIX_FMT_GRAY14BE: - case AV_PIX_FMT_GRAY14LE: - case AV_PIX_FMT_GRAY16BE: - case AV_PIX_FMT_GRAY16LE: - //case AV_PIX_FMT_GRAYF16BE: TODO - //case AV_PIX_FMT_GRAYF16LE: - //case AV_PIX_FMT_YAF16BE: - //case AV_PIX_FMT_YAF16LE: - case AV_PIX_FMT_GRAYF32BE: - case AV_PIX_FMT_GRAYF32LE: - case AV_PIX_FMT_YAF32BE: - case AV_PIX_FMT_YAF32LE: - case AV_PIX_FMT_YA8: - case AV_PIX_FMT_YA16LE: - case AV_PIX_FMT_YA16BE: - *pack_op = (SwsPackOp) {0}; - *rw_op = (SwsReadWriteOp) { - .elems = desc->nb_components, - .packed = desc->nb_components > 1 && !(desc->flags & AV_PIX_FMT_FLAG_PLANAR), - }; - return 0; } return AVERROR(ENOTSUP); -- 2.49.1 >From 018757731ce24f26360b5f8589e14a7b9ba01463 Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Fri, 5 Dec 2025 19:35:39 +0100 Subject: [PATCH 7/8] swscale/format: consolidate format information into a single struct I use a switch/case instead of an array because this is only needed for irregular formats, which are very sparse. --- libswscale/format.c | 226 +++++++++++++++++++------------------------- 1 file changed, 97 insertions(+), 129 deletions(-) diff --git a/libswscale/format.c b/libswscale/format.c index 013ff040b4..d50cf07ade 100644 --- a/libswscale/format.c +++ b/libswscale/format.c @@ -655,6 +655,92 @@ static int is_regular_fmt(enum AVPixelFormat fmt) return total_bits == step * 8; } +typedef struct FmtInfo { + SwsReadWriteOp rw; + SwsSwizzleOp swizzle; + SwsPackOp pack; + int shift; +} FmtInfo; + +#define RW(N, ...) (SwsReadWriteOp) { .elems = N, __VA_ARGS__ } + +#define RGBA SWS_SWIZZLE(0, 1, 2, 3) +#define BGRA SWS_SWIZZLE(2, 1, 0, 3) +#define ARGB SWS_SWIZZLE(3, 0, 1, 2) +#define ABGR SWS_SWIZZLE(3, 2, 1, 0) +#define AVYU SWS_SWIZZLE(3, 2, 0, 1) +#define VYUA SWS_SWIZZLE(2, 0, 1, 3) +#define UYVA SWS_SWIZZLE(1, 0, 2, 3) +#define VUYA BGRA + +static FmtInfo fmt_info_irregular(enum AVPixelFormat fmt) +{ + switch (fmt) { + /* Bitstream formats */ + case AV_PIX_FMT_MONOWHITE: + case AV_PIX_FMT_MONOBLACK: + return (FmtInfo) { RW(1, .frac = 3), RGBA }; + case AV_PIX_FMT_RGB4: + return (FmtInfo) { RW(1, .frac = 1), RGBA, {{ 1, 2, 1 }} }; + case AV_PIX_FMT_BGR4: + return (FmtInfo) { RW(1, .frac = 1), BGRA, {{ 1, 2, 1 }} }; + + /* Sub-packed 8-bit aligned formats */ + case AV_PIX_FMT_RGB4_BYTE: return (FmtInfo) { RW(1), RGBA, {{ 1, 2, 1 }} }; + case AV_PIX_FMT_BGR4_BYTE: return (FmtInfo) { RW(1), BGRA, {{ 1, 2, 1 }} }; + case AV_PIX_FMT_RGB8: return (FmtInfo) { RW(1), RGBA, {{ 3, 3, 2 }} }; + case AV_PIX_FMT_BGR8: return (FmtInfo) { RW(1), BGRA, {{ 2, 3, 3 }} }; + + /* Sub-packed 16-bit aligned formats */ + case AV_PIX_FMT_RGB565LE: + case AV_PIX_FMT_RGB565BE: + return (FmtInfo) { RW(1), RGBA, {{ 5, 6, 5 }} }; + case AV_PIX_FMT_BGR565LE: + case AV_PIX_FMT_BGR565BE: + return (FmtInfo) { RW(1), BGRA, {{ 5, 6, 5 }} }; + case AV_PIX_FMT_RGB555LE: + case AV_PIX_FMT_RGB555BE: + return (FmtInfo) { RW(1), RGBA, {{ 5, 5, 5 }} }; + case AV_PIX_FMT_BGR555LE: + case AV_PIX_FMT_BGR555BE: + return (FmtInfo) { RW(1), BGRA, {{ 5, 5, 5 }} }; + case AV_PIX_FMT_RGB444LE: + case AV_PIX_FMT_RGB444BE: + return (FmtInfo) { RW(1), RGBA, {{ 4, 4, 4 }} }; + case AV_PIX_FMT_BGR444LE: + case AV_PIX_FMT_BGR444BE: + return (FmtInfo) { RW(1), BGRA, {{ 4, 4, 4 }} }; + + /* Sub-packed 32-bit aligned formats */ + case AV_PIX_FMT_X2RGB10LE: + case AV_PIX_FMT_X2RGB10BE: + return (FmtInfo) { RW(1), ARGB, {{ 2, 10, 10, 10 }} }; + case AV_PIX_FMT_X2BGR10LE: + case AV_PIX_FMT_X2BGR10BE: + return (FmtInfo) { RW(1), ABGR, {{ 2, 10, 10, 10 }} }; + case AV_PIX_FMT_XV30LE: + case AV_PIX_FMT_XV30BE: + return (FmtInfo) { RW(1), AVYU, {{ 2, 10, 10, 10 }} }; + case AV_PIX_FMT_V30XLE: + case AV_PIX_FMT_V30XBE: + return (FmtInfo) { RW(1), VYUA, {{ 10, 10, 10, 2 }} }; + /* 3-component formats with extra padding */ + case AV_PIX_FMT_RGB0: return (FmtInfo) { RW(4), RGBA }; + case AV_PIX_FMT_BGR0: return (FmtInfo) { RW(4), BGRA }; + case AV_PIX_FMT_0RGB: return (FmtInfo) { RW(4), ARGB }; + case AV_PIX_FMT_0BGR: return (FmtInfo) { RW(4), ABGR }; + case AV_PIX_FMT_VUYX: return (FmtInfo) { RW(4), VUYA }; + case AV_PIX_FMT_XV36LE: + case AV_PIX_FMT_XV36BE: + return (FmtInfo) { RW(4), UYVA, .shift = 4 }; + case AV_PIX_FMT_XV48LE: + case AV_PIX_FMT_XV48BE: + return (FmtInfo) { RW(4), UYVA }; + } + + return (FmtInfo) {0}; +} + struct comp { int index; int plane; @@ -692,41 +778,8 @@ static SwsSwizzleOp fmt_swizzle(enum AVPixelFormat fmt) return swiz; } - switch (fmt) { - case AV_PIX_FMT_0RGB: - case AV_PIX_FMT_X2RGB10LE: - case AV_PIX_FMT_X2RGB10BE: - return (SwsSwizzleOp) {{ .x = 3, 0, 1, 2 }}; - case AV_PIX_FMT_BGR8: - case AV_PIX_FMT_BGR4: - case AV_PIX_FMT_BGR4_BYTE: - case AV_PIX_FMT_BGR565BE: - case AV_PIX_FMT_BGR565LE: - case AV_PIX_FMT_BGR555BE: - case AV_PIX_FMT_BGR555LE: - case AV_PIX_FMT_BGR444BE: - case AV_PIX_FMT_BGR444LE: - case AV_PIX_FMT_BGR0: - case AV_PIX_FMT_VUYX: - return (SwsSwizzleOp) {{ .x = 2, 1, 0, 3 }}; - case AV_PIX_FMT_0BGR: - case AV_PIX_FMT_X2BGR10LE: - case AV_PIX_FMT_X2BGR10BE: - return (SwsSwizzleOp) {{ .x = 3, 2, 1, 0 }}; - case AV_PIX_FMT_XV30BE: - case AV_PIX_FMT_XV30LE: - return (SwsSwizzleOp) {{ .x = 3, 2, 0, 1 }}; - case AV_PIX_FMT_V30XBE: - case AV_PIX_FMT_V30XLE: - return (SwsSwizzleOp) {{ .x = 2, 0, 1, 3 }}; - case AV_PIX_FMT_XV36BE: - case AV_PIX_FMT_XV36LE: - case AV_PIX_FMT_XV48BE: - case AV_PIX_FMT_XV48LE: - return (SwsSwizzleOp) {{ .x = 1, 0, 2, 3 }}; - default: - return (SwsSwizzleOp) {{ .x = 0, 1, 2, 3 }}; - } + /* This function is only ever called if fmt_read_write() already passes */ + return fmt_info_irregular(fmt).swizzle; } static SwsSwizzleOp swizzle_inv(SwsSwizzleOp swiz) { @@ -747,14 +800,8 @@ static int fmt_shift(enum AVPixelFormat fmt) return desc->comp[0].shift; } - switch (fmt) { - case AV_PIX_FMT_XV36BE: - case AV_PIX_FMT_XV36LE: - return 4; - default: - /* Sub-byte irregular formats are always handled by SWS_OP_(UN)PACK */ - return 0; - } + /* This function is only ever called if fmt_read_write() already passes */ + return fmt_info_irregular(fmt).shift; } /** @@ -809,93 +856,14 @@ static int fmt_read_write(enum AVPixelFormat fmt, SwsReadWriteOp *rw_op, return 0; } - switch (fmt) { - /* Packed bitstream formats */ - case AV_PIX_FMT_MONOWHITE: - case AV_PIX_FMT_MONOBLACK: - *pack_op = (SwsPackOp) {0}; - *rw_op = (SwsReadWriteOp) { - .elems = 1, - .frac = 3, - }; - return 0; - case AV_PIX_FMT_RGB4: - case AV_PIX_FMT_BGR4: - *pack_op = (SwsPackOp) {{ 1, 2, 1 }}; - *rw_op = (SwsReadWriteOp) { - .elems = 1, - .packed = true, - .frac = 1, - }; - return 0; - /* Packed 8-bit aligned formats */ - case AV_PIX_FMT_RGB4_BYTE: - case AV_PIX_FMT_BGR4_BYTE: - *pack_op = (SwsPackOp) {{ 1, 2, 1 }}; - *rw_op = (SwsReadWriteOp) { .elems = 1, .packed = true }; - return 0; - case AV_PIX_FMT_BGR8: - *pack_op = (SwsPackOp) {{ 2, 3, 3 }}; - *rw_op = (SwsReadWriteOp) { .elems = 1, .packed = true }; - return 0; - case AV_PIX_FMT_RGB8: - *pack_op = (SwsPackOp) {{ 3, 3, 2 }}; - *rw_op = (SwsReadWriteOp) { .elems = 1, .packed = true }; - return 0; + FmtInfo info = fmt_info_irregular(fmt); + if (!info.swizzle.mask) + return AVERROR(ENOTSUP); - /* Packed 16-bit aligned formats */ - case AV_PIX_FMT_RGB565BE: - case AV_PIX_FMT_RGB565LE: - case AV_PIX_FMT_BGR565BE: - case AV_PIX_FMT_BGR565LE: - *pack_op = (SwsPackOp) {{ 5, 6, 5 }}; - *rw_op = (SwsReadWriteOp) { .elems = 1, .packed = true }; - return 0; - case AV_PIX_FMT_RGB555BE: - case AV_PIX_FMT_RGB555LE: - case AV_PIX_FMT_BGR555BE: - case AV_PIX_FMT_BGR555LE: - *pack_op = (SwsPackOp) {{ 5, 5, 5 }}; - *rw_op = (SwsReadWriteOp) { .elems = 1, .packed = true }; - return 0; - case AV_PIX_FMT_RGB444BE: - case AV_PIX_FMT_RGB444LE: - case AV_PIX_FMT_BGR444BE: - case AV_PIX_FMT_BGR444LE: - *pack_op = (SwsPackOp) {{ 4, 4, 4 }}; - *rw_op = (SwsReadWriteOp) { .elems = 1, .packed = true }; - return 0; - /* Packed 32-bit aligned 4:4:4 formats */ - case AV_PIX_FMT_X2RGB10BE: - case AV_PIX_FMT_X2RGB10LE: - case AV_PIX_FMT_X2BGR10BE: - case AV_PIX_FMT_X2BGR10LE: - case AV_PIX_FMT_XV30BE: - case AV_PIX_FMT_XV30LE: - *pack_op = (SwsPackOp) {{ 2, 10, 10, 10 }}; - *rw_op = (SwsReadWriteOp) { .elems = 1, .packed = true }; - return 0; - case AV_PIX_FMT_V30XBE: - case AV_PIX_FMT_V30XLE: - *pack_op = (SwsPackOp) {{ 10, 10, 10, 2 }}; - *rw_op = (SwsReadWriteOp) { .elems = 1, .packed = true }; - return 0; - /* 3 component formats with one channel ignored */ - case AV_PIX_FMT_RGB0: - case AV_PIX_FMT_BGR0: - case AV_PIX_FMT_0RGB: - case AV_PIX_FMT_0BGR: - case AV_PIX_FMT_XV36BE: - case AV_PIX_FMT_XV36LE: - case AV_PIX_FMT_XV48BE: - case AV_PIX_FMT_XV48LE: - case AV_PIX_FMT_VUYX: - *pack_op = (SwsPackOp) {0}; - *rw_op = (SwsReadWriteOp) { .elems = 4, .packed = true }; - return 0; - } - - return AVERROR(ENOTSUP); + *pack_op = info.pack; + *rw_op = info.rw; + rw_op->packed = desc->nb_components > 1; + return 0; } static SwsPixelType get_packed_type(SwsPackOp pack) -- 2.49.1 >From 80f16f8d9c87bf4425505f2d44863d6f5d4fabd1 Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Fri, 5 Dec 2025 20:57:55 +0100 Subject: [PATCH 8/8] swscale/format: merge fmt_* helpers into a single fmt_analyze() Handles the split between "regular" and "irregular" pixel formats in a single place, and opens up the door for more complicated formats. --- libswscale/format.c | 161 +++++++++++++++++++++----------------------- 1 file changed, 76 insertions(+), 85 deletions(-) diff --git a/libswscale/format.c b/libswscale/format.c index d50cf07ade..65fda2f424 100644 --- a/libswscale/format.c +++ b/libswscale/format.c @@ -756,13 +756,14 @@ static int cmp_comp(const void *a, const void *b) { return ca->offset - cb->offset; } -static SwsSwizzleOp fmt_swizzle(enum AVPixelFormat fmt) +static int fmt_analyze_regular(enum AVPixelFormat fmt, SwsReadWriteOp *rw_op, + SwsSwizzleOp *swizzle, int *shift) { const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(fmt); if (desc->nb_components == 2) { /* YA formats */ - return SWS_SWIZZLE(0, 3, 1, 2); - } else if (is_regular_fmt(fmt)) { + *swizzle = SWS_SWIZZLE(0, 3, 1, 2); + } else { /* Sort by increasing component order */ struct comp sorted[4] = { {0}, {1}, {2}, {3} }; for (int i = 0; i < desc->nb_components; i++) { @@ -775,11 +776,65 @@ static SwsSwizzleOp fmt_swizzle(enum AVPixelFormat fmt) SwsSwizzleOp swiz = SWS_SWIZZLE(0, 1, 2, 3); for (int i = 0; i < desc->nb_components; i++) swiz.in[i] = sorted[i].index; - return swiz; + *swizzle = swiz; } - /* This function is only ever called if fmt_read_write() already passes */ - return fmt_info_irregular(fmt).swizzle; + *shift = desc->comp[0].shift; + *rw_op = (SwsReadWriteOp) { + .elems = desc->nb_components, + .packed = desc->nb_components > 1 && !(desc->flags & AV_PIX_FMT_FLAG_PLANAR), + }; + return 0; +} + +static int fmt_analyze(enum AVPixelFormat fmt, SwsReadWriteOp *rw_op, + SwsPackOp *pack_op, SwsSwizzleOp *swizzle, int *shift, + SwsPixelType *pixel_type, SwsPixelType *raw_type) +{ + const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(fmt); + if (!desc) + return AVERROR(EINVAL); + + /* No support for subsampled formats at the moment */ + if (desc->log2_chroma_w || desc->log2_chroma_h) + return AVERROR(ENOTSUP); + + /* No support for semi-planar formats at the moment */ + if (desc->flags & AV_PIX_FMT_FLAG_PLANAR && + av_pix_fmt_count_planes(fmt) < desc->nb_components) + return AVERROR(ENOTSUP); + + *pixel_type = *raw_type = fmt_pixel_type(fmt); + if (!*pixel_type) + return AVERROR(ENOTSUP); + + if (is_regular_fmt(fmt)) { + *pack_op = (SwsPackOp) {0}; + return fmt_analyze_regular(fmt, rw_op, swizzle, shift); + } + + FmtInfo info = fmt_info_irregular(fmt); + if (!info.swizzle.mask) + return AVERROR(ENOTSUP); + + *rw_op = info.rw; + *pack_op = info.pack; + *swizzle = info.swizzle; + *shift = info.shift; + rw_op->packed = desc->nb_components > 1; + + if (info.pack.pattern[0]) { + const int sum = info.pack.pattern[0] + info.pack.pattern[1] + + info.pack.pattern[2] + info.pack.pattern[3]; + if (sum > 16) + *raw_type = SWS_PIXEL_U32; + else if (sum > 8) + *raw_type = SWS_PIXEL_U16; + else + *raw_type = SWS_PIXEL_U8; + } + + return 0; } static SwsSwizzleOp swizzle_inv(SwsSwizzleOp swiz) { @@ -792,18 +847,6 @@ static SwsSwizzleOp swizzle_inv(SwsSwizzleOp swiz) { return (SwsSwizzleOp) {{ .x = out[0], out[1], out[2], out[3] }}; } -/* Shift factor for MSB aligned formats */ -static int fmt_shift(enum AVPixelFormat fmt) -{ - if (is_regular_fmt(fmt)) { - const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(fmt); - return desc->comp[0].shift; - } - - /* This function is only ever called if fmt_read_write() already passes */ - return fmt_info_irregular(fmt).shift; -} - /** * This initializes all absent components explicitly to zero. There is no * need to worry about the correct neutral value as fmt_decode() will @@ -827,57 +870,6 @@ static SwsConst fmt_clear(enum AVPixelFormat fmt) return c; } -static int fmt_read_write(enum AVPixelFormat fmt, SwsReadWriteOp *rw_op, - SwsPackOp *pack_op, SwsPixelType *pixel_type) -{ - const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(fmt); - if (!desc) - return AVERROR(EINVAL); - - /* No support for subsampled formats at the moment */ - if (desc->log2_chroma_w || desc->log2_chroma_h) - return AVERROR(ENOTSUP); - - /* No support for semi-planar formats at the moment */ - if (desc->flags & AV_PIX_FMT_FLAG_PLANAR && - av_pix_fmt_count_planes(fmt) < desc->nb_components) - return AVERROR(ENOTSUP); - - *pixel_type = fmt_pixel_type(fmt); - if (!*pixel_type) - return AVERROR(ENOTSUP); - - if (is_regular_fmt(fmt)) { - *pack_op = (SwsPackOp) {0}; - *rw_op = (SwsReadWriteOp) { - .elems = desc->nb_components, - .packed = desc->nb_components > 1 && !(desc->flags & AV_PIX_FMT_FLAG_PLANAR), - }; - return 0; - } - - FmtInfo info = fmt_info_irregular(fmt); - if (!info.swizzle.mask) - return AVERROR(ENOTSUP); - - *pack_op = info.pack; - *rw_op = info.rw; - rw_op->packed = desc->nb_components > 1; - return 0; -} - -static SwsPixelType get_packed_type(SwsPackOp pack) -{ - const int sum = pack.pattern[0] + pack.pattern[1] + - pack.pattern[2] + pack.pattern[3]; - if (sum > 16) - return SWS_PIXEL_U32; - else if (sum > 8) - return SWS_PIXEL_U16; - else - return SWS_PIXEL_U8; -} - #if HAVE_BIGENDIAN # define NATIVE_ENDIAN_FLAG AV_PIX_FMT_FLAG_BE #else @@ -887,15 +879,14 @@ static SwsPixelType get_packed_type(SwsPackOp pack) int ff_sws_decode_pixfmt(SwsOpList *ops, enum AVPixelFormat fmt) { const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(fmt); - SwsPixelType pixel_type; + SwsPixelType pixel_type, raw_type; SwsReadWriteOp rw_op; + SwsSwizzleOp swizzle; SwsPackOp unpack; + int shift; - RET(fmt_read_write(fmt, &rw_op, &unpack, &pixel_type)); - - SwsPixelType raw_type = pixel_type; - if (unpack.pattern[0]) - raw_type = get_packed_type(unpack); + RET(fmt_analyze(fmt, &rw_op, &unpack, &swizzle, &shift, + &pixel_type, &raw_type)); /* TODO: handle subsampled or semipacked input formats */ RET(ff_sws_op_list_append(ops, &(SwsOp) { @@ -928,13 +919,13 @@ int ff_sws_decode_pixfmt(SwsOpList *ops, enum AVPixelFormat fmt) RET(ff_sws_op_list_append(ops, &(SwsOp) { .op = SWS_OP_SWIZZLE, .type = pixel_type, - .swizzle = swizzle_inv(fmt_swizzle(fmt)), + .swizzle = swizzle_inv(swizzle), })); RET(ff_sws_op_list_append(ops, &(SwsOp) { .op = SWS_OP_RSHIFT, .type = pixel_type, - .c.u = fmt_shift(fmt), + .c.u = shift, })); RET(ff_sws_op_list_append(ops, &(SwsOp) { @@ -949,20 +940,19 @@ int ff_sws_decode_pixfmt(SwsOpList *ops, enum AVPixelFormat fmt) int ff_sws_encode_pixfmt(SwsOpList *ops, enum AVPixelFormat fmt) { const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(fmt); - SwsPixelType pixel_type; + SwsPixelType pixel_type, raw_type; SwsReadWriteOp rw_op; + SwsSwizzleOp swizzle; SwsPackOp pack; + int shift; - RET(fmt_read_write(fmt, &rw_op, &pack, &pixel_type)); - - SwsPixelType raw_type = pixel_type; - if (pack.pattern[0]) - raw_type = get_packed_type(pack); + RET(fmt_analyze(fmt, &rw_op, &pack, &swizzle, &shift, + &pixel_type, &raw_type)); RET(ff_sws_op_list_append(ops, &(SwsOp) { .op = SWS_OP_LSHIFT, .type = pixel_type, - .c.u = fmt_shift(fmt), + .c.u = shift, })); if (rw_op.elems > desc->nb_components) { @@ -978,7 +968,7 @@ int ff_sws_encode_pixfmt(SwsOpList *ops, enum AVPixelFormat fmt) RET(ff_sws_op_list_append(ops, &(SwsOp) { .op = SWS_OP_SWIZZLE, .type = pixel_type, - .swizzle = fmt_swizzle(fmt), + .swizzle = swizzle, })); if (pack.pattern[0]) { @@ -1007,6 +997,7 @@ int ff_sws_encode_pixfmt(SwsOpList *ops, enum AVPixelFormat fmt) .type = raw_type, .rw = rw_op, })); + return 0; } -- 2.49.1 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
