PR #24585 opened by Lynne URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24585 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24585.patch
The issue is that the prores simple_idct variant in 32-bit mode masks off all 4 LSBs. But the decoder outputs full-range 16-bits, which means 4 bits fly away, which is enough for even 12 stop sensors to start losing precision off the image. So, introduce a new 16-bit variant, which reuses the 10-bit transform coefficient weights, scaled to 32-bit intermediates, and output 16-bit coeffs without masking any bits. This is around a 20x better approximation of the iDCT, and makes the software decoder actually usable (thought still you'd want to use the Vulkan version, which is better and faster). >From 37e699b5ff837eb77601cd62cca77448fb25bf7a Mon Sep 17 00:00:00 2001 From: Lynne <[email protected]> Date: Sun, 20 Sep 2026 23:39:32 +0900 Subject: [PATCH 1/2] simple_idct: add a 16-bit output variant with 32-bit intermediates For coefficients and output that both span 16 bits, like those of ProRes RAW. The template's 16-bit intermediates cannot hold more fractional bits than the output needs, and the constants of the 12-bit transform, scaled to 15 bits, make W4 32767 rather than 32768, which alone costs 0.8 LSB RMS of a 16-bit output. The new variant uses the 10-bit transform's constants, scaled to 14 bits and exact at W4, with a row shift of 13 and a column shift of 15, the most fractional bits the 32-bit sums of a 16-bit result allow. Against an exact transform, it is within 0.49 LSB RMS and 2.3 LSB peak on real content, with no bias, and passes every IEEE 1180 criterion in the standard's 8-bit units by three orders of magnitude. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_016LFnYMstwe4woBzpxfzDzg --- libavcodec/simple_idct_template.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/libavcodec/simple_idct_template.c b/libavcodec/simple_idct_template.c index e189ef1a8e..051ce17a3b 100644 --- a/libavcodec/simple_idct_template.c +++ b/libavcodec/simple_idct_template.c @@ -60,9 +60,9 @@ #define MUL(a, b) MUL16(a, b) #define MAC(a, b, c) MAC16(a, b, c) -#elif BIT_DEPTH == 10 || BIT_DEPTH == 12 +#elif BIT_DEPTH == 10 || BIT_DEPTH == 12 || BIT_DEPTH == 16 -# if BIT_DEPTH == 10 +# if BIT_DEPTH == 10 || BIT_DEPTH == 16 #define W1 22725 // 90901 #define W2 21407 // 85627 #define W3 19265 // 77062 @@ -71,7 +71,17 @@ #define W6 8867 // 35468 #define W7 4520 // 18081 -# ifdef EXTRA_SHIFT +# if BIT_DEPTH == 16 +/* 16-bit output from 16-bit coefficients. The rows keep two more + * fractional bits than 16-bit intermediates could hold, and the columns + * are shifted by as little as the 32-bit sums of a 16-bit result allow */ +# if IN_IDCT_DEPTH != 32 +#error "The 16-bit iDCT needs 32-bit intermediates" +# endif +#define ROW_SHIFT 13 +#define COL_SHIFT 15 +#define DC_SHIFT 1 +# elif defined(EXTRA_SHIFT) #define ROW_SHIFT 13 #define COL_SHIFT 18 #define DC_SHIFT 1 -- 2.52.0 >From 26711a80d7fd3a1bd8f3e5557bc3d5e4b7e2013c Mon Sep 17 00:00:00 2001 From: Lynne <[email protected]> Date: Sun, 20 Sep 2026 23:39:32 +0900 Subject: [PATCH 2/2] prores_raw: use the 16-bit iDCT The decoder converted the 32-bit output of its transform to 16 bits through an empirically found offset, and added one to every DC coefficient to compensate for the bias of the result. Neither reproduces the transform well: against a bit-exact model of the reference decoder, most samples are off, with a quantizer-dependent bias and errors of up to 357 out of 65535 at the extremes of the range, and the DC offset only helps on HQ material. Use the 16-bit output variant of simple_idct instead, with the dequantized coefficients saturated to 16 bits as the reference does, and the centered transform output biased in the output stage. Against an exact transform, the error drops from 5.6 LSB RMS with a maximum of 21 to 0.49 LSB RMS with a maximum of 2.3 and no bias, on par with the reference decoder's 0.29 and 2.4. The decoded output agrees with the reference decoder on 95% of the samples of a 5.7K RAW HQ frame with a log curve, within 3.5 LSB of a 14-bit sensor in its top stop and within 1 LSB elsewhere. Speed is unchanged. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_016LFnYMstwe4woBzpxfzDzg --- libavcodec/prores_raw.c | 4 ++-- libavcodec/proresdsp.c | 28 ++++++++++++---------------- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/libavcodec/prores_raw.c b/libavcodec/prores_raw.c index dc11b14331..36c3879f73 100644 --- a/libavcodec/prores_raw.c +++ b/libavcodec/prores_raw.c @@ -185,7 +185,7 @@ static int decode_comp(AVCodecContext *avctx, TileContext *tile, if (dc < 0) goto end; int prev_dc = (dc >> 1) ^ -(dc & 1); - block[0] = prev_dc + 1; + block[0] = prev_dc; for (int n = 1; n < nb_blocks; n++) { if ((n & 15) == 1) @@ -202,7 +202,7 @@ static int decode_comp(AVCodecContext *avctx, TileContext *tile, sign = dc_add < 0; prev_dc += dc_add; - block[n*64] = prev_dc + 1; + block[n*64] = prev_dc; } for (int n = nb_blocks; n < nb_codes;) { diff --git a/libavcodec/proresdsp.c b/libavcodec/proresdsp.c index cf4aa9d0cf..919a5d6797 100644 --- a/libavcodec/proresdsp.c +++ b/libavcodec/proresdsp.c @@ -42,9 +42,9 @@ #undef BIT_DEPTH #undef IN_IDCT_DEPTH -/* 32bit iDCT for the ProRes RAW */ +/* 16-bit iDCT for ProRes RAW */ #define IN_IDCT_DEPTH 32 -#define BIT_DEPTH 12 +#define BIT_DEPTH 16 #include "simple_idct_template.c" #undef BIT_DEPTH #undef IN_IDCT_DEPTH @@ -83,21 +83,18 @@ static void prores_idct_12(int16_t *restrict block, const int16_t *restrict qmat } /* - * 32-bit iDCT for the ProRes RAW - * qmat must be s->qmat[i] * scale + * ProRes RAW iDCT: 16-bit coefficients to a 16-bit output centered on zero */ -static void prores_idct_bayer_32(int32_t *restrict block, const int16_t *restrict qmat) +static void prores_idct_bayer_16(int32_t *restrict block, const int16_t *restrict qmat) { for (int i = 0; i < 64; i++) - block[i] = (block[i] * qmat[i]) >> 1; + block[i] = av_clip_int16(block[i] * qmat[i]); for (int i = 0; i < 8; i++) - idctRowCondDC_int32_12bit(block + i*8, 0); + idctRowCondDC_int32_16bit(block + i*8, 0); - for (int i = 0; i < 8; i++) { - block[i] += 8192; - idctSparseCol_int32_12bit(block + i); - } + for (int i = 0; i < 8; i++) + idctSparseCol_int32_16bit(block + i); } #define CLIP_MIN (1 << 2) ///< minimum value for clipping resulting pixels @@ -131,10 +128,9 @@ static inline void put_pixel_bayer_lin_curve_12(uint16_t *dst, ptrdiff_t linesiz { for (int y = 0; y < 8; y++, dst += linesize) { for (int x = 0; x < 8; x++) { - /* Convert the 32-bit input into 16-bits (lrintf(x*16 - 15.5f) = 16) */ - int u = av_clip_uint16(in[(y << 3) + x]*16 - 16); - uint32_t seg = (uint32_t)u >> 13; - uint32_t frac = (uint32_t)u & 0x1FFF; + uint32_t u = av_clip_uint16(in[(y << 3) + x] + 32768); + uint32_t seg = u >> 13; + uint32_t frac = u & 0x1FFF; uint32_t cp0 = lin_curve[seg]; uint32_t cp1 = seg < 7 ? lin_curve[seg + 1] : 0; uint32_t o = (cp0 * 8192 + ((cp1 - cp0) & 0xFFFF) * frac + 4096) >> 13; @@ -169,7 +165,7 @@ static void prores_idct_put_bayer_12_c(uint16_t *out, ptrdiff_t linesize, int32_t *block, const int16_t *qmat, const uint16_t *lin_curve) { - prores_idct_bayer_32(block, qmat); + prores_idct_bayer_16(block, qmat); put_pixel_bayer_lin_curve_12(out, linesize << 1, block, lin_curve); } -- 2.52.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
