PR #25450 opened by Mark Thompson (jkqxz) URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/25450 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/25450.patch
Changes: * Make the checkasm test better exercise the edge cases of the transquant functions. Now runs for all bit depths / QPs / qmatrix, with an attempt to match real values and hit the clipping cases. * Microoptimisations to AVX2 transquant function. Mainly replacing the row matmul with a butterfly fused into other stuff which is going on, for slightly lower instruction count but more gain on some processors due to better use of execution ports. * Precompute per-tile-component values which were previously computed every time inside the transquant function. Significant gains in AVX2 where we can arrange things better; neutral on NEON but probably there are more opportunities there. * Rewrite the multi-symbol entropy decode. Main idea is to make things unconditional or indexed where possible, avoiding branches and corresponding branch prediction misses. The LUT setup changes to store different things to make this work. This change is the largest gain here, and applies everywhere. * Use a local bitstream reader in the entropy decode to get the compiler to do the right thing. I'd guess LTO might already capture this gain, but it's a trivial change to make it always apply. Some numbers, all measured from the outside of the ffmpeg utility so including overheads: Single-threaded with 4:2:2 HQ 1080p: * Intel Cascade Lake: 46.2fps -> 61.7fps * Intel Alder Lake (P): 62.1fps -> 80.2fps * Intel Alder Lake (E): 42.1fps -> 50.1fps * ARM Cortex A53: 12.5fps -> 15.6fps * ARM Cortex A73: 16.2fps -> 20.0fps 24 threads with 4:2:2 HQ 4K: * Intel Cascade Lake: 224fps -> 312fps >From 1bf7844803b37a90a588c979022711c04e09d6da Mon Sep 17 00:00:00 2001 From: Mark Thompson <[email protected]> Date: Sun, 25 Oct 2026 13:32:52 +0000 Subject: [PATCH 1/5] checkasm/apv_dsp: Improve test coverage Test all bit depths, using random QP, qmatrix, and coefficients. This makes the coefficients by sending a random residual through a forward transform and quantisation in order to keep all of the arithmetic within the encoder range. One coefficient is sometimes set to a large value to exercise the dequantisation clipping, which is otherwise never hit here. Benchmarking continues to use a dense block similar to the old test to ensure that results are comparable and represent the worst case. --- tests/checkasm/apv_dsp.c | 150 ++++++++++++++++++++++++++++++++++----- 1 file changed, 131 insertions(+), 19 deletions(-) diff --git a/tests/checkasm/apv_dsp.c b/tests/checkasm/apv_dsp.c index 149c4637638..efd79fa9d4e 100644 --- a/tests/checkasm/apv_dsp.c +++ b/tests/checkasm/apv_dsp.c @@ -21,15 +21,96 @@ #include "checkasm.h" +#include "libavutil/common.h" #include "libavutil/mem_internal.h" #include "libavcodec/apv_dsp.h" -static void check_decode_transquant_10(void) +static const int8_t apv_trans_matrix[8][8] = { + { 64, 64, 64, 64, 64, 64, 64, 64 }, + { 89, 75, 50, 18, -18, -50, -75, -89 }, + { 84, 35, -35, -84, -84, -35, 35, 84 }, + { 75, -18, -89, -50, 50, 89, 18, -75 }, + { 64, -64, -64, 64, 64, -64, -64, 64 }, + { 50, -89, 18, 75, -75, -18, 89, -50 }, + { 35, -84, 84, -35, -35, 84, -84, 35 }, + { 18, -50, 75, -89, 89, -75, 50, -18 }, +}; + +static const uint8_t apv_level_scale[6] = { 40, 45, 51, 57, 64, 71 }; + +// Make a set of quantised coefficients by sending a random residual +// through a forward transform and quantisation: this keeps every +// intermediate in the ranges a real encoder would produce (completely +// random coefficients overflow the use of 16x16->32 multiplies in the +// SIMD paths). +static void generate_block(int16_t *input, const int16_t *qmatrix, + int bit_depth, int qp_shift, int dense) { - LOCAL_ALIGNED_16( int16_t, input, [64]); - LOCAL_ALIGNED_16( int16_t, qmatrix, [64]); - LOCAL_ALIGNED_16(uint16_t, new_output, [64]); - LOCAL_ALIGNED_16(uint16_t, ref_output, [64]); + // 64-bit intermediates here avoid rounding issues. + int64_t residual[8][8], tmp[8][8]; + int amplitude; + + if (dense) { + // Dense blocks for benchmarking are fully random and uniform + // over the range. + amplitude = (1 << bit_depth) - 1; + } else { + // Random blocks for testing bias the amplitude, both to get + // smaller typical coefficients (including many zeroes) and to + // make clipping on the ends of the range more likely. + amplitude = (3 << (bit_depth - 2)) >> (rnd() % 8); + } + for (int y = 0; y < 8; y++) { + for (int x = 0; x < 8; x++) + residual[y][x] = + (int64_t)(rnd() % (2 * amplitude + 1)) - amplitude; + } + + // Forward transform rows. + for (int y = 0; y < 8; y++) { + for (int i = 0; i < 8; i++) { + int64_t sum = 0; + for (int j = 0; j < 8; j++) + sum += apv_trans_matrix[i][j] * residual[y][j]; + tmp[y][i] = sum; + } + } + // Forward transform columns, renormalise, and apply quantisation + // to make the coefficient level. + for (int x = 0; x < 8; x++) { + for (int i = 0; i < 8; i++) { + int64_t coeff, step, level; + + coeff = 0; + for (int j = 0; j < 8; j++) + coeff += apv_trans_matrix[i][j] * tmp[j][x]; + coeff = ROUNDED_DIV(coeff, + INT64_C(1) << (bit_depth + 3)); + + step = qmatrix[i * 8 + x] << qp_shift; + level = ROUNDED_DIV(coeff * (INT64_C(1) << (bit_depth - 2)), + step); + + input[i * 8 + x] = av_clip_int16(level); + } + } + + // Levels made as above never reach the clip after dequantisation, + // so sometimes replace one coefficient with a huge value. A single + // coefficient like this does not overflow the intermediates. + if (!dense && rnd() % 8 == 0) { + int pos = rnd() % 64; + int level = 16384 + (rnd() % 16384); + input[pos] = rnd() % 2 ? level : -level; + } +} + +static void check_decode_transquant(int bit_depth) +{ + LOCAL_ALIGNED_32( int16_t, input, [64]); + LOCAL_ALIGNED_32( int16_t, qmatrix, [64]); + LOCAL_ALIGNED_32(uint16_t, new_output, [64]); + LOCAL_ALIGNED_32(uint16_t, ref_output, [64]); declare_func(void, void *output, @@ -39,23 +120,51 @@ static void check_decode_transquant_10(void) int bit_depth, int qp_shift); - for (int i = 0; i < 64; i++) { - // Any signed 14-bit integer. - input[i] = rnd() % 16384 - 8192; + for (int n = 0; n < 256; n++) { + // Fully random QP for each block. + int qp = rnd() % (3 + 6 * bit_depth + 1); + int level_scale = apv_level_scale[qp % 6]; + int qp_shift = qp / 6; - // qmatrix input is premultiplied by level_scale, so - // range is 1 to 255 * 71. Interesting values are all - // at the low end of that, though. - qmatrix[i] = 16; //rnd() % 16 + 16; + // The reference encoder uses a flat 16-everywhere matrix by + // default, while any other matrix must be supplied manually by + // the user. Use the default matrix half the time, and a + // completely random matrix otherwise. + int flat = rnd() % 2; + if (flat) { + for (int i = 0; i < 64; i++) + qmatrix[i] = level_scale * 16; + } else { + for (int i = 0; i < 64; i++) + qmatrix[i] = level_scale * (1 + rnd() % 255); + } + + generate_block(input, qmatrix, bit_depth, qp_shift, 0); + + call_ref(ref_output, 16, input, qmatrix, bit_depth, qp_shift); + call_new(new_output, 16, input, qmatrix, bit_depth, qp_shift); + + if (memcmp(new_output, ref_output, 64 * sizeof(*ref_output))) { + fail(); + break; + } } - call_ref(ref_output, 16, input, qmatrix, 10, 4); - call_new(new_output, 16, input, qmatrix, 10, 4); + { + // Benchmark on a dense block with QP = 12 and a flat matrix. + // Almost all coefficients will be nonzero. + int qp_shift = 2; + for (int i = 0; i < 64; i++) + qmatrix[i] = apv_level_scale[0] * 16; + generate_block(input, qmatrix, bit_depth, qp_shift, 1); - if (memcmp(new_output, ref_output, 64 * sizeof(*ref_output))) - fail(); + call_ref(ref_output, 16, input, qmatrix, bit_depth, qp_shift); + call_new(new_output, 16, input, qmatrix, bit_depth, qp_shift); + if (memcmp(new_output, ref_output, 64 * sizeof(*ref_output))) + fail(); - bench_new(new_output, 16, input, qmatrix, 10, 4); + bench_new(new_output, 16, input, qmatrix, bit_depth, qp_shift); + } } void checkasm_check_apv_dsp(void) @@ -64,8 +173,11 @@ void checkasm_check_apv_dsp(void) ff_apv_dsp_init(&dsp); - if (check_func(dsp.decode_transquant, "decode_transquant_10")) - check_decode_transquant_10(); + for (int bit_depth = 10; bit_depth <= 16; bit_depth += 2) { + if (check_func(dsp.decode_transquant, + "decode_transquant_%d", bit_depth)) + check_decode_transquant(bit_depth); + } report("decode_transquant"); } -- 2.54.0 >From f88c05dd2922c326dc8d2fd0c9761d929b81a81a Mon Sep 17 00:00:00 2001 From: Mark Thompson <[email protected]> Date: Sun, 25 Oct 2026 13:55:54 +0000 Subject: [PATCH 2/5] lavc/apv: Improve AVX2 transquant Replace the row matmul with a second butterfly step, but fuse it with parts of the transpose and rounding to reduce instructions. CPU: Intel(R) Xeon(R) Gold 6252 CPU @ 2.10GHz decode_transquant_10_c: 332.5 Before: decode_transquant_10_avx2: 87.6 ( 3.80x) After: decode_transquant_10_avx2: 65.4 ( 5.08x) --- libavcodec/x86/apv_dsp.asm | 337 +++++++++++++++++++------------------ 1 file changed, 173 insertions(+), 164 deletions(-) diff --git a/libavcodec/x86/apv_dsp.asm b/libavcodec/x86/apv_dsp.asm index 3ea28f62fda..78098b38def 100644 --- a/libavcodec/x86/apv_dsp.asm +++ b/libavcodec/x86/apv_dsp.asm @@ -22,30 +22,24 @@ SECTION_RODATA 32 -; Full matrix for row transform. -tmatrix_row: - dw 64, 89, 84, 75, 64, 50, 35, 18 - dw 64, -18, -84, 50, 64, -75, -35, 89 - dw 64, 75, 35, -18, -64, -89, -84, -50 - dw 64, -50, -35, 89, -64, -18, 84, -75 - dw 64, 50, -35, -89, -64, 18, 84, 75 - dw 64, -75, 35, 18, -64, 89, -84, 50 - dw 64, 18, -84, -50, 64, 75, -35, -89 - dw 64, -89, 84, -75, 64, -50, 35, -18 +; Constant pairs for the 1-D transform butterfly, expanded so that they +; can be used directly as memory operands. +tmatrix_even_a: times 8 dw 64, 64 +tmatrix_even_b: times 8 dw 64, -64 +tmatrix_even_c: times 8 dw 84, 35 +tmatrix_even_d: times 8 dw 35, -84 +tmatrix_odd_0a: times 8 dw 89, 50 +tmatrix_odd_0b: times 8 dw 75, 18 +tmatrix_odd_1a: times 8 dw 75, -89 +tmatrix_odd_1b: times 8 dw -18, -50 +tmatrix_odd_2a: times 8 dw 50, 18 +tmatrix_odd_2b: times 8 dw -89, 75 +tmatrix_odd_3a: times 8 dw 18, 75 +tmatrix_odd_3b: times 8 dw -50, -89 -; Constant pairs for broadcast in column transform. -tmatrix_col_even: - dw 64, 64, 64, -64 - dw 84, 35, 35, -84 -tmatrix_col_odd: - dw 89, 75, 50, 18 - dw 75, -18, -89, -50 - dw 50, -89, 18, 75 - dw 18, -50, 75, -89 - -; Memory targets for vpbroadcastd (register version requires AVX512). cextern pd_1 cextern pd_64 +vec_1_19: times 8 dd 1 << 19 SECTION .text @@ -58,11 +52,65 @@ SECTION .text INIT_YMM avx2 -cglobal apv_decode_transquant, 5, 7, 16, output, pitch, input, qmatrix, bit_depth, qp_shift, tmp +; One pass of the 1-D DCT, as a butterfly. +; 16-bit input rows in m12-m15 as pairs 04 26 15 37. +; 32-bit output rows in m8-m15. +; The rounding constant is already added to the output. + +%macro TRANSFORM_PASS 1 ; (rounding constant) + ; Even part, from rows 0, 2, 4 and 6. + vpmaddwd m8, m12, [tmatrix_even_a] + vpmaddwd m9, m12, [tmatrix_even_b] + vpmaddwd m10, m13, [tmatrix_even_c] + vpmaddwd m11, m13, [tmatrix_even_d] + vpaddd m8, m8, %1 + vpaddd m9, m9, %1 + vpaddd m0, m8, m10 + vpaddd m1, m9, m11 + vpsubd m2, m9, m11 + vpsubd m3, m8, m10 + + ; Odd part, from rows 1, 3, 5 and 7. + vpmaddwd m8, m14, [tmatrix_odd_0a] + vpmaddwd m9, m15, [tmatrix_odd_0b] + vpmaddwd m10, m14, [tmatrix_odd_1a] + vpmaddwd m11, m15, [tmatrix_odd_1b] + vpaddd m4, m8, m9 + vpaddd m5, m10, m11 + vpmaddwd m8, m14, [tmatrix_odd_2a] + vpmaddwd m9, m15, [tmatrix_odd_2b] + vpmaddwd m10, m14, [tmatrix_odd_3a] + vpmaddwd m11, m15, [tmatrix_odd_3b] + vpaddd m6, m8, m9 + vpaddd m7, m10, m11 + + ; Combine. + vpaddd m8, m0, m4 + vpaddd m9, m1, m5 + vpaddd m10, m2, m6 + vpaddd m11, m3, m7 + vpsubd m12, m3, m7 + vpsubd m13, m2, m6 + vpsubd m14, m1, m5 + vpsubd m15, m0, m4 +%endmacro + +cglobal apv_decode_transquant, 5, 7, 16, 32, output, pitch, input, qmatrix, bit_depth, qp_shift, tmp + + ; Building rounding constant to for the final normalisation and + ; store it on the stack for use later. + + mova m0, [pd_1] + mov tmpd, 19 + sub tmpd, bit_depthd + movd xm1, tmpd + vpslld m0, m0, xm1 + vpaddd m0, m0, [vec_1_19] + mova [rsp], m0 ; Load input and dequantise - vpbroadcastd m10, [pd_1] + mova m10, [pd_1] lea tmpd, [bit_depthd - 2] movd xm8, qp_shiftm movd xm9, tmpd @@ -74,7 +122,7 @@ cglobal apv_decode_transquant, 5, 7, 16, output, pitch, input, qmatrix, bit_dept ; m10 = vector 1 << (bd_shift - 1) ; m11 = qmatrix load -%macro LOAD_AND_DEQUANT 2 ; (xmm input, constant offset) +%macro LOAD_AND_DEQUANT 2 ; (register, constant offset) vpmovsxwd m%1, [inputq + %2] vpmovsxwd m11, [qmatrixq + %2] vpmaddwd m%1, m%1, m11 @@ -94,164 +142,125 @@ cglobal apv_decode_transquant, 5, 7, 16, output, pitch, input, qmatrix, bit_dept LOAD_AND_DEQUANT 7, 0x70 ; mN = row N words 0 1 2 3 0 1 2 3 4 5 6 7 4 5 6 7 + ; Interleave pairs of rows for the butterfly input. - ; Transform columns - ; This applies a 1-D DCT butterfly + vpunpcklwd m12, m0, m4 + vpunpcklwd m13, m2, m6 + vpunpcklwd m14, m1, m5 + vpunpcklwd m15, m3, m7 - vpunpcklwd m12, m0, m4 - vpunpcklwd m13, m2, m6 - vpunpcklwd m14, m1, m3 - vpunpcklwd m15, m5, m7 + ; Transform columns. - ; m12 = rows 0 and 4 interleaved - ; m13 = rows 2 and 6 interleaved - ; m14 = rows 1 and 3 interleaved - ; m15 = rows 5 and 7 interleaved + TRANSFORM_PASS [pd_64] - lea tmpq, [tmatrix_col_even] - vpbroadcastd m0, [tmpq + 0x00] - vpbroadcastd m1, [tmpq + 0x04] - vpbroadcastd m2, [tmpq + 0x08] - vpbroadcastd m3, [tmpq + 0x0c] + ; Mid-transform normalisation. The rounding constant was already + ; added by the transform pass; outputs here are fitted to 16 bits. - vpmaddwd m4, m12, m0 - vpmaddwd m5, m12, m1 - vpmaddwd m6, m13, m2 - vpmaddwd m7, m13, m3 - vpaddd m8, m4, m6 - vpaddd m9, m5, m7 - vpsubd m10, m5, m7 - vpsubd m11, m4, m6 + vpsrad m8, m8, 7 + vpsrad m9, m9, 7 + vpsrad m10, m10, 7 + vpsrad m11, m11, 7 + vpsrad m12, m12, 7 + vpsrad m13, m13, 7 + vpsrad m14, m14, 7 + vpsrad m15, m15, 7 - lea tmpq, [tmatrix_col_odd] - vpbroadcastd m0, [tmpq + 0x00] - vpbroadcastd m1, [tmpq + 0x04] - vpbroadcastd m2, [tmpq + 0x08] - vpbroadcastd m3, [tmpq + 0x0c] + ; Transpose, recombining into the paired rows for the butterfly. - vpmaddwd m4, m14, m0 - vpmaddwd m5, m15, m1 - vpmaddwd m6, m14, m2 - vpmaddwd m7, m15, m3 - vpaddd m12, m4, m5 - vpaddd m13, m6, m7 + vpackssdw m0, m8, m9 + vpackssdw m1, m10, m11 + vpackssdw m2, m12, m13 + vpackssdw m3, m14, m15 - vpbroadcastd m0, [tmpq + 0x10] - vpbroadcastd m1, [tmpq + 0x14] - vpbroadcastd m2, [tmpq + 0x18] - vpbroadcastd m3, [tmpq + 0x1c] + vperm2i128 m4, m0, m2, 0x20 + vperm2i128 m5, m1, m3, 0x20 + vperm2i128 m6, m0, m2, 0x31 + vperm2i128 m7, m1, m3, 0x31 - vpmaddwd m4, m14, m0 - vpmaddwd m5, m15, m1 - vpmaddwd m6, m14, m2 - vpmaddwd m7, m15, m3 - vpaddd m14, m4, m5 - vpaddd m15, m6, m7 + vpslld m8, m6, 16 + vpslld m9, m7, 16 + vpsrld m10, m4, 16 + vpsrld m11, m5, 16 + vpblendw m8, m4, m8, 0xaa + vpblendw m9, m5, m9, 0xaa + vpblendw m10, m10, m6, 0xaa + vpblendw m11, m11, m7, 0xaa - vpaddd m0, m8, m12 - vpaddd m1, m9, m13 - vpaddd m2, m10, m14 - vpaddd m3, m11, m15 - vpsubd m4, m11, m15 - vpsubd m5, m10, m14 - vpsubd m6, m9, m13 - vpsubd m7, m8, m12 + vshufps m12, m8, m9, q2020 + vshufps m13, m8, m9, q3131 + vshufps m14, m10, m11, q2020 + vshufps m15, m10, m11, q3131 - ; Mid-transform normalisation - ; Note that outputs here are fitted to 16 bits + ; Transform rows. - vpbroadcastd m8, [pd_64] + TRANSFORM_PASS [rsp] -%macro NORMALISE 1 - vpaddd m%1, m%1, m8 - vpsrad m%1, m%1, 7 - vpackssdw m%1, m%1, m%1 - vpermq m%1, m%1, q3120 -%endmacro + ; Renormalise, clip and store output. The rounding constant was + ; already added by the transform pass, and here we split the shift + ; on either side of the pack (20 - bit_depth -> 4 and 16 - bit_depth) + ; so that the saturation precisely captures the clip we want. - NORMALISE 0 - NORMALISE 1 - NORMALISE 2 - NORMALISE 3 - NORMALISE 4 - NORMALISE 5 - NORMALISE 6 - NORMALISE 7 - - ; mN = row N words 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 - - ; Transform rows - ; This multiplies the rows directly by the transform matrix, - ; avoiding the need to transpose anything - - lea tmpq, [tmatrix_row] - mova m12, [tmpq + 0x00] - mova m13, [tmpq + 0x20] - mova m14, [tmpq + 0x40] - mova m15, [tmpq + 0x60] - -%macro TRANS_ROW_STEP 1 - vpmaddwd m8, m%1, m12 - vpmaddwd m9, m%1, m13 - vpmaddwd m10, m%1, m14 - vpmaddwd m11, m%1, m15 - vphaddd m8, m8, m9 - vphaddd m10, m10, m11 - vphaddd m%1, m8, m10 -%endmacro - - TRANS_ROW_STEP 0 - TRANS_ROW_STEP 1 - TRANS_ROW_STEP 2 - TRANS_ROW_STEP 3 - TRANS_ROW_STEP 4 - TRANS_ROW_STEP 5 - TRANS_ROW_STEP 6 - TRANS_ROW_STEP 7 - - ; Renormalise, clip and store output - - vpbroadcastd m14, [pd_1] - mov tmpd, 20 + mov tmpd, 16 sub tmpd, bit_depthd - movd xm9, tmpd - dec tmpd - movd xm13, tmpd - movd xm15, bit_depthd - vpslld m8, m14, xm13 - vpslld m12, m14, xm15 - vpsrld m10, m12, 1 - vpsubd m12, m12, m14 + movd xm0, tmpd - ; m8 = vector 1 << (bd_shift - 1) - ; m9 = scalar bd_shift - ; m10 = vector 1 << (bit_depth - 1) - ; m12 = vector (1 << bit_depth) - 1 + vpsrad m8, m8, 4 + vpsrad m9, m9, 4 + vpsrad m10, m10, 4 + vpsrad m11, m11, 4 + vpsrad m12, m12, 4 + vpsrad m13, m13, 4 + vpsrad m14, m14, 4 + vpsrad m15, m15, 4 -%macro NORMALISE_AND_STORE_10 2 - vpaddd m%1, m%1, m8 - vpaddd m%2, m%2, m8 - vpsrad m%1, m%1, xm9 - vpsrad m%2, m%2, xm9 - vpaddd m%1, m%1, m10 - vpaddd m%2, m%2, m10 - vpminsd m%1, m%1, m12 - vpminsd m%2, m%2, m12 - ; m%1 = A0-3 A4-7 - ; m%2 = B0-3 B4-7 - vpackusdw m%1, m%1, m%2 - ; m%1 = A0-3 B0-3 A4-7 B4-7 - vpermq m%1, m%1, q3120 - ; m%1 = A0-3 A4-7 B0-3 B4-7 - mova [outputq], xm%1 - vextracti128 [outputq + pitchq], m%1, 1 - lea outputq, [outputq + 2*pitchq] -%endmacro + vpackusdw m3, m8, m12 + vpackusdw m4, m9, m13 + vpackusdw m5, m10, m14 + vpackusdw m6, m11, m15 - NORMALISE_AND_STORE_10 0, 1 - NORMALISE_AND_STORE_10 2, 3 - NORMALISE_AND_STORE_10 4, 5 - NORMALISE_AND_STORE_10 6, 7 + vpsrlw m3, m3, xm0 + vpsrlw m4, m4, xm0 + vpsrlw m5, m5, xm0 + vpsrlw m6, m6, xm0 + + ; Transpose back for output. + + vpunpcklwd m7, m3, m4 + vpunpckhwd m8, m3, m4 + vpunpcklwd m9, m5, m6 + vpunpckhwd m10, m5, m6 + + vpunpckldq m11, m7, m9 + vpunpckhdq m12, m7, m9 + vpunpckldq m13, m8, m10 + vpunpckhdq m14, m8, m10 + + vpunpcklqdq m0, m11, m13 + vpunpckhqdq m1, m11, m13 + vpunpcklqdq m2, m12, m14 + vpunpckhqdq m3, m12, m14 + + ; mN = rows N and N+4 + + ; Indexing from outputq reaches rows 0, 1, 2 and 4; indexing from a + ; second base at row 7 with the pitch negated reaches rows 7, 6, 5 + ; and 3. The negation means every store relative to outputq has to + ; come first. + + lea tmpq, [outputq + 8*pitchq] + sub tmpq, pitchq + + mova [outputq], xm0 ; row 0 + mova [outputq + pitchq], xm1 ; row 1 + mova [outputq + 2*pitchq], xm2 ; row 2 + vextracti128 [outputq + 4*pitchq], m0, 1 ; row 4 + + neg pitchq + + mova [tmpq + 4*pitchq], xm3 ; row 3 + vextracti128 [tmpq + 2*pitchq], m1, 1 ; row 5 + vextracti128 [tmpq + pitchq], m2, 1 ; row 6 + vextracti128 [tmpq], m3, 1 ; row 7 RET -- 2.54.0 >From 9b649a7e1c470e3c7e384c0a63372bfce13542cf Mon Sep 17 00:00:00 2001 From: Mark Thompson <[email protected]> Date: Sun, 25 Oct 2026 14:21:24 +0000 Subject: [PATCH 3/5] lavc/apv: Precompute transquant constants once per tile component Adds a new dsp function to compute a constants buffer in the form most suitable to the specific implementation. This gets called once per tile component to prepare the constants buffer, and then the transquant functions get a pointer to that buffer to use. CPU: Intel(R) Xeon(R) Gold 6252 CPU @ 2.10GHz Before: decode_transquant_10_c: 333.2 decode_transquant_10_avx2: 65.5 ( 5.09x) After: decode_transquant_10_c: 330.9 decode_transquant_10_avx2: 57.3 ( 5.78x) No significant difference on ARM Cortex-A53/A73. --- libavcodec/aarch64/apv_dsp_constants.h | 31 +++++++++++++ libavcodec/aarch64/apv_dsp_init_aarch64.c | 51 ++++++++++++++++++++-- libavcodec/aarch64/apv_dsp_neon.S | 27 ++++-------- libavcodec/apv_decode.c | 28 +++++------- libavcodec/apv_dsp.c | 39 ++++++++++++++--- libavcodec/apv_dsp.h | 31 +++++++++++-- libavcodec/x86/apv_dsp.asm | 53 +++++++++-------------- libavcodec/x86/apv_dsp_init.c | 45 +++++++++++++++++-- tests/checkasm/apv_dsp.c | 26 ++++++----- 9 files changed, 238 insertions(+), 93 deletions(-) create mode 100644 libavcodec/aarch64/apv_dsp_constants.h diff --git a/libavcodec/aarch64/apv_dsp_constants.h b/libavcodec/aarch64/apv_dsp_constants.h new file mode 100644 index 00000000000..3772be438f9 --- /dev/null +++ b/libavcodec/aarch64/apv_dsp_constants.h @@ -0,0 +1,31 @@ +/* + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef AVCODEC_AARCH64_APV_DSP_CONSTANTS_H +#define AVCODEC_AARCH64_APV_DSP_CONSTANTS_H + +/* Offsets in the constants buffer used by transquant_neon. */ +#define APV_CONST_QMATRIX 0 +#define APV_CONST_QP_SHIFT 128 +#define APV_CONST_DEQUANT_ROUND 132 +#define APV_CONST_NEG_BD_SHIFT 136 +#define APV_CONST_OUTPUT_ROUND 140 +#define APV_CONST_OUTPUT_MAX 144 +#define APV_CONST_OUTPUT_SHIFT 148 + +#endif /* AVCODEC_AARCH64_APV_DSP_CONSTANTS_H */ diff --git a/libavcodec/aarch64/apv_dsp_init_aarch64.c b/libavcodec/aarch64/apv_dsp_init_aarch64.c index 22d4598f91f..ced02901962 100644 --- a/libavcodec/aarch64/apv_dsp_init_aarch64.c +++ b/libavcodec/aarch64/apv_dsp_init_aarch64.c @@ -16,23 +16,66 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ +#include <assert.h> +#include <stddef.h> +#include <string.h> + #include "config.h" #include "libavutil/attributes.h" #include "libavutil/cpu.h" #include "libavutil/aarch64/cpu.h" #include "libavcodec/apv_dsp.h" +#include "apv_dsp_constants.h" + +typedef struct APVConstantsNEON { + int16_t qmatrix[64]; + int32_t qp_shift; + int32_t dequant_round; + int32_t neg_bd_shift; + int32_t output_round; + int32_t output_max; + int32_t output_shift; +} APVConstantsNEON; + +static_assert(sizeof(APVConstantsNEON) <= APV_DSP_CONSTANTS_SIZE, + "APV NEON constants buffer too small"); +static_assert(offsetof(APVConstantsNEON, qmatrix) == APV_CONST_QMATRIX && + offsetof(APVConstantsNEON, qp_shift) == APV_CONST_QP_SHIFT && + offsetof(APVConstantsNEON, dequant_round) == APV_CONST_DEQUANT_ROUND && + offsetof(APVConstantsNEON, neg_bd_shift) == APV_CONST_NEG_BD_SHIFT && + offsetof(APVConstantsNEON, output_round) == APV_CONST_OUTPUT_ROUND && + offsetof(APVConstantsNEON, output_max) == APV_CONST_OUTPUT_MAX && + offsetof(APVConstantsNEON, output_shift) == APV_CONST_OUTPUT_SHIFT, + "APV NEON constants layout does not match assembly"); + +static void apv_prepare_constants_neon(void *constants, + const int16_t *qmatrix, + int bit_depth, + int qp_shift) +{ + APVConstantsNEON *c = constants; + int bd_shift = bit_depth - 2; + + memcpy(c->qmatrix, qmatrix, sizeof(c->qmatrix)); + c->qp_shift = qp_shift; + c->dequant_round = 1 << (bd_shift - 1); + c->neg_bd_shift = -bd_shift; + c->output_round = (1 << (19 - bit_depth)) + (1 << 19); + c->output_max = (1 << bit_depth) - 1; + c->output_shift = bit_depth - 20; +} void ff_apv_decode_transquant_neon(void *output, ptrdiff_t pitch, const int16_t *input, - const int16_t *qmatrix, - int bit_depth, - int qp_shift); + const void *constants); av_cold void ff_apv_dsp_init_aarch64(APVDSPContext *dsp) { int cpu_flags = av_get_cpu_flags(); - if (have_neon(cpu_flags)) + if (have_neon(cpu_flags)) { + dsp->prepare_constants = apv_prepare_constants_neon; dsp->decode_transquant = ff_apv_decode_transquant_neon; + } } diff --git a/libavcodec/aarch64/apv_dsp_neon.S b/libavcodec/aarch64/apv_dsp_neon.S index 3110a4f6fb9..13037ce61af 100644 --- a/libavcodec/aarch64/apv_dsp_neon.S +++ b/libavcodec/aarch64/apv_dsp_neon.S @@ -18,6 +18,7 @@ #include "libavutil/aarch64/asm.S" #include "neon.S" +#include "apv_dsp_constants.h" /* Dequant one row in 16x8 * Input levels at x2 + off, qmatrix at x3 + off (function args) @@ -278,21 +279,15 @@ .endm /* - * void ff_apv_decode_transquant_neon(void *output, = x0 - * ptrdiff_t pitch, = x1 - * const int16_t *input, = x2 - * const int16_t *qmatrix, = x3 - * int bit_depth, = w4 - * int qp_shift) = w5 + * void ff_apv_decode_transquant_neon(void *output, = x0 + * ptrdiff_t pitch, = x1 + * const int16_t *input, = x2 + * const void *constants) = x3 */ function ff_apv_decode_transquant_neon, export=1 - sub w7, w4, #2 /* w7 = bd_shift */ - movz w14, #1 - /* Dequant */ - sub w15, w7, #1 - lsl w10, w14, w15 - neg w15, w7 + ldp w5, w10, [x3, #APV_CONST_QP_SHIFT] + ldr w15, [x3, #APV_CONST_NEG_BD_SHIFT] dup v24.4s, w5 /* v24 = qp_shift */ dup v25.4s, w10 /* v25 = 1 << (bd_shift - 1) */ dup v26.4s, w15 /* v26 = -bd_shift */ @@ -306,11 +301,7 @@ function ff_apv_decode_transquant_neon, export=1 dequant v7, #112 /* Row scaling constants */ - movz w16, #8, lsl #16 /* w16 = 1 << 19 */ - lsr w10, w16, w4 - orr w10, w10, w16 /* w10 = (1 << (19 - bit_depth)) + (1 << 19) */ - lsl w15, w14, w4 - sub w13, w15, #1 /* w13 = (1 << bit_depth) - 1 */ + ldp w10, w13, [x3, #APV_CONST_OUTPUT_ROUND] /* Column butterfly and normalise */ butterfly_mul @@ -323,7 +314,7 @@ function ff_apv_decode_transquant_neon, export=1 butterfly_mul dup v5.4s, w10 /* v5 = output offset */ dup v6.4s, w13 /* v6 = output max clip (min clip is zero) */ - sub w17, w4, #20 /* w17 = bit_depth - 20 (row_combine shift) */ + ldr w17, [x3, #APV_CONST_OUTPUT_SHIFT] row_combine /* Transpose back */ diff --git a/libavcodec/apv_decode.c b/libavcodec/apv_decode.c index a6ca9ba8fb2..7fd93ad0c22 100644 --- a/libavcodec/apv_decode.c +++ b/libavcodec/apv_decode.c @@ -188,9 +188,7 @@ static int apv_decode_block(AVCodecContext *avctx, ptrdiff_t pitch, GetBitContext *gbc, APVEntropyState *entropy_state, - int bit_depth, - int qp_shift, - const uint16_t *qmatrix) + const void *constants) { APVDecodeContext *apv = avctx->priv_data; int err; @@ -202,9 +200,7 @@ static int apv_decode_block(AVCodecContext *avctx, if (err < 0) return err; - apv->dsp.decode_transquant(output, pitch, - coeff, qmatrix, - bit_depth, qp_shift); + apv->dsp.decode_transquant(output, pitch, coeff, constants); return 0; } @@ -242,9 +238,7 @@ static int apv_decode_tile_component(AVCodecContext *avctx, void *data, int blk_mb_width = 2 >> sub_w_shift; int blk_mb_height = 2 >> sub_h_shift; - int bit_depth; - int qp_shift; - LOCAL_ALIGNED_32(uint16_t, qmatrix_scaled, [64]); + LOCAL_ALIGNED_32(uint8_t, constants, [APV_DSP_CONSTANTS_SIZE]); GetBitContext gbc; @@ -265,20 +259,24 @@ static int apv_decode_tile_component(AVCodecContext *avctx, void *data, // Combine the bitstream quantisation matrix with the qp scaling // in advance. (Including qp_shift as well would overflow 16 bits.) - // Fix the row ordering at the same time. + // Fix the row ordering at the same time. Everything else which + // is constant for the tile component is derived from this by the + // DSP code. { static const uint8_t apv_level_scale[6] = { 40, 45, 51, 57, 64, 71 }; int qp = tile->tile_header.tile_qp[comp_index]; int level_scale = apv_level_scale[qp % 6]; - - bit_depth = input->frame_header.frame_info.bit_depth_minus8 + 8; - qp_shift = qp / 6; + int bit_depth = input->frame_header.frame_info.bit_depth_minus8 + 8; + int16_t qmatrix_scaled[64]; for (int y = 0; y < 8; y++) { for (int x = 0; x < 8; x++) qmatrix_scaled[y * 8 + x] = level_scale * input->frame_header.quantization_matrix.q_matrix[comp_index][x][y]; } + + apv->dsp.prepare_constants(constants, qmatrix_scaled, + bit_depth, qp / 6); } for (int mb_y = 0; mb_y < tile_mb_height; mb_y++) { @@ -299,9 +297,7 @@ static int apv_decode_tile_component(AVCodecContext *avctx, void *data, err = apv_decode_block(avctx, block_start, frame_pitch, &gbc, &entropy_state, - bit_depth, - qp_shift, - qmatrix_scaled); + constants); if (err < 0) { // Error in block decode means entropy desync, // so this is not recoverable. diff --git a/libavcodec/apv_dsp.c b/libavcodec/apv_dsp.c index 7351e34b6ae..eabab865c0f 100644 --- a/libavcodec/apv_dsp.c +++ b/libavcodec/apv_dsp.c @@ -16,7 +16,9 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ +#include <assert.h> #include <stdint.h> +#include <string.h> #include "config.h" #include "libavutil/attributes.h" @@ -38,15 +40,35 @@ static const int8_t apv_trans_matrix[8][8] = { { 18, -50, 75, -89, 89, -75, 50, -18 }, }; +typedef struct APVConstantsC { + int16_t qmatrix[8][8]; + int bit_depth; + int qp_shift; +} APVConstantsC; + +static_assert(sizeof(APVConstantsC) <= APV_DSP_CONSTANTS_SIZE, + "APV DSP constants buffer too small"); + +void ff_apv_prepare_constants_c(void *constants, + const int16_t *qmatrix, + int bit_depth, + int qp_shift) +{ + APVConstantsC *c = constants; + + memcpy(c->qmatrix, qmatrix, sizeof(c->qmatrix)); + c->bit_depth = bit_depth; + c->qp_shift = qp_shift; +} + static void apv_decode_transquant_c(void *output, ptrdiff_t pitch, const int16_t *input_flat, - const int16_t *qmatrix_flat, - int bit_depth, - int qp_shift) + const void *constants) { - const int16_t (*input)[8] = (const int16_t(*)[8])input_flat; - const int16_t (*qmatrix)[8] = (const int16_t(*)[8])qmatrix_flat; + const APVConstantsC *c = constants; + const int16_t (*input)[8] = (const int16_t(*)[8])input_flat; + int bit_depth = c->bit_depth; int16_t scaled_coeff[8][8]; int32_t recon_sample[8][8]; @@ -59,8 +81,10 @@ static void apv_decode_transquant_c(void *output, for (int y = 0; y < 8; y++) { for (int x = 0; x < 8; x++) { - int coeff = ((int)(input[y][x] * qmatrix[y][x] * (1U << qp_shift) + - (1 << (bd_shift - 1)))) >> bd_shift; + int coeff = ((int)(input[y][x] * + c->qmatrix[y][x] * + (1U << c->qp_shift) + + (1 << (bd_shift - 1)))) >> bd_shift; scaled_coeff[y][x] = av_clip(coeff, APV_MIN_TRANS_COEFF, @@ -119,6 +143,7 @@ static void apv_decode_transquant_c(void *output, av_cold void ff_apv_dsp_init(APVDSPContext *dsp) { + dsp->prepare_constants = ff_apv_prepare_constants_c; dsp->decode_transquant = apv_decode_transquant_c; #if ARCH_X86_64 && HAVE_X86ASM diff --git a/libavcodec/apv_dsp.h b/libavcodec/apv_dsp.h index c0836f3e2c7..15ee195d0bc 100644 --- a/libavcodec/apv_dsp.h +++ b/libavcodec/apv_dsp.h @@ -23,17 +23,42 @@ #include <stdint.h> +/** + * Size of the buffer holding constants for decode_transquant(), which + * must be large enough for the layout used by any implementation. + * The buffer must be 32-byte aligned. + */ +#define APV_DSP_CONSTANTS_SIZE 512 + typedef struct APVDSPContext { - void (*decode_transquant)(void *output, - ptrdiff_t pitch, - const int16_t *input, + /** + * Fill the constants buffer used by decode_transquant() for all + * blocks with the same quantisation parameters. The layout of the + * buffer is private to the implementation. + */ + void (*prepare_constants)(void *constants, const int16_t *qmatrix, int bit_depth, int qp_shift); + + /** + * Run transquant for a single block. The constants argument must + * have been prepared by a call to prepare_constants() matching the + * implementation being used. + */ + void (*decode_transquant)(void *output, + ptrdiff_t pitch, + const int16_t *input, + const void *constants); } APVDSPContext; void ff_apv_dsp_init(APVDSPContext *dsp); +void ff_apv_prepare_constants_c(void *constants, + const int16_t *qmatrix, + int bit_depth, + int qp_shift); + void ff_apv_dsp_init_x86_64(APVDSPContext *dsp); void ff_apv_dsp_init_aarch64(APVDSPContext *dsp); diff --git a/libavcodec/x86/apv_dsp.asm b/libavcodec/x86/apv_dsp.asm index 78098b38def..7eb5ba17d84 100644 --- a/libavcodec/x86/apv_dsp.asm +++ b/libavcodec/x86/apv_dsp.asm @@ -37,18 +37,23 @@ tmatrix_odd_2b: times 8 dw -89, 75 tmatrix_odd_3a: times 8 dw 18, 75 tmatrix_odd_3b: times 8 dw -50, -89 -cextern pd_1 cextern pd_64 -vec_1_19: times 8 dd 1 << 19 + +; Offsets in the constants buffer. +; This must match APVConstantsAVX2 in apv_dsp_init.c. +%define CONST_QMATRIX 0 +%define CONST_DEQUANT_ROUND 256 +%define CONST_OUTPUT_ROUND 288 +%define CONST_QP_SHIFT 320 +%define CONST_BD_SHIFT 324 +%define CONST_OUTPUT_SHIFT 328 SECTION .text ; void ff_apv_decode_transquant_avx2(void *output, ; ptrdiff_t pitch, ; const int16_t *input, -; const int16_t *qmatrix, -; int bit_depth, -; int qp_shift); +; const void *constants); INIT_YMM avx2 @@ -95,37 +100,23 @@ INIT_YMM avx2 vpsubd m15, m0, m4 %endmacro -cglobal apv_decode_transquant, 5, 7, 16, 32, output, pitch, input, qmatrix, bit_depth, qp_shift, tmp +cglobal apv_decode_transquant, 4, 5, 16, output, pitch, input, constants, tmp - ; Building rounding constant to for the final normalisation and - ; store it on the stack for use later. + ; Load input and dequantise. The qmatrix values are zero-extended + ; in the constants, so vpmaddwd with the sign-extended input gives + ; the product directly. - mova m0, [pd_1] - mov tmpd, 19 - sub tmpd, bit_depthd - movd xm1, tmpd - vpslld m0, m0, xm1 - vpaddd m0, m0, [vec_1_19] - mova [rsp], m0 - - ; Load input and dequantise - - mova m10, [pd_1] - lea tmpd, [bit_depthd - 2] - movd xm8, qp_shiftm - movd xm9, tmpd - vpslld m10, m10, xm9 - vpsrld m10, m10, 1 + mova m10, [constantsq + CONST_DEQUANT_ROUND] + movd xm8, [constantsq + CONST_QP_SHIFT] + movd xm9, [constantsq + CONST_BD_SHIFT] ; m8 = scalar qp_shift ; m9 = scalar bd_shift ; m10 = vector 1 << (bd_shift - 1) - ; m11 = qmatrix load %macro LOAD_AND_DEQUANT 2 ; (register, constant offset) - vpmovsxwd m%1, [inputq + %2] - vpmovsxwd m11, [qmatrixq + %2] - vpmaddwd m%1, m%1, m11 + vpmovsxwd m%1, [inputq + %2] + vpmaddwd m%1, m%1, [constantsq + CONST_QMATRIX + 2 * %2] vpslld m%1, m%1, xm8 vpaddd m%1, m%1, m10 vpsrad m%1, m%1, xm9 @@ -193,16 +184,14 @@ cglobal apv_decode_transquant, 5, 7, 16, 32, output, pitch, input, qmatrix, bit_ ; Transform rows. - TRANSFORM_PASS [rsp] + TRANSFORM_PASS [constantsq + CONST_OUTPUT_ROUND] ; Renormalise, clip and store output. The rounding constant was ; already added by the transform pass, and here we split the shift ; on either side of the pack (20 - bit_depth -> 4 and 16 - bit_depth) ; so that the saturation precisely captures the clip we want. - mov tmpd, 16 - sub tmpd, bit_depthd - movd xm0, tmpd + movd xm0, [constantsq + CONST_OUTPUT_SHIFT] vpsrad m8, m8, 4 vpsrad m9, m9, 4 diff --git a/libavcodec/x86/apv_dsp_init.c b/libavcodec/x86/apv_dsp_init.c index f87292a5281..3d0f08f7747 100644 --- a/libavcodec/x86/apv_dsp_init.c +++ b/libavcodec/x86/apv_dsp_init.c @@ -16,6 +16,9 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ +#include <assert.h> +#include <stddef.h> + #include "config.h" #include "libavutil/attributes.h" #include "libavutil/cpu.h" @@ -24,18 +27,54 @@ #if ARCH_X86_64 +typedef struct APVConstantsAVX2 { + int32_t qmatrix[64]; + int32_t dequant_round[8]; + int32_t output_round[8]; + int32_t qp_shift; + int32_t bd_shift; + int32_t output_shift; +} APVConstantsAVX2; + +static_assert(sizeof(APVConstantsAVX2) <= APV_DSP_CONSTANTS_SIZE, + "APV AVX2 constants buffer too small"); +static_assert(offsetof(APVConstantsAVX2, dequant_round) == 256 && + offsetof(APVConstantsAVX2, output_round) == 288 && + offsetof(APVConstantsAVX2, qp_shift) == 320 && + offsetof(APVConstantsAVX2, bd_shift) == 324 && + offsetof(APVConstantsAVX2, output_shift) == 328, + "APV AVX2 constants layout does not match assembly"); + +static void apv_prepare_constants_avx2(void *constants, + const int16_t *qmatrix, + int bit_depth, + int qp_shift) +{ + APVConstantsAVX2 *c = constants; + int bd_shift = bit_depth - 2; + + for (int i = 0; i < 64; i++) + c->qmatrix[i] = (uint16_t)qmatrix[i]; + for (int i = 0; i < 8; i++) { + c->dequant_round[i] = 1 << (bd_shift - 1); + c->output_round[i] = (1 << (19 - bit_depth)) + (1 << 19); + } + c->qp_shift = qp_shift; + c->bd_shift = bd_shift; + c->output_shift = 16 - bit_depth; +} + void ff_apv_decode_transquant_avx2(void *output, ptrdiff_t pitch, const int16_t *input, - const int16_t *qmatrix, - int bit_depth, - int qp_shift); + const void *constants); av_cold void ff_apv_dsp_init_x86_64(APVDSPContext *dsp) { int cpu_flags = av_get_cpu_flags(); if (EXTERNAL_AVX2_FAST(cpu_flags)) { + dsp->prepare_constants = apv_prepare_constants_avx2; dsp->decode_transquant = ff_apv_decode_transquant_avx2; } } diff --git a/tests/checkasm/apv_dsp.c b/tests/checkasm/apv_dsp.c index efd79fa9d4e..265ffaec99d 100644 --- a/tests/checkasm/apv_dsp.c +++ b/tests/checkasm/apv_dsp.c @@ -105,20 +105,20 @@ static void generate_block(int16_t *input, const int16_t *qmatrix, } } -static void check_decode_transquant(int bit_depth) +static void check_decode_transquant(const APVDSPContext *dsp, int bit_depth) { LOCAL_ALIGNED_32( int16_t, input, [64]); LOCAL_ALIGNED_32( int16_t, qmatrix, [64]); LOCAL_ALIGNED_32(uint16_t, new_output, [64]); LOCAL_ALIGNED_32(uint16_t, ref_output, [64]); + LOCAL_ALIGNED_32( uint8_t, new_constants, [APV_DSP_CONSTANTS_SIZE]); + LOCAL_ALIGNED_32( uint8_t, ref_constants, [APV_DSP_CONSTANTS_SIZE]); declare_func(void, void *output, ptrdiff_t pitch, const int16_t *input, - const int16_t *qmatrix, - int bit_depth, - int qp_shift); + const void *constants); for (int n = 0; n < 256; n++) { // Fully random QP for each block. @@ -141,8 +141,11 @@ static void check_decode_transquant(int bit_depth) generate_block(input, qmatrix, bit_depth, qp_shift, 0); - call_ref(ref_output, 16, input, qmatrix, bit_depth, qp_shift); - call_new(new_output, 16, input, qmatrix, bit_depth, qp_shift); + ff_apv_prepare_constants_c(ref_constants, qmatrix, bit_depth, qp_shift); + dsp->prepare_constants (new_constants, qmatrix, bit_depth, qp_shift); + + call_ref(ref_output, 16, input, ref_constants); + call_new(new_output, 16, input, new_constants); if (memcmp(new_output, ref_output, 64 * sizeof(*ref_output))) { fail(); @@ -158,12 +161,15 @@ static void check_decode_transquant(int bit_depth) qmatrix[i] = apv_level_scale[0] * 16; generate_block(input, qmatrix, bit_depth, qp_shift, 1); - call_ref(ref_output, 16, input, qmatrix, bit_depth, qp_shift); - call_new(new_output, 16, input, qmatrix, bit_depth, qp_shift); + ff_apv_prepare_constants_c(ref_constants, qmatrix, bit_depth, qp_shift); + dsp->prepare_constants (new_constants, qmatrix, bit_depth, qp_shift); + + call_ref(ref_output, 16, input, ref_constants); + call_new(new_output, 16, input, new_constants); if (memcmp(new_output, ref_output, 64 * sizeof(*ref_output))) fail(); - bench_new(new_output, 16, input, qmatrix, bit_depth, qp_shift); + bench_new(new_output, 16, input, new_constants); } } @@ -176,7 +182,7 @@ void checkasm_check_apv_dsp(void) for (int bit_depth = 10; bit_depth <= 16; bit_depth += 2) { if (check_func(dsp.decode_transquant, "decode_transquant_%d", bit_depth)) - check_decode_transquant(bit_depth); + check_decode_transquant(&dsp, bit_depth); } report("decode_transquant"); -- 2.54.0 >From ed89c8c9a2338714079444852a543cee9cad3899 Mon Sep 17 00:00:00 2001 From: Mark Thompson <[email protected]> Date: Sun, 25 Oct 2026 11:25:19 +0000 Subject: [PATCH 4/5] lavc/apv: Improve multi-symbol entropy decode This reduces the number of branches in the fast paths, improving throughput of the entropy decoder. --- libavcodec/apv_decode.c | 5 +- libavcodec/apv_decode.h | 63 +++- libavcodec/apv_entropy.c | 658 ++++++++++++++++----------------------- libavcodec/tests/apv.c | 2 +- 4 files changed, 327 insertions(+), 401 deletions(-) diff --git a/libavcodec/apv_decode.c b/libavcodec/apv_decode.c index 7fd93ad0c22..a5167cc4e73 100644 --- a/libavcodec/apv_decode.c +++ b/libavcodec/apv_decode.c @@ -193,7 +193,10 @@ static int apv_decode_block(AVCodecContext *avctx, APVDecodeContext *apv = avctx->priv_data; int err; - LOCAL_ALIGNED_32(int16_t, coeff, [64]); + // Aligned for SIMD use in transquant. + LOCAL_ALIGNED_32(int16_t, coeff, [APV_ENTROPY_COEFF_SIZE]); + // Zero only the used entries, padding entries may be written but + // are never read. memset(coeff, 0, 64 * sizeof(int16_t)); err = ff_apv_entropy_decode_block(coeff, gbc, entropy_state); diff --git a/libavcodec/apv_decode.h b/libavcodec/apv_decode.h index dd65f1b948c..6a6dc853ff3 100644 --- a/libavcodec/apv_decode.h +++ b/libavcodec/apv_decode.h @@ -19,6 +19,7 @@ #ifndef AVCODEC_APV_DECODE_H #define AVCODEC_APV_DECODE_H +#include <assert.h> #include <stdint.h> #include <stdatomic.h> @@ -33,10 +34,15 @@ // Number of bits in the entropy look-up tables. // It may be desirable to tune this per-architecture, as a larger LUT // trades greater memory use for fewer instructions. -// (N bits -> 24*2^N bytes of tables; 9 -> 12KB of tables.) +// (N bits -> 384*2^N bytes of tables; 9 -> 192KB of tables.) #define APV_VLC_LUT_BITS 9 #define APV_VLC_LUT_SIZE (1 << APV_VLC_LUT_BITS) +// The single-symbol LUT must be able to resolve a symbol with a single +// leading zero for the largest k (5) without reading more bits, which +// takes 2 + 1 + 5 = 8 bits. +static_assert(APV_VLC_LUT_BITS >= 8, "APV VLC LUT too small"); + typedef struct APVSingleVLCLUTEntry { uint16_t result; // Return value if not reading more. uint8_t consume; // Number of bits to consume. @@ -52,26 +58,57 @@ typedef struct APVMultiVLCLUTEntry { uint8_t k_level_0 : 3; // k_level after decoding all symbols. uint8_t k_level_1 : 3; - // Run output values. - uint8_t run[2]; - // Level output values. - int16_t level[2]; - // Bit index of the end of each code. + // Level output values. If fewer than two levels are decoded the + // remaining ones are zero. + int8_t level[2]; + // Scan position of each level relative to the scan position before + // this entry. Missing levels are placed at the end of the entry, + // where writing the level zero is harmless. + uint8_t level_pos[2]; + // Total scan position advance if all symbols are consumed. If not + // all symbols are consumed this can be recalculated from level_pos. + uint8_t scan_advance; + // Bits consumed at the end of each of the four possible symbols. + // Entry 3 always contains the bits from all symbols, even if fewer + // than four symbols are consumed. uint8_t offset[4]; + // Index of the table to use for the next lookup, combining the + // type of the next symbol with k_run and k_level after this entry. + uint8_t next; } APVMultiVLCLUTEntry; +// Types of entry in the entropy table. +enum { + // Entries are run/level/run/level. + APV_ENTROPY_RUN_FIRST = 0, + // Entries are level/run/level/run. + APV_ENTROPY_LEVEL_FIRST = 1, +}; +// Index of the base point in the multi-symbol table for the given +// type, k_run and k_level. +#define APV_TABLE_INDEX(type, k_run, k_level) \ + (((type) * 3 + (k_run)) * 5 + (k_level)) + typedef struct APVVLCLUT { // Single-symbol LUT for VLCs. // Applies to all coefficients, but used only for DC coefficients // in the decoder. APVSingleVLCLUTEntry single_lut[6][APV_VLC_LUT_SIZE]; // Multi-symbol LUT for run/level combinations, decoding up to four - // symbols per step. Comes in two versions, which to use depends on - // whether the next symbol is a run or a level. - APVMultiVLCLUTEntry run_first_lut[3][5][APV_VLC_LUT_SIZE]; - APVMultiVLCLUTEntry level_first_lut[3][5][APV_VLC_LUT_SIZE]; + // symbols per step. + // Indexed by [run first or level first][k_run][k_level][bits]. + APVMultiVLCLUTEntry multi_lut[2][3][5][APV_VLC_LUT_SIZE]; + // Zigzag scan extended past the end of the block, where every + // position maps to a dummy coefficient after the block. + uint8_t zigzag[128]; } APVVLCLUT; + +// Number of coefficients in the block buffer used by entropy decode: +// the block itself and a dummy coefficient slot at the end which is +// used to avoid branching. +#define APV_ENTROPY_COEFF_SIZE 65 + typedef struct APVEntropyState { void *log_ctx; @@ -126,8 +163,10 @@ void ff_apv_entropy_build_decode_lut(APVVLCLUT *decode_lut); * Entropy decode a single 8x8 block to coefficients. * * Outputs nonzero coefficients only to the block row-major order - * (dezigzag is applied within the function). The output block - * must have been filled with zeroes before calling this function. + * (dezigzag is applied within the function). The first 64 elements + * of the output block must have been filled with zeroes before + * calling this function, and there must be space for + * APV_ENTROPY_COEFF_SIZE elements in total. */ int ff_apv_entropy_decode_block(int16_t *restrict coeff, GetBitContext *restrict gbc, diff --git a/libavcodec/apv_entropy.c b/libavcodec/apv_entropy.c index 30f31a14ff5..f72f2c31f64 100644 --- a/libavcodec/apv_entropy.c +++ b/libavcodec/apv_entropy.c @@ -108,101 +108,214 @@ void ff_apv_entropy_build_decode_lut(APVVLCLUT *decode_lut) // Build the multi-symbol VLC table. for (int start_run = 0; start_run <= 2; start_run++) { for (int start_level = 0; start_level <= 4; start_level++) { - for (unsigned int code = 0; code < lut_size; code++) { - APVMultiVLCLUTEntry *ent; - int k_run, k_level; - GetBitContext gbc; - PutBitContext pbc; - uint8_t buffer[16]; - uint8_t run_first_buffer[16]; - uint8_t level_first_buffer[16]; + for (int type = APV_ENTROPY_RUN_FIRST; + type <= APV_ENTROPY_LEVEL_FIRST; type++) { + for (unsigned int code = 0; code < lut_size; code++) { + APVMultiVLCLUTEntry *ent; + int k_run, k_level, adv, nb_levels; + GetBitContext gbc; + PutBitContext pbc; + uint8_t buffer[16]; - memset(buffer, 0, sizeof(buffer)); - init_put_bits(&pbc, buffer, sizeof(buffer)); - put_bits(&pbc, APV_VLC_LUT_BITS, code); - flush_put_bits(&pbc); + memset(buffer, 0, sizeof(buffer)); + init_put_bits(&pbc, buffer, sizeof(buffer)); + put_bits(&pbc, APV_VLC_LUT_BITS, code); + flush_put_bits(&pbc); - memcpy(run_first_buffer, buffer, sizeof(buffer)); - memcpy(level_first_buffer, buffer, sizeof(buffer)); + k_run = start_run; + k_level = start_level; - k_run = start_run; - k_level = start_level; + ent = &decode_lut->multi_lut[type][k_run][k_level][code]; + memset(ent, 0, sizeof(*ent)); + init_get_bits8(&gbc, buffer, sizeof(buffer)); - ent = &decode_lut->run_first_lut[k_run][k_level][code]; - memset(ent, 0, sizeof(*ent)); - init_get_bits8(&gbc, run_first_buffer, sizeof(run_first_buffer)); + // Symbols alternate between run and level, starting + // with whichever the type indicates (so even values + // of sym are runs). Stop at the first one which + // does not fit in the lookup bits. + adv = 0; + nb_levels = 0; + for (int sym = type; sym < type + 4; sym++) { + int value, sign, pos; - ent->count = 0; - for (int i = 0; i <= 1; i++) { - int value, sign, pos; - - value = apv_read_vlc(&gbc, k_run, decode_lut); - pos = get_bits_count(&gbc); - if (pos > APV_VLC_LUT_BITS) - break; - ent->run[i] = value; - ent->offset[ent->count] = pos; - ++ent->count; - k_run = FFMIN(value >> 2, 2); - - value = apv_read_vlc(&gbc, k_level, decode_lut); - sign = get_bits1(&gbc); - pos = get_bits_count(&gbc); - if (pos > APV_VLC_LUT_BITS) - break; - ++value; - ent->level[i] = sign ? -value : value; - ent->offset[ent->count] = pos; - ++ent->count; - k_level = FFMIN(value >> 2, 4); - if (i == 0) - ent->k_level_0 = k_level; + if (sym % 2 == 0) { + // Reading a run. + value = apv_read_vlc(&gbc, k_run, decode_lut); + pos = get_bits_count(&gbc); + if (pos > APV_VLC_LUT_BITS) + break; + adv += value; + k_run = FFMIN(value >> 2, 2); + } else { + // Reading a level. + value = apv_read_vlc(&gbc, k_level, decode_lut); + sign = get_bits1(&gbc); + pos = get_bits_count(&gbc); + if (pos > APV_VLC_LUT_BITS) + break; + ++value; + av_assert0(value <= INT8_MAX); + ent->level[nb_levels] = sign ? -value : value; + ent->level_pos[nb_levels] = adv; + ++nb_levels; + adv += 1; + k_level = FFMIN(value >> 2, 4); + if (nb_levels == 1) + ent->k_level_0 = k_level; + } + ent->offset[ent->count] = pos; + ++ent->count; + } + // Missing levels write zeroes later in the array where + // they will either be overwritten or not used. + for (int i = nb_levels; i < 2; i++) + ent->level_pos[i] = adv; + // Entries are applied at scan positions below 64, so + // this keeps every write inside the extended zigzag. + av_assert0(adv < 64); + ent->scan_advance = adv; + if (ent->count > 0 && ent->count < 4) + ent->offset[3] = ent->offset[ent->count - 1]; + ent->k_run = k_run; + ent->k_level_1 = k_level; + ent->next = APV_TABLE_INDEX(type ^ (ent->count & 1), + k_run, k_level); } - if (ent->count > 0 && ent->count < 4) - ent->offset[3] = ent->offset[ent->count - 1]; - ent->k_run = k_run; - ent->k_level_1 = k_level; - - k_run = start_run; - k_level = start_level; - - ent = &decode_lut->level_first_lut[k_run][k_level][code]; - memset(ent, 0, sizeof(*ent)); - init_get_bits8(&gbc, level_first_buffer, sizeof(level_first_buffer)); - - ent->count = 0; - for (int i = 0; i <= 1; i++) { - int value, sign, pos; - - value = apv_read_vlc(&gbc, k_level, decode_lut); - sign = get_bits1(&gbc); - pos = get_bits_count(&gbc); - if (pos > APV_VLC_LUT_BITS) - break; - ++value; - ent->level[i] = sign ? -value : value; - ent->offset[ent->count] = pos; - ++ent->count; - k_level = FFMIN(value >> 2, 4); - if (i == 0) - ent->k_level_0 = k_level; - - value = apv_read_vlc(&gbc, k_run, decode_lut); - pos = get_bits_count(&gbc); - if (pos > APV_VLC_LUT_BITS) - break; - ent->run[i] = value; - ent->offset[ent->count] = pos; - ++ent->count; - k_run = FFMIN(value >> 2, 2); - } - if (ent->count > 0 && ent->count < 4) - ent->offset[3] = ent->offset[ent->count - 1]; - ent->k_run = k_run; - ent->k_level_1 = k_level; } } } + + // Positions past the end of the block all map to the dummy + // coefficient slot at the end. + for (int i = 0; i < FF_ARRAY_ELEMS(decode_lut->zigzag); i++) + decode_lut->zigzag[i] = i < 64 ? ff_zigzag_direct[i] : 64; +} + +// Decode a run code which is too long for the multi-symbol LUT. +// All such codes fit in the 18 bits already read. +av_always_inline +static int apv_decode_long_run(GetBitContext *restrict gbc, + uint32_t next_bits, int k_run, + APVEntropyState *restrict state) +{ + uint32_t bits, low_bits; + unsigned int leading_zeroes, low_bit_count, low_bit_shift; + + // Remove the prefix bits. + bits = next_bits & 0xffff; + // Determine code length. + leading_zeroes = 15 - av_log2(bits); + if (leading_zeroes >= 6) { + // 6 zeroes implies run > 64, which is always invalid. + av_log(state->log_ctx, AV_LOG_ERROR, + "Out-of-range run value: %d leading zeroes.\n", + leading_zeroes); + return AVERROR_INVALIDDATA; + } + // Extract the low bits. + low_bit_count = leading_zeroes + k_run; + low_bit_shift = 16 - (1 + 2 * leading_zeroes + k_run); + low_bits = av_zero_extend(bits >> low_bit_shift, low_bit_count); + // Skip over the bits just used. + skip_bits(gbc, 2 + leading_zeroes + 1 + low_bit_count); + // Construct run code. + return (2 << k_run) + + ((1 << leading_zeroes) - 1) * (1 << k_run) + + low_bits; +} + +// Decode a level code which is too long for the multi-symbol LUT, +// returning the signed level and setting the absolute value. +// The 18 bits already read are sufficient for the length, but we may +// need to read more bits to complete the value. +av_always_inline +static int apv_decode_long_level(GetBitContext *restrict gbc, + uint32_t next_bits, int k_level, + int *abs_level_out, int scan_pos, + APVEntropyState *restrict state) +{ + uint32_t bits; + unsigned int leading_zeroes; + int level, abs_level, sign; + + // Remove the prefix bits. + bits = next_bits & 0xffff; + // Determine code length. + leading_zeroes = 15 - av_log2(bits); + // Skip the prefix and length bits. + skip_bits(gbc, 2 + leading_zeroes + 1); + // Read the rest of the code and construct the level. + // Include the + 1 offset for nonzero value here. + abs_level = (2 << k_level) + + ((1 << leading_zeroes) - 1) * (1 << k_level) + + get_bits(gbc, leading_zeroes + k_level) + 1; + + sign = get_bits1(gbc); + if (sign) + level = -abs_level; + else + level = abs_level; + + // Check range (not checked in any other case, only a long + // code can be out of range). + if (level < APV_MIN_TRANS_COEFF || + level > APV_MAX_TRANS_COEFF) { + av_log(state->log_ctx, AV_LOG_ERROR, + "Out-of-range AC coefficient value at %d: %d.\n", + scan_pos, level); + return AVERROR_INVALIDDATA; + } + *abs_level_out = abs_level; + return level; +} + +// Apply the symbols from a multi-symbol LUT entry. Both levels are +// always written: absent ones are zero, and any beyond the end of the +// block go to the dummy coefficient. Returns nonzero if the end of +// the block was reached, in which case only the bits up to the symbol +// which reached it are consumed. +av_always_inline +static int apv_apply_multi_entry(int16_t *restrict coeff, + GetBitContext *restrict gbc, + const APVMultiVLCLUTEntry *ent, + const uint8_t *zigzag, + int *scan_pos_ptr, int type) +{ + int scan_pos = *scan_pos_ptr; + + coeff[zigzag[scan_pos + ent->level_pos[0]]] = ent->level[0]; + coeff[zigzag[scan_pos + ent->level_pos[1]]] = ent->level[1]; + + if (scan_pos + ent->scan_advance >= 64) { + // We've gone over the end: find the symbol which reached the + // end and stop there. + int end[4], i; + + if (type == APV_ENTROPY_RUN_FIRST) { + end[0] = ent->level_pos[0]; + end[1] = ent->level_pos[0] + 1; + end[2] = ent->level_pos[1]; + end[3] = ent->level_pos[1] + 1; + } else { + end[0] = 1; + end[1] = ent->level_pos[1]; + end[2] = ent->level_pos[1] + 1; + end[3] = ent->scan_advance; + } + for (i = 0; i < ent->count - 1; i++) { + if (scan_pos + end[i] >= 64) + break; + } + skip_bits(gbc, ent->offset[i]); + *scan_pos_ptr = scan_pos + end[i]; + return 1; + } else { + // All codes used: advance the bitstream and scan postion by + // the full values. + skip_bits(gbc, ent->offset[3]); + *scan_pos_ptr = scan_pos + ent->scan_advance; + return 0; + } } int ff_apv_entropy_decode_block(int16_t *restrict coeff, @@ -210,7 +323,9 @@ int ff_apv_entropy_decode_block(int16_t *restrict coeff, APVEntropyState *restrict state) { const APVVLCLUT *lut = state->decode_lut; - int scan_pos; + const APVMultiVLCLUTEntry *tables = lut->multi_lut[0][0][0]; + const uint8_t *zigzag = lut->zigzag; + int scan_pos, type, table_index; int k_dc = state->prev_k_dc; int k_run, k_level; uint32_t next_bits, lut_bits; @@ -256,349 +371,118 @@ int ff_apv_entropy_decode_block(int16_t *restrict coeff, // single-symbol LUT is never useful here as the multisymbol lookup // has already determined that the code is too long.) - // Run a single iteration of the run-first LUT to start, then a - // single iteration of the level-first LUT if that only read a - // single code. This avoids dealing with the first-AC logic inside - // the normal code lookup sequence. + // The first level in the block sets the k_level for the next block, + // so run the first lookup (and the second, if the first did not + // contain a level) separately. - k_level = state->prev_k_level; + scan_pos = 1; + k_run = 0; + k_level = state->prev_k_level; { next_bits = show_bits(gbc, 18); lut_bits = next_bits >> (18 - APV_VLC_LUT_BITS); - ent = &lut->run_first_lut[0][k_level][lut_bits]; + ent = &lut->multi_lut[APV_ENTROPY_RUN_FIRST][0][k_level][lut_bits]; if (ent->count == 0) { - // One long code. - uint32_t bits, low_bits; - unsigned int leading_zeroes, low_bit_count, low_bit_shift; - int run; - - // Remove the prefix bits. - bits = next_bits & 0xffff; - // Determine code length. - leading_zeroes = 15 - av_log2(bits); - if (leading_zeroes >= 6) { - // 6 zeroes implies run > 64, which is always invalid. - av_log(state->log_ctx, AV_LOG_ERROR, - "Out-of-range run value: %d leading zeroes.\n", - leading_zeroes); - return AVERROR_INVALIDDATA; - } - // Extract the low bits. - low_bit_count = leading_zeroes; - low_bit_shift = 16 - (1 + 2 * leading_zeroes); - low_bits = av_zero_extend(bits >> low_bit_shift, low_bit_count); - // Construct run code. - run = 2 + ((1 << leading_zeroes) - 1) + low_bits; - // Skip over the bits just used. - skip_bits(gbc, 2 + leading_zeroes + 1 + low_bit_count); - - scan_pos = run + 1; + int run = apv_decode_long_run(gbc, next_bits, 0, state); + if (run < 0) + return run; + scan_pos += run; if (scan_pos >= 64) goto end_of_block; k_run = FFMIN(run >> 2, 2); - goto first_level; } else { - // One or more short codes starting with a run; if there is - // a level code then the length needs to be saved for the - // next block. - - scan_pos = ent->run[0] + 1; - if (scan_pos >= 64) { - skip_bits(gbc, ent->offset[0]); - goto end_of_block; - } - if (ent->count > 1) { - coeff[ff_zigzag_direct[scan_pos]] = ent->level[0]; - ++scan_pos; + // The level is only real if the run before it did not + // reach the end of the block. + if (ent->count > 1 && scan_pos + ent->level_pos[0] < 64) state->prev_k_level = ent->k_level_0; - if (scan_pos >= 64) { - skip_bits(gbc, ent->offset[1]); - goto end_of_block; - } - } - if (ent->count > 2) { - scan_pos += ent->run[1]; - if (scan_pos >= 64) { - skip_bits(gbc, ent->offset[2]); - goto end_of_block; - } - } - if (ent->count > 3) { - coeff[ff_zigzag_direct[scan_pos]] = ent->level[1]; - ++scan_pos; - if (scan_pos >= 64) { - skip_bits(gbc, ent->offset[3]); - goto end_of_block; - } - } - skip_bits(gbc, ent->offset[3]); + if (apv_apply_multi_entry(coeff, gbc, ent, zigzag, + &scan_pos, APV_ENTROPY_RUN_FIRST)) + goto end_of_block; k_run = ent->k_run; k_level = ent->k_level_1; - if (ent->count == 1) - goto first_level; - else if (ent->count & 1) - goto next_is_level; - else - goto next_is_run; + if (ent->count > 1) { + type = APV_ENTROPY_RUN_FIRST ^ (ent->count & 1); + goto normal_loop; + } } } - first_level: { + // Only read a run, still need to read the first level. + { next_bits = show_bits(gbc, 18); lut_bits = next_bits >> (18 - APV_VLC_LUT_BITS); - ent = &lut->level_first_lut[k_run][k_level][lut_bits]; + ent = &lut->multi_lut[APV_ENTROPY_LEVEL_FIRST][k_run][k_level][lut_bits]; if (ent->count == 0) { - // One long code. - uint32_t bits; - unsigned int leading_zeroes; - int level, abs_level, sign; - - // Remove the prefix bits. - bits = next_bits & 0xffff; - // Determine code length. - leading_zeroes = 15 - av_log2(bits); - // Skip the prefix and length bits. - skip_bits(gbc, 2 + leading_zeroes + 1); - // Read the rest of the code and construct the level. - // Include the + 1 offset for nonzero value here. - abs_level = (2 << k_level) + - ((1 << leading_zeroes) - 1) * (1 << k_level) + - get_bits(gbc, leading_zeroes + k_level) + 1; - - sign = get_bits(gbc, 1); - if (sign) - level = -abs_level; - else - level = abs_level; - - // Check range (not checked in any other case, only a long - // code can be out of range). - if (level < APV_MIN_TRANS_COEFF || - level > APV_MAX_TRANS_COEFF) { - av_log(state->log_ctx, AV_LOG_ERROR, - "Out-of-range AC coefficient value at %d: %d.\n", - scan_pos, level); - return AVERROR_INVALIDDATA; - } - coeff[ff_zigzag_direct[scan_pos]] = level; + int level, abs_level; + level = apv_decode_long_level(gbc, next_bits, k_level, + &abs_level, scan_pos, state); + if (level < APV_MIN_TRANS_COEFF) + return level; + coeff[zigzag[scan_pos]] = level; ++scan_pos; k_level = FFMIN(abs_level >> 2, 4); state->prev_k_level = k_level; if (scan_pos >= 64) goto end_of_block; - goto next_is_run; - + type = APV_ENTROPY_RUN_FIRST; } else { - // One or more short codes. - - coeff[ff_zigzag_direct[scan_pos]] = ent->level[0]; - ++scan_pos; state->prev_k_level = ent->k_level_0; - if (scan_pos >= 64) { - skip_bits(gbc, ent->offset[0]); + if (apv_apply_multi_entry(coeff, gbc, ent, zigzag, + &scan_pos, APV_ENTROPY_LEVEL_FIRST)) goto end_of_block; - } - if (ent->count > 1) { - scan_pos += ent->run[0]; - if (scan_pos >= 64) { - skip_bits(gbc, ent->offset[1]); - goto end_of_block; - } - } - if (ent->count > 2) { - coeff[ff_zigzag_direct[scan_pos]] = ent->level[1]; - ++scan_pos; - if (scan_pos >= 64) { - skip_bits(gbc, ent->offset[2]); - goto end_of_block; - } - } - if (ent->count > 3) { - scan_pos += ent->run[1]; - if (scan_pos >= 64) { - skip_bits(gbc, ent->offset[3]); - goto end_of_block; - } - } - skip_bits(gbc, ent->offset[3]); k_run = ent->k_run; k_level = ent->k_level_1; - if (ent->count & 1) - goto next_is_run; - else - goto next_is_level; + type = APV_ENTROPY_LEVEL_FIRST ^ (ent->count & 1); } } - next_is_run: { +normal_loop: + // From here the only state carried between lookups is the index of + // the table to use, which each entry supplies for the next one. + table_index = APV_TABLE_INDEX(type, k_run, k_level); + for (;;) { next_bits = show_bits(gbc, 18); lut_bits = next_bits >> (18 - APV_VLC_LUT_BITS); - ent = &lut->run_first_lut[k_run][k_level][lut_bits]; + ent = &tables[table_index * APV_VLC_LUT_SIZE + lut_bits]; if (ent->count == 0) { - // One long code. - uint32_t bits, low_bits; - unsigned int leading_zeroes, low_bit_count, low_bit_shift; - int run; - - // Remove the prefix bits. - bits = next_bits & 0xffff; - // Determine code length. - leading_zeroes = 15 - av_log2(bits); - if (leading_zeroes >= 6) { - // 6 zeroes implies run > 64, which is always invalid. - av_log(state->log_ctx, AV_LOG_ERROR, - "Out-of-range run value: %d leading zeroes.\n", - leading_zeroes); - return AVERROR_INVALIDDATA; + // Missed in the lookup, so need to re-derive the type, + // k_run and k_level from the table index. + k_run = table_index / 5 % 3; + k_level = table_index % 5; + if (table_index < APV_TABLE_INDEX(APV_ENTROPY_LEVEL_FIRST, 0, 0)) { + int run = apv_decode_long_run(gbc, next_bits, k_run, state); + if (run < 0) + return run; + scan_pos += run; + if (scan_pos >= 64) + goto end_of_block; + k_run = FFMIN(run >> 2, 2); + table_index = APV_TABLE_INDEX(APV_ENTROPY_LEVEL_FIRST, k_run, k_level); + } else { + int level, abs_level; + level = apv_decode_long_level(gbc, next_bits, k_level, + &abs_level, scan_pos, state); + if (level < APV_MIN_TRANS_COEFF) + return level; + coeff[zigzag[scan_pos]] = level; + ++scan_pos; + k_level = FFMIN(abs_level >> 2, 4); + if (scan_pos >= 64) + goto end_of_block; + table_index = APV_TABLE_INDEX(APV_ENTROPY_RUN_FIRST, k_run, k_level); } - // Extract the low bits. - low_bit_count = leading_zeroes + k_run; - low_bit_shift = 16 - (1 + 2 * leading_zeroes + k_run); - low_bits = av_zero_extend(bits >> low_bit_shift, low_bit_count); - // Construct run code. - run = (2 << k_run) + - ((1 << leading_zeroes) - 1) * (1 << k_run) + - low_bits; - // Skip over the bits just used. - skip_bits(gbc, 2 + leading_zeroes + 1 + low_bit_count); - - scan_pos += run; - if (scan_pos >= 64) - goto end_of_block; - k_run = FFMIN(run >> 2, 2); - goto next_is_level; - } else { - // One or more short codes. - - scan_pos += ent->run[0]; - if (scan_pos >= 64) { - skip_bits(gbc, ent->offset[0]); + type = table_index >= APV_TABLE_INDEX(APV_ENTROPY_LEVEL_FIRST, 0, 0); + if (apv_apply_multi_entry(coeff, gbc, ent, zigzag, + &scan_pos, type)) goto end_of_block; - } - if (ent->count > 1) { - coeff[ff_zigzag_direct[scan_pos]] = ent->level[0]; - ++scan_pos; - if (scan_pos >= 64) { - skip_bits(gbc, ent->offset[1]); - goto end_of_block; - } - } - if (ent->count > 2) { - scan_pos += ent->run[1]; - if (scan_pos >= 64) { - skip_bits(gbc, ent->offset[2]); - goto end_of_block; - } - } - if (ent->count > 3) { - coeff[ff_zigzag_direct[scan_pos]] = ent->level[1]; - ++scan_pos; - if (scan_pos >= 64) { - skip_bits(gbc, ent->offset[3]); - goto end_of_block; - } - } - skip_bits(gbc, ent->offset[3]); - k_run = ent->k_run; - k_level = ent->k_level_1; - if (ent->count & 1) - goto next_is_level; - else - goto next_is_run; - } - } - - next_is_level: { - next_bits = show_bits(gbc, 18); - lut_bits = next_bits >> (18 - APV_VLC_LUT_BITS); - - ent = &lut->level_first_lut[k_run][k_level][lut_bits]; - - if (ent->count == 0) { - // One long code. - uint32_t bits; - unsigned int leading_zeroes; - int level, abs_level, sign; - - // Remove the prefix bits. - bits = next_bits & 0xffff; - // Determine code length. - leading_zeroes = 15 - av_log2(bits); - // Skip the prefix and length bits. - skip_bits(gbc, 2 + leading_zeroes + 1); - // Read the rest of the code and construct the level. - // Include the + 1 offset for nonzero value here. - abs_level = (2 << k_level) + - ((1 << leading_zeroes) - 1) * (1 << k_level) + - get_bits(gbc, leading_zeroes + k_level) + 1; - - sign = get_bits(gbc, 1); - if (sign) - level = -abs_level; - else - level = abs_level; - - // Check range (not checked in any other case, only a long - // code can be out of range). - if (level < APV_MIN_TRANS_COEFF || - level > APV_MAX_TRANS_COEFF) { - av_log(state->log_ctx, AV_LOG_ERROR, - "Out-of-range AC coefficient value at %d: %d.\n", - scan_pos, level); - return AVERROR_INVALIDDATA; - } - coeff[ff_zigzag_direct[scan_pos]] = level; - ++scan_pos; - k_level = FFMIN(abs_level >> 2, 4); - if (scan_pos >= 64) - goto end_of_block; - goto next_is_run; - - } else { - // One or more short codes. - - coeff[ff_zigzag_direct[scan_pos]] = ent->level[0]; - ++scan_pos; - if (scan_pos >= 64) { - skip_bits(gbc, ent->offset[0]); - goto end_of_block; - } - if (ent->count > 1) { - scan_pos += ent->run[0]; - if (scan_pos >= 64) { - skip_bits(gbc, ent->offset[1]); - goto end_of_block; - } - } - if (ent->count > 2) { - coeff[ff_zigzag_direct[scan_pos]] = ent->level[1]; - ++scan_pos; - if (scan_pos >= 64) { - skip_bits(gbc, ent->offset[2]); - goto end_of_block; - } - } - if (ent->count > 3) { - scan_pos += ent->run[1]; - if (scan_pos >= 64) { - skip_bits(gbc, ent->offset[3]); - goto end_of_block; - } - } - skip_bits(gbc, ent->offset[3]); - k_run = ent->k_run; - k_level = ent->k_level_1; - if (ent->count & 1) - goto next_is_run; - else - goto next_is_level; + table_index = ent->next; } } diff --git a/libavcodec/tests/apv.c b/libavcodec/tests/apv.c index 365a395af1e..2f57539a1de 100644 --- a/libavcodec/tests/apv.c +++ b/libavcodec/tests/apv.c @@ -313,7 +313,7 @@ static int test_apv_entropy_decode_block(void) APVEntropyState state, save_state; int16_t block[64]; int16_t block_test1[64]; - int16_t block_test2[64]; + int16_t block_test2[APV_ENTROPY_COEFF_SIZE]; uint8_t buffer[1024]; PutBitContext pbc; GetBitContext gbc; -- 2.54.0 >From 360b9bd5e3ee09fe363e75d96d7f282e356af77b Mon Sep 17 00:00:00 2001 From: Mark Thompson <[email protected]> Date: Sun, 25 Oct 2026 09:45:01 +0000 Subject: [PATCH 5/5] lavc/apv: Use a local copy of the GetBitContext Having it on the stack lets the compiler see within this translation unit that there are no side effects, improving optimisation. --- libavcodec/apv_entropy.c | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/libavcodec/apv_entropy.c b/libavcodec/apv_entropy.c index f72f2c31f64..3e92a5697a1 100644 --- a/libavcodec/apv_entropy.c +++ b/libavcodec/apv_entropy.c @@ -318,9 +318,10 @@ static int apv_apply_multi_entry(int16_t *restrict coeff, } } -int ff_apv_entropy_decode_block(int16_t *restrict coeff, - GetBitContext *restrict gbc, - APVEntropyState *restrict state) +av_always_inline +static int apv_entropy_decode_block_internal(int16_t *restrict coeff, + GetBitContext *restrict gbc, + APVEntropyState *restrict state) { const APVVLCLUT *lut = state->decode_lut; const APVMultiVLCLUTEntry *tables = lut->multi_lut[0][0][0]; @@ -496,3 +497,16 @@ normal_loop: return 0; } } + +int ff_apv_entropy_decode_block(int16_t *restrict coeff, + GetBitContext *restrict gbc, + APVEntropyState *restrict state) +{ + // Work on a local copy of the bitstream reader so that its state + // can be kept in registers rather than written back after every + // read. + GetBitContext gb = *gbc; + int err = apv_entropy_decode_block_internal(coeff, &gb, state); + *gbc = gb; + return err; +} -- 2.54.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
