This is an automated email from the git hooks/post-receive script. Git pushed a commit to branch master in repository ffmpeg.
commit 0886e50c6b8e812d14ed7acd329684251efbf69e Author: Jun Zhao <[email protected]> AuthorDate: Wed Jan 7 07:50:13 2026 +0800 Commit: Jun Zhao <[email protected]> CommitDate: Sun Jan 25 06:55:26 2026 +0000 lavc/hevc: add aarch64 neon for 8-bit dequant Implement NEON optimization for HEVC dequant at 8-bit depth. The NEON implementation uses srshr (Signed Rounding Shift Right) which does both the add with offset and right shift in a single instruction. Optimization details: - 4x4 (16 coeffs): Single load-process-store sequence - 8x8 (64 coeffs): Fully unrolled, no loop overhead - 16x16 (256 coeffs): Pipelined load/compute/store to hide memory latency - 32x32 (1024 coeffs): Pipelined with all available NEON registers Performance benchmark on Apple M4: ./tests/checkasm/checkasm --test=hevc_dequant --bench hevc_dequant_4x4_8_c: 11.3 ( 1.00x) hevc_dequant_4x4_8_neon: 6.3 ( 1.78x) hevc_dequant_8x8_8_c: 33.9 ( 1.00x) hevc_dequant_8x8_8_neon: 6.6 ( 5.11x) hevc_dequant_16x16_8_c: 153.8 ( 1.00x) hevc_dequant_16x16_8_neon: 9.0 (17.02x) hevc_dequant_32x32_8_c: 78.1 ( 1.00x) hevc_dequant_32x32_8_neon: 31.9 ( 2.45x) Note on Performance Anomaly: The observation that hevc_dequant_32x32_8_c is faster than 16x16 (78.1 vs 153.8) is due to Clang auto-vectorizing only for sizes >= 32x32. Compiler: Apple clang version 17.0.0 (clang-1700.6.3.2) Signed-off-by: Jun Zhao <[email protected]> --- libavcodec/aarch64/Makefile | 1 + libavcodec/aarch64/hevcdsp_dequant_neon.S | 169 ++++++++++++++++++++++++++++++ libavcodec/aarch64/hevcdsp_init_aarch64.c | 18 ++++ tests/checkasm/Makefile | 2 +- tests/checkasm/checkasm.c | 1 + tests/checkasm/checkasm.h | 1 + tests/checkasm/hevc_dequant.c | 76 ++++++++++++++ tests/fate/checkasm.mak | 1 + 8 files changed, 268 insertions(+), 1 deletion(-) diff --git a/libavcodec/aarch64/Makefile b/libavcodec/aarch64/Makefile index 2bf48dfa28..1e838ad901 100644 --- a/libavcodec/aarch64/Makefile +++ b/libavcodec/aarch64/Makefile @@ -71,6 +71,7 @@ NEON-OBJS-$(CONFIG_VP9_DECODER) += aarch64/vp9itxfm_16bpp_neon.o \ aarch64/vp9mc_16bpp_neon.o \ aarch64/vp9mc_neon.o NEON-OBJS-$(CONFIG_HEVC_DECODER) += aarch64/hevcdsp_deblock_neon.o \ + aarch64/hevcdsp_dequant_neon.o \ aarch64/hevcdsp_idct_neon.o \ aarch64/hevcdsp_init_aarch64.o \ aarch64/h26x/epel_neon.o \ diff --git a/libavcodec/aarch64/hevcdsp_dequant_neon.S b/libavcodec/aarch64/hevcdsp_dequant_neon.S new file mode 100644 index 0000000000..a757bac6b3 --- /dev/null +++ b/libavcodec/aarch64/hevcdsp_dequant_neon.S @@ -0,0 +1,169 @@ +/* + * ARM NEON optimised dequant functions for HEVC decoding + * + * Copyright (c) 2026 FFmpeg contributors + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "libavutil/aarch64/asm.S" + +// HEVC dequant for 8-bit depth +// +// Algorithm (from dsp_template.c): +// shift = 15 - BIT_DEPTH - log2_size +// offset = 1 << (shift - 1) +// output = (input + offset) >> shift +// +// This is equivalent to: output = ROUND(input >> shift) +// NEON srshr (Signed Rounding Shift Right) does exactly this in one instruction! +// +// For 8-bit: shift = 15 - 8 - log2_size = 7 - log2_size +// +// Block size | log2_size | shift | operation +// 4x4 | 2 | 5 | srshr #5 +// 8x8 | 3 | 4 | srshr #4 +// 16x16 | 4 | 3 | srshr #3 +// 32x32 | 5 | 2 | srshr #2 + +// void ff_hevc_dequant_4x4_8_neon(int16_t *coeffs) +// 4x4 = 16 coeffs, shift=5 +function ff_hevc_dequant_4x4_8_neon, export=1 + ldp q0, q1, [x0] // load 16 int16_t (32 bytes) + srshr v0.8h, v0.8h, #5 // rounding shift right by 5 + srshr v1.8h, v1.8h, #5 + stp q0, q1, [x0] // store + ret +endfunc + +// void ff_hevc_dequant_8x8_8_neon(int16_t *coeffs) +// 8x8 = 64 coeffs, shift=4 +// Fully unrolled - no loop needed for 64 coeffs +function ff_hevc_dequant_8x8_8_neon, export=1 + ld1 {v0.16b-v3.16b}, [x0], #64 + ld1 {v4.16b-v7.16b}, [x0] + sub x0, x0, #64 + srshr v0.8h, v0.8h, #4 + srshr v1.8h, v1.8h, #4 + srshr v2.8h, v2.8h, #4 + srshr v3.8h, v3.8h, #4 + srshr v4.8h, v4.8h, #4 + srshr v5.8h, v5.8h, #4 + srshr v6.8h, v6.8h, #4 + srshr v7.8h, v7.8h, #4 + st1 {v0.16b-v3.16b}, [x0], #64 + st1 {v4.16b-v7.16b}, [x0] + ret +endfunc + +// void ff_hevc_dequant_16x16_8_neon(int16_t *coeffs) +// 16x16 = 256 coeffs, shift=3 +// Pipelined implementation: interleave load/compute/store to hide memory latency +// Uses .irp macro to unroll 4 iterations, processing 64 coeffs per iteration +// x0 = load pointer, x1 = store pointer (both advance through the buffer) +function ff_hevc_dequant_16x16_8_neon, export=1 + mov x1, x0 + ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #64 +.irp i, 0, 1, 2, 3 + srshr v0.8h, v0.8h, #3 + srshr v1.8h, v1.8h, #3 + ld1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x0], #64 + srshr v2.8h, v2.8h, #3 + srshr v3.8h, v3.8h, #3 + srshr v4.8h, v4.8h, #3 + srshr v5.8h, v5.8h, #3 + st1 {v0.16b - v3.16b}, [x1], #64 + srshr v6.8h, v6.8h, #3 +.if \i < 3 + ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #64 +.endif + srshr v7.8h, v7.8h, #3 + st1 {v4.16b - v7.16b}, [x1], #64 +.endr + ret +endfunc + +// void ff_hevc_dequant_32x32_8_neon(int16_t *coeffs) +// 32x32 = 1024 coeffs, shift=2 +// Process 128 coeffs per iteration (8 iterations) +// Using all available NEON registers for maximum throughput +// AAPCS64: v0-v7 and v16-v31 are volatile (caller-saved) +// We use v0-v7 and v16-v23 to avoid touching callee-saved v8-v15 +function ff_hevc_dequant_32x32_8_neon, export=1 + mov x2, #8 // loop 8 times (128 coeffs each) +1: + // Group A: q0-q3 (64 bytes / 32 coeffs) + ldp q0, q1, [x0] + ldp q2, q3, [x0, #32] + // Group B: q4-q7 (64 bytes / 32 coeffs) + ldp q4, q5, [x0, #64] + ldp q6, q7, [x0, #96] + subs x2, x2, #1 // Decrement loop counter early for better pipelining + + // Calc Group A (shift right with rounding) + srshr v0.8h, v0.8h, #2 + srshr v1.8h, v1.8h, #2 + srshr v2.8h, v2.8h, #2 + srshr v3.8h, v3.8h, #2 + + // Group C: q16-q19 (64 bytes / 32 coeffs) + // Load into volatile high registers to maximize pipeline usage + ldp q16, q17, [x0, #128] + ldp q18, q19, [x0, #160] + + // Calc Group B + srshr v4.8h, v4.8h, #2 + srshr v5.8h, v5.8h, #2 + srshr v6.8h, v6.8h, #2 + srshr v7.8h, v7.8h, #2 + + // Store Group A (Write back results to memory) + stp q0, q1, [x0] + stp q2, q3, [x0, #32] + + // Group D: q20-q23 (64 bytes / 32 coeffs) + ldp q20, q21, [x0, #192] + ldp q22, q23, [x0, #224] + + // Calc Group C + srshr v16.8h, v16.8h, #2 + srshr v17.8h, v17.8h, #2 + srshr v18.8h, v18.8h, #2 + srshr v19.8h, v19.8h, #2 + + // Store Group B + stp q4, q5, [x0, #64] + stp q6, q7, [x0, #96] + + // Calc Group D + srshr v20.8h, v20.8h, #2 + srshr v21.8h, v21.8h, #2 + srshr v22.8h, v22.8h, #2 + srshr v23.8h, v23.8h, #2 + + // Store Group C + stp q16, q17, [x0, #128] + stp q18, q19, [x0, #160] + + // Store Group D + stp q20, q21, [x0, #192] + stp q22, q23, [x0, #224] + + add x0, x0, #256 // Advance pointer by 128 coeffs (256 bytes) + b.ne 1b + ret +endfunc diff --git a/libavcodec/aarch64/hevcdsp_init_aarch64.c b/libavcodec/aarch64/hevcdsp_init_aarch64.c index 8dec58bc7f..80c9d1e2d2 100644 --- a/libavcodec/aarch64/hevcdsp_init_aarch64.c +++ b/libavcodec/aarch64/hevcdsp_init_aarch64.c @@ -22,6 +22,7 @@ #include <stdint.h> #include "libavutil/attributes.h" +#include "libavutil/avassert.h" #include "libavutil/cpu.h" #include "libavutil/aarch64/cpu.h" #include "libavcodec/aarch64/h26x/dsp.h" @@ -97,6 +98,22 @@ void ff_hevc_idct_16x16_dc_12_neon(int16_t *coeffs); void ff_hevc_idct_32x32_dc_12_neon(int16_t *coeffs); void ff_hevc_transform_luma_4x4_neon_8(int16_t *coeffs); +void ff_hevc_dequant_4x4_8_neon(int16_t *coeffs); +void ff_hevc_dequant_8x8_8_neon(int16_t *coeffs); +void ff_hevc_dequant_16x16_8_neon(int16_t *coeffs); +void ff_hevc_dequant_32x32_8_neon(int16_t *coeffs); + +static void hevc_dequant_8_neon(int16_t *coeffs, int16_t log2_size) +{ + switch (log2_size) { + case 2: ff_hevc_dequant_4x4_8_neon(coeffs); break; + case 3: ff_hevc_dequant_8x8_8_neon(coeffs); break; + case 4: ff_hevc_dequant_16x16_8_neon(coeffs); break; + case 5: ff_hevc_dequant_32x32_8_neon(coeffs); break; + default: av_unreachable("log2_size must be 2, 3, 4 or 5"); + } +} + #define NEON8_FNASSIGN(member, v, h, fn, ext) \ member[1][v][h] = ff_hevc_put_hevc_##fn##4_8_neon##ext; \ member[2][v][h] = ff_hevc_put_hevc_##fn##6_8_neon##ext; \ @@ -168,6 +185,7 @@ av_cold void ff_hevc_dsp_init_aarch64(HEVCDSPContext *c, const int bit_depth) c->idct_dc[2] = ff_hevc_idct_16x16_dc_8_neon; c->idct_dc[3] = ff_hevc_idct_32x32_dc_8_neon; c->transform_4x4_luma = ff_hevc_transform_luma_4x4_neon_8; + c->dequant = hevc_dequant_8_neon; c->sao_band_filter[0] = ff_h26x_sao_band_filter_8x8_8_neon; c->sao_band_filter[1] = c->sao_band_filter[2] = diff --git a/tests/checkasm/Makefile b/tests/checkasm/Makefile index 48f358d40d..f4ac5a940f 100644 --- a/tests/checkasm/Makefile +++ b/tests/checkasm/Makefile @@ -42,7 +42,7 @@ AVCODECOBJS-$(CONFIG_HUFFYUV_DECODER) += huffyuvdsp.o AVCODECOBJS-$(CONFIG_JPEG2000_DECODER) += jpeg2000dsp.o AVCODECOBJS-$(CONFIG_OPUS_DECODER) += opusdsp.o AVCODECOBJS-$(CONFIG_PIXBLOCKDSP) += pixblockdsp.o -AVCODECOBJS-$(CONFIG_HEVC_DECODER) += hevc_add_res.o hevc_deblock.o hevc_idct.o hevc_sao.o hevc_pel.o +AVCODECOBJS-$(CONFIG_HEVC_DECODER) += hevc_add_res.o hevc_deblock.o hevc_dequant.o hevc_idct.o hevc_sao.o hevc_pel.o AVCODECOBJS-$(CONFIG_RV34DSP) += rv34dsp.o AVCODECOBJS-$(CONFIG_RV40_DECODER) += rv40dsp.o AVCODECOBJS-$(CONFIG_SVQ1_ENCODER) += svq1enc.o diff --git a/tests/checkasm/checkasm.c b/tests/checkasm/checkasm.c index 7dcdaeb2a4..f9ccb30ce9 100644 --- a/tests/checkasm/checkasm.c +++ b/tests/checkasm/checkasm.c @@ -187,6 +187,7 @@ static const struct { #if CONFIG_HEVC_DECODER { "hevc_add_res", checkasm_check_hevc_add_res }, { "hevc_deblock", checkasm_check_hevc_deblock }, + { "hevc_dequant", checkasm_check_hevc_dequant }, { "hevc_idct", checkasm_check_hevc_idct }, { "hevc_pel", checkasm_check_hevc_pel }, { "hevc_sao", checkasm_check_hevc_sao }, diff --git a/tests/checkasm/checkasm.h b/tests/checkasm/checkasm.h index e3addec21e..b246d0c4da 100644 --- a/tests/checkasm/checkasm.h +++ b/tests/checkasm/checkasm.h @@ -110,6 +110,7 @@ void checkasm_check_h264pred(void); void checkasm_check_h264qpel(void); void checkasm_check_hevc_add_res(void); void checkasm_check_hevc_deblock(void); +void checkasm_check_hevc_dequant(void); void checkasm_check_hevc_idct(void); void checkasm_check_hevc_pel(void); void checkasm_check_hevc_sao(void); diff --git a/tests/checkasm/hevc_dequant.c b/tests/checkasm/hevc_dequant.c new file mode 100644 index 0000000000..20e322994a --- /dev/null +++ b/tests/checkasm/hevc_dequant.c @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2024 FFmpeg contributors + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with FFmpeg; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include <string.h> + +#include "libavutil/intreadwrite.h" +#include "libavutil/mem_internal.h" + +#include "libavcodec/hevc/dsp.h" + +#include "checkasm.h" + +#define randomize_buffers(buf, size) \ + do { \ + int j; \ + for (j = 0; j < size; j++) { \ + int16_t r = rnd() & 0x7FFF; \ + if (rnd() & 1) r = -r; \ + AV_WN16A(buf + j, r); \ + } \ + } while (0) + +static void check_dequant(HEVCDSPContext *h, int bit_depth) +{ + int i; + LOCAL_ALIGNED(32, int16_t, coeffs0, [32 * 32]); + LOCAL_ALIGNED(32, int16_t, coeffs1, [32 * 32]); + + for (i = 2; i <= 5; i++) { + int block_size = 1 << i; + int size = block_size * block_size; + declare_func(void, int16_t *coeffs, int16_t log2_size); + + randomize_buffers(coeffs0, size); + memcpy(coeffs1, coeffs0, sizeof(*coeffs0) * size); + + if (check_func(h->dequant, "hevc_dequant_%dx%d_%d", + block_size, block_size, bit_depth)) { + call_ref(coeffs0, i); + call_new(coeffs1, i); + if (memcmp(coeffs0, coeffs1, sizeof(*coeffs0) * size)) + fail(); + bench_new(coeffs1, i); + } + } +} + +void checkasm_check_hevc_dequant(void) +{ + int bit_depth; + + for (bit_depth = 8; bit_depth <= 12; bit_depth += 2) { + HEVCDSPContext h; + + ff_hevc_dsp_init(&h, bit_depth); + check_dequant(&h, bit_depth); + } + report("dequant"); +} diff --git a/tests/fate/checkasm.mak b/tests/fate/checkasm.mak index b08c1947cd..f55febfbdd 100644 --- a/tests/fate/checkasm.mak +++ b/tests/fate/checkasm.mak @@ -27,6 +27,7 @@ FATE_CHECKASM = fate-checkasm-aacencdsp \ fate-checkasm-h264qpel \ fate-checkasm-hevc_add_res \ fate-checkasm-hevc_deblock \ + fate-checkasm-hevc_dequant \ fate-checkasm-hevc_idct \ fate-checkasm-hevc_pel \ fate-checkasm-hevc_sao \ _______________________________________________ ffmpeg-cvslog mailing list -- [email protected] To unsubscribe send an email to [email protected]
