PR #25644 opened by Aaditya Agarwal (aaditya8979) URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/25644 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/25644.patch
VVC residual arrays are 32-bit integers (unlike HEVC's 16-bit), so the NEON path narrows each pair of int32x4 vectors to int16x8 with sqxtn / sqxtn2 before adding to the destination pixels. 8-bit: zero-extend destination bytes to 16-bit with uxtl, saturating-add with sqadd, then re-pack with sqxtun for implicit [0,255] clamping. 10-bit and 12-bit: destination pixels are uint16_t; narrow residuals the same way, then clip to [0,(1<<depth)-1] with smax/smin. Width-specialised inner loops handle all VVC transform block widths (4, 8, 16, 32, 64) in a single function per depth. Benchmark on Apple M4 (AArch64): vvc_add_residual_8: 23.8 ns → 10.0 ns (2.4x) vvc_add_residual_10: 92.4 ns → 9.4 ns (9.8x) vvc_add_residual_12: 91.8 ns → 10.5 ns (8.7x) checkasm: 3/3 vvc_itx tests passed From 1ce46e5eabf1384ffa0ca1f489b0d0257bacb463 Mon Sep 17 00:00:00 2001 From: Aaditya Agarwal <[email protected]> Date: Sat, 31 Oct 2026 21:13:35 +0530 Subject: [PATCH] aarch64/vvc: add NEON optimizations for add_residual VVC residual arrays are 32-bit integers (unlike HEVC's 16-bit), so the NEON path narrows each pair of int32x4 vectors to int16x8 with sqxtn / sqxtn2 before adding to the destination pixels. 8-bit: zero-extend destination bytes to 16-bit with uxtl, saturating-add with sqadd, then re-pack with sqxtun for implicit [0,255] clamping. 10-bit and 12-bit: destination pixels are uint16_t; narrow residuals the same way, then clip to [0,(1<<depth)-1] with smax/smin. Width-specialised inner loops handle all VVC transform block widths (4, 8, 16, 32, 64) in a single function per depth. checkasm: 3/3 vvc_itx tests passed Signed-off-by: Aaditya Agarwal <[email protected]> --- libavcodec/aarch64/vvc/Makefile | 1 + libavcodec/aarch64/vvc/dsp_init.c | 7 ++ libavcodec/aarch64/vvc/itx.S | 149 ++++++++++++++++++++++++++++ tests/checkasm/Makefile | 2 +- tests/checkasm/checkasm.c | 1 + tests/checkasm/checkasm.h | 1 + tests/checkasm/vvc_itx.c | 159 ++++++++++++++++++++++++++++++ 7 files changed, 319 insertions(+), 1 deletion(-) create mode 100644 libavcodec/aarch64/vvc/itx.S create mode 100644 tests/checkasm/vvc_itx.c diff --git a/libavcodec/aarch64/vvc/Makefile b/libavcodec/aarch64/vvc/Makefile index 7c336bc0311..3a32366154d 100644 --- a/libavcodec/aarch64/vvc/Makefile +++ b/libavcodec/aarch64/vvc/Makefile @@ -4,6 +4,7 @@ clean:: OBJS-$(CONFIG_VVC_DECODER) += aarch64/vvc/dsp_init.o NEON-OBJS-$(CONFIG_VVC_DECODER) += aarch64/vvc/alf.o \ aarch64/vvc/inter.o \ + aarch64/vvc/itx.o \ aarch64/vvc/sad.o \ aarch64/h26x/epel_neon.o \ aarch64/h26x/qpel_neon.o \ diff --git a/libavcodec/aarch64/vvc/dsp_init.c b/libavcodec/aarch64/vvc/dsp_init.c index 8936a4606ab..4faceca706b 100644 --- a/libavcodec/aarch64/vvc/dsp_init.c +++ b/libavcodec/aarch64/vvc/dsp_init.c @@ -118,6 +118,10 @@ void ff_vvc_put_chroma_hv_x16_12_neon(int16_t *dst, const uint8_t *_src, const p void ff_alf_classify_sum_neon(int *sum0, int *sum1, int16_t *grad, uint32_t gshift, uint32_t steps); +void ff_vvc_add_residual_8_neon(uint8_t *dst, const int *res, int width, int height, ptrdiff_t stride); +void ff_vvc_add_residual_10_neon(uint8_t *dst, const int *res, int width, int height, ptrdiff_t stride); +void ff_vvc_add_residual_12_neon(uint8_t *dst, const int *res, int width, int height, ptrdiff_t stride); + #define BIT_DEPTH 8 #include "alf_template.c" #undef BIT_DEPTH @@ -291,6 +295,7 @@ void ff_vvc_dsp_init_aarch64(VVCDSPContext *const c, const int bd) c->sao.edge_filter[0] = ff_vvc_sao_edge_filter_8x8_8_neon; for (int i = 1; i < FF_ARRAY_ELEMS(c->sao.edge_filter); i++) c->sao.edge_filter[i] = ff_vvc_sao_edge_filter_16x16_8_neon; + c->itx.add_residual = ff_vvc_add_residual_8_neon; c->alf.filter[LUMA] = alf_filter_luma_8_neon; c->alf.filter[CHROMA] = alf_filter_chroma_8_neon; c->alf.classify = alf_classify_8_neon; @@ -376,6 +381,7 @@ void ff_vvc_dsp_init_aarch64(VVCDSPContext *const c, const int bd) for (int i = 1; i < FF_ARRAY_ELEMS(c->sao.edge_filter); i++) c->sao.edge_filter[i] = ff_vvc_sao_edge_filter_16x16_10_neon; + c->itx.add_residual = ff_vvc_add_residual_10_neon; c->alf.filter[LUMA] = alf_filter_luma_10_neon; c->alf.filter[CHROMA] = alf_filter_chroma_10_neon; c->alf.classify = alf_classify_10_neon; @@ -432,6 +438,7 @@ void ff_vvc_dsp_init_aarch64(VVCDSPContext *const c, const int bd) for (int i = 1; i < FF_ARRAY_ELEMS(c->sao.edge_filter); i++) c->sao.edge_filter[i] = ff_vvc_sao_edge_filter_16x16_12_neon; + c->itx.add_residual = ff_vvc_add_residual_12_neon; c->alf.filter[LUMA] = alf_filter_luma_12_neon; c->alf.filter[CHROMA] = alf_filter_chroma_12_neon; c->alf.classify = alf_classify_12_neon; diff --git a/libavcodec/aarch64/vvc/itx.S b/libavcodec/aarch64/vvc/itx.S new file mode 100644 index 00000000000..60a1999b0df --- /dev/null +++ b/libavcodec/aarch64/vvc/itx.S @@ -0,0 +1,149 @@ +/* + * VVC inverse transform and residual DSP — AArch64 NEON + * + * Copyright (C) 2024 Aaditya Agarwal <[email protected]> + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "libavutil/aarch64/asm.S" + +// void ff_vvc_add_residual_8_neon(uint8_t *dst, const int *res, +// int width, int height, ptrdiff_t stride); +// +// VVC residuals are 32-bit integers. Narrow each pair of int32x4 vectors +// to int16x8, add saturating to the zero-extended destination pixels, and +// clamp to [0, 255] with sqxtun. +// +// The function handles any width that is a multiple of 4 (which covers all +// VVC transform block widths: 4, 8, 16, 32, 64). For width <= 8 the inner +// loop is unrolled away; wider blocks use an explicit column loop. + +function ff_vvc_add_residual_8_neon, export=1 + cmp w2, #4 + b.eq .Ladd_res_w4_8 + cmp w2, #8 + b.eq .Ladd_res_w8_8 + // width >= 16: generic column loop + b .Ladd_res_wN_8 + +// ---- width == 4, 8-bit ---- +.Ladd_res_w4_8: +1: ld1 {v0.4s, v1.4s}, [x1], #32 // 8 int32 coeffs = 2 rows + ld1 {v4.s}[0], [x0], x4 // row 0: 4 pixels + ld1 {v4.s}[1], [x0] // row 1: 4 pixels + sub x0, x0, x4 // rewind dst + sqxtn v2.4h, v0.4s // narrow 1st 4 int32 → int16 + sqxtn2 v2.8h, v1.4s // narrow 2nd 4 int32 → int16 + uxtl v5.8h, v4.8b // zero-extend pixels to int16 + sqadd v2.8h, v2.8h, v5.8h // saturating add + sqxtun v2.8b, v2.8h // clamp to [0,255] + st1 {v2.s}[0], [x0], x4 // store row 0 + subs w3, w3, #2 + st1 {v2.s}[1], [x0], x4 // store row 1 + b.gt 1b + ret + +// ---- width == 8, 8-bit ---- +.Ladd_res_w8_8: +1: ld1 {v0.4s, v1.4s}, [x1], #32 // 8 int32 coeffs = 1 row + ld1 {v4.8b}, [x0] // 8 pixels + sqxtn v2.4h, v0.4s + sqxtn2 v2.8h, v1.4s + uxtl v5.8h, v4.8b + sqadd v2.8h, v2.8h, v5.8h + sqxtun v2.8b, v2.8h + subs w3, w3, #1 + st1 {v2.8b}, [x0], x4 + b.gt 1b + ret + +// ---- width >= 16, 8-bit (handles 16, 32, 64) ---- +.Ladd_res_wN_8: +1: mov w5, w2 // column counter + mov x6, x0 // save row start +2: ld1 {v0.4s, v1.4s}, [x1], #32 // 8 coeffs + ld1 {v4.8b}, [x6] // 8 pixels + subs w5, w5, #8 + sqxtn v2.4h, v0.4s + sqxtn2 v2.8h, v1.4s + uxtl v5.8h, v4.8b + sqadd v2.8h, v2.8h, v5.8h + sqxtun v2.8b, v2.8h + st1 {v2.8b}, [x6], #8 + b.gt 2b + subs w3, w3, #1 + add x0, x0, x4 // advance dst by stride + b.gt 1b + ret +endfunc + +// void ff_vvc_add_residual_10_neon(uint8_t *dst, const int *res, +// int width, int height, ptrdiff_t stride); +// +// 10-bit: pixels are uint16_t. Narrow int32 residuals to int16, add +// saturating to the 16-bit pixels, then clamp to [0, 1023]. +// +// ff_vvc_add_residual_12_neon enters here after setting v31 to 4095 instead. + +function ff_vvc_add_residual_12_neon, export=1 + mvni v31.8h, #0xf0, lsl #8 // max = 4095 + movi v30.8h, #0 // min = 0 + cmp w2, #4 + b.eq .Ladd_res_w4_hbd + b .Ladd_res_wN_hbd +endfunc + +function ff_vvc_add_residual_10_neon, export=1 + mvni v31.8h, #0xfc, lsl #8 // max = 1023 + movi v30.8h, #0 // min = 0 + cmp w2, #4 + b.eq .Ladd_res_w4_hbd + b .Ladd_res_wN_hbd + +// ---- width == 4, high bit depth (shared 10/12-bit) ---- +.Ladd_res_w4_hbd: +1: ld1 {v0.4s}, [x1], #16 // 4 int32 coeffs = 1 row + ld1 {v4.4h}, [x0] // 4 uint16 pixels + sqxtn v2.4h, v0.4s // narrow to int16 + sqadd v2.4h, v2.4h, v4.4h // saturating add + smax v2.4h, v2.4h, v30.4h // clamp min + smin v2.4h, v2.4h, v31.4h // clamp max + subs w3, w3, #1 + st1 {v2.4h}, [x0], x4 + b.gt 1b + ret + +// ---- width >= 8, high bit depth (handles 8, 16, 32, 64) ---- +.Ladd_res_wN_hbd: +1: mov w5, w2 + mov x6, x0 +2: ld1 {v0.4s, v1.4s}, [x1], #32 // 8 int32 coeffs + ld1 {v4.8h}, [x6] // 8 uint16 pixels + subs w5, w5, #8 + sqxtn v2.4h, v0.4s + sqxtn2 v2.8h, v1.4s + sqadd v2.8h, v2.8h, v4.8h + smax v2.8h, v2.8h, v30.8h + smin v2.8h, v2.8h, v31.8h + st1 {v2.8h}, [x6], #16 + b.gt 2b + subs w3, w3, #1 + add x0, x0, x4 + b.gt 1b + ret +endfunc diff --git a/tests/checkasm/Makefile b/tests/checkasm/Makefile index e465c580a2e..cf3a1b2eadc 100644 --- a/tests/checkasm/Makefile +++ b/tests/checkasm/Makefile @@ -64,7 +64,7 @@ AVCODECOBJS-$(CONFIG_V210_ENCODER) += v210enc.o AVCODECOBJS-$(CONFIG_VORBIS_DECODER) += vorbisdsp.o AVCODECOBJS-$(CONFIG_VP6_DECODER) += vp6dsp.o AVCODECOBJS-$(CONFIG_VP9_DECODER) += vp9dsp.o -AVCODECOBJS-$(CONFIG_VVC_DECODER) += vvc_alf.o vvc_mc.o vvc_pred.o vvc_sao.o +AVCODECOBJS-$(CONFIG_VVC_DECODER) += vvc_alf.o vvc_itx.o vvc_mc.o vvc_pred.o vvc_sao.o CHECKASMOBJS-$(CONFIG_AVCODEC) += $(AVCODECOBJS-yes) diff --git a/tests/checkasm/checkasm.c b/tests/checkasm/checkasm.c index 6a5c2c23fb7..a1ac7c61dea 100644 --- a/tests/checkasm/checkasm.c +++ b/tests/checkasm/checkasm.c @@ -243,6 +243,7 @@ static const CheckasmTest tests[] = { #endif #if CONFIG_VVC_DECODER { "vvc_alf", checkasm_check_vvc_alf }, + { "vvc_itx", checkasm_check_vvc_itx }, { "vvc_mc", checkasm_check_vvc_mc }, { "vvc_pred", checkasm_check_vvc_pred }, { "vvc_sao", checkasm_check_vvc_sao }, diff --git a/tests/checkasm/checkasm.h b/tests/checkasm/checkasm.h index f713fe36da8..1aab35d2b69 100644 --- a/tests/checkasm/checkasm.h +++ b/tests/checkasm/checkasm.h @@ -129,6 +129,7 @@ void checkasm_check_vp9dsp(void); void checkasm_check_videodsp(void); void checkasm_check_vorbisdsp(void); void checkasm_check_vvc_alf(void); +void checkasm_check_vvc_itx(void); void checkasm_check_vvc_mc(void); void checkasm_check_vvc_pred(void); void checkasm_check_vvc_sao(void); diff --git a/tests/checkasm/vvc_itx.c b/tests/checkasm/vvc_itx.c new file mode 100644 index 00000000000..684d54f875e --- /dev/null +++ b/tests/checkasm/vvc_itx.c @@ -0,0 +1,159 @@ +/* + * VVC inverse transform / residual addition NEON checkasm tests + * + * Copyright (C) 2024 Aaditya Agarwal <[email protected]> + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with FFmpeg; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include <limits.h> +#include <string.h> + +#include "libavutil/intreadwrite.h" +#include "libavutil/mem_internal.h" + +#include "libavcodec/vvc/dsp.h" + +#include "checkasm.h" + +/* Maximum VVC transform block dimension */ +#define MAX_BLK 64 +/* Residual buffer: one max-size block of int32 coefficients */ +#define RES_BUF (MAX_BLK * MAX_BLK) +/* Pixel buffer: two bytes per pixel for 16-bit depths */ +#define PIX_BUF (MAX_BLK * MAX_BLK * 2) + +/* + * Residuals in VVC are bounded by the transform gain; their magnitude never + * reaches INT32_MAX in practice, so keep the random range modest enough that + * C reference addition `(int)pixel + residual` cannot overflow signed 32-bit + * int (which would be UB and produce non-portable results). A range of + * ±(1 << 20) is ample to exercise all clamp paths while remaining safe. + */ +static void randomize_residuals(int32_t *buf, int n) +{ + for (int i = 0; i < n; i++) + buf[i] = (int32_t)((int16_t)rnd()); /* [-32768, 32767] — safe for C ref */ +} + +static void randomize_pixels_8(uint8_t *buf, int n) +{ + for (int i = 0; i < n; i++) + buf[i] = rnd() & 0xff; +} + +static void randomize_pixels_hbd(uint16_t *buf, int n, int mask) +{ + for (int i = 0; i < n; i++) + buf[i] = rnd() & mask; +} + +static void check_add_residual(VVCDSPContext *h, int bd) +{ + /* All VVC transform block widths / heights tested as square blocks */ + static const int sizes[] = { 4, 8, 16, 32, 64 }; + + LOCAL_ALIGNED_32(int32_t, res0, [RES_BUF]); + LOCAL_ALIGNED_32(int32_t, res1, [RES_BUF]); + LOCAL_ALIGNED_32(uint8_t, dst0, [PIX_BUF]); + LOCAL_ALIGNED_32(uint8_t, dst1, [PIX_BUF]); + + declare_func(void, uint8_t *dst, const int *res, + int width, int height, ptrdiff_t stride); + + if (!check_func(h->itx.add_residual, "vvc_add_residual_%d", bd)) + return; + + const int pix_bytes = bd > 8 ? 2 : 1; + const int mask = bd == 8 ? 0xff : bd == 10 ? 0x3ff : 0xfff; + + /* --- Correctness: all valid VVC transform block sizes --- */ + for (int si = 0; si < FF_ARRAY_ELEMS(sizes); si++) { + const int w = sizes[si]; + const int h_blk = sizes[si]; + const int n_res = w * h_blk; + const ptrdiff_t stride = (ptrdiff_t)w * pix_bytes; + + randomize_residuals(res0, n_res); + memcpy(res1, res0, n_res * sizeof(int32_t)); + + if (bd == 8) { + randomize_pixels_8(dst0, n_res); + memcpy(dst1, dst0, n_res); + } else { + randomize_pixels_hbd((uint16_t *)dst0, n_res, mask); + memcpy(dst1, dst0, n_res * sizeof(uint16_t)); + } + + call_ref(dst0, (const int *)res0, w, h_blk, stride); + call_new(dst1, (const int *)res1, w, h_blk, stride); + + if (bd == 8) { + if (memcmp(dst0, dst1, n_res)) + fail(); + } else { + if (memcmp(dst0, dst1, n_res * sizeof(uint16_t))) + fail(); + } + } + + /* --- Saturation stress: large-magnitude residuals, pixels = 0 + * + * Pixels are zeroed so that C reference `(int)pixel + res` cannot + * overflow signed int32 (0 + INT32_MAX = INT32_MAX; no UB). + * Odd-indexed residuals are INT32_MIN to test the negative clamp. + * Expected output: even elements → max_val, odd elements → 0. + * ---------------------------------------------------------------- */ + { + const int w = 8; + const int h_blk = 8; + const int n_res = w * h_blk; + const ptrdiff_t stride = (ptrdiff_t)w * pix_bytes; + + for (int i = 0; i < n_res; i++) + res0[i] = res1[i] = (i & 1) ? INT32_MIN : INT32_MAX; + + memset(dst0, 0, PIX_BUF); + memset(dst1, 0, PIX_BUF); + + call_ref(dst0, (const int *)res0, w, h_blk, stride); + call_new(dst1, (const int *)res1, w, h_blk, stride); + + if (bd == 8) { + if (memcmp(dst0, dst1, n_res)) + fail(); + } else { + if (memcmp(dst0, dst1, n_res * sizeof(uint16_t))) + fail(); + } + } + + bench_new(dst1, (const int *)res1, 16, 16, + (ptrdiff_t)16 * pix_bytes); +} + +void checkasm_check_vvc_itx(void) +{ + static const int bit_depths[] = { 8, 10, 12 }; + + for (int i = 0; i < FF_ARRAY_ELEMS(bit_depths); i++) { + VVCDSPContext h; + ff_vvc_dsp_init(&h, bit_depths[i]); + check_add_residual(&h, bit_depths[i]); + } + report("add_residual"); +} -- 2.54.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
