PR #25382 opened by sebokgyul URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/25382 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/25382.patch
NEON add_left_pred, add_left_pred_int16 and add_gradient_pred. The first two commits fix gaps in the llviddsp checkasm test: it only tested widths of 1 modulo 128, and add_left_pred_int16 only with mask 0xFF. Tested with checkasm (1000 seeds) on X2/A710/A510 and X1/A76/A55. On both devices, the lagarith, magicyuv and utvideo FATE samples and huffyuv, ffvhuff, utvideo and magicyuv encode/decode round trips give the same output as with -cpuflags 0. The updated test also passes on x86. Benchmarks are in the commit message. >From f1912552abd592169d1c38bb4738762440d93679 Mon Sep 17 00:00:00 2001 From: Gyula Sebok <[email protected]> Date: Fri, 23 Oct 2026 02:03:13 +0200 Subject: [PATCH 1/3] tests/checkasm/llviddsp: Fix operator precedence in the random width 1 + rnd() % 16*128 is parsed as 1 + (rnd() % 16) * 128, which only gives 16 different widths, all of them 1 modulo 128. Use 1..2048, as intended in 78193e6490. Signed-off-by: Gyula Sebok <[email protected]> --- tests/checkasm/llviddsp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/checkasm/llviddsp.c b/tests/checkasm/llviddsp.c index bc1b1370919..17291452fe6 100644 --- a/tests/checkasm/llviddsp.c +++ b/tests/checkasm/llviddsp.c @@ -193,7 +193,7 @@ void checkasm_check_llviddsp(void) int accRnd = rnd() & 0xFF; size_t align = av_cpu_max_align(); - int width = 1 + rnd() % 16*128; + int width = 1 + rnd() % (16 * 128); size_t aligned_width = FFALIGN(width, align); ff_llviddsp_init(&c); -- 2.54.0 >From e4a9f0dd7b318b396e4e6c444ab5fa5a0ed4bf85 Mon Sep 17 00:00:00 2001 From: Gyula Sebok <[email protected]> Date: Fri, 23 Oct 2026 02:03:13 +0200 Subject: [PATCH 2/3] tests/checkasm/llviddsp: Test add_left_pred_int16 with more masks Only 0xFF was tested, while the decoders use masks up to 0xFFFF (HuffYUV up to 16 bits, UtVideo 10 bits, MagicYUV 10 to 14 bits). Test the masks for all bit depths from 8 to 16. Signed-off-by: Gyula Sebok <[email protected]> --- tests/checkasm/llviddsp.c | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tests/checkasm/llviddsp.c b/tests/checkasm/llviddsp.c index 17291452fe6..4fe4dd679a3 100644 --- a/tests/checkasm/llviddsp.c +++ b/tests/checkasm/llviddsp.c @@ -131,7 +131,7 @@ static void check_add_left_pred(LLVidDSPContext *c, int width, size_t aligned_wi av_free(dst1); } -static void check_add_left_pred_16(LLVidDSPContext *c, unsigned mask, int width, +static void check_add_left_pred_16(LLVidDSPContext *c, int width, size_t align, unsigned acc) { int res0, res1; @@ -149,11 +149,15 @@ static void check_add_left_pred_16(LLVidDSPContext *c, unsigned mask, int width, if (!dst0 || !dst1) fail(); - res0 = call_ref(dst0, src0, mask, width, acc); - res1 = call_new(dst1, src1, mask, width, acc); - if ((res0 & 0xFFFF) != (res1 & 0xFFFF) || memcmp(dst0, dst1, width * sizeof(*dst0))) - fail(); - bench_new(dst1, src1, mask, width, acc); + for (int bits = 8; bits <= 16; bits++) { + unsigned mask = (1U << bits) - 1; + + res0 = call_ref(dst0, src0, mask, width, acc & mask); + res1 = call_new(dst1, src1, mask, width, acc & mask); + if ((res0 & 0xFFFF) != (res1 & 0xFFFF) || memcmp(dst0, dst1, width * sizeof(*dst0))) + fail(); + } + bench_new(dst1, src1, 0xFF, width, acc & 0xFF); av_free(src0); av_free(src1); @@ -215,7 +219,7 @@ void checkasm_check_llviddsp(void) report("add_left_pred_rnd_acc"); if (check_func(c.add_left_pred_int16, "add_left_pred_int16")) - check_add_left_pred_16(&c, 255, width, align, accRnd); + check_add_left_pred_16(&c, width, align, rnd()); report("add_left_pred_int16"); if (check_func(c.add_gradient_pred, "add_gradient_pred")) -- 2.54.0 >From 3c57889e7660124c0d1ed1ee68c19695abb58012 Mon Sep 17 00:00:00 2001 From: Gyula Sebok <[email protected]> Date: Fri, 23 Oct 2026 02:30:18 +0200 Subject: [PATCH 3/3] avcodec/aarch64: Add NEON llviddsp left and gradient prediction Like the x86 versions, these may write past the end, here by up to 15 bytes. The main loops process eight vectors per iteration; with four, the in-order Cortex-A510 stalls on the latency of the prefix sums, and add_left_pred_int16 was only 1.25x faster than C there. checkasm --bench (CPU cycles via perf), seed 12345 (width 983), on a Dimensity 9000: Cortex-X2 Cortex-A710 Cortex-A510 add_gradient_pred_c: 2136.0 2183.8 4586.5 add_gradient_pred_neon: 276.9 (7.71x) 422.8 (5.16x) 818.4 (5.60x) add_left_pred_int16_c: 1926.5 1942.0 2155.9 add_left_pred_int16_neon: 371.5 (5.19x) 630.5 (3.08x) 1058.7 (2.04x) add_left_pred_rnd_acc_c: 1056.5 1033.5 1787.7 add_left_pred_rnd_acc_neon: 202.9 (5.21x) 348.0 (2.97x) 585.3 (3.05x) add_left_pred_zero_c: 1056.0 1033.7 1786.8 add_left_pred_zero_neon: 202.6 (5.21x) 347.7 (2.97x) 592.5 (3.02x) and on a Google Tensor (Pixel 6a): Cortex-X1 Cortex-A76 Cortex-A55 add_gradient_pred_c: 2137.0 2845.8 6892.3 add_gradient_pred_neon: 289.0 (7.40x) 502.3 (5.67x) 1481.8 (4.65x) add_left_pred_int16_c: 1934.9 2187.9 3950.5 add_left_pred_int16_neon: 397.6 (4.87x) 664.3 (3.29x) 1690.2 (2.34x) add_left_pred_rnd_acc_c: 1060.5 1534.2 2967.2 add_left_pred_rnd_acc_neon: 214.4 (4.95x) 378.2 (4.06x) 925.4 (3.21x) add_left_pred_zero_c: 1062.2 1534.1 2967.2 add_left_pred_zero_neon: 214.6 (4.95x) 378.2 (4.06x) 925.4 (3.21x) Signed-off-by: Gyula Sebok <[email protected]> --- libavcodec/aarch64/Makefile | 2 + libavcodec/aarch64/lossless_videodsp_init.c | 42 +++ libavcodec/aarch64/lossless_videodsp_neon.S | 278 ++++++++++++++++++++ libavcodec/lossless_videodsp.c | 4 +- libavcodec/lossless_videodsp.h | 1 + 5 files changed, 326 insertions(+), 1 deletion(-) create mode 100644 libavcodec/aarch64/lossless_videodsp_init.c create mode 100644 libavcodec/aarch64/lossless_videodsp_neon.S diff --git a/libavcodec/aarch64/Makefile b/libavcodec/aarch64/Makefile index 0fe117748f3..0f318e420e2 100644 --- a/libavcodec/aarch64/Makefile +++ b/libavcodec/aarch64/Makefile @@ -8,6 +8,7 @@ OBJS-$(CONFIG_H264PRED) += aarch64/h264pred_init.o OBJS-$(CONFIG_H264QPEL) += aarch64/h264qpel_init_aarch64.o OBJS-$(CONFIG_HPELDSP) += aarch64/hpeldsp_init_aarch64.o OBJS-$(CONFIG_IDCTDSP) += aarch64/idctdsp_init_aarch64.o +OBJS-$(CONFIG_LLVIDDSP) += aarch64/lossless_videodsp_init.o OBJS-$(CONFIG_ME_CMP) += aarch64/me_cmp_init_aarch64.o OBJS-$(CONFIG_MPEGAUDIODSP) += aarch64/mpegaudiodsp_init.o OBJS-$(CONFIG_MPEGVIDEOENCDSP) += aarch64/mpegvideoencdsp_init.o @@ -57,6 +58,7 @@ NEON-OBJS-$(CONFIG_HPELDSP) += aarch64/hpeldsp_neon.o NEON-OBJS-$(CONFIG_HUFFYUVDSP) += aarch64/huffyuvdsp_neon.o NEON-OBJS-$(CONFIG_IDCTDSP) += aarch64/idctdsp_neon.o \ aarch64/simple_idct_neon.o +NEON-OBJS-$(CONFIG_LLVIDDSP) += aarch64/lossless_videodsp_neon.o NEON-OBJS-$(CONFIG_ME_CMP) += aarch64/me_cmp_neon.o NEON-OBJS-$(CONFIG_MPEGAUDIODSP) += aarch64/mpegaudiodsp_neon.o NEON-OBJS-$(CONFIG_MPEGVIDEOENCDSP) += aarch64/mpegvideoencdsp_neon.o diff --git a/libavcodec/aarch64/lossless_videodsp_init.c b/libavcodec/aarch64/lossless_videodsp_init.c new file mode 100644 index 00000000000..2f5149795a0 --- /dev/null +++ b/libavcodec/aarch64/lossless_videodsp_init.c @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2026 Gyula Sebok <[email protected]> + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "libavutil/attributes.h" +#include "libavutil/cpu.h" +#include "libavutil/aarch64/cpu.h" +#include "libavcodec/lossless_videodsp.h" + +int ff_add_left_pred_neon(uint8_t *dst, const uint8_t *src, + ptrdiff_t w, int acc); +int ff_add_left_pred_int16_neon(uint16_t *dst, const uint16_t *src, + unsigned mask, ptrdiff_t w, unsigned acc); +void ff_add_gradient_pred_neon(uint8_t *src, const ptrdiff_t stride, + const ptrdiff_t width); + +av_cold void ff_llviddsp_init_aarch64(LLVidDSPContext *c) +{ + int cpu_flags = av_get_cpu_flags(); + + if (have_neon(cpu_flags)) { + c->add_left_pred = ff_add_left_pred_neon; + c->add_left_pred_int16 = ff_add_left_pred_int16_neon; + c->add_gradient_pred = ff_add_gradient_pred_neon; + } +} diff --git a/libavcodec/aarch64/lossless_videodsp_neon.S b/libavcodec/aarch64/lossless_videodsp_neon.S new file mode 100644 index 00000000000..8d9bcdc6224 --- /dev/null +++ b/libavcodec/aarch64/lossless_videodsp_neon.S @@ -0,0 +1,278 @@ +/* + * Copyright (c) 2026 Gyula Sebok <[email protected]> + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "libavutil/aarch64/asm.S" + +// v31 = 0, clobbers v16-v23 +.macro prefix_step sz, shift, nregs + ext v16.16b, v31.16b, v0.16b, #16 - \shift +.if \nregs >= 4 + ext v17.16b, v31.16b, v1.16b, #16 - \shift + ext v18.16b, v31.16b, v2.16b, #16 - \shift + ext v19.16b, v31.16b, v3.16b, #16 - \shift +.endif +.if \nregs == 8 + ext v20.16b, v31.16b, v4.16b, #16 - \shift + ext v21.16b, v31.16b, v5.16b, #16 - \shift + ext v22.16b, v31.16b, v6.16b, #16 - \shift + ext v23.16b, v31.16b, v7.16b, #16 - \shift +.endif + add v0.\sz, v0.\sz, v16.\sz +.if \nregs >= 4 + add v1.\sz, v1.\sz, v17.\sz + add v2.\sz, v2.\sz, v18.\sz + add v3.\sz, v3.\sz, v19.\sz +.endif +.if \nregs == 8 + add v4.\sz, v4.\sz, v20.\sz + add v5.\sz, v5.\sz, v21.\sz + add v6.\sz, v6.\sz, v22.\sz + add v7.\sz, v7.\sz, v23.\sz +.endif +.endm + +.macro prefix_sum_8 nregs + prefix_step 16b, 1, \nregs + prefix_step 16b, 2, \nregs + prefix_step 16b, 4, \nregs + prefix_step 16b, 8, \nregs +.endm + +.macro prefix_sum_16 nregs + prefix_step 8h, 2, \nregs + prefix_step 8h, 4, \nregs + prefix_step 8h, 8, \nregs +.endm + +// v0-v3 or v0-v7 += v30 + the last elements of the preceding vectors, +// v30 = the new running sum. Only v0 and v1 get v30 directly, the others +// get it through the broadcasts of v1 and v3. mask=1 also ands the +// results with v29. Clobbers v16-v23. +.macro add_carry sz, last, nregs, mask=0 + add v1.\sz, v1.\sz, v30.\sz + dup v16.\sz, v0.\last + dup v18.\sz, v2.\last +.if \nregs == 8 + dup v20.\sz, v4.\last + dup v22.\sz, v6.\last +.endif + add v0.\sz, v0.\sz, v30.\sz + add v1.\sz, v1.\sz, v16.\sz + add v3.\sz, v3.\sz, v18.\sz +.if \nregs == 8 + add v5.\sz, v5.\sz, v20.\sz + add v7.\sz, v7.\sz, v22.\sz +.endif +.if \mask + and v0.16b, v0.16b, v29.16b +.endif + dup v17.\sz, v1.\last +.if \nregs == 8 + dup v21.\sz, v5.\last +.endif +.if \mask + and v1.16b, v1.16b, v29.16b +.endif + add v2.\sz, v2.\sz, v17.\sz + add v3.\sz, v3.\sz, v17.\sz +.if \nregs == 4 + dup v30.\sz, v3.\last +.else + add v6.\sz, v6.\sz, v21.\sz + add v7.\sz, v7.\sz, v21.\sz +.if \mask + and v2.16b, v2.16b, v29.16b +.endif + dup v19.\sz, v3.\last +.if \mask + and v3.16b, v3.16b, v29.16b +.endif + add v4.\sz, v4.\sz, v19.\sz + add v5.\sz, v5.\sz, v19.\sz + add v6.\sz, v6.\sz, v19.\sz + add v7.\sz, v7.\sz, v19.\sz + dup v30.\sz, v7.\last +.endif +.if \mask +.if \nregs == 4 + and v2.16b, v2.16b, v29.16b + and v3.16b, v3.16b, v29.16b +.else + and v4.16b, v4.16b, v29.16b + and v5.16b, v5.16b, v29.16b + and v6.16b, v6.16b, v29.16b + and v7.16b, v7.16b, v29.16b +.endif +.endif +.endm + +// int ff_add_left_pred_neon(uint8_t *dst, const uint8_t *src, +// ptrdiff_t w, int acc) +function ff_add_left_pred_neon, export=1 + cmp x2, #0 + b.le 9f + add x4, x0, x2 // dst + w + movi v31.16b, #0 + dup v30.16b, w3 // running sum + subs x2, x2, #128 + b.lt 2f +1: + ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #64 + ld1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x1], #64 + prefix_sum_8 8 + add_carry 16b, b[15], 8 + subs x2, x2, #128 + st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #64 + st1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x0], #64 + b.ge 1b +2: + tbz x2, #6, 3f // the low 7 bits are w % 128 + ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #64 + prefix_sum_8 4 + add_carry 16b, b[15], 4 + st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #64 +3: + ands x2, x2, #63 + b.eq 5f +4: + ld1 {v0.16b}, [x1], #16 + prefix_sum_8 1 + add v0.16b, v0.16b, v30.16b + subs x2, x2, #16 + dup v30.16b, v0.b[15] + st1 {v0.16b}, [x0], #16 + b.gt 4b +5: + ldurb w0, [x4, #-1] + ret +9: + mov w0, w3 + ret +endfunc + +// int ff_add_left_pred_int16_neon(uint16_t *dst, const uint16_t *src, +// unsigned mask, ptrdiff_t w, unsigned acc) +function ff_add_left_pred_int16_neon, export=1 + cmp x3, #0 + b.le 9f + add x5, x0, x3, lsl #1 // dst + w + movi v31.16b, #0 + dup v29.8h, w2 // mask, (1 << bits) - 1 + dup v30.8h, w4 // running sum + subs x3, x3, #64 + b.lt 2f +1: + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x1], #64 + ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x1], #64 + prefix_sum_16 8 + add_carry 8h, h[7], 8, mask=1 + subs x3, x3, #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], #64 + b.ge 1b +2: + tbz x3, #5, 3f // the low 6 bits are w % 64 + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x1], #64 + prefix_sum_16 4 + add_carry 8h, h[7], 4, mask=1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 +3: + ands x3, x3, #31 + b.eq 5f +4: + ld1 {v0.8h}, [x1], #16 + prefix_sum_16 1 + add v0.8h, v0.8h, v30.8h + subs x3, x3, #8 + dup v30.8h, v0.h[7] + and v0.16b, v0.16b, v29.16b + st1 {v0.8h}, [x0], #16 + b.gt 4b +5: + ldurh w0, [x5, #-2] + ret +9: + mov w0, w4 + ret +endfunc + +// v0-v3 += top - top left, clobbers v16-v23 +.macro gradient4 r0, r1, r2, r3 + ld1 {v16.16b, v17.16b, v18.16b, v19.16b}, [x3], #64 + ld1 {v20.16b, v21.16b, v22.16b, v23.16b}, [x4], #64 + sub v16.16b, v16.16b, v20.16b + sub v17.16b, v17.16b, v21.16b + sub v18.16b, v18.16b, v22.16b + sub v19.16b, v19.16b, v23.16b + add \r0\().16b, \r0\().16b, v16.16b + add \r1\().16b, \r1\().16b, v17.16b + add \r2\().16b, \r2\().16b, v18.16b + add \r3\().16b, \r3\().16b, v19.16b +.endm + +// void ff_add_gradient_pred_neon(uint8_t *src, const ptrdiff_t stride, +// const ptrdiff_t width) +function ff_add_gradient_pred_neon, export=1 + cmp x2, #0 + b.le 9f + sub x3, x0, x1 // top + sub x4, x3, #1 // top left + mov x5, x0 // read pointer + ldurb w6, [x0, #-1] + movi v31.16b, #0 + dup v30.16b, w6 // left + subs x2, x2, #128 + b.lt 2f +1: + ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x5], #64 + gradient4 v0, v1, v2, v3 + ld1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x5], #64 + gradient4 v4, v5, v6, v7 + prefix_sum_8 8 + add_carry 16b, b[15], 8 + subs x2, x2, #128 + st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #64 + st1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x0], #64 + b.ge 1b +2: + tbz x2, #6, 3f + ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x5], #64 + gradient4 v0, v1, v2, v3 + prefix_sum_8 4 + add_carry 16b, b[15], 4 + st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #64 +3: + ands x2, x2, #63 + b.eq 9f +4: + ld1 {v0.16b}, [x5], #16 + ld1 {v16.16b}, [x3], #16 + ld1 {v20.16b}, [x4], #16 + add v0.16b, v0.16b, v16.16b + sub v0.16b, v0.16b, v20.16b + prefix_sum_8 1 + add v0.16b, v0.16b, v30.16b + subs x2, x2, #16 + dup v30.16b, v0.b[15] + st1 {v0.16b}, [x0], #16 + b.gt 4b +9: + ret +endfunc diff --git a/libavcodec/lossless_videodsp.c b/libavcodec/lossless_videodsp.c index bf3a3da90db..ffb6f8a5558 100644 --- a/libavcodec/lossless_videodsp.c +++ b/libavcodec/lossless_videodsp.c @@ -120,7 +120,9 @@ av_cold void ff_llviddsp_init(LLVidDSPContext *c) c->add_left_pred_int16 = add_left_pred_int16_c; c->add_gradient_pred = add_gradient_pred_c; -#if ARCH_PPC +#if ARCH_AARCH64 + ff_llviddsp_init_aarch64(c); +#elif ARCH_PPC ff_llviddsp_init_ppc(c); #elif ARCH_RISCV ff_llviddsp_init_riscv(c); diff --git a/libavcodec/lossless_videodsp.h b/libavcodec/lossless_videodsp.h index 5309ce4be77..f28a3824e6f 100644 --- a/libavcodec/lossless_videodsp.h +++ b/libavcodec/lossless_videodsp.h @@ -40,6 +40,7 @@ typedef struct LLVidDSPContext { } LLVidDSPContext; void ff_llviddsp_init(LLVidDSPContext *llviddsp); +void ff_llviddsp_init_aarch64(LLVidDSPContext *llviddsp); void ff_llviddsp_init_riscv(LLVidDSPContext *llviddsp); void ff_llviddsp_init_x86(LLVidDSPContext *llviddsp); void ff_llviddsp_init_ppc(LLVidDSPContext *llviddsp); -- 2.54.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
