PR #24924 opened by aaditya8979 URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24924 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24924.patch
Two architectural changes eliminate the scalar classification bottleneck: 1. Populate ff_alf_classify_grad_10_neon: the 10-bit gradient stub was empty, causing the entire gradient pass to fall back to C. 2. Replace scalar alf_get_idx with ff_alf_get_idx_neon: cuts 512 scalar calls per 128x128 CTU to a single branchless NEON routine. Benchmarked on Apple M4 (checkasm --test=vvc_alf --bench): classify 8x8 / 10-bit: 717.7 ns -> 41.0 ns (17.49x) classify 128x128 / 12-bit: 111856 ns -> 6401 ns (17.47x) All 4032 checkasm tests pass across 8/10/12-bit. From fd0d582294be110319e4f92e446275631b5d13c4 Mon Sep 17 00:00:00 2001 From: Aaditya Agarwal <[email protected]> Date: Tue, 29 Sep 2026 03:51:35 +0530 Subject: [PATCH 1/9] avdevice/avfoundation: fix potential NULL dereference in avf_read_packet Signed-off-by: Your Name <[email protected]> --- libavdevice/avfoundation.m | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m index 8b7e728279..8fcd39b027 100644 --- a/libavdevice/avfoundation.m +++ b/libavdevice/avfoundation.m @@ -1439,9 +1439,15 @@ static int avf_read_packet(AVFormatContext *s, AVPacket *pkt) } } else if (ctx->current_audio_frame != nil) { CMBlockBufferRef block_buffer = CMSampleBufferGetDataBuffer(ctx->current_audio_frame); - int block_buffer_size = CMBlockBufferGetDataLength(block_buffer); - if (!block_buffer || !block_buffer_size) { + if (!block_buffer) { + unlock_frames(ctx); + return AVERROR(EIO); + } + + int block_buffer_size = CMBlockBufferGetDataLength(block_buffer); + + if (!block_buffer_size) { unlock_frames(ctx); return AVERROR(EIO); } -- 2.52.0 From 8229bd36d9f385750d57cb31100fcc035169ae44 Mon Sep 17 00:00:00 2001 From: Aaditya Agarwal <[email protected]> Date: Tue, 29 Sep 2026 03:51:35 +0530 Subject: [PATCH 2/9] avdevice/avfoundation: remove variable-length arrays Replaces CGDirectDisplayID VLAs with heap allocations via av_malloc_array() to comply with FFmpeg's strict no-VLA policy. Signed-off-by: Your Name <[email protected]> --- libavdevice/avfoundation.m | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m index 8fcd39b027..e9ca8ec0c7 100644 --- a/libavdevice/avfoundation.m +++ b/libavdevice/avfoundation.m @@ -1118,11 +1118,14 @@ static int avf_read_header(AVFormatContext *s) } #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070 if (num_screens > 0) { - CGDirectDisplayID screens[num_screens]; + CGDirectDisplayID *screens = av_malloc_array(num_screens, sizeof(*screens)); + if (!screens) + goto fail; CGGetActiveDisplayList(num_screens, screens, &num_screens); for (int i = 0; i < num_screens; i++) { av_log(ctx, AV_LOG_INFO, "[%d] Capture screen %d\n", ctx->num_video_devices + i, i); } + av_freep(&screens); } #endif @@ -1162,9 +1165,12 @@ static int avf_read_header(AVFormatContext *s) } } else if (ctx->video_device_index < ctx->num_video_devices + num_screens) { #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070 - CGDirectDisplayID screens[num_screens]; + CGDirectDisplayID *screens = av_malloc_array(num_screens, sizeof(*screens)); + if (!screens) + goto fail; CGGetActiveDisplayList(num_screens, screens, &num_screens); AVCaptureScreenInput* capture_screen_input = [[[AVCaptureScreenInput alloc] initWithDisplayID:screens[ctx->video_device_index - ctx->num_video_devices]] autorelease]; + av_freep(&screens); if (ctx->framerate.num > 0) { capture_screen_input.minFrameDuration = CMTimeMake(ctx->framerate.den, ctx->framerate.num); @@ -1217,9 +1223,12 @@ static int avf_read_header(AVFormatContext *s) if (!video_device) { int idx; if(sscanf(ctx->video_filename, "Capture screen %d", &idx) && idx < num_screens) { - CGDirectDisplayID screens[num_screens]; + CGDirectDisplayID *screens = av_malloc_array(num_screens, sizeof(*screens)); + if (!screens) + goto fail; CGGetActiveDisplayList(num_screens, screens, &num_screens); AVCaptureScreenInput* capture_screen_input = [[[AVCaptureScreenInput alloc] initWithDisplayID:screens[idx]] autorelease]; + av_freep(&screens); video_device = (AVCaptureDevice*) capture_screen_input; ctx->video_device_index = ctx->num_video_devices + idx; ctx->video_is_screen = 1; -- 2.52.0 From a3f4cbc39bd31b15cfd373f5c244286caf1de543 Mon Sep 17 00:00:00 2001 From: Aaditya Agarwal <[email protected]> Date: Tue, 29 Sep 2026 13:58:32 +0530 Subject: [PATCH 3/9] avcodec/sanm: replace av_assert0 with proper error return in process_ftch av_assert0 unconditionally calls abort() when the condition is false, which violates FFmpeg's contract that decoders must not crash the host process on malformed input. In process_ftch, the size == 12 branch asserted that at least 12 bytes remain in the GetByteContext before issuing unchecked reads. The outer chunk dispatcher (decode_anim loop) guards bytes-left against the chunk size before calling process_ftch, but that guard uses break rather than return on failure, meaning execution can reach process_ftch with an already-exhausted buffer under specific truncation patterns. A fuzzer-supplied packet that triggers this path would abort the entire host process. Replace the assert with a standard bounds check, an AV_LOG_ERROR diagnostic, and return AVERROR_INVALIDDATA so callers can handle the condition gracefully. Signed-off-by: Aaditya Agarwal <[email protected]> --- libavcodec/sanm.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libavcodec/sanm.c b/libavcodec/sanm.c index 749d003a1b..ef6c331bf6 100644 --- a/libavcodec/sanm.c +++ b/libavcodec/sanm.c @@ -2173,7 +2173,10 @@ static int process_ftch(SANMVideoContext *ctx, int size) xoff = bytestream2_get_le16u(&ctx->gb); yoff = bytestream2_get_le16u(&ctx->gb); } else if (size == 12) { - av_assert0(bytestream2_get_bytes_left(&ctx->gb) >= 12); + if (bytestream2_get_bytes_left(&ctx->gb) < 12) { + av_log(ctx->avctx, AV_LOG_ERROR, "Truncated FTCH chunk.\n"); + return AVERROR_INVALIDDATA; + } bytestream2_skip(&ctx->gb, 4); xoff = bytestream2_get_be32u(&ctx->gb); yoff = bytestream2_get_be32u(&ctx->gb); -- 2.52.0 From c15e0a899bb0005efd0480044c73e2a4387a5c99 Mon Sep 17 00:00:00 2001 From: Aaditya Agarwal <[email protected]> Date: Tue, 29 Sep 2026 13:59:03 +0530 Subject: [PATCH 4/9] avcodec/sanm: validate chunk size fits in signed int in decode_anim The chunk size field is read as uint32_t from the bitstream. Values >= 0x80000000 bypass the existing bytestream2_get_bytes_left() guard (which compares int < uint32_t, and thus promotes correctly), but then truncate to a negative int when passed to bytestream2_init() as its buf_size argument. bytestream2_init() asserts buf_size >= 0 via av_assert0(), which calls abort() on failure, crashing the host process on a malformed packet. The same truncated negative value reaches process_xpal() and process_ftch() as their int size parameter. While both happen to handle negative size safely today, the invariant is fragile and undocumented. Reject any chunk size that exceeds INT_MAX before it reaches any signed-int context. Signed-off-by: Aaditya Agarwal <[email protected]> --- libavcodec/sanm.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libavcodec/sanm.c b/libavcodec/sanm.c index ef6c331bf6..9e911bc383 100644 --- a/libavcodec/sanm.c +++ b/libavcodec/sanm.c @@ -2804,6 +2804,10 @@ static int decode_anim(AVCodecContext *avctx, int *got_frame_ptr) size = bytestream2_get_be32u(&ctx->gb); pos = bytestream2_tell(&ctx->gb); + if (size > INT_MAX) { + av_log(avctx, AV_LOG_ERROR, "Chunk size %"PRIu32" exceeds valid range.\n", size); + return AVERROR_INVALIDDATA; + } if (bytestream2_get_bytes_left(&ctx->gb) < size) { av_log(avctx, AV_LOG_ERROR, "Incorrect chunk size %"PRIu32".\n", size); break; -- 2.52.0 From 517c21e40cc4ad1375571c317c22072d2739df7c Mon Sep 17 00:00:00 2001 From: Aaditya Agarwal <[email protected]> Date: Wed, 30 Sep 2026 01:37:21 +0530 Subject: [PATCH 5/9] aarch64/vvc: add NEON optimizations for 10/12-bit put_uni_pixels This introduces pure NEON strided memory copies for integer-pel unidirectional prediction at 10-bit and 12-bit depths, covering block sizes from 4x4 up to 128x128. Since 10-bit and 12-bit pixels are both 16-bit wide, the 12-bit implementations simply alias the 10-bit routines. Checkasm benchmarks on Apple M-series: put_uni_pixels_luma_10_4x4_c: 8.7 put_uni_pixels_luma_10_4x4_neon: 0.2 (36.0x) put_uni_pixels_luma_10_8x8_c: 17.8 put_uni_pixels_luma_10_8x8_neon: 1.1 (16.6x) put_uni_pixels_luma_10_16x16_c: 46.1 put_uni_pixels_luma_10_16x16_neon: 5.4 ( 8.5x) put_uni_pixels_luma_10_32x32_c: 52.4 put_uni_pixels_luma_10_32x32_neon: 15.4 ( 3.4x) put_uni_pixels_luma_10_64x64_c: 101.5 put_uni_pixels_luma_10_64x64_neon: 64.6 ( 1.6x) Signed-off-by: Aaditya Agarwal <[email protected]> --- libavcodec/aarch64/vvc/dsp_init.c | 46 +++++++++++ libavcodec/aarch64/vvc/inter.S | 132 ++++++++++++++++++++++++++++++ 2 files changed, 178 insertions(+) diff --git a/libavcodec/aarch64/vvc/dsp_init.c b/libavcodec/aarch64/vvc/dsp_init.c index 8936a4606a..6415f5e947 100644 --- a/libavcodec/aarch64/vvc/dsp_init.c +++ b/libavcodec/aarch64/vvc/dsp_init.c @@ -130,6 +130,26 @@ void ff_alf_classify_sum_neon(int *sum0, int *sum1, int16_t *grad, uint32_t gshi #include "alf_template.c" #undef BIT_DEPTH +#define PUT_UNI_PIXELS_PROTO(n, bd) \ +void ff_vvc_put_pel_uni_pixels ## n ## _ ## bd ## _neon( \ + uint8_t *dst, ptrdiff_t dst_stride, \ + const uint8_t *src, ptrdiff_t src_stride, \ + int height, const int8_t *hf, const int8_t *vf, int width); + +PUT_UNI_PIXELS_PROTO(4, 10) +PUT_UNI_PIXELS_PROTO(8, 10) +PUT_UNI_PIXELS_PROTO(16, 10) +PUT_UNI_PIXELS_PROTO(32, 10) +PUT_UNI_PIXELS_PROTO(64, 10) +PUT_UNI_PIXELS_PROTO(128, 10) +PUT_UNI_PIXELS_PROTO(4, 12) +PUT_UNI_PIXELS_PROTO(8, 12) +PUT_UNI_PIXELS_PROTO(16, 12) +PUT_UNI_PIXELS_PROTO(32, 12) +PUT_UNI_PIXELS_PROTO(64, 12) +PUT_UNI_PIXELS_PROTO(128, 12) +#undef PUT_UNI_PIXELS_PROTO + int ff_vvc_sad_neon(const int16_t *src0, const int16_t *src1, int dx, int dy, const int block_w, const int block_h); @@ -334,6 +354,19 @@ void ff_vvc_dsp_init_aarch64(VVCDSPContext *const c, const int bd) c->inter.dmvr[1][1] = ff_vvc_dmvr_hv_10_neon; c->inter.apply_bdof = ff_vvc_apply_bdof_10_neon; + c->inter.put_uni[0][1][0][0] = ff_vvc_put_pel_uni_pixels4_10_neon; + c->inter.put_uni[0][2][0][0] = ff_vvc_put_pel_uni_pixels8_10_neon; + c->inter.put_uni[0][3][0][0] = ff_vvc_put_pel_uni_pixels16_10_neon; + c->inter.put_uni[0][4][0][0] = ff_vvc_put_pel_uni_pixels32_10_neon; + c->inter.put_uni[0][5][0][0] = ff_vvc_put_pel_uni_pixels64_10_neon; + c->inter.put_uni[0][6][0][0] = ff_vvc_put_pel_uni_pixels128_10_neon; + c->inter.put_uni[1][1][0][0] = ff_vvc_put_pel_uni_pixels4_10_neon; + c->inter.put_uni[1][2][0][0] = ff_vvc_put_pel_uni_pixels8_10_neon; + c->inter.put_uni[1][3][0][0] = ff_vvc_put_pel_uni_pixels16_10_neon; + c->inter.put_uni[1][4][0][0] = ff_vvc_put_pel_uni_pixels32_10_neon; + c->inter.put_uni[1][5][0][0] = ff_vvc_put_pel_uni_pixels64_10_neon; + c->inter.put_uni[1][6][0][0] = ff_vvc_put_pel_uni_pixels128_10_neon; + c->inter.put[1][2][0][1] = ff_vvc_put_chroma_h8_10_neon; c->inter.put[1][3][0][1] = ff_vvc_put_chroma_h16_10_neon; c->inter.put[1][4][0][1] = @@ -390,6 +423,19 @@ void ff_vvc_dsp_init_aarch64(VVCDSPContext *const c, const int bd) c->inter.dmvr[1][1] = ff_vvc_dmvr_hv_12_neon; c->inter.apply_bdof = ff_vvc_apply_bdof_12_neon; + c->inter.put_uni[0][1][0][0] = ff_vvc_put_pel_uni_pixels4_12_neon; + c->inter.put_uni[0][2][0][0] = ff_vvc_put_pel_uni_pixels8_12_neon; + c->inter.put_uni[0][3][0][0] = ff_vvc_put_pel_uni_pixels16_12_neon; + c->inter.put_uni[0][4][0][0] = ff_vvc_put_pel_uni_pixels32_12_neon; + c->inter.put_uni[0][5][0][0] = ff_vvc_put_pel_uni_pixels64_12_neon; + c->inter.put_uni[0][6][0][0] = ff_vvc_put_pel_uni_pixels128_12_neon; + c->inter.put_uni[1][1][0][0] = ff_vvc_put_pel_uni_pixels4_12_neon; + c->inter.put_uni[1][2][0][0] = ff_vvc_put_pel_uni_pixels8_12_neon; + c->inter.put_uni[1][3][0][0] = ff_vvc_put_pel_uni_pixels16_12_neon; + c->inter.put_uni[1][4][0][0] = ff_vvc_put_pel_uni_pixels32_12_neon; + c->inter.put_uni[1][5][0][0] = ff_vvc_put_pel_uni_pixels64_12_neon; + c->inter.put_uni[1][6][0][0] = ff_vvc_put_pel_uni_pixels128_12_neon; + c->inter.put[1][2][0][1] = ff_vvc_put_chroma_h8_12_neon; c->inter.put[1][3][0][1] = ff_vvc_put_chroma_h16_12_neon; c->inter.put[1][4][0][1] = diff --git a/libavcodec/aarch64/vvc/inter.S b/libavcodec/aarch64/vvc/inter.S index 318d33cc47..b68f712227 100644 --- a/libavcodec/aarch64/vvc/inter.S +++ b/libavcodec/aarch64/vvc/inter.S @@ -3127,3 +3127,135 @@ endfunc function ff_vvc_put_chroma_hv_x16_12_neon, export=1 put_chroma_hv_x16_xx_neon 4 endfunc + +// ============================================================ +// put_pel_uni_pixels: integer-pel unidirectional copy for +// 10-bit and 12-bit depths. These are pure strided memcpy +// functions — no arithmetic, no saturation. Pixel width is +// in 16-bit units (uint16_t), so a width-N block occupies +// 2N bytes per row. +// +// Function signature (from inter_template.c FUNC(put_uni_pixels)): +// void ff_vvc_put_pel_uni_pixelsN_BD_neon( +// uint8_t *dst, // x0 (uint16_t* aliased) +// ptrdiff_t dst_stride, // x1 (bytes) +// const uint8_t *src, // x2 (uint16_t* aliased) +// ptrdiff_t src_stride, // x3 (bytes) +// int height, // w4 +// intptr_t mx, // x5 (unused) +// intptr_t my, // x6 (unused) +// int width) // w7 (unused — fixed per function) +// +// Both 10-bit and 12-bit are identical in memory layout (uint16_t), +// so a single set of functions covers both depths. +// ============================================================ + +// ---- 4-pixel wide (8 bytes/row) ---- d register per row ---- +// Process 2 rows per iteration to hide load-use latency. +function ff_vvc_put_pel_uni_pixels4_10_neon, export=1 +1: + ldr d0, [x2] + ldr d1, [x2, x3] + subs w4, w4, #2 + add x2, x2, x3, lsl #1 + str d0, [x0] + str d1, [x0, x1] + add x0, x0, x1, lsl #1 + b.hi 1b + ret +endfunc + +function ff_vvc_put_pel_uni_pixels4_12_neon, export=1 + b X(ff_vvc_put_pel_uni_pixels4_10_neon) +endfunc + +// ---- 8-pixel wide (16 bytes/row) ---- q register per row ---- +function ff_vvc_put_pel_uni_pixels8_10_neon, export=1 +1: + ldr q0, [x2] + ldr q1, [x2, x3] + subs w4, w4, #2 + add x2, x2, x3, lsl #1 + str q0, [x0] + str q1, [x0, x1] + add x0, x0, x1, lsl #1 + b.hi 1b + ret +endfunc + +function ff_vvc_put_pel_uni_pixels8_12_neon, export=1 + b X(ff_vvc_put_pel_uni_pixels8_10_neon) +endfunc + +// ---- 16-pixel wide (32 bytes/row) ---- 2 q registers per row ---- +function ff_vvc_put_pel_uni_pixels16_10_neon, export=1 +1: + ld1 {v0.8h, v1.8h}, [x2], x3 + subs w4, w4, #1 + st1 {v0.8h, v1.8h}, [x0], x1 + b.hi 1b + ret +endfunc + +function ff_vvc_put_pel_uni_pixels16_12_neon, export=1 + b X(ff_vvc_put_pel_uni_pixels16_10_neon) +endfunc + +// ---- 32-pixel wide (64 bytes/row) ---- 4 q registers per row ---- +function ff_vvc_put_pel_uni_pixels32_10_neon, export=1 +1: + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x2], x3 + subs w4, w4, #1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + b.hi 1b + ret +endfunc + +function ff_vvc_put_pel_uni_pixels32_12_neon, export=1 + b X(ff_vvc_put_pel_uni_pixels32_10_neon) +endfunc + +// ---- 64-pixel wide (128 bytes/row) ---- 2 x 4 q regs per row ---- +// 128 bytes exceeds the 4-register ld1 limit (64 bytes), so split +// each row into two 64-byte ld1/st1 pairs using post-index #64. +function ff_vvc_put_pel_uni_pixels64_10_neon, export=1 + sub x1, x1, #64 + sub x3, x3, #64 +1: + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x2], #64 + ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x2], x3 + subs w4, w4, #1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], x1 + b.hi 1b + ret +endfunc + +function ff_vvc_put_pel_uni_pixels64_12_neon, export=1 + b X(ff_vvc_put_pel_uni_pixels64_10_neon) +endfunc + +// ---- 128-pixel wide (256 bytes/row) ---- 4 x 4 q regs per row ---- +// Each 256-byte row requires four 64-byte ld1/st1 groups. +// Adjust src/dst strides to compensate for the three post-increments +// of #64 already applied within the row. +function ff_vvc_put_pel_uni_pixels128_10_neon, export=1 + sub x1, x1, #192 // dst_stride -= 3*64 + sub x3, x3, #192 // src_stride -= 3*64 +1: + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x2], #64 + ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x2], #64 + subs w4, w4, #1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], #64 + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x2], #64 + ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x2], x3 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], x1 + b.hi 1b + ret +endfunc + +function ff_vvc_put_pel_uni_pixels128_12_neon, export=1 + b X(ff_vvc_put_pel_uni_pixels128_10_neon) +endfunc -- 2.52.0 From e590d2d795d82a914f3567307f5cfe350282e76e Mon Sep 17 00:00:00 2001 From: Aaditya Agarwal <[email protected]> Date: Wed, 30 Sep 2026 02:06:56 +0530 Subject: [PATCH 6/9] aarch64/vvc: add NEON optimizations for 10/12-bit put_uni_h_luma Add ARMv8.0-A baseline NEON implementations of the 8-tap unidirectional horizontal luma interpolation filter (put_uni_h_luma) for 10-bit and 12-bit VVC inter prediction. Implementation strategy: - Filter coefficients are loaded via a single 'ldr d0 / sxtl v0.8h' sequence, broadcasting all 8 int8_t taps into v0.h[0..7] with zero C-side overhead. - The src pointer is decremented by LUMA_EXTRA_BEFORE*sizeof(uint16_t) = 6 bytes in each function prolog to align tap[0] with src[x-3], matching the LUMA_FILTER macro's reference addressing. - Accumulation uses 'smull / smlal' into 32-bit .4s lanes to prevent overflow (10-bit pixel * 8-tap coefficient sum can exceed int16_t). - 'smlal2' is used alongside 'smlal' for 8-pixel-wide and wider paths, processing the lower and upper halves of each ext result simultaneously without duplicating the sliding window computation. - The 4-pixel-wide path dual-issues two rows per iteration (v30/v31 accumulators) to hide smlal RAW latency on in-order cores. - Saturation pipeline: add rounding offset (32), sshr by 6, sqxtun to narrow to 16-bit (floors negatives to 0), umin to clip to depth max. 10-bit: umin 1023 (0x03FF). 12-bit: umin 4095 (0x0FFF). - Widths 32/64/128 use an inner horizontal loop over 8-pixel chunks; the outer vertical counter is decremented after the inner loop exits so that the final 'subs w4 / b.hi' checks the correct flags. - 12-bit functions are fully independent (not aliased to 10-bit) to ensure the correct saturation ceiling is applied; aliasing to the 10-bit path would destructively clip HDR blocks to 1023. checkasm bench (Apple M-series, AArch64, cycles): 10-bit: put_uni_h_luma_10_4x4: 13.1 -> 4.3 (3.00x) put_uni_h_luma_10_8x8: 38.0 -> 15.9 (2.40x) put_uni_h_luma_10_16x16: 119.6 -> 65.0 (1.84x) put_uni_h_luma_10_32x32: 415.1 -> 261.1 (1.59x) put_uni_h_luma_10_64x64: 1531.8 -> 1045.7 (1.46x) put_uni_h_luma_10_128x128: 5955.3 -> 4335.3 (1.36x) 12-bit: put_uni_h_luma_12_4x4: 13.1 -> 4.3 (3.00x) put_uni_h_luma_12_8x8: 38.0 -> 15.9 (2.40x) put_uni_h_luma_12_16x16: 119.5 -> 64.4 (1.78x) put_uni_h_luma_12_32x32: 415.8 -> 261.2 (1.59x) put_uni_h_luma_12_64x64: 1532.0 -> 1045.8 (1.46x) checkasm: all 1463 vvc_mc tests passed. Signed-off-by: Aaditya Agarwal <[email protected]> Signed-off-by: Aaditya Agarwal <[email protected]> --- libavcodec/aarch64/vvc/dsp_init.c | 34 +++ libavcodec/aarch64/vvc/inter.S | 398 ++++++++++++++++++++++++++++++ 2 files changed, 432 insertions(+) diff --git a/libavcodec/aarch64/vvc/dsp_init.c b/libavcodec/aarch64/vvc/dsp_init.c index 6415f5e947..1b818a54e4 100644 --- a/libavcodec/aarch64/vvc/dsp_init.c +++ b/libavcodec/aarch64/vvc/dsp_init.c @@ -150,6 +150,26 @@ PUT_UNI_PIXELS_PROTO(64, 12) PUT_UNI_PIXELS_PROTO(128, 12) #undef PUT_UNI_PIXELS_PROTO +#define PUT_UNI_H_LUMA_PROTO(n, bd) \ +void ff_vvc_put_uni_pixels_h_luma_ ## n ## _ ## bd ## _neon( \ + uint8_t *dst, ptrdiff_t dst_stride, \ + const uint8_t *src, ptrdiff_t src_stride, \ + int height, const int8_t *hf, const int8_t *vf, int width); + +PUT_UNI_H_LUMA_PROTO(4, 10) +PUT_UNI_H_LUMA_PROTO(8, 10) +PUT_UNI_H_LUMA_PROTO(16, 10) +PUT_UNI_H_LUMA_PROTO(32, 10) +PUT_UNI_H_LUMA_PROTO(64, 10) +PUT_UNI_H_LUMA_PROTO(128, 10) +PUT_UNI_H_LUMA_PROTO(4, 12) +PUT_UNI_H_LUMA_PROTO(8, 12) +PUT_UNI_H_LUMA_PROTO(16, 12) +PUT_UNI_H_LUMA_PROTO(32, 12) +PUT_UNI_H_LUMA_PROTO(64, 12) +PUT_UNI_H_LUMA_PROTO(128, 12) +#undef PUT_UNI_H_LUMA_PROTO + int ff_vvc_sad_neon(const int16_t *src0, const int16_t *src1, int dx, int dy, const int block_w, const int block_h); @@ -367,6 +387,13 @@ void ff_vvc_dsp_init_aarch64(VVCDSPContext *const c, const int bd) c->inter.put_uni[1][5][0][0] = ff_vvc_put_pel_uni_pixels64_10_neon; c->inter.put_uni[1][6][0][0] = ff_vvc_put_pel_uni_pixels128_10_neon; + c->inter.put_uni[0][1][0][1] = ff_vvc_put_uni_pixels_h_luma_4_10_neon; + c->inter.put_uni[0][2][0][1] = ff_vvc_put_uni_pixels_h_luma_8_10_neon; + c->inter.put_uni[0][3][0][1] = ff_vvc_put_uni_pixels_h_luma_16_10_neon; + c->inter.put_uni[0][4][0][1] = ff_vvc_put_uni_pixels_h_luma_32_10_neon; + c->inter.put_uni[0][5][0][1] = ff_vvc_put_uni_pixels_h_luma_64_10_neon; + c->inter.put_uni[0][6][0][1] = ff_vvc_put_uni_pixels_h_luma_128_10_neon; + c->inter.put[1][2][0][1] = ff_vvc_put_chroma_h8_10_neon; c->inter.put[1][3][0][1] = ff_vvc_put_chroma_h16_10_neon; c->inter.put[1][4][0][1] = @@ -436,6 +463,13 @@ void ff_vvc_dsp_init_aarch64(VVCDSPContext *const c, const int bd) c->inter.put_uni[1][5][0][0] = ff_vvc_put_pel_uni_pixels64_12_neon; c->inter.put_uni[1][6][0][0] = ff_vvc_put_pel_uni_pixels128_12_neon; + c->inter.put_uni[0][1][0][1] = ff_vvc_put_uni_pixels_h_luma_4_12_neon; + c->inter.put_uni[0][2][0][1] = ff_vvc_put_uni_pixels_h_luma_8_12_neon; + c->inter.put_uni[0][3][0][1] = ff_vvc_put_uni_pixels_h_luma_16_12_neon; + c->inter.put_uni[0][4][0][1] = ff_vvc_put_uni_pixels_h_luma_32_12_neon; + c->inter.put_uni[0][5][0][1] = ff_vvc_put_uni_pixels_h_luma_64_12_neon; + c->inter.put_uni[0][6][0][1] = ff_vvc_put_uni_pixels_h_luma_128_12_neon; + c->inter.put[1][2][0][1] = ff_vvc_put_chroma_h8_12_neon; c->inter.put[1][3][0][1] = ff_vvc_put_chroma_h16_12_neon; c->inter.put[1][4][0][1] = diff --git a/libavcodec/aarch64/vvc/inter.S b/libavcodec/aarch64/vvc/inter.S index b68f712227..3410378fb3 100644 --- a/libavcodec/aarch64/vvc/inter.S +++ b/libavcodec/aarch64/vvc/inter.S @@ -3259,3 +3259,401 @@ endfunc function ff_vvc_put_pel_uni_pixels128_12_neon, export=1 b X(ff_vvc_put_pel_uni_pixels128_10_neon) endfunc + +// ============================================================ +// put_uni_luma_h: 8-tap horizontal luma filter for 10-bit and +// 12-bit VVC inter prediction. ARMv8.0-A baseline: uses +// smlal / smlal2 exclusively (no sdot). +// +// Arithmetic: +// C reference: val = LUMA_FILTER(src, 1) >> (BIT_DEPTH - 8) +// out = av_clip_pixel((val + offset) >> shift) +// 10-bit: shift=4, offset=8 → combined: (accum + 32) >> 6 +// 12-bit: shift=2, offset=2 → combined: (accum + 8) >> (BIT_DEPTH-8+shift) +// = (accum + 8) >> 6 (same total: 2+4 for 10-bit; BIT_DEPTH-8+14-BD) +// For both 10-bit and 12-bit: total right-shift = 6, rounding = 32. +// +// Register map: +// x0: dst x1: dst_stride x2: src x3: src_stride +// w4: height x5: hf (filter ptr) x6: vf (unused) w7: width +// v0.8h: 8 filter taps [0..7] (sign-extended from int8_t) +// v1.4s: rounding offset vector {32, 32, 32, 32} +// v2.4h: clip max vector {1023, 1023, 1023, 1023} +// +// 4-pixel functions: +// v16.8h + v17.8h: row N source (16 pixels = 32 bytes covers 11-pixel window) +// v18.8h + v19.8h: row N+1 source +// v30.4s: row N accumulator v31.4s: row N+1 accumulator +// +// 8-pixel and wider functions: +// One ld1 pair covers the 15-pixel window needed for an 8-wide output. +// smlal → lower 4 lanes (.4h of v20), smlal2 → upper 4 lanes (.8h of v20) +// This folds two 4-wide accumulators into one 8-wide pass per ext result. +// v30.4s: lower-half accum v31.4s: upper-half accum (→ one output row) +// ============================================================ + +// ----------------------------------------------------------------- +// Shared inner macro: process one tap for 4-wide dual-row pipeline +// ext_byte: byte offset for ext (2 * tap_index, since pixels are .h) +// ----------------------------------------------------------------- +.macro h_luma_4_tap src0_a, src1_a, src0_b, src1_b, acc_a, acc_b, lane, ext_byte + ext v20.16b, \src0_a\().16b, \src1_a\().16b, #\ext_byte + ext v21.16b, \src0_b\().16b, \src1_b\().16b, #\ext_byte + smlal \acc_a\().4s, v20.4h, v0.h[\lane] + smlal \acc_b\().4s, v21.4h, v0.h[\lane] +.endm + +// Shared inner macro: process one tap for 8-wide single-row pipeline +.macro h_luma_8_tap src0, src1, acc_lo, acc_hi, lane, ext_byte + ext v20.16b, \src0\().16b, \src1\().16b, #\ext_byte + smlal \acc_lo\().4s, v20.4h, v0.h[\lane] + smlal2 \acc_hi\().4s, v20.8h, v0.h[\lane] +.endm + +// Saturate, clip, store 4 pixels from a .4s accumulator +.macro h_luma_store_4 acc, dst + add \acc\().4s, \acc\().4s, v1.4s + sshr \acc\().4s, \acc\().4s, #6 + sqxtun \acc\().4h, \acc\().4s + umin \acc\().4h, \acc\().4h, v2.4h + st1 {\acc\().4h}, [\dst] +.endm + +// Saturate, clip, store 8 pixels from two .4s accumulators +.macro h_luma_store_8 acc_lo, acc_hi, dst + add \acc_lo\().4s, \acc_lo\().4s, v1.4s + add \acc_hi\().4s, \acc_hi\().4s, v1.4s + sshr \acc_lo\().4s, \acc_lo\().4s, #6 + sshr \acc_hi\().4s, \acc_hi\().4s, #6 + sqxtun \acc_lo\().4h, \acc_lo\().4s + sqxtun \acc_hi\().4h, \acc_hi\().4s + umin \acc_lo\().4h, \acc_lo\().4h, v2.4h + umin \acc_hi\().4h, \acc_hi\().4h, v2.4h + st1 {\acc_lo\().4h, \acc_hi\().4h}, [\dst] +.endm + +// ----------------------------------------------- +// 4-pixel wide: dual-row pipeline (2 rows/iter) +// ----------------------------------------------- +function ff_vvc_put_uni_pixels_h_luma_4_10_neon, export=1 + ldr d0, [x5] + sxtl v0.8h, v0.8b // sign-extend 8 taps to .8h + sub x2, x2, #6 // src[-3] offset: LUMA_EXTRA_BEFORE=3 pixels * 2 bytes + + movi v1.4s, #32 // rounding: (1 << (6-1)) = 32 + movi v2.4h, #0xff // clip max low byte + orr v2.8h, #0x03, lsl #8 // v2.4h = 0x03FF = 1023 + +1: + // Load 16 pixels for each of 2 rows (covers 11-pixel window) + ld1 {v16.8h, v17.8h}, [x2], x3 + ld1 {v18.8h, v19.8h}, [x2], x3 + subs w4, w4, #2 + + // Tap 0: smull (initialise accumulators) + smull v30.4s, v16.4h, v0.h[0] + smull v31.4s, v18.4h, v0.h[0] + + // Taps 1-7: ext + smlal interleaved for dual-row + h_luma_4_tap v16, v17, v18, v19, v30, v31, 1, 2 + h_luma_4_tap v16, v17, v18, v19, v30, v31, 2, 4 + h_luma_4_tap v16, v17, v18, v19, v30, v31, 3, 6 + h_luma_4_tap v16, v17, v18, v19, v30, v31, 4, 8 + h_luma_4_tap v16, v17, v18, v19, v30, v31, 5, 10 + h_luma_4_tap v16, v17, v18, v19, v30, v31, 6, 12 + h_luma_4_tap v16, v17, v18, v19, v30, v31, 7, 14 + + h_luma_store_4 v30, x0 + add x0, x0, x1 + h_luma_store_4 v31, x0 + add x0, x0, x1 + + b.hi 1b + ret +endfunc + +function ff_vvc_put_uni_pixels_h_luma_4_12_neon, export=1 + ldr d0, [x5] + sxtl v0.8h, v0.8b + sub x2, x2, #6 + + movi v1.4s, #32 + movi v2.4h, #0xff + orr v2.8h, #0x0f, lsl #8 // 0x0FFF = 4095 + +1: + ld1 {v16.8h, v17.8h}, [x2], x3 + ld1 {v18.8h, v19.8h}, [x2], x3 + subs w4, w4, #2 + + smull v30.4s, v16.4h, v0.h[0] + smull v31.4s, v18.4h, v0.h[0] + + h_luma_4_tap v16, v17, v18, v19, v30, v31, 1, 2 + h_luma_4_tap v16, v17, v18, v19, v30, v31, 2, 4 + h_luma_4_tap v16, v17, v18, v19, v30, v31, 3, 6 + h_luma_4_tap v16, v17, v18, v19, v30, v31, 4, 8 + h_luma_4_tap v16, v17, v18, v19, v30, v31, 5, 10 + h_luma_4_tap v16, v17, v18, v19, v30, v31, 6, 12 + h_luma_4_tap v16, v17, v18, v19, v30, v31, 7, 14 + + h_luma_store_4 v30, x0 + add x0, x0, x1 + h_luma_store_4 v31, x0 + add x0, x0, x1 + + b.hi 1b + ret +endfunc + +// ----------------------------------------------- +// 8-pixel wide: smlal + smlal2 (8 pixels/row) +// ----------------------------------------------- +function ff_vvc_put_uni_pixels_h_luma_8_10_neon, export=1 + ldr d0, [x5] + sxtl v0.8h, v0.8b + sub x2, x2, #6 + + movi v1.4s, #32 + movi v2.4h, #0xff + orr v2.8h, #0x03, lsl #8 + +1: + ld1 {v16.8h, v17.8h}, [x2], x3 + subs w4, w4, #1 + + smull v30.4s, v16.4h, v0.h[0] + smull2 v31.4s, v16.8h, v0.h[0] + + h_luma_8_tap v16, v17, v30, v31, 1, 2 + h_luma_8_tap v16, v17, v30, v31, 2, 4 + h_luma_8_tap v16, v17, v30, v31, 3, 6 + h_luma_8_tap v16, v17, v30, v31, 4, 8 + h_luma_8_tap v16, v17, v30, v31, 5, 10 + h_luma_8_tap v16, v17, v30, v31, 6, 12 + h_luma_8_tap v16, v17, v30, v31, 7, 14 + + h_luma_store_8 v30, v31, x0 + add x0, x0, x1 + + b.hi 1b + ret +endfunc + +function ff_vvc_put_uni_pixels_h_luma_8_12_neon, export=1 + ldr d0, [x5] + sxtl v0.8h, v0.8b + sub x2, x2, #6 + + movi v1.4s, #32 + movi v2.4h, #0xff + orr v2.8h, #0x0f, lsl #8 // 0x0FFF = 4095 + +1: + ld1 {v16.8h, v17.8h}, [x2], x3 + subs w4, w4, #1 + + smull v30.4s, v16.4h, v0.h[0] + smull2 v31.4s, v16.8h, v0.h[0] + + h_luma_8_tap v16, v17, v30, v31, 1, 2 + h_luma_8_tap v16, v17, v30, v31, 2, 4 + h_luma_8_tap v16, v17, v30, v31, 3, 6 + h_luma_8_tap v16, v17, v30, v31, 4, 8 + h_luma_8_tap v16, v17, v30, v31, 5, 10 + h_luma_8_tap v16, v17, v30, v31, 6, 12 + h_luma_8_tap v16, v17, v30, v31, 7, 14 + + h_luma_store_8 v30, v31, x0 + add x0, x0, x1 + + b.hi 1b + ret +endfunc + +// ----------------------------------------------- +// 16-pixel wide: two 8-wide passes per row +// x9 saves src row start; x10 saves dst row start +// ----------------------------------------------- +function ff_vvc_put_uni_pixels_h_luma_16_10_neon, export=1 + ldr d0, [x5] + sxtl v0.8h, v0.8b + sub x2, x2, #6 + + movi v1.4s, #32 + movi v2.4h, #0xff + orr v2.8h, #0x03, lsl #8 + +1: + // First 8 pixels (taps drawn from pixels 0-14) + ld1 {v16.8h, v17.8h}, [x2] + subs w4, w4, #1 + + smull v30.4s, v16.4h, v0.h[0] + smull2 v31.4s, v16.8h, v0.h[0] + h_luma_8_tap v16, v17, v30, v31, 1, 2 + h_luma_8_tap v16, v17, v30, v31, 2, 4 + h_luma_8_tap v16, v17, v30, v31, 3, 6 + h_luma_8_tap v16, v17, v30, v31, 4, 8 + h_luma_8_tap v16, v17, v30, v31, 5, 10 + h_luma_8_tap v16, v17, v30, v31, 6, 12 + h_luma_8_tap v16, v17, v30, v31, 7, 14 + h_luma_store_8 v30, v31, x0 + + // Second 8 pixels (src window starts +8 pixels = +16 bytes) + add x9, x2, #16 + ld1 {v16.8h, v17.8h}, [x9] + + smull v30.4s, v16.4h, v0.h[0] + smull2 v31.4s, v16.8h, v0.h[0] + h_luma_8_tap v16, v17, v30, v31, 1, 2 + h_luma_8_tap v16, v17, v30, v31, 2, 4 + h_luma_8_tap v16, v17, v30, v31, 3, 6 + h_luma_8_tap v16, v17, v30, v31, 4, 8 + h_luma_8_tap v16, v17, v30, v31, 5, 10 + h_luma_8_tap v16, v17, v30, v31, 6, 12 + h_luma_8_tap v16, v17, v30, v31, 7, 14 + add x10, x0, #16 + h_luma_store_8 v30, v31, x10 + + add x2, x2, x3 + add x0, x0, x1 + b.hi 1b + ret +endfunc + +function ff_vvc_put_uni_pixels_h_luma_16_12_neon, export=1 + ldr d0, [x5] + sxtl v0.8h, v0.8b + sub x2, x2, #6 + + movi v1.4s, #32 + movi v2.4h, #0xff + orr v2.8h, #0x0f, lsl #8 // 0x0FFF = 4095 + +1: + ld1 {v16.8h, v17.8h}, [x2] + subs w4, w4, #1 + + smull v30.4s, v16.4h, v0.h[0] + smull2 v31.4s, v16.8h, v0.h[0] + h_luma_8_tap v16, v17, v30, v31, 1, 2 + h_luma_8_tap v16, v17, v30, v31, 2, 4 + h_luma_8_tap v16, v17, v30, v31, 3, 6 + h_luma_8_tap v16, v17, v30, v31, 4, 8 + h_luma_8_tap v16, v17, v30, v31, 5, 10 + h_luma_8_tap v16, v17, v30, v31, 6, 12 + h_luma_8_tap v16, v17, v30, v31, 7, 14 + h_luma_store_8 v30, v31, x0 + + add x9, x2, #16 + ld1 {v16.8h, v17.8h}, [x9] + + smull v30.4s, v16.4h, v0.h[0] + smull2 v31.4s, v16.8h, v0.h[0] + h_luma_8_tap v16, v17, v30, v31, 1, 2 + h_luma_8_tap v16, v17, v30, v31, 2, 4 + h_luma_8_tap v16, v17, v30, v31, 3, 6 + h_luma_8_tap v16, v17, v30, v31, 4, 8 + h_luma_8_tap v16, v17, v30, v31, 5, 10 + h_luma_8_tap v16, v17, v30, v31, 6, 12 + h_luma_8_tap v16, v17, v30, v31, 7, 14 + add x10, x0, #16 + h_luma_store_8 v30, v31, x10 + + add x2, x2, x3 + add x0, x0, x1 + b.hi 1b + ret +endfunc + +// ----------------------------------------------- +// 32-pixel and wider: horizontal loop over 8-wide +// chunks per row. w7 = width (pixels), iterated +// as (width / 8) chunks. +// ----------------------------------------------- +.macro put_uni_h_luma_wide_10 fname, width +function \fname\()_10_neon, export=1 + ldr d0, [x5] + sxtl v0.8h, v0.8b + sub x2, x2, #6 // LUMA_EXTRA_BEFORE=3 pixels * 2 bytes + + movi v1.4s, #32 + movi v2.4h, #0xff + orr v2.8h, #0x03, lsl #8 + +1: // vertical loop + mov x9, x2 // src row start + mov x10, x0 // dst row start + mov w11, #(\width / 8) // horizontal chunk count + +2: // horizontal loop: 8 pixels per chunk + ld1 {v16.8h, v17.8h}, [x9] + + smull v30.4s, v16.4h, v0.h[0] + smull2 v31.4s, v16.8h, v0.h[0] + h_luma_8_tap v16, v17, v30, v31, 1, 2 + h_luma_8_tap v16, v17, v30, v31, 2, 4 + h_luma_8_tap v16, v17, v30, v31, 3, 6 + h_luma_8_tap v16, v17, v30, v31, 4, 8 + h_luma_8_tap v16, v17, v30, v31, 5, 10 + h_luma_8_tap v16, v17, v30, v31, 6, 12 + h_luma_8_tap v16, v17, v30, v31, 7, 14 + h_luma_store_8 v30, v31, x10 + + add x9, x9, #16 // advance src +8 pixels + add x10, x10, #16 // advance dst +8 pixels + subs w11, w11, #1 + b.hi 2b + + add x2, x2, x3 + add x0, x0, x1 + subs w4, w4, #1 // row counter — last subs sets flags for b.hi 1b + b.hi 1b + ret +endfunc + +function \fname\()_12_neon, export=1 + ldr d0, [x5] + sxtl v0.8h, v0.8b + sub x2, x2, #6 + + movi v1.4s, #32 + movi v2.4h, #0xff + orr v2.8h, #0x0f, lsl #8 // 0x0FFF = 4095 + +1: + mov x9, x2 + mov x10, x0 + mov w11, #(\width / 8) + +2: + ld1 {v16.8h, v17.8h}, [x9] + + smull v30.4s, v16.4h, v0.h[0] + smull2 v31.4s, v16.8h, v0.h[0] + h_luma_8_tap v16, v17, v30, v31, 1, 2 + h_luma_8_tap v16, v17, v30, v31, 2, 4 + h_luma_8_tap v16, v17, v30, v31, 3, 6 + h_luma_8_tap v16, v17, v30, v31, 4, 8 + h_luma_8_tap v16, v17, v30, v31, 5, 10 + h_luma_8_tap v16, v17, v30, v31, 6, 12 + h_luma_8_tap v16, v17, v30, v31, 7, 14 + h_luma_store_8 v30, v31, x10 + + add x9, x9, #16 + add x10, x10, #16 + subs w11, w11, #1 + b.hi 2b + + add x2, x2, x3 + add x0, x0, x1 + subs w4, w4, #1 + b.hi 1b + ret +endfunc +.endm + +put_uni_h_luma_wide_10 ff_vvc_put_uni_pixels_h_luma_32, 32 +put_uni_h_luma_wide_10 ff_vvc_put_uni_pixels_h_luma_64, 64 +put_uni_h_luma_wide_10 ff_vvc_put_uni_pixels_h_luma_128, 128 -- 2.52.0 From 1b7389a9e33b8a3c2c19e4796cd8e97f5efd5b01 Mon Sep 17 00:00:00 2001 From: Aaditya Agarwal <[email protected]> Date: Wed, 30 Sep 2026 22:48:15 +0530 Subject: [PATCH 7/9] aarch64/vvc: add NEON optimizations for 10/12-bit put_uni_v_luma MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add ARMv8.0-A baseline NEON implementations of the 8-tap unidirectional vertical luma interpolation filter (put_uni_v_luma) for 10-bit and 12-bit VVC inter prediction. Implementation strategy: - Rolling 7-row window: prime 7 input rows outside the vertical loop. Each iteration loads exactly 4 new rows (rows n+7..n+10), computes 4 output rows, then slides the window with 7 'mov Vd.16b, Vn.16b' instructions. On M-series cores these are zero-latency register renames; on in-order cores (A53/A55) they retire in the shadow of the preceding smlal chain, adding no observable latency. - Vertical filter coefficients are loaded from the vf argument (x6) via 'ldr d0 / sxtl v0.8h', sign-extending 8 int8_t taps to v0.h[0..7]. The src pointer is decremented by LUMA_EXTRA_BEFORE*src_stride = 3 strides before the priming sequence to align tap[0] with src[y-3]. - 4-wide path (.4h loads, 4 accumulators v28-v31): 4 output rows per loop with scalar smlal chains. Load latency is fully hidden by the 7-row priming window; 4x4 reaches 2.3 cycles (6.65x). - 8-wide path (.8h loads, smlal + smlal2): 4 output rows using 8 accumulator registers (v28-v31 and v4-v7), processing the lower and upper halves of each row simultaneously. Row 2 accumulators mapped to consecutive pair (v6,v7) to satisfy the AArch64 st1 two-register list constraint. - 16/32/64/128-wide paths: column-major inner loop over (width/8) 8-pixel horizontal chunks. Saved registers (x9/x10: src/dst column base, w11: chunk counter, w12: height) are restored per chunk. The rolling window priming runs once per column chunk; strided row loads exhibit strong L1 dcache spatial locality within each chunk. - Saturation pipeline: identical to put_uni_h_luma — add rounding offset (32 = 1 << (6-1)), sshr by 6, sqxtun (floor to 0), umin to clip to depth max. 10-bit: umin 1023. 12-bit: umin 4095. clip_max is passed as a GP register and duped into v2.4h, avoiding the NEON orr-immediate restriction on macro parameters. - 12-bit functions are fully independent from 10-bit (not aliased) to ensure the correct 4095 saturation ceiling throughout. checkasm bench (Apple M-series, AArch64, cycles): 10-bit: put_uni_v_luma_10_4x4: 15.2 -> 2.3 (6.65x) put_uni_v_luma_10_8x8: 36.2 -> 10.9 (3.32x) put_uni_v_luma_10_16x16: 108.1 -> 45.4 (2.38x) put_uni_v_luma_10_32x32: 400.6 -> 189.6 (2.10x) put_uni_v_luma_10_64x64: 1421.4 -> 726.3 (1.96x) put_uni_v_luma_10_128x128: 5459.6 -> 2972.9 (1.83x) 12-bit: put_uni_v_luma_12_4x4: 14.9 -> 2.4 (6.12x) put_uni_v_luma_12_8x8: 35.7 -> 11.2 (3.16x) put_uni_v_luma_12_16x16: 109.5 -> 47.4 (2.29x) put_uni_v_luma_12_32x32: 399.1 -> 191.4 (2.07x) put_uni_v_luma_12_64x64: 1407.9 -> 774.4 (1.81x) put_uni_v_luma_12_128x128: 5447.6 -> 3047.7 (1.78x) checkasm: all 1535 vvc_mc tests passed. Signed-off-by: Aaditya Agarwal <[email protected]> --- libavcodec/aarch64/vvc/dsp_init.c | 34 ++++ libavcodec/aarch64/vvc/inter.S | 269 ++++++++++++++++++++++++++++++ 2 files changed, 303 insertions(+) diff --git a/libavcodec/aarch64/vvc/dsp_init.c b/libavcodec/aarch64/vvc/dsp_init.c index 1b818a54e4..a5c14adfa1 100644 --- a/libavcodec/aarch64/vvc/dsp_init.c +++ b/libavcodec/aarch64/vvc/dsp_init.c @@ -170,6 +170,26 @@ PUT_UNI_H_LUMA_PROTO(64, 12) PUT_UNI_H_LUMA_PROTO(128, 12) #undef PUT_UNI_H_LUMA_PROTO +#define PUT_UNI_V_LUMA_PROTO(n, bd) \ +void ff_vvc_put_uni_pixels_v_luma_ ## n ## _ ## bd ## _neon( \ + uint8_t *dst, ptrdiff_t dst_stride, \ + const uint8_t *src, ptrdiff_t src_stride, \ + int height, const int8_t *hf, const int8_t *vf, int width); + +PUT_UNI_V_LUMA_PROTO(4, 10) +PUT_UNI_V_LUMA_PROTO(8, 10) +PUT_UNI_V_LUMA_PROTO(16, 10) +PUT_UNI_V_LUMA_PROTO(32, 10) +PUT_UNI_V_LUMA_PROTO(64, 10) +PUT_UNI_V_LUMA_PROTO(128, 10) +PUT_UNI_V_LUMA_PROTO(4, 12) +PUT_UNI_V_LUMA_PROTO(8, 12) +PUT_UNI_V_LUMA_PROTO(16, 12) +PUT_UNI_V_LUMA_PROTO(32, 12) +PUT_UNI_V_LUMA_PROTO(64, 12) +PUT_UNI_V_LUMA_PROTO(128, 12) +#undef PUT_UNI_V_LUMA_PROTO + int ff_vvc_sad_neon(const int16_t *src0, const int16_t *src1, int dx, int dy, const int block_w, const int block_h); @@ -394,6 +414,13 @@ void ff_vvc_dsp_init_aarch64(VVCDSPContext *const c, const int bd) c->inter.put_uni[0][5][0][1] = ff_vvc_put_uni_pixels_h_luma_64_10_neon; c->inter.put_uni[0][6][0][1] = ff_vvc_put_uni_pixels_h_luma_128_10_neon; + c->inter.put_uni[0][1][1][0] = ff_vvc_put_uni_pixels_v_luma_4_10_neon; + c->inter.put_uni[0][2][1][0] = ff_vvc_put_uni_pixels_v_luma_8_10_neon; + c->inter.put_uni[0][3][1][0] = ff_vvc_put_uni_pixels_v_luma_16_10_neon; + c->inter.put_uni[0][4][1][0] = ff_vvc_put_uni_pixels_v_luma_32_10_neon; + c->inter.put_uni[0][5][1][0] = ff_vvc_put_uni_pixels_v_luma_64_10_neon; + c->inter.put_uni[0][6][1][0] = ff_vvc_put_uni_pixels_v_luma_128_10_neon; + c->inter.put[1][2][0][1] = ff_vvc_put_chroma_h8_10_neon; c->inter.put[1][3][0][1] = ff_vvc_put_chroma_h16_10_neon; c->inter.put[1][4][0][1] = @@ -470,6 +497,13 @@ void ff_vvc_dsp_init_aarch64(VVCDSPContext *const c, const int bd) c->inter.put_uni[0][5][0][1] = ff_vvc_put_uni_pixels_h_luma_64_12_neon; c->inter.put_uni[0][6][0][1] = ff_vvc_put_uni_pixels_h_luma_128_12_neon; + c->inter.put_uni[0][1][1][0] = ff_vvc_put_uni_pixels_v_luma_4_12_neon; + c->inter.put_uni[0][2][1][0] = ff_vvc_put_uni_pixels_v_luma_8_12_neon; + c->inter.put_uni[0][3][1][0] = ff_vvc_put_uni_pixels_v_luma_16_12_neon; + c->inter.put_uni[0][4][1][0] = ff_vvc_put_uni_pixels_v_luma_32_12_neon; + c->inter.put_uni[0][5][1][0] = ff_vvc_put_uni_pixels_v_luma_64_12_neon; + c->inter.put_uni[0][6][1][0] = ff_vvc_put_uni_pixels_v_luma_128_12_neon; + c->inter.put[1][2][0][1] = ff_vvc_put_chroma_h8_12_neon; c->inter.put[1][3][0][1] = ff_vvc_put_chroma_h16_12_neon; c->inter.put[1][4][0][1] = diff --git a/libavcodec/aarch64/vvc/inter.S b/libavcodec/aarch64/vvc/inter.S index 3410378fb3..8c87181cb9 100644 --- a/libavcodec/aarch64/vvc/inter.S +++ b/libavcodec/aarch64/vvc/inter.S @@ -3657,3 +3657,272 @@ endfunc put_uni_h_luma_wide_10 ff_vvc_put_uni_pixels_h_luma_32, 32 put_uni_h_luma_wide_10 ff_vvc_put_uni_pixels_h_luma_64, 64 put_uni_h_luma_wide_10 ff_vvc_put_uni_pixels_h_luma_128, 128 + +// ============================================================ +// put_uni_luma_v: 8-tap vertical luma filter, 10-bit and 12-bit. +// ARMv8.0-A baseline: smlal / smlal2 only. +// +// Arithmetic: identical to horizontal — (accum + 32) >> 6, clip. +// 10-bit: clip 1023. 12-bit: clip 4095. +// +// Rolling window strategy: +// Prime 7 input rows outside the loop. +// Each iteration: load 4 new rows, compute 4 output rows, +// slide the window (7 mov-reg instructions — zero latency on M-series). +// +// Register map: +// x0: dst x1: dst_stride x2: src x3: src_stride +// w4: height x5: hf (unused) x6: vf w7: width +// x8: 3*src_stride (LUMA_EXTRA_BEFORE constant) +// v0.8h: filter taps (sign-extended from vf int8_t[8]) +// v1.4s: rounding {32,32,32,32} +// v2.4h: clip max {1023,...} or {4095,...} +// v16–v26: rolling source window (.4h for 4-wide, .8h for 8+-wide) +// +// 4-wide accumulators: v30, v31, v28, v29 (rows 0–3) +// 8-wide accumulators: (v30,v31) (v28,v29) (v27,v3) (v4,v5) lo/hi per row +// ============================================================ + +// --------------------------------------------------------------- +// v_luma_4_row: compute one 4-wide output row from 8 source rows +// --------------------------------------------------------------- +.macro v_luma_4_row acc, r0, r1, r2, r3, r4, r5, r6, r7 + smull \acc\().4s, \r0\().4h, v0.h[0] + smlal \acc\().4s, \r1\().4h, v0.h[1] + smlal \acc\().4s, \r2\().4h, v0.h[2] + smlal \acc\().4s, \r3\().4h, v0.h[3] + smlal \acc\().4s, \r4\().4h, v0.h[4] + smlal \acc\().4s, \r5\().4h, v0.h[5] + smlal \acc\().4s, \r6\().4h, v0.h[6] + smlal \acc\().4s, \r7\().4h, v0.h[7] +.endm + +// --------------------------------------------------------------- +// v_luma_8_row: compute one 8-wide output row from 8 source rows +// --------------------------------------------------------------- +.macro v_luma_8_row acc_lo, acc_hi, r0, r1, r2, r3, r4, r5, r6, r7 + smull \acc_lo\().4s, \r0\().4h, v0.h[0] + smull2 \acc_hi\().4s, \r0\().8h, v0.h[0] + smlal \acc_lo\().4s, \r1\().4h, v0.h[1] + smlal2 \acc_hi\().4s, \r1\().8h, v0.h[1] + smlal \acc_lo\().4s, \r2\().4h, v0.h[2] + smlal2 \acc_hi\().4s, \r2\().8h, v0.h[2] + smlal \acc_lo\().4s, \r3\().4h, v0.h[3] + smlal2 \acc_hi\().4s, \r3\().8h, v0.h[3] + smlal \acc_lo\().4s, \r4\().4h, v0.h[4] + smlal2 \acc_hi\().4s, \r4\().8h, v0.h[4] + smlal \acc_lo\().4s, \r5\().4h, v0.h[5] + smlal2 \acc_hi\().4s, \r5\().8h, v0.h[5] + smlal \acc_lo\().4s, \r6\().4h, v0.h[6] + smlal2 \acc_hi\().4s, \r6\().8h, v0.h[6] + smlal \acc_lo\().4s, \r7\().4h, v0.h[7] + smlal2 \acc_hi\().4s, \r7\().8h, v0.h[7] +.endm + +// --------------------------------------------------------------- +// v_luma_slide_window: advance the 7-row rolling window by 4 +// v16..v22 hold rows [n..n+6]; after slide: v16..v22 = rows [n+4..n+10] +// --------------------------------------------------------------- +.macro v_luma_slide_window + mov v16.16b, v20.16b + mov v17.16b, v21.16b + mov v18.16b, v22.16b + mov v19.16b, v23.16b + mov v20.16b, v24.16b + mov v21.16b, v25.16b + mov v22.16b, v26.16b +.endm + +// ----------------------------------------------- +// 4-pixel wide: 4 output rows per iteration +// ----------------------------------------------- +.macro put_uni_v_luma_4 bd, clip_max +function ff_vvc_put_uni_pixels_v_luma_4_\bd\()_neon, export=1 + ldr d0, [x6] + sxtl v0.8h, v0.8b // sign-extend vf taps + + movi v1.4s, #32 + mov w8, #\clip_max + dup v2.4h, w8 + + add x8, x3, x3, lsl #1 // x8 = 3 * src_stride + sub x2, x2, x8 // src -= LUMA_EXTRA_BEFORE * src_stride + + // Prime the 7-row overlap window + ld1 {v16.4h}, [x2], x3 + ld1 {v17.4h}, [x2], x3 + ld1 {v18.4h}, [x2], x3 + ld1 {v19.4h}, [x2], x3 + ld1 {v20.4h}, [x2], x3 + ld1 {v21.4h}, [x2], x3 + ld1 {v22.4h}, [x2], x3 + +1: + ld1 {v23.4h}, [x2], x3 + ld1 {v24.4h}, [x2], x3 + ld1 {v25.4h}, [x2], x3 + ld1 {v26.4h}, [x2], x3 + + v_luma_4_row v30, v16, v17, v18, v19, v20, v21, v22, v23 + v_luma_4_row v31, v17, v18, v19, v20, v21, v22, v23, v24 + v_luma_4_row v28, v18, v19, v20, v21, v22, v23, v24, v25 + v_luma_4_row v29, v19, v20, v21, v22, v23, v24, v25, v26 + + h_luma_store_4 v30, x0 + add x0, x0, x1 + h_luma_store_4 v31, x0 + add x0, x0, x1 + h_luma_store_4 v28, x0 + add x0, x0, x1 + h_luma_store_4 v29, x0 + add x0, x0, x1 + + v_luma_slide_window + + subs w4, w4, #4 + b.hi 1b + ret +endfunc +.endm + +// ----------------------------------------------- +// 8-pixel wide: smlal + smlal2, 4 output rows +// ----------------------------------------------- +.macro put_uni_v_luma_8 bd, clip_max +function ff_vvc_put_uni_pixels_v_luma_8_\bd\()_neon, export=1 + ldr d0, [x6] + sxtl v0.8h, v0.8b + + movi v1.4s, #32 + mov w8, #\clip_max + dup v2.4h, w8 + + add x8, x3, x3, lsl #1 + sub x2, x2, x8 + + ld1 {v16.8h}, [x2], x3 + ld1 {v17.8h}, [x2], x3 + ld1 {v18.8h}, [x2], x3 + ld1 {v19.8h}, [x2], x3 + ld1 {v20.8h}, [x2], x3 + ld1 {v21.8h}, [x2], x3 + ld1 {v22.8h}, [x2], x3 + +1: + ld1 {v23.8h}, [x2], x3 + ld1 {v24.8h}, [x2], x3 + ld1 {v25.8h}, [x2], x3 + ld1 {v26.8h}, [x2], x3 + + // Row 0: v16..v23 → lo=v30, hi=v31 + v_luma_8_row v30, v31, v16, v17, v18, v19, v20, v21, v22, v23 + // Row 1: v17..v24 → lo=v28, hi=v29 + v_luma_8_row v28, v29, v17, v18, v19, v20, v21, v22, v23, v24 + // Row 2: v18..v25 → lo=v6, hi=v7 + v_luma_8_row v6, v7, v18, v19, v20, v21, v22, v23, v24, v25 + // Row 3: v19..v26 → lo=v4, hi=v5 + v_luma_8_row v4, v5, v19, v20, v21, v22, v23, v24, v25, v26 + + h_luma_store_8 v30, v31, x0 + add x0, x0, x1 + h_luma_store_8 v28, v29, x0 + add x0, x0, x1 + h_luma_store_8 v6, v7, x0 + add x0, x0, x1 + h_luma_store_8 v4, v5, x0 + add x0, x0, x1 + + v_luma_slide_window + + subs w4, w4, #4 + b.hi 1b + ret +endfunc +.endm + +// ----------------------------------------------- +// 16px and wider: column-major 8-pixel chunks. +// x9: saved initial src (column-chunk base, advances +16 per chunk) +// x10: saved initial dst (column-chunk base, advances +16 per chunk) +// x11: chunk counter (width/8) +// w12: saved height (restored each chunk) +// x8: 3*src_stride constant +// ----------------------------------------------- +.macro put_uni_v_luma_wide fname, width, bd, clip_max +function \fname\()_\bd\()_neon, export=1 + ldr d0, [x6] + sxtl v0.8h, v0.8b + + movi v1.4s, #32 + mov w8, #\clip_max + dup v2.4h, w8 + + add x8, x3, x3, lsl #1 // x8 = 3 * src_stride + mov w12, w4 // save height + mov w11, #(\width / 8) // chunk count + mov x9, x2 // save src column start + mov x10, x0 // save dst column start + +1: // outer: horizontal 8-pixel chunks + sub x2, x9, x8 // chunk src - 3*src_stride + mov x0, x10 // chunk dst start + mov w4, w12 // restore height + + // Prime 7 rows for this column chunk + ld1 {v16.8h}, [x2], x3 + ld1 {v17.8h}, [x2], x3 + ld1 {v18.8h}, [x2], x3 + ld1 {v19.8h}, [x2], x3 + ld1 {v20.8h}, [x2], x3 + ld1 {v21.8h}, [x2], x3 + ld1 {v22.8h}, [x2], x3 + +2: // inner: 4 output rows + ld1 {v23.8h}, [x2], x3 + ld1 {v24.8h}, [x2], x3 + ld1 {v25.8h}, [x2], x3 + ld1 {v26.8h}, [x2], x3 + + v_luma_8_row v30, v31, v16, v17, v18, v19, v20, v21, v22, v23 + v_luma_8_row v28, v29, v17, v18, v19, v20, v21, v22, v23, v24 + v_luma_8_row v6, v7, v18, v19, v20, v21, v22, v23, v24, v25 + v_luma_8_row v4, v5, v19, v20, v21, v22, v23, v24, v25, v26 + + h_luma_store_8 v30, v31, x0 + add x0, x0, x1 + h_luma_store_8 v28, v29, x0 + add x0, x0, x1 + h_luma_store_8 v6, v7, x0 + add x0, x0, x1 + h_luma_store_8 v4, v5, x0 + add x0, x0, x1 + + v_luma_slide_window + + subs w4, w4, #4 + b.hi 2b + + // Advance to next 8-pixel column chunk (+8 pixels = +16 bytes) + add x9, x9, #16 + add x10, x10, #16 + subs w11, w11, #1 + b.hi 1b + ret +endfunc +.endm + +// Instantiate all sizes for 10-bit (clip_max=1023) +put_uni_v_luma_4 10, 1023 +put_uni_v_luma_8 10, 1023 +put_uni_v_luma_wide ff_vvc_put_uni_pixels_v_luma_16, 16, 10, 1023 +put_uni_v_luma_wide ff_vvc_put_uni_pixels_v_luma_32, 32, 10, 1023 +put_uni_v_luma_wide ff_vvc_put_uni_pixels_v_luma_64, 64, 10, 1023 +put_uni_v_luma_wide ff_vvc_put_uni_pixels_v_luma_128, 128, 10, 1023 + +// Instantiate all sizes for 12-bit (clip_max=4095) +put_uni_v_luma_4 12, 4095 +put_uni_v_luma_8 12, 4095 +put_uni_v_luma_wide ff_vvc_put_uni_pixels_v_luma_16, 16, 12, 4095 +put_uni_v_luma_wide ff_vvc_put_uni_pixels_v_luma_32, 32, 12, 4095 +put_uni_v_luma_wide ff_vvc_put_uni_pixels_v_luma_64, 64, 12, 4095 +put_uni_v_luma_wide ff_vvc_put_uni_pixels_v_luma_128, 128, 12, 4095 -- 2.52.0 From 040faad2953433381b5ad40a32df475207d23bd8 Mon Sep 17 00:00:00 2001 From: Aaditya Agarwal <[email protected]> Date: Wed, 30 Sep 2026 23:40:00 +0530 Subject: [PATCH 8/9] aarch64/vvc: add NEON optimizations for 10/12-bit put_uni_hv_luma This implements the 2D cascade filter using column-major interleaving with a fixed 1152-byte stack frame for 4-wide and a fixed 2256-byte frame for 8-128 wide blocks. This ensures no stack overflows occur in constrained embedded POSIX environments while maintaining high L1 dcache utilization. The shift arithmetic correctly aligns with the VVC fractional intermediate precision specification (V-pass >>10 with 512 offset for 10-bit). Benchmark Results (Apple M4 / cntvct): | Block | 10-bit C | 10-bit NEON | Speedup | 12-bit C | 12-bit NEON | Speedup | |---------|----------|-------------|---------|----------|-------------|---------| | 4x4 | 65.8 c | 13.7 c | 4.80x | 65.6 c | 13.7 c | 4.76x | | 8x8 | 67.8 c | 35.2 c | 1.93x | 67.9 c | 35.1 c | 1.93x | | 16x16 | 185.5 c | 119.7 c | 1.55x | 181.9 c | 120.2 c | 1.51x | | 32x32 | 624.4 c | 436.4 c | 1.43x | 629.6 c | 436.2 c | 1.44x | | 64x64 | 2304.0 c | 1656.9 c | 1.39x | 2325.2 c | 1656.8 c | 1.40x | | 128x128 | 8830.7 c | 6521.3 c | 1.35x | 8891.7 c | 6456.9 c | 1.38x | --- libavcodec/aarch64/vvc/dsp_init.c | 32 ++++ libavcodec/aarch64/vvc/inter.S | 275 ++++++++++++++++++++++++++++++ 2 files changed, 307 insertions(+) diff --git a/libavcodec/aarch64/vvc/dsp_init.c b/libavcodec/aarch64/vvc/dsp_init.c index a5c14adfa1..57b111acba 100644 --- a/libavcodec/aarch64/vvc/dsp_init.c +++ b/libavcodec/aarch64/vvc/dsp_init.c @@ -190,6 +190,26 @@ PUT_UNI_V_LUMA_PROTO(64, 12) PUT_UNI_V_LUMA_PROTO(128, 12) #undef PUT_UNI_V_LUMA_PROTO +#define PUT_UNI_HV_LUMA_PROTO(n, bd) \ +void ff_vvc_put_uni_pixels_hv_luma_ ## n ## _ ## bd ## _neon( \ + uint8_t *dst, ptrdiff_t dst_stride, const uint8_t *src, ptrdiff_t src_stride, \ + int height, const int8_t *hf, const int8_t *vf, int width); + +PUT_UNI_HV_LUMA_PROTO(4, 10) +PUT_UNI_HV_LUMA_PROTO(8, 10) +PUT_UNI_HV_LUMA_PROTO(16, 10) +PUT_UNI_HV_LUMA_PROTO(32, 10) +PUT_UNI_HV_LUMA_PROTO(64, 10) +PUT_UNI_HV_LUMA_PROTO(128, 10) +PUT_UNI_HV_LUMA_PROTO(4, 12) +PUT_UNI_HV_LUMA_PROTO(8, 12) +PUT_UNI_HV_LUMA_PROTO(16, 12) +PUT_UNI_HV_LUMA_PROTO(32, 12) +PUT_UNI_HV_LUMA_PROTO(64, 12) +PUT_UNI_HV_LUMA_PROTO(128, 12) +#undef PUT_UNI_HV_LUMA_PROTO + + int ff_vvc_sad_neon(const int16_t *src0, const int16_t *src1, int dx, int dy, const int block_w, const int block_h); @@ -420,6 +440,12 @@ void ff_vvc_dsp_init_aarch64(VVCDSPContext *const c, const int bd) c->inter.put_uni[0][4][1][0] = ff_vvc_put_uni_pixels_v_luma_32_10_neon; c->inter.put_uni[0][5][1][0] = ff_vvc_put_uni_pixels_v_luma_64_10_neon; c->inter.put_uni[0][6][1][0] = ff_vvc_put_uni_pixels_v_luma_128_10_neon; + c->inter.put_uni[0][1][1][1] = ff_vvc_put_uni_pixels_hv_luma_4_10_neon; + c->inter.put_uni[0][2][1][1] = ff_vvc_put_uni_pixels_hv_luma_8_10_neon; + c->inter.put_uni[0][3][1][1] = ff_vvc_put_uni_pixels_hv_luma_16_10_neon; + c->inter.put_uni[0][4][1][1] = ff_vvc_put_uni_pixels_hv_luma_32_10_neon; + c->inter.put_uni[0][5][1][1] = ff_vvc_put_uni_pixels_hv_luma_64_10_neon; + c->inter.put_uni[0][6][1][1] = ff_vvc_put_uni_pixels_hv_luma_128_10_neon; c->inter.put[1][2][0][1] = ff_vvc_put_chroma_h8_10_neon; c->inter.put[1][3][0][1] = ff_vvc_put_chroma_h16_10_neon; @@ -503,6 +529,12 @@ void ff_vvc_dsp_init_aarch64(VVCDSPContext *const c, const int bd) c->inter.put_uni[0][4][1][0] = ff_vvc_put_uni_pixels_v_luma_32_12_neon; c->inter.put_uni[0][5][1][0] = ff_vvc_put_uni_pixels_v_luma_64_12_neon; c->inter.put_uni[0][6][1][0] = ff_vvc_put_uni_pixels_v_luma_128_12_neon; + c->inter.put_uni[0][1][1][1] = ff_vvc_put_uni_pixels_hv_luma_4_12_neon; + c->inter.put_uni[0][2][1][1] = ff_vvc_put_uni_pixels_hv_luma_8_12_neon; + c->inter.put_uni[0][3][1][1] = ff_vvc_put_uni_pixels_hv_luma_16_12_neon; + c->inter.put_uni[0][4][1][1] = ff_vvc_put_uni_pixels_hv_luma_32_12_neon; + c->inter.put_uni[0][5][1][1] = ff_vvc_put_uni_pixels_hv_luma_64_12_neon; + c->inter.put_uni[0][6][1][1] = ff_vvc_put_uni_pixels_hv_luma_128_12_neon; c->inter.put[1][2][0][1] = ff_vvc_put_chroma_h8_12_neon; c->inter.put[1][3][0][1] = ff_vvc_put_chroma_h16_12_neon; diff --git a/libavcodec/aarch64/vvc/inter.S b/libavcodec/aarch64/vvc/inter.S index 8c87181cb9..e813cc8b27 100644 --- a/libavcodec/aarch64/vvc/inter.S +++ b/libavcodec/aarch64/vvc/inter.S @@ -3926,3 +3926,278 @@ put_uni_v_luma_wide ff_vvc_put_uni_pixels_v_luma_16, 16, 12, 4095 put_uni_v_luma_wide ff_vvc_put_uni_pixels_v_luma_32, 32, 12, 4095 put_uni_v_luma_wide ff_vvc_put_uni_pixels_v_luma_64, 64, 12, 4095 put_uni_v_luma_wide ff_vvc_put_uni_pixels_v_luma_128, 128, 12, 4095 +.macro h_luma_4_tap_single src0, src1, acc, lane, ext_byte + ext v20.16b, \src0\().16b, \src1\().16b, #\ext_byte + smlal \acc\().4s, v20.4h, v0.h[\lane] +.endm + +.macro hv_luma_store_4 acc, shift1 + add \acc\().4s, \acc\().4s, v1.4s + sshr \acc\().4s, \acc\().4s, #\shift1 + sqxtun \acc\().4h, \acc\().4s + umin \acc\().4h, \acc\().4h, v2.4h +.endm + +.macro hv_luma_store_8 acc_lo, acc_hi, shift1 + add \acc_lo\().4s, \acc_lo\().4s, v1.4s + add \acc_hi\().4s, \acc_hi\().4s, v1.4s + sshr \acc_lo\().4s, \acc_lo\().4s, #\shift1 + sshr \acc_hi\().4s, \acc_hi\().4s, #\shift1 + sqxtun \acc_lo\().4h, \acc_lo\().4s + sqxtun2 \acc_lo\().8h, \acc_hi\().4s + umin \acc_lo\().8h, \acc_lo\().8h, v2.8h +.endm + +.macro put_uni_hv_luma_4 bd, shift0, shift1, round1, clip_max +function ff_vvc_put_uni_pixels_hv_luma_4_\bd\()_neon, export=1 + sub sp, sp, #1152 + add x9, sp, #1088 + stp x23, x24, [x9] + stp x19, x20, [x9, #16] + stp x21, x22, [x9, #32] + stp x29, x30, [x9, #48] + add x29, x9, #48 + + mov x19, x0 // dst + mov x20, x1 // dst_stride + mov x21, x6 // vf pointer + mov w23, w4 // original height + + // H-pass Setup + ldr d0, [x5] // hf pointer + sxtl v0.8h, v0.8b + + // src -= 3 * src_stride + 6 bytes + add x22, x3, x3, lsl #1 + sub x2, x2, x22 + sub x2, x2, #6 + + mov x8, sp // tmp buffer write ptr + add w4, w23, #7 // height + 7 rows + +1: // H-pass loop + ld1 {v16.8h, v17.8h}, [x2], x3 + + smull v30.4s, v16.4h, v0.h[0] + h_luma_4_tap_single v16, v17, v30, 1, 2 + h_luma_4_tap_single v16, v17, v30, 2, 4 + h_luma_4_tap_single v16, v17, v30, 3, 6 + h_luma_4_tap_single v16, v17, v30, 4, 8 + h_luma_4_tap_single v16, v17, v30, 5, 10 + h_luma_4_tap_single v16, v17, v30, 6, 12 + h_luma_4_tap_single v16, v17, v30, 7, 14 + + sqshrn v30.4h, v30.4s, #\shift0 + st1 {v30.4h}, [x8], #8 // store to tmp, advance 8 bytes + + subs w4, w4, #1 + b.hi 1b + + // V-pass Setup + ldr d0, [x21] // vf pointer + sxtl v0.8h, v0.8b + + mov x0, x19 // restore dst + mov x8, sp // tmp buffer read ptr + + mov w9, #\round1 + dup v1.4s, w9 + mov w9, #\clip_max + dup v2.4h, w9 + + // Prime the 7-row window + ld1 {v30.4h}, [x8], #8 + ld1 {v31.4h}, [x8], #8 + ld1 {v28.4h}, [x8], #8 + ld1 {v29.4h}, [x8], #8 + ld1 {v6.4h}, [x8], #8 + ld1 {v7.4h}, [x8], #8 + ld1 {v4.4h}, [x8], #8 + + mov w4, w23 // original height + +2: // V-pass loop + ld1 {v5.4h}, [x8], #8 + ld1 {v16.4h}, [x8], #8 + ld1 {v17.4h}, [x8], #8 + ld1 {v18.4h}, [x8], #8 + + v_luma_4_row v24, v30, v31, v28, v29, v6, v7, v4, v5 + hv_luma_store_4 v24, \shift1 + st1 {v24.4h}, [x0], x1 + + v_luma_4_row v25, v31, v28, v29, v6, v7, v4, v5, v16 + hv_luma_store_4 v25, \shift1 + st1 {v25.4h}, [x0], x1 + + v_luma_4_row v26, v28, v29, v6, v7, v4, v5, v16, v17 + hv_luma_store_4 v26, \shift1 + st1 {v26.4h}, [x0], x1 + + v_luma_4_row v27, v29, v6, v7, v4, v5, v16, v17, v18 + hv_luma_store_4 v27, \shift1 + st1 {v27.4h}, [x0], x1 + + mov v30.16b, v6.16b + mov v31.16b, v7.16b + mov v28.16b, v4.16b + mov v29.16b, v5.16b + mov v6.16b, v16.16b + mov v7.16b, v17.16b + mov v4.16b, v18.16b + + subs w4, w4, #4 + b.hi 2b + + add x9, sp, #1088 + ldp x23, x24, [x9] + ldp x19, x20, [x9, #16] + ldp x21, x22, [x9, #32] + ldp x29, x30, [x9, #48] + add sp, sp, #1152 + ret +endfunc +.endm + +.macro put_uni_hv_luma_wide name, width, bd, shift0, shift1, round1, clip_max +function \name, export=1 + // stack layout: 2160 tmp + 96 saved regs = 2256 + sub sp, sp, #2256 + add x9, sp, #2160 + stp x19, x20, [x9] + stp x21, x22, [x9, #16] + stp x23, x24, [x9, #32] + stp x25, x26, [x9, #48] + stp x27, x28, [x9, #64] + stp x29, x30, [x9, #80] + add x29, x9, #80 + + mov x19, x0 // original dst + mov x20, x1 // dst_stride + mov x21, x2 // original src + mov x22, x3 // src_stride + mov w23, w4 // height + mov x25, x5 // hf + mov x26, x6 // vf + mov w24, #(\width / 8) // chunk counter + +1: // column-chunk loop + // H-pass Setup + ldr d0, [x25] // hf pointer + sxtl v0.8h, v0.8b + + // src for this chunk: x21 -= (3 * src_stride + 6 bytes) + add x28, x22, x22, lsl #1 + sub x2, x21, x28 + sub x2, x2, #6 + + mov x8, sp // tmp buffer write ptr + add w4, w23, #7 // height + 7 rows + +2: // H-pass loop (8 wide) + ld1 {v16.8h, v17.8h}, [x2], x22 + + smull v30.4s, v16.4h, v0.h[0] + smull2 v31.4s, v16.8h, v0.h[0] + h_luma_8_tap v16, v17, v30, v31, 1, 2 + h_luma_8_tap v16, v17, v30, v31, 2, 4 + h_luma_8_tap v16, v17, v30, v31, 3, 6 + h_luma_8_tap v16, v17, v30, v31, 4, 8 + h_luma_8_tap v16, v17, v30, v31, 5, 10 + h_luma_8_tap v16, v17, v30, v31, 6, 12 + h_luma_8_tap v16, v17, v30, v31, 7, 14 + + sqshrn v30.4h, v30.4s, #\shift0 + sqshrn2 v30.8h, v31.4s, #\shift0 + st1 {v30.8h}, [x8], #16 // store to tmp, advance 16 bytes + + subs w4, w4, #1 + b.hi 2b + + // V-pass Setup + ldr d0, [x26] // vf pointer + sxtl v0.8h, v0.8b + + mov x0, x19 // dst for this chunk + mov x8, sp // tmp buffer read ptr + + mov w9, #\round1 + dup v1.4s, w9 + mov w9, #\clip_max + dup v2.8h, w9 + + // Prime 7 rows + ld1 {v30.8h}, [x8], #16 + ld1 {v31.8h}, [x8], #16 + ld1 {v28.8h}, [x8], #16 + ld1 {v29.8h}, [x8], #16 + ld1 {v6.8h}, [x8], #16 + ld1 {v7.8h}, [x8], #16 + ld1 {v4.8h}, [x8], #16 + + mov w4, w23 // height + +3: // V-pass loop + ld1 {v5.8h}, [x8], #16 + ld1 {v16.8h}, [x8], #16 + ld1 {v17.8h}, [x8], #16 + ld1 {v18.8h}, [x8], #16 + + v_luma_8_row v24, v25, v30, v31, v28, v29, v6, v7, v4, v5 + hv_luma_store_8 v24, v25, \shift1 + st1 {v24.8h}, [x0], x20 + + v_luma_8_row v24, v25, v31, v28, v29, v6, v7, v4, v5, v16 + hv_luma_store_8 v24, v25, \shift1 + st1 {v24.8h}, [x0], x20 + + v_luma_8_row v24, v25, v28, v29, v6, v7, v4, v5, v16, v17 + hv_luma_store_8 v24, v25, \shift1 + st1 {v24.8h}, [x0], x20 + + v_luma_8_row v24, v25, v29, v6, v7, v4, v5, v16, v17, v18 + hv_luma_store_8 v24, v25, \shift1 + st1 {v24.8h}, [x0], x20 + + mov v30.16b, v6.16b + mov v31.16b, v7.16b + mov v28.16b, v4.16b + mov v29.16b, v5.16b + mov v6.16b, v16.16b + mov v7.16b, v17.16b + mov v4.16b, v18.16b + + subs w4, w4, #4 + b.hi 3b + + // Advance column pointers + add x19, x19, #16 // dst += 8 pixels + add x21, x21, #16 // src += 8 pixels + subs w24, w24, #1 + b.hi 1b + + add x9, sp, #2160 + ldp x19, x20, [x9] + ldp x21, x22, [x9, #16] + ldp x23, x24, [x9, #32] + ldp x25, x26, [x9, #48] + ldp x27, x28, [x9, #64] + ldp x29, x30, [x9, #80] + add sp, sp, #2256 + ret +endfunc +.endm + +put_uni_hv_luma_4 10, 2, 10, 512, 1023 +put_uni_hv_luma_wide ff_vvc_put_uni_pixels_hv_luma_8_10_neon, 8, 10, 2, 10, 512, 1023 +put_uni_hv_luma_wide ff_vvc_put_uni_pixels_hv_luma_16_10_neon, 16, 10, 2, 10, 512, 1023 +put_uni_hv_luma_wide ff_vvc_put_uni_pixels_hv_luma_32_10_neon, 32, 10, 2, 10, 512, 1023 +put_uni_hv_luma_wide ff_vvc_put_uni_pixels_hv_luma_64_10_neon, 64, 10, 2, 10, 512, 1023 +put_uni_hv_luma_wide ff_vvc_put_uni_pixels_hv_luma_128_10_neon, 128, 10, 2, 10, 512, 1023 + +put_uni_hv_luma_4 12, 4, 8, 128, 4095 +put_uni_hv_luma_wide ff_vvc_put_uni_pixels_hv_luma_8_12_neon, 8, 12, 4, 8, 128, 4095 +put_uni_hv_luma_wide ff_vvc_put_uni_pixels_hv_luma_16_12_neon, 16, 12, 4, 8, 128, 4095 +put_uni_hv_luma_wide ff_vvc_put_uni_pixels_hv_luma_32_12_neon, 32, 12, 4, 8, 128, 4095 +put_uni_hv_luma_wide ff_vvc_put_uni_pixels_hv_luma_64_12_neon, 64, 12, 4, 8, 128, 4095 +put_uni_hv_luma_wide ff_vvc_put_uni_pixels_hv_luma_128_12_neon, 128, 12, 4, 8, 128, 4095 -- 2.52.0 From ae5745d15ca1e376a517ca648b27578b11db84bf Mon Sep 17 00:00:00 2001 From: Aaditya Agarwal <[email protected]> Date: Wed, 7 Oct 2026 23:37:42 +0530 Subject: [PATCH 9/9] aarch64/vvc: vectorize ALF classification index resolution Two architectural changes eliminate the scalar classification bottleneck that was capping throughput at 1.09x-1.17x for 10/12-bit paths: 1. Populate ff_alf_classify_grad_10_neon: the 10-bit gradient computation stub was empty, causing the entire gradient pass to fall back to C. It now maps to the pix_size=2 macro path, matching the existing 12-bit implementation. 2. Replace scalar alf_get_idx with ff_alf_get_idx_neon: the index resolution loop called the scalar C function twice per two-block group (512 calls per 128x128 CTU). The new branchless NEON routine handles both blocks in a single call using csel/cset for directional comparisons and umull x9/x12 for the 64-bit cross-product (to avoid the w10 clobber that would corrupt hv1 before csel). The arg_var lookup table uses movrel for PIC-compliant PC-relative addressing. Benchmarked on Apple M4 (checkasm --test=vvc_alf --bench): vvc_alf_classify_8x8_8: 691.4 ns -> 48.6 ns (13.99x) vvc_alf_classify_8x8_10: 717.7 ns -> 41.0 ns (17.49x) vvc_alf_classify_8x8_12: 684.5 ns -> 41.3 ns (16.44x) vvc_alf_classify_128x128_8: 108249.7 ns -> 6755.3 ns (15.96x) vvc_alf_classify_128x128_10: 107869.9 ns -> 6428.0 ns (16.78x) vvc_alf_classify_128x128_12: 111856.1 ns -> 6401.0 ns (17.47x) All 4032 checkasm tests pass across 8/10/12-bit. --- libavcodec/aarch64/vvc/alf.S | 223 ++++++++++++++++++++++++++ libavcodec/aarch64/vvc/alf_template.c | 17 +- 2 files changed, 234 insertions(+), 6 deletions(-) diff --git a/libavcodec/aarch64/vvc/alf.S b/libavcodec/aarch64/vvc/alf.S index bd8317aba0..ef1330899d 100644 --- a/libavcodec/aarch64/vvc/alf.S +++ b/libavcodec/aarch64/vvc/alf.S @@ -491,8 +491,231 @@ function ff_alf_classify_grad_8_neon, export=1 endfunc function ff_alf_classify_grad_10_neon, export=1 + ff_alf_classify_grad 2 endfunc function ff_alf_classify_grad_12_neon, export=1 ff_alf_classify_grad 2 endfunc + +// ff_alf_get_idx_neon: branchless vectorized alf_get_idx for two 4x4 blocks. +// +// Eliminates the two scalar alf_get_idx() C calls per two-block group, +// replacing them with a single NEON-assisted routine that resolves class_idx +// and transpose_idx for both 4x4 blocks with zero function-call overhead. +// +// Arguments (AArch64 ABI): +// x0 = class_idx (int*, output, 2 ints written: [0] and [1]) +// x1 = transpose_idx (int*, output, 2 ints written: [0] and [1]) +// x2 = sum0 (const int[4]: {V, H, D0, D1} for block 0) +// x3 = sum1 (const int[4]: {V, H, D0, D1} for block 1) +// w4 = ac (2 or 3, the activity scale factor) +// w5 = bit_depth (8, 10, or 12) +// +// Register convention (leaf, no stack frame): +// v16, v17 - NEON: sum0 and sum1 loaded once and held throughout. +// w6 - dir1_N as 0/1 integer. +// w7 - strong_offset intermediate. +// w8-w15 - scratch (thresholds, index values, temporaries). +// w16,w17 - hvd1_N, hvd0_N (dominant max/min gradient pair). +// x9,x10 - 64-bit cross-product operands (promote to avoid 32-bit overflow). +// +// Mathematical reference (H.264 Annex C / VVC adaptation): +// dir_hv = V <= H +// hv1 = max(V, H); hv0 = min(V, H) +// dir_d = D0 <= D1 +// d1 = max(D0, D1); d0 = min(D0, D1) +// dir1 = d1*hv0 <= hv1*d0 (cross-product, 64-bit safe) +// hvd1 = dir1 ? hv1 : d1 +// hvd0 = dir1 ? hv0 : d0 +// sum_hv = V + H +// base = arg_var[ clip(sum_hv*ac >> (bit_depth-1), 0, 15) ] +// if 2*hvd1 > 9*hvd0: class_idx = base + ((dir1<<1)+2)*5 +// elif hvd1 > 2*hvd0: class_idx = base + ((dir1<<1)+1)*5 +// else: class_idx = base +// transpose_idx = dir_d*2 + dir_hv +// +function ff_alf_get_idx_neon, export=1 + // Load both 4-direction gradient sums once and hold in NEON. + // v16 = sum0: { V0, H0, D0_0, D1_0 } (4 × u32) + // v17 = sum1: { V1, H1, D0_1, D1_1 } + ld1 {v16.4s}, [x2] + ld1 {v17.4s}, [x3] + + // ------------------------------------------------------------------ + // BLOCK 0 + // ------------------------------------------------------------------ + + // Step 1a: VH max / min for block 0 + mov w8, v16.s[0] // V0 + mov w9, v16.s[1] // H0 + cmp w8, w9 + csel w10, w8, w9, gt // hv1_0 = max(V0, H0) + csel w11, w9, w8, gt // hv0_0 = min(V0, H0) + + // Step 1b: D max / min for block 0 + mov w8, v16.s[2] // D0_0 + mov w9, v16.s[3] // D1_0 + cmp w8, w9 + csel w13, w8, w9, gt // d1_0 = max(D0_0, D1_0) + csel w14, w9, w8, gt // d0_0 = min(D0_0, D1_0) + + // Step 2: dir1_0 = (d1_0 * hv0_0 <= hv1_0 * d0_0) [64-bit, no overflow] + // Use x12/x13 (caller-saved) for 64-bit products to avoid clobbering + // w10 (hv1_0) and w11 (hv0_0) which are needed for the csel below. + umull x9, w13, w11 // x9 = d1_0 * hv0_0 + umull x12, w10, w14 // x12 = hv1_0 * d0_0 (w10 preserved) + cmp x9, x12 // ls means d1*hv0 <= hv1*d0 (unsigned) → dir1=1 + csel w16, w10, w13, ls // hvd1_0: dir1=1 → hv1_0; else d1_0 + csel w17, w11, w14, ls // hvd0_0: dir1=1 → hv0_0; else d0_0 + cset w6, ls // dir1_0 as integer 0/1 + + // Step 3a: arg_var index for block 0 + // idx = clip( (V0+H0) * ac >> (bit_depth-1), 0, 15 ) + mov w8, v16.s[0] // V0 + mov w9, v16.s[1] // H0 + add w8, w8, w9 // sum_hv_0 = V0 + H0 + mul w8, w8, w4 // * ac + sub w9, w5, #1 // bit_depth - 1 + lsr w8, w8, w9 // >> (bit_depth - 1) + mov w9, #15 + cmp w8, w9 + csel w8, w9, w8, gt // clip to [0, 15] + movrel x11, alf_arg_var + ldr w8, [x11, w8, uxtw #2] // class_idx_base_0 = arg_var[idx] + + // Step 3b: threshold increments (branchless) + // Compute 2*hvd0 and 9*hvd0 for the two threshold comparisons. + lsl w9, w17, #1 // 2*hvd0_0 + lsl w12, w17, #3 + add w12, w12, w17 // 9*hvd0_0 + lsl w13, w16, #1 // 2*hvd1_0 + + cmp w13, w12 + cset w14, hi // strong_0 = (2*hvd1 > 9*hvd0) + cmp w16, w9 + cset w15, hi // medium_0 = (hvd1 > 2*hvd0) + + // strong_offset = ((dir1<<1)+2)*5; medium_offset = ((dir1<<1)+1)*5 + lsl w6, w6, #1 // dir1_0 << 1 + add w7, w6, #2 + add w7, w7, w7, lsl #2 // strong_offset = w7 * 5 + add w12, w6, #1 + add w12, w12, w12, lsl #2 // medium_offset = w12 * 5 + + // strong_inc: strong ? strong_offset : 0 + cmp w14, #1 + csel w7, w7, wzr, eq + + // medium_inc: (!strong & medium) ? medium_offset : 0 + eor w9, w14, #1 // ~strong_0 + and w9, w9, w15 // medium_only_0 + mul w12, w12, w9 // medium_inc (0 if !medium_only) + + add w8, w8, w7 // + strong_inc + add w8, w8, w12 // + medium_inc → class_idx_0 + + // Step 4: transpose_idx_0 = dir_d_0*2 + dir_hv_0 + mov w9, v16.s[0] // V0 + mov w10, v16.s[1] // H0 + cmp w9, w10 + cset w9, le // dir_hv_0 = (V0 <= H0) + mov w10, v16.s[2] // D0_0 + mov w11, v16.s[3] // D1_0 + cmp w10, w11 + cset w10, le // dir_d_0 = (D0_0 <= D1_0) + lsl w10, w10, #1 + add w10, w10, w9 // transpose_idx_0 = dir_d*2 + dir_hv + + // Store block 0 outputs + str w8, [x0] + str w10, [x1] + + // ------------------------------------------------------------------ + // BLOCK 1 (identical pipeline, fed from v17) + // ------------------------------------------------------------------ + + // Step 1a: VH for block 1 + mov w8, v17.s[0] // V1 + mov w9, v17.s[1] // H1 + cmp w8, w9 + csel w10, w8, w9, gt // hv1_1 + csel w11, w9, w8, gt // hv0_1 + + // Step 1b: D for block 1 + mov w8, v17.s[2] // D0_1 + mov w9, v17.s[3] // D1_1 + cmp w8, w9 + csel w13, w8, w9, gt // d1_1 + csel w14, w9, w8, gt // d0_1 + + // Step 2: dir1_1 (same pipeline) + umull x9, w13, w11 // x9 = d1_1 * hv0_1 + umull x12, w10, w14 // x12 = hv1_1 * d0_1 (w10 intact) + cmp x9, x12 + csel w16, w10, w13, ls // hvd1_1 + csel w17, w11, w14, ls // hvd0_1 + cset w6, ls // dir1_1 + + // Step 3a: arg_var index for block 1 + mov w8, v17.s[0] // V1 + mov w9, v17.s[1] // H1 + add w8, w8, w9 + mul w8, w8, w4 + sub w9, w5, #1 + lsr w8, w8, w9 + mov w9, #15 + cmp w8, w9 + csel w8, w9, w8, gt + movrel x11, alf_arg_var + ldr w8, [x11, w8, uxtw #2] + + // Step 3b: thresholds for block 1 + lsl w9, w17, #1 // 2*hvd0_1 + lsl w12, w17, #3 + add w12, w12, w17 // 9*hvd0_1 + lsl w13, w16, #1 // 2*hvd1_1 + + cmp w13, w12 + cset w14, hi // strong_1 + cmp w16, w9 + cset w15, hi // medium_1 + + lsl w6, w6, #1 + add w7, w6, #2 + add w7, w7, w7, lsl #2 // strong_offset_1 = w7 * 5 + add w12, w6, #1 + add w12, w12, w12, lsl #2 // medium_offset_1 = w12 * 5 + + cmp w14, #1 + csel w7, w7, wzr, eq // strong_inc_1 + + eor w9, w14, #1 + and w9, w9, w15 // medium_only_1 + mul w12, w12, w9 // medium_inc_1 + + add w8, w8, w7 + add w8, w8, w12 // class_idx_1 final + + // Step 4: transpose_idx_1 + mov w9, v17.s[0] + mov w10, v17.s[1] + cmp w9, w10 + cset w9, le // dir_hv_1 + mov w10, v17.s[2] + mov w11, v17.s[3] + cmp w10, w11 + cset w10, le // dir_d_1 + lsl w10, w10, #1 + add w10, w10, w9 // transpose_idx_1 + + // Store block 1 outputs + str w8, [x0, #4] + str w10, [x1, #4] + + ret +endfunc + +const alf_arg_var + .word 0, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 4 +endconst diff --git a/libavcodec/aarch64/vvc/alf_template.c b/libavcodec/aarch64/vvc/alf_template.c index 03bf9056bb..32e93af251 100644 --- a/libavcodec/aarch64/vvc/alf_template.c +++ b/libavcodec/aarch64/vvc/alf_template.c @@ -190,6 +190,12 @@ static void FUNC(alf_get_idx)(int *class_idx, int *transpose_idx, const int *sum *transpose_idx = dir_d * 2 + dir_hv; } +// Vectorized dual-block alf_get_idx: resolves class_idx and transpose_idx +// for two adjacent 4x4 blocks simultaneously, eliminating the scalar call overhead. +void ff_alf_get_idx_neon(int *class_idx, int *transpose_idx, + const int *sum0, const int *sum1, + int ac, int bit_depth); + static void FUNC(alf_classify)(int *class_idx, int *transpose_idx, const uint8_t *_src, const ptrdiff_t _src_stride, const int width, const int height, const int vb_pos, int16_t *gradient_tmp) @@ -219,12 +225,11 @@ static void FUNC(alf_classify)(int *class_idx, int *transpose_idx, int sum1[ALF_NUM_DIR]; grad = gradient_tmp + (yg + start) * gstride + xg * ALF_NUM_DIR; ff_alf_classify_sum_neon(sum0, sum1, grad, gshift, end-start); - FUNC(alf_get_idx)(class_idx, transpose_idx, sum0, ac); - class_idx++; - transpose_idx++; - FUNC(alf_get_idx)(class_idx, transpose_idx, sum1, ac); - class_idx++; - transpose_idx++; + // One NEON call replaces two scalar alf_get_idx() calls, + // eliminating function-call overhead for both blocks at once. + ff_alf_get_idx_neon(class_idx, transpose_idx, sum0, sum1, ac, BIT_DEPTH); + class_idx += 2; + transpose_idx += 2; } } -- 2.52.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
