PR #24544 opened by Devin Heitmueller (dheitmueller) URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24544 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24544.patch
Provide assembly implementations for bitpacked yuv422, used mainly in my case for dealing with ST2110-20 streams Individual patches include their respective benchmarks. >From aba08a20d2548db39975b8a461782bacf03607fa Mon Sep 17 00:00:00 2001 From: Devin Heitmueller <[email protected]> Date: Wed, 16 Sep 2026 14:50:28 -0400 Subject: [PATCH 1/4] avcodec/bitpacked: add AArch64 NEON unpacking Unpack packed 10-bit YUV 4:2:2 in 16-pixel NEON blocks and unroll the main loop to 32 pixels. Keep exact-width source loads and use the scalar implementation for decoder tails smaller than eight pixels. The implementation passed 300 consecutive checkasm repetitions and produced bit-exact output in a 100-frame UHD decode. Correctness command: ./tests/checkasm/checkasm --test=bitpackeddec --repeat=300 93000 Benchmark command: ./tests/checkasm/checkasm --bench --test=bitpackeddec \ --duration=1000000 95000 Apple M4 Mac mini, Apple clang 17.0.0: bitpacked_unpack_yuv422p10_c: 856.2 ns bitpacked_unpack_yuv422p10_neon: 228.0 ns (3.75x) Clang auto-vectorizes the C reference with NEON, so the result measures the hand-written implementation against compiler-generated SIMD rather than against purely scalar machine code. --- libavcodec/aarch64/Makefile | 2 + libavcodec/aarch64/bitpacked_init_aarch64.c | 42 ++++++++ libavcodec/aarch64/bitpacked_neon.S | 110 ++++++++++++++++++++ libavcodec/bitpacked_dec.c | 30 +++--- libavcodec/bitpacked_dec.h | 40 +++++++ libavcodec/bitpacked_dec_init.h | 56 ++++++++++ 6 files changed, 267 insertions(+), 13 deletions(-) create mode 100644 libavcodec/aarch64/bitpacked_init_aarch64.c create mode 100644 libavcodec/aarch64/bitpacked_neon.S create mode 100644 libavcodec/bitpacked_dec.h create mode 100644 libavcodec/bitpacked_dec_init.h diff --git a/libavcodec/aarch64/Makefile b/libavcodec/aarch64/Makefile index e0762e3fa1..b1ceb36668 100644 --- a/libavcodec/aarch64/Makefile +++ b/libavcodec/aarch64/Makefile @@ -21,6 +21,7 @@ OBJS-$(CONFIG_VP8DSP) += aarch64/vp8dsp_init_aarch64.o OBJS-$(CONFIG_AAC_DECODER) += aarch64/aacpsdsp_init_aarch64.o \ aarch64/sbrdsp_init_aarch64.o OBJS-$(CONFIG_AAC_ENCODER) += aarch64/aacencdsp_init.o +OBJS-$(CONFIG_BITPACKED_DECODER) += aarch64/bitpacked_init_aarch64.o OBJS-$(CONFIG_DCA_DECODER) += aarch64/dcadsp_init_aarch64.o \ aarch64/synth_filter_init.o OBJS-$(CONFIG_OPUS_DECODER) += aarch64/opusdsp_init.o @@ -64,6 +65,7 @@ NEON-OBJS-$(CONFIG_VP8DSP) += aarch64/vp8dsp_neon.o # decoders/encoders NEON-OBJS-$(CONFIG_AAC_DECODER) += aarch64/aacpsdsp_neon.o +NEON-OBJS-$(CONFIG_BITPACKED_DECODER) += aarch64/bitpacked_neon.o NEON-OBJS-$(CONFIG_DCA_DECODER) += aarch64/dcadsp_neon.o \ aarch64/synth_filter_neon.o NEON-OBJS-$(CONFIG_OPUS_DECODER) += aarch64/opusdsp_neon.o diff --git a/libavcodec/aarch64/bitpacked_init_aarch64.c b/libavcodec/aarch64/bitpacked_init_aarch64.c new file mode 100644 index 0000000000..4c3aa71feb --- /dev/null +++ b/libavcodec/aarch64/bitpacked_init_aarch64.c @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2026 Devin Heitmueller <[email protected]> + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include <stdint.h> + +#include "config.h" +#include "libavutil/attributes.h" +#include "libavutil/aarch64/cpu.h" +#include "libavcodec/bitpacked_dec.h" + +#if HAVE_NEON +void ff_bitpacked_unpack_yuv422p10_neon(const uint8_t *src, + uint16_t *y, uint16_t *u, uint16_t *v, + int width); +#endif + +av_cold void ff_bitpacked_init_aarch64(struct BitpackedContext *s) +{ +#if HAVE_NEON + int cpu_flags = av_get_cpu_flags(); + + if (have_neon(cpu_flags)) + s->unpack_yuv422p10 = ff_bitpacked_unpack_yuv422p10_neon; +#endif +} diff --git a/libavcodec/aarch64/bitpacked_neon.S b/libavcodec/aarch64/bitpacked_neon.S new file mode 100644 index 0000000000..383d20c638 --- /dev/null +++ b/libavcodec/aarch64/bitpacked_neon.S @@ -0,0 +1,110 @@ +/* + * Bitpacked 10-bit YUV 4:2:2 unpacking + * Copyright (c) 2026 Devin Heitmueller <[email protected]> + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "libavutil/aarch64/asm.S" + +const bitpacked_y0_tbl, align=4 + .byte 2, 1, 4, 3, 7, 6, 9, 8, 12, 11, 14, 13, 17, 16, 19, 18 +endconst + +const bitpacked_y1_tbl, align=4 + .byte 22, 21, 24, 23, 27, 26, 29, 28 + .byte 32, 31, 34, 33, 37, 36, 39, 38 +endconst + +const bitpacked_u_tbl, align=4 + .byte 1, 0, 6, 5, 11, 10, 16, 15, 21, 20, 26, 25, 31, 30, 36, 35 +endconst + +const bitpacked_v_tbl, align=4 + .byte 3, 2, 8, 7, 13, 12, 18, 17, 23, 22, 28, 27, 33, 32, 38, 37 +endconst + +const bitpacked_y_mul, align=4 + .hword 4, 64, 4, 64, 4, 64, 4, 64 +endconst + +// Consume 40 bytes and produce 16 Y, 8 U and 8 V samples. TBL reverses +// each big-endian byte pair; 16-bit multiply/shift overflow discards unwanted +// high bits before the common right shift. The main loop unrolls to 80 bytes. +.macro unpack_16 + ld1 {v0.16b, v1.16b}, [x0], #32 + ldr d2, [x0], #8 + + tbl v3.16b, {v0.16b-v2.16b}, v16.16b + tbl v4.16b, {v0.16b-v2.16b}, v17.16b + tbl v5.16b, {v0.16b-v2.16b}, v18.16b + tbl v6.16b, {v0.16b-v2.16b}, v19.16b + + mul v3.8h, v3.8h, v20.8h + mul v4.8h, v4.8h, v20.8h + ushr v3.8h, v3.8h, #6 + ushr v4.8h, v4.8h, #6 + ushr v5.8h, v5.8h, #6 + shl v6.8h, v6.8h, #4 + ushr v6.8h, v6.8h, #6 + + st1 {v3.8h, v4.8h}, [x1], #32 + str q5, [x2], #16 + str q6, [x3], #16 +.endm + +// ff_bitpacked_unpack_yuv422p10_neon(const uint8_t *src, +// uint16_t *y, uint16_t *u, uint16_t *v, +// int width) +function ff_bitpacked_unpack_yuv422p10_neon, export=1 + movrel x5, bitpacked_y0_tbl + ld1 {v16.16b-v19.16b}, [x5], #64 + ldr q20, [x5] + + subs w4, w4, #32 + b.lt 2f + +1: unpack_16 + unpack_16 + subs w4, w4, #32 + b.ge 1b + +2: add w4, w4, #32 + tbz w4, #4, 3f + unpack_16 + sub w4, w4, #16 + +3: cbz w4, 4f + ldr q0, [x0], #16 + ldr s1, [x0] + + tbl v3.16b, {v0.16b-v1.16b}, v16.16b + tbl v5.16b, {v0.16b-v1.16b}, v18.16b + tbl v6.16b, {v0.16b-v1.16b}, v19.16b + + mul v3.8h, v3.8h, v20.8h + ushr v3.8h, v3.8h, #6 + ushr v5.8h, v5.8h, #6 + shl v6.8h, v6.8h, #4 + ushr v6.8h, v6.8h, #6 + + str q3, [x1] + str d5, [x2] + str d6, [x3] + +4: ret +endfunc diff --git a/libavcodec/bitpacked_dec.c b/libavcodec/bitpacked_dec.c index a1d3b7b505..8d88c556cc 100644 --- a/libavcodec/bitpacked_dec.c +++ b/libavcodec/bitpacked_dec.c @@ -27,15 +27,12 @@ */ #include "avcodec.h" +#include "bitpacked_dec.h" +#include "bitpacked_dec_init.h" #include "codec_internal.h" #include "libavutil/imgutils.h" #include "thread.h" -struct BitpackedContext { - int (*decode)(AVCodecContext *avctx, AVFrame *frame, - const AVPacket *pkt); -}; - /* For this format, it's a simple passthrough */ static int bitpacked_decode_uyvy422(AVCodecContext *avctx, AVFrame *frame, const AVPacket *avpkt) @@ -62,11 +59,13 @@ static int bitpacked_decode_uyvy422(AVCodecContext *avctx, AVFrame *frame, static int bitpacked_decode_yuv422p10(AVCodecContext *avctx, AVFrame *frame, const AVPacket *avpkt) { + struct BitpackedContext *bc = avctx->priv_data; uint64_t frame_size = (uint64_t)avctx->width * (uint64_t)avctx->height * 20; uint64_t packet_size = (uint64_t)avpkt->size * 8; - uint8_t *src; + const uint8_t *src; uint16_t *y, *u, *v; - int ret, i, j; + const int bulk = avctx->width & ~7; + int ret, i; ret = ff_thread_get_buffer(avctx, frame, 0); if (ret < 0) @@ -84,13 +83,16 @@ static int bitpacked_decode_yuv422p10(AVCodecContext *avctx, AVFrame *frame, u = (uint16_t*)(frame->data[1] + i * frame->linesize[1]); v = (uint16_t*)(frame->data[2] + i * frame->linesize[2]); - for (j = 0; j < avctx->width; j += 2) { - *u++ = (src[0] << 2) | (src[1] >> 6); - *v++ = ((src[2] << 6) | (src[3] >> 2)) & 0x3ff; - *y++ = ((src[1] << 4) | (src[2] >> 4)) & 0x3ff; - *y++ = ((src[3] << 8) | (src[4])) & 0x3ff; - src += 5; + if (bulk) { + bc->unpack_yuv422p10(src, y, u, v, bulk); + src += bulk * 5 / 2; + y += bulk; + u += bulk / 2; + v += bulk / 2; } + + bitpacked_unpack_yuv422p10_c(src, y, u, v, avctx->width - bulk); + src += (avctx->width - bulk) * 5 / 2; } return 0; @@ -103,6 +105,8 @@ static av_cold int bitpacked_init_decoder(AVCodecContext *avctx) if (!avctx->codec_tag || !avctx->width || !avctx->height) return AVERROR_INVALIDDATA; + ff_bitpackeddec_init(bc); + if (avctx->codec_tag == MKTAG('U', 'Y', 'V', 'Y')) { if (avctx->bits_per_coded_sample == 16 && avctx->pix_fmt == AV_PIX_FMT_UYVY422) diff --git a/libavcodec/bitpacked_dec.h b/libavcodec/bitpacked_dec.h new file mode 100644 index 0000000000..3859c17667 --- /dev/null +++ b/libavcodec/bitpacked_dec.h @@ -0,0 +1,40 @@ +/* + * Unpack bit-packed streams to formats supported by FFmpeg + * Copyright (c) 2026 Devin Heitmueller <[email protected]> + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef AVCODEC_BITPACKED_DEC_H +#define AVCODEC_BITPACKED_DEC_H + +#include <stdint.h> + +#include "avcodec.h" + +struct BitpackedContext { + int (*decode)(AVCodecContext *avctx, AVFrame *frame, + const AVPacket *pkt); + + void (*unpack_yuv422p10)(const uint8_t *src, + uint16_t *y, uint16_t *u, uint16_t *v, + int width); +}; + +void ff_bitpacked_init_aarch64(struct BitpackedContext *s); + +#endif /* AVCODEC_BITPACKED_DEC_H */ diff --git a/libavcodec/bitpacked_dec_init.h b/libavcodec/bitpacked_dec_init.h new file mode 100644 index 0000000000..10b39b207b --- /dev/null +++ b/libavcodec/bitpacked_dec_init.h @@ -0,0 +1,56 @@ +/* + * Bitpacked decoder DSP init + * Copyright (c) 2017 Savoir-faire Linux, Inc + * Copyright (c) 2026 Devin Heitmueller <[email protected]> + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/* Development sponsored by CBC/Radio-Canada */ + +#ifndef AVCODEC_BITPACKED_DEC_INIT_H +#define AVCODEC_BITPACKED_DEC_INIT_H + +#include <stdint.h> + +#include "config.h" +#include "libavutil/attributes.h" +#include "bitpacked_dec.h" + +static void bitpacked_unpack_yuv422p10_c(const uint8_t *src, + uint16_t *y, uint16_t *u, uint16_t *v, + int width) +{ + for (int i = 0; i < width; i += 2) { + *u++ = (src[0] << 2) | (src[1] >> 6); + *v++ = ((src[2] << 6) | (src[3] >> 2)) & 0x3ff; + *y++ = ((src[1] << 4) | (src[2] >> 4)) & 0x3ff; + *y++ = ((src[3] << 8) | src[4]) & 0x3ff; + src += 5; + } +} + +av_unused static av_cold void ff_bitpackeddec_init(struct BitpackedContext *s) +{ + s->unpack_yuv422p10 = bitpacked_unpack_yuv422p10_c; + +#if ARCH_AARCH64 + ff_bitpacked_init_aarch64(s); +#endif +} + +#endif /* AVCODEC_BITPACKED_DEC_INIT_H */ -- 2.52.0 >From 9e77d1209139b66b635f29cb9327e716340f3d10 Mon Sep 17 00:00:00 2001 From: Devin Heitmueller <[email protected]> Date: Wed, 16 Sep 2026 14:50:36 -0400 Subject: [PATCH 2/4] checkasm: add bitpacked decoder test Compare packed input and all three output planes across the NEON block sizes, scalar-tail widths, and a 3840-pixel benchmark case. --- tests/checkasm/Makefile | 1 + tests/checkasm/bitpackeddec.c | 103 ++++++++++++++++++++++++++++++++++ tests/checkasm/checkasm.c | 3 + tests/checkasm/checkasm.h | 1 + tests/fate/checkasm.mak | 1 + 5 files changed, 109 insertions(+) create mode 100644 tests/checkasm/bitpackeddec.c diff --git a/tests/checkasm/Makefile b/tests/checkasm/Makefile index c5b25e753a..244fa0a732 100644 --- a/tests/checkasm/Makefile +++ b/tests/checkasm/Makefile @@ -36,6 +36,7 @@ AVCODECOBJS-$(CONFIG_AAC_DECODER) += aacpsdsp.o \ AVCODECOBJS-$(CONFIG_AAC_ENCODER) += aacencdsp.o AVCODECOBJS-$(CONFIG_ALAC_DECODER) += alacdsp.o AVCODECOBJS-$(CONFIG_APV_DECODER) += apv_dsp.o +AVCODECOBJS-$(CONFIG_BITPACKED_DECODER) += bitpackeddec.o AVCODECOBJS-$(CONFIG_CAVS_DECODER) += cavsdsp.o AVCODECOBJS-$(CONFIG_DCA_DECODER) += dcadsp.o synth_filter.o AVCODECOBJS-$(CONFIG_DIRAC_DECODER) += diracdsp.o diff --git a/tests/checkasm/bitpackeddec.c b/tests/checkasm/bitpackeddec.c new file mode 100644 index 0000000000..9a8aeda8c1 --- /dev/null +++ b/tests/checkasm/bitpackeddec.c @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2026 Devin Heitmueller <[email protected]> + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with FFmpeg; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include <string.h> + +#include "libavutil/mem_internal.h" + +#include "checkasm.h" +#include "libavcodec/bitpacked_dec_init.h" + +#define MAX_WIDTH 3840 + +static void randomize_buffers(uint8_t *src0, uint8_t *src1, int len) +{ + for (int i = 0; i < len; i++) { + uint8_t value = rnd(); + src0[i] = value; + src1[i] = value; + } +} + +void checkasm_check_bitpackeddec(void) +{ + static const int widths[] = { 8, 16, 24, 32, MAX_WIDTH }; + static const int tail_widths[] = { 2, 6, 10, 14, 16 }; + struct BitpackedContext h; + + ff_bitpackeddec_init(&h); + + if (check_func(h.unpack_yuv422p10, "bitpacked_unpack_yuv422p10")) { + LOCAL_ALIGNED_16(uint8_t, src0, [MAX_WIDTH * 5 / 2]); + LOCAL_ALIGNED_16(uint8_t, src1, [MAX_WIDTH * 5 / 2]); + LOCAL_ALIGNED_16(uint16_t, y0, [MAX_WIDTH]); + LOCAL_ALIGNED_16(uint16_t, y1, [MAX_WIDTH]); + LOCAL_ALIGNED_16(uint16_t, u0, [MAX_WIDTH / 2]); + LOCAL_ALIGNED_16(uint16_t, u1, [MAX_WIDTH / 2]); + LOCAL_ALIGNED_16(uint16_t, v0, [MAX_WIDTH / 2]); + LOCAL_ALIGNED_16(uint16_t, v1, [MAX_WIDTH / 2]); + declare_func(void, const uint8_t *src, uint16_t *y, uint16_t *u, + uint16_t *v, int width); + + for (int i = 0; i < FF_ARRAY_ELEMS(widths); i++) { + const int width = widths[i]; + const int src_size = width * 5 / 2; + + randomize_buffers(src0, src1, src_size); + memset(y0, 0, width * sizeof(*y0)); + memset(y1, 0, width * sizeof(*y1)); + memset(u0, 0, width / 2 * sizeof(*u0)); + memset(u1, 0, width / 2 * sizeof(*u1)); + memset(v0, 0, width / 2 * sizeof(*v0)); + memset(v1, 0, width / 2 * sizeof(*v1)); + + call_ref(src0, y0, u0, v0, width); + call_new(src1, y1, u1, v1, width); + if (memcmp(src0, src1, src_size) || + memcmp(y0, y1, width * sizeof(*y0)) || + memcmp(u0, u1, width / 2 * sizeof(*u0)) || + memcmp(v0, v1, width / 2 * sizeof(*v0))) + fail(); + } + + for (int i = 0; i < FF_ARRAY_ELEMS(tail_widths); i++) { + const int width = tail_widths[i]; + const int bulk = width & ~7; + const int src_size = width * 5 / 2; + + randomize_buffers(src0, src1, src_size); + bitpacked_unpack_yuv422p10_c(src0, y0, u0, v0, width); + if (bulk) + h.unpack_yuv422p10(src1, y1, u1, v1, bulk); + bitpacked_unpack_yuv422p10_c(src1 + bulk * 5 / 2, + y1 + bulk, u1 + bulk / 2, v1 + bulk / 2, + width - bulk); + if (memcmp(src0, src1, src_size) || + memcmp(y0, y1, width * sizeof(*y0)) || + memcmp(u0, u1, width / 2 * sizeof(*u0)) || + memcmp(v0, v1, width / 2 * sizeof(*v0))) + fail(); + } + + randomize_buffers(src0, src1, MAX_WIDTH * 5 / 2); + bench_new(src1, y1, u1, v1, MAX_WIDTH); + } + report("bitpacked_unpack_yuv422p10"); +} diff --git a/tests/checkasm/checkasm.c b/tests/checkasm/checkasm.c index f0f6f097de..a70464093c 100644 --- a/tests/checkasm/checkasm.c +++ b/tests/checkasm/checkasm.c @@ -78,6 +78,9 @@ static const CheckasmTest tests[] = { #if CONFIG_AUDIODSP { "audiodsp", checkasm_check_audiodsp }, #endif + #if CONFIG_BITPACKED_DECODER + { "bitpackeddec", checkasm_check_bitpackeddec }, + #endif #if CONFIG_BLOCKDSP { "blockdsp", checkasm_check_blockdsp }, #endif diff --git a/tests/checkasm/checkasm.h b/tests/checkasm/checkasm.h index 84fc0b0c08..81701e201e 100644 --- a/tests/checkasm/checkasm.h +++ b/tests/checkasm/checkasm.h @@ -39,6 +39,7 @@ void checkasm_check_alacdsp(void); void checkasm_check_apv_dsp(void); void checkasm_check_audiodsp(void); void checkasm_check_av_tx(void); +void checkasm_check_bitpackeddec(void); void checkasm_check_blackdetect(void); void checkasm_check_blend(void); void checkasm_check_blockdsp(void); diff --git a/tests/fate/checkasm.mak b/tests/fate/checkasm.mak index fad24e5861..de01ce56a5 100644 --- a/tests/fate/checkasm.mak +++ b/tests/fate/checkasm.mak @@ -8,6 +8,7 @@ FATE_CHECKASM = fate-checkasm-aacencdsp \ fate-checkasm-apv_dsp \ fate-checkasm-audiodsp \ fate-checkasm-av_tx \ + fate-checkasm-bitpackeddec \ fate-checkasm-blockdsp \ fate-checkasm-bswapdsp \ fate-checkasm-cavsdsp \ -- 2.52.0 >From 95114573d247904cd0fab92a29e3de7604d06f3d Mon Sep 17 00:00:00 2001 From: Devin Heitmueller <[email protected]> Date: Wed, 16 Sep 2026 15:48:38 -0400 Subject: [PATCH 3/4] avcodec/bitpacked: add x86 SIMD unpacking Add runtime-dispatched SSSE3 and AVX2 implementations for packed 10-bit YUV 4:2:2. The SSSE3 loop follows the negative-width output-offset pattern used by v210. The AVX2 loop processes 16 pixels at a time and has an internal eight-pixel tail. Both implementations use exact-width input loads and therefore do not rely on padded input or legal overread. They passed 300 consecutive checkasm seeds. Benchmark command: ./tests/checkasm/checkasm --bench --test=bitpackeddec \ --duration=1000000 51000 Intel Xeon E-2356G, GCC 8.5, NASM 2.15.03: bitpacked_unpack_yuv422p10_c: 13516.7 bitpacked_unpack_yuv422p10_ssse3: 3415.8 (3.96x) bitpacked_unpack_yuv422p10_avx2: 2183.1 (6.19x) --- libavcodec/bitpacked_dec.h | 1 + libavcodec/bitpacked_dec_init.h | 2 + libavcodec/x86/Makefile | 2 + libavcodec/x86/bitpacked.asm | 186 ++++++++++++++++++++++++++++++++ libavcodec/x86/bitpacked_init.c | 47 ++++++++ 5 files changed, 238 insertions(+) create mode 100644 libavcodec/x86/bitpacked.asm create mode 100644 libavcodec/x86/bitpacked_init.c diff --git a/libavcodec/bitpacked_dec.h b/libavcodec/bitpacked_dec.h index 3859c17667..1860b7037d 100644 --- a/libavcodec/bitpacked_dec.h +++ b/libavcodec/bitpacked_dec.h @@ -36,5 +36,6 @@ struct BitpackedContext { }; void ff_bitpacked_init_aarch64(struct BitpackedContext *s); +void ff_bitpacked_init_x86(struct BitpackedContext *s); #endif /* AVCODEC_BITPACKED_DEC_H */ diff --git a/libavcodec/bitpacked_dec_init.h b/libavcodec/bitpacked_dec_init.h index 10b39b207b..423ecacc39 100644 --- a/libavcodec/bitpacked_dec_init.h +++ b/libavcodec/bitpacked_dec_init.h @@ -50,6 +50,8 @@ av_unused static av_cold void ff_bitpackeddec_init(struct BitpackedContext *s) #if ARCH_AARCH64 ff_bitpacked_init_aarch64(s); +#elif ARCH_X86 && HAVE_X86ASM + ff_bitpacked_init_x86(s); #endif } diff --git a/libavcodec/x86/Makefile b/libavcodec/x86/Makefile index 456a209482..b1b38ef8aa 100644 --- a/libavcodec/x86/Makefile +++ b/libavcodec/x86/Makefile @@ -48,6 +48,7 @@ X86ASM-OBJS-$(CONFIG_ADPCM_G722_ENCODER) += x86/g722dsp_init.o X86ASM-OBJS-$(CONFIG_ALAC_DECODER) += x86/alacdsp_init.o X86ASM-OBJS-$(CONFIG_APNG_DECODER) += x86/pngdsp_init.o X86ASM-OBJS-$(CONFIG_APV_DECODER) += x86/apv_dsp_init.o +X86ASM-OBJS-$(CONFIG_BITPACKED_DECODER) += x86/bitpacked_init.o X86ASM-OBJS-$(CONFIG_CAVS_DECODER) += x86/cavsdsp.o X86ASM-OBJS-$(CONFIG_CFHD_DECODER) += x86/cfhddsp_init.o X86ASM-OBJS-$(CONFIG_CFHD_ENCODER) += x86/cfhdencdsp_init.o @@ -150,6 +151,7 @@ X86ASM-OBJS-$(CONFIG_ADPCM_G722_ENCODER) += x86/g722dsp.o X86ASM-OBJS-$(CONFIG_ALAC_DECODER) += x86/alacdsp.o X86ASM-OBJS-$(CONFIG_APNG_DECODER) += x86/pngdsp.o X86ASM-OBJS-$(CONFIG_APV_DECODER) += x86/apv_dsp.o +X86ASM-OBJS-$(CONFIG_BITPACKED_DECODER) += x86/bitpacked.o X86ASM-OBJS-$(CONFIG_CAVS_DECODER) += x86/cavsidct.o x86/cavs_qpel.o X86ASM-OBJS-$(CONFIG_CFHD_ENCODER) += x86/cfhdencdsp.o X86ASM-OBJS-$(CONFIG_CFHD_DECODER) += x86/cfhddsp.o diff --git a/libavcodec/x86/bitpacked.asm b/libavcodec/x86/bitpacked.asm new file mode 100644 index 0000000000..db15a2f91f --- /dev/null +++ b/libavcodec/x86/bitpacked.asm @@ -0,0 +1,186 @@ +;****************************************************************************** +;* Bitpacked 10-bit YUV 4:2:2 unpacking +;* Copyright (c) 2026 Devin Heitmueller <[email protected]> +;* +;* This file is part of FFmpeg. +;* +;* FFmpeg is free software; you can redistribute it and/or +;* modify it under the terms of the GNU Lesser General Public +;* License as published by the Free Software Foundation; either +;* version 2.1 of the License, or (at your option) any later version. +;* +;* FFmpeg is distributed in the hope that it will be useful, +;* but WITHOUT ANY WARRANTY; without even the implied warranty of +;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +;* Lesser General Public License for more details. +;* +;* You should have received a copy of the GNU Lesser General Public +;* License along with FFmpeg; if not, write to the Free Software +;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +;****************************************************************************** + +%include "libavutil/x86/x86util.asm" + +SECTION_RODATA 32 + +unpack_shuf_avx2: + db 1,0, 2,1, 3,2, 4,3, 6,5, 7,6, 8,7, 9,8 + db 5,4, 6,5, 7,6, 8,7, 10,9, 11,10, 12,11, 13,12 +u_shuf_ymm: + db 0,1, 8,9, -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 + db 0,1, 8,9, -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 +v_shuf_ymm: + db 4,5, 12,13, -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 + db 4,5, 12,13, -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 + +unpack_shuf: db 1,0, 2,1, 3,2, 4,3, 6,5, 7,6, 8,7, 9,8 +unpack_mult: dw 1,4,16,64, 1,4,16,64 +y_shuf: db 2,3, 6,7, 10,11, 14,15, -1,-1,-1,-1,-1,-1,-1,-1 +u_shuf: db 0,1, 8,9, -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 +v_shuf: db 4,5, 12,13, -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 + +SECTION .text + +; ff_bitpacked_unpack_yuv422p10(const uint8_t *src, +; uint16_t *y, uint16_t *u, uint16_t *v, +; int width) +; Form four big-endian 16-bit windows per five input bytes. Multiplication by +; 1, 4, 16 and 64 aligns all four samples for a common six-bit right shift. +INIT_XMM ssse3 +cglobal bitpacked_unpack_yuv422p10, 5, 5, 8, src, y, u, v, w + movsxdifnidn wq, wd + lea yq, [yq+2*wq] + add uq, wq + add vq, wq + neg wq + + mova m4, [unpack_shuf] + mova m5, [unpack_mult] + mova m6, [y_shuf] + mova m7, [u_shuf] + +.loop: + movq m0, [srcq] + pinsrw m0, [srcq+8], 4 + movq m1, [srcq+10] + pinsrw m1, [srcq+18], 4 + + pshufb m0, m4 + pshufb m1, m4 + pmullw m0, m5 + pmullw m1, m5 + psrlw m0, 6 + psrlw m1, 6 + + mova m2, m0 + mova m3, m1 + pshufb m2, m6 + pshufb m3, m6 + punpcklqdq m2, m3 + movu [yq+2*wq], m2 + + mova m2, m0 + mova m3, m1 + pshufb m2, m7 + pshufb m3, m7 + punpckldq m2, m3 + movq [uq+wq], m2 + + pshufb m0, [v_shuf] + pshufb m1, [v_shuf] + punpckldq m0, m1 + movq [vq+wq], m0 + + add srcq, 20 + add wq, 8 + jl .loop + + RET + +%if HAVE_AVX2_EXTERNAL +%macro unpack_16 0 + ; m0 contains the first and third 10-byte groups; m1 the second and fourth. + movu m0, [srcq] + movq xm1, [srcq+10] + pinsrw xm1, [srcq+18], 4 + movq xm2, [srcq+30] + pinsrw xm2, [srcq+38], 4 + vinserti128 m1, m1, xm2, 1 + + pshufb m0, m4 + pshufb m1, m5 + pmullw m0, m6 + pmullw m1, m6 + psrlw m0, 6 + psrlw m1, 6 + + pshufb m2, m0, m7 + pshufb m3, m1, m7 + punpcklqdq m2, m3 + movu [yq], m2 + + pshufb m2, m0, [u_shuf_ymm] + pshufb m3, m1, [u_shuf_ymm] + punpckldq m2, m3 + vpermq m2, m2, 0xd8 + movu [uq], xm2 + + pshufb m2, m0, [v_shuf_ymm] + pshufb m3, m1, [v_shuf_ymm] + punpckldq m2, m3 + vpermq m2, m2, 0xd8 + movu [vq], xm2 + + add srcq, 40 + add yq, 32 + add uq, 16 + add vq, 16 +%endmacro + +INIT_YMM avx2 +cglobal bitpacked_unpack_yuv422p10, 5, 5, 8, src, y, u, v, w + mova m4, [unpack_shuf_avx2] + VBROADCASTI128 m5, [unpack_shuf] + VBROADCASTI128 m6, [unpack_mult] + VBROADCASTI128 m7, [y_shuf] + + sub wd, 16 + jl .tail +.loop: + unpack_16 + sub wd, 16 + jge .loop + +.tail: + add wd, 16 + jz .ret + movq xm0, [srcq] + pinsrw xm0, [srcq+8], 4 + movq xm1, [srcq+10] + pinsrw xm1, [srcq+18], 4 + vinserti128 m0, m0, xm1, 1 + + pshufb m0, m5 + pmullw m0, m6 + psrlw m0, 6 + + pshufb m1, m0, m7 + vextracti128 xm2, m1, 1 + punpcklqdq xm1, xm2 + movu [yq], xm1 + + VBROADCASTI128 m3, [u_shuf] + pshufb m1, m0, m3 + vextracti128 xm2, m1, 1 + punpckldq xm1, xm2 + movq [uq], xm1 + + VBROADCASTI128 m3, [v_shuf] + pshufb m1, m0, m3 + vextracti128 xm2, m1, 1 + punpckldq xm1, xm2 + movq [vq], xm1 + +.ret: + RET +%endif diff --git a/libavcodec/x86/bitpacked_init.c b/libavcodec/x86/bitpacked_init.c new file mode 100644 index 0000000000..01fce4ba16 --- /dev/null +++ b/libavcodec/x86/bitpacked_init.c @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2026 Devin Heitmueller <[email protected]> + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include <stdint.h> + +#include "libavutil/attributes.h" +#include "libavutil/x86/cpu.h" +#include "libavcodec/bitpacked_dec.h" + +void ff_bitpacked_unpack_yuv422p10_ssse3(const uint8_t *src, + uint16_t *y, uint16_t *u, uint16_t *v, + int width); +#if HAVE_AVX2_EXTERNAL +void ff_bitpacked_unpack_yuv422p10_avx2(const uint8_t *src, + uint16_t *y, uint16_t *u, uint16_t *v, + int width); +#endif + +av_cold void ff_bitpacked_init_x86(struct BitpackedContext *s) +{ + int cpu_flags = av_get_cpu_flags(); + + if (EXTERNAL_SSSE3(cpu_flags)) + s->unpack_yuv422p10 = ff_bitpacked_unpack_yuv422p10_ssse3; + +#if HAVE_AVX2_EXTERNAL + if (EXTERNAL_AVX2(cpu_flags)) + s->unpack_yuv422p10 = ff_bitpacked_unpack_yuv422p10_avx2; +#endif +} -- 2.52.0 >From 845568e0b660acf4d4772eb468fb2e45dbfa2a16 Mon Sep 17 00:00:00 2001 From: Devin Heitmueller <[email protected]> Date: Wed, 16 Sep 2026 15:59:03 -0400 Subject: [PATCH 4/4] avcodec/bitpacked: add AVX-512ICL unpacking Use a masked exact-width load and VBMI byte permutations to expand each 40-byte group, then compact the planar Y, U, and V outputs. Handle an eight-pixel prefix separately so the hot loop can use the negative-width output-offset pattern from v210. The AVX-512ICL feature tier is required for vpermb. CPUs with baseline AVX-512 but without VBMI continue to use AVX2. Reuse one result register for all three output planes so the function also assembles for 32-bit x86. The implementation passed 300 consecutive checkasm seeds and assembled as ELF, Windows COFF, and Mach-O in both 32- and 64-bit modes. Benchmark command: ./tests/checkasm/checkasm --bench --test=bitpackeddec \ --duration=1000000 60300 Intel Xeon E-2356G, GCC 8.5, NASM 2.15.03: bitpacked_unpack_yuv422p10_c: 13502.8 bitpacked_unpack_yuv422p10_ssse3: 3436.6 ( 3.93x) bitpacked_unpack_yuv422p10_avx2: 2222.8 ( 6.07x) bitpacked_unpack_yuv422p10_avx512icl: 1131.5 (11.93x) --- libavcodec/x86/bitpacked.asm | 81 ++++++++++++++++++++++++++++++++- libavcodec/x86/bitpacked_init.c | 10 ++++ 2 files changed, 90 insertions(+), 1 deletion(-) diff --git a/libavcodec/x86/bitpacked.asm b/libavcodec/x86/bitpacked.asm index db15a2f91f..deccdcd557 100644 --- a/libavcodec/x86/bitpacked.asm +++ b/libavcodec/x86/bitpacked.asm @@ -21,8 +21,27 @@ %include "libavutil/x86/x86util.asm" -SECTION_RODATA 32 +SECTION_RODATA 64 +unpack_shuf_avx512icl: + db 1, 0, 2, 1, 3, 2, 4, 3, 6, 5, 7, 6, 8, 7, 9, 8 + db 11,10, 12,11, 13,12, 14,13, 16,15, 17,16, 18,17, 19,18 + db 21,20, 22,21, 23,22, 24,23, 26,25, 27,26, 28,27, 29,28 + db 31,30, 32,31, 33,32, 34,33, 36,35, 37,36, 38,37, 39,38 +y_shuf_avx512icl: + db 2, 3, 6, 7, 10,11, 14,15, 18,19, 22,23, 26,27, 30,31 + db 34,35, 38,39, 42,43, 46,47, 50,51, 54,55, 58,59, 62,63 + times 32 db 0 +u_shuf_avx512icl: + db 0, 1, 8, 9, 16,17, 24,25, 32,33, 40,41, 48,49, 56,57 + times 48 db 0 +v_shuf_avx512icl: + db 4, 5, 12,13, 20,21, 28,29, 36,37, 44,45, 52,53, 60,61 + times 48 db 0 +input_mask_avx512icl: dq 0x000000ffffffffff +tail_mask_avx512icl: dq 0x00000000000fffff + +ALIGN 32 unpack_shuf_avx2: db 1,0, 2,1, 3,2, 4,3, 6,5, 7,6, 8,7, 9,8 db 5,4, 6,5, 7,6, 8,7, 10,9, 11,10, 12,11, 13,12 @@ -184,3 +203,63 @@ cglobal bitpacked_unpack_yuv422p10, 5, 5, 8, src, y, u, v, w .ret: RET %endif + +%if HAVE_AVX512ICL_EXTERNAL +INIT_ZMM avx512icl +cglobal bitpacked_unpack_yuv422p10, 5, 5, 7, src, y, u, v, w + mova m2, [unpack_shuf_avx512icl] + VBROADCASTI128 m3, [unpack_mult] + mova m4, [y_shuf_avx512icl] + mova m5, [u_shuf_avx512icl] + mova m6, [v_shuf_avx512icl] + kmovq k1, [input_mask_avx512icl] + kmovq k2, [tail_mask_avx512icl] + + test wd, 8 + jz .setup + vmovdqu8 m0{k2}{z}, [srcq] + vpermb m0, m2, m0 + pmullw m0, m3 + psrlw m0, 6 + + vpermb m1, m4, m0 + movu [yq], xm1 + vpermb m1, m5, m0 + movq [uq], xm1 + vpermb m1, m6, m0 + movq [vq], xm1 + + add srcq, 20 + add yq, 16 + add uq, 8 + add vq, 8 + sub wd, 8 + jz .ret + +.setup: + movsxdifnidn wq, wd + lea yq, [yq+2*wq] + add uq, wq + add vq, wq + neg wq + +.loop: + vmovdqu8 m0{k1}{z}, [srcq] + vpermb m0, m2, m0 + pmullw m0, m3 + psrlw m0, 6 + + vpermb m1, m4, m0 + movu [yq+2*wq], ym1 + vpermb m1, m5, m0 + movu [uq+wq], xm1 + vpermb m1, m6, m0 + movu [vq+wq], xm1 + + add srcq, 40 + add wq, 16 + jl .loop + +.ret: + RET +%endif diff --git a/libavcodec/x86/bitpacked_init.c b/libavcodec/x86/bitpacked_init.c index 01fce4ba16..4c5295befb 100644 --- a/libavcodec/x86/bitpacked_init.c +++ b/libavcodec/x86/bitpacked_init.c @@ -32,6 +32,11 @@ void ff_bitpacked_unpack_yuv422p10_avx2(const uint8_t *src, uint16_t *y, uint16_t *u, uint16_t *v, int width); #endif +#if HAVE_AVX512ICL_EXTERNAL +void ff_bitpacked_unpack_yuv422p10_avx512icl(const uint8_t *src, + uint16_t *y, uint16_t *u, + uint16_t *v, int width); +#endif av_cold void ff_bitpacked_init_x86(struct BitpackedContext *s) { @@ -44,4 +49,9 @@ av_cold void ff_bitpacked_init_x86(struct BitpackedContext *s) if (EXTERNAL_AVX2(cpu_flags)) s->unpack_yuv422p10 = ff_bitpacked_unpack_yuv422p10_avx2; #endif + +#if HAVE_AVX512ICL_EXTERNAL + if (EXTERNAL_AVX512ICL(cpu_flags)) + s->unpack_yuv422p10 = ff_bitpacked_unpack_yuv422p10_avx512icl; +#endif } -- 2.52.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
