PR #25163 opened by zuxy URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/25163 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/25163.patch
Convert the last MMX (mmx/mmxext) IDCT functions, idct8_dc_add_8, idct_add8_422_8 (plus its plane and DC helpers) and idct_dc_add_10, to SSE2. The DC_ADD macros are renamed to DC_ADD_SSE2 (pshuflw, movq traffic) and shared with the pre-existing sse2/avx IDCT_XMM path. The mmxext DC helper tail-called from SSE2 functions is now XMM, which also removes the MMX-state handoff without emms. Dispatch moves from EXTERNAL_MMX/MMXEXT to EXTERNAL_SSE2. checkasm h264dsp passes. identical performances. >From 48046a4fd8535e957b5a5170c33e709d5bfed88e Mon Sep 17 00:00:00 2001 From: Zuxy Meng <[email protected]> Date: Sat, 17 Oct 2026 19:36:32 -0700 Subject: [PATCH] avcodec/x86/h264: convert IDCT MMX to SSE2 Convert the last MMX (mmx/mmxext) IDCT functions, idct8_dc_add_8, idct_add8_422_8 (plus its plane and DC helpers) and idct_dc_add_10, to SSE2. The DC_ADD macros are renamed to DC_ADD_SSE2 (pshuflw, movq traffic) and shared with the pre-existing sse2/avx IDCT_XMM path. The mmxext DC helper tail-called from SSE2 functions is now XMM, which also removes the MMX-state handoff without emms. Dispatch moves from EXTERNAL_MMX/MMXEXT to EXTERNAL_SSE2. checkasm h264dsp passes. identical performances. --- libavcodec/x86/h264_idct.asm | 107 ++++++++++++++--------------- libavcodec/x86/h264_idct_10bit.asm | 28 ++++---- libavcodec/x86/h264dsp_init.c | 23 ++----- tests/checkasm/h264dsp.c | 84 +++++++++++++++++++++- 4 files changed, 156 insertions(+), 86 deletions(-) diff --git a/libavcodec/x86/h264_idct.asm b/libavcodec/x86/h264_idct.asm index 1d4c9729fa..d9fadabefc 100644 --- a/libavcodec/x86/h264_idct.asm +++ b/libavcodec/x86/h264_idct.asm @@ -68,6 +68,22 @@ SECTION .text %endmacro ; %1=uint8_t *dst, %2=int16_t *block, %3=ptrdiff_t stride +; STORE_DIFFx2 not suitable here: it uses movh +%macro STORE_DIFFx4 8 ; add1, add2, reg1, reg2, zero, shift, source, stride + movd %3, [%7] + movd %4, [%7+%8] + psraw %1, %6 + psraw %2, %6 + punpcklbw %3, %5 + punpcklbw %4, %5 + paddw %3, %1 + paddw %4, %2 + packuswb %3, %5 + packuswb %4, %5 + movd [%7], %3 + movd [%7+%8], %4 +%endmacro + %macro IDCT4_ADD 3 ; Load dct coeffs movq m0, [%2] @@ -99,9 +115,9 @@ SECTION .text movq [%2+24], m7 %endif - STORE_DIFFx2 m0, m1, m4, m5, m7, 6, %1, %3 + STORE_DIFFx4 m0, m1, m4, m5, m7, 6, %1, %3 lea %1, [%1+%3*2] - STORE_DIFFx2 m2, m3, m4, m5, m7, 6, %1, %3 + STORE_DIFFx4 m2, m3, m4, m5, m7, 6, %1, %3 %endmacro %macro IDCT8_1D 2 @@ -230,19 +246,21 @@ cglobal h264_idct8_add_8, 3, 4, 10 IDCT8_ADD_SSE r0, r1, r2, r3 RET -%macro DC_ADD_MMXEXT_INIT 2 +%macro DC_ADD_SSE2_INIT 2 add %1, 32 sar %1, 6 movd m0, %1d lea %1, [%2*3] - pshufw m0, m0, 0 + pshuflw m0, m0, 0 pxor m1, m1 psubw m1, m0 packuswb m0, m0 packuswb m1, m1 + pshufd m0, m0, 0 + pshufd m1, m1, 0 %endmacro -%macro DC_ADD_MMXEXT_OP 4 +%macro DC_ADD_SSE2_OP 4 %1 m2, [%2 ] %1 m3, [%2+%3 ] %1 m4, [%2+%3*2] @@ -261,27 +279,27 @@ cglobal h264_idct8_add_8, 3, 4, 10 %1 [%2+%4 ], m5 %endmacro -INIT_MMX mmxext +INIT_XMM sse2 %if ARCH_X86_64 -; void ff_h264_idct8_dc_add_8_mmxext(uint8_t *dst, int16_t *block, ptrdiff_t stride) -cglobal h264_idct8_dc_add_8, 3, 4, 0 +; void ff_h264_idct8_dc_add_8_sse2(uint8_t *dst, int16_t *block, ptrdiff_t stride) +cglobal h264_idct8_dc_add_8, 3, 4, 6 movsx r3, word [r1] mov dword [r1], 0 - DC_ADD_MMXEXT_INIT r3, r2 - DC_ADD_MMXEXT_OP mova, r0, r2, r3 + DC_ADD_SSE2_INIT r3, r2 + DC_ADD_SSE2_OP movq, r0, r2, r3 lea r0, [r0+r2*4] - DC_ADD_MMXEXT_OP mova, r0, r2, r3 + DC_ADD_SSE2_OP movq, r0, r2, r3 RET %else -; void ff_h264_idct8_dc_add_8_mmxext(uint8_t *dst, int16_t *block, ptrdiff_t stride) -cglobal h264_idct8_dc_add_8, 2, 3, 0 +; void ff_h264_idct8_dc_add_8_sse2(uint8_t *dst, int16_t *block, ptrdiff_t stride) +cglobal h264_idct8_dc_add_8, 2, 3, 6 movsx r2, word [r1] mov dword [r1], 0 mov r1, r2m - DC_ADD_MMXEXT_INIT r2, r1 - DC_ADD_MMXEXT_OP mova, r0, r1, r2 + DC_ADD_SSE2_INIT r2, r1 + DC_ADD_SSE2_OP movq, r0, r1, r2 lea r0, [r0+r1*4] - DC_ADD_MMXEXT_OP mova, r0, r1, r2 + DC_ADD_SSE2_OP movq, r0, r1, r2 RET %endif @@ -304,18 +322,17 @@ cglobal h264_idct8_add4_8, 5, 8 + npicregs, 10, dst1, block_offset, block, strid movsx r6, word [r2] test r6, r6 jz .no_dc -INIT_MMX cpuname mov word [r2], 0 - DC_ADD_MMXEXT_INIT r6, r3 + DC_ADD_SSE2_INIT r6, r3 %if ARCH_X86_64 == 0 %define dst2q r1 %define dst2d r1d %endif mov dst2d, dword [r1+r5*4] add dst2q, r0 - DC_ADD_MMXEXT_OP mova, dst2q, r3, r6 + DC_ADD_SSE2_OP movq, dst2q, r3, r6 lea dst2q, [dst2q+r3*4] - DC_ADD_MMXEXT_OP mova, dst2q, r3, r6 + DC_ADD_SSE2_OP movq, dst2q, r3, r6 %if ARCH_X86_64 == 0 mov r1, r1m %endif @@ -325,7 +342,6 @@ INIT_MMX cpuname jl .nextblock RET .no_dc: -INIT_XMM cpuname mov dst2d, dword [r1+r5*4] add dst2q, r0 IDCT8_ADD_SSE dst2q, r2, r3, r6 @@ -339,8 +355,8 @@ INIT_XMM cpuname jl .nextblock RET -INIT_MMX mmx -cglobal h264_idct_add8_422_8, 5, 8 + npicregs, 0, dst1, block_offset, block, stride, nnzc, cntr, coeff, dst2, picreg +INIT_XMM sse2 +cglobal h264_idct_add8_422_8, 5, 8 + npicregs, 8, dst1, block_offset, block, stride, nnzc, cntr, coeff, dst2, picreg ; dst1, block_offset, block, stride, nnzc, cntr, coeff, dst2, picreg %if PIC lea picregq, [scan8_mem] @@ -352,9 +368,9 @@ cglobal h264_idct_add8_422_8, 5, 8 + npicregs, 0, dst1, block_offset, block, str mov r5, 16 ; i add r2, 512 ; i * 16 * sizeof(dctcoef) ; #define dctcoef int16_t - call h264_idct_add8_mmx_plane + call h264_idct_add8_sse2_plane add r5, 4 - call h264_idct_add8_mmx_plane + call h264_idct_add8_sse2_plane %if ARCH_X86_64 add dst2q, gprsize ; dest[1] @@ -365,11 +381,11 @@ cglobal h264_idct_add8_422_8, 5, 8 + npicregs, 0, dst1, block_offset, block, str add r5, 4 ; set to 32 add r2, 256 ; set to i * 16 * sizeof(dctcoef) - call h264_idct_add8_mmx_plane + call h264_idct_add8_sse2_plane add r5, 4 - TAIL_CALL h264_idct_add8_mmx_plane, 0 + TAIL_CALL h264_idct_add8_sse2_plane, 0 -h264_idct_add8_mmx_plane: +h264_idct_add8_sse2_plane: .nextblock: movzx r6d, byte [scan8+r5] movzx r6d, byte [r4+r6] @@ -394,7 +410,7 @@ h264_idct_add8_mmx_plane: ; r0 = uint8_t *dst, r2 = int16_t *block, r3 = ptrdiff_t stride, r6=clobbered -h264_idct_dc_add8_mmxext: +h264_idct_dc_add8_sse2: movd m0, [r2 ] ; 0 0 X D mov word [r2+ 0], 0 punpcklwd m0, [r2+32] ; x X d D @@ -404,11 +420,12 @@ h264_idct_dc_add8_mmxext: punpcklwd m0, m0 ; d d D D pxor m1, m1 ; 0 0 0 0 psubw m1, m0 ; -d-d-D-D - packuswb m0, m1 ; -d-d-D-D d d D D - pshufw m1, m0, 0xFA ; -d-d-d-d-D-D-D-D + packuswb m0, m1 ; d d D D 0.. -d-d-D-D 0.. + pshufd m1, m0, q3232 ; -d-d-D-D .. + punpcklwd m1, m1 ; -d-d-d-d-D-D-D-D punpcklwd m0, m0 ; d d d d D D D D lea r6, [r3*3] - DC_ADD_MMXEXT_OP movq, r0, r3, r6 + DC_ADD_SSE2_OP movq, r0, r3, r6 ret ALIGN 16 @@ -494,7 +511,7 @@ RET %else add r0, r0m %endif - TAIL_CALL_IF_LAST h264_idct_dc_add8_mmxext, , %1, 7 + TAIL_CALL_IF_LAST h264_idct_dc_add8_sse2, , %1, 7 .cycle%1end: %if %1 < 7 add r2, 64 @@ -543,7 +560,7 @@ RET mov r0, [r0] add r0, dword [r1+(%1&1)*8+64*(1+(%1>>1))] %endif - TAIL_CALL_IF_LAST h264_idct_dc_add8_mmxext, , %1, 3 + TAIL_CALL_IF_LAST h264_idct_dc_add8_sse2, , %1, 3 .cycle%1end: %if %1 == 1 add r2, 384+64 @@ -698,26 +715,6 @@ cglobal h264_luma_dc_dequant_idct, 3, 4, 6 DEQUANT_STORE m6 RET -%ifdef __NASM_VER__ -%if __NASM_MAJOR__ > 2 || (__NASM_MAJOR__ == 2 && __NASM_MINOR__ >= 4) -%unmacro STORE_DIFFx2 8 ; remove macro from x86util.asm but yasm doesn't have this yet -%endif -%endif -%macro STORE_DIFFx2 8 ; add1, add2, reg1, reg2, zero, shift, source, stride - movd %3, [%7] - movd %4, [%7+%8] - psraw %1, %6 - psraw %2, %6 - punpcklbw %3, %5 - punpcklbw %4, %5 - paddw %3, %1 - paddw %4, %2 - packuswb %3, %5 - packuswb %4, %5 - movd [%7], %3 - movd [%7+%8], %4 -%endmacro - %macro DC_ADD_INIT 1 add %1d, 32 sar %1d, 6 @@ -742,7 +739,7 @@ cglobal h264_idct_dc_add_8, 3, 4, 6, dst_, block_, stride_ movsx r3d, word [block_q] mov dword [block_q], 0 DC_ADD_INIT r3 - DC_ADD_MMXEXT_OP movd, dst_q, stride_q, r3 + DC_ADD_SSE2_OP movd, dst_q, stride_q, r3 RET %endmacro diff --git a/libavcodec/x86/h264_idct_10bit.asm b/libavcodec/x86/h264_idct_10bit.asm index 741b79d044..1c3e7c8012 100644 --- a/libavcodec/x86/h264_idct_10bit.asm +++ b/libavcodec/x86/h264_idct_10bit.asm @@ -166,7 +166,9 @@ IDCT_ADD16_10 ;----------------------------------------------------------------------------- ; void ff_h264_idct_dc_add_10(pixel *dst, int16_t *block, ptrdiff_t stride) ;----------------------------------------------------------------------------- -%macro IDCT_DC_ADD_OP_10 3 +; %4 selects the dst traffic: mova for 16-byte (8-wide) rows, movq +; for 8-byte (4-wide) rows whose alignment is only guaranteed to 8. +%macro IDCT_DC_ADD_OP_10 3-4 mova pxor m5, m5 %if avx_enabled paddw m1, m0, [%1+0 ] @@ -174,10 +176,10 @@ IDCT_ADD16_10 paddw m3, m0, [%1+%2*2] paddw m4, m0, [%1+%3 ] %else - mova m1, [%1+0 ] - mova m2, [%1+%2 ] - mova m3, [%1+%2*2] - mova m4, [%1+%3 ] + %4 m1, [%1+0 ] + %4 m2, [%1+%2 ] + %4 m3, [%1+%2*2] + %4 m4, [%1+%3 ] paddw m1, m0 paddw m2, m0 paddw m3, m0 @@ -187,22 +189,22 @@ IDCT_ADD16_10 CLIPW m2, m5, m6 CLIPW m3, m5, m6 CLIPW m4, m5, m6 - mova [%1+0 ], m1 - mova [%1+%2 ], m2 - mova [%1+%2*2], m3 - mova [%1+%3 ], m4 + %4 [%1+0 ], m1 + %4 [%1+%2 ], m2 + %4 [%1+%2*2], m3 + %4 [%1+%3 ], m4 %endmacro -INIT_MMX mmxext -cglobal h264_idct_dc_add_10,3,3 +INIT_XMM sse2 +cglobal h264_idct_dc_add_10,3,3,7 movd m0, [r1] mov dword [r1], 0 paddd m0, [pd_32] psrad m0, 6 lea r1, [r2*3] - pshufw m0, m0, 0 + pshuflw m0, m0, 0 mova m6, [pw_pixel_max] - IDCT_DC_ADD_OP_10 r0, r2, r1 + IDCT_DC_ADD_OP_10 r0, r2, r1, movq RET ;----------------------------------------------------------------------------- diff --git a/libavcodec/x86/h264dsp_init.c b/libavcodec/x86/h264dsp_init.c index 654316c2af..d103fe041f 100644 --- a/libavcodec/x86/h264dsp_init.c +++ b/libavcodec/x86/h264dsp_init.c @@ -35,8 +35,8 @@ IDCT_ADD_FUNC(, 8, avx) IDCT_ADD_FUNC(, 10, sse2) IDCT_ADD_FUNC(_dc, 8, sse2) IDCT_ADD_FUNC(_dc, 8, avx) -IDCT_ADD_FUNC(_dc, 10, mmxext) -IDCT_ADD_FUNC(8_dc, 8, mmxext) +IDCT_ADD_FUNC(_dc, 10, sse2) +IDCT_ADD_FUNC(8_dc, 8, sse2) IDCT_ADD_FUNC(8_dc, 10, sse2) IDCT_ADD_FUNC(8, 8, sse2) IDCT_ADD_FUNC(8, 10, sse2) @@ -70,7 +70,7 @@ IDCT_ADD_REP_FUNC2(, 8, 8, sse2) IDCT_ADD_REP_FUNC2(, 8, 10, sse2) IDCT_ADD_REP_FUNC2(, 8, 10, avx) -IDCT_ADD_REP_FUNC2(, 8_422, 8, mmx) +IDCT_ADD_REP_FUNC2(, 8_422, 8, sse2) IDCT_ADD_REP_FUNC2(, 8_422, 10, sse2) IDCT_ADD_REP_FUNC2(, 8_422, 10, avx) @@ -186,22 +186,15 @@ av_cold void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth, c->loop_filter_strength = ff_h264_loop_filter_strength_mmxext; if (bit_depth == 8) { - if (EXTERNAL_MMX(cpu_flags)) { - if (chroma_format_idc <= 1) { - } else { - c->idct_add8 = ff_h264_idct_add8_422_8_mmx; - } - } - if (EXTERNAL_MMXEXT(cpu_flags)) { - c->idct8_dc_add = ff_h264_idct8_dc_add_8_mmxext; - } if (EXTERNAL_SSE2(cpu_flags)) { c->idct8_add = ff_h264_idct8_add_8_sse2; - + c->idct8_dc_add = ff_h264_idct8_dc_add_8_sse2; c->idct_add16 = ff_h264_idct_add16_8_sse2; c->idct8_add4 = ff_h264_idct8_add4_8_sse2; if (chroma_format_idc <= 1) c->idct_add8 = ff_h264_idct_add8_8_sse2; + else + c->idct_add8 = ff_h264_idct_add8_422_8_sse2; c->idct_add16intra = ff_h264_idct_add16intra_8_sse2; c->luma_dc_dequant_idct = ff_h264_luma_dc_dequant_idct_sse2; @@ -263,10 +256,8 @@ av_cold void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth, c->idct_dc_add = ff_h264_idct_dc_add_8_avx; } } else if (bit_depth == 10) { - if (EXTERNAL_MMXEXT(cpu_flags)) { - c->idct_dc_add = ff_h264_idct_dc_add_10_mmxext; - } if (EXTERNAL_SSE2(cpu_flags)) { + c->idct_dc_add = ff_h264_idct_dc_add_10_sse2; c->idct_add = ff_h264_idct_add_10_sse2; c->idct8_dc_add = ff_h264_idct8_dc_add_10_sse2; diff --git a/tests/checkasm/h264dsp.c b/tests/checkasm/h264dsp.c index 3bd0dec2c3..6e9792a0f6 100644 --- a/tests/checkasm/h264dsp.c +++ b/tests/checkasm/h264dsp.c @@ -184,7 +184,7 @@ static void check_idct(void) LOCAL_ALIGNED_16(int16_t, subcoef1, [8 * 8 * 2]); H264DSPContext h; int bit_depth, sz, align, dc, i; - declare_func_emms(AV_CPU_FLAG_MMX, void, uint8_t *dst, int16_t *block, ptrdiff_t stride); + declare_func(void, uint8_t *dst, int16_t *block, ptrdiff_t stride); for (i = 0; i < FF_ARRAY_ELEMS(depths); i++) { bit_depth = depths[i]; @@ -250,7 +250,7 @@ static void check_idct_multiple(void) LOCAL_ALIGNED_16(uint8_t, nnzc, [15 * 8]); H264DSPContext h; int bit_depth, i, y, func; - declare_func_emms(AV_CPU_FLAG_MMX, void, uint8_t *dst, const int *block_offset, int16_t *block, ptrdiff_t stride, const uint8_t nnzc[15*8]); + declare_func(void, uint8_t *dst, const int *block_offset, int16_t *block, ptrdiff_t stride, const uint8_t nnzc[15*8]); for (bit_depth = 8; bit_depth <= 10; bit_depth++) { ff_h264dsp_init(&h, bit_depth, 1); @@ -325,6 +325,85 @@ static void check_idct_multiple(void) } } +static void check_idct_add8(void) +{ + LOCAL_ALIGNED_16(uint8_t, dst_full, [16 * 32 * 2]); + LOCAL_ALIGNED_16(int16_t, coef_full, [48 * 16 * 2]); + LOCAL_ALIGNED_16(uint8_t, dst0, [16 * 32 * 2]); + LOCAL_ALIGNED_16(uint8_t, dst1, [16 * 32 * 2]); + LOCAL_ALIGNED_16(int16_t, coef0, [48 * 16 * 2]); + LOCAL_ALIGNED_16(int16_t, coef1, [48 * 16 * 2]); + LOCAL_ALIGNED_16(uint8_t, nnzc, [15 * 8]); + H264DSPContext h; + int bit_depth, format, j, k; + int sz = 4; + int block_offset[48] = { 0 }; + declare_func(void, uint8_t **dst, const int *block_offset, + int16_t *block, ptrdiff_t stride, const uint8_t nnzc[15*8]); + + for (bit_depth = 8; bit_depth <= 10; bit_depth += 2) { + for (format = 1; format <= 2; format++) { + void (*idct)(uint8_t **, const int *, int16_t *, ptrdiff_t, const uint8_t[]) = NULL; + const char *name = format == 1 ? "idct_add8" : "idct_add8_422"; + uint8_t *dstp0[2], *dstp1[2]; + + ff_h264dsp_init(&h, bit_depth, format); + idct = h.idct_add8; + + memset(nnzc, 0, 15 * 8); + memset(coef_full, 0, 48 * 16 * SIZEOF_COEF); + for (j = 1; j < 3; j++) { + for (k = 0; k < (format == 1 ? 4 : 8); k++) { + uint8_t src[8 * 8 * 2]; + uint8_t dst[8 * 8 * 2]; + int16_t coef[16 * 2]; + int second = k >= 4; + int i = j * 16 + (k % 4) + (second ? 4 : 0); + int bo = second ? i + 4 : i; + int row = (second ? 8 : 0); + int col = (k % 4) * 4; + int offset = (row * 16 + col) * SIZEOF_PIXEL + + (j - 1) * 16 * 16 * SIZEOF_PIXEL; + int nnz = rnd() % 3; + int16_t *cblk = &coef_full[i * 16 * SIZEOF_COEF / 2]; + + randomize_buffers(bit_depth - 8); + dct4x4(coef, bit_depth); + + if (nnz > 1) + nnz = 16; + memcpy(cblk, coef, nnz * SIZEOF_COEF); + + for (int y = 0; y < 4; y++) + memcpy(&dst_full[offset + y * 16 * SIZEOF_PIXEL], + &dst[y * 4 * SIZEOF_PIXEL], 4 * SIZEOF_PIXEL); + + nnzc[scan8[second ? i + 4 : i]] = nnz; + block_offset[bo] = offset - (j - 1) * 16 * 16 * SIZEOF_PIXEL; + } + } + + dstp0[0] = dst0; + dstp0[1] = dst0 + 16 * 16 * SIZEOF_PIXEL; + dstp1[0] = dst1; + dstp1[1] = dst1 + 16 * 16 * SIZEOF_PIXEL; + + if (check_func(idct, "%s_%dbpp", name, bit_depth)) { + memcpy(coef0, coef_full, 48 * 16 * SIZEOF_COEF); + memcpy(coef1, coef_full, 48 * 16 * SIZEOF_COEF); + memcpy(dst0, dst_full, 16 * 32 * 2); + memcpy(dst1, dst_full, 16 * 32 * 2); + call_ref(dstp0, block_offset, coef0, 16 * SIZEOF_PIXEL, nnzc); + call_new(dstp1, block_offset, coef1, 16 * SIZEOF_PIXEL, nnzc); + if (memcmp(dst0, dst1, 16 * 32 * 2) || + memcmp(coef0, coef1, 48 * 16 * SIZEOF_COEF)) + fail(); + bench_new(dstp1, block_offset, coef1, 16 * SIZEOF_PIXEL, nnzc); + } + } + } +} + static void check_idct_dequant(void) { static const int depths[5] = { 8, 9, 10, 12, 14 }; @@ -617,6 +696,7 @@ void checkasm_check_h264dsp(void) { check_idct(); check_idct_multiple(); + check_idct_add8(); check_idct_dequant(); report("idct"); -- 2.54.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
