PR #25741 opened by Ramiro Polla (ramiro)
URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/25741
Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/25741.patch
This series provides up to 7% speedup of single-threaded H.264 decoding on
x86_64.
I left Claude Code with the task of optimizing the x86_64 assembly for cabac
decoding as much as it could, which ended up in a rewrite of the functions.
Each optimization was then split out and validated.
```
Speedup of the whole series:
P-core E-core
1080p 3M +2.0% +1.5%
1080p 15M +5.5% +3.0%
1080p 30M +7.1% +4.0%
4K 150M +7.0% +3.3%
Geometric Mean +5.4% +2.9%
```
The benchmarks decode the start of four of the Jellyfin H.264 test videos[1]
(High profile, CABAC, 60 fps):
- Test Jellyfin 1080p AVC 3M.mp4 712 frames
- Test Jellyfin 1080p AVC 15M.mp4 328 frames
- Test Jellyfin 1080p AVC 30M.mp4 264 frames
- Test Jellyfin 4K AVC 150M.mp4 80 frames
Using the following command:
```
perf stat -e task-clock taskset -c <core> ./ffmpeg -nostdin -threads 1 -i
<clip> -frames:v <frames> -an -f null -
```
The builds use the default configure (PIC) with gcc 13.3.0, and were tested on
an i9-13900K.
The first 5 commits in the series are cleanup/cosmetics, followed by 4 commits
of optimizations, and the last commit which uses the x86-64 version of
get_cabac() on non-PIC builds too.
[1] https://repo.jellyfin.org/test-videos/SDR/AVC
>From 407685b6c66e9dfc56200ee0d9ce718ea61dbb80 Mon Sep 17 00:00:00 2001
From: Ramiro Polla <[email protected]>
Date: Mon, 2 Nov 2026 17:55:48 +0100
Subject: [PATCH 01/10] avcodec/cabac: remove dead code for CABAC_BITS != 16
CABAC_BITS has been hardcoded to 16 since 5659b509c7 ("refill cabac
variables in 16bit steps, 3% faster get_cabac()"), which changed it
from 8. The code for other values of CABAC_BITS has not been compiled
since then, so remove it.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
Signed-off-by: Ramiro Polla <[email protected]>
---
libavcodec/cabac.c | 5 -----
libavcodec/cabac_functions.h | 14 ++------------
libavcodec/h264_cabac.c | 4 +---
libavcodec/vvc/cabac.c | 4 ----
4 files changed, 3 insertions(+), 24 deletions(-)
diff --git a/libavcodec/cabac.c b/libavcodec/cabac.c
index 7d41cd2ae6f..387be648c4c 100644
--- a/libavcodec/cabac.c
+++ b/libavcodec/cabac.c
@@ -164,7 +164,6 @@ int ff_init_cabac_decoder(CABACContext *c, const uint8_t
*buf, int buf_size){
c->bytestream= buf;
c->bytestream_end= buf + buf_size;
-#if CABAC_BITS == 16
c->low = (*c->bytestream++)<<18;
c->low+= (*c->bytestream++)<<10;
// Keep our fetches on a 2-byte boundary as this should avoid ever having
to
@@ -176,10 +175,6 @@ int ff_init_cabac_decoder(CABACContext *c, const uint8_t
*buf, int buf_size){
else {
c->low += ((*c->bytestream++) << 2) + 2;
}
-#else
- c->low = (*c->bytestream++)<<10;
- c->low+= ((*c->bytestream++)<<2) + 2;
-#endif
c->range= 0x1FE;
if ((c->range<<(CABAC_BITS+1)) < c->low)
return AVERROR_INVALIDDATA;
diff --git a/libavcodec/cabac_functions.h b/libavcodec/cabac_functions.h
index 6a2e9ceca54..eec0e250997 100644
--- a/libavcodec/cabac_functions.h
+++ b/libavcodec/cabac_functions.h
@@ -62,11 +62,7 @@ static const uint8_t * const
ff_h264_last_coeff_flag_offset_8x8 = ff_h264_cabac_
#if !defined(get_cabac_bypass) || !defined(get_cabac_terminate)
static void refill(CABACContext *c){
-#if CABAC_BITS == 16
- c->low+= (c->bytestream[0]<<9) + (c->bytestream[1]<<1);
-#else
- c->low+= c->bytestream[0]<<1;
-#endif
+ c->low+= (c->bytestream[0]<<9) + (c->bytestream[1]<<1);
c->low -= CABAC_MASK;
#if !UNCHECKED_BITSTREAM_READER
if (c->bytestream < c->bytestream_end)
@@ -98,11 +94,7 @@ static void refill2(CABACContext *c){
x= -CABAC_MASK;
-#if CABAC_BITS == 16
- x+= (c->bytestream[0]<<9) + (c->bytestream[1]<<1);
-#else
- x+= c->bytestream[0]<<1;
-#endif
+ x+= (c->bytestream[0]<<9) + (c->bytestream[1]<<1);
c->low += x<<i;
#if !UNCHECKED_BITSTREAM_READER
@@ -205,10 +197,8 @@ av_unused static const uint8_t* skip_bytes(CABACContext
*c, int n) {
if (c->low & 0x1)
ptr--;
-#if CABAC_BITS == 16
if (c->low & 0x1FF)
ptr--;
-#endif
if ((int) (c->bytestream_end - ptr) < n)
return NULL;
if (ff_init_cabac_decoder(c, ptr + n, c->bytestream_end - ptr - n) < 0)
diff --git a/libavcodec/h264_cabac.c b/libavcodec/h264_cabac.c
index 7fc4817634d..4d826caeb61 100644
--- a/libavcodec/h264_cabac.c
+++ b/libavcodec/h264_cabac.c
@@ -2041,9 +2041,7 @@ decode_intra_mb:
// decode, I think it should be done by a function in cabac.h (or
cabac.c).
ptr= sl->cabac.bytestream;
if(sl->cabac.low&0x1) ptr--;
- if(CABAC_BITS==16){
- if(sl->cabac.low&0x1FF) ptr--;
- }
+ if(sl->cabac.low&0x1FF) ptr--;
// The pixels are stored in the same order as levels in h->mb array.
if ((int) (sl->cabac.bytestream_end - ptr) < mb_size)
diff --git a/libavcodec/vvc/cabac.c b/libavcodec/vvc/cabac.c
index 329172e4751..ec45cdee8c2 100644
--- a/libavcodec/vvc/cabac.c
+++ b/libavcodec/vvc/cabac.c
@@ -876,11 +876,7 @@ static void vvc_refill2(CABACContext* c) {
x = -CABAC_MASK;
-#if CABAC_BITS == 16
x += (c->bytestream[0] << 9) + (c->bytestream[1] << 1);
-#else
- x += c->bytestream[0] << 1;
-#endif
c->low += x << i;
#if !UNCHECKED_BITSTREAM_READER
--
2.54.0
>From 298aef4745c4e87d617b8f33b4396430af1194e7 Mon Sep 17 00:00:00 2001
From: Ramiro Polla <[email protected]>
Date: Mon, 2 Nov 2026 00:33:35 +0100
Subject: [PATCH 02/10] avcodec/x86/cabac: remove unused RIP_ARG define
It has been unused since its introduction in 82c71913e4.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
Signed-off-by: Ramiro Polla <[email protected]>
---
libavcodec/x86/cabac.h | 1 -
1 file changed, 1 deletion(-)
diff --git a/libavcodec/x86/cabac.h b/libavcodec/x86/cabac.h
index 396e8e9841b..ce5b831754e 100644
--- a/libavcodec/x86/cabac.h
+++ b/libavcodec/x86/cabac.h
@@ -133,7 +133,6 @@
#else /* BROKEN_RELOCATIONS */
#define TABLES_ARG NAMED_CONSTRAINTS_ARRAY_ADD(ff_h264_cabac_tables)
-#define RIP_ARG
#if HAVE_FAST_CMOV
#define BRANCHLESS_GET_CABAC_UPDATE(ret, low, range, tmp)\
--
2.54.0
>From 93f9d5900a6f2190f2af835f0ed50022003f1d58 Mon Sep 17 00:00:00 2001
From: Ramiro Polla <[email protected]>
Date: Mon, 2 Nov 2026 01:14:12 +0100
Subject: [PATCH 03/10] avcodec/x86/cabac: remove the cmov-free x86-64 PIC
version of get_cabac
HAVE_FAST_CMOV is only disabled on x86-64 when configuring with
--cpu=pentium4, pentium4m, prescott or nocona (or --cpu=host on these
CPUs), on which cmov is supported but slow. Generic x86-64 builds, as
shipped by distributions, already use the cmov version on these CPUs.
Avoiding cmov here is also inconsistent with the rest of these builds,
since gcc freely emits cmov with -march=nocona.
The upcoming cabac optimizations use cmov in the x86-64 version of
BRANCHLESS_GET_CABAC, and keeping a cmov-free version of each of them
for these builds is not worth it, so drop the cmov-free variant.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
Signed-off-by: Ramiro Polla <[email protected]>
---
libavcodec/x86/cabac.h | 30 ++++++------------------------
1 file changed, 6 insertions(+), 24 deletions(-)
diff --git a/libavcodec/x86/cabac.h b/libavcodec/x86/cabac.h
index ce5b831754e..206b4e43466 100644
--- a/libavcodec/x86/cabac.h
+++ b/libavcodec/x86/cabac.h
@@ -52,29 +52,6 @@
#ifdef BROKEN_RELOCATIONS
#define TABLES_ARG , "r"(tables)
-#if HAVE_FAST_CMOV
-#define BRANCHLESS_GET_CABAC_UPDATE(ret, retq, low, range, tmp) \
- "cmp "low" , "tmp" \n\t"\
- "cmova %%ecx , "range" \n\t"\
- "sbb %%rcx , %%rcx \n\t"\
- "and %%ecx , "tmp" \n\t"\
- "xor %%rcx , "retq" \n\t"\
- "sub "tmp" , "low" \n\t"
-#else /* HAVE_FAST_CMOV */
-#define BRANCHLESS_GET_CABAC_UPDATE(ret, retq, low, range, tmp) \
-/* P4 Prescott has crappy cmov,sbb,64-bit shift so avoid them */ \
- "sub "low" , "tmp" \n\t"\
- "sar $31 , "tmp" \n\t"\
- "sub %%ecx , "range" \n\t"\
- "and "tmp" , "range" \n\t"\
- "add %%ecx , "range" \n\t"\
- "shl $17 , %%ecx \n\t"\
- "and "tmp" , %%ecx \n\t"\
- "sub %%ecx , "low" \n\t"\
- "xor "tmp" , "ret" \n\t"\
- "movslq "ret" , "retq" \n\t"
-#endif /* HAVE_FAST_CMOV */
-
#ifdef __LZCNT__
/* renormalisation shift = lzcnt(range) - 23, refill shift = tzcnt(low) - 16 */
#define BRANCHLESS_GET_CABAC_RENORM(range, rangeq, norm_off, tables) \
@@ -116,7 +93,12 @@
"sub "range" , "tmp" \n\t"\
"mov "tmp" , %%ecx \n\t"\
"shl $17 , "tmp" \n\t"\
- BRANCHLESS_GET_CABAC_UPDATE(ret, retq, low, range, tmp) \
+ "cmp "low" , "tmp" \n\t"\
+ "cmova %%ecx , "range" \n\t"\
+ "sbb %%rcx , %%rcx \n\t"\
+ "and %%ecx , "tmp" \n\t"\
+ "xor %%rcx , "retq" \n\t"\
+ "sub "tmp" , "low" \n\t"\
BRANCHLESS_GET_CABAC_RENORM(range, rangeq, norm_off, tables) \
"shl %%cl , "range" \n\t"\
"movzbl "mlps_off"+128("tables", "retq"), "tmp" \n\t"\
--
2.54.0
>From c26bd0e6e80a83d90a6a3d0c6e7d729e9287b9a7 Mon Sep 17 00:00:00 2001
From: Ramiro Polla <[email protected]>
Date: Sun, 1 Nov 2026 23:41:49 +0100
Subject: [PATCH 04/10] avcodec/x86/cabac: use read-write constraints for low
and range operands
c->low and c->range are currently declared as write-only outputs, and
their initial values are also passed as inputs. Use "+" read-write
constraints instead, which is equivalent. No functional change.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
Signed-off-by: Ramiro Polla <[email protected]>
---
libavcodec/x86/cabac.h | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/libavcodec/x86/cabac.h b/libavcodec/x86/cabac.h
index 206b4e43466..371e668ec49 100644
--- a/libavcodec/x86/cabac.h
+++ b/libavcodec/x86/cabac.h
@@ -224,12 +224,11 @@ int get_cabac_inline_x86(CABACContext *c, uint8_t *const
state)
AV_STRINGIFY(H264_LPS_RANGE_OFFSET),
AV_STRINGIFY(H264_MLPS_STATE_OFFSET),
"%8")
- : "=&r"(bit), "=&r"(c->low), "=&r"(c->range), "=&q"(tmp)
+ : "=&r"(bit), "+&r"(c->low), "+&r"(c->range), "=&q"(tmp)
: "r"(state), "r"(c),
"i"(offsetof(CABACContext, bytestream)),
"i"(offsetof(CABACContext, bytestream_end))
TABLES_ARG
- ,"1"(c->low), "2"(c->range)
: "%"FF_REG_c, "memory"
);
return bit & 1;
--
2.54.0
>From 5877df1b61a423455dba0c4447cc47b45a23d5f8 Mon Sep 17 00:00:00 2001
From: Ramiro Polla <[email protected]>
Date: Sun, 1 Nov 2026 21:35:10 +0100
Subject: [PATCH 05/10] avcodec/x86/cabac: use named operands in inline
assembly
This makes the code easier to read and allows adding operands without
renumbering all existing ones. No functional change.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
Signed-off-by: Ramiro Polla <[email protected]>
---
libavcodec/x86/cabac.h | 18 ++---
libavcodec/x86/h264_cabac.c | 133 ++++++++++++++++++------------------
2 files changed, 76 insertions(+), 75 deletions(-)
diff --git a/libavcodec/x86/cabac.h b/libavcodec/x86/cabac.h
index 371e668ec49..249c18f0001 100644
--- a/libavcodec/x86/cabac.h
+++ b/libavcodec/x86/cabac.h
@@ -50,7 +50,7 @@
#endif
#ifdef BROKEN_RELOCATIONS
-#define TABLES_ARG , "r"(tables)
+#define TABLES_ARG , [tables]"r"(tables)
#ifdef __LZCNT__
/* renormalisation shift = lzcnt(range) - 23, refill shift = tzcnt(low) - 16 */
@@ -217,17 +217,17 @@ int get_cabac_inline_x86(CABACContext *c, uint8_t *const
state)
#endif
__asm__ volatile(
- BRANCHLESS_GET_CABAC("%0", "%q0", "(%4)", "%1", "%w1",
- "%2", "%q2", "%3", "%b3",
- "%c6(%5)", "%c7(%5)",
+ BRANCHLESS_GET_CABAC("%[bit]", "%q[bit]", "(%[state])", "%[low]",
"%w[low]",
+ "%[range]", "%q[range]", "%[tmp]", "%b[tmp]",
+ "%c[bs](%[c])", "%c[bsend](%[c])",
AV_STRINGIFY(H264_NORM_SHIFT_OFFSET),
AV_STRINGIFY(H264_LPS_RANGE_OFFSET),
AV_STRINGIFY(H264_MLPS_STATE_OFFSET),
- "%8")
- : "=&r"(bit), "+&r"(c->low), "+&r"(c->range), "=&q"(tmp)
- : "r"(state), "r"(c),
- "i"(offsetof(CABACContext, bytestream)),
- "i"(offsetof(CABACContext, bytestream_end))
+ "%[tables]")
+ : [bit]"=&r"(bit), [low]"+&r"(c->low), [range]"+&r"(c->range),
[tmp]"=&q"(tmp)
+ : [state]"r"(state), [c]"r"(c),
+ [bs]"i"(offsetof(CABACContext, bytestream)),
+ [bsend]"i"(offsetof(CABACContext, bytestream_end))
TABLES_ARG
: "%"FF_REG_c, "memory"
);
diff --git a/libavcodec/x86/h264_cabac.c b/libavcodec/x86/h264_cabac.c
index ac7d07780c9..4f595ec05a9 100644
--- a/libavcodec/x86/h264_cabac.c
+++ b/libavcodec/x86/h264_cabac.c
@@ -65,53 +65,53 @@ static int decode_significance_x86(CABACContext *c, int
max_coeff,
__asm__ volatile(
"3: \n\t"
- BRANCHLESS_GET_CABAC("%4", "%q4", "(%1)", "%3", "%w3",
- "%5", "%q5", "%k0", "%b0",
- "%c11(%6)", "%c12(%6)",
+ BRANCHLESS_GET_CABAC("%[bit]", "%q[bit]", "(%[ctx])", "%[low]",
"%w[low]",
+ "%[range]", "%q[range]", "%k[cnt]", "%b[cnt]",
+ "%c[bs](%[c])", "%c[bsend](%[c])",
AV_STRINGIFY(H264_NORM_SHIFT_OFFSET),
AV_STRINGIFY(H264_LPS_RANGE_OFFSET),
AV_STRINGIFY(H264_MLPS_STATE_OFFSET),
- "%13")
+ "%[tables]")
- "test $1, %4 \n\t"
+ "test $1, %[bit] \n\t"
" jz 4f \n\t"
- "add %10, %1 \n\t"
+ "add %[last_off], %[ctx] \n\t"
- BRANCHLESS_GET_CABAC("%4", "%q4", "(%1)", "%3", "%w3",
- "%5", "%q5", "%k0", "%b0",
- "%c11(%6)", "%c12(%6)",
+ BRANCHLESS_GET_CABAC("%[bit]", "%q[bit]", "(%[ctx])", "%[low]",
"%w[low]",
+ "%[range]", "%q[range]", "%k[cnt]", "%b[cnt]",
+ "%c[bs](%[c])", "%c[bsend](%[c])",
AV_STRINGIFY(H264_NORM_SHIFT_OFFSET),
AV_STRINGIFY(H264_LPS_RANGE_OFFSET),
AV_STRINGIFY(H264_MLPS_STATE_OFFSET),
- "%13")
+ "%[tables]")
- "sub %10, %1 \n\t"
- "mov %2, %0 \n\t"
- "movl %7, %%ecx \n\t"
- "add %1, %%"FF_REG_c" \n\t"
- "movl %%ecx, (%0) \n\t"
+ "sub %[last_off], %[ctx] \n\t"
+ "mov %[index], %[cnt] \n\t"
+ "movl %[minusstart], %%ecx \n\t"
+ "add %[ctx], %%"FF_REG_c" \n\t"
+ "movl %%ecx, (%[cnt]) \n\t"
- "test $1, %4 \n\t"
+ "test $1, %[bit] \n\t"
" jnz 5f \n\t"
- "add"FF_OPSIZE" $4, %2 \n\t"
+ "add"FF_OPSIZE" $4, %[index] \n\t"
"4: \n\t"
- "add $1, %1 \n\t"
- "cmp %8, %1 \n\t"
+ "add $1, %[ctx] \n\t"
+ "cmp %[end], %[ctx] \n\t"
" jb 3b \n\t"
- "mov %2, %0 \n\t"
- "movl %7, %%ecx \n\t"
- "add %1, %%"FF_REG_c" \n\t"
- "movl %%ecx, (%0) \n\t"
+ "mov %[index], %[cnt] \n\t"
+ "movl %[minusstart], %%ecx \n\t"
+ "add %[ctx], %%"FF_REG_c" \n\t"
+ "movl %%ecx, (%[cnt]) \n\t"
"5: \n\t"
- "add %9, %k0 \n\t"
- "shr $2, %k0 \n\t"
- : "=&q"(coeff_count), "+r"(significant_coeff_ctx_base), "+m"(index),
- "+&r"(c->low), "=&r"(bit), "+&r"(c->range)
- : "r"(c), "m"(minusstart), "m"(end), "m"(minusindex), "m"(last_off),
- "i"(offsetof(CABACContext, bytestream)),
- "i"(offsetof(CABACContext, bytestream_end))
+ "add %[minusindex], %k[cnt] \n\t"
+ "shr $2, %k[cnt] \n\t"
+ : [cnt]"=&q"(coeff_count), [ctx]"+r"(significant_coeff_ctx_base),
[index]"+m"(index),
+ [low]"+&r"(c->low), [bit]"=&r"(bit), [range]"+&r"(c->range)
+ : [c]"r"(c), [minusstart]"m"(minusstart), [end]"m"(end),
[minusindex]"m"(minusindex), [last_off]"m"(last_off),
+ [bs]"i"(offsetof(CABACContext, bytestream)),
+ [bsend]"i"(offsetof(CABACContext, bytestream_end))
TABLES_ARG
: "%"FF_REG_c, "memory"
);
@@ -139,66 +139,67 @@ static int decode_significance_8x8_x86(CABACContext *c,
#endif
__asm__ volatile(
- "mov %1, %6 \n\t"
+ "mov %[last], %[state] \n\t"
"3: \n\t"
- "mov %10, %0 \n\t"
- "movzb (%0, %6), %6 \n\t"
- "add %9, %6 \n\t"
+ "mov %[sig_off], %[cnt] \n\t"
+ "movzb (%[cnt], %[state]), %[state] \n\t"
+ "add %[sig_base], %[state] \n\t"
- BRANCHLESS_GET_CABAC("%4", "%q4", "(%6)", "%3", "%w3",
- "%5", "%q5", "%k0", "%b0",
- "%c12(%7)", "%c13(%7)",
+ BRANCHLESS_GET_CABAC("%[bit]", "%q[bit]", "(%[state])", "%[low]",
"%w[low]",
+ "%[range]", "%q[range]", "%k[cnt]", "%b[cnt]",
+ "%c[bs](%[c])", "%c[bsend](%[c])",
AV_STRINGIFY(H264_NORM_SHIFT_OFFSET),
AV_STRINGIFY(H264_LPS_RANGE_OFFSET),
AV_STRINGIFY(H264_MLPS_STATE_OFFSET),
- "%15")
+ "%[tables]")
- "mov %1, %6 \n\t"
- "test $1, %4 \n\t"
+ "mov %[last], %[state] \n\t"
+ "test $1, %[bit] \n\t"
" jz 4f \n\t"
#ifdef BROKEN_RELOCATIONS
- "movzb %c14(%15, %q6), %6\n\t"
+ "movzb %c[lastoff](%[tables], %q[state]), %[state]\n\t"
#else
- "movzb "MANGLE(ff_h264_cabac_tables)"+%c14(%6), %6\n\t"
+ "movzb "MANGLE(ff_h264_cabac_tables)"+%c[lastoff](%[state]),
%[state]\n\t"
#endif
- "add %11, %6 \n\t"
+ "add %[last_base], %[state] \n\t"
- BRANCHLESS_GET_CABAC("%4", "%q4", "(%6)", "%3", "%w3",
- "%5", "%q5", "%k0", "%b0",
- "%c12(%7)", "%c13(%7)",
+ BRANCHLESS_GET_CABAC("%[bit]", "%q[bit]", "(%[state])", "%[low]",
"%w[low]",
+ "%[range]", "%q[range]", "%k[cnt]", "%b[cnt]",
+ "%c[bs](%[c])", "%c[bsend](%[c])",
AV_STRINGIFY(H264_NORM_SHIFT_OFFSET),
AV_STRINGIFY(H264_LPS_RANGE_OFFSET),
AV_STRINGIFY(H264_MLPS_STATE_OFFSET),
- "%15")
+ "%[tables]")
- "mov %2, %0 \n\t"
- "mov %1, %6 \n\t"
- "mov %k6, (%0) \n\t"
+ "mov %[index], %[cnt] \n\t"
+ "mov %[last], %[state] \n\t"
+ "mov %k[state], (%[cnt]) \n\t"
- "test $1, %4 \n\t"
+ "test $1, %[bit] \n\t"
" jnz 5f \n\t"
- "add"FF_OPSIZE" $4, %2 \n\t"
+ "add"FF_OPSIZE" $4, %[index] \n\t"
"4: \n\t"
- "add $1, %6 \n\t"
- "mov %6, %1 \n\t"
- "cmp $63, %6 \n\t"
+ "add $1, %[state] \n\t"
+ "mov %[state], %[last] \n\t"
+ "cmp $63, %[state] \n\t"
" jb 3b \n\t"
- "mov %2, %0 \n\t"
- "mov %k6, (%0) \n\t"
+ "mov %[index], %[cnt] \n\t"
+ "mov %k[state], (%[cnt]) \n\t"
"5: \n\t"
- "addl %8, %k0 \n\t"
- "shr $2, %k0 \n\t"
- : "=&q"(coeff_count), "+"REG64(last), "+"REG64(index), "+&r"(c->low),
- "=&r"(bit), "+&r"(c->range), "=&r"(state)
- : "r"(c), "m"(minusindex), "m"(significant_coeff_ctx_base),
- REG64(sig_off), REG64(last_coeff_ctx_base),
- "i"(offsetof(CABACContext, bytestream)),
- "i"(offsetof(CABACContext, bytestream_end)),
- "i"(H264_LAST_COEFF_FLAG_OFFSET_8x8_OFFSET) TABLES_ARG
+ "addl %[minusindex], %k[cnt] \n\t"
+ "shr $2, %k[cnt] \n\t"
+ : [cnt]"=&q"(coeff_count), [last]"+"REG64(last),
[index]"+"REG64(index), [low]"+&r"(c->low),
+ [bit]"=&r"(bit), [range]"+&r"(c->range), [state]"=&r"(state)
+ : [c]"r"(c), [minusindex]"m"(minusindex),
[sig_base]"m"(significant_coeff_ctx_base),
+ [sig_off]REG64(sig_off), [last_base]REG64(last_coeff_ctx_base),
+ [bs]"i"(offsetof(CABACContext, bytestream)),
+ [bsend]"i"(offsetof(CABACContext, bytestream_end)),
+ [lastoff]"i"(H264_LAST_COEFF_FLAG_OFFSET_8x8_OFFSET)
+ TABLES_ARG
: "%"FF_REG_c, "memory"
);
return coeff_count;
--
2.54.0
>From eed61c119b750405f3a1d81c8b2aec561f59df5b Mon Sep 17 00:00:00 2001
From: Ramiro Polla <[email protected]>
Date: Mon, 2 Nov 2026 01:14:12 +0100
Subject: [PATCH 06/10] avcodec/x86/cabac: improve ILP of the get_cabac LPS
range lookup
Each decision computes its range from the range of the previous one, so
the instructions that go from one range to the next are the critical
path of get_cabac, and their latency limits decoding speed.
The LPS range lookup in the x86-64 PIC version builds its table address
from the state and two bits of the range. It combined the state and the
range first with an lea, so the lea had to wait for the range. Add the
state to the table address instead, which doesn't depend on the range,
and let the load add the range part. This takes the lea off the
critical path:
before: index = state + 2 * (range & 0xC0)
RangeLPS = tables[index]
after: base = tables + state
RangeLPS = base[2 * (range & 0xC0)]
Speedup of single-threaded H.264 decoding on an i9-13900K:
P-core E-core
1080p 3M +0.7% +0.7%
1080p 15M +2.2% +1.0%
1080p 30M +3.1% +1.2%
4K 150M +3.0% +1.1%
Geometric Mean +2.3% +1.0%
Co-Authored-By: Claude Opus 5.5 <[email protected]>
Signed-off-by: Ramiro Polla <[email protected]>
---
libavcodec/x86/cabac.h | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/libavcodec/x86/cabac.h b/libavcodec/x86/cabac.h
index 249c18f0001..308c99a52de 100644
--- a/libavcodec/x86/cabac.h
+++ b/libavcodec/x86/cabac.h
@@ -86,10 +86,10 @@
#define BRANCHLESS_GET_CABAC(ret, retq, statep, low, lowword, range, rangeq,
tmp, tmpbyte, byte, end, norm_off, lps_off, mlps_off, tables) \
"movzbl "statep" , "ret" \n\t"\
+ "lea ("tables", "retq"), %%rcx \n\t"\
"mov "range" , "tmp" \n\t"\
"and $0xC0 , "range" \n\t"\
- "lea ("ret", "range", 2), %%ecx \n\t"\
- "movzbl "lps_off"("tables", %%rcx), "range" \n\t"\
+ "movzbl "lps_off"(%%rcx, "rangeq", 2), "range" \n\t"\
"sub "range" , "tmp" \n\t"\
"mov "tmp" , %%ecx \n\t"\
"shl $17 , "tmp" \n\t"\
--
2.54.0
>From 4e3b6890b167e3fda1b8e2b8019602f3d5be60b7 Mon Sep 17 00:00:00 2001
From: Ramiro Polla <[email protected]>
Date: Mon, 2 Nov 2026 01:14:12 +0100
Subject: [PATCH 07/10] avcodec/x86/cabac: improve ILP of the get_cabac MPS/LPS
decision
Each decision computes its range from the range of the previous one, so
the instructions that go from one range to the next are the critical
path of get_cabac, and their latency limits decoding speed.
The MPS/LPS decision in the x86-64 PIC version compared low against the
MPS part of the range shifted left by 17, so the shift had to wait for
the range. Shift low right instead, which doesn't depend on the range,
and takes the shift off the critical path. The decision for MPS becomes:
before: low < (range - RangeLPS) << 17
after: (low >> 17) + 1 <= range - RangeLPS
The two only differ when low is exactly equal to the shifted MPS part,
which cannot happen because the 16 low bits of low are never all zero
(they hold a marker bit that every refill sets).
This needs one more temporary register in the inline asm.
Speedup of single-threaded H.264 decoding on an i9-13900K:
P-core E-core
1080p 3M +0.8% +0.8%
1080p 15M +1.9% +1.5%
1080p 30M +2.4% +2.2%
4K 150M +2.4% +1.7%
Geometric Mean +1.9% +1.6%
Co-Authored-By: Claude Opus 5.5 <[email protected]>
Signed-off-by: Ramiro Polla <[email protected]>
---
libavcodec/x86/cabac.h | 25 +++++++++++++++++--------
libavcodec/x86/h264_cabac.c | 8 ++++++++
2 files changed, 25 insertions(+), 8 deletions(-)
diff --git a/libavcodec/x86/cabac.h b/libavcodec/x86/cabac.h
index 308c99a52de..8407afb19c9 100644
--- a/libavcodec/x86/cabac.h
+++ b/libavcodec/x86/cabac.h
@@ -51,6 +51,8 @@
#ifdef BROKEN_RELOCATIONS
#define TABLES_ARG , [tables]"r"(tables)
+#define CABAC_TMP_DECL int cabac_t1;
+#define CABAC_TMP_OPERANDS , [cabac_t1]"=&r"(cabac_t1)
#ifdef __LZCNT__
/* renormalisation shift = lzcnt(range) - 23, refill shift = tzcnt(low) - 16 */
@@ -84,20 +86,22 @@
"add "tmp" , "low" \n\t"
#endif /* __LZCNT__ */
-#define BRANCHLESS_GET_CABAC(ret, retq, statep, low, lowword, range, rangeq,
tmp, tmpbyte, byte, end, norm_off, lps_off, mlps_off, tables) \
+#define BRANCHLESS_GET_CABAC(ret, retq, statep, low, lowword, range, rangeq,
tmp, tmpbyte, t1, t1q, byte, end, norm_off, lps_off, mlps_off, tables) \
"movzbl "statep" , "ret" \n\t"\
+ "mov "low" , "t1" \n\t"\
"lea ("tables", "retq"), %%rcx \n\t"\
+ "shr $17 , "t1" \n\t"\
"mov "range" , "tmp" \n\t"\
"and $0xC0 , "range" \n\t"\
+ "add $1 , "t1" \n\t"\
"movzbl "lps_off"(%%rcx, "rangeq", 2), "range" \n\t"\
"sub "range" , "tmp" \n\t"\
- "mov "tmp" , %%ecx \n\t"\
+ "cmp "t1" , "tmp" \n\t"\
+ "cmovae "tmp" , "range" \n\t"\
+ "sbb "t1q" , "t1q" \n\t"\
"shl $17 , "tmp" \n\t"\
- "cmp "low" , "tmp" \n\t"\
- "cmova %%ecx , "range" \n\t"\
- "sbb %%rcx , %%rcx \n\t"\
- "and %%ecx , "tmp" \n\t"\
- "xor %%rcx , "retq" \n\t"\
+ "and "t1" , "tmp" \n\t"\
+ "xor "t1q" , "retq" \n\t"\
"sub "tmp" , "low" \n\t"\
BRANCHLESS_GET_CABAC_RENORM(range, rangeq, norm_off, tables) \
"shl %%cl , "range" \n\t"\
@@ -114,6 +118,8 @@
"2: \n\t"
#else /* BROKEN_RELOCATIONS */
+#define CABAC_TMP_DECL
+#define CABAC_TMP_OPERANDS
#define TABLES_ARG NAMED_CONSTRAINTS_ARRAY_ADD(ff_h264_cabac_tables)
#if HAVE_FAST_CMOV
@@ -172,7 +178,7 @@
"add "tmp" , "low" \n\t"
#endif /* __LZCNT__ */
-#define BRANCHLESS_GET_CABAC(ret, retq, statep, low, lowword, range, rangeq,
tmp, tmpbyte, byte, end, norm_off, lps_off, mlps_off, tables) \
+#define BRANCHLESS_GET_CABAC(ret, retq, statep, low, lowword, range, rangeq,
tmp, tmpbyte, t1, t1q, byte, end, norm_off, lps_off, mlps_off, tables) \
"movzbl "statep" , "ret" \n\t"\
"mov "range" , "tmp" \n\t"\
"and $0xC0 , "range" \n\t"\
@@ -206,6 +212,7 @@ av_always_inline
int get_cabac_inline_x86(CABACContext *c, uint8_t *const state)
{
int bit, tmp;
+ CABAC_TMP_DECL
#ifdef BROKEN_RELOCATIONS
void *tables;
@@ -219,12 +226,14 @@ int get_cabac_inline_x86(CABACContext *c, uint8_t *const
state)
__asm__ volatile(
BRANCHLESS_GET_CABAC("%[bit]", "%q[bit]", "(%[state])", "%[low]",
"%w[low]",
"%[range]", "%q[range]", "%[tmp]", "%b[tmp]",
+ "%[cabac_t1]", "%q[cabac_t1]",
"%c[bs](%[c])", "%c[bsend](%[c])",
AV_STRINGIFY(H264_NORM_SHIFT_OFFSET),
AV_STRINGIFY(H264_LPS_RANGE_OFFSET),
AV_STRINGIFY(H264_MLPS_STATE_OFFSET),
"%[tables]")
: [bit]"=&r"(bit), [low]"+&r"(c->low), [range]"+&r"(c->range),
[tmp]"=&q"(tmp)
+ CABAC_TMP_OPERANDS
: [state]"r"(state), [c]"r"(c),
[bs]"i"(offsetof(CABACContext, bytestream)),
[bsend]"i"(offsetof(CABACContext, bytestream_end))
diff --git a/libavcodec/x86/h264_cabac.c b/libavcodec/x86/h264_cabac.c
index 4f595ec05a9..92f38ebe597 100644
--- a/libavcodec/x86/h264_cabac.c
+++ b/libavcodec/x86/h264_cabac.c
@@ -51,6 +51,7 @@ static int decode_significance_x86(CABACContext *c, int
max_coeff,
int minusindex= 4-(intptr_t)index;
int bit;
x86_reg coeff_count;
+ CABAC_TMP_DECL
#ifdef BROKEN_RELOCATIONS
void *tables;
@@ -67,6 +68,7 @@ static int decode_significance_x86(CABACContext *c, int
max_coeff,
BRANCHLESS_GET_CABAC("%[bit]", "%q[bit]", "(%[ctx])", "%[low]",
"%w[low]",
"%[range]", "%q[range]", "%k[cnt]", "%b[cnt]",
+ "%[cabac_t1]", "%q[cabac_t1]",
"%c[bs](%[c])", "%c[bsend](%[c])",
AV_STRINGIFY(H264_NORM_SHIFT_OFFSET),
AV_STRINGIFY(H264_LPS_RANGE_OFFSET),
@@ -79,6 +81,7 @@ static int decode_significance_x86(CABACContext *c, int
max_coeff,
BRANCHLESS_GET_CABAC("%[bit]", "%q[bit]", "(%[ctx])", "%[low]",
"%w[low]",
"%[range]", "%q[range]", "%k[cnt]", "%b[cnt]",
+ "%[cabac_t1]", "%q[cabac_t1]",
"%c[bs](%[c])", "%c[bsend](%[c])",
AV_STRINGIFY(H264_NORM_SHIFT_OFFSET),
AV_STRINGIFY(H264_LPS_RANGE_OFFSET),
@@ -109,6 +112,7 @@ static int decode_significance_x86(CABACContext *c, int
max_coeff,
"shr $2, %k[cnt] \n\t"
: [cnt]"=&q"(coeff_count), [ctx]"+r"(significant_coeff_ctx_base),
[index]"+m"(index),
[low]"+&r"(c->low), [bit]"=&r"(bit), [range]"+&r"(c->range)
+ CABAC_TMP_OPERANDS
: [c]"r"(c), [minusstart]"m"(minusstart), [end]"m"(end),
[minusindex]"m"(minusindex), [last_off]"m"(last_off),
[bs]"i"(offsetof(CABACContext, bytestream)),
[bsend]"i"(offsetof(CABACContext, bytestream_end))
@@ -127,6 +131,7 @@ static int decode_significance_8x8_x86(CABACContext *c,
x86_reg coeff_count;
x86_reg last=0;
x86_reg state;
+ CABAC_TMP_DECL
#ifdef BROKEN_RELOCATIONS
void *tables;
@@ -148,6 +153,7 @@ static int decode_significance_8x8_x86(CABACContext *c,
BRANCHLESS_GET_CABAC("%[bit]", "%q[bit]", "(%[state])", "%[low]",
"%w[low]",
"%[range]", "%q[range]", "%k[cnt]", "%b[cnt]",
+ "%[cabac_t1]", "%q[cabac_t1]",
"%c[bs](%[c])", "%c[bsend](%[c])",
AV_STRINGIFY(H264_NORM_SHIFT_OFFSET),
AV_STRINGIFY(H264_LPS_RANGE_OFFSET),
@@ -167,6 +173,7 @@ static int decode_significance_8x8_x86(CABACContext *c,
BRANCHLESS_GET_CABAC("%[bit]", "%q[bit]", "(%[state])", "%[low]",
"%w[low]",
"%[range]", "%q[range]", "%k[cnt]", "%b[cnt]",
+ "%[cabac_t1]", "%q[cabac_t1]",
"%c[bs](%[c])", "%c[bsend](%[c])",
AV_STRINGIFY(H264_NORM_SHIFT_OFFSET),
AV_STRINGIFY(H264_LPS_RANGE_OFFSET),
@@ -194,6 +201,7 @@ static int decode_significance_8x8_x86(CABACContext *c,
"shr $2, %k[cnt] \n\t"
: [cnt]"=&q"(coeff_count), [last]"+"REG64(last),
[index]"+"REG64(index), [low]"+&r"(c->low),
[bit]"=&r"(bit), [range]"+&r"(c->range), [state]"=&r"(state)
+ CABAC_TMP_OPERANDS
: [c]"r"(c), [minusindex]"m"(minusindex),
[sig_base]"m"(significant_coeff_ctx_base),
[sig_off]REG64(sig_off), [last_base]REG64(last_coeff_ctx_base),
[bs]"i"(offsetof(CABACContext, bytestream)),
--
2.54.0
>From a1d5202fbb784c4f9742f99c3f2d74dbbeeb1296 Mon Sep 17 00:00:00 2001
From: Ramiro Polla <[email protected]>
Date: Mon, 2 Nov 2026 01:14:12 +0100
Subject: [PATCH 08/10] avcodec/x86/cabac: improve ILP of the get_cabac
renormalisation shift
Each decision computes its range from the range of the previous one, so
the instructions that go from one range to the next are the critical
path of get_cabac, and their latency limits decoding speed.
The renormalisation shift in the x86-64 PIC version was looked up from
the new range, so the lookup had to wait for the MPS/LPS decision.
Compute the shift for both outcomes in parallel instead, and select it
together with the new range. The LPS shift is looked up from RangeLPS.
The MPS part of the range is never below 128, so its shift is either 0
or 1, and two simple instructions compute it without a lookup:
before: shift = norm_shift[new_range]
after: lps_shift = norm_shift[RangeLPS]
mps_shift = (range - RangeLPS) < 256
shift = is_lps ? lps_shift : mps_shift
This needs another temporary register in the inline asm. The lzcnt
version of the renormalisation is removed, since the new code is faster
with lzcnt too.
Speedup of single-threaded H.264 decoding on an i9-13900K:
P-core E-core
1080p 3M +0.1% -0.1%
1080p 15M +1.1% +0.4%
1080p 30M +1.2% +0.3%
4K 150M +1.3% +0.2%
Geometric Mean +0.9% +0.2%
Co-Authored-By: Claude Opus 5.5 <[email protected]>
Signed-off-by: Ramiro Polla <[email protected]>
---
libavcodec/x86/cabac.h | 23 +++++++++++------------
libavcodec/x86/h264_cabac.c | 8 ++++----
2 files changed, 15 insertions(+), 16 deletions(-)
diff --git a/libavcodec/x86/cabac.h b/libavcodec/x86/cabac.h
index 8407afb19c9..3ddd876f5c5 100644
--- a/libavcodec/x86/cabac.h
+++ b/libavcodec/x86/cabac.h
@@ -51,14 +51,11 @@
#ifdef BROKEN_RELOCATIONS
#define TABLES_ARG , [tables]"r"(tables)
-#define CABAC_TMP_DECL int cabac_t1;
-#define CABAC_TMP_OPERANDS , [cabac_t1]"=&r"(cabac_t1)
+#define CABAC_TMP_DECL int cabac_t1, cabac_t2;
+#define CABAC_TMP_OPERANDS , [cabac_t1]"=&r"(cabac_t1),
[cabac_t2]"=&r"(cabac_t2)
#ifdef __LZCNT__
-/* renormalisation shift = lzcnt(range) - 23, refill shift = tzcnt(low) - 16 */
-#define BRANCHLESS_GET_CABAC_RENORM(range, rangeq, norm_off, tables) \
- "lzcnt "range" , %%ecx \n\t"\
- "sub $23 , %%ecx \n\t"
+/* refill shift = tzcnt(low) - 16 */
#define BRANCHLESS_GET_CABAC_REFILL(low, tmp, norm_off, tables) \
"movzwl (%%"FF_REG_c") , "tmp" \n\t"\
"tzcnt "low" , %%ecx \n\t"\
@@ -69,8 +66,6 @@
"shl %%cl , "tmp" \n\t"\
"add "tmp" , "low" \n\t"
#else /* __LZCNT__ */
-#define BRANCHLESS_GET_CABAC_RENORM(range, rangeq, norm_off, tables) \
- "movzbl "norm_off"("tables", "rangeq"), %%ecx \n\t"
#define BRANCHLESS_GET_CABAC_REFILL(low, tmp, norm_off, tables) \
"movzwl (%%"FF_REG_c") , "tmp" \n\t"\
"lea -1("low") , %%ecx \n\t"\
@@ -86,7 +81,7 @@
"add "tmp" , "low" \n\t"
#endif /* __LZCNT__ */
-#define BRANCHLESS_GET_CABAC(ret, retq, statep, low, lowword, range, rangeq,
tmp, tmpbyte, t1, t1q, byte, end, norm_off, lps_off, mlps_off, tables) \
+#define BRANCHLESS_GET_CABAC(ret, retq, statep, low, lowword, range, rangeq,
tmp, tmpbyte, t1, t1q, t2, byte, end, norm_off, lps_off, mlps_off, tables) \
"movzbl "statep" , "ret" \n\t"\
"mov "low" , "t1" \n\t"\
"lea ("tables", "retq"), %%rcx \n\t"\
@@ -96,14 +91,18 @@
"add $1 , "t1" \n\t"\
"movzbl "lps_off"(%%rcx, "rangeq", 2), "range" \n\t"\
"sub "range" , "tmp" \n\t"\
+ "movzbl "norm_off"("tables", "rangeq"), %%ecx \n\t"\
+ "mov "tmp" , "t2" \n\t"\
+ "shr $8 , "t2" \n\t"\
+ "xor $1 , "t2" \n\t"\
"cmp "t1" , "tmp" \n\t"\
"cmovae "tmp" , "range" \n\t"\
+ "cmovae "t2" , %%ecx \n\t"\
"sbb "t1q" , "t1q" \n\t"\
"shl $17 , "tmp" \n\t"\
"and "t1" , "tmp" \n\t"\
"xor "t1q" , "retq" \n\t"\
"sub "tmp" , "low" \n\t"\
- BRANCHLESS_GET_CABAC_RENORM(range, rangeq, norm_off, tables) \
"shl %%cl , "range" \n\t"\
"movzbl "mlps_off"+128("tables", "retq"), "tmp" \n\t"\
"shl %%cl , "low" \n\t"\
@@ -178,7 +177,7 @@
"add "tmp" , "low" \n\t"
#endif /* __LZCNT__ */
-#define BRANCHLESS_GET_CABAC(ret, retq, statep, low, lowword, range, rangeq,
tmp, tmpbyte, t1, t1q, byte, end, norm_off, lps_off, mlps_off, tables) \
+#define BRANCHLESS_GET_CABAC(ret, retq, statep, low, lowword, range, rangeq,
tmp, tmpbyte, t1, t1q, t2, byte, end, norm_off, lps_off, mlps_off, tables) \
"movzbl "statep" , "ret" \n\t"\
"mov "range" , "tmp" \n\t"\
"and $0xC0 , "range" \n\t"\
@@ -226,7 +225,7 @@ int get_cabac_inline_x86(CABACContext *c, uint8_t *const
state)
__asm__ volatile(
BRANCHLESS_GET_CABAC("%[bit]", "%q[bit]", "(%[state])", "%[low]",
"%w[low]",
"%[range]", "%q[range]", "%[tmp]", "%b[tmp]",
- "%[cabac_t1]", "%q[cabac_t1]",
+ "%[cabac_t1]", "%q[cabac_t1]", "%[cabac_t2]",
"%c[bs](%[c])", "%c[bsend](%[c])",
AV_STRINGIFY(H264_NORM_SHIFT_OFFSET),
AV_STRINGIFY(H264_LPS_RANGE_OFFSET),
diff --git a/libavcodec/x86/h264_cabac.c b/libavcodec/x86/h264_cabac.c
index 92f38ebe597..51d098cd7a1 100644
--- a/libavcodec/x86/h264_cabac.c
+++ b/libavcodec/x86/h264_cabac.c
@@ -68,7 +68,7 @@ static int decode_significance_x86(CABACContext *c, int
max_coeff,
BRANCHLESS_GET_CABAC("%[bit]", "%q[bit]", "(%[ctx])", "%[low]",
"%w[low]",
"%[range]", "%q[range]", "%k[cnt]", "%b[cnt]",
- "%[cabac_t1]", "%q[cabac_t1]",
+ "%[cabac_t1]", "%q[cabac_t1]", "%[cabac_t2]",
"%c[bs](%[c])", "%c[bsend](%[c])",
AV_STRINGIFY(H264_NORM_SHIFT_OFFSET),
AV_STRINGIFY(H264_LPS_RANGE_OFFSET),
@@ -81,7 +81,7 @@ static int decode_significance_x86(CABACContext *c, int
max_coeff,
BRANCHLESS_GET_CABAC("%[bit]", "%q[bit]", "(%[ctx])", "%[low]",
"%w[low]",
"%[range]", "%q[range]", "%k[cnt]", "%b[cnt]",
- "%[cabac_t1]", "%q[cabac_t1]",
+ "%[cabac_t1]", "%q[cabac_t1]", "%[cabac_t2]",
"%c[bs](%[c])", "%c[bsend](%[c])",
AV_STRINGIFY(H264_NORM_SHIFT_OFFSET),
AV_STRINGIFY(H264_LPS_RANGE_OFFSET),
@@ -153,7 +153,7 @@ static int decode_significance_8x8_x86(CABACContext *c,
BRANCHLESS_GET_CABAC("%[bit]", "%q[bit]", "(%[state])", "%[low]",
"%w[low]",
"%[range]", "%q[range]", "%k[cnt]", "%b[cnt]",
- "%[cabac_t1]", "%q[cabac_t1]",
+ "%[cabac_t1]", "%q[cabac_t1]", "%[cabac_t2]",
"%c[bs](%[c])", "%c[bsend](%[c])",
AV_STRINGIFY(H264_NORM_SHIFT_OFFSET),
AV_STRINGIFY(H264_LPS_RANGE_OFFSET),
@@ -173,7 +173,7 @@ static int decode_significance_8x8_x86(CABACContext *c,
BRANCHLESS_GET_CABAC("%[bit]", "%q[bit]", "(%[state])", "%[low]",
"%w[low]",
"%[range]", "%q[range]", "%k[cnt]", "%b[cnt]",
- "%[cabac_t1]", "%q[cabac_t1]",
+ "%[cabac_t1]", "%q[cabac_t1]", "%[cabac_t2]",
"%c[bs](%[c])", "%c[bsend](%[c])",
AV_STRINGIFY(H264_NORM_SHIFT_OFFSET),
AV_STRINGIFY(H264_LPS_RANGE_OFFSET),
--
2.54.0
>From d3d40f7b8ff279ad6e618836d6203ddbd8635803 Mon Sep 17 00:00:00 2001
From: Ramiro Polla <[email protected]>
Date: Mon, 2 Nov 2026 00:03:13 +0100
Subject: [PATCH 09/10] avcodec/x86/cabac: don't make the tables address
computation volatile
The lea that loads the address of ff_h264_cabac_tables has no side
effects, so the compiler is free to move it out of loops or merge
multiple instances of it.
Speedup of single-threaded H.264 decoding on an i9-13900K:
P-core E-core
1080p 3M +0.4% +0.1%
1080p 15M +0.1% +0.1%
1080p 30M +0.2% +0.2%
4K 150M +0.1% +0.2%
Geometric Mean +0.2% +0.2%
Co-Authored-By: Claude Opus 5.5 <[email protected]>
Signed-off-by: Ramiro Polla <[email protected]>
---
libavcodec/x86/cabac.h | 2 +-
libavcodec/x86/h264_cabac.c | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/libavcodec/x86/cabac.h b/libavcodec/x86/cabac.h
index 3ddd876f5c5..8d54d2ad40a 100644
--- a/libavcodec/x86/cabac.h
+++ b/libavcodec/x86/cabac.h
@@ -215,7 +215,7 @@ int get_cabac_inline_x86(CABACContext *c, uint8_t *const
state)
#ifdef BROKEN_RELOCATIONS
void *tables;
- __asm__ volatile(
+ __asm__(
"lea "MANGLE(ff_h264_cabac_tables)", %0 \n\t"
: "=&r"(tables)
: NAMED_CONSTRAINTS_ARRAY(ff_h264_cabac_tables)
diff --git a/libavcodec/x86/h264_cabac.c b/libavcodec/x86/h264_cabac.c
index 51d098cd7a1..d0a4541083f 100644
--- a/libavcodec/x86/h264_cabac.c
+++ b/libavcodec/x86/h264_cabac.c
@@ -56,7 +56,7 @@ static int decode_significance_x86(CABACContext *c, int
max_coeff,
#ifdef BROKEN_RELOCATIONS
void *tables;
- __asm__ volatile(
+ __asm__(
"lea "MANGLE(ff_h264_cabac_tables)", %0 \n\t"
: "=&r"(tables)
: NAMED_CONSTRAINTS_ARRAY(ff_h264_cabac_tables)
@@ -136,7 +136,7 @@ static int decode_significance_8x8_x86(CABACContext *c,
#ifdef BROKEN_RELOCATIONS
void *tables;
- __asm__ volatile(
+ __asm__(
"lea "MANGLE(ff_h264_cabac_tables)", %0 \n\t"
: "=&r"(tables)
: NAMED_CONSTRAINTS_ARRAY(ff_h264_cabac_tables)
--
2.54.0
>From 341b5ff96ff380768b2de3af7da0bd44982a29ee Mon Sep 17 00:00:00 2001
From: Ramiro Polla <[email protected]>
Date: Mon, 2 Nov 2026 01:14:12 +0100
Subject: [PATCH 10/10] avcodec/x86/cabac: use the x86-64 version of get_cabac
on non-PIC builds too
The x86-64 version of the get_cabac inline asm keeps the address of the
tables in a register. It was only used on PIC builds, where the tables
can only be addressed relative to the instruction pointer, which can't
be combined with an index register. Non-PIC x86-64 builds used the
x86-32 version, which uses absolute addresses for the tables.
With the previous changes, the x86-64 version is faster on non-PIC
builds as well, so use it on all x86-64 builds.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
Signed-off-by: Ramiro Polla <[email protected]>
---
libavcodec/x86/cabac.h | 8 ++++----
libavcodec/x86/h264_cabac.c | 6 +++---
2 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/libavcodec/x86/cabac.h b/libavcodec/x86/cabac.h
index 8d54d2ad40a..36a534cb789 100644
--- a/libavcodec/x86/cabac.h
+++ b/libavcodec/x86/cabac.h
@@ -49,7 +49,7 @@
"jge 1f \n\t"
#endif
-#ifdef BROKEN_RELOCATIONS
+#if ARCH_X86_64
#define TABLES_ARG , [tables]"r"(tables)
#define CABAC_TMP_DECL int cabac_t1, cabac_t2;
#define CABAC_TMP_OPERANDS , [cabac_t1]"=&r"(cabac_t1),
[cabac_t2]"=&r"(cabac_t2)
@@ -116,7 +116,7 @@
BRANCHLESS_GET_CABAC_REFILL(low, tmp, norm_off, tables) \
"2: \n\t"
-#else /* BROKEN_RELOCATIONS */
+#else /* ARCH_X86_64 */
#define CABAC_TMP_DECL
#define CABAC_TMP_OPERANDS
#define TABLES_ARG NAMED_CONSTRAINTS_ARRAY_ADD(ff_h264_cabac_tables)
@@ -198,7 +198,7 @@
BRANCHLESS_GET_CABAC_REFILL(low, tmp, norm_off, tables) \
"2: \n\t"
-#endif /* BROKEN_RELOCATIONS */
+#endif /* ARCH_X86_64 */
#if HAVE_X86_7REGS && !BROKEN_COMPILER
#define get_cabac_inline get_cabac_inline_x86
@@ -212,7 +212,7 @@ int get_cabac_inline_x86(CABACContext *c, uint8_t *const
state)
{
int bit, tmp;
CABAC_TMP_DECL
-#ifdef BROKEN_RELOCATIONS
+#if ARCH_X86_64
void *tables;
__asm__(
diff --git a/libavcodec/x86/h264_cabac.c b/libavcodec/x86/h264_cabac.c
index d0a4541083f..734879dbd6f 100644
--- a/libavcodec/x86/h264_cabac.c
+++ b/libavcodec/x86/h264_cabac.c
@@ -53,7 +53,7 @@ static int decode_significance_x86(CABACContext *c, int
max_coeff,
x86_reg coeff_count;
CABAC_TMP_DECL
-#ifdef BROKEN_RELOCATIONS
+#if ARCH_X86_64
void *tables;
__asm__(
@@ -133,7 +133,7 @@ static int decode_significance_8x8_x86(CABACContext *c,
x86_reg state;
CABAC_TMP_DECL
-#ifdef BROKEN_RELOCATIONS
+#if ARCH_X86_64
void *tables;
__asm__(
@@ -164,7 +164,7 @@ static int decode_significance_8x8_x86(CABACContext *c,
"test $1, %[bit] \n\t"
" jz 4f \n\t"
-#ifdef BROKEN_RELOCATIONS
+#if ARCH_X86_64
"movzb %c[lastoff](%[tables], %q[state]), %[state]\n\t"
#else
"movzb "MANGLE(ff_h264_cabac_tables)"+%c[lastoff](%[state]),
%[state]\n\t"
--
2.54.0
_______________________________________________
ffmpeg-devel mailing list -- [email protected]
To unsubscribe send an email to [email protected]
