It is sometimes beneficial to prevent preemption for very few instructions, or prevent preemption for some instructions that precede a branch (this latter case will be introduced in the next patches).
To provide such functionality on x86-64, we use an empty REX-prefix (opcode 0x40) as an indication that preemption is disabled for the following instruction. It is expected that this opcode is not in common use. Signed-off-by: Nadav Amit <[email protected]> --- arch/x86/entry/entry_64.S | 10 ++++++++++ arch/x86/include/asm/nospec-branch.h | 12 ++++++++++++ 2 files changed, 22 insertions(+) diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index cb8a5893fd33..31d59aad496e 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -643,6 +643,16 @@ retint_kernel: jnc 1f 0: cmpl $0, PER_CPU_VAR(__preempt_count) jnz 1f + + /* + * Allow to use hint to prevent preemption on a certain instruction. + * Consider an instruction with the first byte having REX prefix + * without any bits set as an indication for preemption disabled. + */ + movq RIP(%rsp), %rax + cmpb $PREEMPT_DISABLE_PREFIX, (%rax) + jz 1f + call preempt_schedule_irq jmp 0b 1: diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h index 80dc14422495..0267611eb247 100644 --- a/arch/x86/include/asm/nospec-branch.h +++ b/arch/x86/include/asm/nospec-branch.h @@ -52,6 +52,12 @@ jnz 771b; \ add $(BITS_PER_LONG/8) * nr, sp; +/* + * An empty REX-prefix is an indication that preemption should not take place on + * this instruction. + */ +#define PREEMPT_DISABLE_PREFIX (0x40) + #ifdef __ASSEMBLY__ /* @@ -148,6 +154,12 @@ #endif .endm +.macro preempt_disable_prefix +#ifdef CONFIG_PREEMPT + .byte PREEMPT_DISABLE_PREFIX +#endif +.endm + #else /* __ASSEMBLY__ */ #define ANNOTATE_NOSPEC_ALTERNATIVE \ -- 2.17.1
