In local_interrupt_enable(), we could avoid re-reading preempt count
because of should_resched() by using the result from
hardirq_disable_exit(), however this means __preempt_count_add_return()
and __preempt_count_sub_return() need to return all the preempt count
bits (including the PREEMPT_NEED_RESCHED bit), since the only user of
__preempt_count_{add,sub}_return() is hardirq_disable_{enter,exit}(),
hence make them return "unsigned long" to optimize this.Signed-off-by: Boqun Feng <[email protected]> --- arch/arm64/include/asm/preempt.h | 12 ++++++------ arch/s390/include/asm/preempt.h | 4 ++-- arch/x86/include/asm/preempt.h | 4 ++-- include/asm-generic/preempt.h | 4 ++-- include/linux/interrupt_rc.h | 31 ++++++++++++++++++++----------- 5 files changed, 32 insertions(+), 23 deletions(-) diff --git a/arch/arm64/include/asm/preempt.h b/arch/arm64/include/asm/preempt.h index 0dd8221d1bef..e9f597d87413 100644 --- a/arch/arm64/include/asm/preempt.h +++ b/arch/arm64/include/asm/preempt.h @@ -55,20 +55,20 @@ static inline void __preempt_count_sub(int val) WRITE_ONCE(current_thread_info()->preempt.count, pc); } -static inline int __preempt_count_add_return(int val) +static inline unsigned long __preempt_count_add_return(int val) { - u32 pc = READ_ONCE(current_thread_info()->preempt.count); + u64 pc = READ_ONCE(current_thread_info()->preempt_count); pc += val; - WRITE_ONCE(current_thread_info()->preempt.count, pc); + WRITE_ONCE(current_thread_info()->preempt_count, pc); return pc; } -static inline int __preempt_count_sub_return(int val) +static inline unsigned long __preempt_count_sub_return(int val) { - u32 pc = READ_ONCE(current_thread_info()->preempt.count); + u64 pc = READ_ONCE(current_thread_info()->preempt_count); pc -= val; - WRITE_ONCE(current_thread_info()->preempt.count, pc); + WRITE_ONCE(current_thread_info()->preempt_count, pc); return pc; } diff --git a/arch/s390/include/asm/preempt.h b/arch/s390/include/asm/preempt.h index 1d5e4d7e9e1b..d0021b979a5d 100644 --- a/arch/s390/include/asm/preempt.h +++ b/arch/s390/include/asm/preempt.h @@ -136,12 +136,12 @@ static __always_inline bool should_resched(int preempt_offset) return unlikely(READ_ONCE(get_lowcore()->preempt_count) == preempt_offset); } -static __always_inline int __preempt_count_add_return(int val) +static __always_inline unsigned long __preempt_count_add_return(int val) { return val + __atomic64_add(val, (long *)&get_lowcore()->preempt_count); } -static __always_inline int __preempt_count_sub_return(int val) +static __always_inline unsigned long __preempt_count_sub_return(int val) { return __preempt_count_add_return(-val); } diff --git a/arch/x86/include/asm/preempt.h b/arch/x86/include/asm/preempt.h index 12353eeebc52..fc1a2799990a 100644 --- a/arch/x86/include/asm/preempt.h +++ b/arch/x86/include/asm/preempt.h @@ -103,12 +103,12 @@ static __always_inline void __preempt_count_sub(int val) __pc_op(add, __preempt_count, -val); } -static __always_inline int __preempt_count_add_return(int val) +static __always_inline unsigned long __preempt_count_add_return(int val) { return __pc_op(add_return, __preempt_count, val); } -static __always_inline int __preempt_count_sub_return(int val) +static __always_inline unsigned long __preempt_count_sub_return(int val) { return __pc_op(add_return, __preempt_count, -val); } diff --git a/include/asm-generic/preempt.h b/include/asm-generic/preempt.h index c8683c046615..7629e23102d1 100644 --- a/include/asm-generic/preempt.h +++ b/include/asm-generic/preempt.h @@ -59,14 +59,14 @@ static __always_inline void __preempt_count_sub(int val) *preempt_count_ptr() -= val; } -static __always_inline int __preempt_count_add_return(int val) +static __always_inline unsigned long __preempt_count_add_return(int val) { *preempt_count_ptr() += val; return *preempt_count_ptr(); } -static __always_inline int __preempt_count_sub_return(int val) +static __always_inline unsigned long __preempt_count_sub_return(int val) { *preempt_count_ptr() -= val; diff --git a/include/linux/interrupt_rc.h b/include/linux/interrupt_rc.h index dd4444c61330..c044dc395452 100644 --- a/include/linux/interrupt_rc.h +++ b/include/linux/interrupt_rc.h @@ -27,7 +27,7 @@ DECLARE_PER_CPU(struct interrupt_disable_state, local_interrupt_disable_state); static inline void local_interrupt_disable(void) { unsigned long flags; - int new_count; + unsigned long new_count; WARN_ON_ONCE(in_nmi()); @@ -41,9 +41,25 @@ static inline void local_interrupt_disable(void) } } +#ifdef CONFIG_PREEMPTION +static inline void local_interrupt_enable_reched(unsigned long pc) +{ + if (pc) + return; + /* No PREEMPT_NEED_RESCHED bit? Check tif_need_resched() */ +#ifndef PREEMPT_NEED_RESCHED + if (!tif_need_resched()) + return; +#endif + __preempt_schedule(); +} +#else +static inline void local_interrupt_enable_reched(unsigned long pc) {} +#endif + static inline void local_interrupt_enable(void) { - int new_count; + unsigned long new_count; new_count = hardirq_disable_exit(); @@ -52,15 +68,8 @@ static inline void local_interrupt_enable(void) flags = raw_cpu_read(local_interrupt_disable_state.flags); local_irq_restore(flags); - /* - * TODO: re-read preempt count can be avoided, but it needs - * should_resched() taking another parameter as the current - * preempt count - */ -#ifdef CONFIG_PREEMPTION - if (should_resched(0)) - __preempt_schedule(); -#endif + + local_interrupt_enable_reched(new_count); } } -- 2.51.0
