An ordinary hardirq removes its original count and lockdep state before the
return path may preempt. Traceable return-tail callbacks then appear to run
in the interrupted task or softirq context and can collide with recursion
state owned by that context.
Use the saved ordinary-hardirq provenance to stage a fresh hardirq-count
attempt and restore lockdep classification after the preemption decision.
Keep it across traceable return-tail work, deferred hrtimer rearming,
IRQ-on preparation, and context-tracking preparation.
Before removing the raw tail state, prepare the conditional count and
lockdep teardown and close late regular or forced-thread softirq work.
Complete any pending RCU quiescent-state transition only after that raw
teardown.
Do not repeat interrupt accounting, RCU or tick entry. Existing non-hardirq
return paths remain unchanged.
Assisted-by: LLM
---
include/linux/hardirq.h | 1 +
include/linux/irq-entry-common.h | 64 ++++++++++++++++++++++++++++++--
kernel/softirq.c | 21 +++++++++++
3 files changed, 83 insertions(+), 3 deletions(-)
diff --git a/include/linux/hardirq.h b/include/linux/hardirq.h
index d4b2754468c09..22b312e674579 100644
--- a/include/linux/hardirq.h
+++ b/include/linux/hardirq.h
@@ -28,6 +28,7 @@ static __always_inline void rcu_irq_enter_check_tick(void)
void irq_enter_rcu_preempt_counted(struct irqentry_hardirq_cookie cookie);
bool irq_exit_rcu_preempt_counted_prepare(struct irqentry_hardirq_cookie
cookie);
+void noinstr irqentry_hardirq_tail_exit(struct irqentry_hardirq_cookie cookie);
/*
* Prepare every potentially instrumentable part of hardirq teardown before
diff --git a/include/linux/irq-entry-common.h b/include/linux/irq-entry-common.h
index 6f649f6613312..f51aafc1b49fd 100644
--- a/include/linux/irq-entry-common.h
+++ b/include/linux/irq-entry-common.h
@@ -383,6 +383,32 @@ irqentry_hardirq_count_state_set(irqentry_state_t *state,
IRQENTRY_HARDIRQ_APPLIED : IRQENTRY_HARDIRQ_REJECTED;
}
+/**
+ * irqentry_hardirq_tail_enter - Stage ordinary-hardirq return-tail state
+ * @state: Pointer to state returned by the matching irqentry_hardirq_enter()
+ *
+ * Make a fresh staged hardirq-count attempt and establish the lockdep
+ * classification which traceable return-tail work needs. Accounting, RCU and
+ * tick handling belong to the completed interrupt and must not be repeated
+ * here.
+ */
+static __always_inline void
+irqentry_hardirq_tail_enter(irqentry_state_t *state)
+{
+ struct irqentry_hardirq_cookie cookie;
+
+ if (!state->hardirq)
+ return;
+
+ cookie = irqentry_hardirq_count_start();
+ lockdep_hardirq_enter();
+
+ instrumentation_begin();
+ irqentry_hardirq_count_finish(cookie);
+ instrumentation_end();
+ irqentry_hardirq_count_state_set(state, cookie);
+}
+
/**
* irqentry_exit_cond_resched - Conditionally reschedule on return from
interrupt
*
@@ -519,7 +545,10 @@ static inline void
irqentry_exit_to_kernel_mode_preempt(struct pt_regs *regs,
*
* The rearm on return to user mode needs no such annotation: no locks are
* held there, so it uses the same lock chains as the rearm in hardirq
- * context.
+ * context. Ordinary-hardirq return paths do not use this wrapper: their saved
+ * provenance makes a fresh hardirq-count attempt and establishes lockdep
+ * nesting across the whole traceable tail. They call
+ * hrtimer_rearm_deferred() directly.
*/
static __always_inline void irqentry_hrtimer_rearm_deferred(void)
{
@@ -534,7 +563,9 @@ static __always_inline void
irqentry_hrtimer_rearm_deferred(void)
* @state: Return value from matching call to
irqentry_enter_from_kernel_mode()
*
* This is to be invoked after irqentry_exit_to_kernel_mode_preempt() and
before
- * actually returning to the interrupted context.
+ * actually returning to the interrupted context. If @state records ordinary
+ * hardirq provenance, the caller must first invoke
+ * irqentry_hardirq_tail_enter() exactly once.
*
* There are no requirements for the CPU state other than being able to
complete
* the tracing, lockdep and RCU state transitions. After this function returns
@@ -543,6 +574,31 @@ static __always_inline void
irqentry_hrtimer_rearm_deferred(void)
static __always_inline void
irqentry_exit_to_kernel_mode_after_preempt(struct pt_regs *regs,
irqentry_state_t state)
{
+ struct ct_irq_exit_state ct_state = { };
+
+ if (state.hardirq) {
+ if (!regs_irqs_disabled(regs)) {
+ instrumentation_begin();
+ hrtimer_rearm_deferred();
+ /* Tell the tracer that the return will enable IRQs. */
+ trace_hardirqs_on_prepare();
+ lockdep_hardirqs_on_prepare_irqtail();
+ instrumentation_end();
+ }
+
+ if (state.exit_rcu)
+ ct_state = ct_irq_exit_prepare();
+
+ irqentry_hardirq_tail_exit(irqentry_hardirq_count_state(state));
+
+ if (state.exit_rcu)
+ ct_irq_exit_finish(ct_state);
+
+ if (!regs_irqs_disabled(regs))
+ lockdep_hardirqs_on(CALLER_ADDR0);
+ return;
+ }
+
if (!regs_irqs_disabled(regs)) {
/*
* If RCU was not watching on entry this needs to be done
@@ -584,7 +640,8 @@ irqentry_exit_to_kernel_mode_after_preempt(struct pt_regs
*regs, irqentry_state_
*
* This is the counterpart of irqentry_enter_from_kernel_mode() and combines
* the calls to irqentry_exit_to_kernel_mode_preempt() and
- * irqentry_exit_to_kernel_mode_after_preempt().
+ * irqentry_exit_to_kernel_mode_after_preempt(). It also establishes the
+ * conditional hardirq return tail between those calls.
*
* The requirement for the CPU state is that it can schedule. After the
function
* returns the tracing, lockdep and RCU state transitions are completed and the
@@ -599,6 +656,7 @@ static __always_inline void
irqentry_exit_to_kernel_mode(struct pt_regs *regs,
irqentry_exit_to_kernel_mode_preempt(regs, state);
instrumentation_end();
+ irqentry_hardirq_tail_enter(&state);
irqentry_exit_to_kernel_mode_after_preempt(regs, state);
}
diff --git a/kernel/softirq.c b/kernel/softirq.c
index 169c559832501..85b167db37c29 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -837,6 +837,27 @@ irq_exit_wake_late_softirqs(unsigned int wake_flags)
irq_exit_wake_ksoftirqd();
}
+/**
+ * irqentry_hardirq_tail_exit - Close work and remove return-tail state
+ * @cookie: Staged hardirq-count cookie from irqentry_hardirq_tail_enter()
+ *
+ * Run every instrumentable teardown operation, including the final worker
+ * closure, before conditionally removing any applied raw hardirq count and
+ * ending lockdep nesting.
+ */
+void noinstr
+irqentry_hardirq_tail_exit(struct irqentry_hardirq_cookie cookie)
+{
+ bool do_sub;
+
+ instrumentation_begin();
+ do_sub = irqentry_hardirq_exit_prepare(cookie);
+ irq_exit_wake_late_softirqs(0);
+ instrumentation_end();
+
+ irqentry_hardirq_exit_finish(do_sub);
+}
+
static __always_inline unsigned int __irq_exit_rcu(bool keep_hardirq_count)
{
unsigned int wake_flags = 0;
--
2.39.5 (Apple Git-154)
