From: Masami Hiramatsu (Google) <[email protected]>

When unoptimize_kprobe() is called with force = true on an actively
optimized probe (whose op->list is empty), force_unoptimize_kprobe()
replaces the relative jump with a breakpoint, but leaves op->list empty
without queueing op to cur->cooling_list or cur->freeing_list.

Because op->list remains empty, kprobe_disarmed() returns true
prematurely as soon as the probe is disabled. When unregister_kprobes()
runs, __unregister_kprobe_top() unhashes the probe immediately, and
unregister_kprobes() frees op and the optinsn slot after synchronize_rcu()
without waiting for Tasks RCU. Any task preempted inside the optinsn slot
then encounters a use-after-free when resuming execution.

Queue forcibly unoptimized probes onto cur->freeing_list (if unused) or
cur->cooling_list (if still in use), and include gen->cooling_list in
optprobe_has_queued_probes() so that the generation is dispatched and
flushed properly. In do_unoptimize_kprobes(), collect newly unoptimized
probes into a separate local list so that probes already placed on
freeing_list are not disarmed a second time.

Fixes: 06e0d3b89da1 ("kprobes: Make optprobe optimizer multi-generational and 
asynchronous")
Signed-off-by: Masami Hiramatsu (Google) <[email protected]>
---
 kernel/kprobes.c |   27 ++++++++++++++++++---------
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/kernel/kprobes.c b/kernel/kprobes.c
index 135fec5d49cf..ceacd56245c2 100644
--- a/kernel/kprobes.c
+++ b/kernel/kprobes.c
@@ -552,17 +552,13 @@ static unsigned long optimizer_passes;
 
 #define OPTIMIZE_DELAY 5
 
-/*
- * Note: gen->cooling_list is not checked here because it is strictly
- * an in-flight holding list for dispatched generations, so it is always
- * empty in optprobe_cur_gen.
- */
 static bool optprobe_has_queued_probes(void)
 {
        struct optprobe_generation *gen = &optprobe_gens[optprobe_cur_gen];
 
        return !list_empty(&gen->optimizing_list) ||
               !list_empty(&gen->unoptimizing_list) ||
+              !list_empty(&gen->cooling_list) ||
               !list_empty(&gen->freeing_list);
 }
 
@@ -612,6 +608,7 @@ static void do_unoptimize_kprobes(struct list_head 
*unopt_list,
                                  struct list_head *cooling_list)
 {
        struct optimized_kprobe *op, *tmp;
+       LIST_HEAD(unoptimized);
 
        lockdep_assert_held(&text_mutex);
        /*
@@ -626,11 +623,13 @@ static void do_unoptimize_kprobes(struct list_head 
*unopt_list,
         */
        lockdep_assert_cpus_held();
 
-       if (!list_empty(unopt_list))
-               arch_unoptimize_kprobes(unopt_list, free_list);
+       if (list_empty(unopt_list))
+               return;
 
-       /* Loop on 'free_list' for disarming and removing from kprobe hash list 
*/
-       list_for_each_entry_safe(op, tmp, free_list, list) {
+       arch_unoptimize_kprobes(unopt_list, &unoptimized);
+
+       /* Loop on newly unoptimized probes for disarming and hash list removal 
*/
+       list_for_each_entry_safe(op, tmp, &unoptimized, list) {
                /* Switching from detour code to origin */
                op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
                /* Disarm probes if marked disabled and not gone */
@@ -643,6 +642,7 @@ static void do_unoptimize_kprobes(struct list_head 
*unopt_list,
                         * (reclaiming is done by do_free_cleaned_kprobes().)
                         */
                        hlist_del_init_rcu(&op->kp.hlist);
+                       list_move(&op->list, free_list);
                } else {
                        /*
                         * Keep on cooling_list until the quiescence period
@@ -1039,6 +1039,15 @@ static void unoptimize_kprobe(struct kprobe *p, bool 
force)
        if (force) {
                /* Forcibly update the code: this is a special case */
                force_unoptimize_kprobe(op);
+               if (kprobe_unused(&op->kp)) {
+                       if (!kprobe_gone(&op->kp))
+                               arch_disarm_kprobe(&op->kp);
+                       hlist_del_init_rcu(&op->kp.hlist);
+                       list_add(&op->list, &cur->freeing_list);
+               } else {
+                       list_add(&op->list, &cur->cooling_list);
+               }
+               kick_kprobe_optimizer();
        } else {
                list_add(&op->list, &cur->unoptimizing_list);
                kick_kprobe_optimizer();


Reply via email to