Diff below does two things: - Use atomic operations for incrementing/decrementing references of anonymous objects. This allows us to manipulate them without holding the KERNEL_LOCK().
- Rewrite the loop from uao_swap_off() to only keep a reference to the next item in the list. This is imported from NetBSD and is necessary to introduce locking around uao_pagein(). ok? Index: uvm/uvm_aobj.c =================================================================== RCS file: /cvs/src/sys/uvm/uvm_aobj.c,v retrieving revision 1.98 diff -u -p -r1.98 uvm_aobj.c --- uvm/uvm_aobj.c 16 Jun 2021 09:02:21 -0000 1.98 +++ uvm/uvm_aobj.c 16 Jun 2021 09:20:26 -0000 @@ -779,19 +779,11 @@ uao_init(void) void uao_reference(struct uvm_object *uobj) { - KERNEL_ASSERT_LOCKED(); - uao_reference_locked(uobj); -} - -void -uao_reference_locked(struct uvm_object *uobj) -{ - /* Kernel object is persistent. */ if (UVM_OBJ_IS_KERN_OBJECT(uobj)) return; - uobj->uo_refs++; + atomic_inc_int(&uobj->uo_refs); } @@ -801,34 +793,19 @@ uao_reference_locked(struct uvm_object * void uao_detach(struct uvm_object *uobj) { - KERNEL_ASSERT_LOCKED(); - uao_detach_locked(uobj); -} - - -/* - * uao_detach_locked: drop a reference to an aobj - * - * => aobj may freed upon return. - */ -void -uao_detach_locked(struct uvm_object *uobj) -{ struct uvm_aobj *aobj = (struct uvm_aobj *)uobj; struct vm_page *pg; /* * Detaching from kernel_object is a NOP. */ - if (UVM_OBJ_IS_KERN_OBJECT(uobj)) { + if (UVM_OBJ_IS_KERN_OBJECT(uobj)) return; - } /* * Drop the reference. If it was the last one, destroy the object. */ - uobj->uo_refs--; - if (uobj->uo_refs) { + if (atomic_dec_int_nv(&uobj->uo_refs) > 0) { return; } @@ -1265,68 +1242,54 @@ uao_dropswap(struct uvm_object *uobj, in boolean_t uao_swap_off(int startslot, int endslot) { - struct uvm_aobj *aobj, *nextaobj, *prevaobj = NULL; + struct uvm_aobj *aobj; /* - * Walk the list of all anonymous UVM objects. + * Walk the list of all anonymous UVM objects. Grab the first. */ mtx_enter(&uao_list_lock); + if ((aobj = LIST_FIRST(&uao_list)) == NULL) { + mtx_leave(&uao_list_lock); + return FALSE; + } + uao_reference(&aobj->u_obj); - for (aobj = LIST_FIRST(&uao_list); - aobj != NULL; - aobj = nextaobj) { + do { + struct uvm_aobj *nextaobj; boolean_t rv; /* - * add a ref to the aobj so it doesn't disappear - * while we're working. - */ - uao_reference_locked(&aobj->u_obj); - - /* - * now it's safe to unlock the uao list. - * note that lock interleaving is alright with IPL_NONE mutexes. + * Prefetch the next object and immediately hold a reference + * on it, so neither the current nor the next entry could + * disappear while we are iterating. */ - mtx_leave(&uao_list_lock); - - if (prevaobj) { - uao_detach_locked(&prevaobj->u_obj); - prevaobj = NULL; + if ((nextaobj = LIST_NEXT(aobj, u_list)) != NULL) { + uao_reference(&nextaobj->u_obj); } + mtx_leave(&uao_list_lock); /* - * page in any pages in the swslot range. - * if there's an error, abort and return the error. + * Page in all pages in the swap slot range. */ rv = uao_pagein(aobj, startslot, endslot); + + /* Drop the reference of the current object. */ + uao_detach(&aobj->u_obj); if (rv) { - uao_detach_locked(&aobj->u_obj); + if (nextaobj) { + uao_detach(&nextaobj->u_obj); + } return rv; } - /* - * we're done with this aobj. - * relock the list and drop our ref on the aobj. - */ + aobj = nextaobj; mtx_enter(&uao_list_lock); - nextaobj = LIST_NEXT(aobj, u_list); - /* - * prevaobj means that we have an object that we need - * to drop a reference for. We can't just drop it now with - * the list locked since that could cause lock recursion in - * the case where we reduce the refcount to 0. It will be - * released the next time we drop the list lock. - */ - prevaobj = aobj; - } + } while (aobj); /* * done with traversal, unlock the list */ mtx_leave(&uao_list_lock); - if (prevaobj) { - uao_detach_locked(&prevaobj->u_obj); - } return FALSE; }