Module Name: src
Committed By: riastradh
Date: Sat Mar 12 15:32:33 UTC 2022
Modified Files:
src/sys/arch/aarch64/aarch64: pmap.c
src/sys/arch/alpha/alpha: pmap.c
src/sys/arch/arm/arm32: pmap.c
src/sys/arch/hppa/hppa: pmap.c
src/sys/arch/ia64/ia64: pmap.c
src/sys/arch/powerpc/oea: pmap.c
src/sys/arch/sparc/sparc: pmap.c
src/sys/arch/sparc64/sparc64: pmap.c
src/sys/dev/hyperv: vmbus.c
src/sys/dev/marvell: mvxpsec.c
src/sys/dev/scsipi: atapiconf.c scsiconf.c scsipi_base.c
src/sys/external/bsd/drm2/linux: linux_stop_machine.c
src/sys/kern: kern_auth.c kern_exec.c kern_mutex_obj.c kern_resource.c
kern_rwlock_obj.c kern_sig.c subr_kcpuset.c sys_futex.c uipc_mbuf.c
vfs_cwd.c vfs_mount.c vfs_vnode.c vfs_wapbl.c
src/sys/net: if.c
src/sys/net/npf: npf_nat.c npf_rproc.c npf_tableset.c
src/sys/uvm: uvm_aobj.c uvm_map.c
src/sys/uvm/pmap: pmap.c
Log Message:
sys: Membar audit around reference count releases.
If two threads are using an object that is freed when the reference
count goes to zero, we need to ensure that all memory operations
related to the object happen before freeing the object.
Using an atomic_dec_uint_nv(&refcnt) == 0 ensures that only one
thread takes responsibility for freeing, but it's not enough to
ensure that the other thread's memory operations happen before the
freeing.
Consider:
Thread A Thread B
obj->foo = 42; obj->baz = 73;
mumble(&obj->bar); grumble(&obj->quux);
/* membar_exit(); */ /* membar_exit(); */
atomic_dec -- not last atomic_dec -- last
/* membar_enter(); */
KASSERT(invariant(obj->foo,
obj->bar));
free_stuff(obj);
The memory barriers ensure that
obj->foo = 42;
mumble(&obj->bar);
in thread A happens before
KASSERT(invariant(obj->foo, obj->bar));
free_stuff(obj);
in thread B. Without them, this ordering is not guaranteed.
So in general it is necessary to do
membar_exit();
if (atomic_dec_uint_nv(&obj->refcnt) != 0)
return;
membar_enter();
to release a reference, for the `last one out hit the lights' style
of reference counting. (This is in contrast to the style where one
thread blocks new references and then waits under a lock for existing
ones to drain with a condvar -- no membar needed thanks to mutex(9).)
I searched for atomic_dec to find all these. Obviously we ought to
have a better abstraction for this because there's so much copypasta.
This is a stop-gap measure to fix actual bugs until we have that. It
would be nice if an abstraction could gracefully handle the different
styles of reference counting in use -- some years ago I drafted an
API for this, but making it cover everything got a little out of hand
(particularly with struct vnode::v_usecount) and I ended up setting
it aside to work on psref/localcount instead for better scalability.
I got bored of adding #ifdef __HAVE_ATOMIC_AS_MEMBAR everywhere, so I
only put it on things that look performance-critical on 5sec review.
We should really adopt membar_enter_preatomic/membar_exit_postatomic
or something (except they are applicable only to atomic r/m/w, not to
atomic_load/store_*, making the naming annoying) and get rid of all
the ifdefs.
To generate a diff of this commit:
cvs rdiff -u -r1.129 -r1.130 src/sys/arch/aarch64/aarch64/pmap.c
cvs rdiff -u -r1.304 -r1.305 src/sys/arch/alpha/alpha/pmap.c
cvs rdiff -u -r1.432 -r1.433 src/sys/arch/arm/arm32/pmap.c
cvs rdiff -u -r1.114 -r1.115 src/sys/arch/hppa/hppa/pmap.c
cvs rdiff -u -r1.40 -r1.41 src/sys/arch/ia64/ia64/pmap.c
cvs rdiff -u -r1.111 -r1.112 src/sys/arch/powerpc/oea/pmap.c
cvs rdiff -u -r1.375 -r1.376 src/sys/arch/sparc/sparc/pmap.c
cvs rdiff -u -r1.313 -r1.314 src/sys/arch/sparc64/sparc64/pmap.c
cvs rdiff -u -r1.15 -r1.16 src/sys/dev/hyperv/vmbus.c
cvs rdiff -u -r1.10 -r1.11 src/sys/dev/marvell/mvxpsec.c
cvs rdiff -u -r1.93 -r1.94 src/sys/dev/scsipi/atapiconf.c
cvs rdiff -u -r1.298 -r1.299 src/sys/dev/scsipi/scsiconf.c
cvs rdiff -u -r1.187 -r1.188 src/sys/dev/scsipi/scsipi_base.c
cvs rdiff -u -r1.2 -r1.3 src/sys/external/bsd/drm2/linux/linux_stop_machine.c
cvs rdiff -u -r1.78 -r1.79 src/sys/kern/kern_auth.c
cvs rdiff -u -r1.515 -r1.516 src/sys/kern/kern_exec.c
cvs rdiff -u -r1.7 -r1.8 src/sys/kern/kern_mutex_obj.c
cvs rdiff -u -r1.187 -r1.188 src/sys/kern/kern_resource.c
cvs rdiff -u -r1.5 -r1.6 src/sys/kern/kern_rwlock_obj.c
cvs rdiff -u -r1.402 -r1.403 src/sys/kern/kern_sig.c
cvs rdiff -u -r1.12 -r1.13 src/sys/kern/subr_kcpuset.c
cvs rdiff -u -r1.15 -r1.16 src/sys/kern/sys_futex.c
cvs rdiff -u -r1.244 -r1.245 src/sys/kern/uipc_mbuf.c
cvs rdiff -u -r1.6 -r1.7 src/sys/kern/vfs_cwd.c
cvs rdiff -u -r1.87 -r1.88 src/sys/kern/vfs_mount.c
cvs rdiff -u -r1.135 -r1.136 src/sys/kern/vfs_vnode.c
cvs rdiff -u -r1.109 -r1.110 src/sys/kern/vfs_wapbl.c
cvs rdiff -u -r1.501 -r1.502 src/sys/net/if.c
cvs rdiff -u -r1.50 -r1.51 src/sys/net/npf/npf_nat.c
cvs rdiff -u -r1.20 -r1.21 src/sys/net/npf/npf_rproc.c
cvs rdiff -u -r1.36 -r1.37 src/sys/net/npf/npf_tableset.c
cvs rdiff -u -r1.153 -r1.154 src/sys/uvm/uvm_aobj.c
cvs rdiff -u -r1.391 -r1.392 src/sys/uvm/uvm_map.c
cvs rdiff -u -r1.62 -r1.63 src/sys/uvm/pmap/pmap.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/aarch64/aarch64/pmap.c
diff -u src/sys/arch/aarch64/aarch64/pmap.c:1.129 src/sys/arch/aarch64/aarch64/pmap.c:1.130
--- src/sys/arch/aarch64/aarch64/pmap.c:1.129 Sat Mar 5 16:53:24 2022
+++ src/sys/arch/aarch64/aarch64/pmap.c Sat Mar 12 15:32:30 2022
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.c,v 1.129 2022/03/05 16:53:24 skrll Exp $ */
+/* $NetBSD: pmap.c,v 1.130 2022/03/12 15:32:30 riastradh Exp $ */
/*
* Copyright (c) 2017 Ryo Shimizu <r...@nerv.org>
@@ -27,7 +27,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.129 2022/03/05 16:53:24 skrll Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.130 2022/03/12 15:32:30 riastradh Exp $");
#include "opt_arm_debug.h"
#include "opt_cpuoptions.h"
@@ -1560,9 +1560,11 @@ pmap_destroy(struct pmap *pm)
if (pm == pmap_kernel())
panic("cannot destroy kernel pmap");
+ membar_exit();
refcnt = atomic_dec_uint_nv(&pm->pm_refcnt);
if (refcnt > 0)
return;
+ membar_enter();
KASSERT(LIST_EMPTY(&pm->pm_pvlist));
pmap_tlb_asid_release_all(pm);
Index: src/sys/arch/alpha/alpha/pmap.c
diff -u src/sys/arch/alpha/alpha/pmap.c:1.304 src/sys/arch/alpha/alpha/pmap.c:1.305
--- src/sys/arch/alpha/alpha/pmap.c:1.304 Thu Dec 9 21:13:18 2021
+++ src/sys/arch/alpha/alpha/pmap.c Sat Mar 12 15:32:31 2022
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.c,v 1.304 2021/12/09 21:13:18 andvar Exp $ */
+/* $NetBSD: pmap.c,v 1.305 2022/03/12 15:32:31 riastradh Exp $ */
/*-
* Copyright (c) 1998, 1999, 2000, 2001, 2007, 2008, 2020
@@ -135,7 +135,7 @@
#include <sys/cdefs.h> /* RCS ID & Copyright macro defns */
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.304 2021/12/09 21:13:18 andvar Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.305 2022/03/12 15:32:31 riastradh Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -1663,6 +1663,7 @@ pmap_destroy(pmap_t pmap)
KASSERT(atomic_load_relaxed(&pmap->pm_count) > 0);
if (atomic_dec_uint_nv(&pmap->pm_count) > 0)
return;
+ PMAP_MP(membar_enter());
pt_entry_t *lev1map = pmap_lev1map(pmap);
Index: src/sys/arch/arm/arm32/pmap.c
diff -u src/sys/arch/arm/arm32/pmap.c:1.432 src/sys/arch/arm/arm32/pmap.c:1.433
--- src/sys/arch/arm/arm32/pmap.c:1.432 Sun Jan 2 11:20:03 2022
+++ src/sys/arch/arm/arm32/pmap.c Sat Mar 12 15:32:31 2022
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.c,v 1.432 2022/01/02 11:20:03 riastradh Exp $ */
+/* $NetBSD: pmap.c,v 1.433 2022/03/12 15:32:31 riastradh Exp $ */
/*
* Copyright 2003 Wasabi Systems, Inc.
@@ -192,7 +192,7 @@
#endif
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.432 2022/01/02 11:20:03 riastradh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.433 2022/03/12 15:32:31 riastradh Exp $");
#include <sys/param.h>
#include <sys/types.h>
@@ -5275,6 +5275,7 @@ pmap_destroy(pmap_t pm)
/*
* Drop reference count
*/
+ membar_exit();
if (atomic_dec_uint_nv(&pm->pm_refs) > 0) {
#ifndef ARM_MMU_EXTENDED
if (pmap_is_current(pm)) {
@@ -5285,6 +5286,7 @@ pmap_destroy(pmap_t pm)
#endif
return;
}
+ membar_enter();
/*
* reference count is zero, free pmap resources and then free pmap.
Index: src/sys/arch/hppa/hppa/pmap.c
diff -u src/sys/arch/hppa/hppa/pmap.c:1.114 src/sys/arch/hppa/hppa/pmap.c:1.115
--- src/sys/arch/hppa/hppa/pmap.c:1.114 Wed Aug 19 07:29:00 2020
+++ src/sys/arch/hppa/hppa/pmap.c Sat Mar 12 15:32:31 2022
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.c,v 1.114 2020/08/19 07:29:00 simonb Exp $ */
+/* $NetBSD: pmap.c,v 1.115 2022/03/12 15:32:31 riastradh Exp $ */
/*-
* Copyright (c) 2001, 2002, 2020 The NetBSD Foundation, Inc.
@@ -65,7 +65,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.114 2020/08/19 07:29:00 simonb Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.115 2022/03/12 15:32:31 riastradh Exp $");
#include "opt_cputype.h"
@@ -1249,8 +1249,10 @@ pmap_destroy(pmap_t pmap)
off_t off;
#endif
+ membar_exit();
if (atomic_dec_uint_nv(&pmap->pm_obj.uo_refs) > 0)
return;
+ membar_enter();
#ifdef DIAGNOSTIC
uvm_page_array_init(&a, &pmap->pm_obj, 0);
Index: src/sys/arch/ia64/ia64/pmap.c
diff -u src/sys/arch/ia64/ia64/pmap.c:1.40 src/sys/arch/ia64/ia64/pmap.c:1.41
--- src/sys/arch/ia64/ia64/pmap.c:1.40 Sat Mar 14 14:05:42 2020
+++ src/sys/arch/ia64/ia64/pmap.c Sat Mar 12 15:32:31 2022
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.c,v 1.40 2020/03/14 14:05:42 ad Exp $ */
+/* $NetBSD: pmap.c,v 1.41 2022/03/12 15:32:31 riastradh Exp $ */
/*-
* Copyright (c) 1998, 1999, 2000, 2001 The NetBSD Foundation, Inc.
@@ -81,7 +81,7 @@
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.40 2020/03/14 14:05:42 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.41 2022/03/12 15:32:31 riastradh Exp $");
#include <sys/param.h>
#include <sys/atomic.h>
@@ -1517,8 +1517,10 @@ pmap_destroy(pmap_t pmap)
UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
UVMHIST_LOG(maphist, "(pm=%p)", pmap, 0, 0, 0);
+ membar_exit();
if (atomic_dec_64_nv(&pmap->pm_refcount) > 0)
return;
+ membar_enter();
KASSERT(pmap->pm_stats.resident_count == 0);
KASSERT(pmap->pm_stats.wired_count == 0);
Index: src/sys/arch/powerpc/oea/pmap.c
diff -u src/sys/arch/powerpc/oea/pmap.c:1.111 src/sys/arch/powerpc/oea/pmap.c:1.112
--- src/sys/arch/powerpc/oea/pmap.c:1.111 Fri Feb 18 19:04:52 2022
+++ src/sys/arch/powerpc/oea/pmap.c Sat Mar 12 15:32:31 2022
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.c,v 1.111 2022/02/18 19:04:52 martin Exp $ */
+/* $NetBSD: pmap.c,v 1.112 2022/03/12 15:32:31 riastradh Exp $ */
/*-
* Copyright (c) 2001 The NetBSD Foundation, Inc.
* All rights reserved.
@@ -63,7 +63,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.111 2022/02/18 19:04:52 martin Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.112 2022/03/12 15:32:31 riastradh Exp $");
#define PMAP_NOOPNAMES
@@ -1230,7 +1230,9 @@ pmap_reference(pmap_t pm)
void
pmap_destroy(pmap_t pm)
{
+ membar_exit();
if (atomic_dec_uint_nv(&pm->pm_refs) == 0) {
+ membar_enter();
pmap_release(pm);
pool_put(&pmap_pool, pm);
}
Index: src/sys/arch/sparc/sparc/pmap.c
diff -u src/sys/arch/sparc/sparc/pmap.c:1.375 src/sys/arch/sparc/sparc/pmap.c:1.376
--- src/sys/arch/sparc/sparc/pmap.c:1.375 Mon Aug 9 21:08:06 2021
+++ src/sys/arch/sparc/sparc/pmap.c Sat Mar 12 15:32:31 2022
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.c,v 1.375 2021/08/09 21:08:06 andvar Exp $ */
+/* $NetBSD: pmap.c,v 1.376 2022/03/12 15:32:31 riastradh Exp $ */
/*
* Copyright (c) 1996
@@ -56,7 +56,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.375 2021/08/09 21:08:06 andvar Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.376 2022/03/12 15:32:31 riastradh Exp $");
#include "opt_ddb.h"
#include "opt_kgdb.h"
@@ -4465,7 +4465,9 @@ pmap_destroy(struct pmap *pm)
{
DPRINTF(PDB_DESTROY, "pmap_destroy[%d](%p)", cpu_number(), pm);
+ membar_exit();
if (atomic_dec_uint_nv(&pm->pm_refcount) == 0) {
+ membar_enter();
pmap_quiet_check(pm);
pool_cache_put(&pmap_cache, pm);
}
Index: src/sys/arch/sparc64/sparc64/pmap.c
diff -u src/sys/arch/sparc64/sparc64/pmap.c:1.313 src/sys/arch/sparc64/sparc64/pmap.c:1.314
--- src/sys/arch/sparc64/sparc64/pmap.c:1.313 Sat Jan 1 11:56:15 2022
+++ src/sys/arch/sparc64/sparc64/pmap.c Sat Mar 12 15:32:31 2022
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.c,v 1.313 2022/01/01 11:56:15 hannken Exp $ */
+/* $NetBSD: pmap.c,v 1.314 2022/03/12 15:32:31 riastradh Exp $ */
/*
*
* Copyright (C) 1996-1999 Eduardo Horvath.
@@ -26,7 +26,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.313 2022/01/01 11:56:15 hannken Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.314 2022/03/12 15:32:31 riastradh Exp $");
#undef NO_VCACHE /* Don't forget the locked TLB in dostart */
#define HWREF
@@ -1510,9 +1510,11 @@ pmap_destroy(struct pmap *pm)
#endif
struct vm_page *pg;
+ membar_exit();
if ((int)atomic_dec_uint_nv(&pm->pm_refs) > 0) {
return;
}
+ membar_enter();
DPRINTF(PDB_DESTROY, ("pmap_destroy: freeing pmap %p\n", pm));
#ifdef MULTIPROCESSOR
CPUSET_CLEAR(pmap_cpus_active);
Index: src/sys/dev/hyperv/vmbus.c
diff -u src/sys/dev/hyperv/vmbus.c:1.15 src/sys/dev/hyperv/vmbus.c:1.16
--- src/sys/dev/hyperv/vmbus.c:1.15 Thu Dec 23 04:06:51 2021
+++ src/sys/dev/hyperv/vmbus.c Sat Mar 12 15:32:31 2022
@@ -1,4 +1,4 @@
-/* $NetBSD: vmbus.c,v 1.15 2021/12/23 04:06:51 yamaguchi Exp $ */
+/* $NetBSD: vmbus.c,v 1.16 2022/03/12 15:32:31 riastradh Exp $ */
/* $OpenBSD: hyperv.c,v 1.43 2017/06/27 13:56:15 mikeb Exp $ */
/*-
@@ -35,7 +35,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: vmbus.c,v 1.15 2021/12/23 04:06:51 yamaguchi Exp $");
+__KERNEL_RCSID(0, "$NetBSD: vmbus.c,v 1.16 2022/03/12 15:32:31 riastradh Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -1452,8 +1452,10 @@ vmbus_channel_detach(struct vmbus_channe
KASSERTMSG(ch->ch_refs > 0, "channel%u: invalid refcnt %d",
ch->ch_id, ch->ch_refs);
+ membar_exit();
refs = atomic_dec_uint_nv(&ch->ch_refs);
if (refs == 0) {
+ membar_enter();
/* Detach the target channel. */
vmbus_devq_enqueue(ch->ch_sc, VMBUS_DEV_TYPE_DETACH, ch);
}
Index: src/sys/dev/marvell/mvxpsec.c
diff -u src/sys/dev/marvell/mvxpsec.c:1.10 src/sys/dev/marvell/mvxpsec.c:1.11
--- src/sys/dev/marvell/mvxpsec.c:1.10 Sat Feb 12 03:24:36 2022
+++ src/sys/dev/marvell/mvxpsec.c Sat Mar 12 15:32:32 2022
@@ -1,4 +1,4 @@
-/* $NetBSD: mvxpsec.c,v 1.10 2022/02/12 03:24:36 riastradh Exp $ */
+/* $NetBSD: mvxpsec.c,v 1.11 2022/03/12 15:32:32 riastradh Exp $ */
/*
* Copyright (c) 2015 Internet Initiative Japan Inc.
* All rights reserved.
@@ -1552,9 +1552,12 @@ mvxpsec_session_unref(struct mvxpsec_ses
{
uint32_t refs;
+ membar_exit();
refs = atomic_dec_32_nv(&mv_s->refs);
- if (refs == 0)
+ if (refs == 0) {
+ membar_enter();
pool_cache_put(mv_s->sc->sc_session_pool, mv_s);
+ }
}
/*
Index: src/sys/dev/scsipi/atapiconf.c
diff -u src/sys/dev/scsipi/atapiconf.c:1.93 src/sys/dev/scsipi/atapiconf.c:1.94
--- src/sys/dev/scsipi/atapiconf.c:1.93 Sat Aug 7 16:19:16 2021
+++ src/sys/dev/scsipi/atapiconf.c Sat Mar 12 15:32:32 2022
@@ -1,4 +1,4 @@
-/* $NetBSD: atapiconf.c,v 1.93 2021/08/07 16:19:16 thorpej Exp $ */
+/* $NetBSD: atapiconf.c,v 1.94 2022/03/12 15:32:32 riastradh Exp $ */
/*
* Copyright (c) 1996, 2001 Manuel Bouyer. All rights reserved.
@@ -25,7 +25,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: atapiconf.c,v 1.93 2021/08/07 16:19:16 thorpej Exp $");
+__KERNEL_RCSID(0, "$NetBSD: atapiconf.c,v 1.94 2022/03/12 15:32:32 riastradh Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -219,8 +219,11 @@ atapibusdetach(device_t self, int flags)
cv_destroy(&chan->chan_cv_comp);
cv_destroy(&chan->chan_cv_thr);
- if (atomic_dec_uint_nv(&chan_running(chan)) == 0)
+ membar_exit();
+ if (atomic_dec_uint_nv(&chan_running(chan)) == 0) {
+ membar_enter();
mutex_destroy(chan_mtx(chan));
+ }
return 0;
}
Index: src/sys/dev/scsipi/scsiconf.c
diff -u src/sys/dev/scsipi/scsiconf.c:1.298 src/sys/dev/scsipi/scsiconf.c:1.299
--- src/sys/dev/scsipi/scsiconf.c:1.298 Sat Feb 5 17:32:59 2022
+++ src/sys/dev/scsipi/scsiconf.c Sat Mar 12 15:32:32 2022
@@ -1,4 +1,4 @@
-/* $NetBSD: scsiconf.c,v 1.298 2022/02/05 17:32:59 hannken Exp $ */
+/* $NetBSD: scsiconf.c,v 1.299 2022/03/12 15:32:32 riastradh Exp $ */
/*-
* Copyright (c) 1998, 1999, 2004 The NetBSD Foundation, Inc.
@@ -48,7 +48,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: scsiconf.c,v 1.298 2022/02/05 17:32:59 hannken Exp $");
+__KERNEL_RCSID(0, "$NetBSD: scsiconf.c,v 1.299 2022/03/12 15:32:32 riastradh Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -365,8 +365,11 @@ scsibusdetach(device_t self, int flags)
cv_destroy(&chan->chan_cv_comp);
cv_destroy(&chan->chan_cv_thr);
- if (atomic_dec_uint_nv(&chan_running(chan)) == 0)
+ membar_exit();
+ if (atomic_dec_uint_nv(&chan_running(chan)) == 0) {
+ membar_enter();
mutex_destroy(chan_mtx(chan));
+ }
return 0;
}
Index: src/sys/dev/scsipi/scsipi_base.c
diff -u src/sys/dev/scsipi/scsipi_base.c:1.187 src/sys/dev/scsipi/scsipi_base.c:1.188
--- src/sys/dev/scsipi/scsipi_base.c:1.187 Thu Sep 17 01:19:41 2020
+++ src/sys/dev/scsipi/scsipi_base.c Sat Mar 12 15:32:32 2022
@@ -1,4 +1,4 @@
-/* $NetBSD: scsipi_base.c,v 1.187 2020/09/17 01:19:41 jakllsch Exp $ */
+/* $NetBSD: scsipi_base.c,v 1.188 2022/03/12 15:32:32 riastradh Exp $ */
/*-
* Copyright (c) 1998, 1999, 2000, 2002, 2003, 2004 The NetBSD Foundation, Inc.
@@ -31,7 +31,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: scsipi_base.c,v 1.187 2020/09/17 01:19:41 jakllsch Exp $");
+__KERNEL_RCSID(0, "$NetBSD: scsipi_base.c,v 1.188 2022/03/12 15:32:32 riastradh Exp $");
#ifdef _KERNEL_OPT
#include "opt_scsi.h"
@@ -2733,8 +2733,10 @@ void
scsipi_adapter_delref(struct scsipi_adapter *adapt)
{
+ membar_exit();
if (atomic_dec_uint_nv(&adapt->adapt_refcnt) == 0
&& adapt->adapt_enable != NULL) {
+ membar_enter();
scsipi_adapter_lock(adapt);
(void) scsipi_adapter_enable(adapt, 0);
scsipi_adapter_unlock(adapt);
Index: src/sys/external/bsd/drm2/linux/linux_stop_machine.c
diff -u src/sys/external/bsd/drm2/linux/linux_stop_machine.c:1.2 src/sys/external/bsd/drm2/linux/linux_stop_machine.c:1.3
--- src/sys/external/bsd/drm2/linux/linux_stop_machine.c:1.2 Sun Dec 19 11:22:22 2021
+++ src/sys/external/bsd/drm2/linux/linux_stop_machine.c Sat Mar 12 15:32:32 2022
@@ -1,4 +1,4 @@
-/* $NetBSD: linux_stop_machine.c,v 1.2 2021/12/19 11:22:22 riastradh Exp $ */
+/* $NetBSD: linux_stop_machine.c,v 1.3 2022/03/12 15:32:32 riastradh Exp $ */
/*-
* Copyright (c) 2018 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: linux_stop_machine.c,v 1.2 2021/12/19 11:22:22 riastradh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: linux_stop_machine.c,v 1.3 2022/03/12 15:32:32 riastradh Exp $");
#include <sys/mutex.h> /* XXX work around cycle x86/mutex.h<->x86/intr.h */
@@ -59,18 +59,19 @@ stop_machine_xcall(void *a, void *b)
s = splhigh();
/* Note that we're ready, and see whether we're the chosen one. */
+ membar_exit();
if (atomic_dec_uint_nv(&S->ncpu) != 0) {
- /* Not the chosen one. Wait until done. */
- while (!S->done)
+ while (!atomic_load_acquire(&S->done))
SPINLOCK_BACKOFF_HOOK;
goto out;
}
+ membar_enter();
/* It's time. Call the callback. */
S->callback(S->cookie);
/* Notify everyone else that we're done. */
- S->done = true;
+ atomic_store_release(&S->done, true);
/* Allow activity again. */
out: splx(s);
Index: src/sys/kern/kern_auth.c
diff -u src/sys/kern/kern_auth.c:1.78 src/sys/kern/kern_auth.c:1.79
--- src/sys/kern/kern_auth.c:1.78 Sat May 16 18:31:50 2020
+++ src/sys/kern/kern_auth.c Sat Mar 12 15:32:32 2022
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_auth.c,v 1.78 2020/05/16 18:31:50 christos Exp $ */
+/* $NetBSD: kern_auth.c,v 1.79 2022/03/12 15:32:32 riastradh Exp $ */
/*-
* Copyright (c) 2005, 2006 Elad Efrat <e...@netbsd.org>
@@ -28,7 +28,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_auth.c,v 1.78 2020/05/16 18:31:50 christos Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_auth.c,v 1.79 2022/03/12 15:32:32 riastradh Exp $");
#include <sys/types.h>
#include <sys/param.h>
@@ -144,8 +144,14 @@ kauth_cred_free(kauth_cred_t cred)
KASSERT(cred->cr_refcnt > 0);
ASSERT_SLEEPABLE();
+#ifndef __HAVE_ATOMIC_AS_MEMBAR
+ membar_exit();
+#endif
if (atomic_dec_uint_nv(&cred->cr_refcnt) > 0)
return;
+#ifndef __HAVE_ATOMIC_AS_MEMBAR
+ membar_enter();
+#endif
kauth_cred_hook(cred, KAUTH_CRED_FREE, NULL, NULL);
specificdata_fini(kauth_domain, &cred->cr_sd);
Index: src/sys/kern/kern_exec.c
diff -u src/sys/kern/kern_exec.c:1.515 src/sys/kern/kern_exec.c:1.516
--- src/sys/kern/kern_exec.c:1.515 Sat Feb 5 23:10:20 2022
+++ src/sys/kern/kern_exec.c Sat Mar 12 15:32:32 2022
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_exec.c,v 1.515 2022/02/05 23:10:20 christos Exp $ */
+/* $NetBSD: kern_exec.c,v 1.516 2022/03/12 15:32:32 riastradh Exp $ */
/*-
* Copyright (c) 2008, 2019, 2020 The NetBSD Foundation, Inc.
@@ -62,7 +62,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_exec.c,v 1.515 2022/02/05 23:10:20 christos Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_exec.c,v 1.516 2022/03/12 15:32:32 riastradh Exp $");
#include "opt_exec.h"
#include "opt_execfmt.h"
@@ -2136,8 +2136,11 @@ exec_sigcode_map(struct proc *p, const s
static void
spawn_exec_data_release(struct spawn_exec_data *data)
{
+
+ membar_exit();
if (atomic_dec_32_nv(&data->sed_refcnt) != 0)
return;
+ membar_enter();
cv_destroy(&data->sed_cv_child_ready);
mutex_destroy(&data->sed_mtx_child);
Index: src/sys/kern/kern_mutex_obj.c
diff -u src/sys/kern/kern_mutex_obj.c:1.7 src/sys/kern/kern_mutex_obj.c:1.8
--- src/sys/kern/kern_mutex_obj.c:1.7 Wed Jan 1 21:34:39 2020
+++ src/sys/kern/kern_mutex_obj.c Sat Mar 12 15:32:32 2022
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_mutex_obj.c,v 1.7 2020/01/01 21:34:39 ad Exp $ */
+/* $NetBSD: kern_mutex_obj.c,v 1.8 2022/03/12 15:32:32 riastradh Exp $ */
/*-
* Copyright (c) 2008, 2019 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_mutex_obj.c,v 1.7 2020/01/01 21:34:39 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_mutex_obj.c,v 1.8 2022/03/12 15:32:32 riastradh Exp $");
#include <sys/param.h>
#include <sys/atomic.h>
@@ -157,9 +157,15 @@ mutex_obj_free(kmutex_t *lock)
"%s: lock %p: mo->mo_refcnt (%#x) == 0",
__func__, mo, mo->mo_refcnt);
+#ifndef __HAVE_ATOMIC_AS_MEMBAR
+ membar_exit();
+#endif
if (atomic_dec_uint_nv(&mo->mo_refcnt) > 0) {
return false;
}
+#ifndef __HAVE_ATOMIC_AS_MEMBAR
+ membar_enter();
+#endif
mutex_destroy(&mo->mo_lock);
pool_cache_put(mutex_obj_cache, mo);
return true;
Index: src/sys/kern/kern_resource.c
diff -u src/sys/kern/kern_resource.c:1.187 src/sys/kern/kern_resource.c:1.188
--- src/sys/kern/kern_resource.c:1.187 Sat May 23 23:42:43 2020
+++ src/sys/kern/kern_resource.c Sat Mar 12 15:32:32 2022
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_resource.c,v 1.187 2020/05/23 23:42:43 ad Exp $ */
+/* $NetBSD: kern_resource.c,v 1.188 2022/03/12 15:32:32 riastradh Exp $ */
/*-
* Copyright (c) 1982, 1986, 1991, 1993
@@ -37,7 +37,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_resource.c,v 1.187 2020/05/23 23:42:43 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_resource.c,v 1.188 2022/03/12 15:32:32 riastradh Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -793,9 +793,11 @@ lim_free(struct plimit *lim)
struct plimit *sv_lim;
do {
+ membar_exit();
if (atomic_dec_uint_nv(&lim->pl_refcnt) > 0) {
return;
}
+ membar_enter();
if (lim->pl_corename != defcorename) {
kmem_free(lim->pl_corename, lim->pl_cnlen);
}
Index: src/sys/kern/kern_rwlock_obj.c
diff -u src/sys/kern/kern_rwlock_obj.c:1.5 src/sys/kern/kern_rwlock_obj.c:1.6
--- src/sys/kern/kern_rwlock_obj.c:1.5 Wed Jan 1 21:34:39 2020
+++ src/sys/kern/kern_rwlock_obj.c Sat Mar 12 15:32:32 2022
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_rwlock_obj.c,v 1.5 2020/01/01 21:34:39 ad Exp $ */
+/* $NetBSD: kern_rwlock_obj.c,v 1.6 2022/03/12 15:32:32 riastradh Exp $ */
/*-
* Copyright (c) 2008, 2009, 2019 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_rwlock_obj.c,v 1.5 2020/01/01 21:34:39 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_rwlock_obj.c,v 1.6 2022/03/12 15:32:32 riastradh Exp $");
#include <sys/param.h>
#include <sys/atomic.h>
@@ -147,9 +147,15 @@ rw_obj_free(krwlock_t *lock)
KASSERT(ro->ro_magic == RW_OBJ_MAGIC);
KASSERT(ro->ro_refcnt > 0);
+#ifndef __HAVE_ATOMIC_AS_MEMBAR
+ membar_exit();
+#endif
if (atomic_dec_uint_nv(&ro->ro_refcnt) > 0) {
return false;
}
+#ifndef __HAVE_ATOMIC_AS_MEMBAR
+ membar_enter();
+#endif
rw_destroy(&ro->ro_lock);
pool_cache_put(rw_obj_cache, ro);
return true;
Index: src/sys/kern/kern_sig.c
diff -u src/sys/kern/kern_sig.c:1.402 src/sys/kern/kern_sig.c:1.403
--- src/sys/kern/kern_sig.c:1.402 Wed Feb 23 21:54:41 2022
+++ src/sys/kern/kern_sig.c Sat Mar 12 15:32:32 2022
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_sig.c,v 1.402 2022/02/23 21:54:41 andvar Exp $ */
+/* $NetBSD: kern_sig.c,v 1.403 2022/03/12 15:32:32 riastradh Exp $ */
/*-
* Copyright (c) 2006, 2007, 2008, 2019 The NetBSD Foundation, Inc.
@@ -70,7 +70,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_sig.c,v 1.402 2022/02/23 21:54:41 andvar Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_sig.c,v 1.403 2022/03/12 15:32:32 riastradh Exp $");
#include "opt_execfmt.h"
#include "opt_ptrace.h"
@@ -311,7 +311,9 @@ void
sigactsfree(struct sigacts *ps)
{
+ membar_exit();
if (atomic_dec_uint_nv(&ps->sa_refcnt) == 0) {
+ membar_enter();
mutex_destroy(&ps->sa_mutex);
pool_cache_put(sigacts_cache, ps);
}
Index: src/sys/kern/subr_kcpuset.c
diff -u src/sys/kern/subr_kcpuset.c:1.12 src/sys/kern/subr_kcpuset.c:1.13
--- src/sys/kern/subr_kcpuset.c:1.12 Fri Jul 26 05:39:55 2019
+++ src/sys/kern/subr_kcpuset.c Sat Mar 12 15:32:32 2022
@@ -1,4 +1,4 @@
-/* $NetBSD: subr_kcpuset.c,v 1.12 2019/07/26 05:39:55 msaitoh Exp $ */
+/* $NetBSD: subr_kcpuset.c,v 1.13 2022/03/12 15:32:32 riastradh Exp $ */
/*-
* Copyright (c) 2011 The NetBSD Foundation, Inc.
@@ -41,7 +41,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: subr_kcpuset.c,v 1.12 2019/07/26 05:39:55 msaitoh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: subr_kcpuset.c,v 1.13 2022/03/12 15:32:32 riastradh Exp $");
#include <sys/param.h>
#include <sys/types.h>
@@ -262,9 +262,11 @@ kcpuset_unuse(kcpuset_t *kcp, kcpuset_t
KASSERT(kc_initialised);
KASSERT(kc->kc_refcnt > 0);
+ membar_exit();
if (atomic_dec_uint_nv(&kc->kc_refcnt) != 0) {
return;
}
+ membar_enter();
KASSERT(kc->kc_next == NULL);
if (lst == NULL) {
kcpuset_destroy(kcp);
Index: src/sys/kern/sys_futex.c
diff -u src/sys/kern/sys_futex.c:1.15 src/sys/kern/sys_futex.c:1.16
--- src/sys/kern/sys_futex.c:1.15 Mon Nov 1 08:35:17 2021
+++ src/sys/kern/sys_futex.c Sat Mar 12 15:32:32 2022
@@ -1,4 +1,4 @@
-/* $NetBSD: sys_futex.c,v 1.15 2021/11/01 08:35:17 chs Exp $ */
+/* $NetBSD: sys_futex.c,v 1.16 2022/03/12 15:32:32 riastradh Exp $ */
/*-
* Copyright (c) 2018, 2019, 2020 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: sys_futex.c,v 1.15 2021/11/01 08:35:17 chs Exp $");
+__KERNEL_RCSID(0, "$NetBSD: sys_futex.c,v 1.16 2022/03/12 15:32:32 riastradh Exp $");
/*
* Futexes
@@ -537,12 +537,18 @@ futex_rele(struct futex *f)
refcnt = atomic_load_relaxed(&f->fx_refcnt);
if (refcnt == 1)
goto trylast;
+#ifndef __HAVE_ATOMIC_AS_MEMBAR
+ membar_exit();
+#endif
} while (atomic_cas_ulong(&f->fx_refcnt, refcnt, refcnt - 1) != refcnt);
return;
trylast:
mutex_enter(&futex_tab.lock);
if (atomic_dec_ulong_nv(&f->fx_refcnt) == 0) {
+#ifndef __HAVE_ATOMIC_AS_MEMBAR
+ membar_enter();
+#endif
if (f->fx_on_tree) {
if (__predict_false(f->fx_shared))
rb_tree_remove_node(&futex_tab.oa, f);
Index: src/sys/kern/uipc_mbuf.c
diff -u src/sys/kern/uipc_mbuf.c:1.244 src/sys/kern/uipc_mbuf.c:1.245
--- src/sys/kern/uipc_mbuf.c:1.244 Wed Oct 6 05:24:54 2021
+++ src/sys/kern/uipc_mbuf.c Sat Mar 12 15:32:32 2022
@@ -1,4 +1,4 @@
-/* $NetBSD: uipc_mbuf.c,v 1.244 2021/10/06 05:24:54 msaitoh Exp $ */
+/* $NetBSD: uipc_mbuf.c,v 1.245 2022/03/12 15:32:32 riastradh Exp $ */
/*
* Copyright (c) 1999, 2001, 2018 The NetBSD Foundation, Inc.
@@ -62,7 +62,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uipc_mbuf.c,v 1.244 2021/10/06 05:24:54 msaitoh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uipc_mbuf.c,v 1.245 2022/03/12 15:32:32 riastradh Exp $");
#ifdef _KERNEL_OPT
#include "opt_mbuftrace.h"
@@ -1914,6 +1914,9 @@ m_ext_free(struct mbuf *m)
if (__predict_true(m->m_ext.ext_refcnt == 1)) {
refcnt = m->m_ext.ext_refcnt = 0;
} else {
+#ifndef __HAVE_ATOMIC_AS_MEMBAR
+ membar_exit();
+#endif
refcnt = atomic_dec_uint_nv(&m->m_ext.ext_refcnt);
}
@@ -1930,6 +1933,9 @@ m_ext_free(struct mbuf *m)
/*
* dropping the last reference
*/
+#ifndef __HAVE_ATOMIC_AS_MEMBAR
+ membar_enter();
+#endif
if (!embedded) {
m->m_ext.ext_refcnt++; /* XXX */
m_ext_free(m->m_ext_ref);
Index: src/sys/kern/vfs_cwd.c
diff -u src/sys/kern/vfs_cwd.c:1.6 src/sys/kern/vfs_cwd.c:1.7
--- src/sys/kern/vfs_cwd.c:1.6 Tue Apr 21 21:42:47 2020
+++ src/sys/kern/vfs_cwd.c Sat Mar 12 15:32:32 2022
@@ -1,4 +1,4 @@
-/* $NetBSD: vfs_cwd.c,v 1.6 2020/04/21 21:42:47 ad Exp $ */
+/* $NetBSD: vfs_cwd.c,v 1.7 2022/03/12 15:32:32 riastradh Exp $ */
/*-
* Copyright (c) 2008, 2020 The NetBSD Foundation, Inc.
@@ -31,7 +31,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: vfs_cwd.c,v 1.6 2020/04/21 21:42:47 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: vfs_cwd.c,v 1.7 2022/03/12 15:32:32 riastradh Exp $");
#include <sys/param.h>
#include <sys/atomic.h>
@@ -138,8 +138,10 @@ void
cwdfree(struct cwdinfo *cwdi)
{
+ membar_exit();
if (atomic_dec_uint_nv(&cwdi->cwdi_refcnt) > 0)
return;
+ membar_enter();
vrele(cwdi->cwdi_cdir);
if (cwdi->cwdi_rdir)
Index: src/sys/kern/vfs_mount.c
diff -u src/sys/kern/vfs_mount.c:1.87 src/sys/kern/vfs_mount.c:1.88
--- src/sys/kern/vfs_mount.c:1.87 Fri Feb 4 15:33:57 2022
+++ src/sys/kern/vfs_mount.c Sat Mar 12 15:32:32 2022
@@ -1,4 +1,4 @@
-/* $NetBSD: vfs_mount.c,v 1.87 2022/02/04 15:33:57 hannken Exp $ */
+/* $NetBSD: vfs_mount.c,v 1.88 2022/03/12 15:32:32 riastradh Exp $ */
/*-
* Copyright (c) 1997-2020 The NetBSD Foundation, Inc.
@@ -67,7 +67,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: vfs_mount.c,v 1.87 2022/02/04 15:33:57 hannken Exp $");
+__KERNEL_RCSID(0, "$NetBSD: vfs_mount.c,v 1.88 2022/03/12 15:32:32 riastradh Exp $");
#include <sys/param.h>
#include <sys/kernel.h>
@@ -283,9 +283,15 @@ void
vfs_rele(struct mount *mp)
{
+#ifndef __HAVE_ATOMIC_AS_MEMBAR
+ membar_exit();
+#endif
if (__predict_true((int)atomic_dec_uint_nv(&mp->mnt_refcnt) > 0)) {
return;
}
+#ifndef __HAVE_ATOMIC_AS_MEMBAR
+ membar_enter();
+#endif
/*
* Nothing else has visibility of the mount: we can now
Index: src/sys/kern/vfs_vnode.c
diff -u src/sys/kern/vfs_vnode.c:1.135 src/sys/kern/vfs_vnode.c:1.136
--- src/sys/kern/vfs_vnode.c:1.135 Wed Mar 9 08:43:28 2022
+++ src/sys/kern/vfs_vnode.c Sat Mar 12 15:32:32 2022
@@ -1,4 +1,4 @@
-/* $NetBSD: vfs_vnode.c,v 1.135 2022/03/09 08:43:28 hannken Exp $ */
+/* $NetBSD: vfs_vnode.c,v 1.136 2022/03/12 15:32:32 riastradh Exp $ */
/*-
* Copyright (c) 1997-2011, 2019, 2020 The NetBSD Foundation, Inc.
@@ -148,7 +148,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: vfs_vnode.c,v 1.135 2022/03/09 08:43:28 hannken Exp $");
+__KERNEL_RCSID(0, "$NetBSD: vfs_vnode.c,v 1.136 2022/03/12 15:32:32 riastradh Exp $");
#ifdef _KERNEL_OPT
#include "opt_pax.h"
@@ -352,11 +352,15 @@ vstate_assert_change(vnode_t *vp, enum v
vnpanic(vp, "state is %s, gate %d does not match at %s:%d\n",
vstate_name(vip->vi_state), gated, func, line);
- /* Open/close the gate for vcache_tryvget(). */
- if (to == VS_LOADED)
+ /* Open/close the gate for vcache_tryvget(). */
+ if (to == VS_LOADED) {
+#ifndef __HAVE_ATOMIC_AS_MEMBAR
+ membar_exit();
+#endif
atomic_or_uint(&vp->v_usecount, VUSECOUNT_GATE);
- else
+ } else {
atomic_and_uint(&vp->v_usecount, ~VUSECOUNT_GATE);
+ }
vip->vi_state = to;
if (from == VS_LOADING)
@@ -395,10 +399,14 @@ vstate_change(vnode_t *vp, enum vnode_st
vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
/* Open/close the gate for vcache_tryvget(). */
- if (to == VS_LOADED)
+ if (to == VS_LOADED) {
+#ifndef __HAVE_ATOMIC_AS_MEMBAR
+ membar_exit();
+#endif
atomic_or_uint(&vp->v_usecount, VUSECOUNT_GATE);
- else
+ } else {
atomic_and_uint(&vp->v_usecount, ~VUSECOUNT_GATE);
+ }
vip->vi_state = to;
if (from == VS_LOADING)
@@ -727,6 +735,9 @@ vtryrele(vnode_t *vp)
{
u_int use, next;
+#ifndef __HAVE_ATOMIC_AS_MEMBAR
+ membar_exit();
+#endif
for (use = atomic_load_relaxed(&vp->v_usecount);; use = next) {
if (__predict_false((use & VUSECOUNT_MASK) == 1)) {
return false;
@@ -827,6 +838,9 @@ retry:
break;
}
}
+#ifndef __HAVE_ATOMIC_AS_MEMBAR
+ membar_enter();
+#endif
if (vrefcnt(vp) <= 0 || vp->v_writecount != 0) {
vnpanic(vp, "%s: bad ref count", __func__);
}
@@ -990,6 +1004,9 @@ out:
break;
}
}
+#ifndef __HAVE_ATOMIC_AS_MEMBAR
+ membar_enter();
+#endif
if (VSTATE_GET(vp) == VS_RECLAIMED && vp->v_holdcnt == 0) {
/*
@@ -1459,6 +1476,9 @@ vcache_tryvget(vnode_t *vp)
next = atomic_cas_uint(&vp->v_usecount,
use, (use + 1) | VUSECOUNT_VGET);
if (__predict_true(next == use)) {
+#ifndef __HAVE_ATOMIC_AS_MEMBAR
+ membar_enter();
+#endif
return 0;
}
}
Index: src/sys/kern/vfs_wapbl.c
diff -u src/sys/kern/vfs_wapbl.c:1.109 src/sys/kern/vfs_wapbl.c:1.110
--- src/sys/kern/vfs_wapbl.c:1.109 Tue Aug 3 20:25:43 2021
+++ src/sys/kern/vfs_wapbl.c Sat Mar 12 15:32:32 2022
@@ -1,4 +1,4 @@
-/* $NetBSD: vfs_wapbl.c,v 1.109 2021/08/03 20:25:43 chs Exp $ */
+/* $NetBSD: vfs_wapbl.c,v 1.110 2022/03/12 15:32:32 riastradh Exp $ */
/*-
* Copyright (c) 2003, 2008, 2009 The NetBSD Foundation, Inc.
@@ -36,7 +36,7 @@
#define WAPBL_INTERNAL
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: vfs_wapbl.c,v 1.109 2021/08/03 20:25:43 chs Exp $");
+__KERNEL_RCSID(0, "$NetBSD: vfs_wapbl.c,v 1.110 2022/03/12 15:32:32 riastradh Exp $");
#include <sys/param.h>
#include <sys/bitops.h>
@@ -2273,7 +2273,9 @@ wapbl_inodetrk_free(struct wapbl *wl)
/* XXX this KASSERT needs locking/mutex analysis */
KASSERT(wl->wl_inohashcnt == 0);
hashdone(wl->wl_inohash, HASH_LIST, wl->wl_inohashmask);
+ membar_exit();
if (atomic_dec_uint_nv(&wapbl_ino_pool_refcount) == 0) {
+ membar_enter();
pool_destroy(&wapbl_ino_pool);
}
}
Index: src/sys/net/if.c
diff -u src/sys/net/if.c:1.501 src/sys/net/if.c:1.502
--- src/sys/net/if.c:1.501 Fri Dec 31 14:26:29 2021
+++ src/sys/net/if.c Sat Mar 12 15:32:32 2022
@@ -1,4 +1,4 @@
-/* $NetBSD: if.c,v 1.501 2021/12/31 14:26:29 riastradh Exp $ */
+/* $NetBSD: if.c,v 1.502 2022/03/12 15:32:32 riastradh Exp $ */
/*-
* Copyright (c) 1999, 2000, 2001, 2008 The NetBSD Foundation, Inc.
@@ -90,7 +90,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: if.c,v 1.501 2021/12/31 14:26:29 riastradh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: if.c,v 1.502 2022/03/12 15:32:32 riastradh Exp $");
#if defined(_KERNEL_OPT)
#include "opt_inet.h"
@@ -1819,9 +1819,15 @@ ifafree(struct ifaddr *ifa)
KASSERT(ifa != NULL);
KASSERTMSG(ifa->ifa_refcnt > 0, "ifa_refcnt=%d", ifa->ifa_refcnt);
- if (atomic_dec_uint_nv(&ifa->ifa_refcnt) == 0) {
- free(ifa, M_IFADDR);
- }
+#ifndef __HAVE_ATOMIC_AS_MEMBAR
+ membar_exit();
+#endif
+ if (atomic_dec_uint_nv(&ifa->ifa_refcnt) != 0)
+ return;
+#ifndef __HAVE_ATOMIC_AS_MEMBAR
+ membar_enter();
+#endif
+ free(ifa, M_IFADDR);
}
bool
Index: src/sys/net/npf/npf_nat.c
diff -u src/sys/net/npf/npf_nat.c:1.50 src/sys/net/npf/npf_nat.c:1.51
--- src/sys/net/npf/npf_nat.c:1.50 Sat May 30 14:16:56 2020
+++ src/sys/net/npf/npf_nat.c Sat Mar 12 15:32:32 2022
@@ -67,7 +67,7 @@
#ifdef _KERNEL
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: npf_nat.c,v 1.50 2020/05/30 14:16:56 rmind Exp $");
+__KERNEL_RCSID(0, "$NetBSD: npf_nat.c,v 1.51 2022/03/12 15:32:32 riastradh Exp $");
#include <sys/param.h>
#include <sys/types.h>
@@ -279,9 +279,15 @@ npf_natpolicy_release(npf_natpolicy_t *n
{
KASSERT(atomic_load_relaxed(&np->n_refcnt) > 0);
+#ifndef __HAVE_ATOMIC_AS_MEMBAR
+ membar_exit();
+#endif
if (atomic_dec_uint_nv(&np->n_refcnt) != 0) {
return;
}
+#ifndef __HAVE_ATOMIC_AS_MEMBAR
+ membar_enter();
+#endif
KASSERT(LIST_EMPTY(&np->n_nat_list));
mutex_destroy(&np->n_lock);
kmem_free(np, sizeof(npf_natpolicy_t));
Index: src/sys/net/npf/npf_rproc.c
diff -u src/sys/net/npf/npf_rproc.c:1.20 src/sys/net/npf/npf_rproc.c:1.21
--- src/sys/net/npf/npf_rproc.c:1.20 Sat May 30 14:16:56 2020
+++ src/sys/net/npf/npf_rproc.c Sat Mar 12 15:32:32 2022
@@ -33,7 +33,7 @@
#ifdef _KERNEL
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: npf_rproc.c,v 1.20 2020/05/30 14:16:56 rmind Exp $");
+__KERNEL_RCSID(0, "$NetBSD: npf_rproc.c,v 1.21 2022/03/12 15:32:32 riastradh Exp $");
#include <sys/param.h>
#include <sys/types.h>
@@ -330,9 +330,15 @@ npf_rproc_release(npf_rproc_t *rp)
{
KASSERT(atomic_load_relaxed(&rp->rp_refcnt) > 0);
+#ifndef __HAVE_ATOMIC_AS_MEMBAR
+ membar_exit();
+#endif
if (atomic_dec_uint_nv(&rp->rp_refcnt) != 0) {
return;
}
+#ifndef __HAVE_ATOMIC_AS_MEMBAR
+ membar_enter();
+#endif
/* XXXintr */
for (unsigned i = 0; i < rp->rp_ext_count; i++) {
npf_ext_t *ext = rp->rp_ext[i];
Index: src/sys/net/npf/npf_tableset.c
diff -u src/sys/net/npf/npf_tableset.c:1.36 src/sys/net/npf/npf_tableset.c:1.37
--- src/sys/net/npf/npf_tableset.c:1.36 Mon Jan 25 17:18:55 2021
+++ src/sys/net/npf/npf_tableset.c Sat Mar 12 15:32:32 2022
@@ -46,7 +46,7 @@
#ifdef _KERNEL
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: npf_tableset.c,v 1.36 2021/01/25 17:18:55 christos Exp $");
+__KERNEL_RCSID(0, "$NetBSD: npf_tableset.c,v 1.37 2022/03/12 15:32:32 riastradh Exp $");
#include <sys/param.h>
#include <sys/types.h>
@@ -158,9 +158,13 @@ npf_tableset_destroy(npf_tableset_t *ts)
for (u_int tid = 0; tid < ts->ts_nitems; tid++) {
npf_table_t *t = ts->ts_map[tid];
- if (t && atomic_dec_uint_nv(&t->t_refcnt) == 0) {
- npf_table_destroy(t);
- }
+ if (t == NULL)
+ continue;
+ membar_exit();
+ if (atomic_dec_uint_nv(&t->t_refcnt) > 0)
+ continue;
+ membar_enter();
+ npf_table_destroy(t);
}
kmem_free(ts, NPF_TABLESET_SIZE(ts->ts_nitems));
}
Index: src/sys/uvm/uvm_aobj.c
diff -u src/sys/uvm/uvm_aobj.c:1.153 src/sys/uvm/uvm_aobj.c:1.154
--- src/sys/uvm/uvm_aobj.c:1.153 Sat Mar 13 15:29:55 2021
+++ src/sys/uvm/uvm_aobj.c Sat Mar 12 15:32:32 2022
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_aobj.c,v 1.153 2021/03/13 15:29:55 skrll Exp $ */
+/* $NetBSD: uvm_aobj.c,v 1.154 2022/03/12 15:32:32 riastradh Exp $ */
/*
* Copyright (c) 1998 Chuck Silvers, Charles D. Cranor and
@@ -38,7 +38,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_aobj.c,v 1.153 2021/03/13 15:29:55 skrll Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_aobj.c,v 1.154 2022/03/12 15:32:32 riastradh Exp $");
#ifdef _KERNEL_OPT
#include "opt_uvmhist.h"
@@ -604,10 +604,16 @@ uao_detach(struct uvm_object *uobj)
KASSERT(uobj->uo_refs > 0);
UVMHIST_LOG(maphist," (uobj=%#jx) ref=%jd",
(uintptr_t)uobj, uobj->uo_refs, 0, 0);
+#ifndef __HAVE_ATOMIC_AS_MEMBAR
+ membar_exit();
+#endif
if (atomic_dec_uint_nv(&uobj->uo_refs) > 0) {
UVMHIST_LOG(maphist, "<- done (rc>0)", 0,0,0,0);
return;
}
+#ifndef __HAVE_ATOMIC_AS_MEMBAR
+ membar_enter();
+#endif
/*
* Remove the aobj from the global list.
Index: src/sys/uvm/uvm_map.c
diff -u src/sys/uvm/uvm_map.c:1.391 src/sys/uvm/uvm_map.c:1.392
--- src/sys/uvm/uvm_map.c:1.391 Thu Nov 25 09:40:45 2021
+++ src/sys/uvm/uvm_map.c Sat Mar 12 15:32:33 2022
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_map.c,v 1.391 2021/11/25 09:40:45 skrll Exp $ */
+/* $NetBSD: uvm_map.c,v 1.392 2022/03/12 15:32:33 riastradh Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -66,7 +66,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_map.c,v 1.391 2021/11/25 09:40:45 skrll Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_map.c,v 1.392 2022/03/12 15:32:33 riastradh Exp $");
#include "opt_ddb.h"
#include "opt_pax.h"
@@ -4234,8 +4234,11 @@ uvmspace_free(struct vmspace *vm)
UVMHIST_FUNC(__func__);
UVMHIST_CALLARGS(maphist,"(vm=%#jx) ref=%jd", (uintptr_t)vm,
vm->vm_refcnt, 0, 0);
+
+ membar_exit();
if (atomic_dec_uint_nv(&vm->vm_refcnt) > 0)
return;
+ membar_enter();
/*
* at this point, there should be no other references to the map.
Index: src/sys/uvm/pmap/pmap.c
diff -u src/sys/uvm/pmap/pmap.c:1.62 src/sys/uvm/pmap/pmap.c:1.63
--- src/sys/uvm/pmap/pmap.c:1.62 Sat Apr 17 01:53:58 2021
+++ src/sys/uvm/pmap/pmap.c Sat Mar 12 15:32:32 2022
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.c,v 1.62 2021/04/17 01:53:58 mrg Exp $ */
+/* $NetBSD: pmap.c,v 1.63 2022/03/12 15:32:32 riastradh Exp $ */
/*-
* Copyright (c) 1998, 2001 The NetBSD Foundation, Inc.
@@ -67,7 +67,7 @@
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.62 2021/04/17 01:53:58 mrg Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.63 2022/03/12 15:32:32 riastradh Exp $");
/*
* Manages physical address maps.
@@ -690,11 +690,13 @@ pmap_destroy(pmap_t pmap)
UVMHIST_FUNC(__func__);
UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx)", (uintptr_t)pmap, 0, 0, 0);
+ membar_exit();
if (atomic_dec_uint_nv(&pmap->pm_count) > 0) {
PMAP_COUNT(dereference);
UVMHIST_LOG(pmaphist, " <-- done (deref)", 0, 0, 0, 0);
return;
}
+ membar_enter();
PMAP_COUNT(destroy);
KASSERT(pmap->pm_count == 0);
