Author: mjg
Date: Sun Feb 5 08:04:11 2017
New Revision: 313275
URL: https://svnweb.freebsd.org/changeset/base/313275
Log:
mtx: move lockstat handling out of inline primitives
Lockstat requires checking if it is enabled and if so, calling a 6 argument
function. Further, determining whether to call it on unlock requires
pre-reading the lock value.
This is problematic in at least 3 ways:
- more branches in the hot path than necessary
- additional cacheline ping pong under contention
- bigger code
Instead, check first if lockstat handling is necessary and if so, just fall
back to regular locking routines. For this purpose a new macro is introduced
(LOCKSTAT_PROFILE_ENABLED).
LOCK_PROFILING uninlines all primitives. Fold in the current inline lock
variant into the _mtx_lock_flags to retain the support. With this change
the inline variants are not used when LOCK_PROFILING is defined and thus
can ignore its existence.
This results in:
text data bss dec hex filename
22259667 1303208 4994976 28557851 1b3c21b kernel.orig
21797315 1303208 4994976 28095499 1acb40b kernel.patched
i.e. about 3% reduction in text size.
A remaining action is to remove spurious arguments for internal kernel
consumers.
Modified:
head/sys/kern/kern_mutex.c
head/sys/sys/lockstat.h
head/sys/sys/mutex.h
head/sys/sys/sdt.h
Modified: head/sys/kern/kern_mutex.c
==============================================================================
--- head/sys/kern/kern_mutex.c Sun Feb 5 06:51:45 2017 (r313274)
+++ head/sys/kern/kern_mutex.c Sun Feb 5 08:04:11 2017 (r313275)
@@ -265,6 +265,7 @@ void
__mtx_lock_flags(volatile uintptr_t *c, int opts, const char *file, int line)
{
struct mtx *m;
+ uintptr_t tid, v;
if (SCHEDULER_STOPPED())
return;
@@ -282,7 +283,13 @@ __mtx_lock_flags(volatile uintptr_t *c,
WITNESS_CHECKORDER(&m->lock_object, (opts & ~MTX_RECURSE) |
LOP_NEWORDER | LOP_EXCLUSIVE, file, line, NULL);
- __mtx_lock(m, curthread, opts, file, line);
+ tid = (uintptr_t)curthread;
+ v = MTX_UNOWNED;
+ if (!_mtx_obtain_lock_fetch(m, &v, tid))
+ _mtx_lock_sleep(m, v, tid, opts, file, line);
+ else
+ LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(adaptive__acquire,
+ m, 0, 0, file, line);
LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file,
line);
WITNESS_LOCK(&m->lock_object, (opts & ~MTX_RECURSE) | LOP_EXCLUSIVE,
@@ -310,7 +317,7 @@ __mtx_unlock_flags(volatile uintptr_t *c
line);
mtx_assert(m, MA_OWNED);
- __mtx_unlock(m, curthread, opts, file, line);
+ __mtx_unlock_sleep(c, opts, file, line);
TD_LOCKS_DEC(curthread);
}
@@ -887,20 +894,17 @@ __mtx_unlock_sleep(volatile uintptr_t *c
{
struct mtx *m;
struct turnstile *ts;
- uintptr_t v;
if (SCHEDULER_STOPPED())
return;
m = mtxlock2mtx(c);
- v = MTX_READ_VALUE(m);
- if (v == (uintptr_t)curthread) {
+ if (!mtx_recursed(m)) {
+ LOCKSTAT_PROFILE_RELEASE_LOCK(adaptive__release, m);
if (_mtx_release_lock(m, (uintptr_t)curthread))
return;
- }
-
- if (mtx_recursed(m)) {
+ } else {
if (--(m->mtx_recurse) == 0)
atomic_clear_ptr(&m->mtx_lock, MTX_RECURSED);
if (LOCK_LOG_TEST(&m->lock_object, opts))
Modified: head/sys/sys/lockstat.h
==============================================================================
--- head/sys/sys/lockstat.h Sun Feb 5 06:51:45 2017 (r313274)
+++ head/sys/sys/lockstat.h Sun Feb 5 08:04:11 2017 (r313275)
@@ -107,6 +107,10 @@ extern int lockstat_enabled;
LOCKSTAT_RECORD1(probe, lp, a); \
} while (0)
+#ifndef LOCK_PROFILING
+#define LOCKSTAT_PROFILE_ENABLED(probe) SDT_PROBE_ENABLED(lockstat, , ,
probe)
+#endif
+
struct lock_object;
uint64_t lockstat_nsecs(struct lock_object *);
@@ -130,6 +134,10 @@ uint64_t lockstat_nsecs(struct lock_obje
#define LOCKSTAT_PROFILE_RELEASE_RWLOCK(probe, lp, a)
\
LOCKSTAT_PROFILE_RELEASE_LOCK(probe, lp)
+#ifndef LOCK_PROFILING
+#define LOCKSTAT_PROFILE_ENABLED(probe) 0
+#endif
+
#endif /* !KDTRACE_HOOKS */
#endif /* _KERNEL */
#endif /* _SYS_LOCKSTAT_H */
Modified: head/sys/sys/mutex.h
==============================================================================
--- head/sys/sys/mutex.h Sun Feb 5 06:51:45 2017 (r313274)
+++ head/sys/sys/mutex.h Sun Feb 5 08:04:11 2017 (r313275)
@@ -171,10 +171,8 @@ void thread_lock_flags_(struct thread *,
#define _mtx_obtain_lock(mp, tid) \
atomic_cmpset_acq_ptr(&(mp)->mtx_lock, MTX_UNOWNED, (tid))
-#define _mtx_obtain_lock_fetch(mp, vp, tid) ({ \
- *vp = MTX_UNOWNED; \
- atomic_fcmpset_rel_ptr(&(mp)->mtx_lock, vp, (tid)); \
-})
+#define _mtx_obtain_lock_fetch(mp, vp, tid) \
+ atomic_fcmpset_rel_ptr(&(mp)->mtx_lock, vp, (tid))
/* Try to release mtx_lock if it is unrecursed and uncontested. */
#define _mtx_release_lock(mp, tid) \
@@ -193,13 +191,11 @@ void thread_lock_flags_(struct thread *,
/* Lock a normal mutex. */
#define __mtx_lock(mp, tid, opts, file, line) do { \
uintptr_t _tid = (uintptr_t)(tid); \
- uintptr_t _v; \
+ uintptr_t _v = MTX_UNOWNED; \
\
- if (!_mtx_obtain_lock_fetch((mp), &_v, _tid)) \
+ if (__predict_false(LOCKSTAT_PROFILE_ENABLED(adaptive__acquire) ||\
+ !_mtx_obtain_lock_fetch((mp), &_v, _tid))) \
_mtx_lock_sleep((mp), _v, _tid, (opts), (file), (line));\
- else \
- LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(adaptive__acquire, \
- mp, 0, 0, file, line); \
} while (0)
/*
@@ -211,7 +207,7 @@ void thread_lock_flags_(struct thread *,
#ifdef SMP
#define __mtx_lock_spin(mp, tid, opts, file, line) do {
\
uintptr_t _tid = (uintptr_t)(tid); \
- uintptr_t _v; \
+ uintptr_t _v = MTX_UNOWNED; \
\
spinlock_enter(); \
if (!_mtx_obtain_lock_fetch((mp), &_v, _tid)) { \
@@ -270,9 +266,8 @@ void thread_lock_flags_(struct thread *,
#define __mtx_unlock(mp, tid, opts, file, line) do { \
uintptr_t _tid = (uintptr_t)(tid); \
\
- if ((mp)->mtx_recurse == 0) \
- LOCKSTAT_PROFILE_RELEASE_LOCK(adaptive__release, mp); \
- if (!_mtx_release_lock((mp), _tid)) \
+ if (__predict_false(LOCKSTAT_PROFILE_ENABLED(adaptive__release) ||\
+ !_mtx_release_lock((mp), _tid))) \
_mtx_unlock_sleep((mp), (opts), (file), (line)); \
} while (0)
Modified: head/sys/sys/sdt.h
==============================================================================
--- head/sys/sys/sdt.h Sun Feb 5 06:51:45 2017 (r313274)
+++ head/sys/sys/sdt.h Sun Feb 5 08:04:11 2017 (r313275)
@@ -160,6 +160,9 @@ SET_DECLARE(sdt_argtypes_set, struct sdt
#define SDT_PROBE_DECLARE(prov, mod, func, name)
\
extern struct sdt_probe sdt_##prov##_##mod##_##func##_##name[1]
+#define SDT_PROBE_ENABLED(prov, mod, func, name)
\
+ __predict_false((sdt_##prov##_##mod##_##func##_##name->id))
+
#define SDT_PROBE(prov, mod, func, name, arg0, arg1, arg2, arg3, arg4) do {
\
if (__predict_false(sdt_##prov##_##mod##_##func##_##name->id))
\
(*sdt_probe_func)(sdt_##prov##_##mod##_##func##_##name->id,
\
_______________________________________________
svn-src-all@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-all
To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
