Author: mjg
Date: Fri Nov 17 02:21:07 2017
New Revision: 325917
URL: https://svnweb.freebsd.org/changeset/base/325917
Log:
sx: avoid branches if in the slow path if lockstat is disabled
Modified:
head/sys/kern/kern_sx.c
Modified: head/sys/kern/kern_sx.c
==============================================================================
--- head/sys/kern/kern_sx.c Fri Nov 17 00:38:00 2017 (r325916)
+++ head/sys/kern/kern_sx.c Fri Nov 17 02:21:07 2017 (r325917)
@@ -88,8 +88,9 @@ PMC_SOFT_DECLARE( , , lock, failed);
int _giantcnt = 0; \
WITNESS_SAVE_DECL(Giant) \
-#define GIANT_SAVE() do {
\
+#define GIANT_SAVE(work) do {
\
if (mtx_owned(&Giant)) { \
+ work++; \
WITNESS_SAVE(&Giant.lock_object, Giant); \
while (mtx_owned(&Giant)) { \
_giantcnt++; \
@@ -513,11 +514,14 @@ _sx_xlock_hard(struct sx *sx, uintptr_t x, uintptr_t t
struct lock_delay_arg lda;
#endif
#ifdef KDTRACE_HOOKS
- uintptr_t state;
u_int sleep_cnt = 0;
int64_t sleep_time = 0;
int64_t all_time = 0;
#endif
+#if defined(KDTRACE_HOOKS) || defined(LOCK_PROFILING)
+ uintptr_t state;
+ int extra_work;
+#endif
if (SCHEDULER_STOPPED())
return (0);
@@ -547,10 +551,17 @@ _sx_xlock_hard(struct sx *sx, uintptr_t x, uintptr_t t
CTR5(KTR_LOCK, "%s: %s contested (lock=%p) at %s:%d", __func__,
sx->lock_object.lo_name, (void *)sx->sx_lock, file, line);
-#ifdef KDTRACE_HOOKS
- all_time -= lockstat_nsecs(&sx->lock_object);
+#ifdef LOCK_PROFILING
+ extra_work = 1;
state = x;
+#elif defined(KDTRACE_HOOKS)
+ extra_work = lockstat_enabled;
+ if (__predict_false(extra_work)) {
+ all_time -= lockstat_nsecs(&sx->lock_object);
+ state = x;
+ }
#endif
+
for (;;) {
if (x == SX_LOCK_UNLOCKED) {
if (atomic_fcmpset_acq_ptr(&sx->sx_lock, &x, tid))
@@ -583,7 +594,7 @@ _sx_xlock_hard(struct sx *sx, uintptr_t x, uintptr_t t
sched_tdname(curthread), "spinning",
"lockname:\"%s\"",
sx->lock_object.lo_name);
- GIANT_SAVE();
+ GIANT_SAVE(extra_work);
do {
lock_delay(&lda);
x = SX_READ_VALUE(sx);
@@ -598,7 +609,7 @@ _sx_xlock_hard(struct sx *sx, uintptr_t x, uintptr_t t
KTR_STATE1(KTR_SCHED, "thread",
sched_tdname(curthread), "spinning",
"lockname:\"%s\"", sx->lock_object.lo_name);
- GIANT_SAVE();
+ GIANT_SAVE(extra_work);
spintries++;
for (i = 0; i < asx_loops; i += n) {
if (LOCK_LOG_TEST(&sx->lock_object, 0))
@@ -705,7 +716,7 @@ _sx_xlock_hard(struct sx *sx, uintptr_t x, uintptr_t t
#ifdef KDTRACE_HOOKS
sleep_time -= lockstat_nsecs(&sx->lock_object);
#endif
- GIANT_SAVE();
+ GIANT_SAVE(extra_work);
sleepq_add(&sx->lock_object, NULL, sx->lock_object.lo_name,
SLEEPQ_SX | ((opts & SX_INTERRUPTIBLE) ?
SLEEPQ_INTERRUPTIBLE : 0), SQ_EXCLUSIVE_QUEUE);
@@ -729,6 +740,10 @@ _sx_xlock_hard(struct sx *sx, uintptr_t x, uintptr_t t
__func__, sx);
x = SX_READ_VALUE(sx);
}
+#if defined(KDTRACE_HOOKS) || defined(LOCK_PROFILING)
+ if (__predict_true(!extra_work))
+ return (error);
+#endif
#ifdef KDTRACE_HOOKS
all_time += lockstat_nsecs(&sx->lock_object);
if (sleep_time)
@@ -856,11 +871,14 @@ _sx_slock_hard(struct sx *sx, int opts, const char *fi
struct lock_delay_arg lda;
#endif
#ifdef KDTRACE_HOOKS
- uintptr_t state;
u_int sleep_cnt = 0;
int64_t sleep_time = 0;
int64_t all_time = 0;
#endif
+#if defined(KDTRACE_HOOKS) || defined(LOCK_PROFILING)
+ uintptr_t state;
+ int extra_work;
+#endif
if (SCHEDULER_STOPPED())
return (0);
@@ -870,9 +888,16 @@ _sx_slock_hard(struct sx *sx, int opts, const char *fi
#elif defined(KDTRACE_HOOKS)
lock_delay_arg_init(&lda, NULL);
#endif
-#ifdef KDTRACE_HOOKS
- all_time -= lockstat_nsecs(&sx->lock_object);
+
+#ifdef LOCK_PROFILING
+ extra_work = 1;
state = x;
+#elif defined(KDTRACE_HOOKS)
+ extra_work = lockstat_enabled;
+ if (__predict_false(extra_work)) {
+ all_time -= lockstat_nsecs(&sx->lock_object);
+ state = x;
+ }
#endif
/*
@@ -908,7 +933,7 @@ _sx_slock_hard(struct sx *sx, int opts, const char *fi
KTR_STATE1(KTR_SCHED, "thread",
sched_tdname(curthread), "spinning",
"lockname:\"%s\"", sx->lock_object.lo_name);
- GIANT_SAVE();
+ GIANT_SAVE(extra_work);
do {
lock_delay(&lda);
x = SX_READ_VALUE(sx);
@@ -982,7 +1007,7 @@ _sx_slock_hard(struct sx *sx, int opts, const char *fi
#ifdef KDTRACE_HOOKS
sleep_time -= lockstat_nsecs(&sx->lock_object);
#endif
- GIANT_SAVE();
+ GIANT_SAVE(extra_work);
sleepq_add(&sx->lock_object, NULL, sx->lock_object.lo_name,
SLEEPQ_SX | ((opts & SX_INTERRUPTIBLE) ?
SLEEPQ_INTERRUPTIBLE : 0), SQ_SHARED_QUEUE);
@@ -1006,6 +1031,10 @@ _sx_slock_hard(struct sx *sx, int opts, const char *fi
__func__, sx);
x = SX_READ_VALUE(sx);
}
+#if defined(KDTRACE_HOOKS) || defined(LOCK_PROFILING)
+ if (__predict_true(!extra_work))
+ return (error);
+#endif
#ifdef KDTRACE_HOOKS
all_time += lockstat_nsecs(&sx->lock_object);
if (sleep_time)
_______________________________________________
[email protected] mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-head
To unsubscribe, send any mail to "[email protected]"
