Module Name: src Committed By: ad Date: Thu Nov 21 17:57:40 UTC 2019
Modified Files: src/sys/kern: kern_timeout.c Log Message: Break the slow path for callout_halt() out into its own routine. No functional change. To generate a diff of this commit: cvs rdiff -u -r1.56 -r1.57 src/sys/kern/kern_timeout.c Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files.
Modified files: Index: src/sys/kern/kern_timeout.c diff -u src/sys/kern/kern_timeout.c:1.56 src/sys/kern/kern_timeout.c:1.57 --- src/sys/kern/kern_timeout.c:1.56 Sun Mar 10 13:44:49 2019 +++ src/sys/kern/kern_timeout.c Thu Nov 21 17:57:40 2019 @@ -1,7 +1,7 @@ -/* $NetBSD: kern_timeout.c,v 1.56 2019/03/10 13:44:49 kre Exp $ */ +/* $NetBSD: kern_timeout.c,v 1.57 2019/11/21 17:57:40 ad Exp $ */ /*- - * Copyright (c) 2003, 2006, 2007, 2008, 2009 The NetBSD Foundation, Inc. + * Copyright (c) 2003, 2006, 2007, 2008, 2009, 2019 The NetBSD Foundation, Inc. * All rights reserved. * * This code is derived from software contributed to The NetBSD Foundation @@ -59,7 +59,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: kern_timeout.c,v 1.56 2019/03/10 13:44:49 kre Exp $"); +__KERNEL_RCSID(0, "$NetBSD: kern_timeout.c,v 1.57 2019/11/21 17:57:40 ad Exp $"); /* * Timeouts are kept in a hierarchical timing wheel. The c_time is the @@ -185,8 +185,10 @@ struct callout_cpu { #ifndef CRASH static void callout_softclock(void *); -static struct callout_cpu callout_cpu0; -static void *callout_sih; +static void callout_wait(callout_impl_t *, void *, kmutex_t *); + +static struct callout_cpu callout_cpu0 __cacheline_aligned; +static void *callout_sih __read_mostly; static inline kmutex_t * callout_lock(callout_impl_t *c) @@ -466,24 +468,42 @@ bool callout_halt(callout_t *cs, void *interlock) { callout_impl_t *c = (callout_impl_t *)cs; - struct callout_cpu *cc; - struct lwp *l; - kmutex_t *lock, *relock; - bool expired; + kmutex_t *lock; + int flags; KASSERT(c->c_magic == CALLOUT_MAGIC); KASSERT(!cpu_intr_p()); KASSERT(interlock == NULL || mutex_owned(interlock)); + /* Fast path. */ lock = callout_lock(c); - relock = NULL; - - expired = ((c->c_flags & CALLOUT_FIRED) != 0); - if ((c->c_flags & CALLOUT_PENDING) != 0) + flags = c->c_flags; + if ((flags & CALLOUT_PENDING) != 0) CIRCQ_REMOVE(&c->c_list); - c->c_flags &= ~(CALLOUT_PENDING|CALLOUT_FIRED); + c->c_flags = flags & ~(CALLOUT_PENDING|CALLOUT_FIRED); + if (__predict_false(flags & CALLOUT_FIRED)) { + callout_wait(c, interlock, lock); + return true; + } + mutex_spin_exit(lock); + return false; +} + +/* + * callout_wait: + * + * Slow path for callout_halt(). Deliberately marked __noinline to + * prevent unneeded overhead in the caller. + */ +static void __noinline +callout_wait(callout_impl_t *c, void *interlock, kmutex_t *lock) +{ + struct callout_cpu *cc; + struct lwp *l; + kmutex_t *relock; l = curlwp; + relock = NULL; for (;;) { cc = c->c_cpu; if (__predict_true(cc->cc_active != c || cc->cc_lwp == l)) @@ -515,8 +535,6 @@ callout_halt(callout_t *cs, void *interl mutex_spin_exit(lock); if (__predict_false(relock != NULL)) mutex_enter(relock); - - return expired; } #ifdef notyet