Module Name:    src
Committed By:   ad
Date:           Thu Nov 21 17:57:40 UTC 2019

Modified Files:
        src/sys/kern: kern_timeout.c

Log Message:
Break the slow path for callout_halt() out into its own routine.
No functional change.


To generate a diff of this commit:
cvs rdiff -u -r1.56 -r1.57 src/sys/kern/kern_timeout.c

Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.

Modified files:

Index: src/sys/kern/kern_timeout.c
diff -u src/sys/kern/kern_timeout.c:1.56 src/sys/kern/kern_timeout.c:1.57
--- src/sys/kern/kern_timeout.c:1.56	Sun Mar 10 13:44:49 2019
+++ src/sys/kern/kern_timeout.c	Thu Nov 21 17:57:40 2019
@@ -1,7 +1,7 @@
-/*	$NetBSD: kern_timeout.c,v 1.56 2019/03/10 13:44:49 kre Exp $	*/
+/*	$NetBSD: kern_timeout.c,v 1.57 2019/11/21 17:57:40 ad Exp $	*/
 
 /*-
- * Copyright (c) 2003, 2006, 2007, 2008, 2009 The NetBSD Foundation, Inc.
+ * Copyright (c) 2003, 2006, 2007, 2008, 2009, 2019 The NetBSD Foundation, Inc.
  * All rights reserved.
  *
  * This code is derived from software contributed to The NetBSD Foundation
@@ -59,7 +59,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_timeout.c,v 1.56 2019/03/10 13:44:49 kre Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_timeout.c,v 1.57 2019/11/21 17:57:40 ad Exp $");
 
 /*
  * Timeouts are kept in a hierarchical timing wheel.  The c_time is the
@@ -185,8 +185,10 @@ struct callout_cpu {
 #ifndef CRASH
 
 static void	callout_softclock(void *);
-static struct callout_cpu callout_cpu0;
-static void *callout_sih;
+static void	callout_wait(callout_impl_t *, void *, kmutex_t *);
+
+static struct callout_cpu callout_cpu0 __cacheline_aligned;
+static void *callout_sih __read_mostly;
 
 static inline kmutex_t *
 callout_lock(callout_impl_t *c)
@@ -466,24 +468,42 @@ bool
 callout_halt(callout_t *cs, void *interlock)
 {
 	callout_impl_t *c = (callout_impl_t *)cs;
-	struct callout_cpu *cc;
-	struct lwp *l;
-	kmutex_t *lock, *relock;
-	bool expired;
+	kmutex_t *lock;
+	int flags;
 
 	KASSERT(c->c_magic == CALLOUT_MAGIC);
 	KASSERT(!cpu_intr_p());
 	KASSERT(interlock == NULL || mutex_owned(interlock));
 
+	/* Fast path. */
 	lock = callout_lock(c);
-	relock = NULL;
-
-	expired = ((c->c_flags & CALLOUT_FIRED) != 0);
-	if ((c->c_flags & CALLOUT_PENDING) != 0)
+	flags = c->c_flags;
+	if ((flags & CALLOUT_PENDING) != 0)
 		CIRCQ_REMOVE(&c->c_list);
-	c->c_flags &= ~(CALLOUT_PENDING|CALLOUT_FIRED);
+	c->c_flags = flags & ~(CALLOUT_PENDING|CALLOUT_FIRED);
+	if (__predict_false(flags & CALLOUT_FIRED)) {
+		callout_wait(c, interlock, lock);
+		return true;
+	}
+	mutex_spin_exit(lock);
+	return false;
+}
+
+/*
+ * callout_wait:
+ *
+ *	Slow path for callout_halt().  Deliberately marked __noinline to
+ *	prevent unneeded overhead in the caller.
+ */
+static void __noinline
+callout_wait(callout_impl_t *c, void *interlock, kmutex_t *lock)
+{
+	struct callout_cpu *cc;
+	struct lwp *l;
+	kmutex_t *relock;
 
 	l = curlwp;
+	relock = NULL;
 	for (;;) {
 		cc = c->c_cpu;
 		if (__predict_true(cc->cc_active != c || cc->cc_lwp == l))
@@ -515,8 +535,6 @@ callout_halt(callout_t *cs, void *interl
 	mutex_spin_exit(lock);
 	if (__predict_false(relock != NULL))
 		mutex_enter(relock);
-
-	return expired;
 }
 
 #ifdef notyet

Reply via email to