Module Name: src
Committed By: pooka
Date: Tue May 18 14:58:42 UTC 2010
Modified Files:
src/lib/librumpuser: rumpuser_int.h rumpuser_pth.c
src/sys/rump/include/rump: rumpuser.h
src/sys/rump/librump/rumpkern: intr.c locks.c rump_private.h
scheduler.c
Log Message:
Make it possible to use the scheduler lock as the rumpuser condvar
interlock. This is applicable in cases where the actual interlock
is the CPU the currently running thread is scheduled on. Borrowing
the scheduler lock as the mutex mandated by pthread_cond_wait()
does away with need to have an additional mutex. This both optimizes
runtime execution and simplifies code, as the extra lock typically
lead to quite some trickeries to avoid the dungeon collapsing due
to zaps from the wand of deadlock.
To generate a diff of this commit:
cvs rdiff -u -r1.2 -r1.3 src/lib/librumpuser/rumpuser_int.h
cvs rdiff -u -r1.1 -r1.2 src/lib/librumpuser/rumpuser_pth.c
cvs rdiff -u -r1.40 -r1.41 src/sys/rump/include/rump/rumpuser.h
cvs rdiff -u -r1.27 -r1.28 src/sys/rump/librump/rumpkern/intr.c
cvs rdiff -u -r1.39 -r1.40 src/sys/rump/librump/rumpkern/locks.c
cvs rdiff -u -r1.44 -r1.45 src/sys/rump/librump/rumpkern/rump_private.h
cvs rdiff -u -r1.13 -r1.14 src/sys/rump/librump/rumpkern/scheduler.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/lib/librumpuser/rumpuser_int.h
diff -u src/lib/librumpuser/rumpuser_int.h:1.2 src/lib/librumpuser/rumpuser_int.h:1.3
--- src/lib/librumpuser/rumpuser_int.h:1.2 Mon Mar 22 09:39:02 2010
+++ src/lib/librumpuser/rumpuser_int.h Tue May 18 14:58:41 2010
@@ -1,4 +1,4 @@
-/* $NetBSD: rumpuser_int.h,v 1.2 2010/03/22 09:39:02 pooka Exp $ */
+/* $NetBSD: rumpuser_int.h,v 1.3 2010/05/18 14:58:41 pooka Exp $ */
/*
* Copyright (c) 2008 Antti Kantee. All Rights Reserved.
@@ -25,6 +25,8 @@
* SUCH DAMAGE.
*/
+#include <stdlib.h>
+
#include <rump/rumpuser.h>
extern kernel_lockfn rumpuser__klock;
@@ -34,9 +36,9 @@
#define KLOCK_WRAP(a) \
do { \
int nlocks; \
- rumpuser__kunlock(0, &nlocks); \
+ rumpuser__kunlock(0, &nlocks, NULL); \
a; \
- rumpuser__klock(nlocks); \
+ rumpuser__klock(nlocks, NULL); \
} while (/*CONSTCOND*/0)
#define DOCALL(rvtype, call) \
@@ -54,9 +56,9 @@
{ \
rvtype rv; \
int nlocks; \
- rumpuser__kunlock(0, &nlocks); \
+ rumpuser__kunlock(0, &nlocks, NULL); \
rv = call; \
- rumpuser__klock(nlocks); \
+ rumpuser__klock(nlocks, NULL); \
if (rv == -1) \
*error = errno; \
else \
Index: src/lib/librumpuser/rumpuser_pth.c
diff -u src/lib/librumpuser/rumpuser_pth.c:1.1 src/lib/librumpuser/rumpuser_pth.c:1.2
--- src/lib/librumpuser/rumpuser_pth.c:1.1 Fri Feb 26 18:54:20 2010
+++ src/lib/librumpuser/rumpuser_pth.c Tue May 18 14:58:41 2010
@@ -1,4 +1,4 @@
-/* $NetBSD: rumpuser_pth.c,v 1.1 2010/02/26 18:54:20 pooka Exp $ */
+/* $NetBSD: rumpuser_pth.c,v 1.2 2010/05/18 14:58:41 pooka Exp $ */
/*
* Copyright (c) 2007-2010 Antti Kantee. All Rights Reserved.
@@ -27,7 +27,7 @@
#include <sys/cdefs.h>
#if !defined(lint)
-__RCSID("$NetBSD: rumpuser_pth.c,v 1.1 2010/02/26 18:54:20 pooka Exp $");
+__RCSID("$NetBSD: rumpuser_pth.c,v 1.2 2010/05/18 14:58:41 pooka Exp $");
#endif /* !lint */
#ifdef __linux__
@@ -131,7 +131,7 @@
int error, dummy;
/* unschedule from CPU. we reschedule before running the interrupt */
- rumpuser__kunlock(0, &dummy);
+ rumpuser__kunlock(0, &dummy, NULL);
assert(dummy == 0);
NOFAIL_ERRNO(pthread_mutex_lock(&rumpuser_aio_mtx.pthmtx));
@@ -169,9 +169,9 @@
#endif
}
}
- rumpuser__klock(0);
+ rumpuser__klock(0, NULL);
biodone(rua->rua_bp, (size_t)rv, error);
- rumpuser__kunlock(0, &dummy);
+ rumpuser__kunlock(0, &dummy, NULL);
rua->rua_bp = NULL;
@@ -443,12 +443,15 @@
void
rumpuser_cv_wait(struct rumpuser_cv *cv, struct rumpuser_mtx *mtx)
{
+ int nlocks;
cv->nwaiters++;
+ rumpuser__kunlock(0, &nlocks, mtx);
assert(mtx->recursion == 1);
mtxexit(mtx);
- KLOCK_WRAP(NOFAIL_ERRNO(pthread_cond_wait(&cv->pthcv, &mtx->pthmtx)));
+ NOFAIL_ERRNO(pthread_cond_wait(&cv->pthcv, &mtx->pthmtx));
mtxenter(mtx);
+ rumpuser__klock(nlocks, mtx);
cv->nwaiters--;
}
@@ -469,15 +472,17 @@
int64_t sec, int64_t nsec)
{
struct timespec ts;
- int rv;
+ int rv, nlocks;
/* LINTED */
ts.tv_sec = sec; ts.tv_nsec = nsec;
cv->nwaiters++;
+ rumpuser__kunlock(0, &nlocks, mtx);
mtxexit(mtx);
- KLOCK_WRAP(rv = pthread_cond_timedwait(&cv->pthcv, &mtx->pthmtx, &ts));
+ rv = pthread_cond_timedwait(&cv->pthcv, &mtx->pthmtx, &ts);
mtxenter(mtx);
+ rumpuser__klock(nlocks, mtx);
cv->nwaiters--;
if (rv != 0 && rv != ETIMEDOUT)
abort();
Index: src/sys/rump/include/rump/rumpuser.h
diff -u src/sys/rump/include/rump/rumpuser.h:1.40 src/sys/rump/include/rump/rumpuser.h:1.41
--- src/sys/rump/include/rump/rumpuser.h:1.40 Wed Apr 28 00:33:45 2010
+++ src/sys/rump/include/rump/rumpuser.h Tue May 18 14:58:41 2010
@@ -1,4 +1,4 @@
-/* $NetBSD: rumpuser.h,v 1.40 2010/04/28 00:33:45 pooka Exp $ */
+/* $NetBSD: rumpuser.h,v 1.41 2010/05/18 14:58:41 pooka Exp $ */
/*
* Copyright (c) 2007 Antti Kantee. All Rights Reserved.
@@ -40,8 +40,8 @@
struct pollfd;
struct sockaddr;
-typedef void (*kernel_lockfn)(int);
-typedef void (*kernel_unlockfn)(int, int *);
+typedef void (*kernel_lockfn)(int, void *);
+typedef void (*kernel_unlockfn)(int, int *, void *);
int rumpuser_getfileinfo(const char *, uint64_t *, int *, int *);
#define RUMPUSER_FT_OTHER 0
Index: src/sys/rump/librump/rumpkern/intr.c
diff -u src/sys/rump/librump/rumpkern/intr.c:1.27 src/sys/rump/librump/rumpkern/intr.c:1.28
--- src/sys/rump/librump/rumpkern/intr.c:1.27 Wed May 12 16:48:21 2010
+++ src/sys/rump/librump/rumpkern/intr.c Tue May 18 14:58:42 2010
@@ -1,4 +1,4 @@
-/* $NetBSD: intr.c,v 1.27 2010/05/12 16:48:21 pooka Exp $ */
+/* $NetBSD: intr.c,v 1.28 2010/05/18 14:58:42 pooka Exp $ */
/*
* Copyright (c) 2008 Antti Kantee. All Rights Reserved.
@@ -26,13 +26,14 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: intr.c,v 1.27 2010/05/12 16:48:21 pooka Exp $");
+__KERNEL_RCSID(0, "$NetBSD: intr.c,v 1.28 2010/05/18 14:58:42 pooka Exp $");
#include <sys/param.h>
#include <sys/cpu.h>
#include <sys/kernel.h>
#include <sys/kmem.h>
#include <sys/kthread.h>
+#include <sys/malloc.h>
#include <sys/intr.h>
#include <sys/timetc.h>
@@ -57,7 +58,6 @@
LIST_ENTRY(softint) si_entries;
};
-static struct rumpuser_mtx *si_mtx;
struct softint_lev {
struct rumpuser_cv *si_cv;
LIST_HEAD(, softint) si_pending;
@@ -140,9 +140,9 @@
}
/*
- * Soft interrupt execution thread. Note that we run without a CPU
- * context until we start processing the interrupt. This is to avoid
- * lock recursion.
+ * Soft interrupt execution thread. This thread is pinned to the
+ * same CPU that scheduled the interrupt, so we don't need to do
+ * lock against si_lvl.
*/
static void
sithread(void *arg)
@@ -155,16 +155,9 @@
struct softint_lev *si_lvlp, *si_lvl;
struct cpu_data *cd = &curcpu()->ci_data;
- rump_unschedule();
-
si_lvlp = cd->cpu_softcpu;
si_lvl = &si_lvlp[mylevel];
- /*
- * XXX: si_mtx is unnecessary, and should open an interface
- * which allows to use schedmtx for the cv wait
- */
- rumpuser_mutex_enter_nowrap(si_mtx);
for (;;) {
if (!LIST_EMPTY(&si_lvl->si_pending)) {
si = LIST_FIRST(&si_lvl->si_pending);
@@ -175,28 +168,19 @@
si->si_flags &= ~SI_ONLIST;
LIST_REMOVE(si, si_entries);
if (si->si_flags & SI_KILLME) {
- rumpuser_mutex_exit(si_mtx);
- rump_schedule();
softint_disestablish(si);
- rump_unschedule();
- rumpuser_mutex_enter_nowrap(si_mtx);
continue;
}
} else {
- rumpuser_cv_wait_nowrap(si_lvl->si_cv, si_mtx);
+ rump_schedlock_cv_wait(si_lvl->si_cv);
continue;
}
- rumpuser_mutex_exit(si_mtx);
- rump_schedule();
if (!mpsafe)
KERNEL_LOCK(1, curlwp);
func(funarg);
if (!mpsafe)
KERNEL_UNLOCK_ONE(curlwp);
- rump_unschedule();
-
- rumpuser_mutex_enter_nowrap(si_mtx);
}
panic("sithread unreachable");
@@ -206,7 +190,6 @@
rump_intr_init()
{
- rumpuser_mutex_init(&si_mtx);
cv_init(&lbolt, "oh kath ra");
}
@@ -233,6 +216,9 @@
}
cd->cpu_softcpu = slev;
+ /* softint might run on different physical CPU */
+ membar_sync();
+
for (i = 0; i < SOFTINT_COUNT; i++) {
rv = kthread_create(PRI_NONE,
KTHREAD_MPSAFE | KTHREAD_INTR, ci,
@@ -261,7 +247,7 @@
{
struct softint *si;
- si = kmem_alloc(sizeof(*si), KM_SLEEP);
+ si = malloc(sizeof(*si), M_TEMP, M_WAITOK);
si->si_func = func;
si->si_arg = arg;
si->si_flags = flags & SOFTINT_MPSAFE ? SI_MPSAFE : 0;
@@ -295,13 +281,11 @@
{
struct softint *si = cook;
- rumpuser_mutex_enter(si_mtx);
if (si->si_flags & SI_ONLIST) {
si->si_flags |= SI_KILLME;
return;
}
- rumpuser_mutex_exit(si_mtx);
- kmem_free(si, sizeof(*si));
+ free(si, M_TEMP);
}
void
Index: src/sys/rump/librump/rumpkern/locks.c
diff -u src/sys/rump/librump/rumpkern/locks.c:1.39 src/sys/rump/librump/rumpkern/locks.c:1.40
--- src/sys/rump/librump/rumpkern/locks.c:1.39 Wed Apr 14 10:34:54 2010
+++ src/sys/rump/librump/rumpkern/locks.c Tue May 18 14:58:42 2010
@@ -1,4 +1,4 @@
-/* $NetBSD: locks.c,v 1.39 2010/04/14 10:34:54 pooka Exp $ */
+/* $NetBSD: locks.c,v 1.40 2010/05/18 14:58:42 pooka Exp $ */
/*
* Copyright (c) 2007, 2008 Antti Kantee. All Rights Reserved.
@@ -29,7 +29,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: locks.c,v 1.39 2010/04/14 10:34:54 pooka Exp $");
+__KERNEL_RCSID(0, "$NetBSD: locks.c,v 1.40 2010/05/18 14:58:42 pooka Exp $");
#include <sys/param.h>
#include <sys/kmem.h>
@@ -332,7 +332,7 @@
if (!rumpuser_mutex_tryenter(rump_giantlock)) {
struct lwp *l = curlwp;
- rump_unschedule_cpu1(l);
+ rump_unschedule_cpu1(l, NULL);
rumpuser_mutex_enter_nowrap(rump_giantlock);
rump_schedule_cpu(l);
}
@@ -367,7 +367,7 @@
}
void
-rump_user_unschedule(int nlocks, int *countp)
+rump_user_unschedule(int nlocks, int *countp, void *interlock)
{
_kernel_unlock(nlocks, countp);
@@ -375,14 +375,14 @@
* XXX: technically we should unschedule_cpu1() here, but that
* requires rump_intr_enter/exit to be implemented.
*/
- rump_unschedule_cpu(curlwp);
+ rump_unschedule_cpu_interlock(curlwp, interlock);
}
void
-rump_user_schedule(int nlocks)
+rump_user_schedule(int nlocks, void *interlock)
{
- rump_schedule_cpu(curlwp);
+ rump_schedule_cpu_interlock(curlwp, interlock);
if (nlocks)
_kernel_lock(nlocks);
Index: src/sys/rump/librump/rumpkern/rump_private.h
diff -u src/sys/rump/librump/rumpkern/rump_private.h:1.44 src/sys/rump/librump/rumpkern/rump_private.h:1.45
--- src/sys/rump/librump/rumpkern/rump_private.h:1.44 Tue Apr 27 23:30:30 2010
+++ src/sys/rump/librump/rumpkern/rump_private.h Tue May 18 14:58:42 2010
@@ -1,4 +1,4 @@
-/* $NetBSD: rump_private.h,v 1.44 2010/04/27 23:30:30 pooka Exp $ */
+/* $NetBSD: rump_private.h,v 1.45 2010/05/18 14:58:42 pooka Exp $ */
/*
* Copyright (c) 2007 Antti Kantee. All Rights Reserved.
@@ -109,11 +109,17 @@
void rump_schedule(void);
void rump_unschedule(void);
void rump_schedule_cpu(struct lwp *);
+void rump_schedule_cpu_interlock(struct lwp *, void *);
void rump_unschedule_cpu(struct lwp *);
-void rump_unschedule_cpu1(struct lwp *);
+void rump_unschedule_cpu_interlock(struct lwp *, void *);
+void rump_unschedule_cpu1(struct lwp *, void *);
-void rump_user_schedule(int);
-void rump_user_unschedule(int, int *);
+void rump_schedlock_cv_wait(struct rumpuser_cv *);
+int rump_schedlock_cv_timedwait(struct rumpuser_cv *,
+ const struct timespec *);
+
+void rump_user_schedule(int, void *);
+void rump_user_unschedule(int, int *, void *);
void rump_cpu_attach(struct cpu_info *);
Index: src/sys/rump/librump/rumpkern/scheduler.c
diff -u src/sys/rump/librump/rumpkern/scheduler.c:1.13 src/sys/rump/librump/rumpkern/scheduler.c:1.14
--- src/sys/rump/librump/rumpkern/scheduler.c:1.13 Wed Apr 28 00:42:16 2010
+++ src/sys/rump/librump/rumpkern/scheduler.c Tue May 18 14:58:42 2010
@@ -1,4 +1,4 @@
-/* $NetBSD: scheduler.c,v 1.13 2010/04/28 00:42:16 pooka Exp $ */
+/* $NetBSD: scheduler.c,v 1.14 2010/05/18 14:58:42 pooka Exp $ */
/*
* Copyright (c) 2009 Antti Kantee. All Rights Reserved.
@@ -29,7 +29,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: scheduler.c,v 1.13 2010/04/28 00:42:16 pooka Exp $");
+__KERNEL_RCSID(0, "$NetBSD: scheduler.c,v 1.14 2010/05/18 14:58:42 pooka Exp $");
#include <sys/param.h>
#include <sys/cpu.h>
@@ -60,6 +60,7 @@
#define RCPU_FREELIST 0x04 /* CPU is on freelist */
static LIST_HEAD(,rumpcpu) cpu_freelist = LIST_HEAD_INITIALIZER(cpu_freelist);
+
static struct rumpuser_mtx *schedmtx;
static struct rumpuser_cv *schedcv, *lwp0cv;
@@ -118,6 +119,23 @@
}
}
+/*
+ * condvar ops using scheduler lock as the rumpuser interlock.
+ */
+void
+rump_schedlock_cv_wait(struct rumpuser_cv *cv)
+{
+
+ rumpuser_cv_wait(cv, schedmtx);
+}
+
+int
+rump_schedlock_cv_timedwait(struct rumpuser_cv *cv, const struct timespec *ts)
+{
+
+ return rumpuser_cv_timedwait(cv, schedmtx, ts->tv_sec, ts->tv_nsec);
+}
+
void
rump_schedule()
{
@@ -159,9 +177,19 @@
void
rump_schedule_cpu(struct lwp *l)
{
+
+ rump_schedule_cpu_interlock(l, NULL);
+}
+
+void
+rump_schedule_cpu_interlock(struct lwp *l, void *interlock)
+{
struct rumpcpu *rcpu;
+ bool schedlock = interlock == schedmtx;
+
+ if (!schedlock)
+ rumpuser_mutex_enter_nowrap(schedmtx);
- rumpuser_mutex_enter_nowrap(schedmtx);
if (l->l_pflag & LP_BOUND) {
KASSERT(l->l_cpu != NULL);
rcpu = &rcpu_storage[l->l_cpu-&rump_cpus[0]];
@@ -236,16 +264,24 @@
rump_unschedule_cpu(struct lwp *l)
{
+ rump_unschedule_cpu_interlock(l, NULL);
+}
+
+void
+rump_unschedule_cpu_interlock(struct lwp *l, void *interlock)
+{
+
if ((l->l_pflag & LP_INTR) == 0)
rump_softint_run(l->l_cpu);
- rump_unschedule_cpu1(l);
+ rump_unschedule_cpu1(l, interlock);
}
void
-rump_unschedule_cpu1(struct lwp *l)
+rump_unschedule_cpu1(struct lwp *l, void *interlock)
{
struct rumpcpu *rcpu;
struct cpu_info *ci;
+ bool schedlock = interlock == schedmtx;
ci = l->l_cpu;
if ((l->l_pflag & LP_BOUND) == 0) {
@@ -256,6 +292,7 @@
KASSERT(rcpu->rcpu_flags & RCPU_BUSY);
rumpuser_mutex_enter_nowrap(schedmtx);
+
if (rcpu->rcpu_flags & RCPU_WANTED) {
/*
* The assumption is that there will usually be max 1
@@ -270,7 +307,9 @@
rumpuser_cv_signal(schedcv);
}
rcpu->rcpu_flags &= ~RCPU_BUSY;
- rumpuser_mutex_exit(schedmtx);
+
+ if (!schedlock)
+ rumpuser_mutex_exit(schedmtx);
}
/* Give up and retake CPU (perhaps a different one) */
