Author: kib
Date: Sun Feb 28 17:52:33 2016
New Revision: 296162
URL: https://svnweb.freebsd.org/changeset/base/296162
Log:
Implement process-shared locks support for libthr.so.3, without
breaking the ABI. Special value is stored in the lock pointer to
indicate shared lock, and offline page in the shared memory is
allocated to store the actual lock.
Reviewed by: vangyzen (previous version)
Discussed with: deischen, emaste, jhb, rwatson,
Martin Simmons <[email protected]>
Tested by: pho
Sponsored by: The FreeBSD Foundation
Added:
head/lib/libthr/thread/thr_pshared.c (contents, props changed)
Modified:
head/bin/sh/miscbltin.c
head/include/pthread.h
head/include/unistd.h
head/lib/libthr/thread/Makefile.inc
head/lib/libthr/thread/thr_barrier.c
head/lib/libthr/thread/thr_barrierattr.c
head/lib/libthr/thread/thr_cond.c
head/lib/libthr/thread/thr_condattr.c
head/lib/libthr/thread/thr_create.c
head/lib/libthr/thread/thr_init.c
head/lib/libthr/thread/thr_mutex.c
head/lib/libthr/thread/thr_mutexattr.c
head/lib/libthr/thread/thr_private.h
head/lib/libthr/thread/thr_rwlock.c
head/lib/libthr/thread/thr_rwlockattr.c
head/sys/kern/kern_resource.c
head/sys/kern/kern_umtx.c
head/sys/kern/uipc_shm.c
head/sys/sys/mman.h
head/sys/sys/resource.h
head/sys/sys/resourcevar.h
head/sys/sys/umtx.h
head/sys/vm/vm_object.c
head/sys/vm/vm_object.h
head/usr.bin/limits/limits.c
head/usr.bin/procstat/procstat_rlimit.c
Modified: head/bin/sh/miscbltin.c
==============================================================================
--- head/bin/sh/miscbltin.c Sun Feb 28 17:42:27 2016 (r296161)
+++ head/bin/sh/miscbltin.c Sun Feb 28 17:52:33 2016 (r296162)
@@ -414,6 +414,9 @@ static const struct limits limits[] = {
#ifdef RLIMIT_KQUEUES
{ "kqueues", (char *)0, RLIMIT_KQUEUES, 1, 'k' },
#endif
+#ifdef RLIMIT_UMTXP
+ { "umtxp", (char *)0, RLIMIT_UMTXP, 1, 'o' },
+#endif
{ (char *) 0, (char *)0, 0, 0, '\0' }
};
Modified: head/include/pthread.h
==============================================================================
--- head/include/pthread.h Sun Feb 28 17:42:27 2016 (r296161)
+++ head/include/pthread.h Sun Feb 28 17:52:33 2016 (r296162)
@@ -69,7 +69,7 @@
#define PTHREAD_EXPLICIT_SCHED 0
/*
- * Flags for read/write lock attributes
+ * Values for process shared/private attributes.
*/
#define PTHREAD_PROCESS_PRIVATE 0
#define PTHREAD_PROCESS_SHARED 1
Modified: head/include/unistd.h
==============================================================================
--- head/include/unistd.h Sun Feb 28 17:42:27 2016 (r296161)
+++ head/include/unistd.h Sun Feb 28 17:52:33 2016 (r296162)
@@ -112,7 +112,7 @@ typedef __useconds_t useconds_t;
#define _POSIX_THREAD_PRIO_INHERIT 200112L
#define _POSIX_THREAD_PRIO_PROTECT 200112L
#define _POSIX_THREAD_PRIORITY_SCHEDULING 200112L
-#define _POSIX_THREAD_PROCESS_SHARED -1
+#define _POSIX_THREAD_PROCESS_SHARED 200112L
#define _POSIX_THREAD_SAFE_FUNCTIONS -1
#define _POSIX_THREAD_SPORADIC_SERVER -1
#define _POSIX_THREADS 200112L
Modified: head/lib/libthr/thread/Makefile.inc
==============================================================================
--- head/lib/libthr/thread/Makefile.inc Sun Feb 28 17:42:27 2016
(r296161)
+++ head/lib/libthr/thread/Makefile.inc Sun Feb 28 17:52:33 2016
(r296162)
@@ -36,6 +36,7 @@ SRCS+= \
thr_mutexattr.c \
thr_once.c \
thr_printf.c \
+ thr_pshared.c \
thr_pspinlock.c \
thr_resume_np.c \
thr_rtld.c \
Modified: head/lib/libthr/thread/thr_barrier.c
==============================================================================
--- head/lib/libthr/thread/thr_barrier.c Sun Feb 28 17:42:27 2016
(r296161)
+++ head/lib/libthr/thread/thr_barrier.c Sun Feb 28 17:52:33 2016
(r296162)
@@ -41,14 +41,25 @@ __weak_reference(_pthread_barrier_destro
int
_pthread_barrier_destroy(pthread_barrier_t *barrier)
{
- pthread_barrier_t bar;
- struct pthread *curthread;
+ pthread_barrier_t bar;
+ struct pthread *curthread;
+ int pshared;
if (barrier == NULL || *barrier == NULL)
return (EINVAL);
+ if (*barrier == THR_PSHARED_PTR) {
+ bar = __thr_pshared_offpage(barrier, 0);
+ if (bar == NULL) {
+ *barrier = NULL;
+ return (0);
+ }
+ pshared = 1;
+ } else {
+ bar = *barrier;
+ pshared = 0;
+ }
curthread = _get_curthread();
- bar = *barrier;
THR_UMUTEX_LOCK(curthread, &bar->b_lock);
if (bar->b_destroying) {
THR_UMUTEX_UNLOCK(curthread, &bar->b_lock);
@@ -71,37 +82,52 @@ _pthread_barrier_destroy(pthread_barrier
THR_UMUTEX_UNLOCK(curthread, &bar->b_lock);
*barrier = NULL;
- free(bar);
+ if (pshared)
+ __thr_pshared_destroy(barrier);
+ else
+ free(bar);
return (0);
}
int
_pthread_barrier_init(pthread_barrier_t *barrier,
- const pthread_barrierattr_t *attr, unsigned count)
+ const pthread_barrierattr_t *attr, unsigned count)
{
- pthread_barrier_t bar;
-
- (void)attr;
+ pthread_barrier_t bar;
+ int pshared;
if (barrier == NULL || count <= 0)
return (EINVAL);
- bar = calloc(1, sizeof(struct pthread_barrier));
- if (bar == NULL)
- return (ENOMEM);
+ if (attr == NULL || *attr == NULL ||
+ (*attr)->pshared == PTHREAD_PROCESS_PRIVATE) {
+ bar = calloc(1, sizeof(struct pthread_barrier));
+ if (bar == NULL)
+ return (ENOMEM);
+ *barrier = bar;
+ pshared = 0;
+ } else {
+ bar = __thr_pshared_offpage(barrier, 1);
+ if (bar == NULL)
+ return (EFAULT);
+ *barrier = THR_PSHARED_PTR;
+ pshared = 1;
+ }
_thr_umutex_init(&bar->b_lock);
_thr_ucond_init(&bar->b_cv);
- bar->b_count = count;
- *barrier = bar;
-
+ if (pshared) {
+ bar->b_lock.m_flags |= USYNC_PROCESS_SHARED;
+ bar->b_cv.c_flags |= USYNC_PROCESS_SHARED;
+ }
+ bar->b_count = count;
return (0);
}
int
_pthread_barrier_wait(pthread_barrier_t *barrier)
{
- struct pthread *curthread = _get_curthread();
+ struct pthread *curthread;
pthread_barrier_t bar;
int64_t cycle;
int ret;
@@ -109,7 +135,14 @@ _pthread_barrier_wait(pthread_barrier_t
if (barrier == NULL || *barrier == NULL)
return (EINVAL);
- bar = *barrier;
+ if (*barrier == THR_PSHARED_PTR) {
+ bar = __thr_pshared_offpage(barrier, 0);
+ if (bar == NULL)
+ return (EINVAL);
+ } else {
+ bar = *barrier;
+ }
+ curthread = _get_curthread();
THR_UMUTEX_LOCK(curthread, &bar->b_lock);
if (++bar->b_waiters == bar->b_count) {
/* Current thread is lastest thread */
Modified: head/lib/libthr/thread/thr_barrierattr.c
==============================================================================
--- head/lib/libthr/thread/thr_barrierattr.c Sun Feb 28 17:42:27 2016
(r296161)
+++ head/lib/libthr/thread/thr_barrierattr.c Sun Feb 28 17:52:33 2016
(r296162)
@@ -56,7 +56,7 @@ _pthread_barrierattr_destroy(pthread_bar
int
_pthread_barrierattr_getpshared(const pthread_barrierattr_t *attr,
- int *pshared)
+ int *pshared)
{
if (attr == NULL || *attr == NULL)
@@ -84,11 +84,9 @@ int
_pthread_barrierattr_setpshared(pthread_barrierattr_t *attr, int pshared)
{
- if (attr == NULL || *attr == NULL)
- return (EINVAL);
-
- /* Only PTHREAD_PROCESS_PRIVATE is supported. */
- if (pshared != PTHREAD_PROCESS_PRIVATE)
+ if (attr == NULL || *attr == NULL ||
+ (pshared != PTHREAD_PROCESS_PRIVATE &&
+ pshared != PTHREAD_PROCESS_SHARED))
return (EINVAL);
(*attr)->pshared = pshared;
Modified: head/lib/libthr/thread/thr_cond.c
==============================================================================
--- head/lib/libthr/thread/thr_cond.c Sun Feb 28 17:42:27 2016
(r296161)
+++ head/lib/libthr/thread/thr_cond.c Sun Feb 28 17:52:33 2016
(r296162)
@@ -1,7 +1,11 @@
/*
* Copyright (c) 2005 David Xu <[email protected]>
+ * Copyright (c) 2015 The FreeBSD Foundation
* All rights reserved.
*
+ * Portions of this software were developed by Konstantin Belousov
+ * under sponsorship from the FreeBSD Foundation.
+ *
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
@@ -63,29 +67,45 @@ __weak_reference(_pthread_cond_broadcast
#define CV_PSHARED(cvp) (((cvp)->__flags & USYNC_PROCESS_SHARED) != 0)
+static void
+cond_init_body(struct pthread_cond *cvp, const struct pthread_cond_attr *cattr)
+{
+
+ if (cattr == NULL) {
+ cvp->__clock_id = CLOCK_REALTIME;
+ } else {
+ if (cattr->c_pshared)
+ cvp->__flags |= USYNC_PROCESS_SHARED;
+ cvp->__clock_id = cattr->c_clockid;
+ }
+}
+
static int
cond_init(pthread_cond_t *cond, const pthread_condattr_t *cond_attr)
{
- struct pthread_cond *cvp;
- int error = 0;
+ struct pthread_cond *cvp;
+ const struct pthread_cond_attr *cattr;
+ int pshared;
- if ((cvp = (pthread_cond_t)
- calloc(1, sizeof(struct pthread_cond))) == NULL) {
- error = ENOMEM;
+ cattr = cond_attr != NULL ? *cond_attr : NULL;
+ if (cattr == NULL || cattr->c_pshared == PTHREAD_PROCESS_PRIVATE) {
+ pshared = 0;
+ cvp = calloc(1, sizeof(struct pthread_cond));
+ if (cvp == NULL)
+ return (ENOMEM);
} else {
- /*
- * Initialise the condition variable structure:
- */
- if (cond_attr == NULL || *cond_attr == NULL) {
- cvp->__clock_id = CLOCK_REALTIME;
- } else {
- if ((*cond_attr)->c_pshared)
- cvp->__flags |= USYNC_PROCESS_SHARED;
- cvp->__clock_id = (*cond_attr)->c_clockid;
- }
- *cond = cvp;
+ pshared = 1;
+ cvp = __thr_pshared_offpage(cond, 1);
+ if (cvp == NULL)
+ return (EFAULT);
}
- return (error);
+
+ /*
+ * Initialise the condition variable structure:
+ */
+ cond_init_body(cvp, cattr);
+ *cond = pshared ? THR_PSHARED_PTR : cvp;
+ return (0);
}
static int
@@ -106,7 +126,11 @@ init_static(struct pthread *thread, pthr
}
#define CHECK_AND_INIT_COND
\
- if (__predict_false((cvp = (*cond)) <= THR_COND_DESTROYED)) {
\
+ if (*cond == THR_PSHARED_PTR) {
\
+ cvp = __thr_pshared_offpage(cond, 0);
\
+ if (cvp == NULL)
\
+ return (EINVAL);
\
+ } else if (__predict_false((cvp = (*cond)) <= THR_COND_DESTROYED)) {
\
if (cvp == THR_COND_INITIALIZER) {
\
int ret;
\
ret = init_static(_get_curthread(), cond);
\
@@ -129,21 +153,22 @@ _pthread_cond_init(pthread_cond_t *cond,
int
_pthread_cond_destroy(pthread_cond_t *cond)
{
- struct pthread_cond *cvp;
- int error = 0;
+ struct pthread_cond *cvp;
+ int error;
- if ((cvp = *cond) == THR_COND_INITIALIZER)
- error = 0;
- else if (cvp == THR_COND_DESTROYED)
+ error = 0;
+ if (*cond == THR_PSHARED_PTR) {
+ cvp = __thr_pshared_offpage(cond, 0);
+ if (cvp != NULL)
+ __thr_pshared_destroy(cond);
+ *cond = THR_COND_DESTROYED;
+ } else if ((cvp = *cond) == THR_COND_INITIALIZER) {
+ /* nothing */
+ } else if (cvp == THR_COND_DESTROYED) {
error = EINVAL;
- else {
+ } else {
cvp = *cond;
*cond = THR_COND_DESTROYED;
-
- /*
- * Free the memory allocated for the condition
- * variable structure:
- */
free(cvp);
}
return (error);
@@ -297,7 +322,13 @@ cond_wait_common(pthread_cond_t *cond, p
CHECK_AND_INIT_COND
- mp = *mutex;
+ if (*mutex == THR_PSHARED_PTR) {
+ mp = __thr_pshared_offpage(mutex, 0);
+ if (mp == NULL)
+ return (EINVAL);
+ } else {
+ mp = *mutex;
+ }
if ((error = _mutex_owned(curthread, mp)) != 0)
return (error);
@@ -385,7 +416,7 @@ cond_signal_common(pthread_cond_t *cond)
td = _sleepq_first(sq);
mp = td->mutex_obj;
cvp->__has_user_waiters = _sleepq_remove(sq, td);
- if (mp->m_owner == curthread) {
+ if (mp->m_owner == TID(curthread)) {
if (curthread->nwaiter_defer >= MAX_DEFER_WAITERS) {
_thr_wake_all(curthread->defer_waiters,
curthread->nwaiter_defer);
@@ -417,7 +448,7 @@ drop_cb(struct pthread *td, void *arg)
struct pthread *curthread = ba->curthread;
mp = td->mutex_obj;
- if (mp->m_owner == curthread) {
+ if (mp->m_owner == TID(curthread)) {
if (curthread->nwaiter_defer >= MAX_DEFER_WAITERS) {
_thr_wake_all(curthread->defer_waiters,
curthread->nwaiter_defer);
Modified: head/lib/libthr/thread/thr_condattr.c
==============================================================================
--- head/lib/libthr/thread/thr_condattr.c Sun Feb 28 17:42:27 2016
(r296161)
+++ head/lib/libthr/thread/thr_condattr.c Sun Feb 28 17:52:33 2016
(r296162)
@@ -105,20 +105,21 @@ _pthread_condattr_setclock(pthread_conda
int
_pthread_condattr_getpshared(const pthread_condattr_t *attr, int *pshared)
{
+
if (attr == NULL || *attr == NULL)
return (EINVAL);
-
- *pshared = PTHREAD_PROCESS_PRIVATE;
+ *pshared = (*attr)->c_pshared;
return (0);
}
int
_pthread_condattr_setpshared(pthread_condattr_t *attr, int pshared)
{
- if (attr == NULL || *attr == NULL)
- return (EINVAL);
- if (pshared != PTHREAD_PROCESS_PRIVATE)
+ if (attr == NULL || *attr == NULL ||
+ (pshared != PTHREAD_PROCESS_PRIVATE &&
+ pshared != PTHREAD_PROCESS_SHARED))
return (EINVAL);
+ (*attr)->c_pshared = pshared;
return (0);
}
Modified: head/lib/libthr/thread/thr_create.c
==============================================================================
--- head/lib/libthr/thread/thr_create.c Sun Feb 28 17:42:27 2016
(r296161)
+++ head/lib/libthr/thread/thr_create.c Sun Feb 28 17:52:33 2016
(r296162)
@@ -56,12 +56,12 @@ _pthread_create(pthread_t * thread, cons
struct thr_param param;
struct sched_param sched_param;
struct rtprio rtp;
- int ret = 0, locked, create_suspended;
sigset_t set, oset;
- cpuset_t *cpusetp = NULL;
- int cpusetsize = 0;
- int old_stack_prot;
+ cpuset_t *cpusetp;
+ int i, cpusetsize, create_suspended, locked, old_stack_prot, ret;
+ cpusetp = NULL;
+ ret = cpusetsize = 0;
_thr_check_init();
/*
@@ -118,8 +118,8 @@ _pthread_create(pthread_t * thread, cons
new_thread->cancel_enable = 1;
new_thread->cancel_async = 0;
/* Initialize the mutex queue: */
- TAILQ_INIT(&new_thread->mutexq);
- TAILQ_INIT(&new_thread->pp_mutexq);
+ for (i = 0; i < TMQ_NITEMS; i++)
+ TAILQ_INIT(&new_thread->mq[i]);
/* Initialise hooks in the thread structure: */
if (new_thread->attr.suspend == THR_CREATE_SUSPENDED) {
Modified: head/lib/libthr/thread/thr_init.c
==============================================================================
--- head/lib/libthr/thread/thr_init.c Sun Feb 28 17:42:27 2016
(r296161)
+++ head/lib/libthr/thread/thr_init.c Sun Feb 28 17:52:33 2016
(r296162)
@@ -91,13 +91,15 @@ struct pthread_attr _pthread_attr_defaul
struct pthread_mutex_attr _pthread_mutexattr_default = {
.m_type = PTHREAD_MUTEX_DEFAULT,
.m_protocol = PTHREAD_PRIO_NONE,
- .m_ceiling = 0
+ .m_ceiling = 0,
+ .m_pshared = PTHREAD_PROCESS_PRIVATE,
};
struct pthread_mutex_attr _pthread_mutexattr_adaptive_default = {
.m_type = PTHREAD_MUTEX_ADAPTIVE_NP,
.m_protocol = PTHREAD_PRIO_NONE,
- .m_ceiling = 0
+ .m_ceiling = 0,
+ .m_pshared = PTHREAD_PROCESS_PRIVATE,
};
/* Default condition variable attributes: */
@@ -387,6 +389,7 @@ static void
init_main_thread(struct pthread *thread)
{
struct sched_param sched_param;
+ int i;
/* Setup the thread attributes. */
thr_self(&thread->tid);
@@ -428,9 +431,9 @@ init_main_thread(struct pthread *thread)
thread->cancel_enable = 1;
thread->cancel_async = 0;
- /* Initialize the mutex queue: */
- TAILQ_INIT(&thread->mutexq);
- TAILQ_INIT(&thread->pp_mutexq);
+ /* Initialize the mutex queues */
+ for (i = 0; i < TMQ_NITEMS; i++)
+ TAILQ_INIT(&thread->mq[i]);
thread->state = PS_RUNNING;
@@ -463,6 +466,7 @@ init_private(void)
_thr_once_init();
_thr_spinlock_init();
_thr_list_init();
+ __thr_pshared_init();
_thr_wake_addr_init();
_sleepq_init();
_single_thread = NULL;
Modified: head/lib/libthr/thread/thr_mutex.c
==============================================================================
--- head/lib/libthr/thread/thr_mutex.c Sun Feb 28 17:42:27 2016
(r296161)
+++ head/lib/libthr/thread/thr_mutex.c Sun Feb 28 17:52:33 2016
(r296162)
@@ -1,8 +1,13 @@
/*
* Copyright (c) 1995 John Birrell <[email protected]>.
* Copyright (c) 2006 David Xu <[email protected]>.
+ * Copyright (c) 2015 The FreeBSD Foundation
+ *
* All rights reserved.
*
+ * Portions of this software were developed by Konstantin Belousov
+ * under sponsorship from the FreeBSD Foundation.
+ *
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
@@ -45,26 +50,6 @@
#include "thr_private.h"
-#if defined(_PTHREADS_INVARIANTS)
-#define MUTEX_INIT_LINK(m) do { \
- (m)->m_qe.tqe_prev = NULL; \
- (m)->m_qe.tqe_next = NULL; \
-} while (0)
-#define MUTEX_ASSERT_IS_OWNED(m) do { \
- if (__predict_false((m)->m_qe.tqe_prev == NULL))\
- PANIC("mutex is not on list"); \
-} while (0)
-#define MUTEX_ASSERT_NOT_OWNED(m) do { \
- if (__predict_false((m)->m_qe.tqe_prev != NULL || \
- (m)->m_qe.tqe_next != NULL)) \
- PANIC("mutex is on list"); \
-} while (0)
-#else
-#define MUTEX_INIT_LINK(m)
-#define MUTEX_ASSERT_IS_OWNED(m)
-#define MUTEX_ASSERT_NOT_OWNED(m)
-#endif
-
/*
* For adaptive mutexes, how many times to spin doing trylock2
* before entering the kernel to block
@@ -122,36 +107,71 @@ __strong_reference(__pthread_mutex_setyi
__weak_reference(_pthread_mutex_getyieldloops_np,
pthread_mutex_getyieldloops_np);
__weak_reference(_pthread_mutex_isowned_np, pthread_mutex_isowned_np);
+static void
+mutex_init_link(struct pthread_mutex *m)
+{
+
+#if defined(_PTHREADS_INVARIANTS)
+ m->m_qe.tqe_prev = NULL;
+ m->m_qe.tqe_next = NULL;
+ m->m_pqe.tqe_prev = NULL;
+ m->m_pqe.tqe_next = NULL;
+#endif
+}
+
+static void
+mutex_assert_is_owned(struct pthread_mutex *m)
+{
+
+#if defined(_PTHREADS_INVARIANTS)
+ if (__predict_false(m->m_qe.tqe_prev == NULL))
+ PANIC("mutex is not on list");
+#endif
+}
+
+static void
+mutex_assert_not_owned(struct pthread_mutex *m)
+{
+
+#if defined(_PTHREADS_INVARIANTS)
+ if (__predict_false(m->m_qe.tqe_prev != NULL ||
+ m->m_qe.tqe_next != NULL))
+ PANIC("mutex is on list");
+#endif
+}
+
static int
-mutex_init(pthread_mutex_t *mutex,
- const struct pthread_mutex_attr *mutex_attr,
- void *(calloc_cb)(size_t, size_t))
+is_pshared_mutex(struct pthread_mutex *m)
{
- const struct pthread_mutex_attr *attr;
- struct pthread_mutex *pmutex;
- if (mutex_attr == NULL) {
- attr = &_pthread_mutexattr_default;
- } else {
- attr = mutex_attr;
- if (attr->m_type < PTHREAD_MUTEX_ERRORCHECK ||
- attr->m_type >= PTHREAD_MUTEX_TYPE_MAX)
- return (EINVAL);
- if (attr->m_protocol < PTHREAD_PRIO_NONE ||
- attr->m_protocol > PTHREAD_PRIO_PROTECT)
- return (EINVAL);
- }
- if ((pmutex = (pthread_mutex_t)
- calloc_cb(1, sizeof(struct pthread_mutex))) == NULL)
- return (ENOMEM);
+ return ((m->m_lock.m_flags & USYNC_PROCESS_SHARED) != 0);
+}
+
+static int
+mutex_check_attr(const struct pthread_mutex_attr *attr)
+{
+
+ if (attr->m_type < PTHREAD_MUTEX_ERRORCHECK ||
+ attr->m_type >= PTHREAD_MUTEX_TYPE_MAX)
+ return (EINVAL);
+ if (attr->m_protocol < PTHREAD_PRIO_NONE ||
+ attr->m_protocol > PTHREAD_PRIO_PROTECT)
+ return (EINVAL);
+ return (0);
+}
+
+static void
+mutex_init_body(struct pthread_mutex *pmutex,
+ const struct pthread_mutex_attr *attr)
+{
pmutex->m_flags = attr->m_type;
- pmutex->m_owner = NULL;
+ pmutex->m_owner = 0;
pmutex->m_count = 0;
pmutex->m_spinloops = 0;
pmutex->m_yieldloops = 0;
- MUTEX_INIT_LINK(pmutex);
- switch(attr->m_protocol) {
+ mutex_init_link(pmutex);
+ switch (attr->m_protocol) {
case PTHREAD_PRIO_NONE:
pmutex->m_lock.m_owner = UMUTEX_UNOWNED;
pmutex->m_lock.m_flags = 0;
@@ -166,13 +186,37 @@ mutex_init(pthread_mutex_t *mutex,
pmutex->m_lock.m_ceilings[0] = attr->m_ceiling;
break;
}
+ if (attr->m_pshared == PTHREAD_PROCESS_SHARED)
+ pmutex->m_lock.m_flags |= USYNC_PROCESS_SHARED;
if (PMUTEX_TYPE(pmutex->m_flags) == PTHREAD_MUTEX_ADAPTIVE_NP) {
pmutex->m_spinloops =
_thr_spinloops ? _thr_spinloops: MUTEX_ADAPTIVE_SPINS;
pmutex->m_yieldloops = _thr_yieldloops;
}
+}
+static int
+mutex_init(pthread_mutex_t *mutex,
+ const struct pthread_mutex_attr *mutex_attr,
+ void *(calloc_cb)(size_t, size_t))
+{
+ const struct pthread_mutex_attr *attr;
+ struct pthread_mutex *pmutex;
+ int error;
+
+ if (mutex_attr == NULL) {
+ attr = &_pthread_mutexattr_default;
+ } else {
+ attr = mutex_attr;
+ error = mutex_check_attr(attr);
+ if (error != 0)
+ return (error);
+ }
+ if ((pmutex = (pthread_mutex_t)
+ calloc_cb(1, sizeof(struct pthread_mutex))) == NULL)
+ return (ENOMEM);
+ mutex_init_body(pmutex, attr);
*mutex = pmutex;
return (0);
}
@@ -187,7 +231,8 @@ init_static(struct pthread *thread, pthr
if (*mutex == THR_MUTEX_INITIALIZER)
ret = mutex_init(mutex, &_pthread_mutexattr_default, calloc);
else if (*mutex == THR_ADAPTIVE_MUTEX_INITIALIZER)
- ret = mutex_init(mutex, &_pthread_mutexattr_adaptive_default,
calloc);
+ ret = mutex_init(mutex, &_pthread_mutexattr_adaptive_default,
+ calloc);
else
ret = 0;
THR_LOCK_RELEASE(thread, &_mutex_static_lock);
@@ -200,7 +245,7 @@ set_inherited_priority(struct pthread *c
{
struct pthread_mutex *m2;
- m2 = TAILQ_LAST(&curthread->pp_mutexq, mutex_queue);
+ m2 = TAILQ_LAST(&curthread->mq[TMQ_NORM_PP], mutex_queue);
if (m2 != NULL)
m->m_lock.m_ceilings[1] = m2->m_lock.m_ceilings[0];
else
@@ -211,7 +256,25 @@ int
__pthread_mutex_init(pthread_mutex_t *mutex,
const pthread_mutexattr_t *mutex_attr)
{
- return mutex_init(mutex, mutex_attr ? *mutex_attr : NULL, calloc);
+ struct pthread_mutex *pmtx;
+ int ret;
+
+ if (mutex_attr != NULL) {
+ ret = mutex_check_attr(*mutex_attr);
+ if (ret != 0)
+ return (ret);
+ }
+ if (mutex_attr == NULL ||
+ (*mutex_attr)->m_pshared == PTHREAD_PROCESS_PRIVATE) {
+ return (mutex_init(mutex, mutex_attr ? *mutex_attr : NULL,
+ calloc));
+ }
+ pmtx = __thr_pshared_offpage(mutex, 1);
+ if (pmtx == NULL)
+ return (EFAULT);
+ *mutex = THR_PSHARED_PTR;
+ mutex_init_body(pmtx, *mutex_attr);
+ return (0);
}
/* This function is used internally by malloc. */
@@ -222,7 +285,8 @@ _pthread_mutex_init_calloc_cb(pthread_mu
static const struct pthread_mutex_attr attr = {
.m_type = PTHREAD_MUTEX_NORMAL,
.m_protocol = PTHREAD_PRIO_NONE,
- .m_ceiling = 0
+ .m_ceiling = 0,
+ .m_pshared = PTHREAD_PROCESS_PRIVATE,
};
int ret;
@@ -232,31 +296,44 @@ _pthread_mutex_init_calloc_cb(pthread_mu
return (ret);
}
-void
-_mutex_fork(struct pthread *curthread)
+/*
+ * Fix mutex ownership for child process.
+ *
+ * Process private mutex ownership is transmitted from the forking
+ * thread to the child process.
+ *
+ * Process shared mutex should not be inherited because owner is
+ * forking thread which is in parent process, they are removed from
+ * the owned mutex list.
+ */
+static void
+queue_fork(struct pthread *curthread, struct mutex_queue *q,
+ struct mutex_queue *qp, uint bit)
{
struct pthread_mutex *m;
- /*
- * Fix mutex ownership for child process.
- * note that process shared mutex should not
- * be inherited because owner is forking thread
- * which is in parent process, they should be
- * removed from the owned mutex list, current,
- * process shared mutex is not supported, so I
- * am not worried.
- */
+ TAILQ_INIT(q);
+ TAILQ_FOREACH(m, qp, m_pqe) {
+ TAILQ_INSERT_TAIL(q, m, m_qe);
+ m->m_lock.m_owner = TID(curthread) | bit;
+ m->m_owner = TID(curthread);
+ }
+}
+
+void
+_mutex_fork(struct pthread *curthread)
+{
- TAILQ_FOREACH(m, &curthread->mutexq, m_qe)
- m->m_lock.m_owner = TID(curthread);
- TAILQ_FOREACH(m, &curthread->pp_mutexq, m_qe)
- m->m_lock.m_owner = TID(curthread) | UMUTEX_CONTESTED;
+ queue_fork(curthread, &curthread->mq[TMQ_NORM],
+ &curthread->mq[TMQ_NORM_PRIV], 0);
+ queue_fork(curthread, &curthread->mq[TMQ_NORM_PP],
+ &curthread->mq[TMQ_NORM_PP_PRIV], UMUTEX_CONTESTED);
}
int
_pthread_mutex_destroy(pthread_mutex_t *mutex)
{
- pthread_mutex_t m;
+ pthread_mutex_t m, m1;
int ret;
m = *mutex;
@@ -265,11 +342,20 @@ _pthread_mutex_destroy(pthread_mutex_t *
} else if (m == THR_MUTEX_DESTROYED) {
ret = EINVAL;
} else {
- if (m->m_owner != NULL) {
+ if (m == THR_PSHARED_PTR) {
+ m1 = __thr_pshared_offpage(mutex, 0);
+ if (m1 != NULL) {
+ mutex_assert_not_owned(m1);
+ __thr_pshared_destroy(mutex);
+ }
+ *mutex = THR_MUTEX_DESTROYED;
+ return (0);
+ }
+ if (m->m_owner != 0) {
ret = EBUSY;
} else {
*mutex = THR_MUTEX_DESTROYED;
- MUTEX_ASSERT_NOT_OWNED(m);
+ mutex_assert_not_owned(m);
free(m);
ret = 0;
}
@@ -278,54 +364,87 @@ _pthread_mutex_destroy(pthread_mutex_t *
return (ret);
}
-#define ENQUEUE_MUTEX(curthread, m) \
- do { \
- (m)->m_owner = curthread; \
- /* Add to the list of owned mutexes: */ \
- MUTEX_ASSERT_NOT_OWNED((m)); \
- if (((m)->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0) \
- TAILQ_INSERT_TAIL(&curthread->mutexq, (m), m_qe);\
- else \
- TAILQ_INSERT_TAIL(&curthread->pp_mutexq, (m), m_qe);\
- } while (0)
-
-#define DEQUEUE_MUTEX(curthread, m) \
- (m)->m_owner = NULL; \
- MUTEX_ASSERT_IS_OWNED(m); \
- if (__predict_true(((m)->m_lock.m_flags & UMUTEX_PRIO_PROTECT)
== 0)) \
- TAILQ_REMOVE(&curthread->mutexq, (m), m_qe);
\
- else { \
- TAILQ_REMOVE(&curthread->pp_mutexq, (m), m_qe); \
- set_inherited_priority(curthread, m); \
- } \
- MUTEX_INIT_LINK(m);
-
-#define CHECK_AND_INIT_MUTEX \
- if (__predict_false((m = *mutex) <= THR_MUTEX_DESTROYED)) { \
- if (m == THR_MUTEX_DESTROYED) \
- return (EINVAL); \
- int ret; \
- ret = init_static(_get_curthread(), mutex); \
- if (ret) \
- return (ret); \
- m = *mutex; \
- }
+static int
+mutex_qidx(struct pthread_mutex *m)
+{
+
+ if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0)
+ return (TMQ_NORM);
+ return (TMQ_NORM_PP);
+}
+
+static void
+enqueue_mutex(struct pthread *curthread, struct pthread_mutex *m)
+{
+ int qidx;
+
+ m->m_owner = TID(curthread);
+ /* Add to the list of owned mutexes: */
+ mutex_assert_not_owned(m);
+ qidx = mutex_qidx(m);
+ TAILQ_INSERT_TAIL(&curthread->mq[qidx], m, m_qe);
+ if (!is_pshared_mutex(m))
+ TAILQ_INSERT_TAIL(&curthread->mq[qidx + 1], m, m_pqe);
+}
+
+static void
+dequeue_mutex(struct pthread *curthread, struct pthread_mutex *m)
+{
+ int qidx;
+
+ m->m_owner = 0;
+ mutex_assert_is_owned(m);
+ qidx = mutex_qidx(m);
+ TAILQ_REMOVE(&curthread->mq[qidx], m, m_qe);
+ if (!is_pshared_mutex(m))
+ TAILQ_REMOVE(&curthread->mq[qidx + 1], m, m_pqe);
+ if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) != 0)
+ set_inherited_priority(curthread, m);
+ mutex_init_link(m);
+}
static int
-mutex_trylock_common(pthread_mutex_t *mutex)
+check_and_init_mutex(pthread_mutex_t *mutex, struct pthread_mutex **m)
{
- struct pthread *curthread = _get_curthread();
- struct pthread_mutex *m = *mutex;
+ int ret;
+
+ *m = *mutex;
+ ret = 0;
+ if (*m == THR_PSHARED_PTR) {
+ *m = __thr_pshared_offpage(mutex, 0);
+ if (*m == NULL)
+ ret = EINVAL;
+ } else if (__predict_false(*m <= THR_MUTEX_DESTROYED)) {
+ if (*m == THR_MUTEX_DESTROYED) {
+ ret = EINVAL;
+ } else {
+ ret = init_static(_get_curthread(), mutex);
+ if (ret == 0)
+ *m = *mutex;
+ }
+ }
+ return (ret);
+}
+
+int
+__pthread_mutex_trylock(pthread_mutex_t *mutex)
+{
+ struct pthread *curthread;
+ struct pthread_mutex *m;
uint32_t id;
int ret;
+ ret = check_and_init_mutex(mutex, &m);
+ if (ret != 0)
+ return (ret);
+ curthread = _get_curthread();
id = TID(curthread);
if (m->m_flags & PMUTEX_FLAG_PRIVATE)
THR_CRITICAL_ENTER(curthread);
ret = _thr_umutex_trylock(&m->m_lock, id);
if (__predict_true(ret == 0)) {
- ENQUEUE_MUTEX(curthread, m);
- } else if (m->m_owner == curthread) {
+ enqueue_mutex(curthread, m);
+ } else if (m->m_owner == id) {
ret = mutex_self_trylock(m);
} /* else {} */
if (ret && (m->m_flags & PMUTEX_FLAG_PRIVATE))
@@ -333,16 +452,6 @@ mutex_trylock_common(pthread_mutex_t *mu
return (ret);
}
-int
-__pthread_mutex_trylock(pthread_mutex_t *mutex)
-{
- struct pthread_mutex *m;
-
- CHECK_AND_INIT_MUTEX
-
- return (mutex_trylock_common(mutex));
-}
-
static int
mutex_lock_sleep(struct pthread *curthread, struct pthread_mutex *m,
const struct timespec *abstime)
@@ -351,10 +460,10 @@ mutex_lock_sleep(struct pthread *curthre
int count;
int ret;
- if (m->m_owner == curthread)
- return mutex_self_lock(m, abstime);
-
id = TID(curthread);
+ if (m->m_owner == id)
+ return (mutex_self_lock(m, abstime));
+
/*
* For adaptive mutexes, spin for a bit in the expectation
* that if the application requests this mutex type then
@@ -406,7 +515,7 @@ sleep_in_kernel:
}
done:
if (ret == 0)
- ENQUEUE_MUTEX(curthread, m);
+ enqueue_mutex(curthread, m);
return (ret);
}
@@ -421,7 +530,7 @@ mutex_lock_common(struct pthread_mutex *
if (!cvattach && m->m_flags & PMUTEX_FLAG_PRIVATE)
THR_CRITICAL_ENTER(curthread);
if (_thr_umutex_trylock2(&m->m_lock, TID(curthread)) == 0) {
- ENQUEUE_MUTEX(curthread, m);
+ enqueue_mutex(curthread, m);
ret = 0;
} else {
ret = mutex_lock_sleep(curthread, m, abstime);
@@ -434,25 +543,28 @@ mutex_lock_common(struct pthread_mutex *
int
__pthread_mutex_lock(pthread_mutex_t *mutex)
{
- struct pthread_mutex *m;
+ struct pthread_mutex *m;
+ int ret;
_thr_check_init();
-
- CHECK_AND_INIT_MUTEX
-
- return (mutex_lock_common(m, NULL, 0));
+ ret = check_and_init_mutex(mutex, &m);
+ if (ret == 0)
+ ret = mutex_lock_common(m, NULL, 0);
+ return (ret);
}
int
-__pthread_mutex_timedlock(pthread_mutex_t *mutex, const struct timespec
*abstime)
+__pthread_mutex_timedlock(pthread_mutex_t *mutex,
+ const struct timespec *abstime)
{
- struct pthread_mutex *m;
+ struct pthread_mutex *m;
+ int ret;
_thr_check_init();
-
- CHECK_AND_INIT_MUTEX
-
- return (mutex_lock_common(m, abstime, 0));
+ ret = check_and_init_mutex(mutex, &m);
+ if (ret == 0)
+ ret = mutex_lock_common(m, abstime, 0);
+ return (ret);
}
int
@@ -460,7 +572,13 @@ _pthread_mutex_unlock(pthread_mutex_t *m
{
struct pthread_mutex *mp;
- mp = *mutex;
+ if (*mutex == THR_PSHARED_PTR) {
+ mp = __thr_pshared_offpage(mutex, 0);
+ if (mp == NULL)
+ return (EINVAL);
+ } else {
+ mp = *mutex;
+ }
return (mutex_unlock_common(mp, 0, NULL));
}
@@ -493,7 +611,7 @@ _mutex_cv_attach(struct pthread_mutex *m
{
struct pthread *curthread = _get_curthread();
- ENQUEUE_MUTEX(curthread, m);
+ enqueue_mutex(curthread, m);
m->m_count = count;
return (0);
}
*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
_______________________________________________
[email protected] mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-all
To unsubscribe, send any mail to "[email protected]"
