Author: kevans Date: Tue Nov 17 03:36:58 2020 New Revision: 367744 URL: https://svnweb.freebsd.org/changeset/base/367744
Log: umtx_op: reduce redundancy required for compat32 All of the compat32 variants are substantially the same, save for copyin/copyout (mostly). Apply the same kind of technique used with kevent here by having the syscall routines supply a umtx_copyops describing the operations needed. umtx_copyops carries the bare minimum needed- size of timespec and _umtx_time are used for determining if copyout is needed in the sem2_wait case. Reviewed by: kib MFC after: 1 week Differential Revision: https://reviews.freebsd.org/D27222 Modified: head/sys/compat/freebsd32/freebsd32.h head/sys/compat/freebsd32/freebsd32_misc.c head/sys/kern/kern_umtx.c head/sys/sys/proc.h head/sys/sys/syscallsubr.h head/sys/sys/umtx.h Modified: head/sys/compat/freebsd32/freebsd32.h ============================================================================== --- head/sys/compat/freebsd32/freebsd32.h Tue Nov 17 03:34:01 2020 (r367743) +++ head/sys/compat/freebsd32/freebsd32.h Tue Nov 17 03:36:58 2020 (r367744) @@ -94,6 +94,27 @@ struct itimerval32 { struct timeval32 it_value; }; +struct umtx_time32 { + struct timespec32 _timeout; + uint32_t _flags; + uint32_t _clockid; +}; + +struct umtx_robust_lists_params_compat32 { + uint32_t robust_list_offset; + uint32_t robust_priv_list_offset; + uint32_t robust_inact_offset; +}; + +struct umutex32 { + volatile __lwpid_t m_owner; /* Owner of the mutex */ + __uint32_t m_flags; /* Flags of the mutex */ + __uint32_t m_ceilings[2]; /* Priority protect ceiling */ + __uint32_t m_rb_lnk; /* Robust linkage */ + __uint32_t m_pad; + __uint32_t m_spare[2]; +}; + #define FREEBSD4_MFSNAMELEN 16 #define FREEBSD4_MNAMELEN (88 - 2 * sizeof(int32_t)) Modified: head/sys/compat/freebsd32/freebsd32_misc.c ============================================================================== --- head/sys/compat/freebsd32/freebsd32_misc.c Tue Nov 17 03:34:01 2020 (r367743) +++ head/sys/compat/freebsd32/freebsd32_misc.c Tue Nov 17 03:36:58 2020 (r367744) @@ -84,6 +84,7 @@ __FBSDID("$FreeBSD$"); #include <sys/thr.h> #include <sys/unistd.h> #include <sys/ucontext.h> +#include <sys/umtx.h> #include <sys/vnode.h> #include <sys/wait.h> #include <sys/ipc.h> @@ -3764,4 +3765,12 @@ freebsd32_sched_rr_get_interval(struct thread *td, error = copyout(&ts32, uap->interval, sizeof(ts32)); } return (error); +} + +int +freebsd32__umtx_op(struct thread *td, struct freebsd32__umtx_op_args *uap) +{ + + return (kern__umtx_op(td, uap->obj, uap->op, uap->val, uap->uaddr, + uap->uaddr2, &umtx_native_ops32)); } Modified: head/sys/kern/kern_umtx.c ============================================================================== --- head/sys/kern/kern_umtx.c Tue Nov 17 03:34:01 2020 (r367743) +++ head/sys/kern/kern_umtx.c Tue Nov 17 03:36:58 2020 (r367744) @@ -219,20 +219,9 @@ struct abs_timeout { struct timespec end; }; -#ifdef COMPAT_FREEBSD32 -struct umutex32 { - volatile __lwpid_t m_owner; /* Owner of the mutex */ - __uint32_t m_flags; /* Flags of the mutex */ - __uint32_t m_ceilings[2]; /* Priority protect ceiling */ - __uint32_t m_rb_lnk; /* Robust linkage */ - __uint32_t m_pad; - __uint32_t m_spare[2]; -}; - _Static_assert(sizeof(struct umutex) == sizeof(struct umutex32), "umutex32"); _Static_assert(__offsetof(struct umutex, m_spare[0]) == __offsetof(struct umutex32, m_spare[0]), "m_spare32"); -#endif int umtx_shm_vnobj_persistent = 0; SYSCTL_INT(_kern_ipc, OID_AUTO, umtx_vnode_persistent, CTLFLAG_RWTUN, @@ -3400,11 +3389,11 @@ do_sem2_wake(struct thread *td, struct _usem2 *sem) } inline int -umtx_copyin_timeout(const void *addr, struct timespec *tsp) +umtx_copyin_timeout(const void *uaddr, struct timespec *tsp) { int error; - error = copyin(addr, tsp, sizeof(struct timespec)); + error = copyin(uaddr, tsp, sizeof(*tsp)); if (error == 0) { if (tsp->tv_sec < 0 || tsp->tv_nsec >= 1000000000 || @@ -3415,16 +3404,16 @@ umtx_copyin_timeout(const void *addr, struct timespec } static inline int -umtx_copyin_umtx_time(const void *addr, size_t size, struct _umtx_time *tp) +umtx_copyin_umtx_time(const void *uaddr, size_t size, struct _umtx_time *tp) { int error; - if (size <= sizeof(struct timespec)) { + if (size <= sizeof(tp->_timeout)) { tp->_clockid = CLOCK_REALTIME; tp->_flags = 0; - error = copyin(addr, &tp->_timeout, sizeof(struct timespec)); + error = copyin(uaddr, &tp->_timeout, sizeof(tp->_timeout)); } else - error = copyin(addr, tp, sizeof(struct _umtx_time)); + error = copyin(uaddr, tp, sizeof(*tp)); if (error != 0) return (error); if (tp->_timeout.tv_sec < 0 || @@ -3434,14 +3423,41 @@ umtx_copyin_umtx_time(const void *addr, size_t size, s } static int -__umtx_op_unimpl(struct thread *td, struct _umtx_op_args *uap) +umtx_copyin_robust_lists(const void *uaddr, size_t size, + struct umtx_robust_lists_params *rb) { + if (size > sizeof(*rb)) + return (EINVAL); + return (copyin(uaddr, &rb, size)); +} + +static int +umtx_copyout_timeout(void *uaddr, size_t sz, struct timespec *tsp) +{ + + /* + * Should be guaranteed by the caller, sz == uaddr1 - sizeof(_umtx_time) + * and we're only called if sz >= sizeof(timespec) as supplied in the + * copyops. + */ + KASSERT(sz >= sizeof(*tsp), + ("umtx_copyops specifies incorrect sizes")); + + return (copyout(tsp, uaddr, sizeof(*tsp))); +} + +static int +__umtx_op_unimpl(struct thread *td, struct _umtx_op_args *uap, + const struct umtx_copyops *ops __unused) +{ + return (EOPNOTSUPP); } static int -__umtx_op_wait(struct thread *td, struct _umtx_op_args *uap) +__umtx_op_wait(struct thread *td, struct _umtx_op_args *uap, + const struct umtx_copyops *ops) { struct _umtx_time timeout, *tm_p; int error; @@ -3449,17 +3465,18 @@ __umtx_op_wait(struct thread *td, struct _umtx_op_args if (uap->uaddr2 == NULL) tm_p = NULL; else { - error = umtx_copyin_umtx_time( + error = ops->copyin_umtx_time( uap->uaddr2, (size_t)uap->uaddr1, &timeout); if (error != 0) return (error); tm_p = &timeout; } - return (do_wait(td, uap->obj, uap->val, tm_p, 0, 0)); + return (do_wait(td, uap->obj, uap->val, tm_p, ops->compat32, 0)); } static int -__umtx_op_wait_uint(struct thread *td, struct _umtx_op_args *uap) +__umtx_op_wait_uint(struct thread *td, struct _umtx_op_args *uap, + const struct umtx_copyops *ops) { struct _umtx_time timeout, *tm_p; int error; @@ -3467,7 +3484,7 @@ __umtx_op_wait_uint(struct thread *td, struct _umtx_op if (uap->uaddr2 == NULL) tm_p = NULL; else { - error = umtx_copyin_umtx_time( + error = ops->copyin_umtx_time( uap->uaddr2, (size_t)uap->uaddr1, &timeout); if (error != 0) return (error); @@ -3477,7 +3494,8 @@ __umtx_op_wait_uint(struct thread *td, struct _umtx_op } static int -__umtx_op_wait_uint_private(struct thread *td, struct _umtx_op_args *uap) +__umtx_op_wait_uint_private(struct thread *td, struct _umtx_op_args *uap, + const struct umtx_copyops *ops) { struct _umtx_time *tm_p, timeout; int error; @@ -3485,7 +3503,7 @@ __umtx_op_wait_uint_private(struct thread *td, struct if (uap->uaddr2 == NULL) tm_p = NULL; else { - error = umtx_copyin_umtx_time( + error = ops->copyin_umtx_time( uap->uaddr2, (size_t)uap->uaddr1, &timeout); if (error != 0) return (error); @@ -3495,7 +3513,8 @@ __umtx_op_wait_uint_private(struct thread *td, struct } static int -__umtx_op_wake(struct thread *td, struct _umtx_op_args *uap) +__umtx_op_wake(struct thread *td, struct _umtx_op_args *uap, + const struct umtx_copyops *ops __unused) { return (kern_umtx_wake(td, uap->obj, uap->val, 0)); @@ -3503,7 +3522,7 @@ __umtx_op_wake(struct thread *td, struct _umtx_op_args #define BATCH_SIZE 128 static int -__umtx_op_nwake_private(struct thread *td, struct _umtx_op_args *uap) +__umtx_op_nwake_private_native(struct thread *td, struct _umtx_op_args *uap) { char *uaddrs[BATCH_SIZE], **upp; int count, error, i, pos, tocopy; @@ -3516,22 +3535,58 @@ __umtx_op_nwake_private(struct thread *td, struct _umt error = copyin(upp + pos, uaddrs, tocopy * sizeof(char *)); if (error != 0) break; - for (i = 0; i < tocopy; ++i) + for (i = 0; i < tocopy; ++i) { kern_umtx_wake(td, uaddrs[i], INT_MAX, 1); + } maybe_yield(); } return (error); } static int -__umtx_op_wake_private(struct thread *td, struct _umtx_op_args *uap) +__umtx_op_nwake_private_compat32(struct thread *td, struct _umtx_op_args *uap) { + uint32_t uaddrs[BATCH_SIZE], *upp; + int count, error, i, pos, tocopy; + upp = (uint32_t *)uap->obj; + error = 0; + for (count = uap->val, pos = 0; count > 0; count -= tocopy, + pos += tocopy) { + tocopy = MIN(count, BATCH_SIZE); + error = copyin(upp + pos, uaddrs, tocopy * sizeof(uint32_t)); + if (error != 0) + break; + for (i = 0; i < tocopy; ++i) { + kern_umtx_wake(td, (void *)(uintptr_t)uaddrs[i], + INT_MAX, 1); + } + maybe_yield(); + } + return (error); +} + +static int +__umtx_op_nwake_private(struct thread *td, struct _umtx_op_args *uap, + const struct umtx_copyops *ops) +{ + + if (ops->compat32) + return (__umtx_op_nwake_private_compat32(td, uap)); + return (__umtx_op_nwake_private_native(td, uap)); +} + +static int +__umtx_op_wake_private(struct thread *td, struct _umtx_op_args *uap, + const struct umtx_copyops *ops __unused) +{ + return (kern_umtx_wake(td, uap->obj, uap->val, 1)); } static int -__umtx_op_lock_umutex(struct thread *td, struct _umtx_op_args *uap) +__umtx_op_lock_umutex(struct thread *td, struct _umtx_op_args *uap, + const struct umtx_copyops *ops) { struct _umtx_time *tm_p, timeout; int error; @@ -3540,7 +3595,7 @@ __umtx_op_lock_umutex(struct thread *td, struct _umtx_ if (uap->uaddr2 == NULL) tm_p = NULL; else { - error = umtx_copyin_umtx_time( + error = ops->copyin_umtx_time( uap->uaddr2, (size_t)uap->uaddr1, &timeout); if (error != 0) return (error); @@ -3550,14 +3605,16 @@ __umtx_op_lock_umutex(struct thread *td, struct _umtx_ } static int -__umtx_op_trylock_umutex(struct thread *td, struct _umtx_op_args *uap) +__umtx_op_trylock_umutex(struct thread *td, struct _umtx_op_args *uap, + const struct umtx_copyops *ops __unused) { return (do_lock_umutex(td, uap->obj, NULL, _UMUTEX_TRY)); } static int -__umtx_op_wait_umutex(struct thread *td, struct _umtx_op_args *uap) +__umtx_op_wait_umutex(struct thread *td, struct _umtx_op_args *uap, + const struct umtx_copyops *ops) { struct _umtx_time *tm_p, timeout; int error; @@ -3566,7 +3623,7 @@ __umtx_op_wait_umutex(struct thread *td, struct _umtx_ if (uap->uaddr2 == NULL) tm_p = NULL; else { - error = umtx_copyin_umtx_time( + error = ops->copyin_umtx_time( uap->uaddr2, (size_t)uap->uaddr1, &timeout); if (error != 0) return (error); @@ -3576,28 +3633,32 @@ __umtx_op_wait_umutex(struct thread *td, struct _umtx_ } static int -__umtx_op_wake_umutex(struct thread *td, struct _umtx_op_args *uap) +__umtx_op_wake_umutex(struct thread *td, struct _umtx_op_args *uap, + const struct umtx_copyops *ops __unused) { return (do_wake_umutex(td, uap->obj)); } static int -__umtx_op_unlock_umutex(struct thread *td, struct _umtx_op_args *uap) +__umtx_op_unlock_umutex(struct thread *td, struct _umtx_op_args *uap, + const struct umtx_copyops *ops __unused) { return (do_unlock_umutex(td, uap->obj, false)); } static int -__umtx_op_set_ceiling(struct thread *td, struct _umtx_op_args *uap) +__umtx_op_set_ceiling(struct thread *td, struct _umtx_op_args *uap, + const struct umtx_copyops *ops __unused) { return (do_set_ceiling(td, uap->obj, uap->val, uap->uaddr1)); } static int -__umtx_op_cv_wait(struct thread *td, struct _umtx_op_args *uap) +__umtx_op_cv_wait(struct thread *td, struct _umtx_op_args *uap, + const struct umtx_copyops *ops) { struct timespec *ts, timeout; int error; @@ -3606,7 +3667,7 @@ __umtx_op_cv_wait(struct thread *td, struct _umtx_op_a if (uap->uaddr2 == NULL) ts = NULL; else { - error = umtx_copyin_timeout(uap->uaddr2, &timeout); + error = ops->copyin_timeout(uap->uaddr2, &timeout); if (error != 0) return (error); ts = &timeout; @@ -3615,21 +3676,24 @@ __umtx_op_cv_wait(struct thread *td, struct _umtx_op_a } static int -__umtx_op_cv_signal(struct thread *td, struct _umtx_op_args *uap) +__umtx_op_cv_signal(struct thread *td, struct _umtx_op_args *uap, + const struct umtx_copyops *ops __unused) { return (do_cv_signal(td, uap->obj)); } static int -__umtx_op_cv_broadcast(struct thread *td, struct _umtx_op_args *uap) +__umtx_op_cv_broadcast(struct thread *td, struct _umtx_op_args *uap, + const struct umtx_copyops *ops __unused) { return (do_cv_broadcast(td, uap->obj)); } static int -__umtx_op_rw_rdlock(struct thread *td, struct _umtx_op_args *uap) +__umtx_op_rw_rdlock(struct thread *td, struct _umtx_op_args *uap, + const struct umtx_copyops *ops) { struct _umtx_time timeout; int error; @@ -3638,7 +3702,7 @@ __umtx_op_rw_rdlock(struct thread *td, struct _umtx_op if (uap->uaddr2 == NULL) { error = do_rw_rdlock(td, uap->obj, uap->val, 0); } else { - error = umtx_copyin_umtx_time(uap->uaddr2, + error = ops->copyin_umtx_time(uap->uaddr2, (size_t)uap->uaddr1, &timeout); if (error != 0) return (error); @@ -3648,7 +3712,8 @@ __umtx_op_rw_rdlock(struct thread *td, struct _umtx_op } static int -__umtx_op_rw_wrlock(struct thread *td, struct _umtx_op_args *uap) +__umtx_op_rw_wrlock(struct thread *td, struct _umtx_op_args *uap, + const struct umtx_copyops *ops) { struct _umtx_time timeout; int error; @@ -3657,7 +3722,7 @@ __umtx_op_rw_wrlock(struct thread *td, struct _umtx_op if (uap->uaddr2 == NULL) { error = do_rw_wrlock(td, uap->obj, 0); } else { - error = umtx_copyin_umtx_time(uap->uaddr2, + error = ops->copyin_umtx_time(uap->uaddr2, (size_t)uap->uaddr1, &timeout); if (error != 0) return (error); @@ -3668,7 +3733,8 @@ __umtx_op_rw_wrlock(struct thread *td, struct _umtx_op } static int -__umtx_op_rw_unlock(struct thread *td, struct _umtx_op_args *uap) +__umtx_op_rw_unlock(struct thread *td, struct _umtx_op_args *uap, + const struct umtx_copyops *ops __unused) { return (do_rw_unlock(td, uap->obj)); @@ -3676,7 +3742,8 @@ __umtx_op_rw_unlock(struct thread *td, struct _umtx_op #if defined(COMPAT_FREEBSD9) || defined(COMPAT_FREEBSD10) static int -__umtx_op_sem_wait(struct thread *td, struct _umtx_op_args *uap) +__umtx_op_sem_wait(struct thread *td, struct _umtx_op_args *uap, + const struct umtx_copyops *ops) { struct _umtx_time *tm_p, timeout; int error; @@ -3685,7 +3752,7 @@ __umtx_op_sem_wait(struct thread *td, struct _umtx_op_ if (uap->uaddr2 == NULL) tm_p = NULL; else { - error = umtx_copyin_umtx_time( + error = ops->copyin_umtx_time( uap->uaddr2, (size_t)uap->uaddr1, &timeout); if (error != 0) return (error); @@ -3695,7 +3762,8 @@ __umtx_op_sem_wait(struct thread *td, struct _umtx_op_ } static int -__umtx_op_sem_wake(struct thread *td, struct _umtx_op_args *uap) +__umtx_op_sem_wake(struct thread *td, struct _umtx_op_args *uap, + const struct umtx_copyops *ops __unused) { return (do_sem_wake(td, uap->obj)); @@ -3703,14 +3771,16 @@ __umtx_op_sem_wake(struct thread *td, struct _umtx_op_ #endif static int -__umtx_op_wake2_umutex(struct thread *td, struct _umtx_op_args *uap) +__umtx_op_wake2_umutex(struct thread *td, struct _umtx_op_args *uap, + const struct umtx_copyops *ops __unused) { return (do_wake2_umutex(td, uap->obj, uap->val)); } static int -__umtx_op_sem2_wait(struct thread *td, struct _umtx_op_args *uap) +__umtx_op_sem2_wait(struct thread *td, struct _umtx_op_args *uap, + const struct umtx_copyops *ops) { struct _umtx_time *tm_p, timeout; size_t uasize; @@ -3722,7 +3792,7 @@ __umtx_op_sem2_wait(struct thread *td, struct _umtx_op tm_p = NULL; } else { uasize = (size_t)uap->uaddr1; - error = umtx_copyin_umtx_time(uap->uaddr2, uasize, &timeout); + error = ops->copyin_umtx_time(uap->uaddr2, uasize, &timeout); if (error != 0) return (error); tm_p = &timeout; @@ -3730,10 +3800,10 @@ __umtx_op_sem2_wait(struct thread *td, struct _umtx_op error = do_sem2_wait(td, uap->obj, tm_p); if (error == EINTR && uap->uaddr2 != NULL && (timeout._flags & UMTX_ABSTIME) == 0 && - uasize >= sizeof(struct _umtx_time) + sizeof(struct timespec)) { - error = copyout(&timeout._timeout, - (struct _umtx_time *)uap->uaddr2 + 1, - sizeof(struct timespec)); + uasize >= ops->umtx_time_sz + ops->timespec_sz) { + error = ops->copyout_timeout( + (void *)((uintptr_t)uap->uaddr2 + ops->umtx_time_sz), + uasize - ops->umtx_time_sz, &timeout._timeout); if (error == 0) { error = EINTR; } @@ -3743,7 +3813,8 @@ __umtx_op_sem2_wait(struct thread *td, struct _umtx_op } static int -__umtx_op_sem2_wake(struct thread *td, struct _umtx_op_args *uap) +__umtx_op_sem2_wake(struct thread *td, struct _umtx_op_args *uap, + const struct umtx_copyops *ops __unused) { return (do_sem2_wake(td, uap->obj)); @@ -4050,418 +4121,203 @@ umtx_shm(struct thread *td, void *addr, u_int flags) } static int -__umtx_op_shm(struct thread *td, struct _umtx_op_args *uap) +__umtx_op_shm(struct thread *td, struct _umtx_op_args *uap, + const struct umtx_copyops *ops __unused) { return (umtx_shm(td, uap->uaddr1, uap->val)); } static int -umtx_robust_lists(struct thread *td, struct umtx_robust_lists_params *rbp) +__umtx_op_robust_lists(struct thread *td, struct _umtx_op_args *uap, + const struct umtx_copyops *ops) { - - td->td_rb_list = rbp->robust_list_offset; - td->td_rbp_list = rbp->robust_priv_list_offset; - td->td_rb_inact = rbp->robust_inact_offset; - return (0); -} - -static int -__umtx_op_robust_lists(struct thread *td, struct _umtx_op_args *uap) -{ struct umtx_robust_lists_params rb; int error; - if (uap->val > sizeof(rb)) - return (EINVAL); bzero(&rb, sizeof(rb)); - error = copyin(uap->uaddr1, &rb, uap->val); + error = ops->copyin_robust_lists(uap->uaddr1, uap->val, &rb); if (error != 0) return (error); - return (umtx_robust_lists(td, &rb)); -} -typedef int (*_umtx_op_func)(struct thread *td, struct _umtx_op_args *uap); + if (ops->compat32) + td->td_pflags2 |= TDP2_COMPAT32RB; + else if ((td->td_pflags2 & TDP2_COMPAT32RB) != 0) + return (EINVAL); -static const _umtx_op_func op_table[] = { - [UMTX_OP_RESERVED0] = __umtx_op_unimpl, - [UMTX_OP_RESERVED1] = __umtx_op_unimpl, - [UMTX_OP_WAIT] = __umtx_op_wait, - [UMTX_OP_WAKE] = __umtx_op_wake, - [UMTX_OP_MUTEX_TRYLOCK] = __umtx_op_trylock_umutex, - [UMTX_OP_MUTEX_LOCK] = __umtx_op_lock_umutex, - [UMTX_OP_MUTEX_UNLOCK] = __umtx_op_unlock_umutex, - [UMTX_OP_SET_CEILING] = __umtx_op_set_ceiling, - [UMTX_OP_CV_WAIT] = __umtx_op_cv_wait, - [UMTX_OP_CV_SIGNAL] = __umtx_op_cv_signal, - [UMTX_OP_CV_BROADCAST] = __umtx_op_cv_broadcast, - [UMTX_OP_WAIT_UINT] = __umtx_op_wait_uint, - [UMTX_OP_RW_RDLOCK] = __umtx_op_rw_rdlock, - [UMTX_OP_RW_WRLOCK] = __umtx_op_rw_wrlock, - [UMTX_OP_RW_UNLOCK] = __umtx_op_rw_unlock, - [UMTX_OP_WAIT_UINT_PRIVATE] = __umtx_op_wait_uint_private, - [UMTX_OP_WAKE_PRIVATE] = __umtx_op_wake_private, - [UMTX_OP_MUTEX_WAIT] = __umtx_op_wait_umutex, - [UMTX_OP_MUTEX_WAKE] = __umtx_op_wake_umutex, -#if defined(COMPAT_FREEBSD9) || defined(COMPAT_FREEBSD10) - [UMTX_OP_SEM_WAIT] = __umtx_op_sem_wait, - [UMTX_OP_SEM_WAKE] = __umtx_op_sem_wake, -#else - [UMTX_OP_SEM_WAIT] = __umtx_op_unimpl, - [UMTX_OP_SEM_WAKE] = __umtx_op_unimpl, -#endif - [UMTX_OP_NWAKE_PRIVATE] = __umtx_op_nwake_private, - [UMTX_OP_MUTEX_WAKE2] = __umtx_op_wake2_umutex, - [UMTX_OP_SEM2_WAIT] = __umtx_op_sem2_wait, - [UMTX_OP_SEM2_WAKE] = __umtx_op_sem2_wake, - [UMTX_OP_SHM] = __umtx_op_shm, - [UMTX_OP_ROBUST_LISTS] = __umtx_op_robust_lists, -}; - -int -sys__umtx_op(struct thread *td, struct _umtx_op_args *uap) -{ - - if ((unsigned)uap->op < nitems(op_table)) - return (*op_table[uap->op])(td, uap); - return (EINVAL); + td->td_rb_list = rb.robust_list_offset; + td->td_rbp_list = rb.robust_priv_list_offset; + td->td_rb_inact = rb.robust_inact_offset; + return (0); } #ifdef COMPAT_FREEBSD32 - -struct umtx_time32 { - struct timespec32 timeout; - uint32_t flags; - uint32_t clockid; -}; - static inline int -umtx_copyin_timeout32(void *addr, struct timespec *tsp) +umtx_copyin_timeout32(const void *uaddr, struct timespec *tsp) { struct timespec32 ts32; int error; - error = copyin(addr, &ts32, sizeof(struct timespec32)); + error = copyin(uaddr, &ts32, sizeof(ts32)); if (error == 0) { if (ts32.tv_sec < 0 || ts32.tv_nsec >= 1000000000 || ts32.tv_nsec < 0) error = EINVAL; else { - tsp->tv_sec = ts32.tv_sec; - tsp->tv_nsec = ts32.tv_nsec; + CP(ts32, *tsp, tv_sec); + CP(ts32, *tsp, tv_nsec); } } return (error); } static inline int -umtx_copyin_umtx_time32(const void *addr, size_t size, struct _umtx_time *tp) +umtx_copyin_umtx_time32(const void *uaddr, size_t size, struct _umtx_time *tp) { struct umtx_time32 t32; int error; - t32.clockid = CLOCK_REALTIME; - t32.flags = 0; - if (size <= sizeof(struct timespec32)) - error = copyin(addr, &t32.timeout, sizeof(struct timespec32)); + t32._clockid = CLOCK_REALTIME; + t32._flags = 0; + if (size <= sizeof(t32._timeout)) + error = copyin(uaddr, &t32._timeout, sizeof(t32._timeout)); else - error = copyin(addr, &t32, sizeof(struct umtx_time32)); + error = copyin(uaddr, &t32, sizeof(t32)); if (error != 0) return (error); - if (t32.timeout.tv_sec < 0 || - t32.timeout.tv_nsec >= 1000000000 || t32.timeout.tv_nsec < 0) + if (t32._timeout.tv_sec < 0 || + t32._timeout.tv_nsec >= 1000000000 || t32._timeout.tv_nsec < 0) return (EINVAL); - tp->_timeout.tv_sec = t32.timeout.tv_sec; - tp->_timeout.tv_nsec = t32.timeout.tv_nsec; - tp->_flags = t32.flags; - tp->_clockid = t32.clockid; + TS_CP(t32, *tp, _timeout); + CP(t32, *tp, _flags); + CP(t32, *tp, _clockid); return (0); } static int -__umtx_op_wait_compat32(struct thread *td, struct _umtx_op_args *uap) +umtx_copyin_robust_lists32(const void *uaddr, size_t size, + struct umtx_robust_lists_params *rbp) { - struct _umtx_time *tm_p, timeout; + struct umtx_robust_lists_params_compat32 rb32; int error; - if (uap->uaddr2 == NULL) - tm_p = NULL; - else { - error = umtx_copyin_umtx_time32(uap->uaddr2, - (size_t)uap->uaddr1, &timeout); - if (error != 0) - return (error); - tm_p = &timeout; - } - return (do_wait(td, uap->obj, uap->val, tm_p, 1, 0)); + if (size > sizeof(rb32)) + return (EINVAL); + bzero(&rb32, sizeof(rb32)); + error = copyin(uaddr, &rb32, size); + if (error != 0) + return (error); + CP(rb32, *rbp, robust_list_offset); + CP(rb32, *rbp, robust_priv_list_offset); + CP(rb32, *rbp, robust_inact_offset); + return (0); } static int -__umtx_op_lock_umutex_compat32(struct thread *td, struct _umtx_op_args *uap) +umtx_copyout_timeout32(void *uaddr, size_t sz, struct timespec *tsp) { - struct _umtx_time *tm_p, timeout; - int error; + struct timespec32 remain32 = { + .tv_sec = tsp->tv_sec, + .tv_nsec = tsp->tv_nsec, + }; - /* Allow a null timespec (wait forever). */ - if (uap->uaddr2 == NULL) - tm_p = NULL; - else { - error = umtx_copyin_umtx_time32(uap->uaddr2, - (size_t)uap->uaddr1, &timeout); - if (error != 0) - return (error); - tm_p = &timeout; - } - return (do_lock_umutex(td, uap->obj, tm_p, 0)); -} + /* + * Should be guaranteed by the caller, sz == uaddr1 - sizeof(_umtx_time) + * and we're only called if sz >= sizeof(timespec) as supplied in the + * copyops. + */ + KASSERT(sz >= sizeof(remain32), + ("umtx_copyops specifies incorrect sizes")); -static int -__umtx_op_wait_umutex_compat32(struct thread *td, struct _umtx_op_args *uap) -{ - struct _umtx_time *tm_p, timeout; - int error; - - /* Allow a null timespec (wait forever). */ - if (uap->uaddr2 == NULL) - tm_p = NULL; - else { - error = umtx_copyin_umtx_time32(uap->uaddr2, - (size_t)uap->uaddr1, &timeout); - if (error != 0) - return (error); - tm_p = &timeout; - } - return (do_lock_umutex(td, uap->obj, tm_p, _UMUTEX_WAIT)); + return (copyout(&remain32, uaddr, sizeof(remain32))); } +#endif /* COMPAT_FREEBSD32 */ -static int -__umtx_op_cv_wait_compat32(struct thread *td, struct _umtx_op_args *uap) -{ - struct timespec *ts, timeout; - int error; +typedef int (*_umtx_op_func)(struct thread *td, struct _umtx_op_args *uap, + const struct umtx_copyops *umtx_ops); - /* Allow a null timespec (wait forever). */ - if (uap->uaddr2 == NULL) - ts = NULL; - else { - error = umtx_copyin_timeout32(uap->uaddr2, &timeout); - if (error != 0) - return (error); - ts = &timeout; - } - return (do_cv_wait(td, uap->obj, uap->uaddr1, ts, uap->val)); -} - -static int -__umtx_op_rw_rdlock_compat32(struct thread *td, struct _umtx_op_args *uap) -{ - struct _umtx_time timeout; - int error; - - /* Allow a null timespec (wait forever). */ - if (uap->uaddr2 == NULL) { - error = do_rw_rdlock(td, uap->obj, uap->val, 0); - } else { - error = umtx_copyin_umtx_time32(uap->uaddr2, - (size_t)uap->uaddr1, &timeout); - if (error != 0) - return (error); - error = do_rw_rdlock(td, uap->obj, uap->val, &timeout); - } - return (error); -} - -static int -__umtx_op_rw_wrlock_compat32(struct thread *td, struct _umtx_op_args *uap) -{ - struct _umtx_time timeout; - int error; - - /* Allow a null timespec (wait forever). */ - if (uap->uaddr2 == NULL) { - error = do_rw_wrlock(td, uap->obj, 0); - } else { - error = umtx_copyin_umtx_time32(uap->uaddr2, - (size_t)uap->uaddr1, &timeout); - if (error != 0) - return (error); - error = do_rw_wrlock(td, uap->obj, &timeout); - } - return (error); -} - -static int -__umtx_op_wait_uint_private_compat32(struct thread *td, struct _umtx_op_args *uap) -{ - struct _umtx_time *tm_p, timeout; - int error; - - if (uap->uaddr2 == NULL) - tm_p = NULL; - else { - error = umtx_copyin_umtx_time32( - uap->uaddr2, (size_t)uap->uaddr1,&timeout); - if (error != 0) - return (error); - tm_p = &timeout; - } - return (do_wait(td, uap->obj, uap->val, tm_p, 1, 1)); -} - -#if defined(COMPAT_FREEBSD9) || defined(COMPAT_FREEBSD10) -static int -__umtx_op_sem_wait_compat32(struct thread *td, struct _umtx_op_args *uap) -{ - struct _umtx_time *tm_p, timeout; - int error; - - /* Allow a null timespec (wait forever). */ - if (uap->uaddr2 == NULL) - tm_p = NULL; - else { - error = umtx_copyin_umtx_time32(uap->uaddr2, - (size_t)uap->uaddr1, &timeout); - if (error != 0) - return (error); - tm_p = &timeout; - } - return (do_sem_wait(td, uap->obj, tm_p)); -} -#endif - -static int -__umtx_op_sem2_wait_compat32(struct thread *td, struct _umtx_op_args *uap) -{ - struct _umtx_time *tm_p, timeout; - size_t uasize; - int error; - - /* Allow a null timespec (wait forever). */ - if (uap->uaddr2 == NULL) { - uasize = 0; - tm_p = NULL; - } else { - uasize = (size_t)uap->uaddr1; - error = umtx_copyin_umtx_time32(uap->uaddr2, uasize, &timeout); - if (error != 0) - return (error); - tm_p = &timeout; - } - error = do_sem2_wait(td, uap->obj, tm_p); - if (error == EINTR && uap->uaddr2 != NULL && - (timeout._flags & UMTX_ABSTIME) == 0 && - uasize >= sizeof(struct umtx_time32) + sizeof(struct timespec32)) { - struct timespec32 remain32 = { - .tv_sec = timeout._timeout.tv_sec, - .tv_nsec = timeout._timeout.tv_nsec - }; - error = copyout(&remain32, - (struct umtx_time32 *)uap->uaddr2 + 1, - sizeof(struct timespec32)); - if (error == 0) { - error = EINTR; - } - } - - return (error); -} - -static int -__umtx_op_nwake_private32(struct thread *td, struct _umtx_op_args *uap) -{ - uint32_t uaddrs[BATCH_SIZE], *upp; - int count, error, i, pos, tocopy; - - upp = (uint32_t *)uap->obj; - error = 0; - for (count = uap->val, pos = 0; count > 0; count -= tocopy, - pos += tocopy) { - tocopy = MIN(count, BATCH_SIZE); - error = copyin(upp + pos, uaddrs, tocopy * sizeof(uint32_t)); - if (error != 0) - break; - for (i = 0; i < tocopy; ++i) - kern_umtx_wake(td, (void *)(uintptr_t)uaddrs[i], - INT_MAX, 1); - maybe_yield(); - } - return (error); -} - -struct umtx_robust_lists_params_compat32 { - uint32_t robust_list_offset; - uint32_t robust_priv_list_offset; - uint32_t robust_inact_offset; -}; - -static int -__umtx_op_robust_lists_compat32(struct thread *td, struct _umtx_op_args *uap) -{ - struct umtx_robust_lists_params rb; - struct umtx_robust_lists_params_compat32 rb32; - int error; - - if (uap->val > sizeof(rb32)) - return (EINVAL); - bzero(&rb, sizeof(rb)); - bzero(&rb32, sizeof(rb32)); - error = copyin(uap->uaddr1, &rb32, uap->val); - if (error != 0) - return (error); - rb.robust_list_offset = rb32.robust_list_offset; - rb.robust_priv_list_offset = rb32.robust_priv_list_offset; - rb.robust_inact_offset = rb32.robust_inact_offset; - return (umtx_robust_lists(td, &rb)); -} - -static const _umtx_op_func op_table_compat32[] = { +static const _umtx_op_func op_table[] = { [UMTX_OP_RESERVED0] = __umtx_op_unimpl, [UMTX_OP_RESERVED1] = __umtx_op_unimpl, - [UMTX_OP_WAIT] = __umtx_op_wait_compat32, + [UMTX_OP_WAIT] = __umtx_op_wait, [UMTX_OP_WAKE] = __umtx_op_wake, [UMTX_OP_MUTEX_TRYLOCK] = __umtx_op_trylock_umutex, - [UMTX_OP_MUTEX_LOCK] = __umtx_op_lock_umutex_compat32, + [UMTX_OP_MUTEX_LOCK] = __umtx_op_lock_umutex, [UMTX_OP_MUTEX_UNLOCK] = __umtx_op_unlock_umutex, [UMTX_OP_SET_CEILING] = __umtx_op_set_ceiling, - [UMTX_OP_CV_WAIT] = __umtx_op_cv_wait_compat32, + [UMTX_OP_CV_WAIT] = __umtx_op_cv_wait, [UMTX_OP_CV_SIGNAL] = __umtx_op_cv_signal, [UMTX_OP_CV_BROADCAST] = __umtx_op_cv_broadcast, - [UMTX_OP_WAIT_UINT] = __umtx_op_wait_compat32, - [UMTX_OP_RW_RDLOCK] = __umtx_op_rw_rdlock_compat32, - [UMTX_OP_RW_WRLOCK] = __umtx_op_rw_wrlock_compat32, + [UMTX_OP_WAIT_UINT] = __umtx_op_wait_uint, + [UMTX_OP_RW_RDLOCK] = __umtx_op_rw_rdlock, + [UMTX_OP_RW_WRLOCK] = __umtx_op_rw_wrlock, [UMTX_OP_RW_UNLOCK] = __umtx_op_rw_unlock, - [UMTX_OP_WAIT_UINT_PRIVATE] = __umtx_op_wait_uint_private_compat32, + [UMTX_OP_WAIT_UINT_PRIVATE] = __umtx_op_wait_uint_private, [UMTX_OP_WAKE_PRIVATE] = __umtx_op_wake_private, - [UMTX_OP_MUTEX_WAIT] = __umtx_op_wait_umutex_compat32, + [UMTX_OP_MUTEX_WAIT] = __umtx_op_wait_umutex, [UMTX_OP_MUTEX_WAKE] = __umtx_op_wake_umutex, #if defined(COMPAT_FREEBSD9) || defined(COMPAT_FREEBSD10) - [UMTX_OP_SEM_WAIT] = __umtx_op_sem_wait_compat32, + [UMTX_OP_SEM_WAIT] = __umtx_op_sem_wait, [UMTX_OP_SEM_WAKE] = __umtx_op_sem_wake, #else [UMTX_OP_SEM_WAIT] = __umtx_op_unimpl, [UMTX_OP_SEM_WAKE] = __umtx_op_unimpl, #endif - [UMTX_OP_NWAKE_PRIVATE] = __umtx_op_nwake_private32, + [UMTX_OP_NWAKE_PRIVATE] = __umtx_op_nwake_private, [UMTX_OP_MUTEX_WAKE2] = __umtx_op_wake2_umutex, - [UMTX_OP_SEM2_WAIT] = __umtx_op_sem2_wait_compat32, + [UMTX_OP_SEM2_WAIT] = __umtx_op_sem2_wait, [UMTX_OP_SEM2_WAKE] = __umtx_op_sem2_wake, [UMTX_OP_SHM] = __umtx_op_shm, - [UMTX_OP_ROBUST_LISTS] = __umtx_op_robust_lists_compat32, + [UMTX_OP_ROBUST_LISTS] = __umtx_op_robust_lists, }; +static const struct umtx_copyops umtx_native_ops = { + .copyin_timeout = umtx_copyin_timeout, + .copyin_umtx_time = umtx_copyin_umtx_time, + .copyin_robust_lists = umtx_copyin_robust_lists, + .copyout_timeout = umtx_copyout_timeout, + .timespec_sz = sizeof(struct timespec), + .umtx_time_sz = sizeof(struct _umtx_time), +}; + *** DIFF OUTPUT TRUNCATED AT 1000 LINES *** _______________________________________________ svn-src-head@freebsd.org mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-head To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"
