> On 3 Sep 2022, at 23:47, Alexander Bluhm <[email protected]> wrote:
>
> Hi,
>
> The next small step towards parallel network stack is to use shared
> netlock in soreceive(). The UDP and IP divert layer provide locking
> of the PCB. If that is possible, use shared instead of exclusive
> netlock in soreceive(). The PCB mutex provides a per socket lock
> against multiple soreceive() running in parallel.
>
> The solock_shared() is a bit hacky. Especially where we have to
> release and regrab both locks in sosleep_nsec(). But it does what
> we need.
>
> The udp_input() keeps exclusive lock for now. Socket splicing and
> IP header chains are not completely MP safe yet. Also udp_input()
> has some tentacles that need more testing.
>
> Ok to procceed with soreceive() unlocking before release? I think
> udp_input() has to wait post 7.2.
>
I have no objections.
> bluhm
>
> Index: kern/uipc_socket.c
> ===================================================================
> RCS file: /data/mirror/openbsd/cvs/src/sys/kern/uipc_socket.c,v
> retrieving revision 1.287
> diff -u -p -r1.287 uipc_socket.c
> --- kern/uipc_socket.c 3 Sep 2022 13:29:33 -0000 1.287
> +++ kern/uipc_socket.c 3 Sep 2022 19:23:43 -0000
> @@ -822,10 +822,10 @@ bad:
> if (mp)
> *mp = NULL;
>
> - solock(so);
> + solock_shared(so);
> restart:
> if ((error = sblock(so, &so->so_rcv, SBLOCKWAIT(flags))) != 0) {
> - sounlock(so);
> + sounlock_shared(so);
> return (error);
> }
>
> @@ -893,7 +893,7 @@ restart:
> sbunlock(so, &so->so_rcv);
> error = sbwait(so, &so->so_rcv);
> if (error) {
> - sounlock(so);
> + sounlock_shared(so);
> return (error);
> }
> goto restart;
> @@ -962,11 +962,11 @@ dontblock:
> sbsync(&so->so_rcv, nextrecord);
> if (controlp) {
> if (pr->pr_domain->dom_externalize) {
> - sounlock(so);
> + sounlock_shared(so);
> error =
> (*pr->pr_domain->dom_externalize)
> (cm, controllen, flags);
> - solock(so);
> + solock_shared(so);
> }
> *controlp = cm;
> } else {
> @@ -1040,9 +1040,9 @@ dontblock:
> SBLASTRECORDCHK(&so->so_rcv, "soreceive uiomove");
> SBLASTMBUFCHK(&so->so_rcv, "soreceive uiomove");
> resid = uio->uio_resid;
> - sounlock(so);
> + sounlock_shared(so);
> uio_error = uiomove(mtod(m, caddr_t) + moff, len, uio);
> - solock(so);
> + solock_shared(so);
> if (uio_error)
> uio->uio_resid = resid - len;
> } else
> @@ -1126,7 +1126,7 @@ dontblock:
> error = sbwait(so, &so->so_rcv);
> if (error) {
> sbunlock(so, &so->so_rcv);
> - sounlock(so);
> + sounlock_shared(so);
> return (0);
> }
> if ((m = so->so_rcv.sb_mb) != NULL)
> @@ -1171,7 +1171,7 @@ dontblock:
> *flagsp |= flags;
> release:
> sbunlock(so, &so->so_rcv);
> - sounlock(so);
> + sounlock_shared(so);
> return (error);
> }
>
> Index: kern/uipc_socket2.c
> ===================================================================
> RCS file: /data/mirror/openbsd/cvs/src/sys/kern/uipc_socket2.c,v
> retrieving revision 1.127
> diff -u -p -r1.127 uipc_socket2.c
> --- kern/uipc_socket2.c 13 Aug 2022 21:01:46 -0000 1.127
> +++ kern/uipc_socket2.c 3 Sep 2022 19:23:43 -0000
> @@ -360,6 +360,24 @@ solock(struct socket *so)
> }
> }
>
> +void
> +solock_shared(struct socket *so)
> +{
> + switch (so->so_proto->pr_domain->dom_family) {
> + case PF_INET:
> + case PF_INET6:
> + if (so->so_proto->pr_usrreqs->pru_lock != NULL) {
> + NET_LOCK_SHARED();
> + pru_lock(so);
> + } else
> + NET_LOCK();
> + break;
> + default:
> + rw_enter_write(&so->so_lock);
> + break;
> + }
> +}
> +
> int
> solock_persocket(struct socket *so)
> {
> @@ -403,6 +421,24 @@ sounlock(struct socket *so)
> }
>
> void
> +sounlock_shared(struct socket *so)
> +{
> + switch (so->so_proto->pr_domain->dom_family) {
> + case PF_INET:
> + case PF_INET6:
> + if (so->so_proto->pr_usrreqs->pru_unlock != NULL) {
> + pru_unlock(so);
> + NET_UNLOCK_SHARED();
> + } else
> + NET_UNLOCK();
> + break;
> + default:
> + rw_exit_write(&so->so_lock);
> + break;
> + }
> +}
> +
> +void
> soassertlocked(struct socket *so)
> {
> switch (so->so_proto->pr_domain->dom_family) {
> @@ -425,7 +461,15 @@ sosleep_nsec(struct socket *so, void *id
> switch (so->so_proto->pr_domain->dom_family) {
> case PF_INET:
> case PF_INET6:
> + if (so->so_proto->pr_usrreqs->pru_unlock != NULL &&
> + rw_status(&netlock) == RW_READ) {
> + pru_unlock(so);
> + }
> ret = rwsleep_nsec(ident, &netlock, prio, wmesg, nsecs);
> + if (so->so_proto->pr_usrreqs->pru_lock != NULL &&
> + rw_status(&netlock) == RW_READ) {
> + pru_lock(so);
> + }
> break;
> default:
> ret = rwsleep_nsec(ident, &so->so_lock, prio, wmesg, nsecs);
> Index: netinet/ip_divert.c
> ===================================================================
> RCS file: /data/mirror/openbsd/cvs/src/sys/netinet/ip_divert.c,v
> retrieving revision 1.85
> diff -u -p -r1.85 ip_divert.c
> --- netinet/ip_divert.c 3 Sep 2022 18:48:50 -0000 1.85
> +++ netinet/ip_divert.c 3 Sep 2022 19:23:43 -0000
> @@ -66,6 +66,8 @@ const struct pr_usrreqs divert_usrreqs =
> .pru_usrreq = divert_usrreq,
> .pru_attach = divert_attach,
> .pru_detach = divert_detach,
> + .pru_lock = divert_lock,
> + .pru_unlock = divert_unlock,
> .pru_bind = divert_bind,
> .pru_shutdown = divert_shutdown,
> .pru_send = divert_send,
> @@ -325,6 +327,24 @@ divert_detach(struct socket *so)
>
> in_pcbdetach(inp);
> return (0);
> +}
> +
> +void
> +divert_lock(struct socket *so)
> +{
> + struct inpcb *inp = sotoinpcb(so);
> +
> + NET_ASSERT_LOCKED();
> + mtx_enter(&inp->inp_mtx);
> +}
> +
> +void
> +divert_unlock(struct socket *so)
> +{
> + struct inpcb *inp = sotoinpcb(so);
> +
> + NET_ASSERT_LOCKED();
> + mtx_leave(&inp->inp_mtx);
> }
>
> int
> Index: netinet/ip_divert.h
> ===================================================================
> RCS file: /data/mirror/openbsd/cvs/src/sys/netinet/ip_divert.h,v
> retrieving revision 1.20
> diff -u -p -r1.20 ip_divert.h
> --- netinet/ip_divert.h 28 Aug 2022 18:44:16 -0000 1.20
> +++ netinet/ip_divert.h 3 Sep 2022 19:23:43 -0000
> @@ -74,6 +74,8 @@ int divert_usrreq(struct socket *,
> int, struct mbuf *, struct mbuf *, struct mbuf *, struct proc *);
> int divert_attach(struct socket *, int);
> int divert_detach(struct socket *);
> +void divert_lock(struct socket *);
> +void divert_unlock(struct socket *);
> int divert_bind(struct socket *, struct mbuf *, struct proc *);
> int divert_shutdown(struct socket *);
> int divert_send(struct socket *, struct mbuf *, struct mbuf *,
> Index: netinet/udp_usrreq.c
> ===================================================================
> RCS file: /data/mirror/openbsd/cvs/src/sys/netinet/udp_usrreq.c,v
> retrieving revision 1.300
> diff -u -p -r1.300 udp_usrreq.c
> --- netinet/udp_usrreq.c 3 Sep 2022 18:48:50 -0000 1.300
> +++ netinet/udp_usrreq.c 3 Sep 2022 19:23:43 -0000
> @@ -126,6 +126,8 @@ const struct pr_usrreqs udp_usrreqs = {
> .pru_usrreq = udp_usrreq,
> .pru_attach = udp_attach,
> .pru_detach = udp_detach,
> + .pru_lock = udp_lock,
> + .pru_unlock = udp_unlock,
> .pru_bind = udp_bind,
> .pru_connect = udp_connect,
> .pru_disconnect = udp_disconnect,
> @@ -141,6 +143,8 @@ const struct pr_usrreqs udp6_usrreqs = {
> .pru_usrreq = udp_usrreq,
> .pru_attach = udp_attach,
> .pru_detach = udp_detach,
> + .pru_lock = udp_lock,
> + .pru_unlock = udp_unlock,
> .pru_bind = udp_bind,
> .pru_connect = udp_connect,
> .pru_disconnect = udp_disconnect,
> @@ -1159,6 +1163,24 @@ udp_detach(struct socket *so)
>
> in_pcbdetach(inp);
> return (0);
> +}
> +
> +void
> +udp_lock(struct socket *so)
> +{
> + struct inpcb *inp = sotoinpcb(so);
> +
> + NET_ASSERT_LOCKED();
> + mtx_enter(&inp->inp_mtx);
> +}
> +
> +void
> +udp_unlock(struct socket *so)
> +{
> + struct inpcb *inp = sotoinpcb(so);
> +
> + NET_ASSERT_LOCKED();
> + mtx_leave(&inp->inp_mtx);
> }
>
> int
> Index: netinet/udp_var.h
> ===================================================================
> RCS file: /data/mirror/openbsd/cvs/src/sys/netinet/udp_var.h,v
> retrieving revision 1.45
> diff -u -p -r1.45 udp_var.h
> --- netinet/udp_var.h 2 Sep 2022 13:12:32 -0000 1.45
> +++ netinet/udp_var.h 3 Sep 2022 19:23:43 -0000
> @@ -147,6 +147,8 @@ int udp_usrreq(struct socket *,
> int, struct mbuf *, struct mbuf *, struct mbuf *, struct proc *);
> int udp_attach(struct socket *, int);
> int udp_detach(struct socket *);
> +void udp_lock(struct socket *);
> +void udp_unlock(struct socket *);
> int udp_bind(struct socket *, struct mbuf *, struct proc *);
> int udp_connect(struct socket *, struct mbuf *);
> int udp_disconnect(struct socket *);
> Index: netinet6/ip6_divert.c
> ===================================================================
> RCS file: /data/mirror/openbsd/cvs/src/sys/netinet6/ip6_divert.c,v
> retrieving revision 1.84
> diff -u -p -r1.84 ip6_divert.c
> --- netinet6/ip6_divert.c 3 Sep 2022 18:48:50 -0000 1.84
> +++ netinet6/ip6_divert.c 3 Sep 2022 19:23:43 -0000
> @@ -67,6 +67,8 @@ const struct pr_usrreqs divert6_usrreqs
> .pru_usrreq = divert6_usrreq,
> .pru_attach = divert6_attach,
> .pru_detach = divert6_detach,
> + .pru_lock = divert6_lock,
> + .pru_unlock = divert6_unlock,
> .pru_bind = divert6_bind,
> .pru_shutdown = divert6_shutdown,
> .pru_send = divert6_send,
> @@ -332,6 +334,24 @@ divert6_detach(struct socket *so)
> in_pcbdetach(inp);
>
> return (0);
> +}
> +
> +void
> +divert6_lock(struct socket *so)
> +{
> + struct inpcb *inp = sotoinpcb(so);
> +
> + NET_ASSERT_LOCKED();
> + mtx_enter(&inp->inp_mtx);
> +}
> +
> +void
> +divert6_unlock(struct socket *so)
> +{
> + struct inpcb *inp = sotoinpcb(so);
> +
> + NET_ASSERT_LOCKED();
> + mtx_leave(&inp->inp_mtx);
> }
>
> int
> Index: netinet6/ip6_divert.h
> ===================================================================
> RCS file: /data/mirror/openbsd/cvs/src/sys/netinet6/ip6_divert.h,v
> retrieving revision 1.18
> diff -u -p -r1.18 ip6_divert.h
> --- netinet6/ip6_divert.h 28 Aug 2022 18:44:17 -0000 1.18
> +++ netinet6/ip6_divert.h 3 Sep 2022 19:23:43 -0000
> @@ -74,6 +74,8 @@ int divert6_usrreq(struct socket *,
> int, struct mbuf *, struct mbuf *, struct mbuf *, struct proc *);
> int divert6_attach(struct socket *, int);
> int divert6_detach(struct socket *);
> +void divert6_lock(struct socket *);
> +void divert6_unlock(struct socket *);
> int divert6_bind(struct socket *, struct mbuf *, struct proc *);
> int divert6_shutdown(struct socket *);
> int divert6_send(struct socket *, struct mbuf *, struct mbuf *,
> Index: sys/protosw.h
> ===================================================================
> RCS file: /data/mirror/openbsd/cvs/src/sys/sys/protosw.h,v
> retrieving revision 1.53
> diff -u -p -r1.53 protosw.h
> --- sys/protosw.h 3 Sep 2022 18:48:50 -0000 1.53
> +++ sys/protosw.h 3 Sep 2022 19:23:43 -0000
> @@ -68,6 +68,8 @@ struct pr_usrreqs {
>
> int (*pru_attach)(struct socket *, int);
> int (*pru_detach)(struct socket *);
> + void (*pru_lock)(struct socket *);
> + void (*pru_unlock)(struct socket *);
> int (*pru_bind)(struct socket *, struct mbuf *, struct proc *);
> int (*pru_listen)(struct socket *);
> int (*pru_connect)(struct socket *, struct mbuf *);
> @@ -277,6 +279,18 @@ static inline int
> pru_detach(struct socket *so)
> {
> return (*so->so_proto->pr_usrreqs->pru_detach)(so);
> +}
> +
> +static inline void
> +pru_lock(struct socket *so)
> +{
> + (*so->so_proto->pr_usrreqs->pru_lock)(so);
> +}
> +
> +static inline void
> +pru_unlock(struct socket *so)
> +{
> + (*so->so_proto->pr_usrreqs->pru_unlock)(so);
> }
>
> static inline int
> Index: sys/socketvar.h
> ===================================================================
> RCS file: /data/mirror/openbsd/cvs/src/sys/sys/socketvar.h,v
> retrieving revision 1.109
> diff -u -p -r1.109 socketvar.h
> --- sys/socketvar.h 1 Sep 2022 05:31:49 -0000 1.109
> +++ sys/socketvar.h 3 Sep 2022 19:23:43 -0000
> @@ -348,9 +348,11 @@ int sockargs(struct mbuf **, const void
>
> int sosleep_nsec(struct socket *, void *, int, const char *, uint64_t);
> void solock(struct socket *);
> +void solock_shared(struct socket *);
> int solock_persocket(struct socket *);
> void solock_pair(struct socket *, struct socket *);
> void sounlock(struct socket *);
> +void sounlock_shared(struct socket *);
>
> int sendit(struct proc *, int, struct msghdr *, int, register_t *);
> int recvit(struct proc *, int, struct msghdr *, caddr_t, register_t *);
>
