Hi,
The next small step towards parallel network stack is to use shared
netlock in soreceive(). The UDP and IP divert layer provide locking
of the PCB. If that is possible, use shared instead of exclusive
netlock in soreceive(). The PCB mutex provides a per socket lock
against multiple soreceive() running in parallel.
The solock_shared() is a bit hacky. Especially where we have to
release and regrab both locks in sosleep_nsec(). But it does what
we need.
The udp_input() keeps exclusive lock for now. Socket splicing and
IP header chains are not completely MP safe yet. Also udp_input()
has some tentacles that need more testing.
Ok to procceed with soreceive() unlocking before release? I think
udp_input() has to wait post 7.2.
bluhm
Index: kern/uipc_socket.c
===================================================================
RCS file: /data/mirror/openbsd/cvs/src/sys/kern/uipc_socket.c,v
retrieving revision 1.287
diff -u -p -r1.287 uipc_socket.c
--- kern/uipc_socket.c 3 Sep 2022 13:29:33 -0000 1.287
+++ kern/uipc_socket.c 3 Sep 2022 19:23:43 -0000
@@ -822,10 +822,10 @@ bad:
if (mp)
*mp = NULL;
- solock(so);
+ solock_shared(so);
restart:
if ((error = sblock(so, &so->so_rcv, SBLOCKWAIT(flags))) != 0) {
- sounlock(so);
+ sounlock_shared(so);
return (error);
}
@@ -893,7 +893,7 @@ restart:
sbunlock(so, &so->so_rcv);
error = sbwait(so, &so->so_rcv);
if (error) {
- sounlock(so);
+ sounlock_shared(so);
return (error);
}
goto restart;
@@ -962,11 +962,11 @@ dontblock:
sbsync(&so->so_rcv, nextrecord);
if (controlp) {
if (pr->pr_domain->dom_externalize) {
- sounlock(so);
+ sounlock_shared(so);
error =
(*pr->pr_domain->dom_externalize)
(cm, controllen, flags);
- solock(so);
+ solock_shared(so);
}
*controlp = cm;
} else {
@@ -1040,9 +1040,9 @@ dontblock:
SBLASTRECORDCHK(&so->so_rcv, "soreceive uiomove");
SBLASTMBUFCHK(&so->so_rcv, "soreceive uiomove");
resid = uio->uio_resid;
- sounlock(so);
+ sounlock_shared(so);
uio_error = uiomove(mtod(m, caddr_t) + moff, len, uio);
- solock(so);
+ solock_shared(so);
if (uio_error)
uio->uio_resid = resid - len;
} else
@@ -1126,7 +1126,7 @@ dontblock:
error = sbwait(so, &so->so_rcv);
if (error) {
sbunlock(so, &so->so_rcv);
- sounlock(so);
+ sounlock_shared(so);
return (0);
}
if ((m = so->so_rcv.sb_mb) != NULL)
@@ -1171,7 +1171,7 @@ dontblock:
*flagsp |= flags;
release:
sbunlock(so, &so->so_rcv);
- sounlock(so);
+ sounlock_shared(so);
return (error);
}
Index: kern/uipc_socket2.c
===================================================================
RCS file: /data/mirror/openbsd/cvs/src/sys/kern/uipc_socket2.c,v
retrieving revision 1.127
diff -u -p -r1.127 uipc_socket2.c
--- kern/uipc_socket2.c 13 Aug 2022 21:01:46 -0000 1.127
+++ kern/uipc_socket2.c 3 Sep 2022 19:23:43 -0000
@@ -360,6 +360,24 @@ solock(struct socket *so)
}
}
+void
+solock_shared(struct socket *so)
+{
+ switch (so->so_proto->pr_domain->dom_family) {
+ case PF_INET:
+ case PF_INET6:
+ if (so->so_proto->pr_usrreqs->pru_lock != NULL) {
+ NET_LOCK_SHARED();
+ pru_lock(so);
+ } else
+ NET_LOCK();
+ break;
+ default:
+ rw_enter_write(&so->so_lock);
+ break;
+ }
+}
+
int
solock_persocket(struct socket *so)
{
@@ -403,6 +421,24 @@ sounlock(struct socket *so)
}
void
+sounlock_shared(struct socket *so)
+{
+ switch (so->so_proto->pr_domain->dom_family) {
+ case PF_INET:
+ case PF_INET6:
+ if (so->so_proto->pr_usrreqs->pru_unlock != NULL) {
+ pru_unlock(so);
+ NET_UNLOCK_SHARED();
+ } else
+ NET_UNLOCK();
+ break;
+ default:
+ rw_exit_write(&so->so_lock);
+ break;
+ }
+}
+
+void
soassertlocked(struct socket *so)
{
switch (so->so_proto->pr_domain->dom_family) {
@@ -425,7 +461,15 @@ sosleep_nsec(struct socket *so, void *id
switch (so->so_proto->pr_domain->dom_family) {
case PF_INET:
case PF_INET6:
+ if (so->so_proto->pr_usrreqs->pru_unlock != NULL &&
+ rw_status(&netlock) == RW_READ) {
+ pru_unlock(so);
+ }
ret = rwsleep_nsec(ident, &netlock, prio, wmesg, nsecs);
+ if (so->so_proto->pr_usrreqs->pru_lock != NULL &&
+ rw_status(&netlock) == RW_READ) {
+ pru_lock(so);
+ }
break;
default:
ret = rwsleep_nsec(ident, &so->so_lock, prio, wmesg, nsecs);
Index: netinet/ip_divert.c
===================================================================
RCS file: /data/mirror/openbsd/cvs/src/sys/netinet/ip_divert.c,v
retrieving revision 1.85
diff -u -p -r1.85 ip_divert.c
--- netinet/ip_divert.c 3 Sep 2022 18:48:50 -0000 1.85
+++ netinet/ip_divert.c 3 Sep 2022 19:23:43 -0000
@@ -66,6 +66,8 @@ const struct pr_usrreqs divert_usrreqs =
.pru_usrreq = divert_usrreq,
.pru_attach = divert_attach,
.pru_detach = divert_detach,
+ .pru_lock = divert_lock,
+ .pru_unlock = divert_unlock,
.pru_bind = divert_bind,
.pru_shutdown = divert_shutdown,
.pru_send = divert_send,
@@ -325,6 +327,24 @@ divert_detach(struct socket *so)
in_pcbdetach(inp);
return (0);
+}
+
+void
+divert_lock(struct socket *so)
+{
+ struct inpcb *inp = sotoinpcb(so);
+
+ NET_ASSERT_LOCKED();
+ mtx_enter(&inp->inp_mtx);
+}
+
+void
+divert_unlock(struct socket *so)
+{
+ struct inpcb *inp = sotoinpcb(so);
+
+ NET_ASSERT_LOCKED();
+ mtx_leave(&inp->inp_mtx);
}
int
Index: netinet/ip_divert.h
===================================================================
RCS file: /data/mirror/openbsd/cvs/src/sys/netinet/ip_divert.h,v
retrieving revision 1.20
diff -u -p -r1.20 ip_divert.h
--- netinet/ip_divert.h 28 Aug 2022 18:44:16 -0000 1.20
+++ netinet/ip_divert.h 3 Sep 2022 19:23:43 -0000
@@ -74,6 +74,8 @@ int divert_usrreq(struct socket *,
int, struct mbuf *, struct mbuf *, struct mbuf *, struct proc *);
int divert_attach(struct socket *, int);
int divert_detach(struct socket *);
+void divert_lock(struct socket *);
+void divert_unlock(struct socket *);
int divert_bind(struct socket *, struct mbuf *, struct proc *);
int divert_shutdown(struct socket *);
int divert_send(struct socket *, struct mbuf *, struct mbuf *,
Index: netinet/udp_usrreq.c
===================================================================
RCS file: /data/mirror/openbsd/cvs/src/sys/netinet/udp_usrreq.c,v
retrieving revision 1.300
diff -u -p -r1.300 udp_usrreq.c
--- netinet/udp_usrreq.c 3 Sep 2022 18:48:50 -0000 1.300
+++ netinet/udp_usrreq.c 3 Sep 2022 19:23:43 -0000
@@ -126,6 +126,8 @@ const struct pr_usrreqs udp_usrreqs = {
.pru_usrreq = udp_usrreq,
.pru_attach = udp_attach,
.pru_detach = udp_detach,
+ .pru_lock = udp_lock,
+ .pru_unlock = udp_unlock,
.pru_bind = udp_bind,
.pru_connect = udp_connect,
.pru_disconnect = udp_disconnect,
@@ -141,6 +143,8 @@ const struct pr_usrreqs udp6_usrreqs = {
.pru_usrreq = udp_usrreq,
.pru_attach = udp_attach,
.pru_detach = udp_detach,
+ .pru_lock = udp_lock,
+ .pru_unlock = udp_unlock,
.pru_bind = udp_bind,
.pru_connect = udp_connect,
.pru_disconnect = udp_disconnect,
@@ -1159,6 +1163,24 @@ udp_detach(struct socket *so)
in_pcbdetach(inp);
return (0);
+}
+
+void
+udp_lock(struct socket *so)
+{
+ struct inpcb *inp = sotoinpcb(so);
+
+ NET_ASSERT_LOCKED();
+ mtx_enter(&inp->inp_mtx);
+}
+
+void
+udp_unlock(struct socket *so)
+{
+ struct inpcb *inp = sotoinpcb(so);
+
+ NET_ASSERT_LOCKED();
+ mtx_leave(&inp->inp_mtx);
}
int
Index: netinet/udp_var.h
===================================================================
RCS file: /data/mirror/openbsd/cvs/src/sys/netinet/udp_var.h,v
retrieving revision 1.45
diff -u -p -r1.45 udp_var.h
--- netinet/udp_var.h 2 Sep 2022 13:12:32 -0000 1.45
+++ netinet/udp_var.h 3 Sep 2022 19:23:43 -0000
@@ -147,6 +147,8 @@ int udp_usrreq(struct socket *,
int, struct mbuf *, struct mbuf *, struct mbuf *, struct proc *);
int udp_attach(struct socket *, int);
int udp_detach(struct socket *);
+void udp_lock(struct socket *);
+void udp_unlock(struct socket *);
int udp_bind(struct socket *, struct mbuf *, struct proc *);
int udp_connect(struct socket *, struct mbuf *);
int udp_disconnect(struct socket *);
Index: netinet6/ip6_divert.c
===================================================================
RCS file: /data/mirror/openbsd/cvs/src/sys/netinet6/ip6_divert.c,v
retrieving revision 1.84
diff -u -p -r1.84 ip6_divert.c
--- netinet6/ip6_divert.c 3 Sep 2022 18:48:50 -0000 1.84
+++ netinet6/ip6_divert.c 3 Sep 2022 19:23:43 -0000
@@ -67,6 +67,8 @@ const struct pr_usrreqs divert6_usrreqs
.pru_usrreq = divert6_usrreq,
.pru_attach = divert6_attach,
.pru_detach = divert6_detach,
+ .pru_lock = divert6_lock,
+ .pru_unlock = divert6_unlock,
.pru_bind = divert6_bind,
.pru_shutdown = divert6_shutdown,
.pru_send = divert6_send,
@@ -332,6 +334,24 @@ divert6_detach(struct socket *so)
in_pcbdetach(inp);
return (0);
+}
+
+void
+divert6_lock(struct socket *so)
+{
+ struct inpcb *inp = sotoinpcb(so);
+
+ NET_ASSERT_LOCKED();
+ mtx_enter(&inp->inp_mtx);
+}
+
+void
+divert6_unlock(struct socket *so)
+{
+ struct inpcb *inp = sotoinpcb(so);
+
+ NET_ASSERT_LOCKED();
+ mtx_leave(&inp->inp_mtx);
}
int
Index: netinet6/ip6_divert.h
===================================================================
RCS file: /data/mirror/openbsd/cvs/src/sys/netinet6/ip6_divert.h,v
retrieving revision 1.18
diff -u -p -r1.18 ip6_divert.h
--- netinet6/ip6_divert.h 28 Aug 2022 18:44:17 -0000 1.18
+++ netinet6/ip6_divert.h 3 Sep 2022 19:23:43 -0000
@@ -74,6 +74,8 @@ int divert6_usrreq(struct socket *,
int, struct mbuf *, struct mbuf *, struct mbuf *, struct proc *);
int divert6_attach(struct socket *, int);
int divert6_detach(struct socket *);
+void divert6_lock(struct socket *);
+void divert6_unlock(struct socket *);
int divert6_bind(struct socket *, struct mbuf *, struct proc *);
int divert6_shutdown(struct socket *);
int divert6_send(struct socket *, struct mbuf *, struct mbuf *,
Index: sys/protosw.h
===================================================================
RCS file: /data/mirror/openbsd/cvs/src/sys/sys/protosw.h,v
retrieving revision 1.53
diff -u -p -r1.53 protosw.h
--- sys/protosw.h 3 Sep 2022 18:48:50 -0000 1.53
+++ sys/protosw.h 3 Sep 2022 19:23:43 -0000
@@ -68,6 +68,8 @@ struct pr_usrreqs {
int (*pru_attach)(struct socket *, int);
int (*pru_detach)(struct socket *);
+ void (*pru_lock)(struct socket *);
+ void (*pru_unlock)(struct socket *);
int (*pru_bind)(struct socket *, struct mbuf *, struct proc *);
int (*pru_listen)(struct socket *);
int (*pru_connect)(struct socket *, struct mbuf *);
@@ -277,6 +279,18 @@ static inline int
pru_detach(struct socket *so)
{
return (*so->so_proto->pr_usrreqs->pru_detach)(so);
+}
+
+static inline void
+pru_lock(struct socket *so)
+{
+ (*so->so_proto->pr_usrreqs->pru_lock)(so);
+}
+
+static inline void
+pru_unlock(struct socket *so)
+{
+ (*so->so_proto->pr_usrreqs->pru_unlock)(so);
}
static inline int
Index: sys/socketvar.h
===================================================================
RCS file: /data/mirror/openbsd/cvs/src/sys/sys/socketvar.h,v
retrieving revision 1.109
diff -u -p -r1.109 socketvar.h
--- sys/socketvar.h 1 Sep 2022 05:31:49 -0000 1.109
+++ sys/socketvar.h 3 Sep 2022 19:23:43 -0000
@@ -348,9 +348,11 @@ int sockargs(struct mbuf **, const void
int sosleep_nsec(struct socket *, void *, int, const char *, uint64_t);
void solock(struct socket *);
+void solock_shared(struct socket *);
int solock_persocket(struct socket *);
void solock_pair(struct socket *, struct socket *);
void sounlock(struct socket *);
+void sounlock_shared(struct socket *);
int sendit(struct proc *, int, struct msghdr *, int, register_t *);
int recvit(struct proc *, int, struct msghdr *, caddr_t, register_t *);
