On Thu, Jul 22, 2021 at 08:38:04PM +0200, Hrvoje Popovski wrote: > On 22.7.2021. 12:21, Hrvoje Popovski wrote: > > Thank you for explanation.. > > > > after hitting box all night, box panic and i was able to reproduce it > > this morning ... I'm not sure but box panic after hour or more of > > sending traffic through iked tunnel .. > > I will try to reproduce it through ipsec tunnel .. > > > with isakmpd i've got panic after 5 or more hours > > r620-1# uvm_fault(0xffffffff822b2158, 0x137, 0, 2) -> e > kernel: page fault trap, code=0 > Stopped at tdb_free+0x9c: movq %rsi,0(%rdi) > TID PID UID PRFLAGS PFLAGS CPU COMMAND > 469292 75215 68 0x10 0 2 isakmpd > *142487 42745 0 0x14000 0x40000200 0K softclock > tdb_free(ffff80000118f6e0) at tdb_free+0x9c > tdb_timeout(ffff80000118f6e0) at tdb_timeout+0x77 > softclock_thread(ffff8000ffffefc0) at softclock_thread+0x16e > end trace frame: 0x0, count: 12 > https://www.openbsd.org/ddb.html describes the minimum info required in > bug reports. Insufficient info makes it difficult to find and fix bugs. >
There is the bluhm@'s diff with serialized `ipsec_in_use' access. Can you try it? Index: sys/net/if.c =================================================================== RCS file: /cvs/src/sys/net/if.c,v retrieving revision 1.643 diff -u -p -r1.643 if.c --- sys/net/if.c 20 Jul 2021 16:32:28 -0000 1.643 +++ sys/net/if.c 22 Jul 2021 20:46:19 -0000 @@ -109,6 +109,10 @@ #include <netinet6/ip6_var.h> #endif +#ifdef IPSEC +#include <netinet/ip_ipsp.h> +#endif + #ifdef MPLS #include <netmpls/mpls.h> #endif @@ -238,7 +242,7 @@ int ifq_congestion; int netisr; -#define NET_TASKQ 1 +#define NET_TASKQ 4 struct taskq *nettqmp[NET_TASKQ]; struct task if_input_task_locked = TASK_INITIALIZER(if_netisr, NULL); @@ -811,10 +815,13 @@ if_output_local(struct ifnet *ifp, struc return (ifiq_enqueue(ifiq, m) == 0 ? 0 : ENOBUFS); } +struct rwlock ipsec_lock = RWLOCK_INITIALIZER("ipsecl"); + void if_input_process(struct ifnet *ifp, struct mbuf_list *ml) { struct mbuf *m; + int exclusive_lock = 0; if (ml_empty(ml)) return; @@ -835,15 +842,30 @@ if_input_process(struct ifnet *ifp, stru * lists and the socket layer. */ + rw_enter_read(&ipsec_lock); +#ifdef IPSEC /* * XXXSMP IPsec data structures are not ready to be accessed * by multiple network threads in parallel. In this case * use an exclusive lock. */ - NET_LOCK(); + if (ipsec_in_use) + exclusive_lock = 1; +#endif + if (exclusive_lock) + NET_LOCK(); + else + NET_RLOCK_IN_SOFTNET(); + + rw_exit_read(&ipsec_lock); + while ((m = ml_dequeue(ml)) != NULL) (*ifp->if_input)(ifp, m); - NET_UNLOCK(); + + if (exclusive_lock) + NET_UNLOCK(); + else + NET_RUNLOCK_IN_SOFTNET(); } void @@ -900,6 +922,12 @@ if_netisr(void *unused) arpintr(); KERNEL_UNLOCK(); } +#endif + if (n & (1 << NETISR_IP)) + ipintr(); +#ifdef INET6 + if (n & (1 << NETISR_IPV6)) + ip6intr(); #endif #if NPPP > 0 if (n & (1 << NETISR_PPP)) { Index: sys/net/if_ethersubr.c =================================================================== RCS file: /cvs/src/sys/net/if_ethersubr.c,v retrieving revision 1.275 diff -u -p -r1.275 if_ethersubr.c --- sys/net/if_ethersubr.c 7 Jul 2021 20:19:01 -0000 1.275 +++ sys/net/if_ethersubr.c 22 Jul 2021 20:46:19 -0000 @@ -222,7 +222,10 @@ ether_resolve(struct ifnet *ifp, struct switch (af) { case AF_INET: + KERNEL_LOCK(); + /* XXXSMP there is a MP race in arpresolve() */ error = arpresolve(ifp, rt, m, dst, eh->ether_dhost); + KERNEL_UNLOCK(); if (error) return (error); eh->ether_type = htons(ETHERTYPE_IP); @@ -245,7 +248,10 @@ ether_resolve(struct ifnet *ifp, struct break; #ifdef INET6 case AF_INET6: + KERNEL_LOCK(); + /* XXXSMP there is a MP race in nd6_resolve() */ error = nd6_resolve(ifp, rt, m, dst, eh->ether_dhost); + KERNEL_UNLOCK(); if (error) return (error); eh->ether_type = htons(ETHERTYPE_IPV6); @@ -271,13 +277,19 @@ ether_resolve(struct ifnet *ifp, struct break; #ifdef INET6 case AF_INET6: + KERNEL_LOCK(); + /* XXXSMP there is a MP race in nd6_resolve() */ error = nd6_resolve(ifp, rt, m, dst, eh->ether_dhost); + KERNEL_UNLOCK(); if (error) return (error); break; #endif case AF_INET: + KERNEL_LOCK(); + /* XXXSMP there is a MP race in arpresolve() */ error = arpresolve(ifp, rt, m, dst, eh->ether_dhost); + KERNEL_UNLOCK(); if (error) return (error); break; @@ -528,12 +540,14 @@ ether_input(struct ifnet *ifp, struct mb case ETHERTYPE_PPPOE: if (m->m_flags & (M_MCAST | M_BCAST)) goto dropanyway; + KERNEL_LOCK(); #ifdef PIPEX if (pipex_enable) { struct pipex_session *session; if ((session = pipex_pppoe_lookup_session(m)) != NULL) { pipex_pppoe_input(m, session); + KERNEL_UNLOCK(); return; } } @@ -542,6 +556,7 @@ ether_input(struct ifnet *ifp, struct mb pppoe_disc_input(m); else pppoe_data_input(m); + KERNEL_UNLOCK(); return; #endif #ifdef MPLS Index: sys/net/ifq.c =================================================================== RCS file: /cvs/src/sys/net/ifq.c,v retrieving revision 1.44 diff -u -p -r1.44 ifq.c --- sys/net/ifq.c 9 Jul 2021 01:22:05 -0000 1.44 +++ sys/net/ifq.c 22 Jul 2021 20:46:19 -0000 @@ -243,7 +243,7 @@ void ifq_init(struct ifqueue *ifq, struct ifnet *ifp, unsigned int idx) { ifq->ifq_if = ifp; - ifq->ifq_softnet = net_tq(ifp->if_index); /* + idx */ + ifq->ifq_softnet = net_tq(ifp->if_index + idx); ifq->ifq_softc = NULL; mtx_init(&ifq->ifq_mtx, IPL_NET); @@ -620,7 +620,7 @@ void ifiq_init(struct ifiqueue *ifiq, struct ifnet *ifp, unsigned int idx) { ifiq->ifiq_if = ifp; - ifiq->ifiq_softnet = net_tq(ifp->if_index); /* + idx */ + ifiq->ifiq_softnet = net_tq(ifp->if_index + idx); ifiq->ifiq_softc = NULL; mtx_init(&ifiq->ifiq_mtx, IPL_NET); Index: sys/net/netisr.h =================================================================== RCS file: /cvs/src/sys/net/netisr.h,v retrieving revision 1.55 diff -u -p -r1.55 netisr.h --- sys/net/netisr.h 5 Jan 2021 20:43:36 -0000 1.55 +++ sys/net/netisr.h 22 Jul 2021 20:46:19 -0000 @@ -41,8 +41,10 @@ * interrupt used for scheduling the network code to calls * on the lowest level routine of each protocol. */ +#define NETISR_IP 2 /* same as AF_INET */ #define NETISR_PFSYNC 5 /* for pfsync "immediate" tx */ #define NETISR_ARP 18 /* same as AF_LINK */ +#define NETISR_IPV6 24 /* same as AF_INET6 */ #define NETISR_PPP 28 /* for PPP processing */ #define NETISR_BRIDGE 29 /* for bridge processing */ #define NETISR_SWITCH 31 /* for switch dataplane */ @@ -57,6 +59,8 @@ extern int netisr; /* scheduling bits extern struct task if_input_task_locked; void arpintr(void); +void ipintr(void); +void ip6intr(void); void pppintr(void); void bridgeintr(void); void switchintr(void); Index: sys/net/pfkeyv2.c =================================================================== RCS file: /cvs/src/sys/net/pfkeyv2.c,v retrieving revision 1.219 diff -u -p -r1.219 pfkeyv2.c --- sys/net/pfkeyv2.c 20 Jul 2021 16:32:28 -0000 1.219 +++ sys/net/pfkeyv2.c 22 Jul 2021 20:46:19 -0000 @@ -407,6 +407,8 @@ pfkeyv2_usrreq(struct socket *so, int re return (error); } +extern struct rwlock ipsec_lock; + int pfkeyv2_output(struct mbuf *mbuf, struct socket *so, struct sockaddr *dstaddr, struct mbuf *control) @@ -440,7 +442,9 @@ pfkeyv2_output(struct mbuf *mbuf, struct */ sounlock(so, SL_LOCKED); + rw_enter_write(&ipsec_lock); error = pfkeyv2_send(so, message, mbuf->m_pkthdr.len); + rw_exit_write(&ipsec_lock); solock(so); ret: Index: sys/netinet/ip_input.c =================================================================== RCS file: /cvs/src/sys/netinet/ip_input.c,v retrieving revision 1.363 diff -u -p -r1.363 ip_input.c --- sys/netinet/ip_input.c 21 Jun 2021 22:09:14 -0000 1.363 +++ sys/netinet/ip_input.c 22 Jul 2021 20:46:19 -0000 @@ -130,6 +130,8 @@ const struct sysctl_bounded_args ipctl_v { IPCTL_ARPDOWN, &arpt_down, 0, INT_MAX }, }; +struct niqueue ipintrq = NIQUEUE_INITIALIZER(IPQ_MAXLEN, NETISR_IP); + struct pool ipqent_pool; struct pool ipq_pool; @@ -143,6 +145,7 @@ static struct mbuf_queue ipsendraw_mq; extern struct niqueue arpinq; int ip_ours(struct mbuf **, int *, int, int); +int ip_local(struct mbuf **, int *, int, int); int ip_dooptions(struct mbuf *, struct ifnet *); int in_ouraddr(struct mbuf *, struct ifnet *, struct rtentry **); @@ -230,6 +233,43 @@ ip_init(void) } /* + * Enqueue packet for local delivery. Queuing is used as a boundary + * between the network layer (input/forward path) running with shared + * NET_RLOCK_IN_SOFTNET() and the transport layer needing it exclusively. + */ +int +ip_ours(struct mbuf **mp, int *offp, int nxt, int af) +{ + /* We are already in a IPv4/IPv6 local deliver loop. */ + if (af != AF_UNSPEC) + return ip_local(mp, offp, nxt, af); + + niq_enqueue(&ipintrq, *mp); + *mp = NULL; + return IPPROTO_DONE; +} + +/* + * Dequeue and process locally delivered packets. + */ +void +ipintr(void) +{ + struct mbuf *m; + int off, nxt; + + while ((m = niq_dequeue(&ipintrq)) != NULL) { +#ifdef DIAGNOSTIC + if ((m->m_flags & M_PKTHDR) == 0) + panic("ipintr no HDR"); +#endif + off = 0; + nxt = ip_local(&m, &off, IPPROTO_IPV4, AF_UNSPEC); + KASSERT(nxt == IPPROTO_DONE); + } +} + +/* * IPv4 input routine. * * Checksum and byte swap header. Process options. Forward or deliver. @@ -514,7 +554,7 @@ ip_input_if(struct mbuf **mp, int *offp, * If fragmented try to reassemble. Pass to next level. */ int -ip_ours(struct mbuf **mp, int *offp, int nxt, int af) +ip_local(struct mbuf **mp, int *offp, int nxt, int af) { struct mbuf *m = *mp; struct ip *ip = mtod(m, struct ip *); @@ -522,6 +562,8 @@ ip_ours(struct mbuf **mp, int *offp, int struct ipqent *ipqe; int mff, hlen; + NET_ASSERT_WLOCKED(); + hlen = ip->ip_hl << 2; /* @@ -1665,7 +1707,8 @@ ip_sysctl(int *name, u_int namelen, void newlen)); #endif case IPCTL_IFQUEUE: - return (EOPNOTSUPP); + return (sysctl_niq(name + 1, namelen - 1, + oldp, oldlenp, newp, newlen, &ipintrq)); case IPCTL_ARPQUEUE: return (sysctl_niq(name + 1, namelen - 1, oldp, oldlenp, newp, newlen, &arpinq)); Index: sys/netinet/ip_var.h =================================================================== RCS file: /cvs/src/sys/netinet/ip_var.h,v retrieving revision 1.88 diff -u -p -r1.88 ip_var.h --- sys/netinet/ip_var.h 30 Mar 2021 08:37:11 -0000 1.88 +++ sys/netinet/ip_var.h 22 Jul 2021 20:46:19 -0000 @@ -248,7 +248,6 @@ void ip_stripoptions(struct mbuf *); int ip_sysctl(int *, u_int, void *, size_t *, void *, size_t); void ip_savecontrol(struct inpcb *, struct mbuf **, struct ip *, struct mbuf *); -void ipintr(void); int ip_input_if(struct mbuf **, int *, int, int, struct ifnet *); int ip_deliver(struct mbuf **, int *, int, int); void ip_forward(struct mbuf *, struct ifnet *, struct rtentry *, int); Index: sys/netinet6/ip6_input.c =================================================================== RCS file: /cvs/src/sys/netinet6/ip6_input.c,v retrieving revision 1.237 diff -u -p -r1.237 ip6_input.c --- sys/netinet6/ip6_input.c 3 Jun 2021 04:47:54 -0000 1.237 +++ sys/netinet6/ip6_input.c 22 Jul 2021 20:46:19 -0000 @@ -115,11 +115,14 @@ #include <netinet/ip_carp.h> #endif +struct niqueue ip6intrq = NIQUEUE_INITIALIZER(IPQ_MAXLEN, NETISR_IPV6); + struct cpumem *ip6counters; uint8_t ip6_soiikey[IP6_SOIIKEY_LEN]; int ip6_ours(struct mbuf **, int *, int, int); +int ip6_local(struct mbuf **, int *, int, int); int ip6_check_rh0hdr(struct mbuf *, int *); int ip6_hbhchcheck(struct mbuf *, int *, int *, int *); int ip6_hopopts_input(u_int32_t *, u_int32_t *, struct mbuf **, int *); @@ -162,6 +165,43 @@ ip6_init(void) ip6counters = counters_alloc(ip6s_ncounters); } +/* + * Enqueue packet for local delivery. Queuing is used as a boundary + * between the network layer (input/forward path) running with shared + * NET_RLOCK_IN_SOFTNET() and the transport layer needing it exclusively. + */ +int +ip6_ours(struct mbuf **mp, int *offp, int nxt, int af) +{ + /* We are already in a IPv4/IPv6 local deliver loop. */ + if (af != AF_UNSPEC) + return ip6_local(mp, offp, nxt, af); + + niq_enqueue(&ip6intrq, *mp); + *mp = NULL; + return IPPROTO_DONE; +} + +/* + * Dequeue and process locally delivered packets. + */ +void +ip6intr(void) +{ + struct mbuf *m; + int off, nxt; + + while ((m = niq_dequeue(&ip6intrq)) != NULL) { +#ifdef DIAGNOSTIC + if ((m->m_flags & M_PKTHDR) == 0) + panic("ip6intr no HDR"); +#endif + off = 0; + nxt = ip6_local(&m, &off, IPPROTO_IPV6, AF_UNSPEC); + KASSERT(nxt == IPPROTO_DONE); + } +} + void ipv6_input(struct ifnet *ifp, struct mbuf *m) { @@ -544,8 +584,10 @@ ip6_input_if(struct mbuf **mp, int *offp } int -ip6_ours(struct mbuf **mp, int *offp, int nxt, int af) +ip6_local(struct mbuf **mp, int *offp, int nxt, int af) { + NET_ASSERT_WLOCKED(); + if (ip6_hbhchcheck(*mp, offp, &nxt, NULL)) return IPPROTO_DONE; @@ -1470,7 +1512,8 @@ ip6_sysctl(int *name, u_int namelen, voi NET_UNLOCK(); return (error); case IPV6CTL_IFQUEUE: - return (EOPNOTSUPP); + return (sysctl_niq(name + 1, namelen - 1, + oldp, oldlenp, newp, newlen, &ip6intrq)); case IPV6CTL_SOIIKEY: return (ip6_sysctl_soiikey(oldp, oldlenp, newp, newlen)); default: