ether_input() called with shared netlock, but pppoe(4) wants it to be exclusive. Do the pppoe(4) input within netisr handler with exclusive netlok held, and remove kernel lock hack from ether_input().
This is the step back, but it makes ether_input() path better then it is now. ok? Index: sys/net/if.c =================================================================== RCS file: /cvs/src/sys/net/if.c,v retrieving revision 1.655 diff -u -p -r1.655 if.c --- sys/net/if.c 28 Jun 2022 08:01:40 -0000 1.655 +++ sys/net/if.c 28 Jun 2022 09:27:29 -0000 @@ -68,6 +68,7 @@ #include "pf.h" #include "pfsync.h" #include "ppp.h" +#include "pppoe.h" #include "if_wg.h" #include <sys/param.h> @@ -917,6 +918,11 @@ if_netisr(void *unused) #ifdef PIPEX if (n & (1 << NETISR_PIPEX)) pipexintr(); +#endif +#if NPPPOE > 0 + if (n & (1 << NETISR_PPPOE)) { + pppoeintr(); + } #endif t |= n; } Index: sys/net/if_ethersubr.c =================================================================== RCS file: /cvs/src/sys/net/if_ethersubr.c,v retrieving revision 1.282 diff -u -p -r1.282 if_ethersubr.c --- sys/net/if_ethersubr.c 27 Jun 2022 20:47:10 -0000 1.282 +++ sys/net/if_ethersubr.c 28 Jun 2022 09:27:29 -0000 @@ -546,12 +546,13 @@ ether_input(struct ifnet *ifp, struct mb pipex_rele_session(session); } #endif - KERNEL_LOCK(); - if (etype == ETHERTYPE_PPPOEDISC) - pppoe_disc_input(m); - else - pppoe_data_input(m); - KERNEL_UNLOCK(); + if (etype == ETHERTYPE_PPPOEDISC) { + if (mq_enqueue(&pppoediscinq, m) == 0) + schednetisr(NETISR_PPPOE); + } else { + if (mq_enqueue(&pppoeinq, m) == 0) + schednetisr(NETISR_PPPOE); + } return; #endif #ifdef MPLS Index: sys/net/if_pppoe.c =================================================================== RCS file: /cvs/src/sys/net/if_pppoe.c,v retrieving revision 1.80 diff -u -p -r1.80 if_pppoe.c --- sys/net/if_pppoe.c 14 May 2022 09:46:15 -0000 1.80 +++ sys/net/if_pppoe.c 28 Jun 2022 09:27:29 -0000 @@ -144,6 +144,8 @@ struct pppoe_softc { }; /* input routines */ +void pppoe_disc_input(struct mbuf *); +void pppoe_data_input(struct mbuf *); static void pppoe_dispatch_disc_pkt(struct mbuf *); /* management routines */ @@ -181,6 +183,26 @@ int pppoe_clone_destroy(struct ifnet *); struct if_clone pppoe_cloner = IF_CLONE_INITIALIZER("pppoe", pppoe_clone_create, pppoe_clone_destroy); +struct mbuf_queue pppoediscinq = MBUF_QUEUE_INITIALIZER( + IFQ_MAXLEN, IPL_SOFTNET); +struct mbuf_queue pppoeinq = MBUF_QUEUE_INITIALIZER( + IFQ_MAXLEN, IPL_SOFTNET); + +void pppoeintr(void) +{ + struct mbuf_list ml; + struct mbuf *m; + + NET_ASSERT_LOCKED(); + + mq_delist(&pppoediscinq, &ml); + while ((m = ml_dequeue(&ml)) != NULL) + pppoe_disc_input(m); + + mq_delist(&pppoeinq, &ml); + while ((m = ml_dequeue(&ml)) != NULL) + pppoe_data_input(m); +} void pppoeattach(int count) Index: sys/net/if_pppoe.h =================================================================== RCS file: /cvs/src/sys/net/if_pppoe.h,v retrieving revision 1.7 diff -u -p -r1.7 if_pppoe.h --- sys/net/if_pppoe.h 4 Jan 2021 21:21:41 -0000 1.7 +++ sys/net/if_pppoe.h 28 Jun 2022 09:27:29 -0000 @@ -66,8 +66,8 @@ struct pppoeconnectionstate { #ifdef _KERNEL -void pppoe_disc_input(struct mbuf *); -void pppoe_data_input(struct mbuf *); +extern struct mbuf_queue pppoediscinq; +extern struct mbuf_queue pppoeinq; #endif /* _KERNEL */ #endif /* _NET_IF_PPPOE_H_ */ Index: sys/net/netisr.h =================================================================== RCS file: /cvs/src/sys/net/netisr.h,v retrieving revision 1.57 diff -u -p -r1.57 netisr.h --- sys/net/netisr.h 28 Jun 2022 08:01:40 -0000 1.57 +++ sys/net/netisr.h 28 Jun 2022 09:27:29 -0000 @@ -48,6 +48,7 @@ #define NETISR_PIPEX 27 /* for pipex processing */ #define NETISR_PPP 28 /* for PPP processing */ #define NETISR_BRIDGE 29 /* for bridge processing */ +#define NETISR_PPPOE 30 /* for pppoe processing */ #define NETISR_SWITCH 31 /* for switch dataplane */ #ifndef _LOCORE @@ -67,6 +68,7 @@ void bridgeintr(void); void switchintr(void); void pfsyncintr(void); void pipexintr(void); +void pppoeintr(void); #define schednetisr(anisr) \ do { \