Hi, The tcp reaper timeout is still imlemented as soft timeout. So it can run while net lock is held by others and it is not synchronized with the other tcp timeouts. Convert it to an ordinary tcp timeout so it is scheduled on the same timeout thread. It grabs the net lock to make sure that softnet has finished its work.
ok? bluhm Index: netinet/tcp_subr.c =================================================================== RCS file: /data/mirror/openbsd/cvs/src/sys/netinet/tcp_subr.c,v retrieving revision 1.167 diff -u -p -r1.167 tcp_subr.c --- netinet/tcp_subr.c 7 Dec 2017 16:52:21 -0000 1.167 +++ netinet/tcp_subr.c 19 Jan 2018 23:05:37 -0000 @@ -436,7 +436,6 @@ tcp_newtcpcb(struct inpcb *inp) TCP_INIT_DELACK(tp); for (i = 0; i < TCPT_NTIMERS; i++) TCP_TIMER_INIT(tp, i); - timeout_set(&tp->t_reap_to, tcp_reaper, tp); tp->sack_enable = tcp_do_sack; tp->t_flags = tcp_do_rfc1323 ? (TF_REQ_SCALE|TF_REQ_TSTMP) : 0; @@ -532,21 +531,12 @@ tcp_close(struct tcpcb *tp) m_free(tp->t_template); tp->t_flags |= TF_DEAD; - timeout_add(&tp->t_reap_to, 0); + TCP_TIMER_ARM(tp, TCPT_REAPER, 0); - inp->inp_ppcb = 0; + inp->inp_ppcb = NULL; soisdisconnected(so); in_pcbdetach(inp); return (NULL); -} - -void -tcp_reaper(void *arg) -{ - struct tcpcb *tp = arg; - - pool_put(&tcpcb_pool, tp); - tcpstat_inc(tcps_closed); } int Index: netinet/tcp_timer.c =================================================================== RCS file: /data/mirror/openbsd/cvs/src/sys/netinet/tcp_timer.c,v retrieving revision 1.60 diff -u -p -r1.60 tcp_timer.c --- netinet/tcp_timer.c 29 Oct 2017 14:56:36 -0000 1.60 +++ netinet/tcp_timer.c 20 Jan 2018 12:36:04 -0000 @@ -70,12 +70,14 @@ void tcp_timer_rexmt(void *); void tcp_timer_persist(void *); void tcp_timer_keep(void *); void tcp_timer_2msl(void *); +void tcp_timer_reaper(void *); const tcp_timer_func_t tcp_timer_funcs[TCPT_NTIMERS] = { tcp_timer_rexmt, tcp_timer_persist, tcp_timer_keep, tcp_timer_2msl, + tcp_timer_reaper, }; /* @@ -462,5 +464,26 @@ tcp_timer_2msl(void *arg) tp = tcp_close(tp); out: + NET_UNLOCK(); +} + +void +tcp_timer_reaper(void *arg) +{ + struct tcpcb *tp = arg; + int i; + + NET_LOCK(); +#ifdef DIAGNOSTIC + if ((tp->t_flags & TF_DEAD) == 0) + panic("%s: tcpcb %p is not dead", __func__, tp); + for (i = 0; i < TCPT_NTIMERS; i++) { + if (TCP_TIMER_ISARMED(tp, i)) + panic("%s: tcpcb %p timer %d is armed", + __func__, tp, i); + } +#endif + pool_put(&tcpcb_pool, tp); + tcpstat_inc(tcps_closed); NET_UNLOCK(); } Index: netinet/tcp_timer.h =================================================================== RCS file: /data/mirror/openbsd/cvs/src/sys/netinet/tcp_timer.h,v retrieving revision 1.14 diff -u -p -r1.14 tcp_timer.h --- netinet/tcp_timer.h 4 Oct 2016 13:54:32 -0000 1.14 +++ netinet/tcp_timer.h 20 Jan 2018 12:35:44 -0000 @@ -39,12 +39,13 @@ * Definitions of the TCP timers. These timers are counted * down PR_SLOWHZ times a second. */ -#define TCPT_NTIMERS 4 +#define TCPT_NTIMERS 5 #define TCPT_REXMT 0 /* retransmit */ #define TCPT_PERSIST 1 /* retransmit persistence */ #define TCPT_KEEP 2 /* keep alive */ #define TCPT_2MSL 3 /* 2*msl quiet time timer */ +#define TCPT_REAPER 4 /* delayed cleanup timeout */ /* * The TCPT_REXMT timer is used to force retransmissions. @@ -108,8 +109,8 @@ #define TCP_DELACK_TICKS (hz / PR_FASTHZ) /* time to delay ACK */ #ifdef TCPTIMERS -const char *tcptimers[] = - { "REXMT", "PERSIST", "KEEP", "2MSL" }; +const char *tcptimers[TCPT_NTIMERS] = + { "REXMT", "PERSIST", "KEEP", "2MSL", "REAPER" }; #endif /* TCPTIMERS */ /* Index: netinet/tcp_var.h =================================================================== RCS file: /data/mirror/openbsd/cvs/src/sys/netinet/tcp_var.h,v retrieving revision 1.128 diff -u -p -r1.128 tcp_var.h --- netinet/tcp_var.h 2 Nov 2017 14:01:18 -0000 1.128 +++ netinet/tcp_var.h 19 Jan 2018 23:01:00 -0000 @@ -193,8 +193,6 @@ struct tcpcb { u_short t_pmtud_ip_hl; /* IP header length from ICMP payload */ int pf; - - struct timeout t_reap_to; /* delayed cleanup timeout */ }; #define intotcpcb(ip) ((struct tcpcb *)(ip)->inp_ppcb) @@ -679,6 +677,7 @@ tcpstat_pkt(enum tcpstat_counters pcount counters_pkt(tcpcounters, pcounter, bcounter, v); } +extern struct pool tcpcb_pool; extern struct inpcbtable tcbtable; /* head of queue of active tcpcb's */ extern u_int32_t tcp_now; /* for RFC 1323 timestamps */ extern int tcp_do_rfc1323; /* enabled/disabled? */ @@ -705,7 +704,6 @@ extern int tcp_syn_cache_active; /* acti void tcp_canceltimers(struct tcpcb *); struct tcpcb * tcp_close(struct tcpcb *); -void tcp_reaper(void *); int tcp_freeq(struct tcpcb *); #ifdef INET6 void tcp6_ctlinput(int, struct sockaddr *, u_int, void *);