The branch main has been updated by glebius: URL: https://cgit.FreeBSD.org/src/commit/?id=50f081ecb7a453213b15804c4b36903f1b248f8f
commit 50f081ecb7a453213b15804c4b36903f1b248f8f Author: Gleb Smirnoff <[email protected]> AuthorDate: 2021-12-02 18:48:48 +0000 Commit: Gleb Smirnoff <[email protected]> CommitDate: 2021-12-02 18:48:48 +0000 tcp_hpts: provide tcp_in_hpts(). It will hide some internal HPTS knowledge from the consumers. Reviewed by: rrs Differential revision: https://reviews.freebsd.org/D33023 --- sys/netinet/tcp_hpts.c | 2 +- sys/netinet/tcp_hpts.h | 7 ++++++ sys/netinet/tcp_stacks/bbr.c | 22 ++++++++--------- sys/netinet/tcp_stacks/rack.c | 56 +++++++++++++++++++++---------------------- 4 files changed, 47 insertions(+), 40 deletions(-) diff --git a/sys/netinet/tcp_hpts.c b/sys/netinet/tcp_hpts.c index 764d46a1580b..ee0b6d8f9d58 100644 --- a/sys/netinet/tcp_hpts.c +++ b/sys/netinet/tcp_hpts.c @@ -55,7 +55,7 @@ __FBSDID("$FreeBSD$"); * its top a check to prevent unwanted calls (from user land or the * arrival of incoming ack's). So it would add something like: * - * if (inp->inp_in_hpts) + * if (tcp_in_hpts(inp)) * return; * * to prevent output processing until the time alotted has gone by. diff --git a/sys/netinet/tcp_hpts.h b/sys/netinet/tcp_hpts.h index bfbd34e65ff5..b93a008fd253 100644 --- a/sys/netinet/tcp_hpts.h +++ b/sys/netinet/tcp_hpts.h @@ -201,6 +201,13 @@ struct tcp_hpts_entry *tcp_cur_hpts(struct inpcb *inp); #define tcp_hpts_remove(a, b) __tcp_hpts_remove(a, b, __LINE__) void __tcp_hpts_remove(struct inpcb *inp, int32_t flags, int32_t line); +static inline bool +tcp_in_hpts(struct inpcb *inp) +{ + + return (inp->inp_in_hpts > 0); +} + /* * To insert a TCB on the hpts you *must* be holding the * INP_WLOCK(). The hpts insert code will then acqurire diff --git a/sys/netinet/tcp_stacks/bbr.c b/sys/netinet/tcp_stacks/bbr.c index eb64a28f8565..74a9fada9174 100644 --- a/sys/netinet/tcp_stacks/bbr.c +++ b/sys/netinet/tcp_stacks/bbr.c @@ -747,7 +747,7 @@ bbr_start_hpts_timer(struct tcp_bbr *bbr, struct tcpcb *tp, uint32_t cts, int32_ uint32_t prev_delay = 0; inp = tp->t_inpcb; - if (inp->inp_in_hpts) { + if (tcp_in_hpts(inp)) { /* A previous call is already set up */ return; } @@ -1058,7 +1058,7 @@ bbr_timer_audit(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts, struct sock */ wrong_timer: if ((bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0) { - if (inp->inp_in_hpts) + if (tcp_in_hpts(inp)) tcp_hpts_remove(inp, HPTS_REMOVE_OUTPUT); bbr_timer_cancel(bbr, __LINE__, cts); bbr_start_hpts_timer(bbr, tp, cts, 1, bbr->r_ctl.rc_last_delay_val, @@ -1883,7 +1883,7 @@ bbr_fill_in_logging_data(struct tcp_bbr *bbr, struct tcp_log_bbr *l, uint32_t ct l->lt_epoch = bbr->r_ctl.rc_lt_epoch; l->pacing_gain = bbr->r_ctl.rc_bbr_hptsi_gain; l->cwnd_gain = bbr->r_ctl.rc_bbr_cwnd_gain; - l->inhpts = bbr->rc_inp->inp_in_hpts; + l->inhpts = tcp_in_hpts(bbr->rc_inp); l->ininput = bbr->rc_inp->inp_in_input; l->use_lt_bw = bbr->rc_lt_use_bw; l->pkts_out = bbr->r_ctl.rc_flight_at_input; @@ -3969,7 +3969,7 @@ bbr_cong_signal(struct tcpcb *tp, struct tcphdr *th, uint32_t type, struct bbr_s bbr->rc_tlp_rtx_out = 0; bbr->r_ctl.recovery_lr = bbr->r_ctl.rc_pkt_epoch_loss_rate; tcp_bbr_tso_size_check(bbr, bbr->r_ctl.rc_rcvtime); - if (bbr->rc_inp->inp_in_hpts && + if (tcp_in_hpts(bbr->rc_inp) && ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_RACK) == 0)) { /* * When we enter recovery, we need to restart @@ -5264,7 +5264,7 @@ bbr_timer_cancel(struct tcp_bbr *bbr, int32_t line, uint32_t cts) if (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK) { uint8_t hpts_removed = 0; - if (bbr->rc_inp->inp_in_hpts && + if (tcp_in_hpts(bbr->rc_inp) && (bbr->rc_timer_first == 1)) { /* * If we are canceling timer's when we have the @@ -7972,7 +7972,7 @@ bbr_exit_persist(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts, int32_t li * don't want to transfer forward the time * for our sum's calculations. */ - if (bbr->rc_inp->inp_in_hpts) { + if (tcp_in_hpts(bbr->rc_inp)) { tcp_hpts_remove(bbr->rc_inp, HPTS_REMOVE_OUTPUT); bbr->rc_timer_first = 0; bbr->r_ctl.rc_hpts_flags = 0; @@ -11653,7 +11653,7 @@ bbr_do_segment_nounlock(struct mbuf *m, struct tcphdr *th, struct socket *so, */ if ((tp->snd_max == tp->snd_una) && ((tp->t_flags & TF_DELACK) == 0) && - (bbr->rc_inp->inp_in_hpts) && + (tcp_in_hpts(bbr->rc_inp)) && (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)) { /* * keep alive not needed if we are hptsi @@ -11661,7 +11661,7 @@ bbr_do_segment_nounlock(struct mbuf *m, struct tcphdr *th, struct socket *so, */ ; } else { - if (bbr->rc_inp->inp_in_hpts) { + if (tcp_in_hpts(bbr->rc_inp)) { tcp_hpts_remove(bbr->rc_inp, HPTS_REMOVE_OUTPUT); if ((bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) && (TSTMP_GT(lcts, bbr->rc_pacer_started))) { @@ -12011,7 +12011,7 @@ bbr_output_wtime(struct tcpcb *tp, const struct timeval *tv) } #endif if (((bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0) && - inp->inp_in_hpts) { + tcp_in_hpts(inp)) { /* * We are on the hpts for some timer but not hptsi output. * Possibly remove from the hpts so we can send/recv etc. @@ -12056,9 +12056,9 @@ bbr_output_wtime(struct tcpcb *tp, const struct timeval *tv) if ((bbr->r_timer_override) || (tp->t_state < TCPS_ESTABLISHED)) { /* Timeouts or early states are exempt */ - if (inp->inp_in_hpts) + if (tcp_in_hpts(inp)) tcp_hpts_remove(inp, HPTS_REMOVE_OUTPUT); - } else if (inp->inp_in_hpts) { + } else if (tcp_in_hpts(inp)) { if ((bbr->r_ctl.rc_last_delay_val) && (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) && delay_calc) { diff --git a/sys/netinet/tcp_stacks/rack.c b/sys/netinet/tcp_stacks/rack.c index a17e598f6477..5a4849566cf9 100644 --- a/sys/netinet/tcp_stacks/rack.c +++ b/sys/netinet/tcp_stacks/rack.c @@ -2294,7 +2294,7 @@ rack_log_retran_reason(struct tcp_rack *rack, struct rack_sendmap *rsm, uint32_t log.u_bbr.flex5 = rsm->r_start; log.u_bbr.flex6 = rsm->r_end; log.u_bbr.flex8 = mod; - log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts; + log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp); log.u_bbr.ininput = rack->rc_inp->inp_in_input; log.u_bbr.timeStamp = tcp_get_usecs(&tv); log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked); @@ -2329,7 +2329,7 @@ rack_log_to_start(struct tcp_rack *rack, uint32_t cts, uint32_t to, int32_t slot log.u_bbr.pkts_out = 0; else log.u_bbr.pkts_out = rack->r_ctl.rc_prr_sndcnt; - log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts; + log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp); log.u_bbr.ininput = rack->rc_inp->inp_in_input; log.u_bbr.timeStamp = tcp_get_usecs(&tv); log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked); @@ -2354,7 +2354,7 @@ rack_log_to_event(struct tcp_rack *rack, int32_t to_num, struct rack_sendmap *rs struct timeval tv; memset(&log.u_bbr, 0, sizeof(log.u_bbr)); - log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts; + log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp); log.u_bbr.ininput = rack->rc_inp->inp_in_input; log.u_bbr.flex8 = to_num; log.u_bbr.flex1 = rack->r_ctl.rc_rack_min_rtt; @@ -2393,7 +2393,7 @@ rack_log_map_chg(struct tcpcb *tp, struct tcp_rack *rack, memset(&log.u_bbr, 0, sizeof(log.u_bbr)); log.u_bbr.flex8 = flag; - log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts; + log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp); log.u_bbr.ininput = rack->rc_inp->inp_in_input; log.u_bbr.cur_del_rate = (uint64_t)prev; log.u_bbr.delRate = (uint64_t)rsm; @@ -2438,7 +2438,7 @@ rack_log_rtt_upd(struct tcpcb *tp, struct tcp_rack *rack, uint32_t t, uint32_t l union tcp_log_stackspecific log; struct timeval tv; memset(&log.u_bbr, 0, sizeof(log.u_bbr)); - log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts; + log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp); log.u_bbr.ininput = rack->rc_inp->inp_in_input; log.u_bbr.flex1 = t; log.u_bbr.flex2 = len; @@ -2588,7 +2588,7 @@ rack_log_progress_event(struct tcp_rack *rack, struct tcpcb *tp, uint32_t tick, struct timeval tv; memset(&log.u_bbr, 0, sizeof(log.u_bbr)); - log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts; + log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp); log.u_bbr.ininput = rack->rc_inp->inp_in_input; log.u_bbr.flex1 = line; log.u_bbr.flex2 = tick; @@ -2615,7 +2615,7 @@ rack_log_type_bbrsnd(struct tcp_rack *rack, uint32_t len, uint32_t slot, uint32_ union tcp_log_stackspecific log; memset(&log.u_bbr, 0, sizeof(log.u_bbr)); - log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts; + log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp); log.u_bbr.ininput = rack->rc_inp->inp_in_input; log.u_bbr.flex1 = slot; if (rack->rack_no_prr) @@ -2661,7 +2661,7 @@ rack_log_doseg_done(struct tcp_rack *rack, uint32_t cts, int32_t nxt_pkt, int32_ log.u_bbr.flex7 <<= 1; log.u_bbr.flex7 |= rack->r_wanted_output; /* Do we want output */ log.u_bbr.flex8 = rack->rc_in_persist; - log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts; + log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp); log.u_bbr.timeStamp = tcp_get_usecs(&tv); log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked); log.u_bbr.use_lt_bw = rack->r_ent_rec_ns; @@ -2717,7 +2717,7 @@ rack_log_type_just_return(struct tcp_rack *rack, uint32_t cts, uint32_t tlen, ui struct timeval tv; memset(&log.u_bbr, 0, sizeof(log.u_bbr)); - log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts; + log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp); log.u_bbr.ininput = rack->rc_inp->inp_in_input; log.u_bbr.flex1 = slot; log.u_bbr.flex2 = rack->r_ctl.rc_hpts_flags; @@ -2750,7 +2750,7 @@ rack_log_to_cancel(struct tcp_rack *rack, int32_t hpts_removed, int line, uint32 union tcp_log_stackspecific log; memset(&log.u_bbr, 0, sizeof(log.u_bbr)); - log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts; + log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp); log.u_bbr.ininput = rack->rc_inp->inp_in_input; log.u_bbr.flex1 = line; log.u_bbr.flex2 = rack->r_ctl.rc_last_output_to; @@ -4466,7 +4466,7 @@ rack_do_goodput_measurement(struct tcpcb *tp, struct tcp_rack *rack, rack->r_ctl.rc_app_limited_cnt, 0, 0, 10, __LINE__, NULL, quality); } - if (rack->rc_inp->inp_in_hpts && + if (tcp_in_hpts(rack->rc_inp) && (rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)) { /* * Ok we can't trust the pacer in this case @@ -5678,7 +5678,7 @@ rack_enter_persist(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts) static void rack_exit_persist(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts) { - if (rack->rc_inp->inp_in_hpts) { + if (tcp_in_hpts(rack->rc_inp)) { tcp_hpts_remove(rack->rc_inp, HPTS_REMOVE_OUTPUT); rack->r_ctl.rc_hpts_flags = 0; } @@ -5811,7 +5811,7 @@ rack_start_hpts_timer(struct tcp_rack *rack, struct tcpcb *tp, uint32_t cts, (tp->t_state == TCPS_LISTEN)) { return; } - if (inp->inp_in_hpts) { + if (tcp_in_hpts(inp)) { /* Already on the pacer */ return; } @@ -7244,7 +7244,7 @@ rack_timer_cancel(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts, int lin } if (rack->r_ctl.rc_hpts_flags & PACE_TMR_MASK) { rack->rc_tmr_stopped = rack->r_ctl.rc_hpts_flags & PACE_TMR_MASK; - if (rack->rc_inp->inp_in_hpts && + if (tcp_in_hpts(rack->rc_inp) && ((rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0)) { /* * Canceling timer's when we have no output being @@ -9983,7 +9983,7 @@ out: (entered_recovery == 0)) { rack_update_prr(tp, rack, changed, th_ack); if ((rsm && (rack->r_ctl.rc_prr_sndcnt >= ctf_fixed_maxseg(tp)) && - ((rack->rc_inp->inp_in_hpts == 0) && + ((tcp_in_hpts(rack->rc_inp) == 0) && ((rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0)))) { /* * If you are pacing output you don't want @@ -13232,7 +13232,7 @@ rack_timer_audit(struct tcpcb *tp, struct tcp_rack *rack, struct sockbuf *sb) * We will force the hpts to be stopped if any, and restart * with the slot set to what was in the saved slot. */ - if (rack->rc_inp->inp_in_hpts) { + if (tcp_in_hpts(rack->rc_inp)) { if (rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) { uint32_t us_cts; @@ -13328,7 +13328,7 @@ rack_log_input_packet(struct tcpcb *tp, struct tcp_rack *rack, struct tcp_ackent } #endif memset(&log.u_bbr, 0, sizeof(log.u_bbr)); - log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts; + log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp); log.u_bbr.ininput = rack->rc_inp->inp_in_input; if (rack->rack_no_prr == 0) log.u_bbr.flex1 = rack->r_ctl.rc_prr_sndcnt; @@ -14320,7 +14320,7 @@ rack_do_segment_nounlock(struct mbuf *m, struct tcphdr *th, struct socket *so, } #endif memset(&log.u_bbr, 0, sizeof(log.u_bbr)); - log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts; + log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp); log.u_bbr.ininput = rack->rc_inp->inp_in_input; if (rack->rack_no_prr == 0) log.u_bbr.flex1 = rack->r_ctl.rc_prr_sndcnt; @@ -14679,13 +14679,13 @@ do_output_now: /* We could not send (probably in the hpts but stopped the timer earlier)? */ if ((tp->snd_max == tp->snd_una) && ((tp->t_flags & TF_DELACK) == 0) && - (rack->rc_inp->inp_in_hpts) && + (tcp_in_hpts(rack->rc_inp)) && (rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)) { /* keep alive not needed if we are hptsi output yet */ ; } else { int late = 0; - if (rack->rc_inp->inp_in_hpts) { + if (tcp_in_hpts(rack->rc_inp)) { if (rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) { us_cts = tcp_get_usecs(NULL); if (TSTMP_GT(rack->r_ctl.rc_last_output_to, us_cts)) { @@ -15611,7 +15611,7 @@ rack_log_fsb(struct tcp_rack *rack, struct tcpcb *tp, struct socket *so, uint32_ struct timeval tv; memset(&log.u_bbr, 0, sizeof(log.u_bbr)); - log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts; + log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp); log.u_bbr.ininput = rack->rc_inp->inp_in_input; log.u_bbr.flex1 = error; log.u_bbr.flex2 = flags; @@ -16127,7 +16127,7 @@ rack_fast_rsm_output(struct tcpcb *tp, struct tcp_rack *rack, struct rack_sendma union tcp_log_stackspecific log; memset(&log.u_bbr, 0, sizeof(log.u_bbr)); - log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts; + log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp); log.u_bbr.ininput = rack->rc_inp->inp_in_input; if (rack->rack_no_prr) log.u_bbr.flex1 = 0; @@ -16628,7 +16628,7 @@ again: union tcp_log_stackspecific log; memset(&log.u_bbr, 0, sizeof(log.u_bbr)); - log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts; + log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp); log.u_bbr.ininput = rack->rc_inp->inp_in_input; if (rack->rack_no_prr) log.u_bbr.flex1 = 0; @@ -16867,7 +16867,7 @@ rack_output(struct tcpcb *tp) cts = tcp_get_usecs(&tv); ms_cts = tcp_tv_to_mssectick(&tv); if (((rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0) && - rack->rc_inp->inp_in_hpts) { + tcp_in_hpts(rack->rc_inp)) { /* * We are on the hpts for some timer but not hptsi output. * Remove from the hpts unconditionally. @@ -16893,7 +16893,7 @@ rack_output(struct tcpcb *tp) } } if (rack->rc_in_persist) { - if (rack->rc_inp->inp_in_hpts == 0) { + if (tcp_in_hpts(rack->rc_inp) == 0) { /* Timer is not running */ rack_start_hpts_timer(rack, tp, cts, 0, 0, 0); } @@ -16907,9 +16907,9 @@ rack_output(struct tcpcb *tp) (delayed) || (tp->t_state < TCPS_ESTABLISHED)) { rack->rc_ack_can_sendout_data = 0; - if (rack->rc_inp->inp_in_hpts) + if (tcp_in_hpts(rack->rc_inp)) tcp_hpts_remove(rack->rc_inp, HPTS_REMOVE_OUTPUT); - } else if (rack->rc_inp->inp_in_hpts) { + } else if (tcp_in_hpts(rack->rc_inp)) { /* * On the hpts you can't pass even if ACKNOW is on, we will * when the hpts fires. @@ -18800,7 +18800,7 @@ send: union tcp_log_stackspecific log; memset(&log.u_bbr, 0, sizeof(log.u_bbr)); - log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts; + log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp); log.u_bbr.ininput = rack->rc_inp->inp_in_input; if (rack->rack_no_prr) log.u_bbr.flex1 = 0;
