On 27.5.2022. 18:25, Jan Klemkow wrote:
> Hi,
>
> The following diff enables the TCP Large Receive Offloading feature for
> ix(4) interfaces. It also includes a default off sysctl(2) switch.
>
> The TCP single stream receiving performance increased from 3.6 Gbit/s to
> 9.4 Gbit/s. Measured from Linux to OpenBSD with tcpbench.
>
> I tested the diff with:
> ix0 at pci3 dev 0 function 0 "Intel 82599" rev 0x01, msix, 12 queues, address
> 00:1b:21:87:fb:2c
>
> If you want to test the diff:
>
> 1. Apply the diff
> 2. Rebuild the kernel
> 3. Rebuild header files
> # cd /usr/src && make includes
Hi,
I'm sorry but I' stuck here.
smc24# cd /usr/src && make includes
cd /usr/src/include && su build -c 'exec make prereq' && exec make
includes
preparing in /usr/src/include/../lib/libcrypto
cat /usr/src/lib/libcrypto/objects/obj_mac.num > obj_mac.num.tmp
/bin/sh: cannot create obj_mac.num.tmp: Permission denied
*** Error 1 in lib/libcrypto (Makefile:460 'obj_mac.h')
*** Error 2 in include (Makefile:81 'prereq': @for i in ../lib/libcrypto
../lib/librpcsvc; do echo preparing in /usr/src/include/$i; cd /u...)
*** Error 2 in /usr/src (Makefile:55 'includes')
I'm doing "make includes" as root ..
> 4. Rebuild sysctl(8)
> # cd /usr/src/sbin/sysctl && make && make install
> 5. Reboot
> 6. Enable the feature
> # sysctl net.inet.tcp.large_recv_offload=1
> # ifconfig ix0 down && ifconfig ix0 up
>
> I tested this diff for a while in different scenarios (receiving,
> routing, relaying) without noticing any problems yet.
>
> bluhm@ already suggested that I could change the feature switch from a
> global sysctl(2) to an per interface ifconfig(8) option. This would
> give the user more control.
>
> Tests with other ix(4) NICs are welcome and needed!
>
> bye,
> Jan
>
> Index: dev/pci/if_ix.c
> ===================================================================
> RCS file: /cvs/src/sys/dev/pci/if_ix.c,v
> retrieving revision 1.185
> diff -u -p -r1.185 if_ix.c
> --- dev/pci/if_ix.c 15 Mar 2022 11:22:10 -0000 1.185
> +++ dev/pci/if_ix.c 23 May 2022 14:39:45 -0000
> @@ -2870,7 +2870,7 @@ ixgbe_initialize_receive_units(struct ix
> {
> struct rx_ring *rxr = sc->rx_rings;
> struct ixgbe_hw *hw = &sc->hw;
> - uint32_t bufsz, fctrl, srrctl, rxcsum;
> + uint32_t bufsz, fctrl, srrctl, rxcsum, rdrxctl;
> uint32_t hlreg;
> int i;
>
> @@ -2894,6 +2894,14 @@ ixgbe_initialize_receive_units(struct ix
> hlreg |= IXGBE_HLREG0_JUMBOEN;
> IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg);
>
> + if (tcp_lro) {
> + /* enable RSCACKC for RSC */
> + rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
> + rdrxctl |= IXGBE_RDRXCTL_RSCACKC;
> + rdrxctl |= IXGBE_RDRXCTL_FCOE_WRFIX;
> + IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
> + }
> +
> bufsz = (sc->rx_mbuf_sz - ETHER_ALIGN) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
>
> for (i = 0; i < sc->num_queues; i++, rxr++) {
> @@ -2909,6 +2917,12 @@ ixgbe_initialize_receive_units(struct ix
> /* Set up the SRRCTL register */
> srrctl = bufsz | IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
> IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
> +
> + if (tcp_lro) {
> + /* Enable Receive Side Coalescing */
> + IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(i),
> + IXGBE_RSCCTL_RSCEN|IXGBE_RSCCTL_MAXDESC_16);
> + }
>
> /* Setup the HW Rx Head and Tail Descriptor Pointers */
> IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0);
> Index: dev/pci/ixgbe.h
> ===================================================================
> RCS file: /cvs/src/sys/dev/pci/ixgbe.h,v
> retrieving revision 1.33
> diff -u -p -r1.33 ixgbe.h
> --- dev/pci/ixgbe.h 8 Feb 2022 03:38:00 -0000 1.33
> +++ dev/pci/ixgbe.h 23 May 2022 14:53:59 -0000
> @@ -61,11 +61,16 @@
> #include <net/if.h>
> #include <net/if_media.h>
> #include <net/toeplitz.h>
> +#include <net/route.h>
>
> #include <netinet/in.h>
> #include <netinet/if_ether.h>
> #include <netinet/ip.h>
> +#include <netinet/ip_ipsp.h>
> #include <netinet/ip6.h>
> +#include <netinet/tcp.h>
> +#include <netinet/tcp_timer.h>
> +#include <netinet/tcp_var.h>
>
> #if NBPFILTER > 0
> #include <net/bpf.h>
> Index: netinet/tcp_input.c
> ===================================================================
> RCS file: /cvs/src/sys/netinet/tcp_input.c,v
> retrieving revision 1.375
> diff -u -p -r1.375 tcp_input.c
> --- netinet/tcp_input.c 4 Jan 2022 06:32:39 -0000 1.375
> +++ netinet/tcp_input.c 23 May 2022 14:41:59 -0000
> @@ -126,6 +126,7 @@ struct timeval tcp_rst_ppslim_last;
> int tcp_ackdrop_ppslim = 100; /* 100pps */
> int tcp_ackdrop_ppslim_count = 0;
> struct timeval tcp_ackdrop_ppslim_last;
> +int tcp_lro = 0; /* TCP Large Receive Offload */
>
> #define TCP_PAWS_IDLE (24 * 24 * 60 * 60 * PR_SLOWHZ)
>
> Index: netinet/tcp_usrreq.c
> ===================================================================
> RCS file: /cvs/src/sys/netinet/tcp_usrreq.c,v
> retrieving revision 1.183
> diff -u -p -r1.183 tcp_usrreq.c
> --- netinet/tcp_usrreq.c 25 Feb 2022 23:51:03 -0000 1.183
> +++ netinet/tcp_usrreq.c 23 May 2022 14:43:41 -0000
> @@ -128,6 +128,7 @@ const struct sysctl_bounded_args tcpctl_
> { TCPCTL_SYN_BUCKET_LIMIT, &tcp_syn_bucket_limit, 1, INT_MAX },
> { TCPCTL_RFC3390, &tcp_do_rfc3390, 0, 2 },
> { TCPCTL_ALWAYS_KEEPALIVE, &tcp_always_keepalive, 0, 1 },
> + { TCPCTL_LRO, &tcp_lro, 0, 1 },
> };
>
> struct inpcbtable tcbtable;
> Index: netinet/tcp_var.h
> ===================================================================
> RCS file: /cvs/src/sys/netinet/tcp_var.h,v
> retrieving revision 1.139
> diff -u -p -r1.139 tcp_var.h
> --- netinet/tcp_var.h 25 Feb 2022 23:51:03 -0000 1.139
> +++ netinet/tcp_var.h 23 May 2022 14:47:49 -0000
> @@ -465,7 +465,8 @@ struct tcpstat {
> #define TCPCTL_SYN_USE_LIMIT 23 /* number of uses before reseeding
> hash */
> #define TCPCTL_ROOTONLY 24 /* return root only port bitmap */
> #define TCPCTL_SYN_HASH_SIZE 25 /* number of buckets in the hash */
> -#define TCPCTL_MAXID 26
> +#define TCPCTL_LRO 26 /* en/disable TCP Large Receive
> Offloading */
> +#define TCPCTL_MAXID 27
>
> #define TCPCTL_NAMES { \
> { 0, 0 }, \
> @@ -494,6 +495,7 @@ struct tcpstat {
> { "synuselimit", CTLTYPE_INT }, \
> { "rootonly", CTLTYPE_STRUCT }, \
> { "synhashsize", CTLTYPE_INT }, \
> + { "large_recv_offload", CTLTYPE_INT }, \
> }
>
> struct tcp_ident_mapping {
> @@ -652,6 +654,7 @@ extern int tcp_syn_bucket_limit;/* max e
> extern int tcp_syn_use_limit; /* number of uses before reseeding
> hash */
> extern struct syn_cache_set tcp_syn_cache[];
> extern int tcp_syn_cache_active; /* active syn cache, may be 0 or 1 */
> +extern int tcp_lro; /* en/disable large receive offloading
> */
>
> void tcp_canceltimers(struct tcpcb *);
> struct tcpcb *
>
