Re: Add LRO counter in ix(4)
On 18.5.2023. 10:46, Jan Klemkow wrote: > On Thu, May 18, 2023 at 12:01:44AM +0200, Alexander Bluhm wrote: >> On Tue, May 16, 2023 at 09:11:48PM +0200, Jan Klemkow wrote: >>> @@ -412,6 +412,10 @@ tcp_stats(char *name) >>> p(tcps_outhwtso, "\t\t%u output TSO packet%s hardware processed\n"); >>> p(tcps_outpkttso, "\t\t%u output TSO packet%s generated\n"); >>> p(tcps_outbadtso, "\t\t%u output TSO packet%s dropped\n"); >>> + p(tcps_inhwlro, "\t\t%u input LRO generated packet%s from hardware\n"); >>> + p(tcps_inpktlro, "\t\t%u input LRO coalesced packet%s from hardware\n"); >> ... coalesced packet%s by hardware > done > >>> + p(tcps_inbadlro, "\t\t%u input bad LRO packet%s from hardware\n"); >>> + >> Move this down to the "packets received" section. You included it >> in "packets sent". > done > >>> + /* >>> +* This function iterates over interleaved descriptors. >>> +* Thus, we reuse ph_mss as global segment counter per >>> +* TCP connection, insteat of introducing a new variable >> s/insteat/instead/ > done > > ok? > > Thanks, > Jan Hi, this diff smc4# netstat -sp tcp | egrep "TSO|LRO" 48 output TSO packets software chopped 776036704 output TSO packets hardware processed 3907800204 output TSO packets generated 0 output TSO packets dropped 152169317 input LRO generated packets by hardware 981432700 input LRO coalesced packets by hardware 0 input bad LRO packets from hardware vs old diff smc4# netstat -sp tcp | egrep "TSO|LRO" 10843097 output TSO packets software chopped 0 output TSO packets hardware processed 136103193 output TSO packets generated 0 output TSO packets dropped 4940412 input LRO generated packets from hardware 74984192 input LRO coalesced packets from hardware 0 input bad LRO packets from hardware
Re: Add LRO counter in ix(4)
On Thu, May 18, 2023 at 12:01:44AM +0200, Alexander Bluhm wrote:
> On Tue, May 16, 2023 at 09:11:48PM +0200, Jan Klemkow wrote:
> > @@ -412,6 +412,10 @@ tcp_stats(char *name)
> > p(tcps_outhwtso, "\t\t%u output TSO packet%s hardware processed\n");
> > p(tcps_outpkttso, "\t\t%u output TSO packet%s generated\n");
> > p(tcps_outbadtso, "\t\t%u output TSO packet%s dropped\n");
> > + p(tcps_inhwlro, "\t\t%u input LRO generated packet%s from hardware\n");
> > + p(tcps_inpktlro, "\t\t%u input LRO coalesced packet%s from hardware\n");
>
> ... coalesced packet%s by hardware
done
> > + p(tcps_inbadlro, "\t\t%u input bad LRO packet%s from hardware\n");
> > +
>
> Move this down to the "packets received" section. You included it
> in "packets sent".
done
> > + /*
> > +* This function iterates over interleaved descriptors.
> > +* Thus, we reuse ph_mss as global segment counter per
> > +* TCP connection, insteat of introducing a new variable
>
> s/insteat/instead/
done
ok?
Thanks,
Jan
diff --git a/sys/dev/pci/if_ix.c b/sys/dev/pci/if_ix.c
index 4119a2416dc..924a6d63236 100644
--- a/sys/dev/pci/if_ix.c
+++ b/sys/dev/pci/if_ix.c
@@ -3214,12 +3214,23 @@ ixgbe_rxeof(struct rx_ring *rxr)
sendmp = rxbuf->fmp;
rxbuf->buf = rxbuf->fmp = NULL;
- if (sendmp != NULL) /* secondary frag */
+ if (sendmp != NULL) { /* secondary frag */
sendmp->m_pkthdr.len += mp->m_len;
- else {
+
+ /*
+* This function iterates over interleaved descriptors.
+* Thus, we reuse ph_mss as global segment counter per
+* TCP connection, instead of introducing a new variable
+* in m_pkthdr.
+*/
+ if (rsccnt)
+ sendmp->m_pkthdr.ph_mss += rsccnt - 1;
+ } else {
/* first desc of a non-ps chain */
sendmp = mp;
sendmp->m_pkthdr.len = mp->m_len;
+ if (rsccnt)
+ sendmp->m_pkthdr.ph_mss = rsccnt - 1;
#if NVLAN > 0
if (sc->vlan_stripping && staterr & IXGBE_RXD_STAT_VP) {
sendmp->m_pkthdr.ether_vtag = vtag;
@@ -3241,6 +3252,21 @@ ixgbe_rxeof(struct rx_ring *rxr)
SET(sendmp->m_pkthdr.csum_flags, M_FLOWID);
}
+ if (sendmp->m_pkthdr.ph_mss == 1)
+ sendmp->m_pkthdr.ph_mss = 0;
+
+ if (sendmp->m_pkthdr.ph_mss > 0) {
+ struct ether_extracted ext;
+ uint16_t pkts = sendmp->m_pkthdr.ph_mss;
+
+ ether_extract_headers(sendmp, );
+ if (ext.tcp)
+ tcpstat_inc(tcps_inhwlro);
+ else
+ tcpstat_inc(tcps_inbadlro);
+ tcpstat_add(tcps_inpktlro, pkts);
+ }
+
ml_enqueue(, sendmp);
}
next_desc:
diff --git a/sys/netinet/tcp_usrreq.c b/sys/netinet/tcp_usrreq.c
index 120e3cc5ea7..3970636cde1 100644
--- a/sys/netinet/tcp_usrreq.c
+++ b/sys/netinet/tcp_usrreq.c
@@ -1340,6 +1340,9 @@ tcp_sysctl_tcpstat(void *oldp, size_t *oldlenp, void
*newp)
ASSIGN(tcps_outhwtso);
ASSIGN(tcps_outpkttso);
ASSIGN(tcps_outbadtso);
+ ASSIGN(tcps_inhwlro);
+ ASSIGN(tcps_inpktlro);
+ ASSIGN(tcps_inbadlro);
#undef ASSIGN
diff --git a/sys/netinet/tcp_var.h b/sys/netinet/tcp_var.h
index 0a9630d719f..e706fedd0e7 100644
--- a/sys/netinet/tcp_var.h
+++ b/sys/netinet/tcp_var.h
@@ -447,6 +447,9 @@ struct tcpstat {
u_int32_t tcps_outhwtso;/* output tso processed by hardware */
u_int32_t tcps_outpkttso; /* packets generated by tso */
u_int32_t tcps_outbadtso; /* output tso failed, packet dropped */
+ u_int32_t tcps_inhwlro; /* input lro from hardware */
+ u_int32_t tcps_inpktlro;/* packets coalessed by hardware lro */
+ u_int32_t tcps_inbadlro;/* input bad lro packets from hardware
*/
};
/*
@@ -625,6 +628,9 @@ enum tcpstat_counters {
tcps_outhwtso,
tcps_outpkttso,
tcps_outbadtso,
+ tcps_inhwlro,
+ tcps_inpktlro,
+ tcps_inbadlro,
tcps_ncounters,
};
diff --git a/usr.bin/netstat/inet.c b/usr.bin/netstat/inet.c
index e04355ed078..6db14673b97 100644
--- a/usr.bin/netstat/inet.c
+++ b/usr.bin/netstat/inet.c
@@ -439,6 +439,9 @@ tcp_stats(char *name)
p(tcps_inswcsum, "\t\t%u packet%s
Re: Add LRO counter in ix(4)
On Tue, May 16, 2023 at 09:11:48PM +0200, Jan Klemkow wrote: > @@ -412,6 +412,10 @@ tcp_stats(char *name) > p(tcps_outhwtso, "\t\t%u output TSO packet%s hardware processed\n"); > p(tcps_outpkttso, "\t\t%u output TSO packet%s generated\n"); > p(tcps_outbadtso, "\t\t%u output TSO packet%s dropped\n"); > + p(tcps_inhwlro, "\t\t%u input LRO generated packet%s from hardware\n"); > + p(tcps_inpktlro, "\t\t%u input LRO coalesced packet%s from hardware\n"); ... coalesced packet%s by hardware > + p(tcps_inbadlro, "\t\t%u input bad LRO packet%s from hardware\n"); > + Move this down to the "packets received" section. You included it in "packets sent". > + /* > + * This function iterates over interleaved descriptors. > + * Thus, we reuse ph_mss as global segment counter per > + * TCP connection, insteat of introducing a new variable s/insteat/instead/ > +struct tdb; > + This will be in tcp_var.h when you commit the other diff.
Re: Add LRO counter in ix(4)
On 16.5.2023. 21:11, Jan Klemkow wrote: > Hi, > > This diff introduces new counters for LRO packets, we get from the > network interface. It shows, how many packets the network interface has > coalesced into LRO packets. > > In followup diff, this packet counter will also be used to set the > ph_mss variable to valid value. So, the stack is able to forward or > redirect this kind of packets. Hi, when LRO is enabled with "ifconfig ix0 tcprecvoffload" LRO counter is increasing if I'm sending or receiving tcp traffic with iperf. Is this ok? TSO counter is increasing only when I'm sending traffic. ix0 at pci5 dev 0 function 0 "Intel X552 SFP+" rev 0x00, msix, 4 queues smc4# ifconfig ix0 tcprecvoffload smc4# ifconfig ix0 ix0: flags=2008843 mtu 1500 smc4# ifconfig ix0 -tcprecvoffload smc4# ifconfig ix0 ix0: flags=8843 mtu 1500 smc4# netstat -sp tcp | egrep "TSO|LRO" 10843097 output TSO packets software chopped 0 output TSO packets hardware processed 136103193 output TSO packets generated 0 output TSO packets dropped 4940412 input LRO generated packets from hardware 74984192 input LRO coalesced packets from hardware 0 input bad LRO packets from hardware It is 0 on "TSO packets hardware processed" because I didn't applied latest bluhm@ "ix hardware tso" diff
Add LRO counter in ix(4)
Hi,
This diff introduces new counters for LRO packets, we get from the
network interface. It shows, how many packets the network interface has
coalesced into LRO packets.
In followup diff, this packet counter will also be used to set the
ph_mss variable to valid value. So, the stack is able to forward or
redirect this kind of packets.
ok?
bye,
Jan
Index: usr.bin/netstat/inet.c
===
RCS file: /cvs/src/usr.bin/netstat/inet.c,v
retrieving revision 1.175
diff -u -p -r1.175 inet.c
--- usr.bin/netstat/inet.c 10 May 2023 12:07:17 - 1.175
+++ usr.bin/netstat/inet.c 16 May 2023 17:55:20 -
@@ -412,6 +412,10 @@ tcp_stats(char *name)
p(tcps_outhwtso, "\t\t%u output TSO packet%s hardware processed\n");
p(tcps_outpkttso, "\t\t%u output TSO packet%s generated\n");
p(tcps_outbadtso, "\t\t%u output TSO packet%s dropped\n");
+ p(tcps_inhwlro, "\t\t%u input LRO generated packet%s from hardware\n");
+ p(tcps_inpktlro, "\t\t%u input LRO coalesced packet%s from hardware\n");
+ p(tcps_inbadlro, "\t\t%u input bad LRO packet%s from hardware\n");
+
p(tcps_rcvtotal, "\t%u packet%s received\n");
p2(tcps_rcvackpack, tcps_rcvackbyte, "\t\t%u ack%s (for %llu
byte%s)\n");
p(tcps_rcvdupack, "\t\t%u duplicate ack%s\n");
Index: sys/dev/pci/if_ix.c
===
RCS file: /cvs/src/sys/dev/pci/if_ix.c,v
retrieving revision 1.194
diff -u -p -r1.194 if_ix.c
--- sys/dev/pci/if_ix.c 16 May 2023 14:32:54 - 1.194
+++ sys/dev/pci/if_ix.c 16 May 2023 18:49:33 -
@@ -3175,12 +3175,23 @@ ixgbe_rxeof(struct rx_ring *rxr)
sendmp = rxbuf->fmp;
rxbuf->buf = rxbuf->fmp = NULL;
- if (sendmp != NULL) /* secondary frag */
+ if (sendmp != NULL) { /* secondary frag */
sendmp->m_pkthdr.len += mp->m_len;
- else {
+
+ /*
+* This function iterates over interleaved descriptors.
+* Thus, we reuse ph_mss as global segment counter per
+* TCP connection, insteat of introducing a new variable
+* in m_pkthdr.
+*/
+ if (rsccnt)
+ sendmp->m_pkthdr.ph_mss += rsccnt - 1;
+ } else {
/* first desc of a non-ps chain */
sendmp = mp;
sendmp->m_pkthdr.len = mp->m_len;
+ if (rsccnt)
+ sendmp->m_pkthdr.ph_mss = rsccnt - 1;
#if NVLAN > 0
if (sc->vlan_stripping && staterr & IXGBE_RXD_STAT_VP) {
sendmp->m_pkthdr.ether_vtag = vtag;
@@ -3200,6 +3211,21 @@ ixgbe_rxeof(struct rx_ring *rxr)
if (hashtype != IXGBE_RXDADV_RSSTYPE_NONE) {
sendmp->m_pkthdr.ph_flowid = hash;
SET(sendmp->m_pkthdr.csum_flags, M_FLOWID);
+ }
+
+ if (sendmp->m_pkthdr.ph_mss == 1)
+ sendmp->m_pkthdr.ph_mss = 0;
+
+ if (sendmp->m_pkthdr.ph_mss > 0) {
+ struct ether_extracted ext;
+ uint16_t pkts = sendmp->m_pkthdr.ph_mss;
+
+ ether_extract_headers(sendmp, );
+ if (ext.tcp)
+ tcpstat_inc(tcps_inhwlro);
+ else
+ tcpstat_inc(tcps_inbadlro);
+ tcpstat_add(tcps_inpktlro, pkts);
}
ml_enqueue(, sendmp);
Index: sys/dev/pci/ixgbe.h
===
RCS file: /cvs/src/sys/dev/pci/ixgbe.h,v
retrieving revision 1.33
diff -u -p -r1.33 ixgbe.h
--- sys/dev/pci/ixgbe.h 8 Feb 2022 03:38:00 - 1.33
+++ sys/dev/pci/ixgbe.h 16 May 2023 17:55:20 -
@@ -60,12 +60,18 @@
#include
#include
+#include
#include
+struct tdb;
+
#include
#include
#include
#include
+#include
+#include
+#include
#if NBPFILTER > 0
#include
Index: sys/netinet/tcp_usrreq.c
===
RCS file: /cvs/src/sys/netinet/tcp_usrreq.c,v
retrieving revision 1.218
diff -u -p -r1.218 tcp_usrreq.c
--- sys/netinet/tcp_usrreq.c10 May 2023 12:07:16 - 1.218
+++ sys/netinet/tcp_usrreq.c16 May 2023 17:55:20 -
@@ -1340,6 +1340,9 @@ tcp_sysctl_tcpstat(void *oldp, size_t *o
ASSIGN(tcps_outhwtso);
ASSIGN(tcps_outpkttso);
ASSIGN(tcps_outbadtso);
+ ASSIGN(tcps_inhwlro);
+ ASSIGN(tcps_inpktlro);
+
