On 8/6/2022 3:04 PM, Jim King wrote:
On 8/5/2022 12:47 PM, mike tancsa wrote:
On 8/5/2022 11:41 AM, Jim King wrote:
If I fiddle with the Flow Control settings it seems to make a
difference. I can do it without bouncing the link. I wonder what
the Linux driver has as default ?
ie.
sysctl -w dev.igc.1.fc=1
sysctl -w dev.igc.0.fc=1
Does it help in your case ?
Thanks for the suggestion. Unfortunately it doesn't seem to help in
my case. All four values for flow control give pretty much the same
results with my iperf test.
Are you able to boot a RELENG_13 stable kernel and test ? I am using
a stable kernel from a couple of days ago and setting the fc=0 (you
set that on all nics right ?) does work for me both at 2.5G and 1G.
Leaving the default causes the nic to bounce on my test boxes. I dont
see any obvious big changes in the driver itself since 13.1 came out
so I am guessing something else fixed or worked around this bug ?
I build stable/13 from yesterday and still haven't found a working
configuration. When I turn off Flow Control on both NICs my first
iperf test always causes the NIC to bounce, and sometimes subsequent
tests will work OK, but only sometimes. Most of the time the
subsequent iperf runs also cause the NIC to bounce.
I also tried -rxcsum and -txcsum on the NIC, which had no effect.
I let my unit run over night and it as well saw some drops. I wonder if
the relatively underpowered CPU doesnt generate enough traffic to cause
it to bounce as often? Not sure if there is anything in the sysctl logs
or not, but this is a before and after below. I guess I will open a PR.
Not sure if this driver is managed by Intel or not. I think the netgate
people did it initially
root@topton1:/var/tmp/reports # diff -u 1659776941b 1659777187a
--- 1659776941b 2022-08-06 05:11:03.514094000 -0400
+++ 1659777187a 2022-08-06 05:14:09.675013000 -0400
@@ -1,35 +1,35 @@
dev.igc.1.wake: 0
dev.igc.1.interrupts.rx_desc_min_thresh: 0
-dev.igc.1.interrupts.asserts: 704721082
-dev.igc.1.mac_stats.tso_txd: 400062204
-dev.igc.1.mac_stats.tx_frames_1024_1522: 4870151883
-dev.igc.1.mac_stats.tx_frames_512_1023: 38990
-dev.igc.1.mac_stats.tx_frames_256_511: 17758
-dev.igc.1.mac_stats.tx_frames_128_255: 4764
-dev.igc.1.mac_stats.tx_frames_65_127: 569597300
-dev.igc.1.mac_stats.tx_frames_64: 36196
+dev.igc.1.interrupts.asserts: 707056694
+dev.igc.1.mac_stats.tso_txd: 400573812
+dev.igc.1.mac_stats.tx_frames_1024_1522: 4882409702
+dev.igc.1.mac_stats.tx_frames_512_1023: 38993
+dev.igc.1.mac_stats.tx_frames_256_511: 17762
+dev.igc.1.mac_stats.tx_frames_128_255: 4769
+dev.igc.1.mac_stats.tx_frames_65_127: 572463728
+dev.igc.1.mac_stats.tx_frames_64: 36307
dev.igc.1.mac_stats.mcast_pkts_txd: 0
dev.igc.1.mac_stats.bcast_pkts_txd: 30
-dev.igc.1.mac_stats.good_pkts_txd: 5439846882
-dev.igc.1.mac_stats.total_pkts_txd: 5439846888
-dev.igc.1.mac_stats.good_octets_txd: 7432797311490
-dev.igc.1.mac_stats.good_octets_recvd: 7400832173605
-dev.igc.1.mac_stats.rx_frames_1024_1522: 4849042867
-dev.igc.1.mac_stats.rx_frames_512_1023: 62119
-dev.igc.1.mac_stats.rx_frames_256_511: 28123
-dev.igc.1.mac_stats.rx_frames_128_255: 5910
-dev.igc.1.mac_stats.rx_frames_65_127: 570440786
-dev.igc.1.mac_stats.rx_frames_64: 49169
+dev.igc.1.mac_stats.good_pkts_txd: 5454971260
+dev.igc.1.mac_stats.total_pkts_txd: 5454971261
+dev.igc.1.mac_stats.good_octets_txd: 7451605352901
+dev.igc.1.mac_stats.good_octets_recvd: 7435895205487
+dev.igc.1.mac_stats.rx_frames_1024_1522: 4872086268
+dev.igc.1.mac_stats.rx_frames_512_1023: 64952
+dev.igc.1.mac_stats.rx_frames_256_511: 29389
+dev.igc.1.mac_stats.rx_frames_128_255: 6311
+dev.igc.1.mac_stats.rx_frames_65_127: 571594440
+dev.igc.1.mac_stats.rx_frames_64: 49437
dev.igc.1.mac_stats.mcast_pkts_recvd: 0
dev.igc.1.mac_stats.bcast_pkts_recvd: 19
-dev.igc.1.mac_stats.good_pkts_recvd: 5419628981
-dev.igc.1.mac_stats.total_pkts_recvd: 5419629039
+dev.igc.1.mac_stats.good_pkts_recvd: 5443830806
+dev.igc.1.mac_stats.total_pkts_recvd: 5443830878
dev.igc.1.mac_stats.xoff_txd: 0
dev.igc.1.mac_stats.xoff_recvd: 14
dev.igc.1.mac_stats.xon_txd: 0
dev.igc.1.mac_stats.xon_recvd: 0
dev.igc.1.mac_stats.alignment_errs: 0
-dev.igc.1.mac_stats.crc_errs: 6
+dev.igc.1.mac_stats.crc_errs: 8
dev.igc.1.mac_stats.recv_errs: 0
dev.igc.1.mac_stats.recv_jabber: 0
dev.igc.1.mac_stats.recv_oversize: 0
@@ -40,43 +40,43 @@
dev.igc.1.mac_stats.defer_count: 0
dev.igc.1.mac_stats.sequence_errors: 0
dev.igc.1.mac_stats.symbol_errors: 0
-dev.igc.1.mac_stats.collision_count: 6
+dev.igc.1.mac_stats.collision_count: 8
dev.igc.1.mac_stats.late_coll: 0
dev.igc.1.mac_stats.multiple_coll: 0
dev.igc.1.mac_stats.single_coll: 0
dev.igc.1.mac_stats.excess_coll: 0
dev.igc.1.queue_rx_3.rx_irq: 0
-dev.igc.1.queue_rx_3.rxd_tail: 907
-dev.igc.1.queue_rx_3.rxd_head: 908
+dev.igc.1.queue_rx_3.rxd_tail: 128
+dev.igc.1.queue_rx_3.rxd_head: 0
dev.igc.1.queue_rx_2.rx_irq: 0
-dev.igc.1.queue_rx_2.rxd_tail: 809
-dev.igc.1.queue_rx_2.rxd_head: 812
+dev.igc.1.queue_rx_2.rxd_tail: 722
+dev.igc.1.queue_rx_2.rxd_head: 723
dev.igc.1.queue_rx_1.rx_irq: 0
-dev.igc.1.queue_rx_1.rxd_tail: 401
-dev.igc.1.queue_rx_1.rxd_head: 402
+dev.igc.1.queue_rx_1.rxd_tail: 83
+dev.igc.1.queue_rx_1.rxd_head: 84
dev.igc.1.queue_rx_0.rx_irq: 0
-dev.igc.1.queue_rx_0.rxd_tail: 521
-dev.igc.1.queue_rx_0.rxd_head: 522
+dev.igc.1.queue_rx_0.rxd_tail: 228
+dev.igc.1.queue_rx_0.rxd_head: 229
dev.igc.1.queue_tx_3.tx_irq: 0
-dev.igc.1.queue_tx_3.txd_tail: 1023
-dev.igc.1.queue_tx_3.txd_head: 1023
+dev.igc.1.queue_tx_3.txd_tail: 750
+dev.igc.1.queue_tx_3.txd_head: 750
dev.igc.1.queue_tx_2.tx_irq: 0
-dev.igc.1.queue_tx_2.txd_tail: 770
-dev.igc.1.queue_tx_2.txd_head: 914
+dev.igc.1.queue_tx_2.txd_tail: 116
+dev.igc.1.queue_tx_2.txd_head: 116
dev.igc.1.queue_tx_1.tx_irq: 0
-dev.igc.1.queue_tx_1.txd_tail: 184
-dev.igc.1.queue_tx_1.txd_head: 643
+dev.igc.1.queue_tx_1.txd_tail: 347
+dev.igc.1.queue_tx_1.txd_head: 347
dev.igc.1.queue_tx_0.tx_irq: 0
-dev.igc.1.queue_tx_0.txd_tail: 210
-dev.igc.1.queue_tx_0.txd_head: 210
+dev.igc.1.queue_tx_0.txd_tail: 17
+dev.igc.1.queue_tx_0.txd_head: 17
dev.igc.1.fc_low_water: 32752
dev.igc.1.fc_high_water: 32768
dev.igc.1.rx_control: 71335966
dev.igc.1.device_control: 404489793
dev.igc.1.watchdog_timeouts: 0
dev.igc.1.rx_overruns: 0
-dev.igc.1.link_irq: 38
-dev.igc.1.dropped: 6
+dev.igc.1.link_irq: 42
+dev.igc.1.dropped: 8
dev.igc.1.eee_control: 1
dev.igc.1.itr: 488
dev.igc.1.tx_abs_int_delay: 66
@@ -90,13 +90,13 @@
CTRL_EXIT 10000040
Interrupt Registers
- ICR 00000001
+ ICR 00000000
RX Registers
RCTL 0440801e
RDLEN 00004000
- RDH 0000020a
- RDT 00000209
+ RDH 000000e5
+ RDT 000000e4
RXDCTL 02040808
RDBAL 02f44000
RDBAH 00000000
@@ -106,8 +106,8 @@
TDBAL 02f06000
TDBAH 00000000
TDLEN 00004000
- TDH 000000d2
- TDT 000000d2
+ TDH 00000011
+ TDT 00000011
TXDCTL 0201011f
TDFH 00000000
TDFT 00000000
@@ -119,38 +119,38 @@
dev.igc.1.debug: -1
dev.igc.1.nvm: -1
dev.igc.1.iflib.rxq3.rxq_fl0.buf_size: 2048
-dev.igc.1.iflib.rxq3.rxq_fl0.credits: 1023
-dev.igc.1.iflib.rxq3.rxq_fl0.cidx: 908
-dev.igc.1.iflib.rxq3.rxq_fl0.pidx: 907
+dev.igc.1.iflib.rxq3.rxq_fl0.credits: 128
+dev.igc.1.iflib.rxq3.rxq_fl0.cidx: 0
+dev.igc.1.iflib.rxq3.rxq_fl0.pidx: 128
dev.igc.1.iflib.rxq3.cpu: 3
dev.igc.1.iflib.rxq2.rxq_fl0.buf_size: 2048
dev.igc.1.iflib.rxq2.rxq_fl0.credits: 1023
-dev.igc.1.iflib.rxq2.rxq_fl0.cidx: 827
-dev.igc.1.iflib.rxq2.rxq_fl0.pidx: 826
+dev.igc.1.iflib.rxq2.rxq_fl0.cidx: 723
+dev.igc.1.iflib.rxq2.rxq_fl0.pidx: 722
dev.igc.1.iflib.rxq2.cpu: 2
dev.igc.1.iflib.rxq1.rxq_fl0.buf_size: 2048
dev.igc.1.iflib.rxq1.rxq_fl0.credits: 1023
-dev.igc.1.iflib.rxq1.rxq_fl0.cidx: 414
-dev.igc.1.iflib.rxq1.rxq_fl0.pidx: 413
+dev.igc.1.iflib.rxq1.rxq_fl0.cidx: 84
+dev.igc.1.iflib.rxq1.rxq_fl0.pidx: 83
dev.igc.1.iflib.rxq1.cpu: 1
dev.igc.1.iflib.rxq0.rxq_fl0.buf_size: 2048
dev.igc.1.iflib.rxq0.rxq_fl0.credits: 1023
-dev.igc.1.iflib.rxq0.rxq_fl0.cidx: 522
-dev.igc.1.iflib.rxq0.rxq_fl0.pidx: 521
+dev.igc.1.iflib.rxq0.rxq_fl0.cidx: 229
+dev.igc.1.iflib.rxq0.rxq_fl0.pidx: 228
dev.igc.1.iflib.rxq0.cpu: 0
dev.igc.1.iflib.txq3.r_abdications: 0
-dev.igc.1.iflib.txq3.r_restarts: 185763
-dev.igc.1.iflib.txq3.r_stalls: 185763
-dev.igc.1.iflib.txq3.r_starts: 22126656
+dev.igc.1.iflib.txq3.r_restarts: 0
+dev.igc.1.iflib.txq3.r_stalls: 0
+dev.igc.1.iflib.txq3.r_starts: 104312
dev.igc.1.iflib.txq3.r_drops: 0
-dev.igc.1.iflib.txq3.r_enqueues: 23334820
-dev.igc.1.iflib.txq3.ring_state: pidx_head: 0430 pidx_tail: 0430 cidx:
0430 state: IDLE
-dev.igc.1.iflib.txq3.txq_cleaned: 106314707
-dev.igc.1.iflib.txq3.txq_processed: 106314747
+dev.igc.1.iflib.txq3.r_enqueues: 104513
+dev.igc.1.iflib.txq3.ring_state: pidx_head: 1960 pidx_tail: 1960 cidx:
1960 state: IDLE
+dev.igc.1.iflib.txq3.txq_cleaned: 208578
+dev.igc.1.iflib.txq3.txq_processed: 208618
dev.igc.1.iflib.txq3.txq_in_use: 44
-dev.igc.1.iflib.txq3.txq_cidx_processed: 1019
-dev.igc.1.iflib.txq3.txq_cidx: 979
-dev.igc.1.iflib.txq3.txq_pidx: 1023
+dev.igc.1.iflib.txq3.txq_cidx_processed: 746
+dev.igc.1.iflib.txq3.txq_cidx: 706
+dev.igc.1.iflib.txq3.txq_pidx: 750
dev.igc.1.iflib.txq3.no_tx_dma_setup: 0
dev.igc.1.iflib.txq3.txd_encap_efbig: 0
dev.igc.1.iflib.txq3.tx_map_failed: 0
@@ -160,18 +160,18 @@
dev.igc.1.iflib.txq3.mbuf_defrag: 0
dev.igc.1.iflib.txq3.cpu: 3
dev.igc.1.iflib.txq2.r_abdications: 0
-dev.igc.1.iflib.txq2.r_restarts: 360876
-dev.igc.1.iflib.txq2.r_stalls: 360877
-dev.igc.1.iflib.txq2.r_starts: 24509109
+dev.igc.1.iflib.txq2.r_restarts: 0
+dev.igc.1.iflib.txq2.r_stalls: 0
+dev.igc.1.iflib.txq2.r_starts: 99386
dev.igc.1.iflib.txq2.r_drops: 0
-dev.igc.1.iflib.txq2.r_enqueues: 26698302
-dev.igc.1.iflib.txq2.ring_state: pidx_head: 0070 pidx_tail: 0070 cidx:
1883 state: STALLED
-dev.igc.1.iflib.txq2.txq_cleaned: 96507779
-dev.igc.1.iflib.txq2.txq_processed: 96507819
-dev.igc.1.iflib.txq2.txq_in_use: 987
-dev.igc.1.iflib.txq2.txq_cidx_processed: 939
-dev.igc.1.iflib.txq2.txq_cidx: 899
-dev.igc.1.iflib.txq2.txq_pidx: 862
+dev.igc.1.iflib.txq2.r_enqueues: 99386
+dev.igc.1.iflib.txq2.ring_state: pidx_head: 0394 pidx_tail: 0394 cidx:
0394 state: IDLE
+dev.igc.1.iflib.txq2.txq_cleaned: 198730
+dev.igc.1.iflib.txq2.txq_processed: 198770
+dev.igc.1.iflib.txq2.txq_in_use: 42
+dev.igc.1.iflib.txq2.txq_cidx_processed: 114
+dev.igc.1.iflib.txq2.txq_cidx: 74
+dev.igc.1.iflib.txq2.txq_pidx: 116
dev.igc.1.iflib.txq2.no_tx_dma_setup: 0
dev.igc.1.iflib.txq2.txd_encap_efbig: 0
dev.igc.1.iflib.txq2.tx_map_failed: 0
@@ -181,18 +181,18 @@
dev.igc.1.iflib.txq2.mbuf_defrag: 0
dev.igc.1.iflib.txq2.cpu: 2
dev.igc.1.iflib.txq1.r_abdications: 0
-dev.igc.1.iflib.txq1.r_restarts: 1133594
-dev.igc.1.iflib.txq1.r_stalls: 1133594
-dev.igc.1.iflib.txq1.r_starts: 19099238
+dev.igc.1.iflib.txq1.r_restarts: 0
+dev.igc.1.iflib.txq1.r_stalls: 0
+dev.igc.1.iflib.txq1.r_starts: 751030
dev.igc.1.iflib.txq1.r_drops: 0
-dev.igc.1.iflib.txq1.r_enqueues: 26635148
-dev.igc.1.iflib.txq1.ring_state: pidx_head: 0088 pidx_tail: 0088 cidx:
0088 state: IDLE
-dev.igc.1.iflib.txq1.txq_cleaned: 134333877
-dev.igc.1.iflib.txq1.txq_processed: 134333917
-dev.igc.1.iflib.txq1.txq_in_use: 771
-dev.igc.1.iflib.txq1.txq_cidx_processed: 477
-dev.igc.1.iflib.txq1.txq_cidx: 437
-dev.igc.1.iflib.txq1.txq_pidx: 184
+dev.igc.1.iflib.txq1.r_enqueues: 751120
+dev.igc.1.iflib.txq1.ring_state: pidx_head: 0145 pidx_tail: 0145 cidx:
0145 state: IDLE
+dev.igc.1.iflib.txq1.txq_cleaned: 6523185
+dev.igc.1.iflib.txq1.txq_processed: 6523225
+dev.igc.1.iflib.txq1.txq_in_use: 42
+dev.igc.1.iflib.txq1.txq_cidx_processed: 345
+dev.igc.1.iflib.txq1.txq_cidx: 305
+dev.igc.1.iflib.txq1.txq_pidx: 347
dev.igc.1.iflib.txq1.no_tx_dma_setup: 0
dev.igc.1.iflib.txq1.txd_encap_efbig: 0
dev.igc.1.iflib.txq1.tx_map_failed: 0
@@ -202,18 +202,18 @@
dev.igc.1.iflib.txq1.mbuf_defrag: 0
dev.igc.1.iflib.txq1.cpu: 1
dev.igc.1.iflib.txq0.r_abdications: 0
-dev.igc.1.iflib.txq0.r_restarts: 825923
-dev.igc.1.iflib.txq0.r_stalls: 825923
-dev.igc.1.iflib.txq0.r_starts: 31553019
+dev.igc.1.iflib.txq0.r_restarts: 0
+dev.igc.1.iflib.txq0.r_stalls: 0
+dev.igc.1.iflib.txq0.r_starts: 8
dev.igc.1.iflib.txq0.r_drops: 0
-dev.igc.1.iflib.txq0.r_enqueues: 37045348
-dev.igc.1.iflib.txq0.ring_state: pidx_head: 1471 pidx_tail: 1471 cidx:
1471 state: IDLE
-dev.igc.1.iflib.txq0.txq_cleaned: 153006246
-dev.igc.1.iflib.txq0.txq_processed: 153006286
-dev.igc.1.iflib.txq0.txq_in_use: 44
-dev.igc.1.iflib.txq0.txq_cidx_processed: 206
-dev.igc.1.iflib.txq0.txq_cidx: 166
-dev.igc.1.iflib.txq0.txq_pidx: 210
+dev.igc.1.iflib.txq0.r_enqueues: 8
+dev.igc.1.iflib.txq0.ring_state: pidx_head: 0816 pidx_tail: 0816 cidx:
0816 state: IDLE
+dev.igc.1.iflib.txq0.txq_cleaned: 0
+dev.igc.1.iflib.txq0.txq_processed: 15
+dev.igc.1.iflib.txq0.txq_in_use: 17
+dev.igc.1.iflib.txq0.txq_cidx_processed: 15
+dev.igc.1.iflib.txq0.txq_cidx: 0
+dev.igc.1.iflib.txq0.txq_pidx: 17
dev.igc.1.iflib.txq0.no_tx_dma_setup: 0
dev.igc.1.iflib.txq0.txd_encap_efbig: 0
dev.igc.1.iflib.txq0.tx_map_failed: 0
