The second generation ix(4) (82599) produce a low latency interrupt when
the RX DMA ring gets full. This interrupt is currently unmoderated. On
overload mclgeti() shrinks the ring to very small sizes and ix(4) will
answer that with a massive interrupt storm. Enabling interrupt moderation
on the LLI is possible and we should enable it since we do not use the
LLI for any other low latency reason.
With this diff the throughput of my test system increased from under
200kpps to almost 500kpps when overloading the system with 1Mpps on the
input side.
Please test on 82598 and 82599.
--
:wq Claudio
Index: if_ix.c
===================================================================
RCS file: /cvs/src/sys/dev/pci/if_ix.c,v
retrieving revision 1.45
diff -u -p -r1.45 if_ix.c
--- if_ix.c 27 Oct 2010 20:48:27 -0000 1.45
+++ if_ix.c 5 Nov 2010 10:20:27 -0000
@@ -657,6 +657,9 @@ ixgbe_init(void *arg)
if (sc->hw.mac.type == ixgbe_mac_82599EB) {
gpie |= IXGBE_SDP1_GPIEN;
gpie |= IXGBE_SDP2_GPIEN;
+ /* set LL interval to max to reduce the number of low latency
+ * interrupts hitting the card when the ring is getting full */
+ gpie |= 0xf << IXGBE_GPIE_LLI_DELAY_SHIFT;
}
/* Enable Fan Failure Interrupt */
@@ -2483,6 +2486,7 @@ ixgbe_initialize_receive_units(struct ix
uint32_t rxctrl, fctrl, srrctl, rxcsum;
uint32_t reta, mrqc, hlreg, linkvec;
uint32_t random[10];
+ uint32_t llimod = 0;
int i;
/*
@@ -2516,8 +2520,10 @@ ixgbe_initialize_receive_units(struct ix
IXGBE_WRITE_REG(&sc->hw, IXGBE_SRRCTL(0), srrctl);
/* Set Queue moderation rate */
+ if (sc->hw.mac.type == ixgbe_mac_82599EB)
+ llimod = IXGBE_EITR_LLI_MOD;
for (i = 0; i < IXGBE_MSGS; i++)
- IXGBE_WRITE_REG(&sc->hw, IXGBE_EITR(i), DEFAULT_ITR);
+ IXGBE_WRITE_REG(&sc->hw, IXGBE_EITR(i), DEFAULT_ITR | llimod);
/* Set Link moderation lower */
linkvec = sc->num_tx_queues + sc->num_rx_queues;
Index: ixgbe_type.h
===================================================================
RCS file: /cvs/src/sys/dev/pci/ixgbe_type.h,v
retrieving revision 1.6
diff -u -p -r1.6 ixgbe_type.h
--- ixgbe_type.h 25 Feb 2010 10:56:07 -0000 1.6
+++ ixgbe_type.h 5 Nov 2010 10:20:27 -0000
@@ -1077,6 +1077,7 @@
#define IXGBE_GPIE_EIMEN 0x00000040 /* Immediate Interrupt Enable */
#define IXGBE_GPIE_EIAME 0x40000000
#define IXGBE_GPIE_PBA_SUPPORT 0x80000000
+#define IXGBE_GPIE_LLI_DELAY_SHIFT 7
#define IXGBE_GPIE_RSC_DELAY_SHIFT 11
#define IXGBE_GPIE_VTMODE_MASK 0x0000C000 /* VT Mode Mask */
#define IXGBE_GPIE_VTMODE_16 0x00004000 /* 16 VFs 8 queues per VF */
