On Tue, Dec 20, 2022 at 01:02:23AM -0500, Geoff Steckel wrote:
>
> On 12/19/22 21:07, Kevin Lo wrote:
> > On Mon, Dec 19, 2022 at 03:50:45PM -0500, Geoff Steckel wrote:
> > > Thanks for all the suggestions:
> > >
> > > sysctl kern.pool_debug=1 = no change
> > > known working board in same slot = no change
> > >
> > > hardware version is indeed 06090000
> > > em(4) in same slot = works
> > > test using old rge(4) board between two Linux systems = works
> > >
> > > Are any other drivers similar enough for me to compare with if_rge.c?
> > > Perhaps the AMD 5600G or the B550 chipset have quirks not seen before?
> > >
> > > I could possibly install FreeBSD if that would give any information.
> > The diff below syncs up the Rx descriptor setup code to the upstream.
> > It should fix the problem.
> >
> > Index: sys/dev/pci/if_rge.c
> > ===================================================================
> > RCS file: /cvs/src/sys/dev/pci/if_rge.c,v
> > retrieving revision 1.20
> > diff -u -p -u -p -r1.20 if_rge.c
> > --- sys/dev/pci/if_rge.c 20 Nov 2022 23:47:51 -0000 1.20
> > +++ sys/dev/pci/if_rge.c 20 Dec 2022 01:54:30 -0000
> > @@ -1104,24 +1104,16 @@ rge_newbuf(struct rge_queues *q)
> > /* Map the segments into RX descriptors. */
> > r = &q->q_rx.rge_rx_list[idx];
> > - if (RGE_OWN(r)) {
> > - printf("%s: tried to map busy RX descriptor\n",
> > - sc->sc_dev.dv_xname);
> > - m_freem(m);
> > - return (ENOBUFS);
> > - }
> > -
> > rxq->rxq_mbuf = m;
> > - r->rge_extsts = 0;
> > - r->rge_addrlo = htole32(RGE_ADDR_LO(rxmap->dm_segs[0].ds_addr));
> > - r->rge_addrhi = htole32(RGE_ADDR_HI(rxmap->dm_segs[0].ds_addr));
> > + r->hi_qword1.rx_qword4.rge_extsts = 0;
> > + r->hi_qword0.rge_addr = htole64(rxmap->dm_segs[0].ds_addr);
> > - r->rge_cmdsts = htole32(rxmap->dm_segs[0].ds_len);
> > + r->hi_qword1.rx_qword4.rge_cmdsts = htole32(rxmap->dm_segs[0].ds_len);
> > if (idx == RGE_RX_LIST_CNT - 1)
> > - r->rge_cmdsts |= htole32(RGE_RDCMDSTS_EOR);
> > + r->hi_qword1.rx_qword4.rge_cmdsts |= htole32(RGE_RDCMDSTS_EOR);
> > - r->rge_cmdsts |= htole32(RGE_RDCMDSTS_OWN);
> > + r->hi_qword1.rx_qword4.rge_cmdsts |= htole32(RGE_RDCMDSTS_OWN);
> > bus_dmamap_sync(sc->sc_dmat, q->q_rx.rge_rx_list_map,
> > idx * sizeof(struct rge_rx_desc), sizeof(struct rge_rx_desc),
> > @@ -1140,11 +1132,11 @@ rge_discard_rxbuf(struct rge_queues *q,
> > r = &q->q_rx.rge_rx_list[idx];
> > - r->rge_cmdsts = htole32(RGE_JUMBO_FRAMELEN);
> > - r->rge_extsts = 0;
> > + r->hi_qword1.rx_qword4.rge_cmdsts = htole32(RGE_JUMBO_FRAMELEN);
> > + r->hi_qword1.rx_qword4.rge_extsts = 0;
> > if (idx == RGE_RX_LIST_CNT - 1)
> > - r->rge_cmdsts |= htole32(RGE_RDCMDSTS_EOR);
> > - r->rge_cmdsts |= htole32(RGE_RDCMDSTS_OWN);
> > + r->hi_qword1.rx_qword4.rge_cmdsts |= htole32(RGE_RDCMDSTS_EOR);
> > + r->hi_qword1.rx_qword4.rge_cmdsts |= htole32(RGE_RDCMDSTS_OWN);
> > bus_dmamap_sync(sc->sc_dmat, q->q_rx.rge_rx_list_map,
> > idx * sizeof(struct rge_rx_desc), sizeof(struct rge_rx_desc),
> > @@ -1219,8 +1211,8 @@ rge_rxeof(struct rge_queues *q)
> > if (RGE_OWN(cur_rx))
> > break;
> > - rxstat = letoh32(cur_rx->rge_cmdsts);
> > - extsts = letoh32(cur_rx->rge_extsts);
> > + rxstat = letoh32(cur_rx->hi_qword1.rx_qword4.rge_cmdsts);
> > + extsts = letoh32(cur_rx->hi_qword1.rx_qword4.rge_extsts);
> >
> > total_len = RGE_RXBYTES(cur_rx);
> > rxq = &q->q_rx.rge_rxq[i];
> > @@ -1282,16 +1274,16 @@ rge_rxeof(struct rge_queues *q)
> > (total_len - ETHER_CRC_LEN);
> > /* Check IP header checksum. */
> > - if (!(rxstat & RGE_RDCMDSTS_IPCSUMERR) &&
> > + if (!(extsts & RGE_RDEXTSTS_IPCSUMERR) &&
> > (extsts & RGE_RDEXTSTS_IPV4))
> > m->m_pkthdr.csum_flags |= M_IPV4_CSUM_IN_OK;
> > /* Check TCP/UDP checksum. */
> > if ((extsts & (RGE_RDEXTSTS_IPV4 | RGE_RDEXTSTS_IPV6)) &&
> > - (((rxstat & RGE_RDCMDSTS_TCPPKT) &&
> > - !(rxstat & RGE_RDCMDSTS_TCPCSUMERR)) ||
> > - ((rxstat & RGE_RDCMDSTS_UDPPKT) &&
> > - !(rxstat & RGE_RDCMDSTS_UDPCSUMERR))))
> > + (((extsts & RGE_RDEXTSTS_TCPPKT) &&
> > + !(extsts & RGE_RDEXTSTS_TCPCSUMERR)) ||
> > + ((extsts & RGE_RDEXTSTS_UDPPKT) &&
> > + !(extsts & RGE_RDEXTSTS_UDPCSUMERR))))
> > m->m_pkthdr.csum_flags |= M_TCP_CSUM_IN_OK |
> > M_UDP_CSUM_IN_OK;
> > Index: sys/dev/pci/if_rgereg.h
> > ===================================================================
> > RCS file: /cvs/src/sys/dev/pci/if_rgereg.h,v
> > retrieving revision 1.8
> > diff -u -p -u -p -r1.8 if_rgereg.h
> > --- sys/dev/pci/if_rgereg.h 20 Nov 2022 23:47:51 -0000 1.8
> > +++ sys/dev/pci/if_rgereg.h 20 Dec 2022 01:54:30 -0000
> > @@ -189,9 +189,10 @@
> > #define RGE_NEXT_RX_DESC(x) (((x) + 1) % RGE_RX_LIST_CNT)
> > #define RGE_ADDR_LO(y) ((uint64_t) (y) & 0xffffffff)
> > #define RGE_ADDR_HI(y) ((uint64_t) (y) >> 32)
> > -#define RGE_OWN(x) (letoh32((x)->rge_cmdsts) & RGE_RDCMDSTS_OWN)
> > -#define RGE_RXBYTES(x) (letoh32((x)->rge_cmdsts) & \
> > - RGE_RDCMDSTS_FRAGLEN)
> > +#define RGE_OWN(x) \
> > + (letoh32((x)->hi_qword1.rx_qword4.rge_cmdsts) & RGE_RDCMDSTS_OWN)
> > +#define RGE_RXBYTES(x)
> > \
> > + (letoh32((x)->hi_qword1.rx_qword4.rge_cmdsts) & RGE_RDCMDSTS_FRAGLEN)
> > #define RGE_ADV_2500TFDX 0x0080
> > @@ -219,26 +220,67 @@ struct rge_tx_desc {
> > /* Rx descriptor */
> > struct rge_rx_desc {
> > - uint32_t rge_cmdsts;
> > - uint32_t rge_extsts;
> > - uint32_t rge_addrlo;
> > - uint32_t rge_addrhi;
> > -};
> > -
> > -#define RGE_RDCMDSTS_TCPCSUMERR 0x00004000
> > -#define RGE_RDCMDSTS_UDPCSUMERR 0x00008000
> > -#define RGE_RDCMDSTS_IPCSUMERR 0x00010000
> > -#define RGE_RDCMDSTS_TCPPKT 0x00020000
> > -#define RGE_RDCMDSTS_UDPPKT 0x00040000
> > -#define RGE_RDCMDSTS_RXERRSUM 0x00200000
> > -#define RGE_RDCMDSTS_EOF 0x10000000
> > -#define RGE_RDCMDSTS_SOF 0x20000000
> > + union {
> > + struct {
> > + uint32_t rsvd0;
> > + uint32_t rsvd1;
> > + } rx_qword0;
> > + } lo_qword0;
> > +
> > + union {
> > + struct {
> > + uint32_t rss;
> > + uint16_t length;
> > + uint16_t hdr_info;
> > + } rx_qword1;
> > +
> > + struct {
> > + uint32_t rsvd2;
> > + uint32_t rsvd3;
> > + } rx_qword2;
> > + } lo_qword1;
> > +
> > + union {
> > + uint64_t rge_addr;
> > +
> > + struct {
> > + uint64_t timestamp;
> > + } rx_timestamp;
> > +
> > + struct {
> > + uint32_t rsvd4;
> > + uint32_t rsvd5;
> > + } rx_qword3;
> > + } hi_qword0;
> > +
> > + union {
> > + struct {
> > + uint32_t rge_extsts;
> > + uint32_t rge_cmdsts;
> > + } rx_qword4;
> > +
> > + struct {
> > + uint16_t rsvd6;
> > + uint16_t rsvd7;
> > + uint32_t rsvd8;
> > + } rx_ptp;
> > + } hi_qword1;
> > +};
> > +
> > +#define RGE_RDCMDSTS_RXERRSUM 0x00100000
> > +#define RGE_RDCMDSTS_EOF 0x01000000
> > +#define RGE_RDCMDSTS_SOF 0x02000000
> > #define RGE_RDCMDSTS_EOR 0x40000000
> > #define RGE_RDCMDSTS_OWN 0x80000000
> > #define RGE_RDCMDSTS_FRAGLEN 0x00003fff
> > #define RGE_RDEXTSTS_VTAG 0x00010000
> > #define RGE_RDEXTSTS_VLAN_MASK 0x0000ffff
> > +#define RGE_RDEXTSTS_TCPCSUMERR 0x01000000
> > +#define RGE_RDEXTSTS_UDPCSUMERR 0x02000000
> > +#define RGE_RDEXTSTS_IPCSUMERR 0x04000000
> > +#define RGE_RDEXTSTS_TCPPKT 0x10000000
> > +#define RGE_RDEXTSTS_UDPPKT 0x20000000
> > #define RGE_RDEXTSTS_IPV4 0x40000000
> > #define RGE_RDEXTSTS_IPV6 0x80000000
> > @@ -332,7 +374,7 @@ enum rge_mac_type {
> > ETHER_VLAN_ENCAP_LEN)
> > #define RGE_TXCFG_CONFIG 0x03000700
> > -#define RGE_RXCFG_CONFIG 0x40c00700
> > +#define RGE_RXCFG_CONFIG 0x41c00700
> > struct kstat;
> The above patch worked afaik perfectly. 1TB of data worth, at least.
> Thanks very much to Mr. Lo and everyone else who looked at this!
Thank you for testing. I have committed the fix.
