On Mon, Jun 23, 2025 at 03:23:27PM +0100, Mark de Vries wrote:
> Greetings,
> 
> I am also seeing increased latency and jitter using an Octeon system -
> Edgerouter 6p. It's the same issue for both OpenBSD 7.6 and 7.7. I don't see
> it on a amd64 machine with the same configuration.
> 
> Attached is a graph showing on the left ping data collected with traffic
> going through a PC-Engines APU1 board with OpenBSD 7.7. To the right is
> traffic going through an Edgerouter 6p with OpenBSD 7.6 . It's the same for
> OpenBSD 7.7.
> 
> In my case the config is with OpenBGPD on a vlan. This data is through NAT
> but it's the same without NAT and it's also the same using PPPoE rather than
> BGPD.
> 
> From what I've read on the list it has been observed for Octeon, and also
> occasionally on other architectures. As my data shows there are cases where
> the amd64 architecture performs as expected.
> 
> Is it time for a bug report?
> 
> Here is some further info on my system, though note it's 7.6 at the moment,
> which I put on temporarily to check if it is unique to version 7.7, which it
> isn't.
> 
> 
> > pfctl -si
> 
> Status: Enabled for 0 days 01:20:57              Debug: err
> 
> State Table                          Total             Rate
>   current entries                       10
>   half-open tcp                          0
>   searches                           67352           13.9/s
>   inserts                             1262            0.3/s
>   removals                            1252            0.3/s
> Counters
>   match                              15580            3.2/s
>   bad-offset                             0            0.0/s
>   fragment                               0            0.0/s
>   short                                  0            0.0/s
>   normalize                              4            0.0/s
>   memory                                 0            0.0/s
>   bad-timestamp                          0            0.0/s
>   congestion                             0            0.0/s
>   ip-option                              0            0.0/s
>   proto-cksum                            0            0.0/s
>   state-mismatch                        22            0.0/s
>   state-insert                           0            0.0/s
>   state-limit                            0            0.0/s
>   src-limit                              1            0.0/s
>   synproxy                               0            0.0/s
>   translate                              0            0.0/s
>   no-route                               0            0.0/s
> 
> > netstat -m
> 1551 mbufs in use:
>         1538 mbufs allocated to data
>         5 mbufs allocated to packet headers
>         8 mbufs allocated to socket names and addresses
> 1536/1712 mbuf 2048 byte clusters in use (current/peak)
> 0/0 mbuf 2112 byte clusters in use (current/peak)
> 0/40 mbuf 4096 byte clusters in use (current/peak)
> 0/24 mbuf 8192 byte clusters in use (current/peak)
> 0/0 mbuf 9216 byte clusters in use (current/peak)
> 0/0 mbuf 12288 byte clusters in use (current/peak)
> 0/0 mbuf 16384 byte clusters in use (current/peak)
> 0/8 mbuf 65536 byte clusters in use (current/peak)
> 4672/4720/131072 Kbytes allocated to network (current/peak/max)
> 0 requests for memory denied
> 0 requests for memory delayed
> 0 calls to protocol drain routines
> 0 defrag mbuf allocation
> 1001 prepend mbuf allocation
> 0 pullup mbuf allocation
> 0 pullup memory copy
> 0 pulldown mbuf allocation
> 0 pulldown memory copy
> 
> > systat vm
> 
> 
>   2 users Load 0.00 0.00 0.00                        15:14:23
> 
>             memory totals (in KB)            PAGING   SWAPPING Interrupts
>            real   virtual     free           in  out   in  out      576
> total
> Active    95280     95280   630160   ops                                soft
> All      380976    380976   841072   pages                          528
> clock
>                                                                       3
> cnmac1
> Proc:r  d  s  w    Csw   Trp   Sys   Int   Sof  Flt       forks       6
> cnmac2
>           58        57     7    63   576   100   13       fkppw com0
>                                                           fksvm octmmc0
>    0.0%Int   0.0%Spn   0.1%Sys   0.0%Usr  99.9%Idle       pwait      39 ipi
> |    |    |    |    |    |    |    |    |    |    |       relck
>                                                           rlkok
>                                                           noram
> Namei         Sys-cache    Proc-cache    No-cache       1 ndcpy
>     Calls     hits    %    hits     %    miss   %         fltcp
>        10       10  100                                 2 zfod
>                                                           cow
> Disks   sd0                                          2106 fmin
> seeks                                                2808 ftarg
> xfers                                                     itarg
> speed                                                   2 wired
>   sec                                                     pdfre
>                                                           pdscn
>                                                           pzidl      14
> IPKTS
>                                                         8 kmape      12
> OPKTS
> 
> > systat mbufs
> 
> 
> cnmac4   2 users Load 0.00 0.00 0.00                     15:15:15
> 
> IFACE            RING      LIVELOCKS  SIZE ALIVE   LWM   HWM   CWM
> System           mbufs             0   256  1551          27
>                  mcl2k                2048  1536         214
>                  mcl4k                4096     0           5
>                  mcl8k                8192     0           3
>                  mcl64k              65536     0           1
> lo0
> cnmac0
> cnmac1
> cnmac2
> cnmac3
> cnmac4
> cnmac5
> enc0
> vether0
> vlan100
> 
> dmesg:
> root@hrr67:~$ dmesg
> [ using 773576 bytes of bsd ELF symbol table ]
> Copyright (c) 1982, 1986, 1989, 1991, 1993
>         The Regents of the University of California.  All rights reserved.
> Copyright (c) 1995-2024 OpenBSD. All rights reserved.
> https://www.OpenBSD.org
> 
> OpenBSD 7.6 (GENERIC.MP) #233: Mon Sep 30 09:41:48 MDT 2024
>     dera...@octeon.openbsd.org:/usr/src/sys/arch/octeon/compile/GENERIC.MP
> real mem = 1073741824 (1024MB)
> avail mem = 1035321344 (987MB)
> random: good seed from bootblocks
> mainbus0 at root: board 20300 rev 1.23, model cavium,ubnt_e300
> cpu0 at mainbus0: CN70xx/CN71xx CPU rev 0.2 1000 MHz, CN70xx/CN71xx FPU rev
> 0.0
> cpu0: cache L1-I 78KB 39 way D 32KB 32 way, L2 1024KB 8 way
> cpu1 at mainbus0: CN70xx/CN71xx CPU rev 0.2 1000 MHz, CN70xx/CN71xx FPU rev
> 0.0
> cpu1: cache L1-I 78KB 39 way D 32KB 32 way, L2 1024KB 8 way
> cpu2 at mainbus0: CN70xx/CN71xx CPU rev 0.2 1000 MHz, CN70xx/CN71xx FPU rev
> 0.0
> cpu2: cache L1-I 78KB 39 way D 32KB 32 way, L2 1024KB 8 way
> cpu3 at mainbus0: CN70xx/CN71xx CPU rev 0.2 1000 MHz, CN70xx/CN71xx FPU rev
> 0.0
> cpu3: cache L1-I 78KB 39 way D 32KB 32 way, L2 1024KB 8 way
> clock0 at mainbus0: int 5
> octcrypto0 at mainbus0
> iobus0 at mainbus0
> simplebus0 at iobus0: "soc"
> "bootbus" at simplebus0 not configured
> octciu0 at simplebus0
> octcib0 at simplebus0: max-bits 23
> octcib1 at simplebus0: max-bits 12
> octcib2 at simplebus0: max-bits 6
> octcib3 at simplebus0: max-bits 15
> octcib4 at simplebus0: max-bits 4
> octcib5 at simplebus0: max-bits 11
> octcib6 at simplebus0: max-bits 11
> octgpio0 at simplebus0: 20 pins, xbit 16
> octsmi0 at simplebus0
> octsmi1 at simplebus0
> octpip0 at simplebus0
> octgmx0 at octpip0 interface 0
> cnmac0 at octgmx0: port 0 SGMII, address 18:e8:29:ba:19:1b
> ukphy0 at cnmac0 phy 4: Generic IEEE 802.3u media interface, rev. 2: OUI
> 0x0001c1, model 0x000c
> cnmac1 at octgmx0: port 1 SGMII, address 18:e8:29:ba:19:1c
> ukphy1 at cnmac1 phy 5: Generic IEEE 802.3u media interface, rev. 2: OUI
> 0x0001c1, model 0x000c
> cnmac2 at octgmx0: port 2 SGMII, address 18:e8:29:ba:19:1d
> ukphy2 at cnmac2 phy 6: Generic IEEE 802.3u media interface, rev. 2: OUI
> 0x0001c1, model 0x000c
> cnmac3 at octgmx0: port 3 SGMII, address 18:e8:29:ba:19:1e
> ukphy3 at cnmac3 phy 7: Generic IEEE 802.3u media interface, rev. 2: OUI
> 0x0001c1, model 0x000c
> octgmx1 at octpip0 interface 1
> cnmac4 at octgmx1: port 16 SGMII, address 18:e8:29:ba:19:1f
> ukphy4 at cnmac4 phy 8: Generic IEEE 802.3u media interface, rev. 0: OUI
> 0x0001c1, model 0x0027
> cnmac5 at octgmx1: port 17 SGMII, address 18:e8:29:ba:19:20
> ukphy5 at cnmac5 phy 9: Generic IEEE 802.3u media interface, rev. 0: OUI
> 0x0001c1, model 0x0027
> octsctl0 at simplebus0: disabled
> octxctl0 at simplebus0: DWC3 rev 0x250a
> xhci0 at octxctl0, xHCI 1.0
> usb0 at xhci0: USB revision 3.0
> uhub0 at usb0 configuration 1 interface 0 "Generic xHCI root hub" rev
> 3.00/1.00 addr 1
> octxctl1 at simplebus0: DWC3 rev 0x250a
> xhci1 at octxctl1, xHCI 1.0
> usb1 at xhci1: USB revision 3.0
> uhub1 at usb1 configuration 1 interface 0 "Generic xHCI root hub" rev
> 3.00/1.00 addr 1
> "i2c" at simplebus0 not configured
> "i2c" at simplebus0 not configured
> com0 at simplebus0: ns16550a, 64 byte fifo
> com0: console
> com1 at simplebus0: ns16550a, 64 byte fifo
> com1: probed fifo depth: 0 bytes
> octmmc0 at simplebus0
> sdmmc0 at octmmc0: 8-bit, mmc high-speed
> "spi" at simplebus0 not configured
> "ocla0" at simplebus0 not configured
> "dma-engine" at simplebus0 not configured
> "dma-engine" at simplebus0 not configured
> octrng0 at iobus0 base 0x1400000000000 irq 0
> octpcie0 at iobus0: 3 ports
> octpcie0 port 0: link timeout
> octpcie0 port 1: reset timeout
> octpcie0 port 2: reset timeout
> scsibus0 at sdmmc0: 2 targets, initiator 0
> sd0 at scsibus0 targ 1 lun 0: <Kingston, MMC4GB, 0000> removable
> sd0: 3728MB, 512 bytes/sector, 7634944 sectors
> vscsi0 at root
> scsibus1 at vscsi0: 256 targets
> softraid0 at root
> scsibus2 at softraid0: 256 targets
> root on sd0a (285e489508f0f02b.a) swap on sd0b dump on sd0b

can you try this?

Index: uipc_socket.c
===================================================================
RCS file: /cvs/src/sys/kern/uipc_socket.c,v
diff -u -p -r1.344 uipc_socket.c
--- uipc_socket.c       31 Oct 2024 12:51:55 -0000      1.344
+++ uipc_socket.c       1 Nov 2024 23:53:25 -0000
@@ -787,48 +787,36 @@ m_getuio(struct mbuf **mp, int atomic, l
 {
        struct mbuf *m, *top = NULL;
        struct mbuf **nextp = &top;
-       u_long len, mlen;
-       size_t resid = uio->uio_resid;
+       u_long len, mlen, alen;
+       int align = atomic ? roundup(max_hdr, sizeof(long)) : 0;
        int error;
 
-       do {
-               if (top == NULL) {
-                       MGETHDR(m, M_WAIT, MT_DATA);
-                       mlen = MHLEN;
-               } else {
-                       MGET(m, M_WAIT, MT_DATA);
-                       mlen = MLEN;
-               }
+       m = m_gethdr(M_WAIT, MT_DATA);
+       mlen = MHLEN;
+
+       for (;;) {
                /* chain mbuf together */
                *nextp = m;
                nextp = &m->m_next;
 
-               resid = ulmin(resid, space);
-               if (resid >= MINCLSIZE) {
-                       MCLGETL(m, M_NOWAIT, ulmin(resid, MAXMCLBYTES));
-                       if ((m->m_flags & M_EXT) == 0)
+               /* How much data we want to put in this mbuf? */
+               len = ulmin(uio->uio_resid, space);
+               /* How much space are we allocating for that data? */
+               alen = align + len;
+               if (alen > mlen) {
+                       MCLGETL(m, M_NOWAIT, ulmin(alen, MAXMCLBYTES));
+                       if (!ISSET(m->m_flags, M_EXT) && alen > MCLBYTES)
                                MCLGETL(m, M_NOWAIT, MCLBYTES);
-                       if ((m->m_flags & M_EXT) == 0)
-                               goto nopages;
-                       mlen = m->m_ext.ext_size;
-                       len = ulmin(mlen, resid);
-                       /*
-                        * For datagram protocols, leave room
-                        * for protocol headers in first mbuf.
-                        */
-                       if (atomic && m == top && len < mlen - max_hdr)
-                               m->m_data += max_hdr;
-               } else {
-nopages:
-                       len = ulmin(mlen, resid);
-                       /*
-                        * For datagram protocols, leave room
-                        * for protocol headers in first mbuf.
-                        */
-                       if (atomic && m == top && len < mlen - max_hdr)
-                               m_align(m, len);
+                       if (ISSET(m->m_flags, M_EXT))
+                               mlen = m->m_ext.ext_size;
                }
 
+               /* Avoid pain from a stupid max_hdr value */
+               if (align < mlen)
+                       m->m_data += align;
+
+               /* How much data can we put in this mbuf? */
+               len = ulmin(mlen, len);
                error = uiomove(mtod(m, caddr_t), len, uio);
                if (error) {
                        m_freem(top);
@@ -836,13 +824,19 @@ nopages:
                }
 
                /* adjust counters */
-               resid = uio->uio_resid;
                space -= len;
                m->m_len = len;
                top->m_pkthdr.len += len;
 
                /* Is there more space and more data? */
-       } while (space > 0 && resid > 0);
+               if (space == 0 || uio->uio_resid == 0)
+                       break;
+
+               align = 0;
+
+               m = m_get(M_WAIT, MT_DATA);
+               mlen = MLEN;
+       }
 
        *mp = top;
        return 0;

Reply via email to