Re: add in6 multicast support to vxlan(4) ; question on mbufs

[Vincent Gross]

On Tue, 1 Nov 2016 18:51:08 +0100
Mike Belopuhov  wrote:

> On 1 November 2016 at 18:23, Vincent Gross 
> wrote:
> > On Tue, 4 Oct 2016 01:07:51 +0200
> > Vincent Gross  wrote:
> >  
> >> On Sat, 24 Sep 2016 10:58:10 +0200
> >> Vincent Gross  wrote:
> >>  
> >> > Hi,
> >> >  
> >> [snip]  
> >> >
> >> > Aside from the mbuf issue, is this Ok ?  
> >>
> >> I will go back on the mbuff stuff later.
> >>
> >> Diff rebased, ok anyone ?
> >>  
> >
> > New rebase, tested on amd64 and macppc, Ok ?
> >  
> 
> Why have you kept the m_adj with ETHER_ALIGN?
> 

Derp. Lost track while reading the mbuf thread.

New diff w/o the mbuf dance, again tested on amd64 and macppc.


Index: sys/net/if_vxlan.c
===
RCS file: /cvs/src/sys/net/if_vxlan.c,v
retrieving revision 1.51
diff -u -p -r1.51 if_vxlan.c
--- sys/net/if_vxlan.c  25 Oct 2016 16:31:08 -  1.51
+++ sys/net/if_vxlan.c  1 Nov 2016 21:58:24 -
@@ -47,6 +47,8 @@
 #include 
 #include 
 
+#include 
+
 #if NPF > 0
 #include 
 #endif
@@ -61,7 +63,12 @@ struct vxlan_softc {
struct arpcomsc_ac;
struct ifmedia   sc_media;
 
-   struct ip_moptions   sc_imo;
+   union {
+   struct ip_moptions   u_imo;
+   struct ip6_moptions  u_imo6;
+   } sc_imu;
+#define sc_imo sc_imu.u_imo
+#define sc_im6osc_imu.u_imo6
void*sc_ahcookie;
void*sc_lhcookie;
void*sc_dhcookie;
@@ -129,10 +136,6 @@ vxlan_clone_create(struct if_clone *ifc,
M_DEVBUF, M_NOWAIT|M_ZERO)) == NULL)
return (ENOMEM);
 
-   sc->sc_imo.imo_membership = malloc(
-   (sizeof(struct in_multi *) * IP_MIN_MEMBERSHIPS),
M_IPMOPTS,
-   M_WAITOK|M_ZERO);
-   sc->sc_imo.imo_max_memberships = IP_MIN_MEMBERSHIPS;
sc->sc_dstport = htons(VXLAN_PORT);
sc->sc_vnetid = VXLAN_VNI_UNSET;
 
@@ -190,7 +193,6 @@ vxlan_clone_destroy(struct ifnet *ifp)
ifmedia_delete_instance(>sc_media, IFM_INST_ANY);
ether_ifdetach(ifp);
if_detach(ifp);
-   free(sc->sc_imo.imo_membership, M_IPMOPTS, 0);
free(sc, M_DEVBUF, sizeof(*sc));
 
return (0);
@@ -199,11 +201,33 @@ vxlan_clone_destroy(struct ifnet *ifp)
 void
 vxlan_multicast_cleanup(struct ifnet *ifp)
 {
-   struct vxlan_softc  *sc = (struct vxlan_softc
*)ifp->if_softc;
-   struct ip_moptions  *imo = >sc_imo;
-   struct ifnet*mifp;
+   struct vxlan_softc   *sc = (struct vxlan_softc
*)ifp->if_softc;
+   struct ip_moptions   *imo;
+   struct in_multi **imm;
+   struct ip6_moptions  *im6o;
+   struct in6_multi_mship   *im6m, *im6m_next;
+   struct ifnet *mifp = NULL;
+
+   switch (sc->sc_dst.ss_family) {
+   case AF_INET:
+   imo = >sc_imo;
+   mifp = if_get(imo->imo_ifidx);
+   imm = imo->imo_membership;
+   while (imo->imo_num_memberships > 0)
+   in_delmulti(imm[--imo->imo_num_memberships]);
+   free(imm, M_IPMOPTS,
+   sizeof(struct in_multi *) *
imo->imo_num_memberships);
+   break;
+   case AF_INET6:
+   im6o = >sc_im6o;
+   mifp = if_get(im6o->im6o_ifidx);
+   LIST_FOREACH_SAFE(im6m, >im6o_memberships,
i6mm_chain,
+   im6m_next)
+   in6_leavegroup(im6m);
+   break;
+   }
+   bzero(>sc_imu, sizeof(sc->sc_imu));
 
-   mifp = if_get(imo->imo_ifidx);
if (mifp != NULL) {
if (sc->sc_ahcookie != NULL) {
hook_disestablish(mifp->if_addrhooks,
sc->sc_ahcookie); @@ -219,14 +243,9 @@ vxlan_multicast_cleanup(struct
ifnet *if sc->sc_dhcookie);
sc->sc_dhcookie = NULL;
}
-
-   if_put(mifp);
}
 
-   if (imo->imo_num_memberships > 0) {
-
in_delmulti(imo->imo_membership[--imo->imo_num_memberships]);
-   imo->imo_ifidx = 0;
-   }
+   if_put(mifp);
 }
 
 int
@@ -234,55 +253,140 @@ vxlan_multicast_join(struct ifnet *ifp, 
 struct sockaddr *dst)
 {
struct vxlan_softc  *sc = ifp->if_softc;
-   struct ip_moptions  *imo = >sc_imo;
+   struct ip_moptions  *imo;
+   struct ip6_moptions *im6o;
+   struct in6_multi_mship  *imm;
struct sockaddr_in  *src4, *dst4;
 #ifdef INET6
-   struct sockaddr_in6 *dst6;
+   struct sockaddr_in6 *src6, *dst6;
 #endif /* INET6 */
struct ifaddr   *ifa;
-   struct ifnet*mifp;
+   struct ifnet*mifp = NULL;
+   struct rtentry  *rt;
+   int  error;
 
switch 

Re: add in6 multicast support to vxlan(4) ; question on mbufs

[Mike Belopuhov]

On 1 November 2016 at 18:23, Vincent Gross  wrote:
> On Tue, 4 Oct 2016 01:07:51 +0200
> Vincent Gross  wrote:
>
>> On Sat, 24 Sep 2016 10:58:10 +0200
>> Vincent Gross  wrote:
>>
>> > Hi,
>> >
>> [snip]
>> >
>> > Aside from the mbuf issue, is this Ok ?
>>
>> I will go back on the mbuff stuff later.
>>
>> Diff rebased, ok anyone ?
>>
>
> New rebase, tested on amd64 and macppc, Ok ?
>

Why have you kept the m_adj with ETHER_ALIGN?

match rules for spamlogd?

[Peter Hessler]

the line checking for PF_PASS has not changed since 2006, which predates
the existence of match rules in PF.

We check for permissions in one place, but if they are (eventually)
permitted, we want spamlogd to handle the packet.

Is there a better way than checking the PF_MATCH state?  Is there an
"ultimately allowed" state that we can check?

OK?

Index: libexec/spamlogd/spamlogd.c
===
RCS file: /cvs/openbsd/src/libexec/spamlogd/spamlogd.c,v
retrieving revision 1.27
diff -u -p -u -p -r1.27 spamlogd.c
--- libexec/spamlogd/spamlogd.c 16 Mar 2016 14:47:04 -  1.27
+++ libexec/spamlogd/spamlogd.c 1 Nov 2016 15:56:48 -
@@ -173,7 +173,7 @@ logpkt_handler(u_char *user, const struc
}
 
/* We're interested in passed packets */
-   if (hdr->action != PF_PASS)
+   if (hdr->action != PF_PASS || hdr->action != PF_MATCH)
return;
 
af = hdr->af;



-- 
An age is called Dark not because the light fails to shine, but because
people refuse to see it.
-- James Michener, "Space"

Re: add in6 multicast support to vxlan(4) ; question on mbufs

[Vincent Gross]

On Tue, 4 Oct 2016 01:07:51 +0200
Vincent Gross  wrote:

> On Sat, 24 Sep 2016 10:58:10 +0200
> Vincent Gross  wrote:
> 
> > Hi,
> >   
> [snip]
> > 
> > Aside from the mbuf issue, is this Ok ?  
> 
> I will go back on the mbuff stuff later.
> 
> Diff rebased, ok anyone ?
> 

New rebase, tested on amd64 and macppc, Ok ?

Index: sys/net/if_vxlan.c
===
RCS file: /cvs/src/sys/net/if_vxlan.c,v
retrieving revision 1.51
diff -u -p -r1.51 if_vxlan.c
--- sys/net/if_vxlan.c  25 Oct 2016 16:31:08 -  1.51
+++ sys/net/if_vxlan.c  1 Nov 2016 17:18:15 -
@@ -47,6 +47,8 @@
 #include 
 #include 
 
+#include 
+
 #if NPF > 0
 #include 
 #endif
@@ -61,7 +63,12 @@ struct vxlan_softc {
struct arpcomsc_ac;
struct ifmedia   sc_media;
 
-   struct ip_moptions   sc_imo;
+   union {
+   struct ip_moptions   u_imo;
+   struct ip6_moptions  u_imo6;
+   } sc_imu;
+#define sc_imo sc_imu.u_imo
+#define sc_im6osc_imu.u_imo6
void*sc_ahcookie;
void*sc_lhcookie;
void*sc_dhcookie;
@@ -129,10 +136,6 @@ vxlan_clone_create(struct if_clone *ifc,
M_DEVBUF, M_NOWAIT|M_ZERO)) == NULL)
return (ENOMEM);
 
-   sc->sc_imo.imo_membership = malloc(
-   (sizeof(struct in_multi *) * IP_MIN_MEMBERSHIPS), M_IPMOPTS,
-   M_WAITOK|M_ZERO);
-   sc->sc_imo.imo_max_memberships = IP_MIN_MEMBERSHIPS;
sc->sc_dstport = htons(VXLAN_PORT);
sc->sc_vnetid = VXLAN_VNI_UNSET;
 
@@ -190,7 +193,6 @@ vxlan_clone_destroy(struct ifnet *ifp)
ifmedia_delete_instance(>sc_media, IFM_INST_ANY);
ether_ifdetach(ifp);
if_detach(ifp);
-   free(sc->sc_imo.imo_membership, M_IPMOPTS, 0);
free(sc, M_DEVBUF, sizeof(*sc));
 
return (0);
@@ -199,11 +201,33 @@ vxlan_clone_destroy(struct ifnet *ifp)
 void
 vxlan_multicast_cleanup(struct ifnet *ifp)
 {
-   struct vxlan_softc  *sc = (struct vxlan_softc *)ifp->if_softc;
-   struct ip_moptions  *imo = >sc_imo;
-   struct ifnet*mifp;
+   struct vxlan_softc   *sc = (struct vxlan_softc *)ifp->if_softc;
+   struct ip_moptions   *imo;
+   struct in_multi **imm;
+   struct ip6_moptions  *im6o;
+   struct in6_multi_mship   *im6m, *im6m_next;
+   struct ifnet *mifp = NULL;
+
+   switch (sc->sc_dst.ss_family) {
+   case AF_INET:
+   imo = >sc_imo;
+   mifp = if_get(imo->imo_ifidx);
+   imm = imo->imo_membership;
+   while (imo->imo_num_memberships > 0)
+   in_delmulti(imm[--imo->imo_num_memberships]);
+   free(imm, M_IPMOPTS,
+   sizeof(struct in_multi *) * imo->imo_num_memberships);
+   break;
+   case AF_INET6:
+   im6o = >sc_im6o;
+   mifp = if_get(im6o->im6o_ifidx);
+   LIST_FOREACH_SAFE(im6m, >im6o_memberships, i6mm_chain,
+   im6m_next)
+   in6_leavegroup(im6m);
+   break;
+   }
+   bzero(>sc_imu, sizeof(sc->sc_imu));
 
-   mifp = if_get(imo->imo_ifidx);
if (mifp != NULL) {
if (sc->sc_ahcookie != NULL) {
hook_disestablish(mifp->if_addrhooks, sc->sc_ahcookie);
@@ -219,14 +243,9 @@ vxlan_multicast_cleanup(struct ifnet *if
sc->sc_dhcookie);
sc->sc_dhcookie = NULL;
}
-
-   if_put(mifp);
}
 
-   if (imo->imo_num_memberships > 0) {
-   in_delmulti(imo->imo_membership[--imo->imo_num_memberships]);
-   imo->imo_ifidx = 0;
-   }
+   if_put(mifp);
 }
 
 int
@@ -234,55 +253,140 @@ vxlan_multicast_join(struct ifnet *ifp, 
 struct sockaddr *dst)
 {
struct vxlan_softc  *sc = ifp->if_softc;
-   struct ip_moptions  *imo = >sc_imo;
+   struct ip_moptions  *imo;
+   struct ip6_moptions *im6o;
+   struct in6_multi_mship  *imm;
struct sockaddr_in  *src4, *dst4;
 #ifdef INET6
-   struct sockaddr_in6 *dst6;
+   struct sockaddr_in6 *src6, *dst6;
 #endif /* INET6 */
struct ifaddr   *ifa;
-   struct ifnet*mifp;
+   struct ifnet*mifp = NULL;
+   struct rtentry  *rt;
+   int  error;
 
switch (dst->sa_family) {
case AF_INET:
dst4 = satosin(dst);
+   src4 = satosin(src);
if (!IN_MULTICAST(dst4->sin_addr.s_addr))
return (0);
+   if (src4->sin_addr.s_addr == INADDR_ANY ||
+   IN_MULTICAST(src4->sin_addr.s_addr))
+   

iwm host interrupt operation mode flag

[Stefan Sperling]

The global host_interrupt_operation_mode variable can be represented
as a flag. This is just a cosmetic change.

Index: if_iwm.c
===
RCS file: /cvs/src/sys/dev/pci/if_iwm.c,v
retrieving revision 1.146
diff -u -p -r1.146 if_iwm.c
--- if_iwm.c1 Nov 2016 13:49:50 -   1.146
+++ if_iwm.c1 Nov 2016 14:58:05 -
@@ -1409,11 +1409,11 @@ iwm_apm_init(struct iwm_softc *sc)
goto out;
}
 
-   if (sc->host_interrupt_operation_mode) {
+   if (sc->sc_flags & IWM_FLAG_HOST_INT_OPER_MODE) {
/*
-* This is a bit of an abuse - This is needed for 7260 / 3160
-* only check host_interrupt_operation_mode even if this is
-* not related to host_interrupt_operation_mode.
+* This is needed for 7260 / 3160.
+* XXX We check the otherwise unrelated HOST_INT_OPER_MODE flag
+* to detect these devices.
 *
 * Enable the oscillator to count wake up time for L1 exit. This
 * consumes slightly more power (100uA) - but allows to be sure
@@ -1627,7 +1627,7 @@ iwm_nic_rx_init(struct iwm_softc *sc)
IWM_WRITE_1(sc, IWM_CSR_INT_COALESCING, IWM_HOST_INT_TIMEOUT_DEF);
 
/* W/A for interrupt coalescing bug in 7260 and 3160 */
-   if (sc->host_interrupt_operation_mode)
+   if (sc->sc_flags & IWM_FLAG_HOST_INT_OPER_MODE)
IWM_SETBITS(sc, IWM_CSR_INT_COALESCING, IWM_HOST_INT_OPER_MODE);
 
/*
@@ -6969,35 +6969,32 @@ iwm_attach(struct device *parent, struct
case PCI_PRODUCT_INTEL_WL_3160_1:
case PCI_PRODUCT_INTEL_WL_3160_2:
sc->sc_fwname = "iwm-3160-16";
-   sc->host_interrupt_operation_mode = 1;
+   sc->sc_flags |= IWM_FLAG_HOST_INT_OPER_MODE;
sc->sc_device_family = IWM_DEVICE_FAMILY_7000;
sc->sc_fwdmasegsz = IWM_FWDMASEGSZ;
break;
case PCI_PRODUCT_INTEL_WL_3165_1:
case PCI_PRODUCT_INTEL_WL_3165_2:
sc->sc_fwname = "iwm-7265-16";
-   sc->host_interrupt_operation_mode = 0;
sc->sc_device_family = IWM_DEVICE_FAMILY_7000;
sc->sc_fwdmasegsz = IWM_FWDMASEGSZ;
break;
case PCI_PRODUCT_INTEL_WL_7260_1:
case PCI_PRODUCT_INTEL_WL_7260_2:
sc->sc_fwname = "iwm-7260-16";
-   sc->host_interrupt_operation_mode = 1;
+   sc->sc_flags |= IWM_FLAG_HOST_INT_OPER_MODE;
sc->sc_device_family = IWM_DEVICE_FAMILY_7000;
sc->sc_fwdmasegsz = IWM_FWDMASEGSZ;
break;
case PCI_PRODUCT_INTEL_WL_7265_1:
case PCI_PRODUCT_INTEL_WL_7265_2:
sc->sc_fwname = "iwm-7265-16";
-   sc->host_interrupt_operation_mode = 0;
sc->sc_device_family = IWM_DEVICE_FAMILY_7000;
sc->sc_fwdmasegsz = IWM_FWDMASEGSZ;
break;
case PCI_PRODUCT_INTEL_WL_8260_1:
case PCI_PRODUCT_INTEL_WL_8260_2:
sc->sc_fwname = "iwm-8000C-16";
-   sc->host_interrupt_operation_mode = 0;
sc->sc_device_family = IWM_DEVICE_FAMILY_8000;
sc->sc_fwdmasegsz = IWM_FWDMASEGSZ_8000;
break;
Index: if_iwmvar.h
===
RCS file: /cvs/src/sys/dev/pci/if_iwmvar.h,v
retrieving revision 1.24
diff -u -p -r1.24 if_iwmvar.h
--- if_iwmvar.h 21 Sep 2016 13:53:18 -  1.24
+++ if_iwmvar.h 1 Nov 2016 14:57:38 -
@@ -279,11 +279,12 @@ struct iwm_rx_ring {
int cur;
 };
 
-#define IWM_FLAG_USE_ICT   0x01
-#define IWM_FLAG_HW_INITED 0x02
-#define IWM_FLAG_STOPPED   0x04
-#define IWM_FLAG_RFKILL0x08
-#define IWM_FLAG_SCANNING  0x10
+#define IWM_FLAG_USE_ICT   0x01
+#define IWM_FLAG_HW_INITED 0x02
+#define IWM_FLAG_STOPPED   0x04
+#define IWM_FLAG_RFKILL0x08
+#define IWM_FLAG_SCANNING  0x10
+#define IWM_FLAG_HOST_INT_OPER_MODE0x20
 
 struct iwm_ucode_status {
uint32_t uc_error_event_table;
@@ -473,8 +474,6 @@ struct iwm_softc {
 
struct iwm_notif_statistics sc_stats;
int sc_noise;
-
-   int host_interrupt_operation_mode;
 
 #if NBPFILTER > 0
caddr_t sc_drvbpf;

Re: iwm(4): Also reset sc_uc.uc_intr flag on 8000 hw. Gets rid of 1s tsleep.

[Stefan Sperling]

On Tue, Oct 11, 2016 at 04:50:09PM +0200, Stefan Sperling wrote:
> On Tue, Oct 11, 2016 at 04:29:24PM +0200, Imre Vadasz wrote:
> > Hi,
> > 
> > The sc->sc_uc.uc_intr flag isn't getting reset to 0 in
> > iwm_load_firmware_8000(), so the
> > /* wait for the firmware to load */
> > for (w = 0; !sc->sc_uc.uc_intr && w < 10; w++) {
> > err = tsleep(>sc_uc, 0, "iwmuc", hz/10);
> > }
> > loop is immediately aborting on AC8260 hardware without properly waiting
> > for the IWM_ALIVE notification (except on the very first time a firmware
> > was loaded). This issue was workarounded by the fixed 1s tsleep afterwards.
> > /*
> >  * Give the firmware some time to initialize.
> >  * Accessing it too early causes errors.
> >  */
> > tsleep(, PCATCH, "iwmfwinit", hz);
> > 
> > 
> > Everything works fine in iwm(4) without that 1s tsleep if I also reset
> > sc->sc_uc.uc_intr in the family 8000 case.
> 
> Thank you! OK stsp@ (I can't test + commit this myself right now.)

I have committed this. Thanks again.
 
> > 
> > Index: sys/dev/pci/if_iwm.c
> > ===
> > RCS file: /cvs/src/sys/dev/pci/if_iwm.c,v
> > retrieving revision 1.144
> > diff -u -r1.144 if_iwm.c
> > --- sys/dev/pci/if_iwm.c8 Oct 2016 14:37:48 -   1.144
> > +++ sys/dev/pci/if_iwm.c11 Oct 2016 14:08:05 -
> > @@ -2820,8 +2820,6 @@
> > uint32_t dlen;
> > uint32_t offset;
> >  
> > -   sc->sc_uc.uc_intr = 0;
> > -
> > fws = >sc_fw.fw_sects[ucode_type];
> > for (i = 0; i < fws->fw_count; i++) {
> > data = fws->fw_sect[i].fws_data;
> > @@ -2924,7 +2922,6 @@
> > int first_ucode_section;
> >  
> > fws = >sc_fw.fw_sects[ucode_type];
> > -
> > /* configure the ucode to be ready to get the secured image */
> > /* release CPU reset */
> > iwm_write_prph(sc, IWM_RELEASE_CPU_RESET, IWM_RELEASE_CPU_RESET_BIT);
> > @@ -2943,6 +2940,8 @@
> >  {
> > int err, w;
> >  
> > +   sc->sc_uc.uc_intr = 0;
> > +
> > if (sc->sc_device_family == IWM_DEVICE_FAMILY_8000)
> > err = iwm_load_firmware_8000(sc, ucode_type);
> > else
> > @@ -2957,12 +2956,6 @@
> > }
> > if (err || !sc->sc_uc.uc_ok)
> > printf("%s: could not load firmware\n", DEVNAME(sc));
> > -
> > -   /*
> > -* Give the firmware some time to initialize.
> > -* Accessing it too early causes errors.
> > -*/
> > -   tsleep(, PCATCH, "iwmfwinit", hz);
> >  
> > return err;
> >  }
> > 
> 

Re: per-cpu caches for pools

[David Gwynne]

Nice, thanks for testing. It's good to know it's reliable. Were you
watching pool stats while running the tests?

Either way, I'm going to commit this so it will get broader testing.

Cheers,
dlg

On 1 Nov 2016 4:14 a.m., "Simon Mages"  wrote:

> Hi,
>
> today i did some performance messurements on OpenBSD-current, with and
> without your diff.
>
> I use a custom HTTP proxy. Performance in this case is requests/s and
> bandwidth.
> To messure requests/s i use requests with a very small response. To
> messure bandwidth
> i use big responses. This custom proxy uses socket splicing to improve
> the performace for
> bigger junks of data. The following results are a average over
> multiple test runs.
>
> without your diff:
> requests/s = 3929
> bandwidth in Mbit/s = 1196
>
> with your diff:
> requests/s = 4093
> bandwidth in Mbit/s = 1428
>
> Just ask if you want more details on the testsetup.
>
> BR
>
> Simon
>
> 2016-10-27 5:54 GMT+02:00, David Gwynne :
> > On Tue, Oct 25, 2016 at 10:35:45AM +1000, David Gwynne wrote:
> >> On Mon, Oct 24, 2016 at 04:24:13PM +1000, David Gwynne wrote:
> >> > ive posted this before as part of a much bigger diff, but smaller
> >> > is better.
> >> >
> >> > it basically lets things ask for per-cpu item caches to be enabled
> >> > on pools. the most obvious use case for this is the mbuf pools.
> >> >
> >> > the caches are modelled on whats described in the "Magazines and
> >> > Vmem: Extending the Slab Allocator to Many CPUs and Arbitrary
> >> > Resources" paper by Jeff Bonwick and Jonathan Adams. pools are
> >> > modelled on slabs, which bonwick described in a previous paper.
> >> >
> >> > the main inspiration the paper provided was around how many objects
> >> > to cache on each cpu, and how often to move sets of objects between
> >> > the cpu caches and a global list of objects. unlike the paper we
> >> > do not care about maintaining constructed objects on the free lists,
> >> > so we reuse the objects themselves to build the free list.
> >> >
> >> > id like to get this in so we can tinker with it in tree. the things
> >> > i think we need to tinker with are what poisioning we can get away
> >> > with on the per cpu caches, and what limits can we enforce at the
> >> > pool level.
> >> >
> >> > i think poisioning will be relatively simple to add. the limits one
> >> > is more challenging because we dont want the pools to have to
> >> > coordinate between cpus for every get or put operation. my thought
> >> > there was to limit the number of pages that a pool can allocate
> >> > from its backend rather than limit the items the pool can provide.
> >> > limiting the pages could also be done at a lower level. eg, the
> >> > mbuf clusters could share a common backend that limits the pages
> >> > the pools can get, rather than have the cluster pools account for
> >> > pages separately.
> >> >
> >> > anyway, either way i would like to get this in so we can work on
> >> > this stuff.
> >> >
> >> > ok?
> >>
> >> this adds per-cpu caches to the mbuf pools so people can actually
> >> try and see if the code works or not.
> >
> > this fixes a crash hrvoje and i found independently. avoid holding
> > a mutex when calling yield().
> >
> > also some whitespace fixes.
> >
> > Index: kern/subr_pool.c
> > ===
> > RCS file: /cvs/src/sys/kern/subr_pool.c,v
> > retrieving revision 1.198
> > diff -u -p -r1.198 subr_pool.c
> > --- kern/subr_pool.c  15 Sep 2016 02:00:16 -  1.198
> > +++ kern/subr_pool.c  27 Oct 2016 03:51:10 -
> > @@ -42,6 +42,7 @@
> >  #include 
> >  #include 
> >  #include 
> > +#include 
> >
> >  #include 
> >
> > @@ -96,6 +97,33 @@ struct pool_item {
> >  };
> >  #define POOL_IMAGIC(ph, pi) ((u_long)(pi) ^ (ph)->ph_magic)
> >
> > +#ifdef MULTIPROCESSOR
> > +struct pool_list {
> > + struct pool_list*pl_next;   /* next in list */
> > + unsigned longpl_cookie;
> > + struct pool_list*pl_nextl;  /* next list */
> > + unsigned longpl_nitems; /* items in list */
> > +};
> > +
> > +struct pool_cache {
> > + struct pool_list*pc_actv;
> > + unsigned longpc_nactv;  /* cache pc_actv nitems */
> > + struct pool_list*pc_prev;
> > +
> > + uint64_t pc_gen;/* generation number */
> > + uint64_t pc_gets;
> > + uint64_t pc_puts;
> > + uint64_t pc_fails;
> > +
> > + int  pc_nout;
> > +};
> > +
> > +void *pool_cache_get(struct pool *);
> > +void  pool_cache_put(struct pool *, void *);
> > +void  pool_cache_destroy(struct pool *);
> > +#endif
> > +void  pool_cache_info(struct pool *, struct kinfo_pool *);
> > +
> >  #ifdef POOL_DEBUG
> >  int  pool_debug = 1;
> >  #else
> > @@ -355,6 +383,11 @@ pool_destroy(struct pool *pp)
> >   struct pool_item_header