Author: markj
Date: Sun Dec  6 22:45:50 2020
New Revision: 368400
URL: https://svnweb.freebsd.org/changeset/base/368400

Log:
  uma: Make uma_zone_set_maxcache() work better with small limits
  
  The old implementation chose the largest bucket zone such that if the
  per-CPU caches are fully populated, the total number of items cached is
  no larger than the specified limit.  If no such zone existed, UMA would
  not do any caching.
  
  We can now use uz_bucket_size_max to set a precise limit on the number
  of items in a zone's bucket, so the total size of per-CPU caches can be
  bounded more easily.  Implement a new policy in uma_zone_set_maxcache():
  choose a bucket size such that up to half of the limit can be cached in
  per-CPU caches, with the rest going to the full bucket cache.  This
  fixes a problem with the kstack_cache zone: the limit of 4 * mp_ncpus
  items meant that the zone would not do any caching, defeating the whole
  purpose of the zone.  That's because the smallest bucket size holds up
  to 2 items and we may cache up to 3 full buckets per CPU, and
  2 * 3 * mp_ncpus > 4 * mp_ncpus.
  
  Reported by:  mjg
  Sponsored by: The FreeBSD Foundation
  Differential Revision:        https://reviews.freebsd.org/D27168

Modified:
  head/sys/vm/uma.h
  head/sys/vm/uma_core.c

Modified: head/sys/vm/uma.h
==============================================================================
--- head/sys/vm/uma.h   Sun Dec  6 22:45:39 2020        (r368399)
+++ head/sys/vm/uma.h   Sun Dec  6 22:45:50 2020        (r368400)
@@ -492,7 +492,7 @@ void uma_zone_reserve(uma_zone_t zone, int nitems);
 int uma_zone_reserve_kva(uma_zone_t zone, int nitems);
 
 /*
- * Sets a high limit on the number of items allowed in a zone
+ * Sets an upper limit on the number of items allocated from a zone
  *
  * Arguments:
  *     zone  The zone to limit
@@ -504,7 +504,7 @@ int uma_zone_reserve_kva(uma_zone_t zone, int nitems);
 int uma_zone_set_max(uma_zone_t zone, int nitems);
 
 /*
- * Sets a high limit on the number of items allowed in zone's bucket cache
+ * Sets an upper limit on the number of items allowed in zone's caches
  *
  * Arguments:
  *      zone  The zone to limit

Modified: head/sys/vm/uma_core.c
==============================================================================
--- head/sys/vm/uma_core.c      Sun Dec  6 22:45:39 2020        (r368399)
+++ head/sys/vm/uma_core.c      Sun Dec  6 22:45:50 2020        (r368400)
@@ -438,27 +438,6 @@ bucket_zone_lookup(int entries)
        return (ubz);
 }
 
-static struct uma_bucket_zone *
-bucket_zone_max(uma_zone_t zone, int nitems)
-{
-       struct uma_bucket_zone *ubz;
-       int bpcpu;
-
-       bpcpu = 2;
-       if ((zone->uz_flags & UMA_ZONE_FIRSTTOUCH) != 0)
-               /* Count the cross-domain bucket. */
-               bpcpu++;
-
-       for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++)
-               if (ubz->ubz_entries * bpcpu * mp_ncpus > nitems)
-                       break;
-       if (ubz == &bucket_zones[0])
-               ubz = NULL;
-       else
-               ubz--;
-       return (ubz);
-}
-
 static int
 bucket_select(int size)
 {
@@ -2478,10 +2457,10 @@ zone_alloc_sysctl(uma_zone_t zone, void *unused)
        SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
            "items", CTLFLAG_RD | CTLTYPE_U64 | CTLFLAG_MPSAFE,
            zone, 0, sysctl_handle_uma_zone_items, "QU",
-           "current number of allocated items if limit is set");
+           "Current number of allocated items if limit is set");
        SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
            "max_items", CTLFLAG_RD, &zone->uz_max_items, 0,
-           "Maximum number of cached items");
+           "Maximum number of allocated and cached items");
        SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
            "sleepers", CTLFLAG_RD, &zone->uz_sleepers, 0,
            "Number of threads sleeping at limit");
@@ -4570,20 +4549,19 @@ zone_free_item(uma_zone_t zone, void *item, void *udat
 int
 uma_zone_set_max(uma_zone_t zone, int nitems)
 {
-       struct uma_bucket_zone *ubz;
-       int count;
 
        /*
+        * If the limit is small, we may need to constrain the maximum per-CPU
+        * cache size, or disable caching entirely.
+        */
+       uma_zone_set_maxcache(zone, nitems);
+
+       /*
         * XXX This can misbehave if the zone has any allocations with
         * no limit and a limit is imposed.  There is currently no
         * way to clear a limit.
         */
        ZONE_LOCK(zone);
-       ubz = bucket_zone_max(zone, nitems);
-       count = ubz != NULL ? ubz->ubz_entries : 0;
-       zone->uz_bucket_size_max = zone->uz_bucket_size = count;
-       if (zone->uz_bucket_size_min > zone->uz_bucket_size_max)
-               zone->uz_bucket_size_min = zone->uz_bucket_size_max;
        zone->uz_max_items = nitems;
        zone->uz_flags |= UMA_ZFLAG_LIMIT;
        zone_update_caches(zone);
@@ -4598,24 +4576,35 @@ uma_zone_set_max(uma_zone_t zone, int nitems)
 void
 uma_zone_set_maxcache(uma_zone_t zone, int nitems)
 {
-       struct uma_bucket_zone *ubz;
-       int bpcpu;
+       int bpcpu, bpdom, bsize, nb;
 
        ZONE_LOCK(zone);
-       ubz = bucket_zone_max(zone, nitems);
-       if (ubz != NULL) {
-               bpcpu = 2;
-               if ((zone->uz_flags & UMA_ZONE_FIRSTTOUCH) != 0)
-                       /* Count the cross-domain bucket. */
-                       bpcpu++;
-               nitems -= ubz->ubz_entries * bpcpu * mp_ncpus;
-               zone->uz_bucket_size_max = ubz->ubz_entries;
-       } else {
-               zone->uz_bucket_size_max = zone->uz_bucket_size = 0;
+
+       /*
+        * Compute a lower bound on the number of items that may be cached in
+        * the zone.  Each CPU gets at least two buckets, and for cross-domain
+        * frees we use an additional bucket per CPU and per domain.  Select the
+        * largest bucket size that does not exceed half of the requested limit,
+        * with the left over space given to the full bucket cache.
+        */
+       bpdom = 0;
+       bpcpu = 2;
+#ifdef NUMA
+       if ((zone->uz_flags & UMA_ZONE_FIRSTTOUCH) != 0 && vm_ndomains > 1) {
+               bpcpu++;
+               bpdom++;
        }
+#endif
+       nb = bpcpu * mp_ncpus + bpdom * vm_ndomains;
+       bsize = nitems / nb / 2;
+       if (bsize > BUCKET_MAX)
+               bsize = BUCKET_MAX;
+       else if (bsize == 0 && nitems / nb > 0)
+               bsize = 1;
+       zone->uz_bucket_size_max = zone->uz_bucket_size = bsize;
        if (zone->uz_bucket_size_min > zone->uz_bucket_size_max)
                zone->uz_bucket_size_min = zone->uz_bucket_size_max;
-       zone->uz_bucket_max = nitems / vm_ndomains;
+       zone->uz_bucket_max = nitems - nb * bsize;
        ZONE_UNLOCK(zone);
 }
 
_______________________________________________
svn-src-head@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-head
To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"

Reply via email to