the following adds a mutex to malloc and free to protect their
internal state. this should be enough to make the api mpsafe,
assuming the way they interact with uvm is mpsafe.

this only uses a single mutex around the entire malloc subsystem,
but shuffles the code slightly to avoid holding it around calls
into uvm. the shuffling is mostly to account for an allocation (ie,
bump memuse early) before releasing the mutex to try and allocate
it from uvm. if uvm fails, it rolls this back.

this is modelled on how the item accounting and locking is done in
pools.

can anyone comment on the uvm safety?

also, it is obvious that the malloc debug code has started to rot
a bit. ive fixed it up a bit, but now there's also a possible mp
race when items are moved from the used list to the free list, and
when the item is unmapped between that. id like to remove the malloc
debug code and improve the robustness of malloc itself. pools have
shows they can be fast and strict at the same time.

Index: kern_malloc.c
===================================================================
RCS file: /cvs/src/sys/kern/kern_malloc.c,v
retrieving revision 1.129
diff -u -p -r1.129 kern_malloc.c
--- kern_malloc.c       7 Jun 2017 13:30:36 -0000       1.129
+++ kern_malloc.c       3 Jul 2017 10:14:49 -0000
@@ -39,6 +39,7 @@
 #include <sys/systm.h>
 #include <sys/sysctl.h>
 #include <sys/time.h>
+#include <sys/mutex.h>
 #include <sys/rwlock.h>
 
 #include <uvm/uvm_extern.h>
@@ -94,6 +95,7 @@ u_int nkmempages_min = 0;
 #endif
 u_int  nkmempages_max = 0;
 
+struct mutex malloc_mtx = MUTEX_INITIALIZER(IPL_VM);
 struct kmembuckets bucket[MINBUCKET + 16];
 #ifdef KMEMSTATS
 struct kmemstats kmemstats[M_LAST];
@@ -151,8 +153,8 @@ malloc(size_t size, int type, int flags)
        struct kmemusage *kup;
        struct kmem_freelist *freep;
        long indx, npg, allocsize;
-       int s;
        caddr_t va, cp;
+       int s;
 #ifdef DIAGNOSTIC
        int freshalloc;
        char *savedtype;
@@ -164,9 +166,6 @@ malloc(size_t size, int type, int flags)
                panic("malloc: bogus type %d", type);
 #endif
 
-       if (!cold)
-               KERNEL_ASSERT_LOCKED();
-
        KASSERT(flags & (M_WAITOK | M_NOWAIT));
 
        if ((flags & M_NOWAIT) == 0) {
@@ -176,10 +175,6 @@ malloc(size_t size, int type, int flags)
                if (pool_debug == 2)
                        yield();
 #endif
-               if (!cold && pool_debug) {
-                       KERNEL_UNLOCK();
-                       KERNEL_LOCK();
-               }
        }
 
 #ifdef MALLOC_DEBUG
@@ -193,6 +188,7 @@ malloc(size_t size, int type, int flags)
        if (size > 65535 * PAGE_SIZE) {
                if (flags & M_CANFAIL) {
 #ifndef SMALL_KERNEL
+                       /* XXX lock */
                        if (ratecheck(&malloc_lasterr, &malloc_errintvl))
                                printf("malloc(): allocation too large, "
                                    "type = %d, size = %lu\n", type, size);
@@ -204,33 +200,38 @@ malloc(size_t size, int type, int flags)
        }
 
        indx = BUCKETINDX(size);
+       if (size > MAXALLOCSAVE)
+               allocsize = round_page(size);
+       else
+               allocsize = 1 << indx;
        kbp = &bucket[indx];
-       s = splvm();
+       mtx_enter(&malloc_mtx);
 #ifdef KMEMSTATS
        while (ksp->ks_memuse >= ksp->ks_limit) {
                if (flags & M_NOWAIT) {
-                       splx(s);
+                       mtx_leave(&malloc_mtx);
                        return (NULL);
                }
                if (ksp->ks_limblocks < 65535)
                        ksp->ks_limblocks++;
-               tsleep(ksp, PSWP+2, memname[type], 0);
+               msleep(ksp, &malloc_mtx, PSWP+2, memname[type], 0);
        }
+       ksp->ks_memuse += allocsize; /* account for this early */
        ksp->ks_size |= 1 << indx;
 #endif
-       if (size > MAXALLOCSAVE)
-               allocsize = round_page(size);
-       else
-               allocsize = 1 << indx;
        if (XSIMPLEQ_FIRST(&kbp->kb_freelist) == NULL) {
+               mtx_leave(&malloc_mtx);
                npg = atop(round_page(allocsize));
+               s = splvm();
                va = (caddr_t)uvm_km_kmemalloc_pla(kmem_map, NULL,
                    (vsize_t)ptoa(npg), 0,
                    ((flags & M_NOWAIT) ? UVM_KMF_NOWAIT : 0) |
                    ((flags & M_CANFAIL) ? UVM_KMF_CANFAIL : 0),
                    no_constraint.ucr_low, no_constraint.ucr_high,
                    0, 0, 0);
+               splx(s);
                if (va == NULL) {
+                       long wake;
                        /*
                         * Kmem_malloc() can return NULL, even if it can
                         * wait, if there is no map space available, because
@@ -241,9 +242,18 @@ malloc(size_t size, int type, int flags)
                         */
                        if ((flags & (M_NOWAIT|M_CANFAIL)) == 0)
                                panic("malloc: out of space in kmem_map");
-                       splx(s);
+
+                       mtx_enter(&malloc_mtx);
+                       ksp->ks_memuse -= allocsize;
+                       wake = ksp->ks_memuse + allocsize >= ksp->ks_limit &&
+                           ksp->ks_memuse < ksp->ks_limit;
+                       mtx_leave(&malloc_mtx);
+                       if (wake)
+                               wakeup(ksp);
+                       
                        return (NULL);
                }
+               mtx_enter(&malloc_mtx);
 #ifdef KMEMSTATS
                kbp->kb_total += kbp->kb_elmpercl;
 #endif
@@ -254,9 +264,6 @@ malloc(size_t size, int type, int flags)
 #endif
                if (allocsize > MAXALLOCSAVE) {
                        kup->ku_pagecnt = npg;
-#ifdef KMEMSTATS
-                       ksp->ks_memuse += allocsize;
-#endif
                        goto out;
                }
 #ifdef KMEMSTATS
@@ -334,17 +341,14 @@ malloc(size_t size, int type, int flags)
                panic("malloc: lost data");
        kup->ku_freecnt--;
        kbp->kb_totalfree--;
-       ksp->ks_memuse += 1 << indx;
 out:
        kbp->kb_calls++;
        ksp->ks_inuse++;
        ksp->ks_calls++;
-       if (ksp->ks_memuse > ksp->ks_maxused)
-               ksp->ks_maxused = ksp->ks_memuse;
 #else
 out:
 #endif
-       splx(s);
+       mtx_leave(&malloc_mtx);
 
        if ((flags & M_ZERO) && va != NULL)
                memset(va, 0, size);
@@ -369,9 +373,6 @@ free(void *addr, int type, size_t freeds
        struct kmemstats *ksp = &kmemstats[type];
 #endif
 
-       if (!cold)
-               KERNEL_ASSERT_LOCKED();
-
        if (addr == NULL)
                return;
 
@@ -391,7 +392,6 @@ free(void *addr, int type, size_t freeds
        kbp = &bucket[kup->ku_indx];
        if (size > MAXALLOCSAVE)
                size = kup->ku_pagecnt << PAGE_SHIFT;
-       s = splvm();
 #ifdef DIAGNOSTIC
        if (freedsize != 0 && freedsize > size)
                panic("free: size too large %zu > %ld (%p) type %s",
@@ -412,8 +412,11 @@ free(void *addr, int type, size_t freeds
                        addr, size, memname[type], alloc);
 #endif /* DIAGNOSTIC */
        if (size > MAXALLOCSAVE) {
+               s = splvm();
                uvm_km_free(kmem_map, (vaddr_t)addr, ptoa(kup->ku_pagecnt));
+               splx(s);
 #ifdef KMEMSTATS
+               mtx_enter(&malloc_mtx);
                ksp->ks_memuse -= size;
                kup->ku_indx = 0;
                kup->ku_pagecnt = 0;
@@ -422,11 +425,12 @@ free(void *addr, int type, size_t freeds
                        wakeup(ksp);
                ksp->ks_inuse--;
                kbp->kb_total -= 1;
+               mtx_leave(&malloc_mtx);
 #endif
-               splx(s);
                return;
        }
        freep = (struct kmem_freelist *)addr;
+       mtx_enter(&malloc_mtx);
 #ifdef DIAGNOSTIC
        /*
         * Check for multiple frees. Use a quick check to see if
@@ -468,7 +472,7 @@ free(void *addr, int type, size_t freeds
        ksp->ks_inuse--;
 #endif
        XSIMPLEQ_INSERT_TAIL(&kbp->kb_freelist, freep, kf_flist);
-       splx(s);
+       mtx_leave(&malloc_mtx);
 }
 
 /*
@@ -578,6 +582,9 @@ sysctl_malloc(int *name, u_int namelen, 
     size_t newlen, struct proc *p)
 {
        struct kmembuckets kb;
+#ifdef KMEMSTATS
+       struct kmemstats km;
+#endif
 #if defined(KMEMSTATS) || defined(DIAGNOSTIC) || defined(FFS_SOFTUPDATES)
        int error;
 #endif
@@ -606,15 +613,19 @@ sysctl_malloc(int *name, u_int namelen, 
                return (sysctl_rdstring(oldp, oldlenp, newp, buckstring));
 
        case KERN_MALLOC_BUCKET:
+               mtx_enter(&malloc_mtx);
                memcpy(&kb, &bucket[BUCKETINDX(name[1])], sizeof(kb));
+               mtx_leave(&malloc_mtx);
                memset(&kb.kb_freelist, 0, sizeof(kb.kb_freelist));
                return (sysctl_rdstruct(oldp, oldlenp, newp, &kb, sizeof(kb)));
        case KERN_MALLOC_KMEMSTATS:
 #ifdef KMEMSTATS
                if ((name[1] < 0) || (name[1] >= M_LAST))
                        return (EINVAL);
-               return (sysctl_rdstruct(oldp, oldlenp, newp,
-                   &kmemstats[name[1]], sizeof(struct kmemstats)));
+               mtx_enter(&malloc_mtx);
+               memcpy(&km, &kmemstats[name[1]], sizeof(km));
+               mtx_leave(&malloc_mtx);
+               return (sysctl_rdstruct(oldp, oldlenp, newp, &km, sizeof(km)));
 #else
                return (EOPNOTSUPP);
 #endif
Index: kern_malloc_debug.c
===================================================================
RCS file: /cvs/src/sys/kern/kern_malloc_debug.c,v
retrieving revision 1.34
diff -u -p -r1.34 kern_malloc_debug.c
--- kern_malloc_debug.c 16 Nov 2014 12:31:00 -0000      1.34
+++ kern_malloc_debug.c 3 Jul 2017 10:14:49 -0000
@@ -56,8 +56,10 @@
 #include <sys/malloc.h>
 #include <sys/systm.h>
 #include <sys/pool.h>
+#include <sys/mutex.h>
 
 #include <uvm/uvm_extern.h>
+#include <uvm/uvm.h>
 
 /*
  * debug_malloc_type and debug_malloc_size define the type and size of
@@ -104,12 +106,13 @@ int debug_malloc_chunks_on_freelist;
 int debug_malloc_initialized;
 
 struct pool debug_malloc_pool;
+struct mutex debug_malloc_mtx = MUTEX_INITIALIZER(IPL_VM);
 
 int
 debug_malloc(unsigned long size, int type, int flags, void **addr)
 {
        struct debug_malloc_entry *md = NULL;
-       int s, wait = (flags & M_NOWAIT) == 0;
+       int wait = (flags & M_NOWAIT) == 0;
 
        /* Careful not to compare unsigned long to int -1 */
        if (((type != debug_malloc_type && debug_malloc_type != 0) ||
@@ -123,13 +126,14 @@ debug_malloc(unsigned long size, int typ
        if (size > PAGE_SIZE)
                return (0);
 
-       s = splvm();
        if (debug_malloc_chunks_on_freelist < MALLOC_DEBUG_CHUNKS)
                debug_malloc_allocate_free(wait);
 
+       mtx_enter(&debug_malloc_mtx);
+
        md = TAILQ_FIRST(&debug_malloc_freelist);
        if (md == NULL) {
-               splx(s);
+               mtx_leave(&debug_malloc_mtx);
                return (0);
        }
        TAILQ_REMOVE(&debug_malloc_freelist, md, md_list);
@@ -137,14 +141,15 @@ debug_malloc(unsigned long size, int typ
 
        TAILQ_INSERT_HEAD(&debug_malloc_usedlist, md, md_list);
        debug_malloc_allocs++;
-       splx(s);
-
-       pmap_kenter_pa(md->md_va, md->md_pa, PROT_READ | PROT_WRITE);
-       pmap_update(pmap_kernel());
 
        md->md_size = size;
        md->md_type = type;
 
+       mtx_leave(&debug_malloc_mtx);
+
+       pmap_kenter_pa(md->md_va, md->md_pa, PROT_READ | PROT_WRITE);
+       pmap_update(pmap_kernel());
+
        /*
         * Align the returned addr so that it ends where the first page
         * ends. roundup to get decent alignment.
@@ -158,7 +163,6 @@ debug_free(void *addr, int type)
 {
        struct debug_malloc_entry *md;
        vaddr_t va;
-       int s;
 
        if (type != debug_malloc_type && debug_malloc_type != 0 &&
            type != M_DEBUG)
@@ -169,7 +173,7 @@ debug_free(void *addr, int type)
         */
        va = trunc_page((vaddr_t)addr);
 
-       s = splvm();
+       mtx_enter(&debug_malloc_mtx);
        TAILQ_FOREACH(md, &debug_malloc_usedlist, md_list)
                if (md->md_va == va)
                        break;
@@ -185,21 +189,24 @@ debug_free(void *addr, int type)
                TAILQ_FOREACH(md, &debug_malloc_freelist, md_list)
                        if (md->md_va == va)
                                panic("debug_free: already free");
-               splx(s);
+               mtx_leave(&debug_malloc_mtx);
                return (0);
        }
 
        debug_malloc_frees++;
        TAILQ_REMOVE(&debug_malloc_usedlist, md, md_list);
+       mtx_leave(&debug_malloc_mtx);
 
-       TAILQ_INSERT_TAIL(&debug_malloc_freelist, md, md_list);
-       debug_malloc_chunks_on_freelist++;
        /*
         * unmap the page.
         */
        pmap_kremove(md->md_va, PAGE_SIZE);
        pmap_update(pmap_kernel());
-       splx(s);
+
+       mtx_enter(&debug_malloc_mtx);
+       TAILQ_INSERT_TAIL(&debug_malloc_freelist, md, md_list);
+       debug_malloc_chunks_on_freelist++;
+       mtx_leave(&debug_malloc_mtx);
 
        return (1);
 }
@@ -217,7 +224,7 @@ debug_malloc_init(void)
        debug_malloc_chunks_on_freelist = 0;
 
        pool_init(&debug_malloc_pool, sizeof(struct debug_malloc_entry),
-           0, 0, 0, "mdbepl", NULL);
+           0, 0, IPL_VM, "mdbepl", NULL);
 
        debug_malloc_initialized = 1;
 }
@@ -233,19 +240,18 @@ debug_malloc_allocate_free(int wait)
        vaddr_t va, offset;
        struct vm_page *pg;
        struct debug_malloc_entry *md;
-
-       splassert(IPL_VM);
+       int s;
 
        md = pool_get(&debug_malloc_pool, wait ? PR_WAITOK : PR_NOWAIT);
        if (md == NULL)
                return;
 
+       s = splvm();
+
        va = uvm_km_kmemalloc(kmem_map, NULL, PAGE_SIZE * 2,
            UVM_KMF_VALLOC | (wait ? 0: UVM_KMF_NOWAIT));
-       if (va == 0) {
-               pool_put(&debug_malloc_pool, md);
-               return;
-       }
+       if (va == 0)
+               goto put;
 
        offset = va - vm_map_min(kernel_map);
        for (;;) {
@@ -258,20 +264,31 @@ debug_malloc_allocate_free(int wait)
                if (pg)
                        break;
 
-               if (wait == 0) {
-                       uvm_unmap(kmem_map, va, va + PAGE_SIZE * 2);
-                       pool_put(&debug_malloc_pool, md);
-                       return;
-               }
+               if (wait == 0)
+                       goto unmap;
+
                uvm_wait("debug_malloc");
        }
 
+       splx(s);
+
        md->md_va = va;
        md->md_pa = VM_PAGE_TO_PHYS(pg);
 
+       mtx_enter(&debug_malloc_mtx);
        debug_malloc_pages++;
        TAILQ_INSERT_HEAD(&debug_malloc_freelist, md, md_list);
        debug_malloc_chunks_on_freelist++;
+       mtx_leave(&debug_malloc_mtx);
+
+       return;
+
+unmap:
+       uvm_unmap(kmem_map, va, va + PAGE_SIZE * 2);
+put:
+       splx(s);
+       pool_put(&debug_malloc_pool, md);
+       return;
 }
 
 void
@@ -311,7 +328,7 @@ debug_malloc_printit(
                        if (addr >= md->md_va &&
                            addr < md->md_va + 2 * PAGE_SIZE) {
                                (*pr)("Memory at address 0x%lx is in a freed "
-                                     "area. type %d, size: %d\n ",
+                                     "area. type %d, size: %zu\n ",
                                      addr, md->md_type, md->md_size);
                                return;
                        }
@@ -320,7 +337,7 @@ debug_malloc_printit(
                        if (addr >= md->md_va + PAGE_SIZE &&
                            addr < md->md_va + 2 * PAGE_SIZE) {
                                (*pr)("Memory at address 0x%lx is just outside "
-                                     "an allocated area. type %d, size: %d\n",
+                                     "an allocated area. type %d, size: %zu\n",
                                      addr, md->md_type, md->md_size);
                                return;
                        }

Reply via email to