Author: markj Date: Wed Feb 15 01:50:58 2017 New Revision: 313756 URL: https://svnweb.freebsd.org/changeset/base/313756
Log: Apply MADV_FREE to exec_map entries only after a lowmem event. This effectively provides the same benefit as applying MADV_FREE inline upon every execve, since the page daemon invokes lowmem handlers prior to scanning the inactive queue. It also has less overhead; the cost of applying MADV_FREE is very noticeable on many-CPU systems since it includes that of a TLB shootdown of global PTEs. For instance, this change nearly halves the system CPU usage during a buildkernel on a 128-vCPU EC2 instance (with some other patches applied). Benchmarked by: cperciva (earlier version) Reviewed by: kib MFC after: 2 weeks Differential Revision: https://reviews.freebsd.org/D9586 Modified: head/sys/kern/kern_exec.c Modified: head/sys/kern/kern_exec.c ============================================================================== --- head/sys/kern/kern_exec.c Wed Feb 15 01:35:26 2017 (r313755) +++ head/sys/kern/kern_exec.c Wed Feb 15 01:50:58 2017 (r313756) @@ -1320,6 +1320,7 @@ err_exit: struct exec_args_kva { vm_offset_t addr; + u_int gen; SLIST_ENTRY(exec_args_kva) next; }; @@ -1327,6 +1328,7 @@ static DPCPU_DEFINE(struct exec_args_kva static SLIST_HEAD(, exec_args_kva) exec_args_kva_freelist; static struct mtx exec_args_kva_mtx; +static u_int exec_args_gen; static void exec_prealloc_args_kva(void *arg __unused) @@ -1339,6 +1341,7 @@ exec_prealloc_args_kva(void *arg __unuse for (i = 0; i < exec_map_entries; i++) { argkva = malloc(sizeof(*argkva), M_PARGS, M_WAITOK); argkva->addr = kmap_alloc_wait(exec_map, exec_map_entry_size); + argkva->gen = exec_args_gen; SLIST_INSERT_HEAD(&exec_args_kva_freelist, argkva, next); } } @@ -1364,15 +1367,16 @@ exec_alloc_args_kva(void **cookie) } static void -exec_free_args_kva(void *cookie) +exec_release_args_kva(struct exec_args_kva *argkva, u_int gen) { - struct exec_args_kva *argkva; vm_offset_t base; - argkva = cookie; base = argkva->addr; - - vm_map_madvise(exec_map, base, base + exec_map_entry_size, MADV_FREE); + if (argkva->gen != gen) { + vm_map_madvise(exec_map, base, base + exec_map_entry_size, + MADV_FREE); + argkva->gen = gen; + } if (!atomic_cmpset_ptr((uintptr_t *)DPCPU_PTR(exec_args_kva), (uintptr_t)NULL, (uintptr_t)argkva)) { mtx_lock(&exec_args_kva_mtx); @@ -1382,6 +1386,46 @@ exec_free_args_kva(void *cookie) } } +static void +exec_free_args_kva(void *cookie) +{ + + exec_release_args_kva(cookie, exec_args_gen); +} + +static void +exec_args_kva_lowmem(void *arg __unused) +{ + SLIST_HEAD(, exec_args_kva) head; + struct exec_args_kva *argkva; + u_int gen; + int i; + + gen = atomic_fetchadd_int(&exec_args_gen, 1) + 1; + + /* + * Force an madvise of each KVA range. Any currently allocated ranges + * will have MADV_FREE applied once they are freed. + */ + SLIST_INIT(&head); + mtx_lock(&exec_args_kva_mtx); + SLIST_SWAP(&head, &exec_args_kva_freelist, exec_args_kva); + mtx_unlock(&exec_args_kva_mtx); + while ((argkva = SLIST_FIRST(&head)) != NULL) { + SLIST_REMOVE_HEAD(&head, next); + exec_release_args_kva(argkva, gen); + } + + CPU_FOREACH(i) { + argkva = (void *)atomic_readandclear_ptr( + (uintptr_t *)DPCPU_ID_PTR(i, exec_args_kva)); + if (argkva != NULL) + exec_release_args_kva(argkva, gen); + } +} +EVENTHANDLER_DEFINE(vm_lowmem, exec_args_kva_lowmem, NULL, + EVENTHANDLER_PRI_ANY); + /* * Allocate temporary demand-paged, zero-filled memory for the file name, * argument, and environment strings. _______________________________________________ svn-src-all@freebsd.org mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-all To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
