This patch to libgo makes the implementation of function trampolines,
used for nested functions that refer to variables in an enclosing
function scope, much more efficient. Previously we were allocating a
separate page for each trampoline. This patch packs them on a single
page as much as possible. Bootstrapped and ran Go testsuite on
x86_64-unknown-linux-gnu. Commited to mainline and 4.7 branch.
Ian
diff -r 9dad4e6b17e9 libgo/runtime/go-trampoline.c
--- a/libgo/runtime/go-trampoline.c Fri May 25 14:12:42 2012 -0700
+++ b/libgo/runtime/go-trampoline.c Fri May 25 14:45:55 2012 -0700
@@ -14,40 +14,100 @@
#include <sys/mman.h>
#endif
-#include "go-alloc.h"
+#include "runtime.h"
+#include "arch.h"
+#include "malloc.h"
#include "go-assert.h"
-/* In order to build a trampoline we need space which is both writable
- and executable. We currently just allocate a whole page. This
- needs to be more system dependent. */
+/* Trampolines need to run in memory that is both writable and
+ executable. In order to implement them, we grab a page of memory
+ and mprotect it. We fill in the page with trampolines as they are
+ required. When we run out of space, we drop the pointer to the
+ page and allocate a new one. The page will be freed by the garbage
+ collector when there are no more variables of type func pointing to
+ it. */
+
+/* A lock to control access to the page of closures. */
+
+static Lock trampoline_lock;
+
+/* The page of closures. */
+
+static unsigned char *trampoline_page;
+
+/* The size of trampoline_page. */
+
+static uintptr_t trampoline_page_size;
+
+/* The number of bytes we have used on trampoline_page. */
+
+static uintptr_t trampoline_page_used;
+
+/* Allocate a trampoline of SIZE bytes that will use the closure in
+ CLOSURE. */
void *
__go_allocate_trampoline (uintptr_t size, void *closure)
{
- unsigned int page_size;
- void *ret;
- size_t off;
+ uintptr_t ptr_size;
+ uintptr_t full_size;
+ unsigned char *ret;
- page_size = getpagesize ();
- __go_assert (page_size >= size);
- ret = __go_alloc (2 * page_size - 1);
- ret = (void *) (((uintptr_t) ret + page_size - 1)
- & ~ ((uintptr_t) page_size - 1));
+ /* Because the garbage collector only looks at aligned addresses, we
+ need to store the closure at an aligned address to ensure that it
+ sees it. */
+ ptr_size = sizeof (void *);
+ full_size = (((size + ptr_size - 1) / ptr_size) * ptr_size);
+ full_size += ptr_size;
- /* Because the garbage collector only looks at correct address
- offsets, we need to ensure that it will see the closure
- address. */
- off = ((size + sizeof (void *) - 1) / sizeof (void *)) * sizeof (void *);
- __go_assert (size + off + sizeof (void *) <= page_size);
- __builtin_memcpy (ret + off, &closure, sizeof (void *));
+ runtime_lock (&trampoline_lock);
+
+ if (full_size < trampoline_page_size - trampoline_page_used)
+ trampoline_page = NULL;
+
+ if (trampoline_page == NULL)
+ {
+ uintptr_t page_size;
+ unsigned char *page;
+
+ page_size = getpagesize ();
+ __go_assert (page_size >= full_size);
+ page = (unsigned char *) runtime_mallocgc (2 * page_size - 1, 0, 0, 0);
+ page = (unsigned char *) (((uintptr_t) page + page_size - 1)
+ & ~ (page_size - 1));
#ifdef HAVE_SYS_MMAN_H
- {
- int i;
- i = mprotect (ret, size, PROT_READ | PROT_WRITE | PROT_EXEC);
- __go_assert (i == 0);
- }
+ {
+ int i;
+
+ i = mprotect (page, page_size, PROT_READ | PROT_WRITE | PROT_EXEC);
+ __go_assert (i == 0);
+ }
#endif
- return ret;
+ trampoline_page = page;
+ trampoline_page_size = page_size;
+ trampoline_page_used = 0;
+ }
+
+ ret = trampoline_page + trampoline_page_used;
+ trampoline_page_used += full_size;
+
+ runtime_unlock (&trampoline_lock);
+
+ __builtin_memcpy (ret + full_size - ptr_size, &closure, ptr_size);
+
+ return (void *) ret;
}
+
+/* Scan the trampoline page when running the garbage collector. This
+ just makes sure that the garbage collector sees the pointer in
+ trampoline_page, so that the page itself is not freed if there are
+ no other references to it. */
+
+void
+runtime_trampoline_scan (void (*scan) (byte *, int64))
+{
+ if (trampoline_page != NULL)
+ scan ((byte *) &trampoline_page, sizeof trampoline_page);
+}
diff -r 9dad4e6b17e9 libgo/runtime/mgc0.c
--- a/libgo/runtime/mgc0.c Fri May 25 14:12:42 2012 -0700
+++ b/libgo/runtime/mgc0.c Fri May 25 14:45:55 2012 -0700
@@ -703,6 +703,7 @@
scan((byte*)&runtime_allm, sizeof runtime_allm);
runtime_MProf_Mark(scan);
runtime_time_scan(scan);
+ runtime_trampoline_scan(scan);
// mark stacks
for(gp=runtime_allg; gp!=nil; gp=gp->alllink) {
diff -r 9dad4e6b17e9 libgo/runtime/runtime.h
--- a/libgo/runtime/runtime.h Fri May 25 14:12:42 2012 -0700
+++ b/libgo/runtime/runtime.h Fri May 25 14:45:55 2012 -0700
@@ -490,6 +490,7 @@
void runtime_setprof(bool);
void runtime_time_scan(void (*)(byte*, int64));
+void runtime_trampoline_scan(void (*)(byte *, int64));
void runtime_setsig(int32, bool, bool);
#define runtime_setitimer setitimer
