The diff below implements __cxa_thread_atexit(). Calls to this function are emitted by the compiler to schedule running desctructors for thread_local objects when a thread terminates or calls exit(3). The Linux implementation prevents unloading of shared libraries that registered such destructors to prevent things from crashing. This diff does not implement that functionality yet. I plan to add that later. I expect this to be a bit of a corner case.
I've chosen to implement __cxa_thread_atexit() directly instead of __cxa_thread_atexit_impl(). I think that is cleaner. It means we don't need to make changes to libc++ for this to start working. It looks like modern libstdc++ version will detect __cxa_thread_atexit(). This adds a member to the "TIB". Tis means that you'll need to install the new headers and rebuild ld.so before rebuilding libpthread and libc. This will require a libc minor bump. But maybe it could ride the scheduled major bump? Index: include/tib.h =================================================================== RCS file: /cvs/src/include/tib.h,v retrieving revision 1.4 diff -u -p -r1.4 tib.h --- include/tib.h 20 Apr 2017 16:07:52 -0000 1.4 +++ include/tib.h 11 Aug 2017 14:20:30 -0000 @@ -135,6 +135,7 @@ */ struct tib { + void *tib_atexit; int tib_thread_flags; /* internal to libpthread */ pid_t tib_tid; int tib_cantcancel; @@ -182,6 +183,7 @@ struct tib { int tib_cantcancel; pid_t tib_tid; int tib_thread_flags; /* internal to libpthread */ + void *tib_atexit; #if !defined(__LP64__) && !defined(__i386) int __tib_padding; /* padding for 8byte alignment */ #endif @@ -207,6 +209,7 @@ struct tib { #define TIB_INIT(tib, dtv, thread) do { \ (tib)->tib_thread = (thread); \ + (tib)->tib_atexit = NULL; \ (tib)->tib_locale = NULL; \ (tib)->tib_cantcancel = 0; \ (tib)->tib_cancel_point = 0; \ Index: lib/librthread/rthread.c =================================================================== RCS file: /cvs/src/lib/librthread/rthread.c,v retrieving revision 1.95 diff -u -p -r1.95 rthread.c --- lib/librthread/rthread.c 27 Jul 2017 16:35:08 -0000 1.95 +++ lib/librthread/rthread.c 11 Aug 2017 14:20:30 -0000 @@ -331,6 +331,12 @@ pthread_exit(void *retval) oclfn->fn(oclfn->arg); free(oclfn); } + while (tib->tib_atexit) { + struct thread_atexit_fn *fnp = tib->tib_atexit; + tib->tib_atexit = fnp->next; + fnp->func(fnp->arg); + free(fnp); + } _rthread_tls_destructors(thread); _spinlock(&_thread_lock); LIST_REMOVE(thread, threads); Index: lib/libc/Symbols.list =================================================================== RCS file: /cvs/src/lib/libc/Symbols.list,v retrieving revision 1.54 diff -u -p -r1.54 Symbols.list --- lib/libc/Symbols.list 19 Jun 2017 03:06:26 -0000 1.54 +++ lib/libc/Symbols.list 11 Aug 2017 14:20:30 -0000 @@ -1420,6 +1420,7 @@ random /* stdlib */ _Exit __cxa_atexit +__cxa_thread_atexit __cxa_finalize __isthreaded a64l Index: lib/libc/include/thread_private.h =================================================================== RCS file: /cvs/src/lib/libc/include/thread_private.h,v retrieving revision 1.29 diff -u -p -r1.29 thread_private.h --- lib/libc/include/thread_private.h 15 Oct 2016 18:24:40 -0000 1.29 +++ lib/libc/include/thread_private.h 11 Aug 2017 14:20:31 -0000 @@ -96,6 +96,12 @@ struct thread_callbacks { __pid_t (*tc_vfork)(void); }; +struct thread_atexit_fn { + void (*func)(void *); + void *arg; + struct thread_atexit_fn *next; +}; + __BEGIN_PUBLIC_DECLS /* * Set the callbacks used by libc Index: lib/libc/stdlib/atexit.c =================================================================== RCS file: /cvs/src/lib/libc/stdlib/atexit.c,v retrieving revision 1.24 diff -u -p -r1.24 atexit.c --- lib/libc/stdlib/atexit.c 10 Nov 2015 04:14:03 -0000 1.24 +++ lib/libc/stdlib/atexit.c 11 Aug 2017 14:20:31 -0000 @@ -38,6 +38,8 @@ #include "atfork.h" #include "thread_private.h" +#include "tib.h" + struct atexit *__atexit; static int restartloop; @@ -121,6 +123,25 @@ atexit(void (*fn)(void)) } DEF_STRONG(atexit); +int +__cxa_thread_atexit(void (*func)(void *), void *arg, void *dso) +{ + struct thread_atexit_fn *fnp; + struct tib *tib = TIB_GET(); + + fnp = calloc(1, sizeof(struct thread_atexit_fn)); + if (fnp == NULL) + return -1; + + fnp->func = func; + fnp->arg = arg; + fnp->next = tib->tib_atexit; + tib->tib_atexit = fnp; + + return 0; +} +DEF_STRONG(__cxa_thread_atexit); + /* * Call all handlers registered with __cxa_atexit() for the shared * object owning 'dso'. @@ -129,10 +150,20 @@ DEF_STRONG(atexit); void __cxa_finalize(void *dso) { + struct tib *tib = TIB_GET(); struct atexit *p, *q; struct atexit_fn fn; int n, pgsize = getpagesize(); static int call_depth; + + if (dso == NULL) { + while (tib->tib_atexit) { + struct thread_atexit_fn *fnp = tib->tib_atexit; + tib->tib_atexit = fnp->next; + fnp->func(fnp->arg); + free(fnp); + } + } _ATEXIT_LOCK(); call_depth++; Index: lib/libc/stdlib/atexit.h =================================================================== RCS file: /cvs/src/lib/libc/stdlib/atexit.h,v retrieving revision 1.10 diff -u -p -r1.10 atexit.h --- lib/libc/stdlib/atexit.h 25 Oct 2015 18:01:24 -0000 1.10 +++ lib/libc/stdlib/atexit.h 11 Aug 2017 14:20:31 -0000 @@ -46,7 +46,9 @@ extern struct atexit *__atexit; /* poin __END_HIDDEN_DECLS int __cxa_atexit(void (*)(void *), void *, void *); +int __cxa_thread_atexit(void (*)(void *), void *, void *); void __cxa_finalize(void *); PROTO_NORMAL(__cxa_atexit); +PROTO_NORMAL(__cxa_thread_atexit); PROTO_NORMAL(__cxa_finalize);