On Tue, Oct 27, 2026 at 08:39:57AM -0400, Martin Cracauer wrote: > Konstantin Belousov wrote on Tue, Oct 27, 2026 at 02:33:12PM +0200: > > On Tue, Oct 27, 2026 at 03:23:28PM +0300, Dima Panov wrote: > > > > > > > > > On 27.10.2026 15:05, Konstantin Belousov wrote: > > > > On Tue, Oct 27, 2026 at 01:52:43PM +0300, Dima Panov wrote: > > > > > Hello! > > > > > > > > > > > > > > > Houston, we have a problem! > > > > > > > > > > When building latest stabweek world in poudriere on my aarch64 > > > > > machine, I've got an error in libthr when pkg command is called: > > > > > > > > > > > > > > > ===> Creating FreeBSD-kernel-generic-nodebug-16.snap20261026210619 > > > > > Fatal error 'mutex 0x41814480 own 0x8c935 is on list 0x0 0x409e0008' > > > > > at line 154 in file > > > > > /opt/poudriere/jails/16aarch64/usr/src/lib/libthr/thread/thr_mutex.c > > > > > (errno = 0) > > > > > Abort trap (core dumped) > > > > > > > > > > Host system was a 16.0-CURRENT #0 main-461da47e45e7: Sun Oct 18 > > > > > 02:22:28 MSK 2026 > > > > > Same error on host FreeBSD 16.0-CURRENT #0 main-5250c9890848: Tue Oct > > > > > 27 00:49:17 MSK 2026 > > > > > > > > > > But when I revert host to 16.0-CURRENT #0 main-17af098383ba: Fri Sep > > > > > 18 23:23:19 MSK 2026 > > > > > all builds runs without any errors > > > > > > > > > > Looks like some of latest commits between Sept 18 and Oct18 in libthr > > > > > causes this failure > > > > > > > > > > 4472a048cf95c9c1593a8b04655a59096b44b0c9 > > > > > 192781bf5d047cceff3a1947bfc754c8b5eb0835 > > > > > 1d21e67e720a08992e81d626cff9d76524a81f22 > > > > > 627e7d5b72cf967eae3cd9e9e0707ce0ff63a4da > > > > > 99edcc3bd52614c6cf972ea8424a7807ed3fe1e3 > > > > > de2c0f157a9cce0b763da9ae97ada870fa4d1824 > > > > > 305c3044e0249ea24a9b4571dfa6ae319a5bcdec > > > > > 5fb07bbcd5d5d9a865d9e7765fb93e9ba5edef1c > > > > > a0e152973613e0d0d55257f4e1f614509b7ec4c9 > > > > > 894f70a2bd8c7f67f131b76b9ce122a752818af3 > > > > > > > > What is the program that asserted? > > > > > > pkg when generates packages for recently build jail > > > > > > > Get the core dump and backtrace. > > > > > > /usr/obj/opt/poudriere/jails/16aarch64/usr/src/arm64.aarch64/packages/utilities$ > > > sudo gdb pkg pkg.core > > > GNU gdb (GDB) 15.1 [GDB v15.1 for FreeBSD] > > > [skip] > > > Type "apropos word" to search for commands related to "word"... > > > Reading symbols from pkg... > > > Reading symbols from /usr/lib/debug//usr/sbin/pkg.debug... > > > [New LWP 170934] > > > Core was generated by `pkg -o ABI=FreeBSD:16:aarch64 -o OSVERSION=1600029 > > > -o ALLOW_BASE_SHLIBS=yes -o S'. > > > Program terminated with signal SIGABRT, Aborted. > > > Sent by kill() from pid 45354 and user 0. > > > #0 0x0000000040fd52f4 in ?? () > > > (gdb) bt > > > #0 0x0000000040fd52f4 in ?? () > > > #1 0x00000000002fb784 in ?? () > > > Backtrace stopped: previous frame identical to this frame (corrupt stack?) > > > (gdb) > > > > You should build the system libraries with the debug symbols. > > Otherwise the backtrace is not useful. > > I don't think that would have helped in this specific instance. It > looks like a smashed stack.
Why? I would believe that unwinder become confused because there is no proper unwind info, but I am not sure about aarch64. Anyway, I think the following patch would provide some minimal useful information, but it is not a substitute for the full backtrace. commit cce9267d4c66eee4dd1be28024910944b6511939 Author: Konstantin Belousov <[email protected]> Date: Tue Oct 27 15:05:59 2026 +0200 libthr/thr_mutex.c: make mutex asserts more useful Print the line number of the caller where the assert fired. Sponsored by: The FreeBSD Foundation MFC after: 1 week diff --git a/lib/libthr/thread/thr_mutex.c b/lib/libthr/thread/thr_mutex.c index 9c0939aa6080..9c6f6d8c5dd9 100644 --- a/lib/libthr/thread/thr_mutex.c +++ b/lib/libthr/thread/thr_mutex.c @@ -131,41 +131,51 @@ mutex_init_link(struct pthread_mutex *m __unused) #endif } -static void -mutex_assert_is_owned(struct pthread_mutex *m __unused) -{ - #if defined(_PTHREADS_INVARIANTS) - if (__predict_false(m->m_qe.tqe_prev == NULL)) - PANIC("mutex %p own %#x is not on list %p %p", +static void +mutex_assert_is_owned(struct pthread_mutex *m, u_int line) +{ + if (__predict_false(m->m_qe.tqe_prev == NULL)) { + PANIC("%s:%u: mutex %p own %#x is not on list %p %p", + __FILE__, line, m, m->m_lock.m_owner, m->m_qe.tqe_prev, m->m_qe.tqe_next); -#endif + } } static void -mutex_assert_not_owned(struct pthread *curthread __unused, - struct pthread_mutex *m __unused) +mutex_assert_not_owned(struct pthread_mutex *m, u_int line) { + struct pthread *curthread; -#if defined(_PTHREADS_INVARIANTS) + curthread = _get_curthread(); if (__predict_false(m->m_qe.tqe_prev != NULL || - m->m_qe.tqe_next != NULL)) - PANIC("mutex %p own %#x is on list %p %p", + m->m_qe.tqe_next != NULL)) { + PANIC("%s:%u: mutex %p own %#x is on list %p %p", + __FILE__, line, m, m->m_lock.m_owner, m->m_qe.tqe_prev, m->m_qe.tqe_next); + } if (__predict_false(is_robust_mutex(m) && (m->m_lock.m_rb_lnk != 0 || m->m_rb_prev != NULL || (is_pshared_mutex(m) && curthread->robust_list == (uintptr_t)&m->m_lock) || (!is_pshared_mutex(m) && curthread->priv_robust_list == - (uintptr_t)&m->m_lock)))) + (uintptr_t)&m->m_lock)))) { PANIC( - "mutex %p own %#x is on robust linkage %p %p head %p phead %p", + "%s:%u: mutex %p own %#x is on robust linkage %p %p head %p phead %p", + __FILE__, line, m, m->m_lock.m_owner, (void *)m->m_lock.m_rb_lnk, m->m_rb_prev, (void *)curthread->robust_list, (void *)curthread->priv_robust_list); -#endif + } } +#define MUTEX_ASSERT_IS_OWNED(m) mutex_assert_is_owned(m, __LINE__) +#define MUTEX_ASSERT_NOT_OWNED(m) mutex_assert_not_owned(m, __LINE__) +#else +#define MUTEX_ASSERT_IS_OWNED(m) do {} while (0) +#define MUTEX_ASSERT_NOT_OWNED(m) do {} while (0) +#endif + static bool is_pshared_mutex(struct pthread_mutex *m) { @@ -477,8 +487,7 @@ _thr_mutex_destroy(pthread_mutex_t *mutex) if (m1 != NULL) { if ((uint32_t)m1->m_lock.m_owner != UMUTEX_RB_OWNERDEAD) { - mutex_assert_not_owned( - _get_curthread(), m1); + MUTEX_ASSERT_NOT_OWNED(m1); } __thr_pshared_destroy(mutex); } @@ -490,7 +499,7 @@ _thr_mutex_destroy(pthread_mutex_t *mutex) ret = EBUSY; } else { *mutex = THR_MUTEX_DESTROYED; - mutex_assert_not_owned(_get_curthread(), m); + MUTEX_ASSERT_NOT_OWNED(m); __thr_free(m); ret = 0; } @@ -530,7 +539,7 @@ enqueue_mutex(struct pthread *curthread, struct pthread_mutex *m, /* Add to the list of owned mutexes: */ if (error != EOWNERDEAD) - mutex_assert_not_owned(curthread, m); + MUTEX_ASSERT_NOT_OWNED(m); qidx = mutex_qidx(m); TAILQ_INSERT_TAIL(&curthread->mq[qidx], m, m_qe); if (!is_pshared_mutex(m)) @@ -558,7 +567,7 @@ dequeue_mutex(struct pthread *curthread, struct pthread_mutex *m) struct pthread_mutex *mp, *mn; int qidx; - mutex_assert_is_owned(m); + MUTEX_ASSERT_IS_OWNED(m); qidx = mutex_qidx(m); if (is_robust_mutex(m)) { mp = m->m_rb_prev; @@ -1054,7 +1063,7 @@ _pthread_mutex_setprioceiling(pthread_mutex_t * __restrict mutex, curthread = _get_curthread(); if (PMUTEX_OWNER_ID(m) == TID(curthread)) { - mutex_assert_is_owned(m); + MUTEX_ASSERT_IS_OWNED(m); m1 = TAILQ_PREV(m, mutex_queue, m_qe); m2 = TAILQ_NEXT(m, m_qe); if ((m1 != NULL && m1->m_lock.m_ceilings[0] > (u_int)ceiling) ||
