OK great.
Here is the patch for --enable-mutex-debug.
make check will have to come a bit later. I have many scripts that do
more or less that, but they are mostly Python, and focused on the
journal's failure modes; I will need to consolidate them and convert
them to shell.
As for this patch: a system built from current master with
--enable-mutex-debug does not boot. The ext2fs bootstrap filesystem
dies on the first lookup through a translator, at the double unlock in
diskfs_S_dir_lookup. Patch 1 of the v2 series ("libdiskfs: Unlock the
directory once a lookup has found its node") fixes it. With that
series applied, it boots, and a large apt upgrade ran through with
every mutex checked.
Thanks again,
Milos
On Mon, Nov 2, 2026 at 12:23 AM Samuel Thibault <[email protected]>
wrote:
> Milos Nikic, le dim. 01 nov. 2026 22:29:45 -0800, a ecrit:
> > To be sure we mean the same thing: --enable-mutex-debug only changes
> > the compiler flags, so it works for a native build on the Hurd as well
> > as for a cross build.
>
> Yes
>
> > make check always runs inside a Hurd system,
> > since it mounts the image with settrans; in a cross build it would only
> > say so and skip.
>
> Yes.
>
> > The two are independent, but used together, a lock
> > bug that the operations reach fails the check.
>
> The first one can still be used on a live system (could be a sub-hurd)
> to easily run packages' testsuites etc.
>
> > Where would you like the new header to live? Maybe
> > libshouldbeinlibc/, so that it can apply to the whole tree rather than
> > only libdiskfs and the filesystems: libports, libpager and the other
> > translators may have the same kind of bugs.
>
> Yes.
>
> Samuel
>
From da36c151f95148092ab70b1e1f374f3ac3159e71 Mon Sep 17 00:00:00 2001
From: Milos Nikic <[email protected]>
Date: Mon, 2 Nov 2026 08:50:50 -0800
Subject: [PATCH] Add --enable-mutex-debug, for error-checking mutexes
A default mutex records no owner, so an unlock by a thread that does
not hold it, or of one nobody holds, passes silently and takes the lock
from its holder: such a bug only shows when another thread races for
that lock, far from the faulty unlock.
With --enable-mutex-debug, every C file is compiled with
libshouldbeinlibc/mutex-debug.h included first. It makes the mutexes
that are initialized with no attributes, or with
PTHREAD_MUTEX_INITIALIZER, error-checking ones, and stops the program
with a backtrace when a lock or unlock fails. It only changes the
compiler flags, so it works for a native build and for a cross build,
and a system or a sub-Hurd built with it can run packages' test suites
with every unbalanced lock caught where it happens.
---
Makeconf | 6 +++
config.make.in | 1 +
configure.ac | 6 +++
libshouldbeinlibc/mutex-debug.h | 94 +++++++++++++++++++++++++++++++++
4 files changed, 107 insertions(+)
create mode 100644 libshouldbeinlibc/mutex-debug.h
diff --git a/Makeconf b/Makeconf
index 288e7daae..4d18ebade 100644
--- a/Makeconf
+++ b/Makeconf
@@ -90,6 +90,12 @@ include $(..)config.make
configured = yes
endif
+# --enable-mutex-debug, once config.make has set mutex-debug. In CFLAGS,
+# not CPPFLAGS, which also preprocesses the MIG definitions.
+ifeq ($(mutex-debug),yes)
+CFLAGS += -include $(top_srcdir)/libshouldbeinlibc/mutex-debug.h
+endif
+
# If we're not configured, don't do deps; then `make TAGS' and such can work.
ifndef configured
no_deps = t
diff --git a/config.make.in b/config.make.in
index 1a30626b8..b4521dd58 100644
--- a/config.make.in
+++ b/config.make.in
@@ -11,6 +11,7 @@ asm_syntax = @asm_syntax@
# Build options.
build-profiled = @enable_profile@
+mutex-debug = @enable_mutex_debug@
build-static = @enable_static_progs@
boot-store-types = @boot_store_types@
diff --git a/configure.ac b/configure.ac
index 4ddb1fe9c..1e63f84f5 100644
--- a/configure.ac
+++ b/configure.ac
@@ -58,6 +58,12 @@ AC_ARG_ENABLE(profile,
[ --disable-profile do not build profiled libraries and programs])
AC_SUBST(enable_profile)
+AC_ARG_ENABLE(mutex-debug,
+[ --enable-mutex-debug make mutexes error-checking, and stop a program
+ with a backtrace when a lock or unlock fails],
+[], [enable_mutex_debug=no])
+AC_SUBST(enable_mutex_debug)
+
define([default_static],['ext2fs,iso9660fs,rumpdisk,pci-arbiter,acpi'])dnl
AC_ARG_ENABLE(static-progs,
[ --enable-static-progs=PROGRAMS...
diff --git a/libshouldbeinlibc/mutex-debug.h b/libshouldbeinlibc/mutex-debug.h
new file mode 100644
index 000000000..d5bbe59df
--- /dev/null
+++ b/libshouldbeinlibc/mutex-debug.h
@@ -0,0 +1,94 @@
+/* Error-checking mutexes, for --enable-mutex-debug.
+
+ Copyright (C) 2026 Free Software Foundation, Inc.
+ Written by Milos Nikic.
+
+ This program is free software; you can redistribute it and/or
+ modify it under the terms of the GNU General Public License as
+ published by the Free Software Foundation; either version 2, or (at
+ your option) any later version.
+
+ This program is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */
+
+/* A default mutex records no owner: its unlock clears the lock whoever
+ calls it, so an unlock by a thread that does not hold the mutex, or of
+ one nobody holds, passes silently, and takes the lock from a thread
+ that holds it. An error-checking mutex records its owner: its unlock
+ by another thread fails with EPERM, and its relock by the owner with
+ EDEADLK.
+
+ With --enable-mutex-debug, every C file of the tree is compiled with
+ this header included first (Makeconf). Every mutex initialized with
+ no attributes, or with PTHREAD_MUTEX_INITIALIZER, is then an
+ error-checking one, and a lock or unlock that fails stops the program
+ with a backtrace. */
+
+#ifndef _HURD_MUTEX_DEBUG_H
+#define _HURD_MUTEX_DEBUG_H
+
+/* Before the macros below, so that its declarations are not expanded. */
+#include <pthread.h>
+#include <execinfo.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+/* Print the failure and a backtrace through stderr, and abort. Not
+ through descriptor 2, as backtrace_symbols_fd and assert_backtrace
+ do: a bootstrap filesystem's stderr goes to the Mach console. The
+ addresses resolve with addr2line against the unstripped program. */
+static void __attribute__ ((noreturn, unused))
+mutex_debug_fail (const char *what, int err, const char *file, int line)
+{
+ void *bt[64];
+ int n = backtrace (bt, 64);
+ char **names = backtrace_symbols (bt, n);
+
+ fprintf (stderr, "mutex-debug: %s:%d: pthread_mutex_%s: %s\n",
+ file, line, what, strerror (err));
+ for (int i = 1; i < n; i++)
+ fprintf (stderr, " #%d %p %s\n", i, bt[i], names ? names[i] : "");
+ fflush (stderr);
+ abort ();
+}
+
+static inline int
+mutex_debug_init (pthread_mutex_t *m, const pthread_mutexattr_t *attr)
+{
+ pthread_mutexattr_t a;
+ int err;
+
+ if (attr)
+ return (pthread_mutex_init) (m, attr);
+ pthread_mutexattr_init (&a);
+ pthread_mutexattr_settype (&a, PTHREAD_MUTEX_ERRORCHECK);
+ err = (pthread_mutex_init) (m, &a);
+ pthread_mutexattr_destroy (&a);
+ return err;
+}
+
+#define pthread_mutex_init(m, attr) mutex_debug_init ((m), (attr))
+
+#undef PTHREAD_MUTEX_INITIALIZER
+#define PTHREAD_MUTEX_INITIALIZER __PTHREAD_ERRORCHECK_MUTEX_INITIALIZER
+
+#define pthread_mutex_lock(m) \
+ ({ int __mutex_debug_err = (pthread_mutex_lock) (m); \
+ if (__mutex_debug_err) \
+ mutex_debug_fail ("lock", __mutex_debug_err, __FILE__, __LINE__); \
+ __mutex_debug_err; })
+
+#define pthread_mutex_unlock(m) \
+ ({ int __mutex_debug_err = (pthread_mutex_unlock) (m); \
+ if (__mutex_debug_err) \
+ mutex_debug_fail ("unlock", __mutex_debug_err, __FILE__, __LINE__); \
+ __mutex_debug_err; })
+
+#endif /* _HURD_MUTEX_DEBUG_H */
--
2.56.0
