Paul Eggert wrote: > > Is that what you need for GNU sed? > > Yes, the idea is that if mbrtoc32 succeeds in the C locale, it should yield a > char32_t that can be fed back into c32rtomb and get the original byte.
Understood and implemented below. > > Currently it fails on: glibc, OpenBSD, Cygwin, mingw, MSVC. (Which is a > > little > > more than the platforms from > > <https://lists.gnu.org/archive/html/bug-gnulib/2026-08/msg00308.html>.) > The test also fails on NetBSD 11.0 because in the C locale its mbrtoc32 > treats bytes in the range 0x80-0xFF as encoding errors, like glibc does. It > seems that NetBSD 11.0 was the first NetBSD version to support mbrtoc32. Right, thanks for the reminder. I've tested this on NetBSD 11 now. > And the test succeeds on FreeBSD 15.1 but I think that may be due to another > bug, where Gnulib's btoc32(0x80) yields 0x80; isn't btoc32(0x80) supposed to > yield 0xDF80 in the C locale, so that we can run c32isprint on the result? If the test succeeds on FreeBSD, then all is fine. We now have tests - for consistency of c32isprint with isprint, - for consistency of c32rtomb with mbrtoc32, and both succeed on FreeBSD 15.1. Whether 0x80..0xFF gets mapped to U+0080..U+00FF or U+DF80..U+DFFF, is irrelevant. Bruno 2026-09-16 Bruno Haible <[email protected]> c32rtomb: Make consistent with mbrtoc32 in the C locale. Suggested by Paul Eggert. * m4/c32rtomb.m4 (gl_FUNC_C32RTOMB): Require gl_MBRTOC32_C_LOCALE, gl_MBRTOC32_C_LOCALE_LIKE_ISO_8859. When mbrtoc32 gets overridden due to the C locale, set also REPLACE_C32RTOMB to 1. * lib/c32rtomb.c: Include hard-locale.h, <locale.h>. (c32rtomb): On OpenBSD, NetBSD ≥ 11, glibc, Cygwin, mingw, MSVC, do special handling of non-ASCII characters in the C locale. * modules/c32rtomb (Depends-on): Add hard-locale. * tests/test-c32rtomb.c (main): Check that c32rtomb does the inverse of mbrtoc32 in the C locale. diff --git a/lib/c32rtomb.c b/lib/c32rtomb.c index 74d38ed754..cb8274f14a 100644 --- a/lib/c32rtomb.c +++ b/lib/c32rtomb.c @@ -32,11 +32,19 @@ # include "lc-charset-unicode.h" #endif +#if (MBRTOC32_IN_C_LOCALE_MAYBE_LIKE_ISO_8859 \ + || MBRTOC32_IN_C_LOCALE_MAYBE_EILSEQ \ + || (HAVE_WORKING_MBRTOC32 && HAVE_WORKING_C32RTOMB && !_GL_WCHAR_T_IS_UCS4) \ + || _GL_SMALL_WCHAR_T) +# include "hard-locale.h" +# include <locale.h> +#endif + size_t c32rtomb (char *s, char32_t wc, mbstate_t *ps) #undef c32rtomb { -#if HAVE_WORKING_MBRTOC32 && HAVE_WORKING_C32RTOMB +#if !GNULIB_defined_mbstate_t && HAVE_WORKING_MBRTOC32 && HAVE_WORKING_C32RTOMB # if C32RTOMB_RETVAL_BUG if (s == NULL) @@ -44,9 +52,30 @@ c32rtomb (char *s, char32_t wc, mbstate_t *ps) return 1; # endif +# if MBRTOC32_IN_C_LOCALE_MAYBE_LIKE_ISO_8859 /* OpenBSD */ \ + || !_GL_WCHAR_T_IS_UCS4 /* NetBSD ≥ 11 */ + if ((wc >= 0xDF80 && wc <= 0xDFFF) && !hard_locale (LC_CTYPE)) + { + /* In the "C" locale, map the code points U+DF80..U+DFFF back to the bytes + 0x80..0xFF, for consistency with the mbrtoc32 and btoc32 functions. */ + s[0] = (unsigned char) (wc - 0xDF00); + return 1; + } +# endif + +# if MBRTOC32_IN_C_LOCALE_MAYBE_EILSEQ + if ((wc >= 0x0080 && wc <= 0x00FF) && !hard_locale (LC_CTYPE)) + { + /* In the "C" locale, map the code points U+0080..U+00FF back to the bytes + 0x80..0xFF, for consistency with the mbrtoc32 and btoc32 functions. */ + s[0] = (unsigned char) wc; + return 1; + } +# endif + return c32rtomb (s, wc, ps); -#elif _GL_SMALL_WCHAR_T +#elif !GNULIB_defined_mbstate_t && _GL_SMALL_WCHAR_T /* Cygwin, mingw, MSVC */ if (s == NULL) return wcrtomb (NULL, 0, ps); @@ -100,6 +129,27 @@ c32rtomb (char *s, char32_t wc, mbstate_t *ps) return count; } } + else if (!hard_locale (LC_CTYPE)) + { + /* In the "C" locale, map the code points U+DF80..U+DFFF back + to the bytes 0x80..0xFF, for consistency with the mbrtoc32 and + btoc32 functions. */ + if (wc >= 0x00 && wc <= 0x7F) + { + s[0] = (unsigned char) wc; + return 1; + } + else if (wc >= 0xDF80 && wc <= 0xDFFF) + { + s[0] = (unsigned char) (wc - 0xDF00); + return 1; + } + else + { + errno = EILSEQ; + return (size_t)(-1); + } + } else { if ((wchar_t) wc == wc) diff --git a/m4/c32rtomb.m4 b/m4/c32rtomb.m4 index 8382e834ca..e40e9e891b 100644 --- a/m4/c32rtomb.m4 +++ b/m4/c32rtomb.m4 @@ -1,5 +1,5 @@ # c32rtomb.m4 -# serial 10 +# serial 11 dnl Copyright (C) 2020-2026 Free Software Foundation, Inc. dnl This file is free software; the Free Software Foundation dnl gives unlimited permission to copy and/or distribute it, @@ -12,6 +12,8 @@ AC_DEFUN([gl_FUNC_C32RTOMB] AC_REQUIRE([AC_CANONICAL_HOST]) AC_REQUIRE([gl_MBRTOC32_SANITYCHECK]) + AC_REQUIRE([gl_MBRTOC32_C_LOCALE]) + AC_REQUIRE([gl_MBRTOC32_C_LOCALE_LIKE_ISO_8859]) AC_REQUIRE([gl_C32RTOMB_SANITYCHECK]) AC_REQUIRE([gl_CHECK_FUNC_C32RTOMB]) @@ -23,6 +25,20 @@ AC_DEFUN([gl_FUNC_C32RTOMB] if test $HAVE_WORKING_MBRTOC32 = 0; then REPLACE_C32RTOMB=1 fi + dnl Likewise when we override it because of its behaviour in the C locale. + case "$gl_cv_func_mbrtoc32_C_locale_sans_EILSEQ" in + *yes) + case "$gl_cv_func_mbrtoc32_C_locale_like_iso_8859" in + *yes) + case "$host_os" in + solaris*) ;; + *) REPLACE_C32RTOMB=1 ;; + esac + ;; + esac + ;; + *) REPLACE_C32RTOMB=1 ;; + esac AC_CACHE_CHECK([whether c32rtomb return value is correct], [gl_cv_func_c32rtomb_retval], [ diff --git a/modules/c32rtomb b/modules/c32rtomb index 1b1bd000d0..449d885b9a 100644 --- a/modules/c32rtomb +++ b/modules/c32rtomb @@ -15,6 +15,7 @@ uchar-h attribute [test $HAVE_C32RTOMB = 0 || test $REPLACE_C32RTOMB = 1] wchar-h [test $HAVE_C32RTOMB = 0 || test $REPLACE_C32RTOMB = 1] wcrtomb [test $HAVE_C32RTOMB = 0 || test $REPLACE_C32RTOMB = 1] +hard-locale [test $HAVE_C32RTOMB = 0 || test $REPLACE_C32RTOMB = 1] localcharset [{ test $HAVE_C32RTOMB = 0 || test $REPLACE_C32RTOMB = 1; } && test $SMALL_WCHAR_T = 1] streq-opt [{ test $HAVE_C32RTOMB = 0 || test $REPLACE_C32RTOMB = 1; } && test $SMALL_WCHAR_T = 1] diff --git a/tests/test-c32rtomb.c b/tests/test-c32rtomb.c index 20cbc85baf..b3b4ddc963 100644 --- a/tests/test-c32rtomb.c +++ b/tests/test-c32rtomb.c @@ -116,6 +116,23 @@ main (int argc, char *argv[]) { case '1': /* C locale; tested above. */ +#if !defined __ANDROID__ + /* On Android ≥ 5.0, the default locale is the "C.UTF-8" locale, not the + "C" locale. Furthermore, when you attempt to set the "C" or "POSIX" + locale via setlocale(), what you get is a "C" locale with UTF-8 + encoding, that is, effectively the "C.UTF-8" locale. */ + /* Check that c32rtomb does the inverse of mbrtoc32, in the C locale. + Recall that POSIX:2024 says about mbrtoc32: + "In the POSIX locale an [EILSEQ] error cannot occur since all + byte values are valid characters." + Above we have only tested the ISO C "basic character set". */ + for (int c = 0; c < 0x100; c++) + { + ret = c32rtomb (buf, btoc32 (c), NULL); + ASSERT (ret == 1); + ASSERT (buf[0] == (char) c); + } +#endif return test_exit_status; case '2':
