Hi Pádraig,
> Weird, NetBSD 10 was the only VM I tested the previous fix on.
> Also it's weird because it suggests a mismatch between isprint()
> and wcwidth() which is generally -1 for non printable chars.
>
> Ah I see NetBSD 11 is now used in CI.
Yes. Sorry, I forgot to mention that NetBSD 11 has a different behaviour
than NetBSD 10 regarding some of the Unicode related stuff.
> and gnulib's width only returns -1 for some control chars.
> How about something like the following to return -1 for all control chars?
> ...
>
> diff --git a/lib/uniwidth/width.c b/lib/uniwidth/width.c
> index 0b440ac395..0063d005ef 100644
> --- a/lib/uniwidth/width.c
> +++ b/lib/uniwidth/width.c
> @@ -23,6 +23,7 @@
> #include <stdcountof.h>
>
> #include "cjk.h"
> +#include "unictype.h"
>
> /* The non-spacing attribute table consists of:
> * Non-spacing characters; generated from PropList.txt or
> @@ -57,18 +58,16 @@
> int
> uc_width (ucs4_t uc, const char *encoding)
> {
> - /* Test for non-spacing or control character. */
> + if (uc != 0 && uc_is_cntrl (uc))
> + return -1;
> +
> + /* Test for non-spacing character. */
> if ((uc >> 9) < countof (nonspacing_table_ind))
> {
> int ind = nonspacing_table_ind[uc >> 9];
> if (ind >= 0)
> if ((nonspacing_table_data[64*ind + ((uc >> 3) & 63)] >> (uc & 7))
> & 1)
> - {
> - if (uc > 0 && uc < 0xa0)
> - return -1;
> - else
> - return 0;
> - }
> + return 0;
> }
> else if ((uc >> 9) == (0xe0000 >> 9))
> {
For all control chars, that would be too much of a change.
But the change that you are suggesting is that uc_width returns -1 for
U+2028 LINE SEPARATOR
and U+2029 PARAGRAPH SEPARATOR.
And that actually makes sense, because U+2028 is much like U+000A,
and U+2029 is like U+000B.
Done as follows:
2026-11-17 Bruno Haible <[email protected]>
uniwidth/width: Return -1 for characters U+2028, U+2029.
Suggested by Pádraig Brady in
<https://lists.gnu.org/archive/html/coreutils/2026-11/msg00152.html>.
* lib/gen-uni-tables.c (is_nonspacing): Return true also for the
LINE SEPARATOR and PARAGRAPH SEPARATOR characters.
(symbolic_width): Update accordingly.
* lib/uniwidth/width0.h: Regenerated.
* lib/uniwidth/width.c (uc_width): Return -1 for the LINE SEPARATOR and
PARAGRAPH SEPARATOR characters.
* tests/uniwidth/test-uc_width2.sh: Update expected result.
* modules/uniwidth/*: Bump required libunistring version.
* modules/unilbrk/*-width-linebreaks: Likewise.
* modules/unistdio/ulc-*printf: Likewise.
diff --git a/lib/gen-uni-tables.c b/lib/gen-uni-tables.c
index c890489b61..86abbb2fed 100644
--- a/lib/gen-uni-tables.c
+++ b/lib/gen-uni-tables.c
@@ -6703,6 +6703,11 @@ fill_width (const char *width_filename)
Rationale for the Prepended_Concatenation_Mark exception:
The Unicode standard says "Unlike most other format characters,
however, they should be rendered with a visible glyph".
+ * Line and paragraph separators; generated from
+ "grep '^[^;]*;[^;]*;Z[lp];' UnicodeData.txt"
+ Rationale: These characters move the "cursor" to column 0.
+ The line separator U+2028 is like U+000A.
+ The paragraph separator U+2029 is like U+000B.
* Zero width characters; generated from
"grep '^[^;]*;ZERO WIDTH ' UnicodeData.txt"
* Hangul Jamo characters that have conjoining behaviour:
@@ -6730,6 +6735,7 @@ is_nonspacing (unsigned int ch)
|| is_category_Cc (ch)
|| (is_category_Cf (ch)
&& !is_property_prepended_concatenation_mark (ch))
+ || is_category_Zl (ch) || is_category_Zp (ch)
|| str_startswith (unicode_attributes[ch].name, "ZERO WIDTH ")
|| (ch >= 0x1160 && ch <= 0x11A7) || (ch >= 0xD7B0 && ch <=
0xD7C6) /* jungseong */
|| (ch >= 0x11A8 && ch <= 0x11FF) || (ch >= 0xD7CB && ch <=
0xD7FB) /* jongseong */
@@ -6975,7 +6981,8 @@ symbolic_width (unsigned int ch)
else
{
/* Test for non-spacing or control character. */
- if (is_category_Cc (ch) && ch < 0x00A0)
+ if ((is_category_Cc (ch) && ch < 0x00A0)
+ || is_category_Zl (ch) || is_category_Zp (ch))
return 0;
if (is_nonspacing (ch))
return '0';
diff --git a/lib/uniwidth/width.c b/lib/uniwidth/width.c
index 0b440ac395..b42c732e4d 100644
--- a/lib/uniwidth/width.c
+++ b/lib/uniwidth/width.c
@@ -29,6 +29,8 @@
"grep '^[^;]*;[^;]*;[^;]*;[^;]*;NSM;' UnicodeData.txt"
* Format control characters; generated from
"grep '^[^;]*;[^;]*;Cf;' UnicodeData.txt"
+ * Line and paragraph separators; generated from
+ "grep '^[^;]*;[^;]*;Z[lp];' UnicodeData.txt"
* Zero width characters; generated from
"grep '^[^;]*;ZERO WIDTH ' UnicodeData.txt"
* Hangul Jamo characters that have conjoining behaviour:
@@ -64,7 +66,7 @@ uc_width (ucs4_t uc, const char *encoding)
if (ind >= 0)
if ((nonspacing_table_data[64*ind + ((uc >> 3) & 63)] >> (uc & 7)) & 1)
{
- if (uc > 0 && uc < 0xa0)
+ if ((uc > 0 && uc < 0xa0) || (uc & ~1) == 0x2028)
return -1;
else
return 0;
diff --git a/lib/uniwidth/width0.h b/lib/uniwidth/width0.h
index 3e27107f1d..0baa2e50a8 100644
--- a/lib/uniwidth/width0.h
+++ b/lib/uniwidth/width0.h
@@ -145,7 +145,7 @@ static const unsigned char nonspacing_table_data[50*64] = {
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1d80-0x1dbf */
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x1dc0-0x1dff */
/* 0x2000-0x21ff */
- 0x00, 0xf8, 0x00, 0x00, 0x00, 0x7c, 0x00, 0x00, /* 0x2000-0x203f */
+ 0x00, 0xf8, 0x00, 0x00, 0x00, 0x7f, 0x00, 0x00, /* 0x2000-0x203f */
0x00, 0x00, 0x00, 0x00, 0xdf, 0xff, 0x00, 0x00, /* 0x2040-0x207f */
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2080-0x20bf */
0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x01, 0x00, /* 0x20c0-0x20ff */
diff --git a/modules/unilbrk/u16-width-linebreaks
b/modules/unilbrk/u16-width-linebreaks
index 2ce127dc8f..6a5d3f8abc 100644
--- a/modules/unilbrk/u16-width-linebreaks
+++ b/modules/unilbrk/u16-width-linebreaks
@@ -11,7 +11,7 @@ uniwidth/width
unistr/u16-mbtouc-unsafe
configure.ac:
-gl_LIBUNISTRING_MODULE([1.5], [unilbrk/u16-width-linebreaks])
+gl_LIBUNISTRING_MODULE([1.6], [unilbrk/u16-width-linebreaks])
Makefile.am:
if LIBUNISTRING_COMPILE_UNILBRK_U16_WIDTH_LINEBREAKS
diff --git a/modules/unilbrk/u32-width-linebreaks
b/modules/unilbrk/u32-width-linebreaks
index 7d3469df2c..167c502f98 100644
--- a/modules/unilbrk/u32-width-linebreaks
+++ b/modules/unilbrk/u32-width-linebreaks
@@ -10,7 +10,7 @@ unilbrk/u32-possible-linebreaks
uniwidth/width
configure.ac:
-gl_LIBUNISTRING_MODULE([1.5], [unilbrk/u32-width-linebreaks])
+gl_LIBUNISTRING_MODULE([1.6], [unilbrk/u32-width-linebreaks])
Makefile.am:
if LIBUNISTRING_COMPILE_UNILBRK_U32_WIDTH_LINEBREAKS
diff --git a/modules/unilbrk/u8-width-linebreaks
b/modules/unilbrk/u8-width-linebreaks
index 9cfab4cc9a..ef1f4a0ea7 100644
--- a/modules/unilbrk/u8-width-linebreaks
+++ b/modules/unilbrk/u8-width-linebreaks
@@ -12,7 +12,7 @@ uniwidth/width
unistr/u8-mbtouc-unsafe
configure.ac:
-gl_LIBUNISTRING_MODULE([1.5], [unilbrk/u8-width-linebreaks])
+gl_LIBUNISTRING_MODULE([1.6], [unilbrk/u8-width-linebreaks])
Makefile.am:
if LIBUNISTRING_COMPILE_UNILBRK_U8_WIDTH_LINEBREAKS
diff --git a/modules/unilbrk/ulc-width-linebreaks
b/modules/unilbrk/ulc-width-linebreaks
index 08409737c0..a67d3621ef 100644
--- a/modules/unilbrk/ulc-width-linebreaks
+++ b/modules/unilbrk/ulc-width-linebreaks
@@ -12,7 +12,7 @@ uniconv/u8-conv-from-enc
c-ctype
configure.ac:
-gl_LIBUNISTRING_MODULE([1.5], [unilbrk/ulc-width-linebreaks])
+gl_LIBUNISTRING_MODULE([1.6], [unilbrk/ulc-width-linebreaks])
Makefile.am:
if LIBUNISTRING_COMPILE_UNILBRK_ULC_WIDTH_LINEBREAKS
diff --git a/modules/unistdio/ulc-asnprintf b/modules/unistdio/ulc-asnprintf
index c813f6c091..564ad1143f 100644
--- a/modules/unistdio/ulc-asnprintf
+++ b/modules/unistdio/ulc-asnprintf
@@ -11,7 +11,7 @@ unistdio/base
unistdio/ulc-vasnprintf
configure.ac:
-gl_LIBUNISTRING_MODULE(m4_ifdef([gl_PRINTF_SUPPORT_N_DIRECTIVE],[999.9],[1.5]),
[unistdio/ulc-asnprintf])
+gl_LIBUNISTRING_MODULE(m4_ifdef([gl_PRINTF_SUPPORT_N_DIRECTIVE],[999.9],[1.6]),
[unistdio/ulc-asnprintf])
Makefile.am:
if LIBUNISTRING_COMPILE_UNISTDIO_ULC_ASNPRINTF
diff --git a/modules/unistdio/ulc-asprintf b/modules/unistdio/ulc-asprintf
index 59f5c07bd2..53ba93c10c 100644
--- a/modules/unistdio/ulc-asprintf
+++ b/modules/unistdio/ulc-asprintf
@@ -10,7 +10,7 @@ unistdio/base
unistdio/ulc-vasprintf
configure.ac:
-gl_LIBUNISTRING_MODULE(m4_ifdef([gl_PRINTF_SUPPORT_N_DIRECTIVE],[999.9],[1.5]),
[unistdio/ulc-asprintf])
+gl_LIBUNISTRING_MODULE(m4_ifdef([gl_PRINTF_SUPPORT_N_DIRECTIVE],[999.9],[1.6]),
[unistdio/ulc-asprintf])
Makefile.am:
if LIBUNISTRING_COMPILE_UNISTDIO_ULC_ASPRINTF
diff --git a/modules/unistdio/ulc-fprintf b/modules/unistdio/ulc-fprintf
index f28cad2e40..6cd7a7537c 100644
--- a/modules/unistdio/ulc-fprintf
+++ b/modules/unistdio/ulc-fprintf
@@ -11,7 +11,7 @@ fseterr
errno-h
configure.ac:
-gl_LIBUNISTRING_MODULE(m4_ifdef([gl_PRINTF_SUPPORT_N_DIRECTIVE],[999.9],[1.5]),
[unistdio/ulc-fprintf])
+gl_LIBUNISTRING_MODULE(m4_ifdef([gl_PRINTF_SUPPORT_N_DIRECTIVE],[999.9],[1.6]),
[unistdio/ulc-fprintf])
Makefile.am:
if LIBUNISTRING_COMPILE_UNISTDIO_ULC_FPRINTF
diff --git a/modules/unistdio/ulc-snprintf b/modules/unistdio/ulc-snprintf
index e936fae87c..4555535b2e 100644
--- a/modules/unistdio/ulc-snprintf
+++ b/modules/unistdio/ulc-snprintf
@@ -10,7 +10,7 @@ unistdio/base
unistdio/ulc-vsnprintf
configure.ac:
-gl_LIBUNISTRING_MODULE(m4_ifdef([gl_PRINTF_SUPPORT_N_DIRECTIVE],[999.9],[1.5]),
[unistdio/ulc-snprintf])
+gl_LIBUNISTRING_MODULE(m4_ifdef([gl_PRINTF_SUPPORT_N_DIRECTIVE],[999.9],[1.6]),
[unistdio/ulc-snprintf])
Makefile.am:
if LIBUNISTRING_COMPILE_UNISTDIO_ULC_SNPRINTF
diff --git a/modules/unistdio/ulc-sprintf b/modules/unistdio/ulc-sprintf
index d9d69a927b..4b0d6cd816 100644
--- a/modules/unistdio/ulc-sprintf
+++ b/modules/unistdio/ulc-sprintf
@@ -10,7 +10,7 @@ unistdio/base
unistdio/ulc-vsprintf
configure.ac:
-gl_LIBUNISTRING_MODULE(m4_ifdef([gl_PRINTF_SUPPORT_N_DIRECTIVE],[999.9],[1.5]),
[unistdio/ulc-sprintf])
+gl_LIBUNISTRING_MODULE(m4_ifdef([gl_PRINTF_SUPPORT_N_DIRECTIVE],[999.9],[1.6]),
[unistdio/ulc-sprintf])
Makefile.am:
if LIBUNISTRING_COMPILE_UNISTDIO_ULC_SPRINTF
diff --git a/modules/unistdio/ulc-vasnprintf b/modules/unistdio/ulc-vasnprintf
index ed383a7547..032580075d 100644
--- a/modules/unistdio/ulc-vasnprintf
+++ b/modules/unistdio/ulc-vasnprintf
@@ -48,7 +48,7 @@ streq
configure.ac:
gl_PREREQ_VASNPRINTF_WITH_POSIX_EXTRAS
-gl_LIBUNISTRING_MODULE(m4_ifdef([gl_PRINTF_SUPPORT_N_DIRECTIVE],[999.9],[1.5]),
[unistdio/ulc-vasnprintf])
+gl_LIBUNISTRING_MODULE(m4_ifdef([gl_PRINTF_SUPPORT_N_DIRECTIVE],[999.9],[1.6]),
[unistdio/ulc-vasnprintf])
Makefile.am:
if LIBUNISTRING_COMPILE_UNISTDIO_ULC_VASNPRINTF
diff --git a/modules/unistdio/ulc-vasprintf b/modules/unistdio/ulc-vasprintf
index 69ee61e736..e49577d6ad 100644
--- a/modules/unistdio/ulc-vasprintf
+++ b/modules/unistdio/ulc-vasprintf
@@ -11,7 +11,7 @@ unistdio/ulc-vasnprintf
errno-h
configure.ac:
-gl_LIBUNISTRING_MODULE(m4_ifdef([gl_PRINTF_SUPPORT_N_DIRECTIVE],[999.9],[1.5]),
[unistdio/ulc-vasprintf])
+gl_LIBUNISTRING_MODULE(m4_ifdef([gl_PRINTF_SUPPORT_N_DIRECTIVE],[999.9],[1.6]),
[unistdio/ulc-vasprintf])
Makefile.am:
if LIBUNISTRING_COMPILE_UNISTDIO_ULC_VASPRINTF
diff --git a/modules/unistdio/ulc-vfprintf b/modules/unistdio/ulc-vfprintf
index 28819f4166..ca2840238b 100644
--- a/modules/unistdio/ulc-vfprintf
+++ b/modules/unistdio/ulc-vfprintf
@@ -11,7 +11,7 @@ fseterr
errno-h
configure.ac:
-gl_LIBUNISTRING_MODULE(m4_ifdef([gl_PRINTF_SUPPORT_N_DIRECTIVE],[999.9],[1.5]),
[unistdio/ulc-vfprintf])
+gl_LIBUNISTRING_MODULE(m4_ifdef([gl_PRINTF_SUPPORT_N_DIRECTIVE],[999.9],[1.6]),
[unistdio/ulc-vfprintf])
Makefile.am:
if LIBUNISTRING_COMPILE_UNISTDIO_ULC_VFPRINTF
diff --git a/modules/unistdio/ulc-vsnprintf b/modules/unistdio/ulc-vsnprintf
index 14e3dfb038..f2b7b85075 100644
--- a/modules/unistdio/ulc-vsnprintf
+++ b/modules/unistdio/ulc-vsnprintf
@@ -11,7 +11,7 @@ unistdio/ulc-vasnprintf
errno-h
configure.ac:
-gl_LIBUNISTRING_MODULE(m4_ifdef([gl_PRINTF_SUPPORT_N_DIRECTIVE],[999.9],[1.5]),
[unistdio/ulc-vsnprintf])
+gl_LIBUNISTRING_MODULE(m4_ifdef([gl_PRINTF_SUPPORT_N_DIRECTIVE],[999.9],[1.6]),
[unistdio/ulc-vsnprintf])
Makefile.am:
if LIBUNISTRING_COMPILE_UNISTDIO_ULC_VSNPRINTF
diff --git a/modules/unistdio/ulc-vsprintf b/modules/unistdio/ulc-vsprintf
index 5065eac2ed..974ad8b44d 100644
--- a/modules/unistdio/ulc-vsprintf
+++ b/modules/unistdio/ulc-vsprintf
@@ -12,7 +12,7 @@ stdint-h
errno-h
configure.ac:
-gl_LIBUNISTRING_MODULE(m4_ifdef([gl_PRINTF_SUPPORT_N_DIRECTIVE],[999.9],[1.5]),
[unistdio/ulc-vsprintf])
+gl_LIBUNISTRING_MODULE(m4_ifdef([gl_PRINTF_SUPPORT_N_DIRECTIVE],[999.9],[1.6]),
[unistdio/ulc-vsprintf])
Makefile.am:
if LIBUNISTRING_COMPILE_UNISTDIO_ULC_VSPRINTF
diff --git a/modules/uniwidth/u16-strwidth b/modules/uniwidth/u16-strwidth
index c46bb8e1cd..392c7018ad 100644
--- a/modules/uniwidth/u16-strwidth
+++ b/modules/uniwidth/u16-strwidth
@@ -10,7 +10,7 @@ uniwidth/u16-width
unistr/u16-strlen
configure.ac:
-gl_LIBUNISTRING_MODULE([1.5], [uniwidth/u16-strwidth])
+gl_LIBUNISTRING_MODULE([1.6], [uniwidth/u16-strwidth])
Makefile.am:
if LIBUNISTRING_COMPILE_UNIWIDTH_U16_STRWIDTH
diff --git a/modules/uniwidth/u16-width b/modules/uniwidth/u16-width
index 84f3c6c9ed..e0cb04b203 100644
--- a/modules/uniwidth/u16-width
+++ b/modules/uniwidth/u16-width
@@ -10,7 +10,7 @@ uniwidth/width
unistr/u16-mbtouc-unsafe
configure.ac:
-gl_LIBUNISTRING_MODULE([1.5], [uniwidth/u16-width])
+gl_LIBUNISTRING_MODULE([1.6], [uniwidth/u16-width])
Makefile.am:
if LIBUNISTRING_COMPILE_UNIWIDTH_U16_WIDTH
diff --git a/modules/uniwidth/u32-strwidth b/modules/uniwidth/u32-strwidth
index 064166113f..0b77938b8a 100644
--- a/modules/uniwidth/u32-strwidth
+++ b/modules/uniwidth/u32-strwidth
@@ -10,7 +10,7 @@ uniwidth/u32-width
unistr/u32-strlen
configure.ac:
-gl_LIBUNISTRING_MODULE([1.5], [uniwidth/u32-strwidth])
+gl_LIBUNISTRING_MODULE([1.6], [uniwidth/u32-strwidth])
Makefile.am:
if LIBUNISTRING_COMPILE_UNIWIDTH_U32_STRWIDTH
diff --git a/modules/uniwidth/u32-width b/modules/uniwidth/u32-width
index 85d9fd603b..b91fbde1d6 100644
--- a/modules/uniwidth/u32-width
+++ b/modules/uniwidth/u32-width
@@ -9,7 +9,7 @@ uniwidth/base
uniwidth/width
configure.ac:
-gl_LIBUNISTRING_MODULE([1.5], [uniwidth/u32-width])
+gl_LIBUNISTRING_MODULE([1.6], [uniwidth/u32-width])
Makefile.am:
if LIBUNISTRING_COMPILE_UNIWIDTH_U32_WIDTH
diff --git a/modules/uniwidth/u8-strwidth b/modules/uniwidth/u8-strwidth
index 96fbcb20b7..610ac35d27 100644
--- a/modules/uniwidth/u8-strwidth
+++ b/modules/uniwidth/u8-strwidth
@@ -10,7 +10,7 @@ uniwidth/u8-width
unistr/u8-strlen
configure.ac:
-gl_LIBUNISTRING_MODULE([1.5], [uniwidth/u8-strwidth])
+gl_LIBUNISTRING_MODULE([1.6], [uniwidth/u8-strwidth])
Makefile.am:
if LIBUNISTRING_COMPILE_UNIWIDTH_U8_STRWIDTH
diff --git a/modules/uniwidth/u8-width b/modules/uniwidth/u8-width
index 93934d25bd..7c24e1ff41 100644
--- a/modules/uniwidth/u8-width
+++ b/modules/uniwidth/u8-width
@@ -10,7 +10,7 @@ uniwidth/width
unistr/u8-mbtouc-unsafe
configure.ac:
-gl_LIBUNISTRING_MODULE([1.5], [uniwidth/u8-width])
+gl_LIBUNISTRING_MODULE([1.6], [uniwidth/u8-width])
Makefile.am:
if LIBUNISTRING_COMPILE_UNIWIDTH_U8_WIDTH
diff --git a/modules/uniwidth/width b/modules/uniwidth/width
index 86d7dc7c55..1962302960 100644
--- a/modules/uniwidth/width
+++ b/modules/uniwidth/width
@@ -14,7 +14,7 @@ streq-opt
stdcountof-h
configure.ac:
-gl_LIBUNISTRING_MODULE([1.5], [uniwidth/width])
+gl_LIBUNISTRING_MODULE([1.6], [uniwidth/width])
Makefile.am:
if LIBUNISTRING_COMPILE_UNIWIDTH_WIDTH
diff --git a/tests/uniwidth/test-uc_width2.sh b/tests/uniwidth/test-uc_width2.sh
index ccfb574180..ea19da19e4 100755
--- a/tests/uniwidth/test-uc_width2.sh
+++ b/tests/uniwidth/test-uc_width2.sh
@@ -340,7 +340,7 @@ cat > uc_width.ok <<\EOF
1DC0..1DFF 0
1E00..200A A
200B..200F 0
-2010..2029 A
+2010..2027 A
202A..202E 0
202F..205F A
2060..2064 0