On 2023-04-28 23:54, Jim Meyering wrote:
I've made some small adjustments and tidied up the ChangeLog in the attached.
One question about that patch (both original and as revised). Why do we
need a new binary_safe slot in struct pcre_comp? Shouldn't the
binary_safe stuff be done at compile-time rather than run-time?
Proposed revised patch attached. It also tweaks commentary slightly, and
uses a more uniform style in the test comments (something like what
Carlo suggested, but a bit wordier since it names the characters).From 15594ee2c611f35ca6251c33b337b94270d6e536 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Carlo=20Marcelo=20Arenas=20Bel=C3=B3n?= <care...@gmail.com>
Date: Thu, 20 Apr 2023 18:37:20 -0700
Subject: [PATCH] pcre: work around a PCRE2_MATCH_INVALID_UTF bug
PCRE2 has a bug when using PCRE2_MATCH_INVALID_UTF: it would
sometimes fail to match patterns using negative classes
like \W and \D.
* NEWS (Bug fixes): Mention it.
* src/pcre2search.c: Restrict impact of the bug.
Do not use the problematic flag with broken versions of PCRE2.
Also, generate locale tables only for single-byte locales,
as the PCRE2 documentation recommends this.
* tests/Makefile.am (TESTS): Add the file name
* tests/pcre-utf8-bug224: New file, to test for this.
---
NEWS | 5 +++++
src/pcresearch.c | 29 +++++++++++++++++------------
tests/Makefile.am | 1 +
tests/pcre-utf8-bug224 | 31 +++++++++++++++++++++++++++++++
4 files changed, 54 insertions(+), 12 deletions(-)
create mode 100755 tests/pcre-utf8-bug224
diff --git a/NEWS b/NEWS
index c15764c..53c1427 100644
--- a/NEWS
+++ b/NEWS
@@ -15,6 +15,11 @@ GNU grep NEWS -*- outline -*-
when running on 32-bit x86 and ARM hosts using glibc 2.34+.
[bug introduced in grep 3.9]
+ grep -P no longer fails to match patterns using negated classes
+ like \D or \W when linked with PCRE2 10.34 or newer.
+ [bug introduced in grep 3.8]
+
+
** Changes in behavior
grep --version now prints a line describing the version of PCRE2 it uses.
diff --git a/src/pcresearch.c b/src/pcresearch.c
index e867f49..44262ac 100644
--- a/src/pcresearch.c
+++ b/src/pcresearch.c
@@ -39,6 +39,15 @@
# define PCRE2_EXTRA_ASCII_BSD 0
#endif
+/* Use PCRE2_MATCH_INVALID_UTF if supported and not buggy;
+ see <https://github.com/PCRE2Project/pcre2/issues/224>.
+ Assume the bug will be fixed after PCRE2 10.42. */
+#if defined PCRE2_MATCH_INVALID_UTF && 10 < PCRE2_MAJOR + (42 < PCRE2_MINOR)
+enum { MATCH_INVALID_UTF = PCRE2_MATCH_INVALID_UTF };
+#else
+enum { MATCH_INVALID_UTF = 0 };
+#endif
+
struct pcre_comp
{
/* General context for PCRE operations. */
@@ -130,16 +139,11 @@ jit_exec (struct pcre_comp *pc, char const *subject, idx_t search_bytes,
}
}
-/* Return true if E is an error code for bad UTF-8, and if pcre2_match
- could return E because PCRE lacks PCRE2_MATCH_INVALID_UTF. */
+/* Return true if E is an error code for bad UTF-8. */
static bool
bad_utf8_from_pcre2 (int e)
{
-#ifdef PCRE2_MATCH_INVALID_UTF
- return false;
-#else
return PCRE2_ERROR_UTF8_ERR21 <= e && e <= PCRE2_ERROR_UTF8_ERR1;
-#endif
}
/* Compile the -P style PATTERN, containing SIZE bytes that are
@@ -168,6 +172,9 @@ Pcompile (char *pattern, idx_t size, reg_syntax_t ignored, bool exact)
flags |= PCRE2_UTF;
+ /* If supported, consider invalid UTF-8 as a barrier not an error. */
+ flags |= MATCH_INVALID_UTF;
+
/* If PCRE2_EXTRA_ASCII_BSD is available, use PCRE2_UCP
so that \d does not have the undesirable effect of matching
non-ASCII digits. Otherwise (i.e., with PCRE2 10.42 and earlier),
@@ -179,10 +186,6 @@ Pcompile (char *pattern, idx_t size, reg_syntax_t ignored, bool exact)
#if 0
/* Do not match individual code units but only UTF-8. */
flags |= PCRE2_NEVER_BACKSLASH_C;
-#endif
-#ifdef PCRE2_MATCH_INVALID_UTF
- /* Consider invalid UTF-8 as a barrier, instead of error. */
- flags |= PCRE2_MATCH_INVALID_UTF;
#endif
}
@@ -226,7 +229,9 @@ Pcompile (char *pattern, idx_t size, reg_syntax_t ignored, bool exact)
size = re_size;
}
- pcre2_set_character_tables (ccontext, pcre2_maketables (gcontext));
+ if (!localeinfo.multibyte)
+ pcre2_set_character_tables (ccontext, pcre2_maketables (gcontext));
+
pc->cre = pcre2_compile ((PCRE2_SPTR) pattern, size, flags,
&ec, &e, ccontext);
if (!pc->cre)
@@ -313,7 +318,7 @@ Pexecute (void *vcp, char const *buf, idx_t size, idx_t *match_size,
e = jit_exec (pc, subject, line_end - subject,
search_offset, options);
- if (!bad_utf8_from_pcre2 (e))
+ if (MATCH_INVALID_UTF || !bad_utf8_from_pcre2 (e))
break;
idx_t valid_bytes = pcre2_get_startchar (pc->data);
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 7718f24..9b4422e 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -155,6 +155,7 @@ TESTS = \
pcre-jitstack \
pcre-o \
pcre-utf8 \
+ pcre-utf8-bug224 \
pcre-utf8-w \
pcre-w \
pcre-wx-backref \
diff --git a/tests/pcre-utf8-bug224 b/tests/pcre-utf8-bug224
new file mode 100755
index 0000000..7884501
--- /dev/null
+++ b/tests/pcre-utf8-bug224
@@ -0,0 +1,31 @@
+#!/bin/sh
+# Ensure negated Perl classes match multibyte characters in UTF mode.
+#
+# Copyright (C) 2023 Free Software Foundation, Inc.
+#
+# Copying and distribution of this file, with or without modification,
+# are permitted in any medium without royalty provided the copyright
+# notice and this notice are preserved.
+
+. "${srcdir=.}/init.sh"; path_prepend_ ../src
+require_en_utf8_locale_
+LC_ALL=en_US.UTF-8
+export LC_ALL
+require_pcre_
+
+echo . | grep -qP '(*UTF).' 2>/dev/null \
+ || skip_ 'PCRE unicode support is compiled out'
+
+fail=0
+
+# 'ñ' - U+00F1 LATIN SMALL LETTER N WITH TILDE
+printf '\302\221\n' > in || framework_failure_
+grep -P '\D' in > out || fail=1
+compare in out || fail=1
+
+# '𝄞' - U+1D11E MUSICAL SYMBOL G CLEF
+printf '\360\235\204\236\n' > in || framework_failure_
+grep -P '\W' in > out || fail=1
+compare in out || fail=1
+
+Exit $fail
--
2.40.0
