Hi, Thank you for working on this!
On Tue, 15 Sept 2026 at 14:31, Andrew Dunstan <[email protected]> wrote: > > On 2026-09-15 Tu 12:22 AM, Chao Li wrote: > > > > The attached is my test script. > > > > Great, thanks for the review and tests. What do you think about continuing from where text_ascii_check() is left? I wrote a patch for this and benchmarked with Chao's script. Timings are v1 vs v2, not master vs v2. # unicode_is_normalized() * all ascii: 89.115ms | 87.803ms * mixed: 249.239ms | 126.011ms -> improvement * non-ascii: 1243.496ms | 1244.052ms * late-non-ascii: 3632.410ms | 370.966ms -> improvement # unicode_normalize_func() * all ascii: 84.458ms | 84.636ms * mixed: 578.684ms | 226.093ms -> improvement * non-ascii: 3651.573ms | 3644.026ms * late-non-ascii: 10513.861ms | 942.470ms -> improvement # unicode_assigned() * all ascii: 61.147ms | 60.658ms * mixed: 124.193ms | 86.186ms -> improvement * non-ascii: 507.130ms | 510.247ms * late-non-ascii: 1169.617ms | 166.924ms -> improvement Do you think these results worth the additional complexity? -- Regards, Nazir Bilal Yavuz Microsoft
From 10adbaa8cef3908f9bae4f848cd5b9803f6460c9 Mon Sep 17 00:00:00 2001 From: Andrew Dunstan <[email protected]> Date: Sun, 13 Sep 2026 17:13:15 -0400 Subject: [PATCH v2 1/2] Add ASCII fast path to Unicode normalization functions unicode_is_normalized(), unicode_assigned(), and unicode_normalize_func() each decoded its input to an array of char32_t codepoints, one utf8_to_unicode()/pg_utf_mblen() call at a time, before doing any real work. A string that is entirely ASCII (code points 0-127) has no canonical or compatibility decomposition anywhere in it, and a combining class of zero throughout, so it is trivially normalized under NFC/NFKC/NFD/NFKD alike. Every one of its code points is assigned too. Detect that case up front with a byte scan using the existing SIMD-vectorized is_valid_ascii() (already used by pg_utf8_verifystr()), and skip the decode loop and per-codepoint work entirely. Add regression coverage for the ASCII-hit case in all three functions, plus a chunk/remainder boundary sweep that plants a non-NFC sequence at varying offsets to catch any off-by-one in text_is_ascii()'s SIMD/ scalar split. --- src/backend/utils/adt/varlena.c | 47 ++++++++++++++++ src/test/regress/expected/unicode.out | 78 +++++++++++++++++++++++++++ src/test/regress/sql/unicode.sql | 14 +++++ 3 files changed, 139 insertions(+) diff --git a/src/backend/utils/adt/varlena.c b/src/backend/utils/adt/varlena.c index 01cc61c5778..3065b837537 100644 --- a/src/backend/utils/adt/varlena.c +++ b/src/backend/utils/adt/varlena.c @@ -35,6 +35,7 @@ #include "parser/scansup.h" #include "port/pg_bswap.h" #include "regex/regex.h" +#include "utils/ascii.h" #include "utils/builtins.h" #include "utils/guc.h" #include "utils/lsyscache.h" @@ -167,6 +168,7 @@ static void text_format_string_conversion(StringInfo buf, char conversion, int flags, int width); static void text_format_append_string(StringInfo buf, const char *str, int flags, int width); +static bool text_is_ascii(text *t); /***************************************************************************** @@ -5480,6 +5482,39 @@ icu_unicode_version(PG_FUNCTION_ARGS) PG_RETURN_NULL(); } +/* + * Check whether a text value is pure ASCII. + * + * Pure ASCII (code points 0-127) is unaffected by Unicode normalization + * and is always an assigned code point, independent of server encoding, + * so callers can use this to skip multibyte decoding entirely. This is + * byte-oriented (checking raw bytes for the high bit, rather than + * comparing byte length to codepoint count) so it stays correct if ever + * reused somewhere the server encoding isn't already known to be UTF8. + * + * is_valid_ascii() rejects embedded zero bytes as well as high-bit + * bytes, and requires its length argument to be a multiple of the SIMD + * chunk size (sizeof(Vector8)), so scan the largest chunk-aligned + * prefix with it, then apply the same zero-byte/high-bit check + * byte-at-a-time to the remainder for consistency. + */ +static bool +text_is_ascii(text *t) +{ + unsigned char *s = (unsigned char *) VARDATA_ANY(t); + int len = VARSIZE_ANY_EXHDR(t); + int chunk_len = len - (len % sizeof(Vector8)); + + if (chunk_len > 0 && !is_valid_ascii(s, chunk_len)) + return false; + + for (int i = chunk_len; i < len; i++) + if (s[i] == 0 || IS_HIGHBIT_SET(s[i])) + return false; + + return true; +} + /* * Check whether the string contains only assigned Unicode code * points. Requires that the database encoding is UTF-8. @@ -5495,6 +5530,10 @@ unicode_assigned(PG_FUNCTION_ARGS) ereport(ERROR, (errmsg("Unicode categorization can only be performed if server encoding is UTF8"))); + /* ASCII code points are always assigned */ + if (text_is_ascii(input)) + PG_RETURN_BOOL(true); + /* convert to char32_t */ size = pg_mbstrlen_with_len(VARDATA_ANY(input), VARSIZE_ANY_EXHDR(input)); p = (unsigned char *) VARDATA_ANY(input); @@ -5527,6 +5566,10 @@ unicode_normalize_func(PG_FUNCTION_ARGS) form = unicode_norm_form_from_string(formstr); + /* ASCII code points are unaffected by normalization */ + if (text_is_ascii(input)) + PG_RETURN_TEXT_P(input); + /* convert to char32_t */ size = pg_mbstrlen_with_len(VARDATA_ANY(input), VARSIZE_ANY_EXHDR(input)); input_chars = palloc_array(char32_t, size + 1); @@ -5595,6 +5638,10 @@ unicode_is_normalized(PG_FUNCTION_ARGS) form = unicode_norm_form_from_string(formstr); + /* ASCII code points are always normalized, in any of the four forms */ + if (text_is_ascii(input)) + PG_RETURN_BOOL(true); + /* convert to char32_t */ size = pg_mbstrlen_with_len(VARDATA_ANY(input), VARSIZE_ANY_EXHDR(input)); input_chars = palloc_array(char32_t, size + 1); diff --git a/src/test/regress/expected/unicode.out b/src/test/regress/expected/unicode.out index 63e48d3a961..e716aa7fca2 100644 --- a/src/test/regress/expected/unicode.out +++ b/src/test/regress/expected/unicode.out @@ -26,6 +26,12 @@ SELECT unicode_assigned(U&'abc\+10FFFF'); f (1 row) +SELECT unicode_assigned('abc'); + unicode_assigned +------------------ + t +(1 row) + SELECT normalize(''); normalize ----------- @@ -68,6 +74,19 @@ SELECT normalize(U&'\00E4\24D1c', NFKD) = U&'\0061\0308bc' COLLATE "C" AS test_n t (1 row) +SELECT normalize('abc') = 'abc' COLLATE "C" AS test_ascii_idem; + test_ascii_idem +----------------- + t +(1 row) + +SELECT normalize(val) = val COLLATE "C" AS test_ascii_idem_column +FROM (VALUES ('abc'::text)) v(val); + test_ascii_idem_column +------------------------ + t +(1 row) + SELECT "normalize"('abc', 'def'); -- run-time error ERROR: invalid normalization form: def SELECT U&'\00E4\24D1c' IS NORMALIZED AS test_default; @@ -82,6 +101,12 @@ SELECT U&'\00E4\24D1c' IS NFC NORMALIZED AS test_nfc; t (1 row) +SELECT 'abc' IS NORMALIZED AS test_ascii; + test_ascii +------------ + t +(1 row) + SELECT num, val, val IS NFC NORMALIZED AS NFC, val IS NFD NORMALIZED AS NFD, @@ -105,6 +130,59 @@ ORDER BY num; SELECT is_normalized('abc', 'def'); -- run-time error ERROR: invalid normalization form: def +-- Exercise the ASCII fast-path's chunk/remainder boundary handling: a +-- non-NFC-normalized codepoint sequence must still be detected as such +-- regardless of how much pure-ASCII padding surrounds it. +SELECT len, + (repeat('a', len) || U&'\0061\0308') IS NORMALIZED AS non_nfc_at_end, + (U&'\0061\0308' || repeat('a', len)) IS NORMALIZED AS non_nfc_at_start +FROM generate_series(0, 40) AS len +ORDER BY len; + len | non_nfc_at_end | non_nfc_at_start +-----+----------------+------------------ + 0 | f | f + 1 | f | f + 2 | f | f + 3 | f | f + 4 | f | f + 5 | f | f + 6 | f | f + 7 | f | f + 8 | f | f + 9 | f | f + 10 | f | f + 11 | f | f + 12 | f | f + 13 | f | f + 14 | f | f + 15 | f | f + 16 | f | f + 17 | f | f + 18 | f | f + 19 | f | f + 20 | f | f + 21 | f | f + 22 | f | f + 23 | f | f + 24 | f | f + 25 | f | f + 26 | f | f + 27 | f | f + 28 | f | f + 29 | f | f + 30 | f | f + 31 | f | f + 32 | f | f + 33 | f | f + 34 | f | f + 35 | f | f + 36 | f | f + 37 | f | f + 38 | f | f + 39 | f | f + 40 | f | f +(41 rows) + -- Hangul NFC recomposition tests -- L+V -> LV composition (first and last) SELECT normalize(U&'\1100\1161', NFC) = U&'\AC00' COLLATE "C" AS hangul_lv_first; diff --git a/src/test/regress/sql/unicode.sql b/src/test/regress/sql/unicode.sql index 951f86a336e..e3c030c3744 100644 --- a/src/test/regress/sql/unicode.sql +++ b/src/test/regress/sql/unicode.sql @@ -8,6 +8,7 @@ SELECT U&'\0061\0308bc' <> U&'\00E4bc' COLLATE "C" AS sanity_check; SELECT unicode_version() IS NOT NULL; SELECT unicode_assigned(U&'abc'); SELECT unicode_assigned(U&'abc\+10FFFF'); +SELECT unicode_assigned('abc'); SELECT normalize(''); SELECT normalize(U&'\0061\0308\24D1c') = U&'\00E4\24D1c' COLLATE "C" AS test_default; @@ -16,11 +17,15 @@ SELECT normalize(U&'\00E4bc', NFC) = U&'\00E4bc' COLLATE "C" AS test_nfc_idem; SELECT normalize(U&'\00E4\24D1c', NFD) = U&'\0061\0308\24D1c' COLLATE "C" AS test_nfd; SELECT normalize(U&'\0061\0308\24D1c', NFKC) = U&'\00E4bc' COLLATE "C" AS test_nfkc; SELECT normalize(U&'\00E4\24D1c', NFKD) = U&'\0061\0308bc' COLLATE "C" AS test_nfkd; +SELECT normalize('abc') = 'abc' COLLATE "C" AS test_ascii_idem; +SELECT normalize(val) = val COLLATE "C" AS test_ascii_idem_column +FROM (VALUES ('abc'::text)) v(val); SELECT "normalize"('abc', 'def'); -- run-time error SELECT U&'\00E4\24D1c' IS NORMALIZED AS test_default; SELECT U&'\00E4\24D1c' IS NFC NORMALIZED AS test_nfc; +SELECT 'abc' IS NORMALIZED AS test_ascii; SELECT num, val, val IS NFC NORMALIZED AS NFC, @@ -37,6 +42,15 @@ ORDER BY num; SELECT is_normalized('abc', 'def'); -- run-time error +-- Exercise the ASCII fast-path's chunk/remainder boundary handling: a +-- non-NFC-normalized codepoint sequence must still be detected as such +-- regardless of how much pure-ASCII padding surrounds it. +SELECT len, + (repeat('a', len) || U&'\0061\0308') IS NORMALIZED AS non_nfc_at_end, + (U&'\0061\0308' || repeat('a', len)) IS NORMALIZED AS non_nfc_at_start +FROM generate_series(0, 40) AS len +ORDER BY len; + -- Hangul NFC recomposition tests -- L+V -> LV composition (first and last) SELECT normalize(U&'\1100\1161', NFC) = U&'\AC00' COLLATE "C" AS hangul_lv_first; -- 2.47.3
From 4ad231668103c9e95562fce9253bfa7f94ae58d0 Mon Sep 17 00:00:00 2001 From: Nazir Bilal Yavuz <[email protected]> Date: Tue, 15 Sep 2026 15:48:47 +0300 Subject: [PATCH v2 2/2] Return index at text_ascii_check() --- src/backend/utils/adt/varlena.c | 75 +++++++++++++++++++++++---------- 1 file changed, 53 insertions(+), 22 deletions(-) diff --git a/src/backend/utils/adt/varlena.c b/src/backend/utils/adt/varlena.c index 3065b837537..f9eb0e6491f 100644 --- a/src/backend/utils/adt/varlena.c +++ b/src/backend/utils/adt/varlena.c @@ -168,7 +168,7 @@ static void text_format_string_conversion(StringInfo buf, char conversion, int flags, int width); static void text_format_append_string(StringInfo buf, const char *str, int flags, int width); -static bool text_is_ascii(text *t); +static int text_ascii_check(text *t); /***************************************************************************** @@ -5485,6 +5485,10 @@ icu_unicode_version(PG_FUNCTION_ARGS) /* * Check whether a text value is pure ASCII. * + * Return -1 if it is, otherwise the zero-based byte offset of the first + * failing SIMD chunk, or the failing byte in the scalar remainder. All + * bytes before the returned offset are known to be nonzero ASCII. + * * Pure ASCII (code points 0-127) is unaffected by Unicode normalization * and is always an assigned code point, independent of server encoding, * so callers can use this to skip multibyte decoding entirely. This is @@ -5494,25 +5498,26 @@ icu_unicode_version(PG_FUNCTION_ARGS) * * is_valid_ascii() rejects embedded zero bytes as well as high-bit * bytes, and requires its length argument to be a multiple of the SIMD - * chunk size (sizeof(Vector8)), so scan the largest chunk-aligned - * prefix with it, then apply the same zero-byte/high-bit check + * chunk size (sizeof(Vector8)), so check one chunk at a time to locate + * the first failure, then apply the same zero-byte/high-bit check * byte-at-a-time to the remainder for consistency. */ -static bool -text_is_ascii(text *t) +static int +text_ascii_check(text *t) { unsigned char *s = (unsigned char *) VARDATA_ANY(t); int len = VARSIZE_ANY_EXHDR(t); int chunk_len = len - (len % sizeof(Vector8)); - if (chunk_len > 0 && !is_valid_ascii(s, chunk_len)) - return false; + for (int i = 0; i < chunk_len; i += sizeof(Vector8)) + if (!is_valid_ascii(s + i, sizeof(Vector8))) + return i; for (int i = chunk_len; i < len; i++) if (s[i] == 0 || IS_HIGHBIT_SET(s[i])) - return false; + return i; - return true; + return -1; } /* @@ -5525,18 +5530,21 @@ unicode_assigned(PG_FUNCTION_ARGS) text *input = PG_GETARG_TEXT_PP(0); unsigned char *p; int size; + int start; if (GetDatabaseEncoding() != PG_UTF8) ereport(ERROR, (errmsg("Unicode categorization can only be performed if server encoding is UTF8"))); /* ASCII code points are always assigned */ - if (text_is_ascii(input)) + start = text_ascii_check(input); + if (start == -1) PG_RETURN_BOOL(true); - /* convert to char32_t */ - size = pg_mbstrlen_with_len(VARDATA_ANY(input), VARSIZE_ANY_EXHDR(input)); - p = (unsigned char *) VARDATA_ANY(input); + /* Convert only the suffix not already known to be ASCII to char32_t. */ + size = pg_mbstrlen_with_len(VARDATA_ANY(input) + start, + VARSIZE_ANY_EXHDR(input) - start); + p = (unsigned char *) VARDATA_ANY(input) + start; for (int i = 0; i < size; i++) { char32_t uchar = utf8_to_unicode(p); @@ -5563,17 +5571,28 @@ unicode_normalize_func(PG_FUNCTION_ARGS) unsigned char *p; text *result; size_t i; + int start; form = unicode_norm_form_from_string(formstr); /* ASCII code points are unaffected by normalization */ - if (text_is_ascii(input)) + start = text_ascii_check(input); + if (start == -1) PG_RETURN_TEXT_P(input); - /* convert to char32_t */ - size = pg_mbstrlen_with_len(VARDATA_ANY(input), VARSIZE_ANY_EXHDR(input)); + /* + * Keep the last ASCII character in the suffix, since a following + * combining mark could compose with it. ASCII characters have combining + * class zero, so normalization cannot affect any earlier characters. + */ + if (start > 0) + start--; + + /* convert the suffix to char32_t */ + size = pg_mbstrlen_with_len(VARDATA_ANY(input) + start, + VARSIZE_ANY_EXHDR(input) - start); input_chars = palloc_array(char32_t, size + 1); - p = (unsigned char *) VARDATA_ANY(input); + p = (unsigned char *) VARDATA_ANY(input) + start; for (i = 0; i < size; i++) { input_chars[i] = utf8_to_unicode(p); @@ -5586,7 +5605,7 @@ unicode_normalize_func(PG_FUNCTION_ARGS) output_chars = unicode_normalize(form, input_chars); /* convert back to UTF-8 string */ - size = 0; + size = start; for (char32_t *wp = output_chars; *wp; wp++) { unsigned char buf[4]; @@ -5599,6 +5618,8 @@ unicode_normalize_func(PG_FUNCTION_ARGS) SET_VARSIZE(result, size + VARHDRSZ); p = (unsigned char *) VARDATA_ANY(result); + memcpy(p, VARDATA_ANY(input), start); + p += start; for (char32_t *wp = output_chars; *wp; wp++) { unicode_to_utf8(*wp, p); @@ -5635,17 +5656,27 @@ unicode_is_normalized(PG_FUNCTION_ARGS) UnicodeNormalizationQC quickcheck; size_t output_size; bool result; + int start; form = unicode_norm_form_from_string(formstr); /* ASCII code points are always normalized, in any of the four forms */ - if (text_is_ascii(input)) + start = text_ascii_check(input); + if (start == -1) PG_RETURN_BOOL(true); - /* convert to char32_t */ - size = pg_mbstrlen_with_len(VARDATA_ANY(input), VARSIZE_ANY_EXHDR(input)); + /* + * Include the preceding ASCII character for possible composition with a + * following combining mark, as in unicode_normalize_func(). + */ + if (start > 0) + start--; + + /* convert the suffix to char32_t */ + size = pg_mbstrlen_with_len(VARDATA_ANY(input) + start, + VARSIZE_ANY_EXHDR(input) - start); input_chars = palloc_array(char32_t, size + 1); - p = (unsigned char *) VARDATA_ANY(input); + p = (unsigned char *) VARDATA_ANY(input) + start; for (i = 0; i < size; i++) { input_chars[i] = utf8_to_unicode(p); -- 2.47.3
