On Fri, 2026-06-26 at 12:38 +0800, Chao Li wrote:
> Sounds like 0001 will be back patched. In that case, the commit
> message "defend against invalid UTF8” seems too broad. Does it make
> sense to add some brief description about the defend behavior to the
> function header comment and the commit message?
Committed 0001 and backported to 17. I think adding too much
explanation about behavior we don't expect to actually see would just
add confusion.
> Then I continue to review 0002-0005:
>
> 0002 - overall looks good. A small comment is:
> ```
> + for (int i = offset; i > 0;)
>
> + for (int i = offset + ulen; i < len;)
> ```
Committed 0002 and backported to 18. The 'int' is pre-existing, so I
left it as-is.
> 0003 - looks good.
Committed to master only.
> 0004 - looks good. This commit introduces a new helper utf8decode()
> that will resolve my previous concern on 0001.
>
> 0005 - Mostly looks good. This commit applies the new help and my
> previous concern is resolved. But from what you talked, I guess 0004
> and 0005 will only be pushed to HEAD.
>
> Just one tiny comment on 0005:
> ```
> + /* invalid UTF8: surrogates */
> + needed = unicode_strfold(NULL, 0, "abc\xED\xA0\x81xyz", 7,
> &consumed, false);
> + Assert(needed == 3 && consumed == 3);
> ```
>
> This test passes a 10-char string but uses 7 as srclen. I know that
> doesn’t affect the test result, but it just adds unnecessary
> confusion to readers. So maybe change 7 to 10 to reflect to the real
> string length.
Thank you, attached with fix.
I'd like to wait for more comments before I commit these last two
patches, to see if the functions are generally useful for other callers
as well.
Regards,
Jeff Davis
From 5ab174963e2ab04455e59ea5b077238687151449 Mon Sep 17 00:00:00 2001
From: Jeff Davis <[email protected]>
Date: Thu, 4 Jun 2026 12:08:51 -0700
Subject: [PATCH v6 1/2] Validating, iterator-friendly UTF8 encoder/decoder
API.
Discussion: https://postgr.es/m/[email protected]
---
src/include/mb/pg_wchar.h | 139 +++++++++++++++++++++++++++++++++++++-
1 file changed, 137 insertions(+), 2 deletions(-)
diff --git a/src/include/mb/pg_wchar.h b/src/include/mb/pg_wchar.h
index deee2a832c3..7afbf25fc18 100644
--- a/src/include/mb/pg_wchar.h
+++ b/src/include/mb/pg_wchar.h
@@ -27,6 +27,11 @@
*/
typedef unsigned int pg_wchar;
+/*
+ * Returned for decoding failures in utf8decode() and utf8_to_unicode().
+ */
+#define PG_INVALID_CODEPOINT 0xFFFFFFFF
+
/*
* Maximum byte length of multibyte characters in any backend encoding
*/
@@ -392,11 +397,140 @@ surrogate_pair_to_codepoint(char16_t first, char16_t second)
return ((first & 0x3FF) << 10) + 0x10000 + (second & 0x3FF);
}
+/*
+ * Encode the codepoint as UTF8 and return the number of bytes required. If
+ * the number of bytes required exceeds dstsize, just return the number of
+ * bytes required without modifying dst. If dstsize is zero, dst may be
+ * NULL. If codepoint is not a valid Unicode Scalar, return -1.
+ */
+static inline int
+utf8encode(unsigned char *dst, size_t dstsize, char32_t codepoint)
+{
+ int nbytes;
+
+ if (codepoint <= 0x7F)
+ nbytes = 1;
+ else if (codepoint <= 0x7FF)
+ nbytes = 2;
+ else if (codepoint <= 0xFFFF)
+ {
+ /* surrogate halves not valid for UTF8 */
+ if (codepoint >= 0xD800 && codepoint <= 0xDFFF)
+ return -1;
+ nbytes = 3;
+ }
+ else if (codepoint <= 0x10FFFF)
+ nbytes = 4;
+ else
+ return -1;
+
+ if ((size_t) nbytes > dstsize)
+ return nbytes;
+
+ if (codepoint <= 0x7F)
+ {
+ dst[0] = codepoint;
+ }
+ else if (codepoint <= 0x7FF)
+ {
+ dst[0] = 0xC0 | ((codepoint >> 6) & 0x1F);
+ dst[1] = 0x80 | (codepoint & 0x3F);
+ }
+ else if (codepoint <= 0xFFFF)
+ {
+ dst[0] = 0xE0 | ((codepoint >> 12) & 0x0F);
+ dst[1] = 0x80 | ((codepoint >> 6) & 0x3F);
+ dst[2] = 0x80 | (codepoint & 0x3F);
+ }
+ else if (codepoint <= 0x10FFFF)
+ {
+ dst[0] = 0xF0 | ((codepoint >> 18) & 0x07);
+ dst[1] = 0x80 | ((codepoint >> 12) & 0x3F);
+ dst[2] = 0x80 | ((codepoint >> 6) & 0x3F);
+ dst[3] = 0x80 | (codepoint & 0x3F);
+ }
+
+ return nbytes;
+}
+
+/*
+ * Decode the next Unicode codepoint from UTF8 at src, reading no more than
+ * srclen bytes (which must be at least 1). On success, *pcodepoint will be a
+ * valid Unicode Scalar; otherwise it will be set to PG_INVALID_CODEPOINT.
+ *
+ * Returns the number of bytes consumed. If srclen is not large enough
+ * (i.e. src is truncated in the middle of a sequence), returns 0. If invalid,
+ * returns -1.
+ */
+static inline int
+utf8decode(char32_t *pcodepoint, const unsigned char *src, size_t srclen)
+{
+ int nbytes;
+ char32_t codepoint;
+ char32_t min;
+
+ Assert(srclen >= 1);
+
+ if ((*src & 0x80) == 0)
+ {
+ *pcodepoint = (char32_t) src[0];
+ return 1;
+ }
+ else if ((*src & 0xe0) == 0xc0)
+ {
+ nbytes = 2;
+ min = 0x80;
+ codepoint = (char32_t) src[0] & 0x1f;
+ }
+ else if ((*src & 0xf0) == 0xe0)
+ {
+ nbytes = 3;
+ min = 0x800;
+ codepoint = (char32_t) src[0] & 0x0f;
+ }
+ else if ((*src & 0xf8) == 0xf0)
+ {
+ nbytes = 4;
+ min = 0x10000;
+ codepoint = (char32_t) src[0] & 0x07;
+ }
+ else
+ goto invalid; /* invalid leading byte */
+
+ /* truncated multibyte sequence */
+ if (srclen < (size_t) nbytes)
+ {
+ *pcodepoint = PG_INVALID_CODEPOINT;
+ return 0;
+ }
+
+ for (int i = 1; i < nbytes; i++)
+ {
+ if ((src[i] & 0xc0) != 0x80)
+ goto invalid;
+ codepoint = (codepoint << 6) | (src[i] & 0x3f);
+ }
+
+ /* reject overlong, surrogate, and out-of-range */
+ if (codepoint < min || codepoint > 0x10FFFF ||
+ (codepoint >= 0xD800 && codepoint <= 0xDFFF))
+ goto invalid;
+
+ *pcodepoint = codepoint;
+ return nbytes;
+
+invalid:
+ *pcodepoint = PG_INVALID_CODEPOINT;
+ return -1;
+}
+
/*
* Convert a UTF-8 character to a Unicode code point.
* This is a one-character version of pg_utf2wchar_with_len.
*
* No error checks here, c must point to a long-enough string.
+ *
+ * XXX: Callers should consider utf8decode() instead.
*/
static inline char32_t
utf8_to_unicode(const unsigned char *c)
@@ -416,13 +550,14 @@ utf8_to_unicode(const unsigned char *c)
((c[2] & 0x3f) << 6) |
(c[3] & 0x3f));
else
- /* that is an invalid code on purpose */
- return 0xffffffff;
+ return PG_INVALID_CODEPOINT;
}
/*
* Map a Unicode code point to UTF-8. utf8string must have at least
* unicode_utf8len(c) bytes available.
+ *
+ * XXX: Callers should consider utf8encode() instead.
*/
static inline unsigned char *
unicode_to_utf8(char32_t c, unsigned char *utf8string)
--
2.43.0
From 2d9c03f786db067bfa0d89e8ab7f3b1ee5e1b0c0 Mon Sep 17 00:00:00 2001
From: Jeff Davis <[email protected]>
Date: Fri, 19 Jun 2026 14:58:47 -0700
Subject: [PATCH v6 2/2] unicode_case.c: use new utf8encode/utf8decode APIs.
Reviewed-by: Chao Li <[email protected]>
Discussion: https://postgr.es/m/[email protected]
---
src/backend/utils/adt/pg_locale_builtin.c | 10 +--
src/common/unicode/case_test.c | 45 +++++++++----
src/common/unicode_case.c | 77 +++++++++++------------
3 files changed, 77 insertions(+), 55 deletions(-)
diff --git a/src/backend/utils/adt/pg_locale_builtin.c b/src/backend/utils/adt/pg_locale_builtin.c
index 5619daf43c3..63516e7174f 100644
--- a/src/backend/utils/adt/pg_locale_builtin.c
+++ b/src/backend/utils/adt/pg_locale_builtin.c
@@ -62,22 +62,22 @@ initcap_wbnext(void *state)
while (wbstate->offset < wbstate->len)
{
- int ulen = pg_utf_mblen((const unsigned char *) wbstate->str +
- wbstate->offset);
+ int ulen;
char32_t u;
bool curr_alnum;
size_t prev_offset = wbstate->offset;
+ ulen = utf8decode(&u, (const unsigned char *) wbstate->str + wbstate->offset,
+ wbstate->len - wbstate->offset);
+
/* invalid UTF8 */
- if (wbstate->offset + ulen > wbstate->len)
+ if (ulen <= 0)
{
wbstate->init = true;
wbstate->offset = wbstate->len;
return prev_offset;
}
- u = utf8_to_unicode((const unsigned char *) wbstate->str +
- wbstate->offset);
curr_alnum = pg_u_isalnum(u, wbstate->posix);
if (!wbstate->init || curr_alnum != wbstate->prev_alnum)
diff --git a/src/common/unicode/case_test.c b/src/common/unicode/case_test.c
index 08421d9e5ca..22f3f9064b6 100644
--- a/src/common/unicode/case_test.c
+++ b/src/common/unicode/case_test.c
@@ -52,24 +52,35 @@ initcap_wbnext(void *state)
{
struct WordBoundaryState *wbstate = (struct WordBoundaryState *) state;
- while (wbstate->offset < wbstate->len &&
- wbstate->str[wbstate->offset] != '\0')
+ while (wbstate->offset < wbstate->len)
{
- char32_t u = utf8_to_unicode((const unsigned char *) wbstate->str +
- wbstate->offset);
- bool curr_alnum = pg_u_isalnum(u, wbstate->posix);
+ int ulen;
+ char32_t u;
+ bool curr_alnum;
+ size_t prev_offset = wbstate->offset;
- if (!wbstate->init || curr_alnum != wbstate->prev_alnum)
+ ulen = utf8decode(&u, (const unsigned char *) wbstate->str + wbstate->offset,
+ wbstate->len - wbstate->offset);
+
+ /* invalid UTF8 */
+ if (ulen <= 0)
{
- size_t prev_offset = wbstate->offset;
+ wbstate->init = true;
+ wbstate->offset = wbstate->len;
+ return prev_offset;
+ }
+
+ curr_alnum = pg_u_isalnum(u, wbstate->posix);
+ if (!wbstate->init || curr_alnum != wbstate->prev_alnum)
+ {
wbstate->init = true;
- wbstate->offset += unicode_utf8len(u);
+ wbstate->offset += ulen;
wbstate->prev_alnum = curr_alnum;
return prev_offset;
}
- wbstate->offset += unicode_utf8len(u);
+ wbstate->offset += ulen;
}
return wbstate->len;
@@ -179,7 +190,7 @@ test_icu(void)
{
pg_unicode_category category = unicode_category(code);
- if (category != PG_U_UNASSIGNED)
+ if (category != PG_U_UNASSIGNED && category != PG_U_SURROGATE)
{
uint8_t icu_category = u_charType(code);
char code_str[5] = {0};
@@ -191,7 +202,7 @@ test_icu(void)
}
icu_test_simple(code);
- unicode_to_utf8(code, (unsigned char *) code_str);
+ utf8encode((unsigned char *) code_str, 5, code);
icu_test_full(code_str);
successful++;
@@ -337,6 +348,18 @@ test_convert_case(void)
/* invalid UTF8: leading byte invalid length */
needed = unicode_strfold(NULL, 0, "abc\xF8xyz", 7, &consumed, false);
Assert(needed == 3 && consumed == 3);
+ /* invalid UTF8: surrogates */
+ needed = unicode_strfold(NULL, 0, "abc\xED\xA0\x81xyz", 9, &consumed, false);
+ Assert(needed == 3 && consumed == 3);
+ /* invalid UTF8: continuation with no leading byte */
+ needed = unicode_strfold(NULL, 0, "abc\x80xyz", 7, &consumed, false);
+ Assert(needed == 3 && consumed == 3);
+ /* invalid UTF8: out of range */
+ needed = unicode_strfold(NULL, 0, "abc\xF5\x80\x80\x80xyz", 10, &consumed, false);
+ Assert(needed == 3 && consumed == 3);
+ /* invalid UTF8: overlong */
+ needed = unicode_strfold(NULL, 0, "abc\xC1\xBFxyz", 8, &consumed, false);
+ Assert(needed == 3 && consumed == 3);
#ifdef USE_ICU
icu_test_full("");
diff --git a/src/common/unicode_case.c b/src/common/unicode_case.c
index 24753aaab09..4d8ee71e8dc 100644
--- a/src/common/unicode_case.c
+++ b/src/common/unicode_case.c
@@ -194,22 +194,6 @@ unicode_strfold(char *dst, size_t dstsize, const char *src, size_t srclen,
NULL, NULL);
}
-/* local version of pg_utf_mblen() to be inlinable */
-static int
-utf8_mblen(const unsigned char *s)
-{
- if ((*s & 0x80) == 0)
- return 1;
- else if ((*s & 0xe0) == 0xc0)
- return 2;
- else if ((*s & 0xf0) == 0xe0)
- return 3;
- else if ((*s & 0xf8) == 0xf0)
- return 4;
- else
- return -1;
-}
-
/*
* Implement Unicode Default Case Conversion algorithm.
*
@@ -248,18 +232,19 @@ convert_case(char *dst, size_t dstsize, const char *src, size_t srclen,
while (srcoff < srclen)
{
- int u1len = utf8_mblen((const unsigned char *) src + srcoff);
char32_t u1;
+ int u1len;
char32_t simple = 0;
const char32_t *special = NULL;
enum CaseMapResult casemap_result;
+ u1len = utf8decode(&u1, (const unsigned char *) src + srcoff,
+ srclen - srcoff);
+
/* invalid UTF8 */
- if (u1len < 0 || srcoff + u1len > srclen)
+ if (u1len <= 0)
break;
- u1 = utf8_to_unicode((const unsigned char *) src + srcoff);
-
if (str_casekind == CaseTitle)
{
if (srcoff == boundary)
@@ -280,6 +265,7 @@ convert_case(char *dst, size_t dstsize, const char *src, size_t srclen,
/* no mapping; copy bytes from src */
Assert(simple == 0);
Assert(special == NULL);
+
if (result_len + u1len <= dstsize)
memcpy(dst + result_len, src + srcoff, u1len);
@@ -289,11 +275,19 @@ convert_case(char *dst, size_t dstsize, const char *src, size_t srclen,
{
/* replace with single character */
char32_t u2 = simple;
- char32_t u2len = unicode_utf8len(u2);
+ int u2len;
+ size_t remaining = 0;
+ unsigned char *p = NULL;
+
+ if (dstsize > result_len)
+ {
+ remaining = dstsize - result_len;
+ p = (unsigned char *) dst + result_len;
+ }
Assert(special == NULL);
- if (result_len + u2len <= dstsize)
- unicode_to_utf8(u2, (unsigned char *) dst + result_len);
+ u2len = utf8encode(p, remaining, u2);
+ Assert(u2len > 0);
result_len += u2len;
}
@@ -304,10 +298,18 @@ convert_case(char *dst, size_t dstsize, const char *src, size_t srclen,
for (int i = 0; i < MAX_CASE_EXPANSION && special[i]; i++)
{
char32_t u2 = special[i];
- size_t u2len = unicode_utf8len(u2);
+ int u2len;
+ size_t remaining = 0;
+ unsigned char *p = NULL;
+
+ if (dstsize > result_len)
+ {
+ remaining = dstsize - result_len;
+ p = (unsigned char *) dst + result_len;
+ }
- if (result_len + u2len <= dstsize)
- unicode_to_utf8(u2, (unsigned char *) dst + result_len);
+ u2len = utf8encode(p, remaining, u2);
+ Assert(u2len > 0);
result_len += u2len;
}
@@ -352,13 +354,10 @@ check_final_sigma(const unsigned char *str, size_t len, size_t offset)
/* now at leading byte of previous sequence */
Assert((str[i] & 0x80) == 0 || (str[i] & 0xC0) == 0xC0);
- ulen = utf8_mblen((const unsigned char *) str + i);
-
- /* invalid UTF8 */
- if (ulen < 0 || i + ulen > len)
- return false;
+ ulen = utf8decode(&curr, (const unsigned char *) str + i, len - i);
- curr = utf8_to_unicode((const unsigned char *) str + i);
+ if (ulen <= 0)
+ return false; /* invalid UTF8 */
if (!pg_u_prop_case_ignorable(curr))
{
@@ -367,18 +366,18 @@ check_final_sigma(const unsigned char *str, size_t len, size_t offset)
}
}
- ulen = utf8_mblen((const unsigned char *) str + offset);
+ ulen = utf8decode(&curr, (const unsigned char *) str + offset,
+ len - offset);
+ if (ulen <= 0)
+ return false; /* invalid UTF8 */
/* iterate forward looking for following character */
for (int i = offset + ulen; i < len;)
{
- ulen = utf8_mblen((const unsigned char *) str + i);
-
- /* invalid UTF8 */
- if (ulen < 0 || i + ulen > len)
- return false;
+ ulen = utf8decode(&curr, (const unsigned char *) str + i, len - i);
- curr = utf8_to_unicode((const unsigned char *) str + i);
+ if (ulen <= 0)
+ return false; /* invalid UTF8 */
if (!pg_u_prop_case_ignorable(curr))
{
--
2.43.0
