The documentation for the functions in the fields of pg_wchar_tbl was
spread out all over the place and was missing a lot of detail. I
propose the attached patch that moves the important material into the
definition of the struct and also expands the details a bit, so that all
the important info is in one easy to find place. Some of those details
I gathered and inferred from various places, and some of those used to
be more like "it works this way" where I'm now somewhat elevating it to
"it is supposed to work this way", so maybe that's worth double-checking.
I'm also removing some unnecessary typedefs for those functions, except
where they were used elsewhere to hold local variables. This
indirection made the code documentation harder to follow. And some of
those typedefs had nonsensical names, like *_converter for functions
that didn't do any converting.
(I was looking for example at struct TableAmRoutine in access/tableam.h
for inspiration for a well-commented callback table definition.)
From 3ebaaada53f1a04dc7e807b31ba54f420c967b42 Mon Sep 17 00:00:00 2001
From: Peter Eisentraut <[email protected]>
Date: Tue, 27 Oct 2026 14:12:49 +0100
Subject: [PATCH] Clean up code comments for pg_wchar_tbl
The documentation for the functions in the fields of pg_wchar_tbl was
spread out all over the place and was missing a lot of detail. Move
and expand this at the definition of the struct. (See also struct
TableAmRoutine in access/tableam.h for inspiration.)
Also remove unnecessary typedefs, except where they were used
elsewhere as well. This indirection makes the code documentation
harder to follow. And some of those typedefs had nonsensical names,
like *_converter for functions that didn't do any converting.
---
src/backend/utils/mb/mbutils.c | 10 +--
src/common/wchar.c | 60 +--------------
src/include/mb/pg_wchar.h | 125 +++++++++++++++++++++++++------
src/tools/pgindent/typedefs.list | 8 +-
4 files changed, 113 insertions(+), 90 deletions(-)
diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c
index b57dba4ead1..bdfdd9eeace 100644
--- a/src/backend/utils/mb/mbutils.c
+++ b/src/backend/utils/mb/mbutils.c
@@ -1222,7 +1222,7 @@ int
pg_encoding_mbcliplen(int encoding, const char *mbstr,
int len, int limit)
{
- mblen_converter mblen_fn;
+ mblen_function mblen_fn;
int clen = 0;
int l;
@@ -1452,15 +1452,15 @@ static bool
pg_generic_charinc(unsigned char *charptr, int len)
{
unsigned char *lastbyte = charptr + len - 1;
- mbchar_verifier mbverify;
+ mbverifychar_function mbverifychar;
/* We can just invoke the character verifier directly. */
- mbverify = pg_wchar_table[GetDatabaseEncoding()].mbverifychar;
+ mbverifychar = pg_wchar_table[GetDatabaseEncoding()].mbverifychar;
while (*lastbyte < (unsigned char) 255)
{
(*lastbyte)++;
- if ((*mbverify) (charptr, len) == len)
+ if ((*mbverifychar) (charptr, len) == len)
return true;
}
@@ -1723,7 +1723,7 @@ pg_verify_mbstr(int encoding, const char *mbstr, int len,
bool noError)
int
pg_verify_mbstr_len(int encoding, const char *mbstr, int len, bool noError)
{
- mbchar_verifier mbverifychar;
+ mbverifychar_function mbverifychar;
int mb_len;
Assert(PG_VALID_ENCODING(encoding));
diff --git a/src/common/wchar.c b/src/common/wchar.c
index 926823cabec..23bbdbd2ae5 100644
--- a/src/common/wchar.c
+++ b/src/common/wchar.c
@@ -37,32 +37,6 @@
#define NONUTF8_INVALID_BYTE1 (' ')
-/*
- * Operations on multi-byte encodings are driven by a table of helper
- * functions.
- *
- * To add an encoding support, define mblen(), dsplen(), verifychar() and
- * verifystr() for the encoding. For server-encodings, also define mb2wchar()
- * and wchar2mb() conversion functions.
- *
- * These functions generally assume that their input is validly formed.
- * The "verifier" functions, further down in the file, have to be more
- * paranoid.
- *
- * We expect that mblen() does not need to examine more than the first byte
- * of the character to discover the correct length. GB18030 is an exception
- * to that rule, though, as it also looks at second byte. But even that
- * behaves in a predictable way, if you only pass the first byte: it will
- * treat 4-byte encoded characters as two 2-byte encoded characters, which is
- * good enough for all current uses.
- *
- * Note: for the display output of psql to work properly, the return values
- * of the dsplen functions must conform to the Unicode standard. In particular
- * the NUL character is zero width and control characters are generally
- * width -1. It is recommended that non-ASCII encodings refer their ASCII
- * subset to the ASCII routines to ensure consistency.
- */
-
/* No error-reporting facility. Ignore incomplete trailing byte sequence. */
#define MB2CHAR_NEED_AT_LEAST(len, need) if ((len) < (need)) break
@@ -390,9 +364,6 @@ pg_euctw_dsplen(const unsigned char *s)
/*
* Convert pg_wchar to EUC_* encoding.
- * caller must allocate enough space for "to", including a trailing zero!
- * len: length of from.
- * "from" not necessarily null terminated.
*/
static int
pg_wchar2euc_with_len(const pg_wchar *from, unsigned char *to, int len)
@@ -454,9 +425,6 @@ pg_johab_dsplen(const unsigned char *s)
/*
* convert UTF8 string to pg_wchar (UCS-4)
- * caller must allocate enough space for "to", including a trailing zero!
- * len: length of from.
- * "from" not necessarily null terminated.
*/
static int
pg_utf2wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
@@ -517,9 +485,6 @@ pg_utf2wchar_with_len(const unsigned char *from, pg_wchar
*to, int len)
/*
* Trivial conversion from pg_wchar to UTF-8.
- * caller should allocate enough space for "to"
- * len: length of from.
- * "from" not necessarily null terminated.
*/
static int
pg_wchar2utf_with_len(const pg_wchar *from, unsigned char *to, int len)
@@ -704,9 +669,6 @@ pg_latin12wchar_with_len(const unsigned char *from,
pg_wchar *to, int len)
/*
* Trivial conversion from pg_wchar to single byte encoding. Just ignores
* high bits.
- * caller should allocate enough space for "to"
- * len: length of from.
- * "from" not necessarily null terminated.
*/
static int
pg_wchar2single_with_len(const pg_wchar *from, unsigned char *to, int len)
@@ -891,25 +853,9 @@ pg_gb18030_dsplen(const unsigned char *s)
/*
*-------------------------------------------------------------------
* multibyte sequence validators
- *
- * The verifychar functions accept "s", a pointer to the first byte of a
- * string, and "len", the remaining length of the string. If there is a
- * validly encoded character beginning at *s, return its length in bytes;
- * else return -1.
- *
- * The verifystr functions also accept "s", a pointer to a string and "len",
- * the length of the string. They verify the whole string, and return the
- * number of input bytes (<= len) that are valid. In other words, if the
- * whole string is valid, verifystr returns "len", otherwise it returns the
- * byte offset of the first invalid character. The verifystr functions must
- * test for and reject zeroes in the input.
- *
- * The verifychar functions can assume that len > 0 and that *s != '\0', but
- * they must test for and reject zeroes in any additional bytes of a
- * multibyte character. Note that this definition allows the function for a
- * single-byte encoding to be just "return 1".
*-------------------------------------------------------------------
*/
+
static int
pg_ascii_verifychar(const unsigned char *s, int len)
{
@@ -1982,7 +1928,7 @@ pg_encoding_dsplen(int encoding, const char *mbstr)
/*
* Verify the first multibyte character of the given string.
- * Return its byte length if good, -1 if bad. (See comments above for
+ * Return its byte length if good, -1 if bad. (See comments at pg_wchar_tbl
for
* full details of the mbverifychar API.)
*/
int
@@ -1996,7 +1942,7 @@ pg_encoding_verifymbchar(int encoding, const char *mbstr,
int len)
/*
* Verify that a string is valid for the given encoding.
* Returns the number of input bytes (<= len) that form a valid string.
- * (See comments above for full details of the mbverifystr API.)
+ * (See comments at pg_wchar_tbl for full details of the mbverifystr API.)
*/
int
pg_encoding_verifymbstr(int encoding, const char *mbstr, int len)
diff --git a/src/include/mb/pg_wchar.h b/src/include/mb/pg_wchar.h
index 9f0e4223748..72d960bde6d 100644
--- a/src/include/mb/pg_wchar.h
+++ b/src/include/mb/pg_wchar.h
@@ -208,35 +208,115 @@ extern PGDLLIMPORT const char *pg_enc2gettext_tbl[];
/*
* pg_wchar stuff
*/
-typedef int (*mb2wchar_with_len_converter) (const unsigned char *from,
-
pg_wchar *to,
-
int len);
-typedef int (*wchar2mb_with_len_converter) (const pg_wchar *from,
-
unsigned char *to,
-
int len);
+typedef int (*mblen_function) (const unsigned char *mbstr);
+typedef int (*mbverifychar_function) (const unsigned char *mbstr, int len);
-typedef int (*mblen_converter) (const unsigned char *mbstr);
+/*
+ * Operations on multi-byte encodings are driven by a table of helper
+ * functions.
+ *
+ * All functions are required, except the conversion functions
+ * .mb2wchar_with_len and .wchar2mb_with_len, which are only required for
+ * server encodings.
+ *
+ * These functions generally assume that their input is validly formed, and
+ * unless specified otherwise have no special return value for errors. The
+ * exceptions are the two "verify" functions, which must check their input and
+ * signal invalid input through their return value. None of these functions
+ * report errors via ereport().
+ */
+typedef struct
+{
+ /*
+ * Convert a multibyte string to a pg_wchar string.
+ *
+ * "len" is the length of "from" in bytes (not characters). The input
+ * string need not be zero-terminated, but conversion will stop early
on a
+ * zero byte. The output is written to "to". "to" must be allocated to
+ * accommodate at least len + 1 pg_wchar's. A terminating 0 is always
+ * written. The return value is the number of pg_wchar's written,
without
+ * the terminator.
+ *
+ * If a multibyte sequence at the end is cut off by len, it is silently
+ * dropped.
+ */
+ int (*mb2wchar_with_len) (const unsigned char
*from, pg_wchar *to, int len);
-typedef int (*mbdisplaylen_converter) (const unsigned char *mbstr);
+ /*
+ * Convert a pg_wchar string to a multibyte string.
+ *
+ * "len" is the length of "from" in pg_wchar characters (not bytes).
The
+ * input need not be terminated by a 0 pg_wchar, but conversion will
stop
+ * early on a 0 pg_wchar. The output is written to "to". "to" must be
+ * allocated as len * maxmblen + 1 bytes. A terminating 0 byte is
always
+ * written. The return value is the number of bytes written, without
the
+ * terminator.
+ */
+ int (*wchar2mb_with_len) (const pg_wchar *from,
unsigned char *to, int len);
-typedef bool (*mbcharacter_incrementer) (unsigned char *mbstr, int len);
+ /*
+ * Get byte length of the first character of the input string.
+ *
+ * If the input string has length zero, then 1 is returned. The return
+ * value is always between 1 and maxmblen inclusive.
+ *
+ * We expect that mblen does not need to examine more than the first
byte
+ * of the character to discover the correct length. GB18030 is an
+ * exception; see pg_gb18030_mblen().
+ */
+ mblen_function mblen;
-typedef int (*mbchar_verifier) (const unsigned char *mbstr, int len);
+ /*
+ * Get the display width of the first character of the input string.
+ *
+ * The possible results are -1, 0, 1 and 2. Control characters return
-1;
+ * callers can handle that specially. Unlike mblen, an empty string
+ * returns 0. (But the converse is not true: Zero-width characters also
+ * return 0.)
+ *
+ * Note: for the display output of psql to work properly, the return
+ * values of the dsplen functions must conform to the Unicode standard.
In
+ * particular the NUL character is zero width and control characters are
+ * generally width -1. It is recommended that non-ASCII encodings refer
+ * their ASCII subset to the ASCII routines to ensure consistency.
+ */
+ int (*dsplen) (const unsigned char *mbstr);
-typedef int (*mbstr_verifier) (const unsigned char *mbstr, int len);
+ /*
+ * Verify multibyte character.
+ *
+ * The verifychar functions accept "mbstr", a pointer to the first byte
of
+ * a string, and "len", the remaining length of the string. If there
is a
+ * validly encoded character beginning at *mbstr, return its length in
+ * bytes; else return -1.
+ *
+ * The verifychar functions can assume that len > 0 and that *mbstr !=
+ * '\0', but they must test for and reject zeroes in any additional
bytes
+ * of a multibyte character. Note that this definition allows the
+ * function for a single-byte encoding to be just "return 1".
+ */
+ mbverifychar_function mbverifychar;
-typedef struct
-{
- mb2wchar_with_len_converter mb2wchar_with_len; /* convert a multibyte
-
* string to a wchar */
- wchar2mb_with_len_converter wchar2mb_with_len; /* convert a wchar
string
-
* to a multibyte */
- mblen_converter mblen; /* get byte length of a char */
- mbdisplaylen_converter dsplen; /* get display width of a char */
- mbchar_verifier mbverifychar; /* verify multibyte character */
- mbstr_verifier mbverifystr; /* verify multibyte string */
- int maxmblen; /* max bytes for a char
in this encoding */
+ /*
+ * Verify multibyte string.
+ *
+ * The verifystr functions accept "mbstr", a pointer to a string and
+ * "len", the length of the string. They verify the whole string, and
+ * return the number of input bytes (<= len) that are valid. In other
+ * words, if the whole string is valid, verifystr returns "len",
otherwise
+ * it returns the byte offset of the first invalid character. The
+ * verifystr functions must test for and reject zeroes in the input.
+ */
+ int (*mbverifystr) (const unsigned char *mbstr, int
len);
+
+ /*
+ * The maximum number of bytes a character in this encoding could
occupy.
+ *
+ * This is used to size output buffers of conversion routines, so it
must
+ * not be too small.
+ */
+ int maxmblen;
} pg_wchar_tbl;
extern PGDLLIMPORT const pg_wchar_tbl pg_wchar_table[];
@@ -547,6 +627,7 @@ extern int pg_encoding_mbcliplen(int encoding, const char
*mbstr,
int len, int
limit);
extern int pg_mbcharcliplen(const char *mbstr, int len, int limit);
extern int pg_database_encoding_max_length(void);
+typedef bool (*mbcharacter_incrementer) (unsigned char *mbstr, int len);
extern mbcharacter_incrementer
pg_database_encoding_character_incrementer(void);
extern int PrepareClientEncoding(int encoding);
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index cfef572df80..7393e27a534 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -3952,12 +3952,9 @@ manifest_wal_range
manifest_writer
map_variable_attnos_context
max_parallel_hazard_context
-mb2wchar_with_len_converter
-mbchar_verifier
mbcharacter_incrementer
-mbdisplaylen_converter
-mblen_converter
-mbstr_verifier
+mblen_function
+mbverifychar_function
memoize_hash
memoize_iterator
metastring
@@ -4440,7 +4437,6 @@ walrcv_receive_fn
walrcv_send_fn
walrcv_server_version_fn
walrcv_startstreaming_fn
-wchar2mb_with_len_converter
wchar_t
win32_deadchild_waitinfo
wint_t
--
2.56.0
