This is an automated email from the ASF dual-hosted git repository. reshke pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/cloudberry.git
commit ab889cec1503726fb30afd402a13ceb4d855a14d Author: Tom Lane <[email protected]> AuthorDate: Mon Aug 10 06:38:23 2026 -0700 Harden tsvector code against overflows. The core of this patch is to prevent array_to_tsvector() from generating invalid tsvectors. It did not check for overly-long lexemes (so that WordEntry.len fields could overflow), nor did it check that the total "datalen" fits within MAXSTRPOS (so that WordEntry.pos fields could overflow, and the number of entries in the tsvector could be much more than the normal limit). While the field overflows couldn't do anything much worse than produce a corrupted tsvector value, a sufficiently large number of tsvector entries could cause integer overflows in later processing, such as tsvectorout. Another important fix is to prevent tsvectorrecv() from accepting invalid tsvectors. The main problem there is that it did not reject empty-string lexemes. Hence, even though it did (mostly) enforce the MAXSTRPOS limit, it could still produce a result with an unreasonable number of tsvector entries, if they were primarily empty strings. Also, fix tsvectorout's calculation of its required output buffer size: it was multiplying the string lengths by pg_database_encoding_max_length() for no reason. That contributed to the risk of integer overflow there. With valid tsvector input, there's no risk, but there's still no reason to make the output buffer several times bigger than needed. I also tried to make a couple of related routines more robust, and spent some effort on improving the comments in ts_type.h. Also, standardize on a single spelling of the "string is too long for tsvector" message, using %zu instead of an assortment of formats. These changes aren't security per se but came out of inspecting the code for problems. Reported-by: Yuhang Wu <[email protected]> and Zhenpeng Lin Reported-by: Zheng Yu <[email protected]> Reported-by: Hcamael <[email protected]> Author: Tom Lane <[email protected]> Reviewed-by: Amit Langote <[email protected]> Backpatch-through: 14 Security: CVE-2026-14662 --- src/backend/tsearch/to_tsany.c | 23 +++++++++++++++----- src/backend/tsearch/ts_parse.c | 27 ++++++++++++++++++----- src/backend/utils/adt/tsvector.c | 28 ++++++++++++++++++------ src/backend/utils/adt/tsvector_op.c | 40 +++++++++++++++++++++++++++++----- src/include/tsearch/ts_type.h | 43 ++++++++++++++++++++++++++----------- 5 files changed, 126 insertions(+), 35 deletions(-) diff --git a/src/backend/tsearch/to_tsany.c b/src/backend/tsearch/to_tsany.c index fe39d6c4b93..be53631810b 100644 --- a/src/backend/tsearch/to_tsany.c +++ b/src/backend/tsearch/to_tsany.c @@ -166,8 +166,8 @@ TSVector make_tsvector(ParsedText *prs) { int i, - j, - lenstr = 0, + j; + size_t lenstr = 0, totallen; TSVector in; WordEntry *ptr; @@ -178,10 +178,22 @@ make_tsvector(ParsedText *prs) if (prs->curwords > 0) prs->curwords = uniqueWORD(prs->words, prs->curwords); - /* Determine space needed */ + /* + * Determine space needed. Since what we are calculating is equivalent to + * the size of a portion of the input data structure, lenstr surely can't + * overflow size_t. + */ for (i = 0; i < prs->curwords; i++) { - lenstr += prs->words[i].len; + int toklen = prs->words[i].len; + + /* Double-check that caller passed only lexemes of valid lengths */ + if (toklen <= 0 || toklen > MAXSTRLEN) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("lexeme is too long for tsvector (%zu bytes, max %zu bytes)", + (size_t) toklen, (size_t) MAXSTRLEN))); + lenstr += toklen; if (prs->words[i].alen) { lenstr = SHORTALIGN(lenstr); @@ -192,7 +204,8 @@ make_tsvector(ParsedText *prs) if (lenstr > MAXSTRPOS) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), - errmsg("string is too long for tsvector (%d bytes, max %d bytes)", lenstr, MAXSTRPOS))); + errmsg("string is too long for tsvector (%zu bytes, max %zu bytes)", + lenstr, (size_t) MAXSTRPOS))); totallen = CALCDATASIZE(prs->curwords, lenstr); in = (TSVector) palloc0(totallen); diff --git a/src/backend/tsearch/ts_parse.c b/src/backend/tsearch/ts_parse.c index c347129009e..bc5d5197e71 100644 --- a/src/backend/tsearch/ts_parse.c +++ b/src/backend/tsearch/ts_parse.c @@ -401,12 +401,30 @@ parsetext(Oid cfgId, ParsedText *prs, char *buf, int buflen) while ((norms = LexizeExec(&ldata, NULL)) != NULL) { - TSLexeme *ptr = norms; - prs->pos++; /* set pos */ - while (ptr->lexeme) + for (TSLexeme *ptr = norms; ptr->lexeme; ptr++) { + size_t lexeme_len = strlen(ptr->lexeme); + + if (lexeme_len > MAXSTRLEN) + { +#ifdef IGNORE_LONGLEXEME + ereport(NOTICE, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("word is too long to be indexed"), + errdetail("Words longer than %d characters are ignored.", + MAXSTRLEN))); + continue; +#else + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("word is too long to be indexed"), + errdetail("Words longer than %d characters are ignored.", + MAXSTRLEN))); +#endif + } + if (prs->curwords == prs->lenwords) { prs->lenwords *= 2; @@ -415,13 +433,12 @@ parsetext(Oid cfgId, ParsedText *prs, char *buf, int buflen) if (ptr->flags & TSL_ADDPOS) prs->pos++; - prs->words[prs->curwords].len = strlen(ptr->lexeme); + prs->words[prs->curwords].len = lexeme_len; prs->words[prs->curwords].word = ptr->lexeme; prs->words[prs->curwords].nvariant = ptr->nvariant; prs->words[prs->curwords].flags = ptr->flags & TSL_PREFIX; prs->words[prs->curwords].alen = 0; prs->words[prs->curwords].pos.pos = LIMITPOS(prs->pos); - ptr++; prs->curwords++; } pfree(norms); diff --git a/src/backend/utils/adt/tsvector.c b/src/backend/utils/adt/tsvector.c index 39e16f8a7cd..d3981bafd37 100644 --- a/src/backend/utils/adt/tsvector.c +++ b/src/backend/utils/adt/tsvector.c @@ -218,8 +218,8 @@ tsvectorin(PG_FUNCTION_ARGS) if (cur - tmpbuf > MAXSTRPOS) ereturn(escontext, (Datum) 0, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), - errmsg("string is too long for tsvector (%ld bytes, max %ld bytes)", - (long) (cur - tmpbuf), (long) MAXSTRPOS))); + errmsg("string is too long for tsvector (%zu bytes, max %zu bytes)", + (size_t) (cur - tmpbuf), (size_t) MAXSTRPOS))); /* * Enlarge buffers if needed @@ -272,7 +272,8 @@ tsvectorin(PG_FUNCTION_ARGS) if (buflen > MAXSTRPOS) ereturn(escontext, (Datum) 0, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), - errmsg("string is too long for tsvector (%d bytes, max %d bytes)", buflen, MAXSTRPOS))); + errmsg("string is too long for tsvector (%zu bytes, max %zu bytes)", + (size_t) buflen, (size_t) MAXSTRPOS))); totallen = CALCDATASIZE(len, buflen); in = (TSVector) palloc0(totallen); @@ -317,8 +318,8 @@ tsvectorout(PG_FUNCTION_ARGS) TSVector out = PG_GETARG_TSVECTOR(0); char *outbuf; int32 i, - lenbuf = 0, pp; + size_t lenbuf; WordEntry *ptr = ARRPTR(out); char *curin, *curout; @@ -327,7 +328,7 @@ tsvectorout(PG_FUNCTION_ARGS) lenbuf = out->size * 2 /* '' */ + out->size - 1 /* space */ + 2 /* \0 */ ; for (i = 0; i < out->size; i++) { - lenbuf += ptr[i].len * 2 * pg_database_encoding_max_length() /* for escape */ ; + lenbuf += ptr[i].len * 2 /* allow for escapes */ ; if (ptr[i].haspos) lenbuf += 1 /* : */ + 7 /* int2 + , + weight */ * POSDATALEN(out, &(ptr[i])); } @@ -459,12 +460,14 @@ tsvectorrecv(PG_FUNCTION_ARGS) bool needSort = false; nentries = pq_getmsgint(buf, sizeof(int32)); - if (nentries < 0 || nentries > (MaxAllocSize / sizeof(WordEntry))) + + /* We disallow empty lexemes, so more than MAXSTRPOS of them can't fit */ + if (nentries < 0 || nentries > MAXSTRPOS) elog(ERROR, "invalid size of tsvector"); hdrlen = DATAHDRSIZE + sizeof(WordEntry) * nentries; - len = hdrlen * 2; /* times two to make room for lexemes */ + len = hdrlen * 2; /* times two to make some room for lexemes */ vec = (TSVector) palloc0(len); vec->size = nentries; @@ -481,6 +484,8 @@ tsvectorrecv(PG_FUNCTION_ARGS) /* sanity checks */ lex_len = strlen(lexeme); + if (lex_len == 0) + elog(ERROR, "invalid tsvector: empty lexeme"); if (lex_len > MAXSTRLEN) elog(ERROR, "invalid tsvector: lexeme too long"); @@ -546,6 +551,15 @@ tsvectorrecv(PG_FUNCTION_ARGS) } } + /* + * Enforce that datalen is still within MAXSTRPOS, ie the last lexeme + * didn't go past that. We could allow that, since no "pos" field + * overflowed, but tsvectorrecv shouldn't accept values that other + * tsvector-constructing routines wouldn't. + */ + if (datalen > MAXSTRPOS) + elog(ERROR, "invalid tsvector: maximum total lexeme length exceeded"); + SET_VARSIZE(vec, hdrlen + datalen); if (needSort) diff --git a/src/backend/utils/adt/tsvector_op.c b/src/backend/utils/adt/tsvector_op.c index ae90e750604..b0f6257fc7d 100644 --- a/src/backend/utils/adt/tsvector_op.c +++ b/src/backend/utils/adt/tsvector_op.c @@ -175,6 +175,7 @@ tsvector_strip(PG_FUNCTION_ARGS) *arrout; char *cur; + /* Output can't be bigger than input, so no need for overflow checks */ for (i = 0; i < in->size; i++) len += arrin[i].len; @@ -499,6 +500,8 @@ tsvector_delete_by_indices(TSVector tsv, int *indices_to_delete, /* * Copy tsv to tsout, skipping lexemes listed in indices_to_delete. + * + * Output can't be bigger than input, so no need for overflow checks. */ arrout = ARRPTR(tsout); dataout = STRPTR(tsout); @@ -727,7 +730,7 @@ tsvector_to_array(PG_FUNCTION_ARGS) int i; ArrayType *array; - elements = palloc(tsin->size * sizeof(Datum)); + elements = palloc_array(Datum, tsin->size); for (i = 0; i < tsin->size; i++) { @@ -762,20 +765,29 @@ array_to_tsvector(PG_FUNCTION_ARGS) deconstruct_array_builtin(v, TEXTOID, &dlexemes, &nulls, &nitems); /* - * Reject nulls and zero length strings (maybe we should just ignore them, - * instead?) + * Reject nulls and zero-length or over-length strings (maybe we should + * just ignore them, instead?) */ for (i = 0; i < nitems; i++) { + int toklen; + if (nulls[i]) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("lexeme array may not contain nulls"))); - if (VARSIZE(dlexemes[i]) - VARHDRSZ == 0) + toklen = VARSIZE(dlexemes[i]) - VARHDRSZ; + if (toklen == 0) ereport(ERROR, (errcode(ERRCODE_ZERO_LENGTH_CHARACTER_STRING), errmsg("lexeme array may not contain empty strings"))); + if (toklen >= MAXSTRLEN) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("word is too long (%d bytes, max %d bytes)", + toklen, + MAXSTRLEN - 1))); } /* Sort and de-dup, because this is required for a valid tsvector. */ @@ -789,6 +801,11 @@ array_to_tsvector(PG_FUNCTION_ARGS) /* Calculate space needed for surviving lexemes. */ for (i = 0; i < nitems; i++) datalen += VARSIZE(dlexemes[i]) - VARHDRSZ; + if (datalen > MAXSTRPOS) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("string is too long for tsvector (%zu bytes, max %zu bytes)", + (size_t) datalen, (size_t) MAXSTRPOS))); tslen = CALCDATASIZE(nitems, datalen); /* Allocate and fill tsvector. */ @@ -872,9 +889,15 @@ tsvector_filter(PG_FUNCTION_ARGS) } } + /* + * The output tsvector might be smaller than the input, but it can't be + * bigger, so VARSIZE(tsin) is surely enough space. Also, we don't need + * to worry about overflows below. + */ tsout = (TSVector) palloc0(VARSIZE(tsin)); tsout->size = tsin->size; arrout = ARRPTR(tsout); + /* worst-case location of output's lexemes; we may need to adjust below */ dataout = STRPTR(tsout); for (i = j = 0; i < tsin->size; i++) @@ -974,6 +997,12 @@ tsvector_concat(PG_FUNCTION_ARGS) * Conservative estimate of space needed. We might need all the data in * both inputs, and conceivably add a pad byte before position data for * each item where there was none before. + * + * Note: since the MAXSTRPOS limit constrains each input tsvector to be + * considerably less than MaxAllocSize, we don't need to worry about + * integer overflow here, nor in the data-copying steps below. We do need + * to enforce that the result meets the MAXSTRPOS limit, but we check that + * once at the end. */ output_bytes = VARSIZE(in1) + VARSIZE(in2) + i1 + i2; @@ -1125,7 +1154,8 @@ tsvector_concat(PG_FUNCTION_ARGS) if (dataoff > MAXSTRPOS) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), - errmsg("string is too long for tsvector (%d bytes, max %d bytes)", dataoff, MAXSTRPOS))); + errmsg("string is too long for tsvector (%zu bytes, max %zu bytes)", + (size_t) dataoff, (size_t) MAXSTRPOS))); /* * Adjust sizes (asserting that we didn't overrun the original estimates) diff --git a/src/include/tsearch/ts_type.h b/src/include/tsearch/ts_type.h index b076039c1c1..23f6a7ec593 100644 --- a/src/include/tsearch/ts_type.h +++ b/src/include/tsearch/ts_type.h @@ -35,7 +35,9 @@ * * The positions for each lexeme must be sorted. * - * Note, tsvectorsend/recv believe that sizeof(WordEntry) == 4 + * Note that while the WordEntry items must be sorted per tsCompareString(), + * the per-lexeme data storage could be in some other order, ie the series + * of WordEntry->pos values need not be strictly ascending. */ typedef struct @@ -46,13 +48,15 @@ typedef struct pos:20; /* MAX 1Mb */ } WordEntry; -#define MAXSTRLEN ( (1<<11) - 1) -#define MAXSTRPOS ( (1<<20) - 1) +#define MAXSTRLEN ( (1<<11) - 1) /* maximum value of WordEntry.len */ +#define MAXSTRPOS ( (1<<20) - 1) /* maximum value of WordEntry.pos */ extern int compareWordEntryPos(const void *a, const void *b); /* - * Equivalent to + * Representation of positions (and weights) associated with a lexeme. + * + * WordEntryPos is equivalent to * typedef struct { * uint16 * weight:2, @@ -75,40 +79,53 @@ typedef struct WordEntryPos pos[1]; } WordEntryPosVector1; +#define MAXNUMPOS (256) /* semi-arbitrary limit on npos */ +/* Macros for getting/setting the fields of a WordEntryPos */ #define WEP_GETWEIGHT(x) ( (x) >> 14 ) #define WEP_GETPOS(x) ( (x) & 0x3fff ) #define WEP_SETWEIGHT(x,v) ( (x) = ( (v) << 14 ) | ( (x) & 0x3fff ) ) #define WEP_SETPOS(x,v) ( (x) = ( (x) & 0xc000 ) | ( (v) & 0x3fff ) ) -#define MAXENTRYPOS (1<<14) -#define MAXNUMPOS (256) +#define MAXENTRYPOS (1<<14) /* max value of WordEntryPos pos field, +1 */ +/* Macro for clamping a position to what will fit in WordEntryPos pos field */ #define LIMITPOS(x) ( ( (x) >= MAXENTRYPOS ) ? (MAXENTRYPOS-1) : (x) ) /* This struct represents a complete tsvector datum */ typedef struct { int32 vl_len_; /* varlena header (do not touch directly!) */ - int32 size; + int32 size; /* number of entries[] items */ WordEntry entries[FLEXIBLE_ARRAY_MEMBER]; /* lexemes follow the entries[] array */ } TSVectorData; typedef TSVectorData *TSVector; +/* + * Calculate the size of a TSVector given the number of WordEntries and + * the total space needed for lexeme text and positions. NOTE: callers + * must enforce lenstr <= MAXSTRPOS, which ensures that WordEntry.pos + * fields will not overflow, and also protects against integer overflow here. + * (Since we prohibit empty lexemes, nentries can't exceed lenstr.) + */ #define DATAHDRSIZE (offsetof(TSVectorData, entries)) #define CALCDATASIZE(nentries, lenstr) (DATAHDRSIZE + (nentries) * sizeof(WordEntry) + (lenstr) ) /* pointer to start of a tsvector's WordEntry array */ -#define ARRPTR(x) ( (x)->entries ) +#define ARRPTR(tsv) ( (tsv)->entries ) /* pointer to start of a tsvector's lexeme storage */ -#define STRPTR(x) ( (char *) &(x)->entries[(x)->size] ) - -#define _POSVECPTR(x, e) ((WordEntryPosVector *)(STRPTR(x) + SHORTALIGN((e)->pos + (e)->len))) -#define POSDATALEN(x,e) ( ( (e)->haspos ) ? (_POSVECPTR(x,e)->npos) : 0 ) -#define POSDATAPTR(x,e) (_POSVECPTR(x,e)->pos) +#define STRPTR(tsv) ( (char *) &(tsv)->entries[(tsv)->size] ) + +/* pointer to WordEntryPosVector for a WordEntry */ +#define _POSVECPTR(tsv,we) ((WordEntryPosVector *) \ + (STRPTR(tsv) + SHORTALIGN((we)->pos + (we)->len))) +/* number of positions stored for a WordEntry */ +#define POSDATALEN(tsv,we) ( (we)->haspos ? _POSVECPTR(tsv,we)->npos : 0 ) +/* pointer to start of positions stored for a WordEntry */ +#define POSDATAPTR(tsv,we) (_POSVECPTR(tsv,we)->pos) /* * fmgr interface functions --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
