This is an automated email from the ASF dual-hosted git repository.
raulcd pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new e611f48081 GH-50707: [C++][Gandiva] fix out-of-bounds read in mask
utf8proc length (#50709)
e611f48081 is described below
commit e611f48081ec927e18e5bcb72fbd071c15c76b08
Author: Abdul Rawoof Khan <[email protected]>
AuthorDate: Fri Aug 14 13:39:26 2026 +0530
GH-50707: [C++][Gandiva] fix out-of-bounds read in mask utf8proc length
(#50709)
### Rationale for this change
The Gandiva mask stubs walk a utf8 string calling `utf8proc_iterate(data +
bytes_read, data_len, ...)`, passing the total `data_len` as the remaining byte
count even though the pointer has already advanced by `bytes_read`.
`utf8proc_iterate` reads up to `strlen` bytes, so a value ending in a truncated
multi-byte glyph (for example a lead byte `0xF0` as the final byte of an
exactly sized value buffer) makes it read continuation bytes past `data +
data_len`. This is reachable from `mask [...]
### What changes are included in this PR?
Pass `data_len - bytes_read` (the real remaining length) at all four
`utf8proc_iterate` call sites so utf8proc reports the truncated glyph instead
of reading past the buffer. `mask_utf8_utf8_utf8_utf8` also lacked the
`char_len < 0` check that the other two paths already have, so I added it there
to reject the now-detected invalid input rather than advancing by a negative
length.
### Are these changes tested?
Yes. `TestMaskTruncatedUtf8NoOverread` feeds a value whose reported length
stops one byte short of a complete euro sign; before the change both functions
consumed the out-of-range byte and returned a masked result, after it they
report the truncated input. The existing mask tests still pass.
### Are there any user-facing changes?
No change for valid utf8. A value that ends in a truncated multi-byte glyph
now surfaces an invalid-utf8 error instead of being silently masked using bytes
past its end.
**This PR contains a "Critical Fix".** The wrong length argument lets
`utf8proc_iterate` read past the end of an exactly sized input buffer.
* GitHub Issue: #50707
Authored-by: abdul rawoof <[email protected]>
Signed-off-by: Raúl Cumplido <[email protected]>
---
cpp/src/gandiva/gdv_function_stubs.cc | 23 +++++++++++++++----
cpp/src/gandiva/gdv_function_stubs_test.cc | 37 ++++++++++++++++++++++++++++++
2 files changed, 56 insertions(+), 4 deletions(-)
diff --git a/cpp/src/gandiva/gdv_function_stubs.cc
b/cpp/src/gandiva/gdv_function_stubs.cc
index 1b359524c2..6b3e9935b0 100644
--- a/cpp/src/gandiva/gdv_function_stubs.cc
+++ b/cpp/src/gandiva/gdv_function_stubs.cc
@@ -489,7 +489,7 @@ const char* gdv_mask_first_n_utf8_int32(int64_t context,
const char* data,
while ((chars_masked < n_to_mask) && (bytes_masked < data_len)) {
auto char_len =
utf8proc_iterate(reinterpret_cast<const utf8proc_uint8_t*>(data +
bytes_masked),
- data_len, &utf8_char);
+ data_len - bytes_masked, &utf8_char);
if (char_len < 0) {
gdv_fn_context_set_error_msg(context, utf8proc_errmsg(char_len));
@@ -597,7 +597,12 @@ const char* gdv_mask_last_n_utf8_int32(int64_t context,
const char* data,
while ((bytes_read < data_len) && (chars_counter < (num_of_chars -
n_to_mask))) {
auto char_len =
utf8proc_iterate(reinterpret_cast<const utf8proc_uint8_t*>(data +
bytes_read),
- data_len, &utf8_char);
+ data_len - bytes_read, &utf8_char);
+ if (char_len < 0) {
+ gdv_fn_context_set_error_msg(context, utf8proc_errmsg(char_len));
+ *out_len = 0;
+ return nullptr;
+ }
chars_counter++;
bytes_read += static_cast<int>(char_len);
}
@@ -611,7 +616,12 @@ const char* gdv_mask_last_n_utf8_int32(int64_t context,
const char* data,
while (bytes_read < data_len) {
auto char_len =
utf8proc_iterate(reinterpret_cast<const utf8proc_uint8_t*>(data +
bytes_read),
- data_len, &utf8_char);
+ data_len - bytes_read, &utf8_char);
+ if (char_len < 0) {
+ gdv_fn_context_set_error_msg(context, utf8proc_errmsg(char_len));
+ *out_len = 0;
+ return nullptr;
+ }
switch (utf8proc_category(utf8_char)) {
case 1:
out[out_idx] = 'X';
@@ -700,7 +710,12 @@ const char* mask_utf8_utf8_utf8_utf8(int64_t context,
const char* data, int32_t
while (bytes_read < data_len) {
auto char_len =
utf8proc_iterate(reinterpret_cast<const utf8proc_uint8_t*>(data +
bytes_read),
- data_len, &utf8_char);
+ data_len - bytes_read, &utf8_char);
+ if (char_len < 0) {
+ gdv_fn_context_set_error_msg(context, utf8proc_errmsg(char_len));
+ *out_len = 0;
+ return nullptr;
+ }
switch (utf8proc_category(utf8_char)) {
case UTF8PROC_CATEGORY_LU:
memcpy(out + out_index, upper, upper_length);
diff --git a/cpp/src/gandiva/gdv_function_stubs_test.cc
b/cpp/src/gandiva/gdv_function_stubs_test.cc
index 27ae60694b..c0197eb815 100644
--- a/cpp/src/gandiva/gdv_function_stubs_test.cc
+++ b/cpp/src/gandiva/gdv_function_stubs_test.cc
@@ -1151,6 +1151,43 @@ TEST(TestGdvFnStubs, TestMaskLastN) {
EXPECT_EQ(expected, std::string(result, out_len));
}
+TEST(TestGdvFnStubs, TestMaskTruncatedUtf8NoOverread) {
+ gandiva::ExecutionContext ctx;
+ int64_t ctx_ptr = reinterpret_cast<int64_t>(&ctx);
+ int32_t out_len = -1;
+
+ // A byte > 127 routes the mask functions through the utf8proc multi-byte
path.
+ // The buffer holds a complete euro sign (0xE2 0x82 0xAC) but the reported
+ // length stops one byte short, so the trailing glyph is truncated. The
+ // functions must not read the byte past data_len: utf8proc_iterate has to be
+ // told only data_len - offset bytes remain, otherwise it consumes the
+ // out-of-range continuation byte and decodes a full glyph instead of
+ // reporting the truncated input.
+ const char buf[] = {'a', static_cast<char>(0xE2), static_cast<char>(0x82),
+ static_cast<char>(0xAC)};
+ const int32_t truncated_len = 3; // 'a' + first two bytes of the euro sign
+
+ ctx.Reset();
+ gdv_mask_first_n_utf8_int32(ctx_ptr, buf, truncated_len, 4, &out_len);
+ EXPECT_EQ(out_len, 0);
+ EXPECT_TRUE(ctx.has_error());
+
+ out_len = -1;
+ ctx.Reset();
+ mask_utf8(ctx_ptr, buf, truncated_len, &out_len);
+ EXPECT_EQ(out_len, 0);
+ EXPECT_TRUE(ctx.has_error());
+
+ // gdv_mask_last_n_utf8_int32 catches the truncated glyph in its
+ // utf8proc_decompose pre-pass, so it reports the invalid input rather than
+ // reading past data_len in the iterate loop.
+ out_len = -1;
+ ctx.Reset();
+ gdv_mask_last_n_utf8_int32(ctx_ptr, buf, truncated_len, 4, &out_len);
+ EXPECT_EQ(out_len, 0);
+ EXPECT_TRUE(ctx.has_error());
+}
+
TEST(TestGdvFnStubs, TestTranslate) {
gandiva::ExecutionContext ctx;
int64_t ctx_ptr = reinterpret_cast<int64_t>(&ctx);
