This is an automated email from the ASF dual-hosted git repository.
kou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new 57139b0db2 GH-50136: [C++][Gandiva] Enhance CHR to work with unicode
(#50137)
57139b0db2 is described below
commit 57139b0db2c6f85bf2cdd9525107398f646e5c97
Author: Logan Riggs <[email protected]>
AuthorDate: Thu Aug 13 14:28:50 2026 -0700
GH-50136: [C++][Gandiva] Enhance CHR to work with unicode (#50137)
### Rationale for this change
`CHR(n)` only worked for ASCII (0–127). Values ≥ 128 emitted a single raw
byte
(invalid UTF‑8), causing "Error during planning". Goal: emit the proper
multi‑byte **UTF‑8 encoding** of the Unicode code point, consistent with
PostgreSQL/Snowflake.
### What changes are included in this PR?
#### Arrow (C++ / Gandiva)
| File | Change |
|------|--------|
| `cpp/src/gandiva/precompiled/string_ops.cc` | `chr_int64` rewritten to
UTF‑8‑encode the code point (1–4 bytes) and error on invalid input (negative, >
0x10FFFF, surrogate range 0xD800–0xDFFF). `chr_int32` now delegates to it. |
| `cpp/src/gandiva/precompiled/string_ops_test.cc` | `TestChrBigInt`
rewritten for UTF‑8 semantics: every byte‑length boundary (1/2/3/4‑byte,
low+high), í/€/日/😀, and the three invalid‑input error cases. |
### Are these changes tested?
Yes, unit tests.
### Are there any user-facing changes?
Yes, the CHR gandiva function now supports unicode characters.
* GitHub Issue: #50136
Authored-by: [email protected] <[email protected]>
Signed-off-by: Sutou Kouhei <[email protected]>
---
cpp/src/gandiva/precompiled/string_ops.cc | 67 ++++++++++-----
cpp/src/gandiva/precompiled/string_ops_test.cc | 110 ++++++++++++++++---------
cpp/src/gandiva/tests/projector_test.cc | 44 ++++++++--
3 files changed, 156 insertions(+), 65 deletions(-)
diff --git a/cpp/src/gandiva/precompiled/string_ops.cc
b/cpp/src/gandiva/precompiled/string_ops.cc
index 2bd8e4076b..b7c6b518b4 100644
--- a/cpp/src/gandiva/precompiled/string_ops.cc
+++ b/cpp/src/gandiva/precompiled/string_ops.cc
@@ -1417,29 +1417,32 @@ gdv_int32 ascii_utf8(const char* data, gdv_int32
data_len) {
return static_cast<gdv_int32>(static_cast<signed char>(data[0]));
}
-// Returns the ASCII character having the binary equivalent to A.
-// If A is larger than 256 the result is equivalent to chr(A % 256).
+// Returns the UTF-8 encoding of the Unicode code point A.
+// Raises an error if A is not a valid code point, i.e. it is negative, greater
+// than 0x10FFFF, or falls within the UTF-16 surrogate range 0xD800-0xDFFF.
FORCE_INLINE
-const char* chr_int32(gdv_int64 context, gdv_int32 in, gdv_int32* out_len) {
- in = in % 256;
- *out_len = 1;
-
- char* ret = reinterpret_cast<char*>(gdv_fn_context_arena_malloc(context,
*out_len));
- if (ret == nullptr) {
- gdv_fn_context_set_error_msg(context, "Could not allocate memory for
output string");
+const char* chr_int64(gdv_int64 context, gdv_int64 in, gdv_int32* out_len) {
+ if (in < 0 || in > 0x10FFFF || (in >= 0xD800 && in <= 0xDFFF)) {
+ char err_msg[128];
+ snprintf(err_msg, sizeof(err_msg),
+ "Input %" PRId64
+ " is not a valid Unicode code point in the range 0 to 1114111,
excluding "
+ "the surrogate range 0xD800–0xDFFF",
+ in);
+ gdv_fn_context_set_error_msg(context, err_msg);
*out_len = 0;
return "";
}
- ret[0] = char(in);
- return ret;
-}
-// Returns the ASCII character having the binary equivalent to A.
-// If A is larger than 256 the result is equivalent to chr(A % 256).
-FORCE_INLINE
-const char* chr_int64(gdv_int64 context, gdv_int64 in, gdv_int32* out_len) {
- in = in % 256;
- *out_len = 1;
+ if (in <= 0x7F) {
+ *out_len = 1;
+ } else if (in <= 0x7FF) {
+ *out_len = 2;
+ } else if (in <= 0xFFFF) {
+ *out_len = 3;
+ } else {
+ *out_len = 4;
+ }
char* ret = reinterpret_cast<char*>(gdv_fn_context_arena_malloc(context,
*out_len));
if (ret == nullptr) {
@@ -1447,10 +1450,36 @@ const char* chr_int64(gdv_int64 context, gdv_int64 in,
gdv_int32* out_len) {
*out_len = 0;
return "";
}
- ret[0] = char(in);
+
+ switch (*out_len) {
+ case 1:
+ ret[0] = static_cast<char>(in);
+ break;
+ case 2:
+ ret[0] = static_cast<char>(0xC0 | (in >> 6));
+ ret[1] = static_cast<char>(0x80 | (in & 0x3F));
+ break;
+ case 3:
+ ret[0] = static_cast<char>(0xE0 | (in >> 12));
+ ret[1] = static_cast<char>(0x80 | ((in >> 6) & 0x3F));
+ ret[2] = static_cast<char>(0x80 | (in & 0x3F));
+ break;
+ case 4:
+ ret[0] = static_cast<char>(0xF0 | (in >> 18));
+ ret[1] = static_cast<char>(0x80 | ((in >> 12) & 0x3F));
+ ret[2] = static_cast<char>(0x80 | ((in >> 6) & 0x3F));
+ ret[3] = static_cast<char>(0x80 | (in & 0x3F));
+ break;
+ }
return ret;
}
+// Returns the UTF-8 encoding of the Unicode code point A. See chr_int64.
+FORCE_INLINE
+const char* chr_int32(gdv_int64 context, gdv_int32 in, gdv_int32* out_len) {
+ return chr_int64(context, in, out_len);
+}
+
FORCE_INLINE
const char* convert_fromUTF8_binary(gdv_int64 context, const char* bin_in,
gdv_int32 len,
gdv_int32* out_len) {
diff --git a/cpp/src/gandiva/precompiled/string_ops_test.cc
b/cpp/src/gandiva/precompiled/string_ops_test.cc
index 25dbb339fe..3faeb31623 100644
--- a/cpp/src/gandiva/precompiled/string_ops_test.cc
+++ b/cpp/src/gandiva/precompiled/string_ops_test.cc
@@ -58,11 +58,12 @@ TEST(TestStringOps, TestAscii) {
}
TEST(TestStringOps, TestChrBigInt) {
- // CHR
+ // CHR returns the UTF-8 encoding of the given Unicode code point.
gandiva::ExecutionContext ctx;
uint64_t ctx_ptr = reinterpret_cast<gdv_int64>(&ctx);
int32_t out_len = 0;
+ // 1-byte ASCII code points.
auto out = chr_int32(ctx_ptr, 88, &out_len);
EXPECT_EQ(std::string(out, out_len), "X");
@@ -72,62 +73,89 @@ TEST(TestStringOps, TestChrBigInt) {
out = chr_int32(ctx_ptr, 49, &out_len);
EXPECT_EQ(std::string(out, out_len), "1");
- out = chr_int64(ctx_ptr, 84, &out_len);
- EXPECT_EQ(std::string(out, out_len), "T");
+ out = chr_int32(ctx_ptr, 33, &out_len);
+ EXPECT_EQ(std::string(out, out_len), "!");
- out = chr_int32(ctx_ptr, 340, &out_len);
- EXPECT_EQ(std::string(out, out_len), "T");
+ out = chr_int64(ctx_ptr, 0, &out_len);
+ EXPECT_EQ(std::string(out, out_len), std::string("\0", 1));
- out = chr_int64(ctx_ptr, 256, &out_len);
- EXPECT_EQ(std::strcmp(out, "\0"), 0);
+ // BACKSPACE
+ out = chr_int64(ctx_ptr, 8, &out_len);
+ EXPECT_EQ(std::string(out, out_len), "\b");
- out = chr_int32(ctx_ptr, 33, &out_len);
- EXPECT_EQ(std::string(out, out_len), "!");
+ // ESCAPE (ESC)
+ out = chr_int64(ctx_ptr, 27, &out_len);
+ EXPECT_EQ(std::string(out, out_len), "\x1B");
- out = chr_int64(ctx_ptr, 46, &out_len);
- EXPECT_EQ(std::string(out, out_len), ".");
+ // Highest 1-byte code point (U+007F, DELETE).
+ out = chr_int32(ctx_ptr, 0x7F, &out_len);
+ EXPECT_EQ(std::string(out, out_len), "\x7F");
- out = chr_int32(ctx_ptr, 63, &out_len);
- EXPECT_EQ(std::string(out, out_len), "?");
+ // 2-byte code points.
+ // Lowest 2-byte code point (U+0080).
+ out = chr_int32(ctx_ptr, 0x80, &out_len);
+ EXPECT_EQ(std::string(out, out_len), "\xC2\x80");
- out = chr_int64(ctx_ptr, 0, &out_len);
- EXPECT_EQ(std::strcmp(out, "\0"), 0);
+ // í (U+00ED)
+ out = chr_int32(ctx_ptr, 237, &out_len);
+ EXPECT_EQ(std::string(out, out_len), "\xC3\xAD");
- out = chr_int32(ctx_ptr, -158, &out_len);
- EXPECT_EQ(std::string(out, out_len), "b");
+ // Highest 2-byte code point (U+07FF).
+ out = chr_int64(ctx_ptr, 0x7FF, &out_len);
+ EXPECT_EQ(std::string(out, out_len), "\xDF\xBF");
- out = chr_int64(ctx_ptr, -5, &out_len);
- EXPECT_EQ(std::string(out, out_len), "\xFB");
+ // 3-byte code points.
+ // Lowest 3-byte code point (U+0800).
+ out = chr_int32(ctx_ptr, 0x800, &out_len);
+ EXPECT_EQ(std::string(out, out_len), "\xE0\xA0\x80");
- out = chr_int32(ctx_ptr, -340, &out_len);
- EXPECT_EQ(std::string(out, out_len), "\xAC");
+ // € (U+20AC)
+ out = chr_int32(ctx_ptr, 8364, &out_len);
+ EXPECT_EQ(std::string(out, out_len), "\xE2\x82\xAC");
- out = chr_int64(ctx_ptr, -66, &out_len);
- EXPECT_EQ(std::string(out, out_len), "\xBE");
+ // 日 (U+65E5)
+ out = chr_int64(ctx_ptr, 26085, &out_len);
+ EXPECT_EQ(std::string(out, out_len), "\xE6\x97\xA5");
- // €
- out = chr_int32(ctx_ptr, 128, &out_len);
- EXPECT_EQ(std::string(out, out_len), "\x80");
+ // Highest 3-byte code point (U+FFFF).
+ out = chr_int32(ctx_ptr, 0xFFFF, &out_len);
+ EXPECT_EQ(std::string(out, out_len), "\xEF\xBF\xBF");
- // œ
- out = chr_int64(ctx_ptr, 156, &out_len);
- EXPECT_EQ(std::string(out, out_len), "\x9C");
+ // 4-byte code points.
+ // Lowest 4-byte code point (U+10000).
+ out = chr_int64(ctx_ptr, 0x10000, &out_len);
+ EXPECT_EQ(std::string(out, out_len), "\xF0\x90\x80\x80");
- // ÿ
- out = chr_int32(ctx_ptr, 255, &out_len);
- EXPECT_EQ(std::string(out, out_len), "\xFF");
+ // 😀 (U+1F600)
+ out = chr_int64(ctx_ptr, 0x1F600, &out_len);
+ EXPECT_EQ(std::string(out, out_len), "\xF0\x9F\x98\x80");
- // BACKSPACE
- out = chr_int64(ctx_ptr, 8, &out_len);
- EXPECT_EQ(std::string(out, out_len), "\b");
+ // Highest valid code point (U+10FFFF).
+ out = chr_int64(ctx_ptr, 0x10FFFF, &out_len);
+ EXPECT_EQ(std::string(out, out_len), "\xF4\x8F\xBF\xBF");
- // DEVICE CONTROL 3 (DC3)
- out = chr_int32(ctx_ptr, 19, &out_len);
- EXPECT_EQ(std::string(out, out_len), "\x13");
+ EXPECT_FALSE(ctx.has_error());
- // ESCAPE (ESC)
- out = chr_int64(ctx_ptr, 27, &out_len);
- EXPECT_EQ(std::string(out, out_len), "\x1B");
+ // Invalid code points raise an error that includes the offending value.
+ chr_int64(ctx_ptr, -1, &out_len);
+ EXPECT_EQ(out_len, 0);
+ EXPECT_TRUE(ctx.get_error().find("not a valid Unicode code point") !=
std::string::npos)
+ << ctx.get_error();
+ EXPECT_TRUE(ctx.get_error().find("-1") != std::string::npos) <<
ctx.get_error();
+ ctx.Reset();
+
+ chr_int64(ctx_ptr, 0x110000, &out_len);
+ EXPECT_EQ(out_len, 0);
+ EXPECT_TRUE(ctx.get_error().find("not a valid Unicode code point") !=
std::string::npos)
+ << ctx.get_error();
+ ctx.Reset();
+
+ // UTF-16 surrogate range is not a valid code point.
+ chr_int32(ctx_ptr, 0xD800, &out_len);
+ EXPECT_EQ(out_len, 0);
+ EXPECT_TRUE(ctx.get_error().find("not a valid Unicode code point") !=
std::string::npos)
+ << ctx.get_error();
+ ctx.Reset();
}
TEST(TestStringOps, TestBeginsEnds) {
diff --git a/cpp/src/gandiva/tests/projector_test.cc
b/cpp/src/gandiva/tests/projector_test.cc
index facd1084ca..27beb8595c 100644
--- a/cpp/src/gandiva/tests/projector_test.cc
+++ b/cpp/src/gandiva/tests/projector_test.cc
@@ -1325,13 +1325,16 @@ TEST_F(TestProjector, TestChr) {
auto status = Projector::Make(schema, {chr_expr}, TestConfiguration(),
&projector);
EXPECT_TRUE(status.ok()) << status.message();
- // Create a row-batch with some sample data
+ // Create a row-batch with code points spanning 1- to 4-byte UTF-8 encodings.
+ // 65 -> "A", 237 -> "í" (U+00ED), 8364 -> "€" (U+20AC), 26085 -> "日"
(U+65E5),
+ // 128512 -> "😀" (U+1F600).
int num_records = 5;
auto array0 =
- MakeArrowArrayInt64({65, 84, 255, 340, -5}, {true, true, true, true,
true});
- // expected output
- auto exp_chr =
- MakeArrowArrayUtf8({"A", "T", "\xFF", "T", "\xFB"}, {true, true, true,
true, true});
+ MakeArrowArrayInt64({65, 237, 8364, 26085, 128512}, {true, true, true,
true, true});
+ // expected UTF-8 output
+ auto exp_chr = MakeArrowArrayUtf8(
+ {"A", "\xC3\xAD", "\xE2\x82\xAC", "\xE6\x97\xA5", "\xF0\x9F\x98\x80"},
+ {true, true, true, true, true});
// prepare input record batch
auto in_batch = arrow::RecordBatch::Make(schema, num_records, {array0});
@@ -1345,6 +1348,37 @@ TEST_F(TestProjector, TestChr) {
EXPECT_ARROW_ARRAY_EQUALS(exp_chr, outputs.at(0));
}
+TEST_F(TestProjector, TestChrInvalidCodePoint) {
+ // schema for input fields
+ auto field0 = field("f0", int64());
+ auto schema = arrow::schema({field0});
+
+ // output fields
+ auto field_chr = field("chr", arrow::utf8());
+
+ // Build expression
+ auto chr_expr = TreeExprBuilder::MakeExpression("chr", {field0}, field_chr);
+
+ std::shared_ptr<Projector> projector;
+ auto status = Projector::Make(schema, {chr_expr}, TestConfiguration(),
&projector);
+ EXPECT_TRUE(status.ok()) << status.message();
+
+ // A code point outside the valid Unicode range (here, a negative value) must
+ // fail evaluation rather than wrap around.
+ int num_records = 1;
+ auto array0 = MakeArrowArrayInt64({-5}, {true});
+ auto in_batch = arrow::RecordBatch::Make(schema, num_records, {array0});
+
+ // Evaluate expression
+ arrow::ArrayVector outputs;
+ status = projector->Evaluate(*in_batch, pool_, &outputs);
+ EXPECT_FALSE(status.ok());
+ EXPECT_NE(status.message().find("not a valid Unicode code point"),
std::string::npos)
+ << status.message();
+ // The message should include the offending value for debuggability.
+ EXPECT_NE(status.message().find("-5"), std::string::npos) <<
status.message();
+}
+
TEST_F(TestProjector, TestBase64) {
// schema for input fields
auto field0 = field("f0", arrow::binary());
