This is an automated email from the ASF dual-hosted git repository.
pitrou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new 79e074ace3 GH-51225: [C++] utf8_normalize: compose for NFC and NFKC
(#51237)
79e074ace3 is described below
commit 79e074ace3a7c4ca26f211dfd84a1984ad53c362
Author: singhpratech <[email protected]>
AuthorDate: Thu Sep 10 12:02:24 2026 -0400
GH-51225: [C++] utf8_normalize: compose for NFC and NFKC (#51237)
### Rationale for this change
`utf8_normalize` with `form=NFC` or `NFKC` returned the decomposed forms
(NFD, NFKD). `Utf8NormalizeBase`
builds the right `utf8proc` options for each form but only calls
`utf8proc_decompose()`, which
decomposes regardless of `UTF8PROC_COMPOSE`; the composition step lives in
`utf8proc_normalize_utf32()`, which the kernel never called. See #51225.
### What changes are included in this PR?
- After a successful `utf8proc_decompose()`, when the options include
`UTF8PROC_COMPOSE`, call
`utf8proc_normalize_utf32()` on the scratch buffer. It composes in place
and returns the new code
point count; the existing UTF-8 encode loop is unchanged. NFD and NFKD
take the same path as before.
- Fix the `json_composed` fixture in `scalar_string_test.cc`: its bytes
were the decomposed form
(`61 CC 81`), the same string as `json_decomposed`, so the compose
assertions were comparing a value
with itself and passed with the bug. Thanks to `@ Santoshkumarpuppala`
for spotting that on the issue.
- Add composed/decomposed pairs (U+00E9, and a Hangul syllable with its
jamo) to
`test_utf8_normalize` in pyarrow; the existing input, U+00B2, is its own
NFC.
### Are these changes tested?
Yes. With the corrected fixture, `TestStringKernels.Utf8Normalize` fails on
the unpatched kernel and
passes with this change; the pyarrow test covers the composed forms from
Python.
### Are there any user-facing changes?
Yes: `utf8_normalize` with `NFC` and `NFKC` now returns composed output.
Callers that depended on the
previous (decomposed) result for those forms will see different bytes.
* GitHub Issue: #51225
Authored-by: singhpratech <[email protected]>
Signed-off-by: Antoine Pitrou <[email protected]>
---
.../arrow/compute/kernels/scalar_string_test.cc | 35 +++++++++++++++++++++-
.../arrow/compute/kernels/scalar_string_utf8.cc | 10 +++++++
python/pyarrow/tests/test_compute.py | 9 ++++++
3 files changed, 53 insertions(+), 1 deletion(-)
diff --git a/cpp/src/arrow/compute/kernels/scalar_string_test.cc
b/cpp/src/arrow/compute/kernels/scalar_string_test.cc
index 4969beada6..74650c1d76 100644
--- a/cpp/src/arrow/compute/kernels/scalar_string_test.cc
+++ b/cpp/src/arrow/compute/kernels/scalar_string_test.cc
@@ -1245,7 +1245,7 @@ TYPED_TEST(TestStringKernels, Utf8Normalize) {
// decomposed: U+0061(LATIN SMALL LETTER A) + U+0301(COMBINING ACUTE ACCENT)
// composed: U+00E1(LATIN SMALL LETTER A WITH ACUTE)
- const char* json_composed = "[\"foo\", \"á\"]";
+ const char* json_composed = "[\"foo\", \"\xc3\xa1\"]";
const char* json_decomposed = "[\"foo\", \"a\xcc\x81\"]";
for (const auto& options : compose_options) {
this->CheckUnary("utf8_normalize", json_decomposed, this->type(),
json_composed,
@@ -1260,6 +1260,39 @@ TYPED_TEST(TestStringKernels, Utf8Normalize) {
&options);
}
+ // decomposed: U+1112(HANGUL CHOSEONG HIEUH) + U+1161(HANGUL JUNGSEONG A) +
+ // U+11AB(HANGUL JONGSEONG NIEUN)
+ // composed: U+D55C(HANGUL SYLLABLE HAN)
+ json_composed = "[\"\xed\x95\x9c\"]";
+ json_decomposed = "[\"\xe1\x84\x92\xe1\x85\xa1\xe1\x86\xab\"]";
+ for (const auto& options : compose_options) {
+ this->CheckUnary("utf8_normalize", json_decomposed, this->type(),
json_composed,
+ &options);
+ this->CheckUnary("utf8_normalize", json_composed, this->type(),
json_composed,
+ &options);
+ }
+ for (const auto& options : decompose_options) {
+ this->CheckUnary("utf8_normalize", json_composed, this->type(),
json_decomposed,
+ &options);
+ this->CheckUnary("utf8_normalize", json_decomposed, this->type(),
json_decomposed,
+ &options);
+ }
+
+ // singleton: U+212B(ANGSTROM SIGN) decomposes to U+0041(LATIN CAPITAL
LETTER A) +
+ // U+030A(COMBINING RING ABOVE), which composes to U+00C5(LATIN
CAPITAL
+ // LETTER A WITH RING ABOVE), so the composed form differs from
the input
+ const char* json_singleton = "[\"\xe2\x84\xab\"]";
+ json_composed = "[\"\xc3\x85\"]";
+ json_decomposed = "[\"A\xcc\x8a\"]";
+ for (const auto& options : compose_options) {
+ this->CheckUnary("utf8_normalize", json_singleton, this->type(),
json_composed,
+ &options);
+ }
+ for (const auto& options : decompose_options) {
+ this->CheckUnary("utf8_normalize", json_singleton, this->type(),
json_decomposed,
+ &options);
+ }
+
// canonical: U+00B2(Superscript Two)
// compatibility: "2"
const char* json_canonical = "[\"01\xc2\xb2!\"]";
diff --git a/cpp/src/arrow/compute/kernels/scalar_string_utf8.cc
b/cpp/src/arrow/compute/kernels/scalar_string_utf8.cc
index 582d559dde..6d4525c286 100644
--- a/cpp/src/arrow/compute/kernels/scalar_string_utf8.cc
+++ b/cpp/src/arrow/compute/kernels/scalar_string_utf8.cc
@@ -547,6 +547,16 @@ struct Utf8NormalizeBase {
if (res < 0) {
return Status::Invalid("Cannot normalize utf8 string: ",
utf8proc_errmsg(res));
}
+ if (decompose_options_ & UTF8PROC_COMPOSE) {
+ // utf8proc_decompose() only decomposes; the canonical composition step
for
+ // NFC and NFKC is done in-place by utf8proc_normalize_utf32().
+ res = utf8proc_normalize_utf32(
+ reinterpret_cast<utf8proc_int32_t*>(codepoints_.data()), res,
+ decompose_options_);
+ if (res < 0) {
+ return Status::Invalid("Cannot normalize utf8 string: ",
utf8proc_errmsg(res));
+ }
+ }
return res;
}
diff --git a/python/pyarrow/tests/test_compute.py
b/python/pyarrow/tests/test_compute.py
index 6fac20a5ad..68fe9a9937 100644
--- a/python/pyarrow/tests/test_compute.py
+++ b/python/pyarrow/tests/test_compute.py
@@ -3831,6 +3831,15 @@ def test_utf8_normalize():
assert pc.utf8_normalize(arr, form="NFKC") == pa.array(["0123"])
assert pc.utf8_normalize(arr, "NFD") == arr
assert pc.utf8_normalize(arr, "NFKD") == pa.array(["0123"])
+ # GH-51225: composing forms must compose, not only decompose
+ composed = pa.array(["\u00e9", "\ud55c", None])
+ decomposed = pa.array(["e\u0301", "\u1112\u1161\u11ab", None])
+ for form in ("NFC", "NFKC"):
+ assert pc.utf8_normalize(decomposed, form=form) == composed
+ assert pc.utf8_normalize(composed, form=form) == composed
+ for form in ("NFD", "NFKD"):
+ assert pc.utf8_normalize(composed, form=form) == decomposed
+ assert pc.utf8_normalize(decomposed, form=form) == decomposed
with pytest.raises(
ValueError,
match='"NFZ" is not a valid Unicode normalization form'):
