maartenbreddels commented on a change in pull request #7449:
URL: https://github.com/apache/arrow/pull/7449#discussion_r447155149
##########
File path: cpp/src/arrow/compute/kernels/scalar_string.cc
##########
@@ -39,6 +73,103 @@ struct AsciiLength {
}
};
+template <typename Type, typename Derived>
+struct Utf8Transform {
+ using offset_type = typename Type::offset_type;
+ using ArrayType = typename TypeTraits<Type>::ArrayType;
+
+ static offset_type Transform(const uint8_t* input, offset_type
input_string_ncodeunits,
+ uint8_t* output) {
+ uint8_t* output_end = arrow::util::Utf8Transform(
+ input, input + input_string_ncodeunits, output,
Derived::TransformCodepoint);
+ return static_cast<offset_type>(output_end - output);
+ }
+
+ static void Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ if (batch[0].kind() == Datum::ARRAY) {
+ EnsureLookupTablesFilled();
+ const ArrayData& input = *batch[0].array();
+ ArrayType input_boxed(batch[0].array());
+ ArrayData* output = out->mutable_array();
+
+ offset_type input_ncodeunits = input_boxed.total_values_length();
+ offset_type input_nstrings = static_cast<offset_type>(input.length);
+
+ // Section 5.18 of the Unicode spec claim that the number of codepoints
for case
+ // mapping can grow by a factor of 3. This means grow by a factor of 3
in bytes
+ // However, since we don't support all casings (SpecialCasing.txt) the
growth
+ // is actually only at max 3/2 (as covered by the unittest).
+ // Note that rounding down the 3/2 is ok, since only codepoints encoded
by
+ // two code units (even) can grow to 3 code units.
+
+ int64_t output_ncodeunits_max = ((int64_t)input_ncodeunits) * 3 / 2;
Review comment:
Old habit, doing my best :)
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
