linrrzqqq commented on code in PR #68131: URL: https://github.com/apache/doris/pull/68131#discussion_r4057617305
########## be/src/exprs/function/function_character_encoding.cpp: ########## @@ -0,0 +1,359 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include <unicode/ucnv.h> +#include <unicode/ucnv_err.h> + +#include <array> +#include <cstddef> +#include <cstdint> +#include <limits> +#include <memory> +#include <string> +#include <string_view> +#include <type_traits> +#include <vector> + +#include "common/status.h" +#include "core/assert_cast.h" +#include "core/block/block.h" +#include "core/column/column.h" +#include "core/column/column_const.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_varbinary.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_varbinary.h" +#include "core/string_ref.h" +#include "exprs/function/function.h" +#include "exprs/function/simple_function_factory.h" + +namespace doris { +namespace { + +enum class CharacterSet : uint8_t { + US_ASCII, + ISO_8859_1, + UTF_8, + UTF_16BE, + UTF_16LE, + UTF_16, + SIZE, +}; + +constexpr std::array<std::string_view, static_cast<size_t>(CharacterSet::SIZE)> + SUPPORTED_CHARACTER_SETS = {"US-ASCII", "ISO-8859-1", "UTF-8", + "UTF-16BE", "UTF-16LE", "UTF-16"}; + +bool equals_ignore_case(StringRef value, std::string_view expected) { + if (value.size != expected.size()) { + return false; + } + for (size_t i = 0; i < value.size; ++i) { + const char current = value.data[i] >= 'a' && value.data[i] <= 'z' + ? value.data[i] - ('a' - 'A') + : value.data[i]; + if (current != expected[i]) { + return false; + } + } + return true; +} + +Status parse_character_set(StringRef value, CharacterSet& character_set) { + for (size_t i = 0; i < SUPPORTED_CHARACTER_SETS.size(); ++i) { + if (equals_ignore_case(value, SUPPORTED_CHARACTER_SETS[i])) { + character_set = static_cast<CharacterSet>(i); + return Status::OK(); + } + } + return Status::InvalidArgument( + "Unsupported character set '{}'. Supported character sets are US-ASCII, " + "ISO-8859-1, UTF-8, UTF-16BE, UTF-16LE, and UTF-16", + std::string(value.data, value.size)); +} + +using ConverterPtr = std::unique_ptr<UConverter, decltype(&ucnv_close)>; + +class ConverterPair { +public: + ConverterPair() : _source(nullptr, ucnv_close), _target(nullptr, ucnv_close) {} + + Status open(std::string_view source_name, std::string_view target_name) { + UErrorCode error = U_ZERO_ERROR; + _source.reset(ucnv_open(source_name.data(), &error)); + if (U_FAILURE(error)) { + return Status::InternalError("Failed to open ICU converter '{}': {}", source_name, + u_errorName(error)); + } + + error = U_ZERO_ERROR; + ucnv_setToUCallBack(_source.get(), UCNV_TO_U_CALLBACK_STOP, nullptr, nullptr, nullptr, + &error); + if (U_FAILURE(error)) { + return Status::InternalError("Failed to configure ICU converter '{}': {}", source_name, + u_errorName(error)); + } + + error = U_ZERO_ERROR; + _target.reset(ucnv_open(target_name.data(), &error)); + if (U_FAILURE(error)) { + return Status::InternalError("Failed to open ICU converter '{}': {}", target_name, + u_errorName(error)); + } + + error = U_ZERO_ERROR; + ucnv_setFromUCallBack(_target.get(), UCNV_FROM_U_CALLBACK_STOP, nullptr, nullptr, nullptr, + &error); + if (U_FAILURE(error)) { + return Status::InternalError("Failed to configure ICU converter '{}': {}", target_name, + u_errorName(error)); + } + return Status::OK(); + } + + Status convert(StringRef input, std::string_view character_set_name, std::string& output) { Review Comment: ### P1：每行做了四次 ICU 扫描，实测慢约 2 倍 每个非空输入依次执行： 1. `ucnv_toUChars` 预计算 UTF-16 大小 2. `ucnv_toUChars` 真正转换 3. `ucnv_fromUChars` 预计算结果大小 4. `ucnv_fromUChars` 真正转换 这会完整扫描输入/中间结果四次，并生成 `_utf16` 中间缓冲区，最后还要把 `std::string converted` 再复制进结果列。 我用仓库自带 ICU、复用 converter，做了隔离转换内核的本地 microbenchmark，UTF8 → UTF16BE 的结果为： | 输入长度 | 当前实现 | `ucnv_convertEx` 直接转换 | 差距 | | -------- | ----------- | ------------------------- | ----- | | 15 B | 200 ns/row | 75 ns/row | 2.67× | | 63 B | 383 ns/row | 157 ns/row | 2.43× | | 1023 B | 3978 ns/row | 1991 ns/row | 2.00× | | 65535 B | 272 μs/row | 124 μs/row | 2.19× | 吞吐测试： - Encode UTF8 → UTF16BE：230 MiB/s → 499 MiB/s - Decode Latin1 → UTF8：659 MiB/s → 1336 MiB/s 这不是完整 Doris 端到端 benchmark，但已经能证明转换内核有约 2 倍的优化空间。建议使用 `ucnv_convertEx` 的 pivot buffer 做直接转换，并使用可增长的目标 buffer。 此外： - `_utf16` 是普通 `std::vector<UChar>` - 输出 scratch 是普通 `std::string` - 一个 block 最多缓存 7 个 converter，每个 converter 都可能保留自己的最大 `_utf16` 容量 大字符串和混合 charset block 下，这些显著 scratch allocation 没有使用 Doris allocator。直接转换既能减少内存峰值，也能避免这部分中间缓冲。 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
