nishant94 commented on code in PR #64667: URL: https://github.com/apache/doris/pull/64667#discussion_r3894018914
########## be/src/storage/index/inverted/analyzer/kuromoji/kuromoji_viterbi.cpp: ########## @@ -0,0 +1,300 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/inverted/analyzer/kuromoji/kuromoji_viterbi.h" + +#include <algorithm> +#include <cstddef> +#include <limits> + +namespace doris::segment_v2::inverted_index::kuromoji { + +namespace { + +constexpr int64_t KMJ_INF = std::numeric_limits<int64_t>::max() / 4; +constexpr uint32_t MAX_UNKNOWN_GROUP_CHARS = 1024; + +// Search/Extended-mode compound-decomposition penalties, matching Lucene's +// JapaneseTokenizer. Lengths are counted in code points. A token longer than the +// length threshold is penalized so the minimum-cost path prefers its shorter +// parts: all-kanji runs over KANJI_LENGTH chars, other runs over OTHER_LENGTH. +constexpr uint32_t SEARCH_MODE_KANJI_LENGTH = 2; +constexpr int64_t SEARCH_MODE_KANJI_PENALTY = 3000; +constexpr uint32_t SEARCH_MODE_OTHER_LENGTH = 7; +constexpr int64_t SEARCH_MODE_OTHER_PENALTY = 1700; + +struct DecodedCp { + char32_t cp; + uint32_t len; +}; + +// Decode one UTF-8 code point at text[pos]. +DecodedCp decode_utf8(std::string_view text, std::size_t pos) { + auto b0 = static_cast<unsigned char>(text[pos]); + const std::size_t avail = text.size() - pos; + if (b0 < 0x80) { + return {b0, 1}; + } + auto cont = [&](std::size_t i) { + return (static_cast<unsigned char>(text[pos + i]) & 0xC0U) == 0x80U; + }; + if ((b0 >> 5) == 0x6 && avail >= 2 && cont(1)) { + auto b1 = static_cast<unsigned char>(text[pos + 1]); + const auto cp = static_cast<char32_t>(((b0 & 0x1FU) << 6) | (b1 & 0x3FU)); + if (cp >= 0x80) { // reject overlong + return {cp, 2}; + } + } else if ((b0 >> 4) == 0xE && avail >= 3 && cont(1) && cont(2)) { + auto b1 = static_cast<unsigned char>(text[pos + 1]); + auto b2 = static_cast<unsigned char>(text[pos + 2]); + const auto cp = + static_cast<char32_t>(((b0 & 0x0FU) << 12) | ((b1 & 0x3FU) << 6) | (b2 & 0x3FU)); + if (cp >= 0x800 && (cp < 0xD800 || cp > 0xDFFF)) { // reject overlong + surrogates + return {cp, 3}; + } + } else if ((b0 >> 3) == 0x1E && avail >= 4 && cont(1) && cont(2) && cont(3)) { + auto b1 = static_cast<unsigned char>(text[pos + 1]); + auto b2 = static_cast<unsigned char>(text[pos + 2]); + auto b3 = static_cast<unsigned char>(text[pos + 3]); + const auto cp = static_cast<char32_t>(((b0 & 0x07U) << 18) | ((b1 & 0x3FU) << 12) | + ((b2 & 0x3FU) << 6) | (b3 & 0x3FU)); + if (cp >= 0x10000 && cp <= 0x10FFFF) { // reject overlong + out of range + return {cp, 4}; + } + } + return {b0, 1}; +} + +// Lucene JapaneseTokenizer's search-mode penalty for the token covering +// [start, end) bytes: penalize long compounds so the Viterbi prefers their +// shorter parts. Returns 0 for tokens at or under the length thresholds. +int64_t compute_penalty(const KuromojiDictionary& dict, std::string_view text, uint32_t start, + uint32_t end) { + uint32_t length = 0; + bool all_kanji = true; + for (uint32_t p = start; p < end;) { + const DecodedCp d = decode_utf8(text, p); + if (dict.char_category(d.cp) != CAT_KANJI) { Review Comment: run-be-ut.sh resets DORIS_HOME to the test dir, so that path never found the repo dict. The test now uses a compiled-in repo path, so it runs once the dict is generated (still skips otherwise). ########## be/test/storage/index/inverted/analyzer/kuromoji/kuromoji_real_dict_test.cpp: ########## @@ -0,0 +1,173 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include <gtest/gtest.h> +#include <sys/stat.h> + +#include <algorithm> +#include <cstdlib> +#include <iostream> +#include <memory> +#include <string> +#include <vector> + +#include "CLucene.h" +#include "common/config.h" +#include "storage/index/inverted/analyzer/analyzer.h" +#include "storage/index/inverted/analyzer/kuromoji/dict/kuromoji_dictionary.h" +#include "storage/index/inverted/analyzer/kuromoji/kuromoji_viterbi.h" +#include "storage/index/inverted/inverted_index_parser.h" + +// End-to-end against the REAL IPADIC dictionary generated under be/dict/kuromoji. +// Skips if the dictionary has not been generated on this host (e.g. plain CI), +// so it is safe to keep in the suite. +namespace doris::segment_v2::inverted_index::kuromoji { + +static std::string real_dict_dir() { + const char* home = std::getenv("DORIS_HOME"); + return std::string(home != nullptr ? home : ".") + "/be/dict/kuromoji"; Review Comment: fixed -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
