nishant94 commented on code in PR #64667: URL: https://github.com/apache/doris/pull/64667#discussion_r3746739046
########## be/src/storage/index/inverted/analyzer/kuromoji/KuromojiTokenizer.cpp: ########## @@ -0,0 +1,164 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/inverted/analyzer/kuromoji/KuromojiTokenizer.h" + +#include <algorithm> +#include <string_view> + +#include "common/exception.h" +#include "storage/index/inverted/analyzer/kuromoji/kuromoji_normalize.h" + +namespace doris::segment_v2 { + +namespace { +// Returns the idx-th comma-separated field of an IPADIC feature string +// (0=POS1 ... 6=base form, 7=reading, 8=pronunciation), or empty. +std::string_view feature_field(std::string_view feat, int idx) { + int cur = 0; + std::size_t start = 0; + for (std::size_t i = 0; i <= feat.size(); ++i) { + if (i == feat.size() || feat[i] == ',') { + if (cur == idx) { + return feat.substr(start, i - start); + } + ++cur; + start = i + 1; + } + } + return {}; +} + +// Part-of-speech (POS1) classes dropped for full-text search. A coarse subset of +// Lucene/OpenSearch's ja stoptags: particles, auxiliary verbs, conjunctions, +// symbols, fillers. (Full stoptags fidelity is a later refinement.) +bool is_stop_pos(std::string_view pos1) { + return pos1 == "\xE5\x8A\xA9\xE8\xA9\x9E" || // 助詞 (particle) + pos1 == "\xE5\x8A\xA9\xE5\x8B\x95\xE8\xA9\x9E" || // 助動詞 (auxiliary verb) + pos1 == "\xE6\x8E\xA5\xE7\xB6\x9A\xE8\xA9\x9E" || // 接続詞 (conjunction) + pos1 == "\xE8\xA8\x98\xE5\x8F\xB7" || // 記号 (symbol) + pos1 == "\xE3\x83\x95\xE3\x82\xA3\xE3\x83\xA9\xE3\x83\xBC"; // フィラー (filler) +} + +void ascii_lower(std::string& s) { + for (char& c : s) { + if (c >= 'A' && c <= 'Z') { + c = static_cast<char>(c - 'A' + 'a'); + } + } +} + +// Decode the first UTF-8 code point of text[start, start+len]. +char32_t first_codepoint(std::string_view text, uint32_t start, uint32_t len) { + if (len == 0 || start >= text.size()) { + return 0; + } + const auto b0 = static_cast<unsigned char>(text[start]); + const std::size_t avail = std::min<std::size_t>(len, text.size() - start); + auto cont = [&](uint32_t i) { return static_cast<unsigned char>(text[start + i]) & 0x3FU; }; + if (b0 < 0x80) { + return b0; + } + if ((b0 >> 5) == 0x6 && avail >= 2) { + return static_cast<char32_t>(((b0 & 0x1FU) << 6) | cont(1)); + } + if ((b0 >> 4) == 0xE && avail >= 3) { + return static_cast<char32_t>(((b0 & 0x0FU) << 12) | (cont(1) << 6) | cont(2)); + } + if ((b0 >> 3) == 0x1E && avail >= 4) { + return static_cast<char32_t>(((b0 & 0x07U) << 18) | (cont(1) << 12) | (cont(2) << 6) | + cont(3)); + } + return b0; +} +} // namespace + +KuromojiTokenizer::KuromojiTokenizer(KuromojiMode mode, bool lower_case, bool own_reader, + const inverted_index::kuromoji::KuromojiDictionary* dict) + : mode_(mode), dict_(dict) { + this->lowercase = lower_case; + this->ownReader = own_reader; +} + +void KuromojiTokenizer::reset(lucene::util::Reader* reader) { + this->input = reader; + buffer_index_ = 0; + data_length_ = 0; + tokens_text_.clear(); + + // Read the entire input. readCopy returns the count read, or <= 0 at EOF. + std::string text; + char buf[4096]; + int32_t n = 0; + while ((n = reader->readCopy(buf, 0, static_cast<int32_t>(sizeof(buf)))) > 0) { + text.append(buf, n); + } + + if (dict_ == nullptr) { + throw doris::Exception(doris::ErrorCode::INVERTED_INDEX_ANALYZER_ERROR, + "kuromoji tokenizer requires a loaded dictionary"); + } + + // Viterbi morphological segmentation, then OpenSearch-default-style filtering: + // drop stop part-of-speech (particles/auxiliaries/...), emit the dictionary + // base form for conjugated words, and lowercase embedded ASCII. + inverted_index::kuromoji::KuromojiViterbi viterbi(*dict_, mode_); + std::vector<inverted_index::kuromoji::KuromojiMorpheme> morphemes; + viterbi.segment(text, &morphemes); + tokens_text_.reserve(morphemes.size()); + for (const auto& m : morphemes) { + if (!m.known) { + const auto cat = dict_->char_category(first_codepoint(text, m.byte_start, m.byte_len)); + if (cat == inverted_index::kuromoji::CAT_SPACE || + cat == inverted_index::kuromoji::CAT_SYMBOL) { + continue; + } + } + const std::string_view feat = + m.known ? dict_->feature(dict_->word(m.word_id)) + : dict_->unknown_feature(dict_->unknown_word(m.word_id)); + if (is_stop_pos(feature_field(feat, 0))) { + continue; // part-of-speech stop filtering + } + const std::string_view base = feature_field(feat, 6); + std::string term = (base.empty() || base == "*") ? text.substr(m.byte_start, m.byte_len) + : std::string(base); + term = inverted_index::kuromoji::cjk_width_normalize( + term); // full-width ASCII -> ASCII before lowercase + if (this->lowercase) { + ascii_lower(term); + } + if (!term.empty()) { + tokens_text_.push_back(std::move(term)); + } + } + data_length_ = static_cast<int32_t>(tokens_text_.size()); +} + +Token* KuromojiTokenizer::next(Token* token) { + if (buffer_index_ >= data_length_) { + return nullptr; + } + std::string& token_text = tokens_text_[buffer_index_++]; + // reset() already segmented and normalized the terms; hand them out one at a + // time, capped at the CLucene maximum term length. + size_t size = std::min(token_text.size(), static_cast<size_t>(LUCENE_MAX_WORD_LEN)); Review Comment: This truncation is framework-wide, not kuromoji-specific: IKTokenizer, basic_tokenizer, icu_tokenizer, and pinyin_tokenizer all do the identical std::min(size, LUCENE_MAX_WORD_LEN) + setNoCopy, so the mid-UTF8/prefix-aliasing behavior is the same for every analyzer. No changes required for this. ########## regression-test/suites/inverted_index_p0/analyzer/test_japanese_analyzer.groovy: ########## @@ -0,0 +1,84 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_japanese_analyzer", "p0") { + def tableName = "test_japanese_analyzer" + + def backendId_to_backendIP = [:] + def backendId_to_backendHttpPort = [:] + getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort) + def set_be_config = { key, value -> + for (String backend_id : backendId_to_backendIP.keySet()) { + update_be_config(backendId_to_backendIP.get(backend_id), + backendId_to_backendHttpPort.get(backend_id), key, value) + } + } + + sql "DROP TABLE IF EXISTS ${tableName}" + // kuromoji is disabled by default; enable it for this test. + set_be_config("enable_kuromoji_analyzer", "true") + try { + sql """ + CREATE TABLE ${tableName} ( + `id` int(11) NULL COMMENT "", + `content` text NULL COMMENT "", + INDEX content_idx (`content`) USING INVERTED PROPERTIES("parser" = "kuromoji", "parser_mode" = "search") COMMENT '', + ) ENGINE=OLAP + DUPLICATE KEY(`id`) + COMMENT "OLAP" + DISTRIBUTED BY RANDOM BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + + sql """ INSERT INTO ${tableName} VALUES (1, "東京都に住んでいます"); """ + sql """ INSERT INTO ${tableName} VALUES (2, "私は寿司が好きです"); """ + sql """ INSERT INTO ${tableName} VALUES (3, "Apache Doris は高速です"); """ + sql "sync" + + // The kuromoji IPADIC dictionary ships with the package (built by the + // kuromoji_dict target), so these queries exercise real morphological + // analysis on the deterministic dictionary output. + + // Search mode decomposes the compound 東京都 into 東京 + 都, so a 東京 query + // matches row 1. + qt_tokyo """ SELECT id FROM ${tableName} WHERE content MATCH '東京' ORDER BY id """ + + // The full compound 東京都 still matches row 1: query-time analysis applies + // the same search-mode decomposition, so 東京都 -> 東京 + 都 matches the + // indexed parts. (Decomposition does not drop compound recall.) + qt_compound """ SELECT id FROM ${tableName} WHERE content MATCH '東京都' ORDER BY id """ + + // 寿司 is segmented as its own morpheme in 私は寿司が好きです. + qt_sushi """ SELECT id FROM ${tableName} WHERE content MATCH '寿司' ORDER BY id """ + + // Base-form normalization: the conjugated 住ん(でいます) is indexed under its + // dictionary base form 住む, so a 住む query matches row 1. + qt_live """ SELECT id FROM ${tableName} WHERE content MATCH '住む' ORDER BY id """ + + // Directly show search mode emits the 東京 part of the 東京都 compound. + // (A contains-check rather than qt_: the full TOKENIZE JSON pins byte + // offsets/positions that are not the point of this assertion.) + def tokens = sql """SELECT TOKENIZE('東京都', '"parser"="kuromoji","parser_mode"="search"');""" + def tokenStr = tokens[0][0].toString() + assertTrue(tokenStr.contains('"token": "東京"')) + } finally { + sql "DROP TABLE IF EXISTS ${tableName}" + set_be_config("enable_kuromoji_analyzer", "false") Review Comment: Fixed -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
