github-actions[bot] commented on code in PR #64667:
URL: https://github.com/apache/doris/pull/64667#discussion_r3743122933
##########
be/CMakeLists.txt:
##########
@@ -966,6 +986,41 @@ if (BUILD_META_TOOL OR BUILD_INDEX_TOOL)
add_subdirectory(${SRC_DIR}/tools)
endif()
+if (NOT MAKE_TEST)
+ # Offline generator: compiles the UTF-8 mecab-ipadic source into binary
files.
+ add_executable(kuromoji_build_dict EXCLUDE_FROM_ALL
${SRC_DIR}/tools/kuromoji_build_dict.cpp)
+ target_include_directories(kuromoji_build_dict PRIVATE
${PROJECT_SOURCE_DIR}/..)
+ pch_reuse(kuromoji_build_dict)
+ set_target_properties(kuromoji_build_dict PROPERTIES ENABLE_EXPORTS 1)
+ if (COMPILER_CLANG)
+ target_compile_options(kuromoji_build_dict PRIVATE
+ -Wno-implicit-int-conversion
+ -Wno-shorten-64-to-32)
+ endif()
+ target_link_libraries(kuromoji_build_dict ${DORIS_LINK_LIBS})
+
+ set(KUROMOJI_IPADIC_SRC
"${THIRDPARTY_DIR}/share/mecab-ipadic-2.7.0-20250920"
+ CACHE PATH "UTF-8 mecab-ipadic source directory used to generate the
kuromoji dictionary")
+ set(KUROMOJI_DICT_OUT "${BASE_DIR}/dict/kuromoji")
+ file(GLOB KUROMOJI_IPADIC_SRC_FILES CONFIGURE_DEPENDS
+ "${KUROMOJI_IPADIC_SRC}/*.csv"
+ "${KUROMOJI_IPADIC_SRC}/*.def")
+ get_filename_component(KUROMOJI_LIBJVM_DIR "${LIB_JVM}" DIRECTORY)
+ add_custom_command(
+ OUTPUT "${KUROMOJI_DICT_OUT}/system.bin"
"${KUROMOJI_DICT_OUT}/matrix.bin"
+ "${KUROMOJI_DICT_OUT}/chardef.bin"
"${KUROMOJI_DICT_OUT}/unkdict.bin"
+ COMMAND ${CMAKE_COMMAND} -E make_directory "${KUROMOJI_DICT_OUT}"
+ COMMAND ${CMAKE_COMMAND} -E env
+ --modify
"DYLD_LIBRARY_PATH=path_list_prepend:${KUROMOJI_LIBJVM_DIR}"
Review Comment:
[Major] Keep this command compatible with the declared CMake minimum. `cmake
-E env --modify` was added in CMake 3.25, while this project still accepts
3.19.2 and `docker/compilation/Dockerfile.gcc10` pins 3.22.1. On those
supported toolchains configuration succeeds, but the `ALL` `kuromoji_dict` edge
fails before producing the four binaries because CMake does not recognize
`--modify`. Use a 3.19-compatible environment wrapper/syntax, or raise and
enforce the minimum before configuring.
##########
be/src/storage/index/inverted/analyzer/kuromoji/dict/kuromoji_dictionary.cpp:
##########
@@ -0,0 +1,326 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "storage/index/inverted/analyzer/kuromoji/dict/kuromoji_dictionary.h"
+
+#include <fcntl.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include <cstring>
+#include <limits>
+#include <map>
+#include <mutex>
+
+#include "common/logging.h"
+
+namespace doris::segment_v2::inverted_index::kuromoji {
+
+namespace {
+Status check_region(const char* what, uint64_t offset, uint64_t count,
uint64_t elem,
+ uint64_t min_offset, std::size_t size) {
+ if (elem != 0 && count > std::numeric_limits<uint64_t>::max() / elem) {
+ return Status::Corruption("kuromoji dict: {} count overflow ({} x
{})", what, count, elem);
+ }
+ const uint64_t bytes = count * elem;
+ if (offset < min_offset || offset > size || bytes >
static_cast<uint64_t>(size) - offset) {
+ return Status::Corruption("kuromoji dict: {} out of range (offset={},
bytes={}, file={})",
+ what, offset, bytes, size);
+ }
+ return Status::OK();
+}
+} // namespace
+
+MappedFile::~MappedFile() {
+ if (_data != nullptr) {
+ ::munmap(const_cast<uint8_t*>(_data), _size);
+ _data = nullptr;
+ _size = 0;
+ }
+}
+
+Status MappedFile::open(const std::string& path) {
+ int fd = ::open(path.c_str(), O_RDONLY);
+ if (fd < 0) {
+ return Status::IOError("kuromoji dict: cannot open {}", path);
+ }
+ struct stat st {};
+ if (::fstat(fd, &st) != 0 || st.st_size <= 0) {
+ ::close(fd);
+ return Status::IOError("kuromoji dict: cannot stat {}", path);
+ }
+ auto bytes = static_cast<std::size_t>(st.st_size);
+ void* m = ::mmap(nullptr, bytes, PROT_READ, MAP_PRIVATE, fd, 0);
+ ::close(fd);
+ if (m == MAP_FAILED) {
+ return Status::IOError("kuromoji dict: mmap failed for {}", path);
+ }
+ _data = static_cast<const uint8_t*>(m);
+ _size = bytes;
+ return Status::OK();
+}
+
+Status KuromojiDictionary::check_header(const uint8_t* p, std::size_t size,
KmjFileKind kind) {
+ if (size < sizeof(KmjFileHeader)) {
+ return Status::Corruption("kuromoji dict: file too small");
+ }
+ KmjFileHeader h {};
+ std::memcpy(&h, p, sizeof(h));
+ if (std::memcmp(h.magic, KMJ_MAGIC, sizeof(h.magic)) != 0) {
+ return Status::Corruption("kuromoji dict: bad magic");
+ }
+ if (h.format_version != KMJ_FORMAT_VERSION) {
+ return Status::Corruption("kuromoji dict: version {} != {}",
h.format_version,
+ KMJ_FORMAT_VERSION);
+ }
+ if (h.file_kind != static_cast<uint32_t>(kind)) {
+ return Status::Corruption("kuromoji dict: wrong file_kind {}",
h.file_kind);
+ }
+ if (h.file_size != size) {
+ return Status::Corruption("kuromoji dict: file_size {} != actual {}",
h.file_size, size);
+ }
+ return Status::OK();
+}
+
+std::string_view KuromojiDictionary::feature_at(const uint8_t* blob, uint64_t
blob_bytes,
+ uint32_t off) {
+ if (off == KMJ_NO_FEATURE || blob == nullptr || static_cast<uint64_t>(off)
+ 2 > blob_bytes) {
+ return {};
+ }
+ auto len = static_cast<uint16_t>(static_cast<uint16_t>(blob[off]) |
+ static_cast<uint16_t>(blob[off + 1] <<
8));
+ if (static_cast<uint64_t>(off) + 2 + len > blob_bytes) {
+ return {};
+ }
+ return {reinterpret_cast<const char*>(blob + off + 2), len};
+}
+
+Status KuromojiDictionary::map_system(const std::string& path) {
+ RETURN_IF_ERROR(_system_map.open(path));
+ const uint8_t* p = _system_map.data();
+ const std::size_t size = _system_map.size();
+ RETURN_IF_ERROR(check_header(p, size, KMJ_KIND_SYSTEM));
+ constexpr uint64_t kHdrEnd = sizeof(KmjFileHeader) +
sizeof(KmjSystemHeader);
+ if (size < kHdrEnd) {
+ return Status::Corruption("kuromoji dict: system.bin truncated
sub-header");
+ }
+ KmjSystemHeader s {};
+ std::memcpy(&s, p + sizeof(KmjFileHeader), sizeof(s));
+ // The trie is read as 4-byte Darts units, so both offset and length must
be
+ // 4-byte aligned/sized before set_array() walks them.
+ if (s.trie_offset % 4 != 0 || s.trie_bytes % 4 != 0) {
+ return Status::Corruption("kuromoji dict: trie not 4-byte aligned");
+ }
+ if (s.trie_bytes == 0) {
+ return Status::Corruption("kuromoji dict: system.bin has an empty
trie");
+ }
+ RETURN_IF_ERROR(check_region("system trie", s.trie_offset, s.trie_bytes,
1, kHdrEnd, size));
+ RETURN_IF_ERROR(check_region("system runs", s.runs_offset, s.runs_count,
sizeof(WordIdRun),
+ kHdrEnd, size));
+ RETURN_IF_ERROR(check_region("system entries", s.entries_offset,
s.entries_count,
+ sizeof(WordEntry), kHdrEnd, size));
+ RETURN_IF_ERROR(
+ check_region("system features", s.features_offset,
s.features_bytes, 1, kHdrEnd, size));
+ _runs = reinterpret_cast<const WordIdRun*>(p + s.runs_offset);
+ _runs_count = s.runs_count;
+ _entries = reinterpret_cast<const WordEntry*>(p + s.entries_offset);
+ _entries_count = s.entries_count;
+ _features = p + s.features_offset;
+ _features_bytes = s.features_bytes;
+ // trie_bytes is non-zero and in 4-byte units; the mmap
+ // outlives _trie (both owned by this object).
+ _trie.set_array(p + s.trie_offset, static_cast<std::size_t>(s.trie_bytes /
4));
+ return Status::OK();
+}
+
+Status KuromojiDictionary::map_matrix(const std::string& path) {
+ RETURN_IF_ERROR(_matrix_map.open(path));
+ const uint8_t* p = _matrix_map.data();
+ const std::size_t size = _matrix_map.size();
+ RETURN_IF_ERROR(check_header(p, size, KMJ_KIND_MATRIX));
+ constexpr uint64_t kHdrEnd = sizeof(KmjFileHeader) +
sizeof(KmjMatrixHeader);
+ if (size < kHdrEnd) {
+ return Status::Corruption("kuromoji dict: matrix.bin truncated
sub-header");
+ }
+ KmjMatrixHeader m {};
+ std::memcpy(&m, p + sizeof(KmjFileHeader), sizeof(m));
+ if (m.forward_size == 0 || m.backward_size == 0) {
+ return Status::Corruption("kuromoji dict: matrix has a zero
dimension");
+ }
+ const uint64_t cells = static_cast<uint64_t>(m.forward_size) *
m.backward_size;
+ RETURN_IF_ERROR(
+ check_region("matrix cells", m.cells_offset, cells,
sizeof(int16_t), kHdrEnd, size));
+ _forward_size = m.forward_size;
+ _backward_size = m.backward_size;
+ _cells = reinterpret_cast<const int16_t*>(p + m.cells_offset);
+ return Status::OK();
+}
+
+Status KuromojiDictionary::map_chardef(const std::string& path) {
+ RETURN_IF_ERROR(_chardef_map.open(path));
+ const uint8_t* p = _chardef_map.data();
+ const std::size_t size = _chardef_map.size();
+ RETURN_IF_ERROR(check_header(p, size, KMJ_KIND_CHARDEF));
+ constexpr uint64_t kHdrEnd = sizeof(KmjFileHeader) +
sizeof(KmjCharDefHeader);
+ if (size < kHdrEnd) {
+ return Status::Corruption("kuromoji dict: chardef.bin truncated
sub-header");
+ }
+ KmjCharDefHeader c {};
+ std::memcpy(&c, p + sizeof(KmjFileHeader), sizeof(c));
+ if (c.class_count != CAT_CLASS_COUNT) {
+ return Status::Corruption("kuromoji dict: chardef class_count {} !=
{}", c.class_count,
+ static_cast<uint32_t>(CAT_CLASS_COUNT));
+ }
+ // catmap is exactly one byte per BMP code point.
+ RETURN_IF_ERROR(check_region("chardef catmap", c.catmap_offset, 0x10000,
1, kHdrEnd, size));
+ RETURN_IF_ERROR(check_region("chardef defs", c.defs_offset, c.class_count,
sizeof(CategoryDef),
+ kHdrEnd, size));
+ _catmap = p + c.catmap_offset;
+ _defs = reinterpret_cast<const CategoryDef*>(p + c.defs_offset);
+ return Status::OK();
+}
+
+Status KuromojiDictionary::map_unkdict(const std::string& path) {
+ RETURN_IF_ERROR(_unk_map.open(path));
+ const uint8_t* p = _unk_map.data();
+ const std::size_t size = _unk_map.size();
+ RETURN_IF_ERROR(check_header(p, size, KMJ_KIND_UNKDICT));
+ constexpr uint64_t kHdrEnd = sizeof(KmjFileHeader) + sizeof(KmjUnkHeader);
+ if (size < kHdrEnd) {
+ return Status::Corruption("kuromoji dict: unkdict.bin truncated
sub-header");
+ }
+ KmjUnkHeader u {};
+ std::memcpy(&u, p + sizeof(KmjFileHeader), sizeof(u));
+ if (u.class_count != CAT_CLASS_COUNT) {
+ return Status::Corruption("kuromoji dict: unkdict class_count {} !=
{}", u.class_count,
+ static_cast<uint32_t>(CAT_CLASS_COUNT));
+ }
+ RETURN_IF_ERROR(check_region("unk runs", u.runs_offset, u.class_count,
sizeof(WordIdRun),
+ kHdrEnd, size));
+ RETURN_IF_ERROR(check_region("unk entries", u.entries_offset,
u.entries_count,
+ sizeof(WordEntry), kHdrEnd, size));
+ RETURN_IF_ERROR(
+ check_region("unk features", u.features_offset, u.features_bytes,
1, kHdrEnd, size));
+ _unk_runs = reinterpret_cast<const WordIdRun*>(p + u.runs_offset);
+ _unk_runs_count = u.class_count;
+ _unk_entries = reinterpret_cast<const WordEntry*>(p + u.entries_offset);
+ _unk_entries_count = u.entries_count;
+ _unk_features = p + u.features_offset;
+ _unk_features_bytes = u.features_bytes;
+ return Status::OK();
+}
+
+Status KuromojiDictionary::validate_ranges() const {
+ // Every run must reference a valid [entry_start, entry_start + count)
slice.
+ auto check_runs = [](const WordIdRun* runs, uint64_t run_count, uint64_t
entries_count,
+ const char* what) -> Status {
+ for (uint64_t i = 0; i < run_count; ++i) {
+ if (static_cast<uint64_t>(runs[i].entry_start) + runs[i].count >
entries_count) {
+ return Status::Corruption(
+ "kuromoji dict: {} run {} references entries past the
end", what, i);
+ }
+ }
+ return Status::OK();
+ };
+ // Every entry's context ids must index the connection matrix (used
directly
+ // as offsets into _cells at query time).
+ auto check_entries = [this](const WordEntry* entries, uint64_t count,
+ const char* what) -> Status {
+ for (uint64_t i = 0; i < count; ++i) {
+ const WordEntry& e = entries[i];
+ if (e.left_id < 0 || static_cast<uint32_t>(e.left_id) >=
_backward_size ||
+ e.right_id < 0 || static_cast<uint32_t>(e.right_id) >=
_forward_size) {
+ return Status::Corruption("kuromoji dict: {} entry {} has
out-of-range context id",
+ what, i);
+ }
+ }
+ return Status::OK();
+ };
+ RETURN_IF_ERROR(check_runs(_runs, _runs_count, _entries_count, "system"));
+ RETURN_IF_ERROR(check_entries(_entries, _entries_count, "system"));
+ RETURN_IF_ERROR(check_runs(_unk_runs, _unk_runs_count, _unk_entries_count,
"unk"));
+ RETURN_IF_ERROR(check_entries(_unk_entries, _unk_entries_count, "unk"));
+ for (uint32_t cp = 0; cp < 0x10000; ++cp) {
+ if (_catmap[cp] >= CAT_CLASS_COUNT) {
+ return Status::Corruption(
+ "kuromoji dict: chardef catmap has out-of-range category
{} at code point {}",
+ static_cast<uint32_t>(_catmap[cp]), cp);
+ }
+ }
+ if (_unk_entries_count == 0) {
+ return Status::Corruption("kuromoji dict: unknown dictionary has no
entries");
+ }
+ return Status::OK();
+}
+
+Status KuromojiDictionary::load(const std::string& dir,
std::unique_ptr<KuromojiDictionary>* out) {
+ auto dict = std::make_unique<KuromojiDictionary>();
+ RETURN_IF_ERROR(dict->map_system(dir + "/system.bin"));
+ RETURN_IF_ERROR(dict->map_matrix(dir + "/matrix.bin"));
+ RETURN_IF_ERROR(dict->map_chardef(dir + "/chardef.bin"));
+ RETURN_IF_ERROR(dict->map_unkdict(dir + "/unkdict.bin"));
+ // Cross-file checks need every file mapped (entries vs. matrix bounds).
+ RETURN_IF_ERROR(dict->validate_ranges());
+ *out = std::move(dict);
+ return Status::OK();
+}
+
+const KuromojiDictionary* KuromojiDictionary::get_or_load(const std::string&
dir) {
+ static std::mutex mu;
+ static std::map<std::string, std::unique_ptr<KuromojiDictionary>> cache;
+ std::lock_guard<std::mutex> lock(mu);
+ auto it = cache.find(dir);
+ if (it != cache.end()) {
+ return it->second.get(); // may be nullptr if a prior load failed
+ }
+ std::unique_ptr<KuromojiDictionary> dict;
+ Status st = load(dir, &dict);
+ if (!st.ok()) {
+ LOG(WARNING) << "kuromoji: failed to load dictionary from " << dir <<
": " << st;
+ cache.emplace(dir, nullptr);
Review Comment:
[Major] Do not cache a failed dictionary load for the lifetime of the BE. If
the first request observes a missing, corrupt, or temporarily unavailable file,
this inserts `nullptr` for the directory; every later analyzer creation returns
that cached failure even after all four valid artifacts are repaired or
restored at the same path. The mutable feature gate makes first access after
startup a supported lifecycle, but neither toggling it nor repairing the files
can recover this BE. Cache successful immutable dictionaries only, or give
negative entries a bounded retry/invalidation path, and cover
fail-then-install-then-retry at one directory.
##########
regression-test/suites/inverted_index_p0/analyzer/test_japanese_analyzer.groovy:
##########
@@ -0,0 +1,84 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_japanese_analyzer", "p0") {
+ def tableName = "test_japanese_analyzer"
+
+ def backendId_to_backendIP = [:]
+ def backendId_to_backendHttpPort = [:]
+ getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort)
+ def set_be_config = { key, value ->
+ for (String backend_id : backendId_to_backendIP.keySet()) {
+ update_be_config(backendId_to_backendIP.get(backend_id),
+ backendId_to_backendHttpPort.get(backend_id),
key, value)
+ }
+ }
+
+ sql "DROP TABLE IF EXISTS ${tableName}"
+ // kuromoji is disabled by default; enable it for this test.
+ set_be_config("enable_kuromoji_analyzer", "true")
+ try {
+ sql """
+ CREATE TABLE ${tableName} (
+ `id` int(11) NULL COMMENT "",
+ `content` text NULL COMMENT "",
+ INDEX content_idx (`content`) USING INVERTED PROPERTIES("parser" =
"kuromoji", "parser_mode" = "search") COMMENT '',
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`id`)
+ COMMENT "OLAP"
+ DISTRIBUTED BY RANDOM BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1"
+ );
+ """
+
+ sql """ INSERT INTO ${tableName} VALUES (1, "東京都に住んでいます"); """
+ sql """ INSERT INTO ${tableName} VALUES (2, "私は寿司が好きです"); """
+ sql """ INSERT INTO ${tableName} VALUES (3, "Apache Doris は高速です"); """
+ sql "sync"
+
+ // The kuromoji IPADIC dictionary ships with the package (built by the
+ // kuromoji_dict target), so these queries exercise real morphological
+ // analysis on the deterministic dictionary output.
+
+ // Search mode decomposes the compound 東京都 into 東京 + 都, so a 東京 query
+ // matches row 1.
+ qt_tokyo """ SELECT id FROM ${tableName} WHERE content MATCH '東京'
ORDER BY id """
+
+ // The full compound 東京都 still matches row 1: query-time analysis
applies
+ // the same search-mode decomposition, so 東京都 -> 東京 + 都 matches the
+ // indexed parts. (Decomposition does not drop compound recall.)
+ qt_compound """ SELECT id FROM ${tableName} WHERE content MATCH '東京都'
ORDER BY id """
+
+ // 寿司 is segmented as its own morpheme in 私は寿司が好きです.
+ qt_sushi """ SELECT id FROM ${tableName} WHERE content MATCH '寿司'
ORDER BY id """
+
+ // Base-form normalization: the conjugated 住ん(でいます) is indexed under
its
+ // dictionary base form 住む, so a 住む query matches row 1.
+ qt_live """ SELECT id FROM ${tableName} WHERE content MATCH '住む' ORDER
BY id """
+
+ // Directly show search mode emits the 東京 part of the 東京都 compound.
+ // (A contains-check rather than qt_: the full TOKENIZE JSON pins byte
+ // offsets/positions that are not the point of this assertion.)
+ def tokens = sql """SELECT TOKENIZE('東京都',
'"parser"="kuromoji","parser_mode"="search"');"""
+ def tokenStr = tokens[0][0].toString()
+ assertTrue(tokenStr.contains('"token": "東京"'))
+ } finally {
+ sql "DROP TABLE IF EXISTS ${tableName}"
+ set_be_config("enable_kuromoji_analyzer", "false")
Review Comment:
[Minor] Restore each backend's original `enable_kuromoji_analyzer` value
instead of forcing `false` here. The setting is mutable and `true` is a valid
pre-suite state (with different BEs potentially configured differently), so a
successful or failed run currently changes the shared cluster for later suites.
Snapshot the value per BE before enabling it, check each update result, and
restore those values in `finally`.
##########
be/src/storage/index/inverted/analyzer/kuromoji/KuromojiTokenizer.cpp:
##########
@@ -0,0 +1,164 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "storage/index/inverted/analyzer/kuromoji/KuromojiTokenizer.h"
+
+#include <algorithm>
+#include <string_view>
+
+#include "common/exception.h"
+#include "storage/index/inverted/analyzer/kuromoji/kuromoji_normalize.h"
+
+namespace doris::segment_v2 {
+
+namespace {
+// Returns the idx-th comma-separated field of an IPADIC feature string
+// (0=POS1 ... 6=base form, 7=reading, 8=pronunciation), or empty.
+std::string_view feature_field(std::string_view feat, int idx) {
+ int cur = 0;
+ std::size_t start = 0;
+ for (std::size_t i = 0; i <= feat.size(); ++i) {
+ if (i == feat.size() || feat[i] == ',') {
+ if (cur == idx) {
+ return feat.substr(start, i - start);
+ }
+ ++cur;
+ start = i + 1;
+ }
+ }
+ return {};
+}
+
+// Part-of-speech (POS1) classes dropped for full-text search. A coarse subset
of
+// Lucene/OpenSearch's ja stoptags: particles, auxiliary verbs, conjunctions,
+// symbols, fillers. (Full stoptags fidelity is a later refinement.)
+bool is_stop_pos(std::string_view pos1) {
+ return pos1 == "\xE5\x8A\xA9\xE8\xA9\x9E" || // 助詞
(particle)
+ pos1 == "\xE5\x8A\xA9\xE5\x8B\x95\xE8\xA9\x9E" || // 助動詞
(auxiliary verb)
+ pos1 == "\xE6\x8E\xA5\xE7\xB6\x9A\xE8\xA9\x9E" || // 接続詞
(conjunction)
+ pos1 == "\xE8\xA8\x98\xE5\x8F\xB7" || // 記号
(symbol)
+ pos1 == "\xE3\x83\x95\xE3\x82\xA3\xE3\x83\xA9\xE3\x83\xBC"; // フィラー
(filler)
+}
+
+void ascii_lower(std::string& s) {
+ for (char& c : s) {
+ if (c >= 'A' && c <= 'Z') {
+ c = static_cast<char>(c - 'A' + 'a');
+ }
+ }
+}
+
+// Decode the first UTF-8 code point of text[start, start+len].
+char32_t first_codepoint(std::string_view text, uint32_t start, uint32_t len) {
+ if (len == 0 || start >= text.size()) {
+ return 0;
+ }
+ const auto b0 = static_cast<unsigned char>(text[start]);
+ const std::size_t avail = std::min<std::size_t>(len, text.size() - start);
+ auto cont = [&](uint32_t i) { return static_cast<unsigned char>(text[start
+ i]) & 0x3FU; };
+ if (b0 < 0x80) {
+ return b0;
+ }
+ if ((b0 >> 5) == 0x6 && avail >= 2) {
+ return static_cast<char32_t>(((b0 & 0x1FU) << 6) | cont(1));
+ }
+ if ((b0 >> 4) == 0xE && avail >= 3) {
+ return static_cast<char32_t>(((b0 & 0x0FU) << 12) | (cont(1) << 6) |
cont(2));
+ }
+ if ((b0 >> 3) == 0x1E && avail >= 4) {
+ return static_cast<char32_t>(((b0 & 0x07U) << 18) | (cont(1) << 12) |
(cont(2) << 6) |
+ cont(3));
+ }
+ return b0;
+}
+} // namespace
+
+KuromojiTokenizer::KuromojiTokenizer(KuromojiMode mode, bool lower_case, bool
own_reader,
+ const
inverted_index::kuromoji::KuromojiDictionary* dict)
+ : mode_(mode), dict_(dict) {
+ this->lowercase = lower_case;
+ this->ownReader = own_reader;
+}
+
+void KuromojiTokenizer::reset(lucene::util::Reader* reader) {
+ this->input = reader;
+ buffer_index_ = 0;
+ data_length_ = 0;
+ tokens_text_.clear();
+
+ // Read the entire input. readCopy returns the count read, or <= 0 at EOF.
+ std::string text;
+ char buf[4096];
+ int32_t n = 0;
+ while ((n = reader->readCopy(buf, 0, static_cast<int32_t>(sizeof(buf)))) >
0) {
+ text.append(buf, n);
+ }
+
+ if (dict_ == nullptr) {
+ throw doris::Exception(doris::ErrorCode::INVERTED_INDEX_ANALYZER_ERROR,
+ "kuromoji tokenizer requires a loaded
dictionary");
+ }
+
+ // Viterbi morphological segmentation, then OpenSearch-default-style
filtering:
+ // drop stop part-of-speech (particles/auxiliaries/...), emit the
dictionary
+ // base form for conjugated words, and lowercase embedded ASCII.
+ inverted_index::kuromoji::KuromojiViterbi viterbi(*dict_, mode_);
+ std::vector<inverted_index::kuromoji::KuromojiMorpheme> morphemes;
+ viterbi.segment(text, &morphemes);
+ tokens_text_.reserve(morphemes.size());
+ for (const auto& m : morphemes) {
+ if (!m.known) {
+ const auto cat = dict_->char_category(first_codepoint(text,
m.byte_start, m.byte_len));
+ if (cat == inverted_index::kuromoji::CAT_SPACE ||
+ cat == inverted_index::kuromoji::CAT_SYMBOL) {
+ continue;
+ }
+ }
+ const std::string_view feat =
+ m.known ? dict_->feature(dict_->word(m.word_id))
+ :
dict_->unknown_feature(dict_->unknown_word(m.word_id));
+ if (is_stop_pos(feature_field(feat, 0))) {
+ continue; // part-of-speech stop filtering
+ }
+ const std::string_view base = feature_field(feat, 6);
+ std::string term = (base.empty() || base == "*") ?
text.substr(m.byte_start, m.byte_len)
+ : std::string(base);
+ term = inverted_index::kuromoji::cjk_width_normalize(
+ term); // full-width ASCII -> ASCII before lowercase
+ if (this->lowercase) {
+ ascii_lower(term);
+ }
+ if (!term.empty()) {
+ tokens_text_.push_back(std::move(term));
+ }
+ }
+ data_length_ = static_cast<int32_t>(tokens_text_.size());
+}
+
+Token* KuromojiTokenizer::next(Token* token) {
+ if (buffer_index_ >= data_length_) {
+ return nullptr;
+ }
+ std::string& token_text = tokens_text_[buffer_index_++];
+ // reset() already segmented and normalized the terms; hand them out one
at a
+ // time, capped at the CLucene maximum term length.
+ size_t size = std::min(token_text.size(),
static_cast<size_t>(LUCENE_MAX_WORD_LEN));
Review Comment:
[Major] Do not silently truncate a Kuromoji morpheme to 255 bytes. Grouped
OOV tokens can be up to 1,024 code points, so two terms with the same first 255
bytes but different suffixes become identical in both the index and query
analyzer, creating false MATCH-family hits; a Japanese/supplementary token can
also be cut in the middle of a UTF-8 code point and leak malformed output
through TOKENIZE. Reject/drop over-limit terms under one explicit index/query
policy, or split them on validated UTF-8 boundaries, and test differing long
suffixes plus a multibyte boundary.
##########
be/dict/kuromoji/README.md:
##########
@@ -0,0 +1,55 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Kuromoji (Japanese) dictionary
+
+This directory holds the compiled IPADIC dictionary consumed at runtime by the
+`kuromoji` inverted-index analyzer (`KuromojiAnalyzer` → `KuromojiDictionary`):
+
+- `system.bin` — surface→word Darts trie + word entries + feature blob
+- `matrix.bin` — connection-cost matrix (1316×1316)
+- `chardef.bin` — character-category map + per-category flags
+- `unkdict.bin` — unknown-word entries per category
+
+These `*.bin` files are **generated** (not committed; see `.gitignore`). The
+runtime resolves them at `${inverted_index_dict_path}/kuromoji`
+(default `${DORIS_HOME}/dict/kuromoji`); `be/CMakeLists.txt` installs this
+directory into the BE package.
+
+## How it's (re)generated
+
+Source: the UTF-8 IPADIC from <https://github.com/lindera/mecab-ipadic>
+(tag `2.7.0-20250920`) — the original `mecab-ipadic-2.7.0-20070801` lexicon
+converted to UTF-8 (license: NAIST-2003, see
`dist/licenses/LICENSE-ipadic.txt`).
+
+Automated, two steps:
+
+```bash
+# 1. thirdparty fetches + stages the UTF-8 IPADIC source into
+# ${DORIS_THIRDPARTY}/installed/share/mecab-ipadic-2.7.0-20250920
+sh thirdparty/build-thirdparty.sh mecab_ipadic
+
+# 2. the CMake target builds the offline compiler and produces the *.bin here
+ninja -C be/ut_build_RELEASE kuromoji_dict
Review Comment:
[Minor] Point this command at a build tree that actually defines
`kuromoji_dict`. The named `be/ut_build_RELEASE` directory is configured by
`run-be-ut.sh` with `MAKE_TEST=ON`, while `be/CMakeLists.txt` creates both
Kuromoji generator targets only under `if (NOT MAKE_TEST)`, so following these
checked-in regeneration instructions fails with an unknown target. Document a
normal `MAKE_TEST=OFF` build directory, or intentionally expose the target in
the documented graph.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
