This is an automated email from the ASF dual-hosted git repository.

airborne12 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/master by this push:
     new 2b64096faf4 [improvement](be) Move jieba into the Doris tree and cut 
chinese text over a flat dictionary (#70266)
2b64096faf4 is described below

commit 2b64096faf4d796d5f836a88f6a22f2d4588c0ef
Author: Jack <[email protected]>
AuthorDate: Sat Nov 21 19:10:44 2026 +0800

    [improvement](be) Move jieba into the Doris tree and cut chinese text over 
a flat dictionary (#70266)
    
    ### What problem does this PR solve?
    
    Issue Number: None
    
    Related PR: #70265 (depends on)
    
    Problem Summary: Move jieba into the Doris tree for the chinese parser
    and replace its trie of hash maps with flat arrays: 3.2x (coarse) and
    2.3x (fine) as fast, less memory, the same tokens.
---
 .licenserc.yaml                                    |   5 +
 LICENSE.txt                                        |  19 +
 .../index/inverted/analysis/analyzer/analyzer.cpp  |  58 +--
 .../inverted/analysis/tokenizer/jieba/dict_trie.h  | 200 +++++++++
 .../inverted/analysis/tokenizer/jieba/flat_dict.h  | 349 +++++++++++++++
 .../inverted/analysis/tokenizer/jieba/hmm_model.h  | 160 +++++++
 .../inverted/analysis/tokenizer/jieba/jieba.h      |  85 ++++
 .../analysis/tokenizer/jieba/jieba_scratch.h       |  61 +++
 .../analysis/tokenizer/jieba/jieba_tokenizer.cpp   | 112 +++++
 .../analysis/tokenizer/jieba/jieba_tokenizer.h     | 102 +++++
 .../inverted/analysis/tokenizer/jieba/segmenter.h  | 408 +++++++++++++++++
 .../analysis/tokenizer/jieba/string_util.h         |  70 +++
 .../inverted/analysis/tokenizer/jieba/unicode.h    |  36 ++
 .../inverted/analysis/component_stream_test.cpp    |  13 +
 .../analysis/tokenizer/jieba_tokenizer_test.cpp    | 496 +++++++++++++++++++++
 build-support/check-header-deps.py                 |   4 -
 dist/LICENSE-dist.txt                              |   2 +
 dist/licenses/LICENSE-cppjieba.txt                 |  20 +
 dist/licenses/LICENSE-limonp.txt                   |  20 +
 19 files changed, 2167 insertions(+), 53 deletions(-)

diff --git a/.licenserc.yaml b/.licenserc.yaml
index 5cc8026f6e4..f8ae024a78f 100644
--- a/.licenserc.yaml
+++ b/.licenserc.yaml
@@ -135,6 +135,11 @@ header:
     - "be/src/util/sse2neo.h"
     - "be/src/util/sse2neon.h"
     - "be/src/storage/index/inverted/analysis/analyzer/kuromoji/dict/darts.h"
+    - "be/src/storage/index/inverted/analysis/tokenizer/jieba/dict_trie.h"
+    - "be/src/storage/index/inverted/analysis/tokenizer/jieba/hmm_model.h"
+    - "be/src/storage/index/inverted/analysis/tokenizer/jieba/segmenter.h"
+    - "be/src/storage/index/inverted/analysis/tokenizer/jieba/string_util.h"
+    - "be/src/storage/index/inverted/analysis/tokenizer/jieba/unicode.h"
     - "be/src/pch/*"
     - "be/test/data"
     - "be/test/expected_result"
diff --git a/LICENSE.txt b/LICENSE.txt
index b54c0b65503..5f50887ef91 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -378,6 +378,25 @@ 
be/src/storage/index/inverted/analysis/analyzer/kuromoji/dict/darts.h : 2-Clause
 
 
--------------------------------------------------------------------------------
 
+be/src/storage/index/inverted/analysis/tokenizer/jieba/dict_trie.h
+be/src/storage/index/inverted/analysis/tokenizer/jieba/hmm_model.h
+be/src/storage/index/inverted/analysis/tokenizer/jieba/segmenter.h
+be/src/storage/index/inverted/analysis/tokenizer/jieba/unicode.h : MIT license 
(cppjieba)
+
+  Copyright (c) 2013
+  Derived from https://github.com/yanyiwu/cppjieba
+  Full text: dist/licenses/LICENSE-cppjieba.txt
+
+--------------------------------------------------------------------------------
+
+be/src/storage/index/inverted/analysis/tokenizer/jieba/string_util.h : MIT 
license (limonp)
+
+  Copyright (c) 2013
+  Derived from https://github.com/yanyiwu/limonp
+  Full text: dist/licenses/LICENSE-limonp.txt
+
+--------------------------------------------------------------------------------
+
 be/src/util/murmur_hash3.h
 be/src/util/murmur_hash3.cpp
 
diff --git a/be/src/storage/index/inverted/analysis/analyzer/analyzer.cpp 
b/be/src/storage/index/inverted/analysis/analyzer/analyzer.cpp
index 1a837527954..ef7b1eefdc5 100644
--- a/be/src/storage/index/inverted/analysis/analyzer/analyzer.cpp
+++ b/be/src/storage/index/inverted/analysis/analyzer/analyzer.cpp
@@ -19,21 +19,6 @@
 
 #include <glog/logging.h>
 
-#include "CLucene.h"
-#include "CLucene/analysis/LanguageBasedAnalyzer.h"
-
-#ifdef __clang__
-#pragma clang diagnostic push
-#pragma clang diagnostic ignored "-Wshadow-field"
-#endif
-// clang-format off
-#include "common/compile_check_avoid_begin.h"
-#include "CLucene/analysis/standard95/StandardAnalyzer.h"
-#include "common/compile_check_avoid_end.h"
-// clang-format on
-#ifdef __clang__
-#pragma clang diagnostic pop
-#endif
 #include "runtime/exec_env.h"
 #include "runtime/index_policy/index_policy_mgr.h"
 #include "storage/index/inverted/analysis/analyzer/chain_analyzer.h"
@@ -51,9 +36,9 @@
 #include 
"storage/index/inverted/analysis/tokenizer/english/english_tokenizer.h"
 #include 
"storage/index/inverted/analysis/tokenizer/icu/icu_tokenizer_factory.h"
 #include "storage/index/inverted/analysis/tokenizer/ik/ik_tokenizer_factory.h"
+#include "storage/index/inverted/analysis/tokenizer/jieba/jieba_tokenizer.h"
 #include 
"storage/index/inverted/analysis/tokenizer/kuromoji/kuromoji_tokenizer_factory.h"
 #include "storage/index/inverted/analysis/tokenizer/standard/uax29_tokenizer.h"
-#include "storage/index/inverted/clucene/lucene_backed_text_analyzer.h"
 
 namespace doris::segment_v2::inverted_index {
 namespace {
@@ -188,23 +173,6 @@ bool builtin_lowercase(const std::string& lower_case, bool 
parser_default) {
     return parser_default;
 }
 
-// A builtin parser whose analyzer is still CLucene's, with lower_case and 
stop_words applied.
-TextAnalyzerPtr 
lucene_builtin_analyzer(std::shared_ptr<lucene::analysis::Analyzer> analyzer,
-                                        const std::string& lower_case,
-                                        const std::string& stop_words) {
-    if (lower_case == INVERTED_INDEX_PARSER_TRUE) {
-        analyzer->set_lowercase(true);
-    } else if (lower_case == INVERTED_INDEX_PARSER_FALSE) {
-        analyzer->set_lowercase(false);
-    }
-    if (stop_words == "none") {
-        analyzer->set_stopwords(nullptr);
-    } else {
-        analyzer->set_stopwords(&lucene::analysis::standard95::stop_words);
-    }
-    return std::make_shared<LuceneBackedTextAnalyzer>(std::move(analyzer));
-}
-
 // `tokenizer`, followed by the lowercase filter when `lowercase`.
 TextAnalyzerPtr tokenizer_chain(TokenizerFactoryPtr tokenizer, bool lowercase) 
{
     std::vector<TokenFilterFactoryPtr> token_filters;
@@ -230,15 +198,13 @@ TextAnalyzerPtr 
build_builtin_analyzer(InvertedIndexParserType parser_type,
                 std::vector<TokenFilterFactoryPtr> {});
     }
     if (parser_type == InvertedIndexParserType::PARSER_CHINESE) {
-        auto chinese_analyzer =
-                
std::make_shared<lucene::analysis::LanguageBasedAnalyzer>(L"chinese", false);
-        chinese_analyzer->initDict(config::inverted_index_dict_path);
-        if (parser_mode == INVERTED_INDEX_PARSER_COARSE_GRANULARITY) {
-            chinese_analyzer->setMode(lucene::analysis::AnalyzerMode::Default);
-        } else {
-            chinese_analyzer->setMode(lucene::analysis::AnalyzerMode::All);
-        }
-        return lucene_builtin_analyzer(std::move(chinese_analyzer), 
lower_case, stop_words);
+        // The fine mode cuts every dictionary word; coarse_grained mixes the 
dictionary and the HMM.
+        return std::make_shared<ChainAnalyzer>(
+                std::vector<CharFilterFactoryPtr> {},
+                std::make_shared<JiebaTokenizerFactory>(
+                        parser_mode != 
INVERTED_INDEX_PARSER_COARSE_GRANULARITY,
+                        builtin_lowercase(lower_case, false), 
config::inverted_index_dict_path),
+                std::vector<TokenFilterFactoryPtr> {});
     }
     if (parser_type == InvertedIndexParserType::PARSER_ICU) {
         return tokenizer_chain(std::make_shared<ICUTokenizerFactory>(),
@@ -300,13 +266,7 @@ TextAnalyzerPtr 
InvertedIndexAnalyzer::create_builtin_analyzer(InvertedIndexPars
                                                                const 
std::string& parser_mode,
                                                                const 
std::string& lower_case,
                                                                const 
std::string& stop_words) {
-    try {
-        return build_builtin_analyzer(parser_type, parser_mode, lower_case, 
stop_words);
-    } catch (const CLuceneError& e) {
-        throw Exception(ErrorCode::INVERTED_INDEX_ANALYZER_ERROR,
-                        "building the {} analyzer failed: {}",
-                        inverted_index_parser_type_to_string(parser_type), 
e.what());
-    }
+    return build_builtin_analyzer(parser_type, parser_mode, lower_case, 
stop_words);
 }
 
 TextAnalyzerPtr InvertedIndexAnalyzer::create_analyzer(const 
InvertedIndexAnalyzerConfig* config) {
diff --git a/be/src/storage/index/inverted/analysis/tokenizer/jieba/dict_trie.h 
b/be/src/storage/index/inverted/analysis/tokenizer/jieba/dict_trie.h
new file mode 100644
index 00000000000..fca697ca7ab
--- /dev/null
+++ b/be/src/storage/index/inverted/analysis/tokenizer/jieba/dict_trie.h
@@ -0,0 +1,200 @@
+// Derived from cppjieba's DictTrie.hpp (https://github.com/yanyiwu/cppjieba), 
MIT License,
+// Copyright (c) 2013. See dist/licenses/LICENSE-cppjieba.txt.
+
+#pragma once
+
+#include <algorithm>
+#include <cmath>
+#include <cstddef>
+#include <cstdlib>
+#include <fstream>
+#include <string>
+#include <unordered_set>
+#include <vector>
+
+#include "common/exception.h"
+#include "common/logging.h"
+#include "storage/index/inverted/analysis/tokenizer/jieba/string_util.h"
+#include "storage/index/inverted/analysis/tokenizer/jieba/unicode.h"
+
+namespace doris::segment_v2::inverted_index::jieba {
+
+inline constexpr double MIN_DOUBLE = -3.14e+100;
+inline constexpr size_t DICT_COLUMN_NUM = 3;
+
+// A dictionary word and its weight. Upstream's part-of-speech tag is gone: 
only tagging read it.
+struct DictUnit {
+    Unicode word;
+    double weight = 0.0;
+};
+
+// The words of the main dictionary and the user dictionaries, with their 
weights. FlatDict builds
+// the trie the cuts walk out of them; upstream built a trie of pointers to 
these units here.
+//
+// A main dictionary line is "word frequency tag", three columns split at 
single spaces, with the
+// frequency read by atof; any other line fails the load. A word's weight is 
log(frequency / the sum
+// of all frequencies). A user dictionary line is "word", "word tag" or "word 
frequency tag"; the
+// first two take the median weight of the main dictionary (upstream's default 
WordWeightMedian, the
+// only option Jieba used), the third log(frequency / the main dictionary's 
sum), with the frequency
+// read by atoi. A user line with more columns adds a unit with no word, and 
every user word of one
+// rune is remembered in user_dict_single_chinese_words(), whatever its script.
+class DictTrie {
+public:
+    // `user_dict_paths` lists the user dictionaries, separated by '|' or ';'.
+    DictTrie(const std::string& dict_path, const std::string& user_dict_paths) 
{
+        init(dict_path, user_dict_paths);
+    }
+
+    // One unit for every line of the main dictionary and then of the user 
dictionaries, in the
+    // order of the files and their lines. A unit whose word did not decode 
has an empty word and
+    // is no word of the trie; of several units of the same word, the trie 
takes the last, which
+    // lets a user word replace the main dictionary's unit of that word, as 
upstream's trie did
+    // when it inserted the units in this order.
+    const std::vector<DictUnit>& units() const { return _static_node_infos; }
+
+    const std::unordered_set<Rune>& user_dict_single_chinese_words() const {
+        return _user_dict_single_chinese_word;
+    }
+
+    double get_min_weight() const { return _min_weight; }
+
+private:
+    // Upstream also shrank the units to fit before it pointed its trie into 
them; nothing keeps
+    // them now.
+    void init(const std::string& dict_path, const std::string& 
user_dict_paths) {
+        load_dict(dict_path);
+        _freq_sum = calc_freq_sum(_static_node_infos);
+        calculate_weight(_static_node_infos, _freq_sum);
+        set_static_word_weights(dict_path);
+
+        // Upstream skipped this for an empty list, which splits into no file 
anyway.
+        load_user_dict(user_dict_paths);
+    }
+
+    // Decodes `word` into the unit and sets its weight. A word that does not 
decode leaves the
+    // unit with an empty word and its weight as it was, and returns false; no 
caller looks at the
+    // result, as upstream's did not.
+    static bool make_node_info(DictUnit& node_info, const std::string& word, 
double weight) {
+        if (!decode_unicode(word, node_info.word)) {
+            return false;
+        }
+        node_info.weight = weight;
+        return true;
+    }
+
+    void load_dict(const std::string& file_path) {
+        std::ifstream ifs(file_path);
+        if (!ifs.is_open()) {
+            throw Exception(ErrorCode::INVERTED_INDEX_ANALYZER_ERROR,
+                            "jieba main dictionary file not found: {}", 
file_path);
+        }
+        std::string line;
+        std::vector<std::string> buf;
+
+        // One unit for every line, as upstream: a line whose word does not 
decode adds a unit with
+        // no word and the frequency of the line before it, which then counts 
twice in the sum of
+        // the frequencies. (A first line that does not decode added an 
indeterminate frequency
+        // upstream, and adds 0 here.)
+        DictUnit node_info;
+        while (std::getline(ifs, line)) {
+            split(line, buf, " ");
+            if (buf.size() != DICT_COLUMN_NUM) {
+                throw Exception(ErrorCode::INVERTED_INDEX_ANALYZER_ERROR,
+                                "jieba main dictionary file {} has a line that 
is not {} columns "
+                                "split at spaces: {}",
+                                file_path, DICT_COLUMN_NUM, line);
+            }
+            make_node_info(node_info, buf[0], std::atof(buf[1].c_str()));
+            _static_node_infos.push_back(node_info);
+        }
+    }
+
+    // A user dictionary file that cannot be opened fails the load; an empty 
line is skipped.
+    void load_user_dict(const std::string& file_paths) {
+        const std::vector<std::string> files = split(file_paths, "|;");
+        for (const std::string& file : files) {
+            std::ifstream ifs(file);
+            if (!ifs.is_open()) {
+                throw Exception(ErrorCode::INVERTED_INDEX_ANALYZER_ERROR,
+                                "jieba user dictionary file not found: {}", 
file);
+            }
+            std::string line;
+            while (std::getline(ifs, line)) {
+                if (line.empty()) {
+                    continue;
+                }
+                insert_user_dict_node(line);
+            }
+        }
+    }
+
+    // Upstream's InserUserDictNode.
+    void insert_user_dict_node(const std::string& line) {
+        std::vector<std::string> buf;
+        DictUnit node_info;
+        split(line, buf, " ");
+        if (buf.size() == 1 || buf.size() == 2) {
+            // The second column is the tag.
+            make_node_info(node_info, buf[0], _user_word_default_weight);
+        } else if (buf.size() == 3) {
+            int freq = std::atoi(buf[1].c_str());
+            DCHECK(_freq_sum > 0.0);
+            double weight = std::log(1.0 * freq / _freq_sum);
+            make_node_info(node_info, buf[0], weight);
+        }
+        _static_node_infos.push_back(node_info);
+        if (node_info.word.size() == 1) {
+            _user_dict_single_chinese_word.insert(node_info.word[0]);
+        }
+    }
+
+    static bool weight_compare(const DictUnit& lhs, const DictUnit& rhs) {
+        return lhs.weight < rhs.weight;
+    }
+
+    // Sorts a copy of the main dictionary's units by weight for the smallest 
and the median, as
+    // upstream; the median is the default weight of a user word. Only these 
two order statistics
+    // are read, which any sort gives alike; std::ranges::sort is libstdc++'s 
std::sort anyway.
+    void set_static_word_weights(const std::string& dict_path) {
+        if (_static_node_infos.empty()) {
+            throw Exception(ErrorCode::INVERTED_INDEX_ANALYZER_ERROR,
+                            "jieba main dictionary file {} is empty", 
dict_path);
+        }
+        std::vector<DictUnit> x = _static_node_infos;
+        std::ranges::sort(x, weight_compare);
+        _min_weight = x[0].weight;
+        _median_weight = x[x.size() / 2].weight;
+        _user_word_default_weight = _median_weight;
+    }
+
+    // In the order of the file.
+    static double calc_freq_sum(const std::vector<DictUnit>& node_infos) {
+        double sum = 0.0;
+        for (const DictUnit& node_info : node_infos) {
+            sum += node_info.weight;
+        }
+        return sum;
+    }
+
+    // The DCHECKs are upstream's asserts. A debug build stops at a frequency, 
or a sum of them, of
+    // 0 or less, as upstream's did, and so at an empty dictionary before 
set_static_word_weights()
+    // can throw; a release build loads such a dictionary with weights of -inf 
or NaN, as upstream's
+    // did, and throws for an empty one.
+    static void calculate_weight(std::vector<DictUnit>& node_infos, double 
sum) {
+        DCHECK(sum > 0.0);
+        for (DictUnit& node_info : node_infos) {
+            DCHECK(node_info.weight > 0.0);
+            node_info.weight = std::log(node_info.weight / sum);
+        }
+    }
+
+    std::vector<DictUnit> _static_node_infos;
+
+    double _freq_sum = 0.0;
+    double _min_weight = 0.0;
+    double _median_weight = 0.0;
+    double _user_word_default_weight = 0.0;
+    std::unordered_set<Rune> _user_dict_single_chinese_word;
+};
+
+} // namespace doris::segment_v2::inverted_index::jieba
diff --git a/be/src/storage/index/inverted/analysis/tokenizer/jieba/flat_dict.h 
b/be/src/storage/index/inverted/analysis/tokenizer/jieba/flat_dict.h
new file mode 100644
index 00000000000..772ce375265
--- /dev/null
+++ b/be/src/storage/index/inverted/analysis/tokenizer/jieba/flat_dict.h
@@ -0,0 +1,349 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <algorithm>
+#include <array>
+#include <bit>
+#include <cstddef>
+#include <cstdint>
+#include <fstream>
+#include <string>
+#include <string_view>
+#include <unordered_map>
+#include <unordered_set>
+#include <vector>
+
+#include "storage/index/inverted/analysis/tokenizer/jieba/dict_trie.h"
+#include "storage/index/inverted/analysis/tokenizer/jieba/hmm_model.h"
+#include "storage/index/inverted/analysis/tokenizer/jieba/unicode.h"
+
+namespace doris::segment_v2::inverted_index::jieba {
+
+// What the two cuts read, built once from the loaded dictionaries, model and 
stop words, which
+// can be dropped then, and laid out so that every lookup of a cut is an array 
probe or a short
+// scan instead of a hash lookup behind a pointer.
+//
+// The dictionary is a trie of flat arrays. Node 0 is the root and the other 
nodes are numbered
+// breadth first, so the children of node n are the consecutive nodes 
child_begin[n] to
+// child_begin[n + 1] - 1, in increasing rune, and key[c] is the rune that 
leads to child c. A
+// node at which a word ends has has_value set and the word's weight. Say the 
words are A, AB, ABC
+// and C, each letter a rune and A < C: the root's children are 1 (A) and 2 
(C), node 1's child is
+// 3 (B) and node 3's is 4 (C); child_begin is {1, 3, 4, 4, 5, 5}, so node 2 
has 4 - 4 = no child,
+// and has_value is set for nodes 1 to 4.
+//
+// The trie keeps the words of the units with a word, the last unit of a word 
where there are
+// several, which is the trie of pointers upstream built by inserting the 
units in order. Nothing
+// is cached per value: the cuts keep what a value needs in a JiebaScratch.
+class FlatDict {
+public:
+    // The root, which is nobody's child, also stands for no node.
+    static constexpr uint32_t kNoNode = 0;
+
+    FlatDict(const DictTrie& dict_trie, const HMMModel& model, const 
std::string& stop_word_path)
+            : _min_weight(dict_trie.get_min_weight()),
+              _start_prob(model.start_prob),
+              _trans_prob(model.trans_prob) {
+        build_trie(dict_trie.units());
+        build_user_single_words(dict_trie.user_dict_single_chinese_words());
+        build_emit_prob(model);
+        load_stop_words(stop_word_path);
+    }
+    FlatDict(const FlatDict&) = delete;
+    FlatDict& operator=(const FlatDict&) = delete;
+
+    // The child of the root for `rune`, or kNoNode: a table for the first 
plane, a search among
+    // the root's children above it.
+    uint32_t root_child(Rune rune) const {
+        return rune < kDirectRunes ? _root_children[rune] : child(kNoNode, 
rune);
+    }
+
+    // The child of `node` for `rune`, or kNoNode. A node of up to 
kLinearChildren children is
+    // scanned, one of more is searched by bisection.
+    uint32_t child(uint32_t node, Rune rune) const {
+        const uint32_t begin = _child_begin[node];
+        const uint32_t end = _child_begin[node + 1];
+        if (end - begin <= kLinearChildren) {
+            for (uint32_t c = begin; c < end; ++c) {
+                if (_key[c] == rune) {
+                    return c;
+                }
+            }
+            return kNoNode;
+        }
+        const Rune* const first = _key.data() + begin;
+        const Rune* const last = _key.data() + end;
+        const Rune* const it = std::lower_bound(first, last, rune);
+        return it != last && *it == rune ? static_cast<uint32_t>(it - 
_key.data()) : kNoNode;
+    }
+
+    // Whether a word ends at `node`, and its weight.
+    bool has_value(uint32_t node) const { return _has_value[node] != 0; }
+    double weight(uint32_t node) const { return _weight[node]; }
+
+    // The weight of a rune that is no word: the smallest weight of the main 
dictionary.
+    double min_weight() const { return _min_weight; }
+
+    // Whether a user dictionary has `rune` alone as a word, upstream's
+    // IsUserDictSingleChineseWord.
+    bool is_user_single_word(Rune rune) const {
+        if (rune < kDirectRunes) {
+            return ((_user_single_words[rune / 64] >> (rune % 64)) & 1U) != 0;
+        }
+        return _user_single_supplementary.contains(rune);
+    }
+
+    const std::array<double, HMMModel::STATUS_SUM>& start_prob() const { 
return _start_prob; }
+    const std::array<std::array<double, HMMModel::STATUS_SUM>, 
HMMModel::STATUS_SUM>& trans_prob()
+            const {
+        return _trans_prob;
+    }
+
+    // The emission probabilities of `rune` for B, E, M and S, in this order, 
with MIN_DOUBLE for a
+    // status the model gives no probability for `rune`, as upstream's 
GetEmitProb.
+    const double* emit_prob(Rune rune) const {
+        if (rune < kDirectRunes) {
+            return _emit_prob.data() + size_t {rune} * HMMModel::STATUS_SUM;
+        }
+        const auto it = _supplementary_emit_prob.find(rune);
+        return it == _supplementary_emit_prob.end() ? kNoEmitProb.data() : 
it->second.data();
+    }
+
+    // Whether `word` is a line of the stop word file. A word of up to 
kPackedStopWordBytes bytes
+    // is probed for in an open-addressing table keyed by its bytes, a longer 
one searched for among
+    // the longer stop words.
+    bool is_stop_word(std::string_view word) const {
+        if (word.size() > _longest_stop_word) {
+            return false;
+        }
+        if (word.size() > kPackedStopWordBytes) {
+            return std::ranges::binary_search(_long_stop_words, word);
+        }
+        const uint64_t bytes = pack_bytes(word);
+        const size_t mask = _stop_word_slots.size() - 1;
+        for (size_t slot = stop_word_slot(bytes, word.size());; slot = (slot + 
1) & mask) {
+            const StopWordSlot& entry = _stop_word_slots[slot];
+            if (entry.size == 0) {
+                return false;
+            }
+            if (entry.bytes == bytes && entry.size == word.size()) {
+                return true;
+            }
+        }
+    }
+
+private:
+    // The runes below this have a table of their own: the first rune of a 
word, whether a user
+    // word is that rune alone, and the emission probabilities.
+    static constexpr Rune kDirectRunes = 0x10000;
+    static constexpr uint32_t kLinearChildren = 8;
+    static constexpr size_t kPackedStopWordBytes = 8;
+    // 30 times the 542 distinct stop words of up to 8 bytes in the shipped 
file, so that a probe
+    // nearly always ends at its first slot; a file with more of them gets a 
table of at least
+    // twice as many slots as words, so that a probe always meets an empty 
slot.
+    static constexpr size_t kMinStopWordSlots = size_t {1} << 14;
+    static constexpr std::array<double, HMMModel::STATUS_SUM> kNoEmitProb = {
+            MIN_DOUBLE, MIN_DOUBLE, MIN_DOUBLE, MIN_DOUBLE};
+
+    // A stop word of up to 8 bytes; an empty slot has size 0, which no stop 
word in the table has.
+    struct StopWordSlot {
+        uint64_t bytes = 0;
+        uint64_t size = 0;
+    };
+
+    // The units the trie keeps, by word, breadth first. For each node the 
words below it are a
+    // run of the sorted words, and those below a child are the run that has 
the child's rune
+    // after the node's runes; a word that ends at the node sorts first in its 
run.
+    void build_trie(const std::vector<DictUnit>& units) {
+        std::vector<uint32_t> words;
+        words.reserve(units.size());
+        for (uint32_t i = 0; i < units.size(); ++i) {
+            if (!units[i].word.empty()) {
+                words.push_back(i);
+            }
+        }
+        // Stable, so that the units of one word stay in the order of the 
files; the last of them
+        // is the one upstream's trie kept.
+        std::ranges::stable_sort(words, [&units](uint32_t lhs, uint32_t rhs) {
+            return units[lhs].word < units[rhs].word;
+        });
+        size_t kept = 0;
+        for (size_t i = 0; i < words.size(); ++i) {
+            if (i + 1 == words.size() || units[words[i]].word != units[words[i 
+ 1]].word) {
+                words[kept++] = words[i];
+            }
+        }
+        words.resize(kept);
+
+        // The words below each node: words[begin, end), which all start with 
the node's `depth`
+        // runes. Nodes are appended in the order they are numbered, so that a 
node is handled
+        // after every node of a smaller number has appended its children.
+        struct Below {
+            uint32_t begin;
+            uint32_t end;
+            uint32_t depth;
+        };
+        std::vector<Below> below;
+        below.push_back({.begin = 0, .end = 
static_cast<uint32_t>(words.size()), .depth = 0});
+        _key.push_back(0);
+        _has_value.push_back(0);
+        _weight.push_back(0.0);
+        for (size_t node = 0; node < below.size(); ++node) {
+            auto [begin, end, depth] = below[node];
+            _child_begin.push_back(static_cast<uint32_t>(below.size()));
+            if (begin < end && units[words[begin]].word.size() == depth) {
+                ++begin;
+            }
+            while (begin < end) {
+                const DictUnit& first = units[words[begin]];
+                const Rune rune = first.word[depth];
+                uint32_t run_end = begin + 1;
+                while (run_end < end && units[words[run_end]].word[depth] == 
rune) {
+                    ++run_end;
+                }
+                const bool ends_here = first.word.size() == depth + 1;
+                below.push_back({.begin = begin, .end = run_end, .depth = 
depth + 1});
+                _key.push_back(rune);
+                _has_value.push_back(ends_here ? 1 : 0);
+                _weight.push_back(ends_here ? first.weight : 0.0);
+                begin = run_end;
+            }
+        }
+        _child_begin.push_back(static_cast<uint32_t>(below.size()));
+        _child_begin.shrink_to_fit();
+        _key.shrink_to_fit();
+        _has_value.shrink_to_fit();
+        _weight.shrink_to_fit();
+
+        _root_children.assign(kDirectRunes, kNoNode);
+        for (uint32_t c = _child_begin[kNoNode]; c < _child_begin[kNoNode + 
1]; ++c) {
+            if (_key[c] < kDirectRunes) {
+                _root_children[_key[c]] = c;
+            }
+        }
+    }
+
+    void build_user_single_words(const std::unordered_set<Rune>& runes) {
+        _user_single_words.assign(kDirectRunes / 64, 0);
+        for (const Rune rune : runes) {
+            if (rune < kDirectRunes) {
+                _user_single_words[rune / 64] |= uint64_t {1} << (rune % 64);
+            } else {
+                _user_single_supplementary.insert(rune);
+            }
+        }
+    }
+
+    // Rune by rune instead of status by status, so that the four 
probabilities Viterbi reads for
+    // a rune are next to each other.
+    void build_emit_prob(const HMMModel& model) {
+        _emit_prob.assign(size_t {kDirectRunes} * HMMModel::STATUS_SUM, 
MIN_DOUBLE);
+        for (size_t status = 0; status < HMMModel::STATUS_SUM; ++status) {
+            for (const auto& [rune, prob] : *model.emit_prob_vec[status]) {
+                if (rune < kDirectRunes) {
+                    _emit_prob[size_t {rune} * HMMModel::STATUS_SUM + status] 
= prob;
+                } else {
+                    // A status the model has no pair of the rune for keeps 
MIN_DOUBLE.
+                    auto& probs =
+                            _supplementary_emit_prob.try_emplace(rune, 
kNoEmitProb).first->second;
+                    probs[status] = prob;
+                }
+            }
+        }
+    }
+
+    // A stop word file that cannot be opened leaves no stop word, as 
upstream. Every line is a
+    // stop word as it is, untrimmed; an empty line is one too, but no word is 
empty, so it is not
+    // kept.
+    void load_stop_words(const std::string& file_path) {
+        std::vector<std::string> short_words;
+        std::ifstream ifs(file_path);
+        if (ifs.is_open()) {
+            std::string line;
+            while (std::getline(ifs, line)) {
+                _longest_stop_word = std::max(_longest_stop_word, line.size());
+                if (line.size() > kPackedStopWordBytes) {
+                    _long_stop_words.push_back(line);
+                } else if (!line.empty()) {
+                    short_words.push_back(line);
+                }
+            }
+        }
+        std::ranges::sort(_long_stop_words);
+        const auto [last, end] = std::ranges::unique(_long_stop_words);
+        _long_stop_words.erase(last, end);
+
+        size_t slots = kMinStopWordSlots;
+        while (slots < 2 * short_words.size()) {
+            slots *= 2;
+        }
+        _stop_word_slots.assign(slots, StopWordSlot {});
+        _stop_word_shift = 64 - std::countr_zero(slots);
+        for (const std::string& word : short_words) {
+            const uint64_t bytes = pack_bytes(word);
+            size_t slot = stop_word_slot(bytes, word.size());
+            while (_stop_word_slots[slot].size != 0 &&
+                   (_stop_word_slots[slot].bytes != bytes ||
+                    _stop_word_slots[slot].size != word.size())) {
+                slot = (slot + 1) & (slots - 1);
+            }
+            _stop_word_slots[slot].bytes = bytes;
+            _stop_word_slots[slot].size = word.size();
+        }
+    }
+
+    // The bytes of a word of at most 8 bytes as one integer, the first byte 
lowest.
+    static uint64_t pack_bytes(std::string_view word) {
+        uint64_t bytes = 0;
+        for (size_t i = 0; i < word.size(); ++i) {
+            bytes |= uint64_t {static_cast<uint8_t>(word[i])} << (8 * i);
+        }
+        return bytes;
+    }
+
+    // Fibonacci hashing: the top bits of the product, as many as the table 
has slots.
+    size_t stop_word_slot(uint64_t bytes, size_t size) const {
+        return static_cast<size_t>(((bytes ^ size) * 0x9E3779B97F4A7C15ULL) >> 
_stop_word_shift);
+    }
+
+    // The trie.
+    std::vector<uint32_t> _child_begin;
+    std::vector<Rune> _key;
+    std::vector<uint8_t> _has_value;
+    std::vector<double> _weight;
+    std::vector<uint32_t> _root_children;
+    double _min_weight = 0.0;
+
+    // The user words of one rune: a bit for each rune of the first plane, a 
set above it.
+    std::vector<uint64_t> _user_single_words;
+    std::unordered_set<Rune> _user_single_supplementary;
+
+    // The HMM.
+    std::array<double, HMMModel::STATUS_SUM> _start_prob;
+    std::array<std::array<double, HMMModel::STATUS_SUM>, HMMModel::STATUS_SUM> 
_trans_prob;
+    std::vector<double> _emit_prob;
+    std::unordered_map<Rune, std::array<double, HMMModel::STATUS_SUM>> 
_supplementary_emit_prob;
+
+    // The stop words.
+    std::vector<StopWordSlot> _stop_word_slots;
+    int _stop_word_shift = 0;
+    std::vector<std::string> _long_stop_words;
+    size_t _longest_stop_word = 0;
+};
+
+} // namespace doris::segment_v2::inverted_index::jieba
diff --git a/be/src/storage/index/inverted/analysis/tokenizer/jieba/hmm_model.h 
b/be/src/storage/index/inverted/analysis/tokenizer/jieba/hmm_model.h
new file mode 100644
index 00000000000..9322dfec441
--- /dev/null
+++ b/be/src/storage/index/inverted/analysis/tokenizer/jieba/hmm_model.h
@@ -0,0 +1,160 @@
+// Derived from cppjieba's HMMModel.hpp (https://github.com/yanyiwu/cppjieba), 
MIT License,
+// Copyright (c) 2013. See dist/licenses/LICENSE-cppjieba.txt.
+
+#pragma once
+
+#include <array>
+#include <cstddef>
+#include <cstdlib>
+#include <fstream>
+#include <string>
+#include <string_view>
+#include <unordered_map>
+#include <vector>
+
+#include "common/exception.h"
+#include "common/logging.h"
+#include "storage/index/inverted/analysis/tokenizer/jieba/string_util.h"
+#include "storage/index/inverted/analysis/tokenizer/jieba/unicode.h"
+
+namespace doris::segment_v2::inverted_index::jieba {
+
+using EmitProbMap = std::unordered_map<Rune, double>;
+
+// The HMM of jieba's unknown-word segmentation. Every line of the model file 
is trimmed of white
+// space, and blank lines and lines that start with '#' are skipped. The rest 
are one line of the
+// 4 start probabilities, 4 lines of 4 transition probabilities, all split at 
single spaces and
+// read by atof, and one line of emission probabilities for each of B, E, M 
and S:
+// "rune:probability" pairs split at ','. A missing or illegal line fails the 
load; a later pair of
+// the same rune replaces the earlier one, and what follows the S line is not 
read. Upstream's
+// statMap, which only printing read, is gone, and so is GetEmitProb: the cuts 
read the model as
+// FlatDict lays it out.
+struct HMMModel {
+    /*
+     * STATUS:
+     * 0: HMMModel::B, 1: HMMModel::E, 2: HMMModel::M, 3:HMMModel::S
+     * */
+    enum { B = 0, E = 1, M = 2, S = 3, STATUS_SUM = 4 };
+
+    explicit HMMModel(const std::string& model_path) {
+        emit_prob_vec.push_back(&emit_prob_b);
+        emit_prob_vec.push_back(&emit_prob_e);
+        emit_prob_vec.push_back(&emit_prob_m);
+        emit_prob_vec.push_back(&emit_prob_s);
+        load_model(model_path);
+    }
+    // emit_prob_vec points at the maps of this model.
+    HMMModel(const HMMModel&) = delete;
+    HMMModel& operator=(const HMMModel&) = delete;
+
+    void load_model(const std::string& file_path) {
+        std::ifstream ifile(file_path);
+        if (!ifile.is_open()) {
+            throw Exception(ErrorCode::INVERTED_INDEX_ANALYZER_ERROR,
+                            "jieba hmm model file not found: {}", file_path);
+        }
+        std::string line;
+        std::vector<std::string> tmp;
+        //Load start_prob
+        if (!get_line(ifile, line)) {
+            throw_illegal(file_path, "start probability");
+        }
+        split(line, tmp, " ");
+        if (tmp.size() != STATUS_SUM) {
+            throw_illegal(file_path, "start probability");
+        }
+        for (size_t j = 0; j < tmp.size(); j++) {
+            start_prob[j] = std::atof(tmp[j].c_str());
+        }
+
+        //Load trans_prob
+        for (size_t i = 0; i < STATUS_SUM; i++) {
+            if (!get_line(ifile, line)) {
+                throw_illegal(file_path, "transition probability");
+            }
+            split(line, tmp, " ");
+            if (tmp.size() != STATUS_SUM) {
+                throw_illegal(file_path, "transition probability");
+            }
+            for (size_t j = 0; j < STATUS_SUM; j++) {
+                trans_prob[i][j] = std::atof(tmp[j].c_str());
+            }
+        }
+
+        //Load emit_prob_b
+        if (!get_line(ifile, line) || !load_emit_prob(line, emit_prob_b)) {
+            throw_illegal(file_path, "B emission probability");
+        }
+
+        //Load emit_prob_e
+        if (!get_line(ifile, line) || !load_emit_prob(line, emit_prob_e)) {
+            throw_illegal(file_path, "E emission probability");
+        }
+
+        //Load emit_prob_m
+        if (!get_line(ifile, line) || !load_emit_prob(line, emit_prob_m)) {
+            throw_illegal(file_path, "M emission probability");
+        }
+
+        //Load emit_prob_s
+        if (!get_line(ifile, line) || !load_emit_prob(line, emit_prob_s)) {
+            throw_illegal(file_path, "S emission probability");
+        }
+    }
+
+    // The next line that is neither blank nor a comment, trimmed.
+    static bool get_line(std::ifstream& ifile, std::string& line) {
+        while (std::getline(ifile, line)) {
+            trim(line);
+            if (line.empty()) {
+                continue;
+            }
+            if (line.starts_with("#")) {
+                continue;
+            }
+            return true;
+        }
+        return false;
+    }
+
+    // Fails on a pair that is not two pieces split at ':' and on a rune that 
does not decode to
+    // exactly one code point. The pairs before it are in `mp` by then, which 
does not matter: the
+    // load fails.
+    static bool load_emit_prob(const std::string& line, EmitProbMap& mp) {
+        // Upstream returned false for an empty line, which get_line() never 
returns.
+        DCHECK(!line.empty());
+        std::vector<std::string> tmp;
+        std::vector<std::string> tmp2;
+        Unicode unicode;
+        split(line, tmp, ",");
+        for (const std::string& item : tmp) {
+            split(item, tmp2, ":");
+            if (2 != tmp2.size()) {
+                return false;
+            }
+            if (!decode_unicode(tmp2[0], unicode) || unicode.size() != 1) {
+                return false;
+            }
+            mp[unicode[0]] = std::atof(tmp2[1].c_str());
+        }
+        return true;
+    }
+
+    std::array<double, STATUS_SUM> start_prob {};
+    std::array<std::array<double, STATUS_SUM>, STATUS_SUM> trans_prob {};
+    EmitProbMap emit_prob_b;
+    EmitProbMap emit_prob_e;
+    EmitProbMap emit_prob_m;
+    EmitProbMap emit_prob_s;
+    // Indexed by status.
+    std::vector<EmitProbMap*> emit_prob_vec;
+
+private:
+    // A line of the model that upstream's checks failed the load on.
+    [[noreturn]] static void throw_illegal(const std::string& file_path, 
std::string_view part) {
+        throw Exception(ErrorCode::INVERTED_INDEX_ANALYZER_ERROR,
+                        "jieba hmm model file {} has no legal {} line", 
file_path, part);
+    }
+};
+
+} // namespace doris::segment_v2::inverted_index::jieba
diff --git a/be/src/storage/index/inverted/analysis/tokenizer/jieba/jieba.h 
b/be/src/storage/index/inverted/analysis/tokenizer/jieba/jieba.h
new file mode 100644
index 00000000000..f08638672dd
--- /dev/null
+++ b/be/src/storage/index/inverted/analysis/tokenizer/jieba/jieba.h
@@ -0,0 +1,85 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <string>
+#include <string_view>
+#include <vector>
+
+#include "storage/index/inverted/analysis/tokenizer/jieba/dict_trie.h"
+#include "storage/index/inverted/analysis/tokenizer/jieba/flat_dict.h"
+#include "storage/index/inverted/analysis/tokenizer/jieba/hmm_model.h"
+#include "storage/index/inverted/analysis/tokenizer/jieba/jieba_scratch.h"
+#include "storage/index/inverted/analysis/tokenizer/jieba/segmenter.h"
+
+namespace doris::segment_v2::inverted_index::jieba {
+
+// The dictionaries of the chinese parser, loaded once, and its two cuts. 
Loading reads the main
+// dictionary and the user dictionaries, then the model, then the stop words, 
and throws
+// INVERTED_INDEX_ANALYZER_ERROR when the dictionary, a user dictionary or the 
model cannot be
+// opened or has an illegal line; a stop word file that cannot be opened 
leaves no stop word. The
+// loaders' results are laid out in a FlatDict and dropped once it is built. 
Unlike CLucene's
+// Jieba it reads no idf file and has no keyword extractor, whose loading also 
failed on a stop
+// word file that could not be opened.
+//
+// Cutting is const and keeps what it needs in the caller's scratch, so one 
Jieba serves every
+// thread.
+class Jieba {
+public:
+    Jieba(const std::string& dict_path, const std::string& model_path,
+          const std::string& user_dict_path, const std::string& stop_word_path)
+            : _dict(load(dict_path, model_path, user_dict_path, 
stop_word_path)) {}
+    Jieba(const Jieba&) = delete;
+    Jieba& operator=(const Jieba&) = delete;
+
+    // The coarse mode, CLucene's Cut with the HMM. The words are views of 
`sentence`.
+    void cut(std::string_view sentence, JiebaScratch& scratch,
+             std::vector<std::string_view>& words) const {
+        Segmenter(_dict, sentence, scratch, words).mix_cut();
+    }
+
+    // The fine mode, CLucene's CutAll. The words are views of `sentence`.
+    void cut_all(std::string_view sentence, JiebaScratch& scratch,
+                 std::vector<std::string_view>& words) const {
+        Segmenter(_dict, sentence, scratch, words).full_cut();
+    }
+
+    // The same with a scratch of their own, for a caller that cuts once.
+    void cut(std::string_view sentence, std::vector<std::string_view>& words) 
const {
+        JiebaScratch scratch;
+        cut(sentence, scratch, words);
+    }
+    void cut_all(std::string_view sentence, std::vector<std::string_view>& 
words) const {
+        JiebaScratch scratch;
+        cut_all(sentence, scratch, words);
+    }
+
+private:
+    // In this order, as CLucene's Jieba built its members, so that a bad 
dictionary fails the
+    // load before a bad model does.
+    static FlatDict load(const std::string& dict_path, const std::string& 
model_path,
+                         const std::string& user_dict_path, const std::string& 
stop_word_path) {
+        const DictTrie dict_trie(dict_path, user_dict_path);
+        const HMMModel model(model_path);
+        return {dict_trie, model, stop_word_path};
+    }
+
+    const FlatDict _dict;
+};
+
+} // namespace doris::segment_v2::inverted_index::jieba
diff --git 
a/be/src/storage/index/inverted/analysis/tokenizer/jieba/jieba_scratch.h 
b/be/src/storage/index/inverted/analysis/tokenizer/jieba/jieba_scratch.h
new file mode 100644
index 00000000000..012d7faba84
--- /dev/null
+++ b/be/src/storage/index/inverted/analysis/tokenizer/jieba/jieba_scratch.h
@@ -0,0 +1,61 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+#include <vector>
+
+#include "storage/index/inverted/analysis/scratch.h"
+
+namespace doris::segment_v2::inverted_index::jieba {
+
+// What a cut works on besides the dictionary, all of it in proportion to the 
value: the caller
+// keeps one from value to value, so that a steady stream of values allocates 
nothing, and calls
+// release_oversized() between values. The dictionary is shared and read only; 
this is not.
+struct JiebaScratch {
+    // The code point of every rune of the value and its byte offset, and 
after the last rune the
+    // size of the value: the runes [l, r] are the bytes [offsets[l], 
offsets[r + 1]).
+    std::vector<uint32_t> runes;
+    std::vector<uint32_t> offsets;
+    // The maximum-probability cut of a range: from each rune, the best weight 
of the rest of the
+    // range and the runes of the word that starts it.
+    std::vector<double> dp;
+    std::vector<uint32_t> best_len;
+    // Viterbi over a run of runes: for each rune and status, the status of 
the rune before; then
+    // the status of each rune.
+    std::vector<int8_t> path;
+    std::vector<uint8_t> status;
+
+    void release_oversized() {
+        release_oversized_scratch(runes);
+        release_oversized_scratch(offsets);
+        release_oversized_scratch(dp);
+        release_oversized_scratch(best_len);
+        release_oversized_scratch(path);
+        release_oversized_scratch(status);
+    }
+
+    size_t reserved_bytes() const {
+        return runes.capacity() * sizeof(uint32_t) + offsets.capacity() * 
sizeof(uint32_t) +
+               dp.capacity() * sizeof(double) + best_len.capacity() * 
sizeof(uint32_t) +
+               path.capacity() * sizeof(int8_t) + status.capacity() * 
sizeof(uint8_t);
+    }
+};
+
+} // namespace doris::segment_v2::inverted_index::jieba
diff --git 
a/be/src/storage/index/inverted/analysis/tokenizer/jieba/jieba_tokenizer.cpp 
b/be/src/storage/index/inverted/analysis/tokenizer/jieba/jieba_tokenizer.cpp
new file mode 100644
index 00000000000..fa5ef0206a7
--- /dev/null
+++ b/be/src/storage/index/inverted/analysis/tokenizer/jieba/jieba_tokenizer.cpp
@@ -0,0 +1,112 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "storage/index/inverted/analysis/tokenizer/jieba/jieba_tokenizer.h"
+
+#include <algorithm>
+#include <cstdint>
+#include <fstream>
+#include <mutex>
+#include <unordered_map>
+#include <utility>
+
+#include "common/exception.h"
+#include "storage/index/inverted/analysis/common/rune.h"
+#include "storage/index/inverted/analysis/scratch.h"
+#include "storage/index/inverted/analysis/token_batch.h"
+#include "storage/index/inverted/analysis/tokenizer/byte_classifier.h"
+#include "storage/index/inverted/analysis/tokenizer/jieba/jieba.h"
+
+namespace doris::segment_v2::inverted_index {
+
+namespace {
+
+bool has_ascii_upper(std::string_view term) {
+    return std::ranges::any_of(term,
+                               [](char c) { return 
is_ascii_upper(static_cast<uint8_t>(c)); });
+}
+
+} // namespace
+
+std::shared_ptr<const jieba::Jieba> load_jieba(const std::string& dict_dir) {
+    static std::mutex mutex;
+    static std::unordered_map<std::string, std::shared_ptr<const 
jieba::Jieba>> loaded;
+    std::lock_guard lock(mutex);
+    if (const auto it = loaded.find(dict_dir); it != loaded.end()) {
+        return it->second;
+    }
+    // idf.utf8, which CLucene's keyword extractor read, is no longer needed. 
The segmenters skip a
+    // stop word file they cannot open, which the extractor refused, so an 
unreadable file is
+    // refused here.
+    for (const char* name :
+         {"jieba.dict.utf8", "hmm_model.utf8", "user.dict.utf8", 
"stop_words.utf8"}) {
+        const std::string path = dict_dir + "/" + name;
+        if (!std::ifstream(path).is_open()) {
+            throw Exception(ErrorCode::INVERTED_INDEX_ANALYZER_ERROR,
+                            "chinese tokenizer dict file not found: " + path);
+        }
+    }
+    auto jieba = std::make_shared<const jieba::Jieba>(
+            dict_dir + "/jieba.dict.utf8", dict_dir + "/hmm_model.utf8",
+            dict_dir + "/user.dict.utf8", dict_dir + "/stop_words.utf8");
+    loaded.emplace(dict_dir, jieba);
+    return jieba;
+}
+
+JiebaTokenizer::JiebaTokenizer(std::shared_ptr<const jieba::Jieba> jieba, bool 
fine, bool lowercase)
+        : _jieba(std::move(jieba)), _fine(fine), _lowercase(lowercase) {}
+
+JiebaTokenizer::~JiebaTokenizer() = default;
+
+void JiebaTokenizer::reset(std::string_view text) {
+    _next = 0;
+    release_oversized_scratch(_value);
+    release_oversized_scratch(_words);
+    _scratch.release_oversized();
+    _value.assign(text);
+    if (_fine) {
+        _jieba->cut_all(_value, _scratch, _words);
+    } else {
+        _jieba->cut(_value, _scratch, _words);
+    }
+}
+
+bool JiebaTokenizer::next_batch(TokenBatch& batch) {
+    batch.clear();
+    while (_next < _words.size() && batch.size() < kBatchTokens) {
+        const std::string_view word = _words[_next++];
+        if (_lowercase && !word.empty() && static_cast<uint8_t>(word[0]) < 
0x80) {
+            // The whole word, even past what the term keeps: an overlapping 
word reads those bytes.
+            char* const begin = _value.data() + (word.data() - _value.data());
+            for (size_t i = 0; i < word.size(); ++i) {
+                begin[i] = ascii_lower(begin[i]);
+            }
+        }
+        const std::string_view term = word.substr(0, MAX_WORD_LEN);
+        if (_lowercase && has_ascii_upper(term)) {
+            batch.push_copy(term, 1);
+        } else {
+            batch.push(term, 1);
+        }
+    }
+    return batch.size() != 0;
+}
+
+JiebaTokenizerFactory::JiebaTokenizerFactory(bool fine, bool lowercase, const 
std::string& dict_dir)
+        : _fine(fine), _lowercase(lowercase), _jieba(load_jieba(dict_dir)) {}
+
+} // namespace doris::segment_v2::inverted_index
diff --git 
a/be/src/storage/index/inverted/analysis/tokenizer/jieba/jieba_tokenizer.h 
b/be/src/storage/index/inverted/analysis/tokenizer/jieba/jieba_tokenizer.h
new file mode 100644
index 00000000000..29a01a83c44
--- /dev/null
+++ b/be/src/storage/index/inverted/analysis/tokenizer/jieba/jieba_tokenizer.h
@@ -0,0 +1,102 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <cstddef>
+#include <memory>
+#include <string>
+#include <string_view>
+#include <vector>
+
+#include "storage/index/inverted/analysis/text_token_stream.h"
+#include "storage/index/inverted/analysis/tokenizer/jieba/jieba_scratch.h"
+#include "storage/index/inverted/analysis/tokenizer/tokenizer_factory.h"
+
+namespace doris::segment_v2::inverted_index {
+
+namespace jieba {
+class Jieba;
+} // namespace jieba
+
+// The dictionaries of the chinese parser in `dict_dir`: jieba.dict.utf8, 
hmm_model.utf8,
+// user.dict.utf8 and stop_words.utf8. They are loaded at the first call for a 
directory and shared
+// by every stream after it; a load that fails throws, and the next call tries 
again.
+std::shared_ptr<const jieba::Jieba> load_jieba(const std::string& dict_dir);
+
+// The tokenizer of the builtin chinese parser, which makes the tokens 
CLucene's ChineseTokenizer
+// made: jieba's words of the value, cut by MixSegment with the HMM in the 
coarse mode and by
+// FullSegment, which hands out every dictionary word it finds, in the fine 
mode, with jieba's stop
+// words dropped before anything else happens to them. A value jieba cannot 
decode has no token. A
+// word is cut to MAX_WORD_LEN bytes, which may split a character, and every 
word is one position
+// after the last.
+//
+// With `lowercase`, a word whose first byte is ASCII is lowercased whole, and 
a word that starts with
+// another byte is left as it is, capitals included: with "HHOK" in the user 
dictionary, H standing
+// for a Han character, the coarse mode makes "HHOK", and "QHH" makes "qHH". 
The words are
+// lowercased where they lie, in the copy
+// of the value, when they are handed out, as the old tokenizer did to its 
buffer, so a word sees the
+// lowercasing of the words handed out before it. That matters where the fine 
mode's words overlap:
+// it hands out a run of ASCII letters and digits whole and then the runs of 
letters and of digits
+// in it, and jieba reads a stray continuation byte and the byte after it as 
one character, so
+// "€" "7abcB" is the run "€7abcB", which starts with a byte that is not ASCII 
and keeps its
+// capital, followed by "abcB", which lowers it. A batch holds many words at 
once, so a word that
+// still has a capital after its turn is copied into the batch, where a later 
word cannot lower it.
+class JiebaTokenizer final : public TextTokenizer {
+public:
+    JiebaTokenizer(std::shared_ptr<const jieba::Jieba> jieba, bool fine, bool 
lowercase);
+    ~JiebaTokenizer() override;
+
+    void reset(std::string_view text) override;
+    bool next_batch(TokenBatch& batch) override;
+    // The copy of the value, the list of its words and what jieba cut them 
with.
+    size_t reserved_bytes() const override {
+        return _value.capacity() + _words.capacity() * 
sizeof(std::string_view) +
+               _scratch.reserved_bytes();
+    }
+
+private:
+    const std::shared_ptr<const jieba::Jieba> _jieba;
+    const bool _fine;
+    const bool _lowercase;
+    // The value, which jieba cuts and lowercasing rewrites; the words are 
views of it.
+    std::string _value;
+    std::vector<std::string_view> _words;
+    // The runes of the value and the arrays of the cut, kept for the next 
value.
+    jieba::JiebaScratch _scratch;
+    size_t _next = 0;
+};
+
+// The tokenizer of the builtin chinese parser, which is no custom tokenizer. 
It loads the
+// dictionaries when it is made, so that a missing file fails the analyzer, 
not its first stream.
+class JiebaTokenizerFactory final : public TokenizerFactory {
+public:
+    JiebaTokenizerFactory(bool fine, bool lowercase, const std::string& 
dict_dir);
+
+    void initialize(const Settings& settings) override {}
+
+    std::unique_ptr<TextTokenizer> create() const override {
+        return std::make_unique<JiebaTokenizer>(_jieba, _fine, _lowercase);
+    }
+
+private:
+    const bool _fine;
+    const bool _lowercase;
+    const std::shared_ptr<const jieba::Jieba> _jieba;
+};
+
+} // namespace doris::segment_v2::inverted_index
diff --git a/be/src/storage/index/inverted/analysis/tokenizer/jieba/segmenter.h 
b/be/src/storage/index/inverted/analysis/tokenizer/jieba/segmenter.h
new file mode 100644
index 00000000000..ae38e618437
--- /dev/null
+++ b/be/src/storage/index/inverted/analysis/tokenizer/jieba/segmenter.h
@@ -0,0 +1,408 @@
+// Derived from cppjieba's MPSegment.hpp, HMMSegment.hpp, MixSegment.hpp, 
FullSegment.hpp and
+// PreFilter.hpp (https://github.com/yanyiwu/cppjieba), MIT License, Copyright 
(c) 2013. See
+// dist/licenses/LICENSE-cppjieba.txt.
+
+#pragma once
+
+#include <algorithm>
+#include <array>
+#include <cstddef>
+#include <cstdint>
+#include <limits>
+#include <string_view>
+#include <vector>
+
+#include "common/logging.h"
+#include "storage/index/inverted/analysis/common/rune.h"
+#include "storage/index/inverted/analysis/tokenizer/jieba/dict_trie.h"
+#include "storage/index/inverted/analysis/tokenizer/jieba/flat_dict.h"
+#include "storage/index/inverted/analysis/tokenizer/jieba/hmm_model.h"
+#include "storage/index/inverted/analysis/tokenizer/jieba/jieba_scratch.h"
+
+namespace doris::segment_v2::inverted_index::jieba {
+
+// The longest dictionary word a cut looks for, in runes.
+inline constexpr uint32_t MAX_WORD_LENGTH = 512;
+
+// One cut of a sentence into words, which are views of it, with the stop 
words dropped as they
+// are, before anything lowercases them. A sentence that does not decode by 
decode_rune() has no
+// word at all.
+//
+// Both cuts split the sentence into ranges first, as upstream's PreFilter: 
every separator (space,
+// tab, newline, the fullwidth comma U+FF0C and the ideographic full stop 
U+3002, upstream's
+// SPECIAL_SEPARATORS) is a range of its own, and the runes between two 
separators are one range;
+// the full cut also makes each run of ASCII letters and digits a range of its 
own. "ab,c d" has
+// the ranges "ab" "," "c" " " "d" in both cuts; "Hab", H being a Han 
character, is one range in
+// the mix cut and "H" "ab" in the full cut. A separator is always one word, 
so it is handed out
+// without a cut: the maximum-probability cut of one rune is that rune, which 
the HMM keeps whole
+// too, and the full cut hands out the only rune of a range, since no word 
starts there but the
+// rune alone and none before it covers it.
+//
+// Upstream decoded into runes of 20 bytes, built a Dag of a few hundred bytes 
for every rune, and
+// allocated each step's vectors anew. Here the runes and the steps' arrays 
are in the scratch, and
+// the dictionary walk from a rune feeds the step that reads it directly, in 
the order the Dag held
+// the words: the rune alone, a word or not, then the longer words in 
increasing end.
+class Segmenter {
+public:
+    Segmenter(const FlatDict& dict, std::string_view sentence, JiebaScratch& 
scratch,
+              std::vector<std::string_view>& words)
+            : _dict(dict), _sentence(sentence), _scratch(scratch), 
_words(words) {}
+
+    // Jieba's default cut, upstream's MixSegment::Cut with the HMM: the 
maximum-probability cut of
+    // each range, with every run of its single runes that are no user word of 
one rune cut again
+    // by the HMM.
+    void mix_cut() {
+        _words.clear();
+        if (!decode()) {
+            return;
+        }
+        const uint32_t* const runes = _scratch.runes.data();
+        const auto count = static_cast<uint32_t>(_scratch.runes.size());
+        uint32_t begin = 0;
+        for (uint32_t cursor = 0; cursor < count; ++cursor) {
+            if (is_separator(runes[cursor])) {
+                if (begin < cursor) {
+                    mix_range(begin, cursor);
+                }
+                emit(cursor, cursor);
+                begin = cursor + 1;
+            }
+        }
+        if (begin < count) {
+            mix_range(begin, count);
+        }
+    }
+
+    // Jieba's full cut, upstream's FullSegment::Cut: every dictionary word of 
two runes or more in
+    // a range, overlapping, and the runes no word covers; a run of ASCII 
letters and digits is a
+    // word, and so is each run of two or more letters or of two or more 
digits inside it.
+    void full_cut() {
+        _words.clear();
+        if (!decode()) {
+            return;
+        }
+        const uint32_t* const runes = _scratch.runes.data();
+        const auto count = static_cast<uint32_t>(_scratch.runes.size());
+        uint32_t cursor = 0;
+        while (cursor < count) {
+            const uint32_t begin = cursor;
+            if (is_separator(runes[cursor])) {
+                emit(cursor, cursor);
+                ++cursor;
+            } else if (is_alnum(runes[cursor])) {
+                do {
+                    ++cursor;
+                } while (cursor < count && is_alnum(runes[cursor]));
+                alnum_range(begin, cursor);
+            } else {
+                do {
+                    ++cursor;
+                } while (cursor < count && !is_separator(runes[cursor]) &&
+                         !is_alnum(runes[cursor]));
+                full_range(begin, cursor);
+            }
+        }
+    }
+
+private:
+    // Decodes the sentence by decode_rune(), the rules of decode_runes(): the 
lead byte alone
+    // sets a rune's length, so a stray continuation byte reads as the lead of 
a two-byte rune, and
+    // "\x80" "7" is U+0037. False when a rune does not decode.
+    bool decode() {
+        DCHECK(_sentence.size() < std::numeric_limits<uint32_t>::max());
+        _scratch.runes.resize(_sentence.size());
+        _scratch.offsets.resize(_sentence.size() + 1);
+        uint32_t* const runes = _scratch.runes.data();
+        uint32_t* const offsets = _scratch.offsets.data();
+        uint32_t count = 0;
+        for (size_t offset = 0; offset < _sentence.size();) {
+            const RuneStrLite decoded =
+                    decode_rune(_sentence.data() + offset, _sentence.size() - 
offset);
+            if (decoded.len == 0) {
+                return false;
+            }
+            runes[count] = decoded.rune;
+            offsets[count] = static_cast<uint32_t>(offset);
+            ++count;
+            offset += decoded.len;
+        }
+        offsets[count] = static_cast<uint32_t>(_sentence.size());
+        _scratch.runes.resize(count);
+        _scratch.offsets.resize(count + 1);
+        return true;
+    }
+
+    // MixSegment::Cut over the runes [lo, hi): the maximum-probability cut, 
whose words of two
+    // runes or more, and single runes that are user words of one rune, are 
words, and whose other
+    // single runes are cut again by the HMM, run by run.
+    void mix_range(uint32_t lo, uint32_t hi) {
+        const uint32_t size = hi - lo;
+        max_probability_cut(lo, size);
+        const uint32_t* const runes = _scratch.runes.data() + lo;
+        const uint32_t* const best_len = _scratch.best_len.data();
+        uint32_t i = 0;
+        while (i < size) {
+            const uint32_t len = best_len[i];
+            if (len > 1 || _dict.is_user_single_word(runes[i])) {
+                emit(lo + i, lo + i + len - 1);
+                i += len;
+                continue;
+            }
+            uint32_t j = i + 1;
+            while (j < size && best_len[j] == 1 && 
!_dict.is_user_single_word(runes[j])) {
+                ++j;
+            }
+            hmm_range(lo + i, lo + j);
+            i = j;
+        }
+    }
+
+    // MPSegment's CalcDP over the `size` runes from `lo`, from the last rune 
to the first: the best
+    // weight of the rest of the range from each rune, and the runes of the 
word that starts it. A
+    // rune that is no word weighs the smallest weight of the dictionary. The 
weight of a word is
+    // added to the best weight after it, in that order, and the strict > 
keeps the first of equal
+    // sums: a tie goes to the shorter word. dp[size] is 0.0, where upstream 
added nothing: 0.0 +
+    // 0.0 + w is 0.0 + w bit for bit, whatever w is.
+    void max_probability_cut(uint32_t lo, uint32_t size) {
+        _scratch.dp.resize(size + 1);
+        _scratch.best_len.resize(size);
+        double* const dp = _scratch.dp.data();
+        uint32_t* const best_len = _scratch.best_len.data();
+        const uint32_t* const runes = _scratch.runes.data() + lo;
+        dp[size] = 0.0;
+        for (uint32_t i = size; i-- > 0;) {
+            uint32_t node = _dict.root_child(runes[i]);
+            double val = 0.0;
+            val += dp[i + 1];
+            val += node != FlatDict::kNoNode && _dict.has_value(node) ? 
_dict.weight(node)
+                                                                      : 
_dict.min_weight();
+            // Upstream started at MIN_DOUBLE and took the rune alone when it 
was above: a NaN, or
+            // a sum of MIN_DOUBLE or less, leaves MIN_DOUBLE and still a cut 
of the rune alone.
+            double best = std::max(MIN_DOUBLE, val);
+            uint32_t len = 1;
+            const uint32_t end = walk_end(i, size);
+            for (uint32_t k = i + 1; node != FlatDict::kNoNode && k < end; 
++k) {
+                node = _dict.child(node, runes[k]);
+                if (node != FlatDict::kNoNode && _dict.has_value(node)) {
+                    val = 0.0;
+                    val += dp[k + 1];
+                    val += _dict.weight(node);
+                    if (val > best) {
+                        best = val;
+                        len = k - i + 1;
+                    }
+                }
+            }
+            dp[i] = best;
+            best_len[i] = len;
+        }
+    }
+
+    // HMMSegment::Cut over the runes [lo, hi): an ASCII rune ends the run of 
other runes before
+    // it, which Viterbi cuts, and starts a word of its own: a letter followed 
by letters and
+    // digits, a digit followed by digits and '.', or else the rune alone, 
none reaching past hi.
+    void hmm_range(uint32_t lo, uint32_t hi) {
+        const uint32_t* const runes = _scratch.runes.data();
+        uint32_t left = lo;
+        uint32_t right = lo;
+        while (right != hi) {
+            if (runes[right] >= 0x80) {
+                ++right;
+                continue;
+            }
+            if (left != right) {
+                viterbi(left, right);
+            }
+            left = right;
+            right = ascii_word_end(runes, left, hi);
+            emit(left, right - 1);
+            left = right;
+        }
+        if (left != right) {
+            viterbi(left, right);
+        }
+    }
+
+    // The end of the word that starts at the ASCII rune `begin`: upstream's 
sequential letter
+    // rule, its numbers rule, or the rune alone.
+    static uint32_t ascii_word_end(const uint32_t* runes, uint32_t begin, 
uint32_t end) {
+        const uint32_t first = runes[begin];
+        uint32_t cursor = begin + 1;
+        if (is_letter(first)) {
+            while (cursor != end && (is_letter(runes[cursor]) || 
is_digit(runes[cursor]))) {
+                ++cursor;
+            }
+        } else if (is_digit(first)) {
+            while (cursor != end && (is_digit(runes[cursor]) || runes[cursor] 
== '.')) {
+                ++cursor;
+            }
+        }
+        return cursor;
+    }
+
+    // HMMSegment's Viterbi over the runes [lo, hi), which is not empty, and 
the words it makes: a
+    // word ends at every rune tagged E or S, and the last rune is always one 
of them. The sums run
+    // weight + transition + emission in that order; the strict > keeps the 
first of equal previous
+    // statuses, and E where none beats MIN_DOUBLE; the last rune is E when E 
and S are equal. Only
+    // the path is kept for every rune: the next rune needs the weights of 
this one alone.
+    void viterbi(uint32_t lo, uint32_t hi) {
+        constexpr size_t kStatuses = HMMModel::STATUS_SUM;
+        const uint32_t size = hi - lo;
+        _scratch.path.resize(size_t {size} * kStatuses);
+        _scratch.status.resize(size);
+        int8_t* const path = _scratch.path.data();
+        uint8_t* const status = _scratch.status.data();
+        const uint32_t* const runes = _scratch.runes.data() + lo;
+        const auto& start_prob = _dict.start_prob();
+        const auto& trans_prob = _dict.trans_prob();
+
+        std::array<double, kStatuses> weight;
+        const double* emit_prob = _dict.emit_prob(runes[0]);
+        for (size_t y = 0; y < kStatuses; ++y) {
+            weight[y] = start_prob[y] + emit_prob[y];
+            path[y] = -1;
+        }
+        for (uint32_t x = 1; x < size; ++x) {
+            emit_prob = _dict.emit_prob(runes[x]);
+            std::array<double, kStatuses> next;
+            for (size_t y = 0; y < kStatuses; ++y) {
+                double best = MIN_DOUBLE;
+                auto from = static_cast<int8_t>(HMMModel::E);
+                for (size_t pre_y = 0; pre_y < kStatuses; ++pre_y) {
+                    const double tmp = weight[pre_y] + trans_prob[pre_y][y] + 
emit_prob[y];
+                    if (tmp > best) {
+                        best = tmp;
+                        from = static_cast<int8_t>(pre_y);
+                    }
+                }
+                next[y] = best;
+                path[size_t {x} * kStatuses + y] = from;
+            }
+            weight = next;
+        }
+
+        int stat = weight[HMMModel::E] >= weight[HMMModel::S] ? HMMModel::E : 
HMMModel::S;
+        // The path of the first rune is -1, which `stat` takes after its last 
use.
+        for (uint32_t x = size; x-- > 0;) {
+            status[x] = static_cast<uint8_t>(stat);
+            stat = path[size_t {x} * kStatuses + static_cast<size_t>(stat)];
+        }
+        uint32_t left = lo;
+        for (uint32_t x = 0; x < size; ++x) {
+            // E and S are odd.
+            if (status[x] % 2 != 0) {
+                emit(left, lo + x);
+                left = lo + x + 1;
+            }
+        }
+    }
+
+    // FullSegment::Cut over the runes [lo, hi). Every word of two runes or 
more is handed out. A
+    // rune that starts no longer word is handed out, a word or not, when 
max_idx, the end of the
+    // words before it as far as word_len tells, does not reach past it.
+    //
+    // word_len keeps its value from one word to the next and from one rune to 
the next: a rune
+    // that is no word does not set it, yet still raises max_idx to u_idx + 
the length of the last
+    // word seen. This looks like a bug, but it decides which single runes are 
handed out, so it is
+    // kept as upstream has it. Say A, B, C and D are CJK runes, and AB is the 
only word of the
+    // dictionary that starts with any of them. The runes ABCD give only AB: B 
is skipped because AB
+    // covers it (max_idx 2 > u_idx 1), and the stale 2 then raises max_idx to 
3; C meets 3 > 2, is
+    // skipped and raises it to 4, and D meets 4 > 3. With a word_len of 1 for 
a rune that is no
+    // word, max_idx would stay 2 after B, and C and D would both be handed 
out.
+    //
+    // Upstream had the words of a rune in its Dag before it looked at the 
rune alone, and handed
+    // the rune out only when no longer word started there. The walk here 
takes the rune alone at
+    // the first longer word, where it hands nothing out, or after the walk 
when there is none;
+    // word_len and max_idx change in upstream's order either way.
+    void full_range(uint32_t lo, uint32_t hi) {
+        const uint32_t size = hi - lo;
+        const uint32_t* const runes = _scratch.runes.data() + lo;
+        size_t max_idx = 0;
+        size_t word_len = 0;
+        for (uint32_t u_idx = 0; u_idx < size; ++u_idx) {
+            uint32_t node = _dict.root_child(runes[u_idx]);
+            const bool alone_is_word = node != FlatDict::kNoNode && 
_dict.has_value(node);
+            bool has_longer = false;
+            const uint32_t end = walk_end(u_idx, size);
+            for (uint32_t k = u_idx + 1; node != FlatDict::kNoNode && k < end; 
++k) {
+                node = _dict.child(node, runes[k]);
+                if (node == FlatDict::kNoNode || !_dict.has_value(node)) {
+                    continue;
+                }
+                if (!has_longer) {
+                    has_longer = true;
+                    if (alone_is_word) {
+                        word_len = 1;
+                    }
+                    max_idx = std::max(max_idx, u_idx + word_len);
+                }
+                word_len = size_t {k} - u_idx + 1;
+                emit(lo + u_idx, lo + k);
+                max_idx = std::max(max_idx, u_idx + word_len);
+            }
+            if (!has_longer) {
+                if (max_idx <= u_idx) {
+                    emit(lo + u_idx, lo + u_idx);
+                }
+                if (alone_is_word) {
+                    word_len = 1;
+                }
+                max_idx = std::max(max_idx, u_idx + word_len);
+            }
+        }
+    }
+
+    // FullSegment::CutAlnum over a run of ASCII letters and digits [lo, hi): 
the whole run, then
+    // the runs of two or more digits or of two or more letters in it, unless 
one is the whole run:
+    // "abc123" gives "abc123", "abc" and "123", and "abc" only "abc".
+    void alnum_range(uint32_t lo, uint32_t hi) {
+        emit(lo, hi - 1);
+        const uint32_t* const runes = _scratch.runes.data();
+        uint32_t cursor = lo;
+        while (cursor != hi) {
+            const bool digits = is_digit(runes[cursor]);
+            DCHECK(digits || is_letter(runes[cursor]));
+            const uint32_t start = cursor;
+            do {
+                ++cursor;
+            } while (cursor != hi && (digits ? is_digit(runes[cursor]) : 
is_letter(runes[cursor])));
+            if (cursor - start > 1 && (start != lo || cursor != hi)) {
+                emit(start, cursor - 1);
+            }
+        }
+    }
+
+    // Where the dictionary walk from the rune `begin` of a range of `size` 
runes stops: a word of
+    // more than MAX_WORD_LENGTH runes is never found, as upstream.
+    static uint32_t walk_end(uint32_t begin, uint32_t size) {
+        return static_cast<uint32_t>(std::min<size_t>(size, size_t {begin} + 
MAX_WORD_LENGTH));
+    }
+
+    // Hands out the runes [left, right] as a word, unless it is a stop word.
+    void emit(uint32_t left, uint32_t right) {
+        const uint32_t begin = _scratch.offsets[left];
+        const std::string_view word(_sentence.data() + begin, 
_scratch.offsets[right + 1] - begin);
+        if (!_dict.is_stop_word(word)) {
+            _words.push_back(word);
+        }
+    }
+
+    static bool is_separator(uint32_t rune) {
+        return rune == ' ' || rune == '\t' || rune == '\n' || rune == 0xFF0C 
|| rune == 0x3002;
+    }
+
+    static bool is_digit(uint32_t rune) { return rune >= '0' && rune <= '9'; }
+
+    static bool is_letter(uint32_t rune) {
+        return (rune >= 'A' && rune <= 'Z') || (rune >= 'a' && rune <= 'z');
+    }
+
+    static bool is_alnum(uint32_t rune) { return is_digit(rune) || 
is_letter(rune); }
+
+    const FlatDict& _dict;
+    const std::string_view _sentence;
+    JiebaScratch& _scratch;
+    std::vector<std::string_view>& _words;
+};
+
+} // namespace doris::segment_v2::inverted_index::jieba
diff --git 
a/be/src/storage/index/inverted/analysis/tokenizer/jieba/string_util.h 
b/be/src/storage/index/inverted/analysis/tokenizer/jieba/string_util.h
new file mode 100644
index 00000000000..9c8029dd714
--- /dev/null
+++ b/be/src/storage/index/inverted/analysis/tokenizer/jieba/string_util.h
@@ -0,0 +1,70 @@
+// Derived from limonp's StringUtil.hpp (https://github.com/yanyiwu/limonp), 
MIT License,
+// Copyright (c) 2013. See dist/licenses/LICENSE-limonp.txt.
+//
+// Only Split and Trim, which jieba's dictionary and model loading use.
+
+#pragma once
+
+#include <algorithm>
+#include <cctype>
+#include <cstddef>
+#include <string>
+#include <vector>
+
+namespace doris::segment_v2::inverted_index::jieba {
+
+// Splits `src` at every byte that is one of `pattern`. Two separators in a 
row give the empty piece
+// between them and a separator at the start an empty first piece, but a 
separator at the end gives
+// no piece after it: "a  b " split at " " is {"a", "", "b"}, and "" is no 
piece at all. Upstream's
+// maxsplit parameter is gone: no caller passed one, and with its default of 
npos it never stopped
+// a split.
+inline void split(const std::string& src, std::vector<std::string>& res,
+                  const std::string& pattern) {
+    res.clear();
+    size_t start = 0;
+    size_t end = 0;
+    while (start < src.size()) {
+        end = src.find_first_of(pattern, start);
+        if (std::string::npos == end) {
+            res.push_back(src.substr(start));
+            return;
+        }
+        res.push_back(src.substr(start, end - start));
+        start = end + 1;
+    }
+}
+
+inline std::vector<std::string> split(const std::string& src, const 
std::string& pattern) {
+    std::vector<std::string> res;
+    split(src, res, pattern);
+    return res;
+}
+
+// Whether `c` is a space by std::isspace, so by the C locale, which the BE 
never changes. A char
+// is sign-extended on the way in, so where char is signed a byte of 0x80 and 
above is above 0xFF
+// here, and no space.
+inline bool is_space(unsigned c) {
+    // when passing large int as the argument of isspace, it core dump, so 
here need a type cast.
+    return c > 0xff ? false : std::isspace(static_cast<int>(c & 0xff)) != 0;
+}
+
+inline std::string& ltrim(std::string& s) {
+    s.erase(s.begin(), std::ranges::find_if(s, [](const char& c) {
+                return !is_space(static_cast<unsigned>(c));
+            }));
+    return s;
+}
+
+inline std::string& rtrim(std::string& s) {
+    s.erase(std::ranges::find_if(s.rbegin(), s.rend(),
+                                 [](const char& c) { return 
!is_space(static_cast<unsigned>(c)); })
+                    .base(),
+            s.end());
+    return s;
+}
+
+inline std::string& trim(std::string& s) {
+    return ltrim(rtrim(s));
+}
+
+} // namespace doris::segment_v2::inverted_index::jieba
diff --git a/be/src/storage/index/inverted/analysis/tokenizer/jieba/unicode.h 
b/be/src/storage/index/inverted/analysis/tokenizer/jieba/unicode.h
new file mode 100644
index 00000000000..59ffa55a69c
--- /dev/null
+++ b/be/src/storage/index/inverted/analysis/tokenizer/jieba/unicode.h
@@ -0,0 +1,36 @@
+// Derived from cppjieba's Unicode.hpp (https://github.com/yanyiwu/cppjieba), 
MIT License,
+// Copyright (c) 2013. See dist/licenses/LICENSE-cppjieba.txt.
+//
+// Only what the dictionary and model loading use. The decoder comes from 
analysis/common/rune.h,
+// which carries jieba's decoding rules; the cuts decode with it too 
(segmenter.h).
+
+#pragma once
+
+#include <cstdint>
+#include <string>
+#include <vector>
+
+#include "storage/index/inverted/analysis/common/rune.h"
+
+namespace doris::segment_v2::inverted_index::jieba {
+
+using Rune = uint32_t;
+// The code points of a dictionary word: the key of the trie.
+using Unicode = std::vector<Rune>;
+
+// The code points of `s`, decoded with decode_runes(). Returns false with no 
code point when one of
+// them does not decode. Upstream's DecodeRunesInString(const string&, 
Unicode&).
+inline bool decode_unicode(const std::string& s, Unicode& unicode) {
+    unicode.clear();
+    RuneStrArray runes;
+    if (!decode_runes(s.data(), s.size(), runes)) {
+        return false;
+    }
+    unicode.reserve(runes.size());
+    for (const RuneStr& rune : runes) {
+        unicode.push_back(rune.rune);
+    }
+    return true;
+}
+
+} // namespace doris::segment_v2::inverted_index::jieba
diff --git a/be/test/storage/index/inverted/analysis/component_stream_test.cpp 
b/be/test/storage/index/inverted/analysis/component_stream_test.cpp
index d2f1fc1fff3..da8984885c5 100644
--- a/be/test/storage/index/inverted/analysis/component_stream_test.cpp
+++ b/be/test/storage/index/inverted/analysis/component_stream_test.cpp
@@ -50,6 +50,7 @@
 #include "storage/index/inverted/analysis/tokenizer/basic/basic_tokenizer.h"
 #include 
"storage/index/inverted/analysis/tokenizer/english/english_tokenizer.h"
 #include "storage/index/inverted/analysis/tokenizer/icu/icu_tokenizer.h"
+#include "storage/index/inverted/analysis/tokenizer/jieba/jieba_tokenizer.h"
 #include "storage/index/inverted/analysis/tokenizer/ngram/gram_tokenizer.h"
 #include "storage/index/inverted/analysis/tokenizer/ngram/ngram_tokenizer.h"
 #include "storage/index/inverted/analysis/tokenizer/pinyin/pinyin_tokenizer.h"
@@ -284,6 +285,18 @@ TEST(ComponentStreamTest, 
EnglishTokenizerKeepsNoRowSizedScratch) {
     }
 }
 
+TEST(ComponentStreamTest, JiebaTokenizerReleasesItsCopyAndWordList) {
+    const JiebaTokenizerFactory factory(
+            false, true, 
"./contrib/clucene/src/contribs-lib/CLucene/analysis/jieba/dict");
+    expect_text_stream_contract([&] { return factory.create(); }, 
contract_texts());
+    auto tokenizer = factory.create();
+    auto& jieba = as<JiebaTokenizer>(*tokenizer);
+    expect_released_at_next_row(*tokenizer, repeat_to("我们 ", kLongRowBytes), 
"short", true, [&] {
+        return jieba._value.capacity() + jieba._words.capacity() * 
sizeof(std::string_view) +
+               jieba._scratch.reserved_bytes();
+    });
+}
+
 TEST(ComponentStreamTest, Uax29TokenizerReleasesItsLoweredCopy) {
     for (bool lowercase : {true, false}) {
         SCOPED_TRACE(lowercase);
diff --git 
a/be/test/storage/index/inverted/analysis/tokenizer/jieba_tokenizer_test.cpp 
b/be/test/storage/index/inverted/analysis/tokenizer/jieba_tokenizer_test.cpp
new file mode 100644
index 00000000000..80ad6e7636f
--- /dev/null
+++ b/be/test/storage/index/inverted/analysis/tokenizer/jieba_tokenizer_test.cpp
@@ -0,0 +1,496 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "storage/index/inverted/analysis/tokenizer/jieba/jieba_tokenizer.h"
+
+#include <CLucene.h> // IWYU pragma: keep
+#include <gtest/gtest.h>
+#include <unistd.h>
+
+#include <algorithm>
+#include <cstdlib>
+#include <filesystem>
+#include <fstream>
+#include <iostream>
+#include <memory>
+#include <random>
+#include <string>
+#include <string_view>
+#include <thread>
+#include <vector>
+
+#include "CLucene/analysis/LanguageBasedAnalyzer.h"
+#include "common/config.h"
+#include "common/exception.h"
+#include "storage/index/inverted/analysis/analyzer/analyzer.h"
+#include "storage/index/inverted/analysis/analyzer/chain_analyzer.h"
+#include "storage/index/inverted/analysis/common/rune.h"
+#include "storage/index/inverted/analysis/golden/golden_driver.h"
+#include "storage/index/inverted/analysis/text_stream_contract.h"
+#include "storage/index/inverted/analysis/token_batch.h"
+#include "storage/index/inverted/analysis/tokenizer/byte_classifier.h"
+#include "storage/index/inverted/analysis/tokenizer/jieba/jieba.h"
+
+namespace doris::segment_v2::inverted_index {
+namespace {
+
+const std::filesystem::path kShippedDict =
+        "./contrib/clucene/src/contribs-lib/CLucene/analysis/jieba/dict";
+
+// Points the dictionary path at jieba's shipped dictionaries while a test 
runs.
+class JiebaDictScope {
+public:
+    JiebaDictScope() : _saved(config::inverted_index_dict_path) {
+        config::inverted_index_dict_path = kShippedDict.string();
+    }
+    ~JiebaDictScope() { config::inverted_index_dict_path = _saved; }
+
+private:
+    const std::string _saved;
+};
+
+// A dictionary directory of a test's own: the shipped files, with 
user.dict.utf8 replaced by
+// `user_words` and the files in `missing` left out.
+class OwnDict {
+public:
+    explicit OwnDict(const std::vector<std::string>& user_words,
+                     const std::vector<std::string>& missing = {}) {
+        static int counter = 0;
+        _dir = std::filesystem::temp_directory_path() /
+               ("jieba_own_dict_" + std::to_string(::getpid()) + "_" + 
std::to_string(counter++));
+        std::filesystem::create_directories(_dir);
+        for (const char* name :
+             {"jieba.dict.utf8", "hmm_model.utf8", "stop_words.utf8", 
"idf.utf8"}) {
+            if (std::ranges::find(missing, name) == missing.end()) {
+                std::filesystem::copy_file(kShippedDict / name, _dir / name);
+            }
+        }
+        if (std::ranges::find(missing, "user.dict.utf8") == missing.end()) {
+            std::ofstream user(_dir / "user.dict.utf8");
+            for (const std::string& word : user_words) {
+                user << word << "\n";
+            }
+        }
+    }
+    ~OwnDict() { std::filesystem::remove_all(_dir); }
+
+    std::string dir() const { return _dir.string(); }
+    void add(const char* name) const {
+        std::filesystem::copy_file(kShippedDict / name, _dir / name);
+    }
+
+private:
+    std::filesystem::path _dir;
+};
+
+// The chinese parser as production builds it.
+TextAnalyzerPtr chinese_analyzer(bool fine, bool lowercase) {
+    return 
InvertedIndexAnalyzer::create_builtin_analyzer(InvertedIndexParserType::PARSER_CHINESE,
+                                                          fine ? 
"fine_grained" : "coarse_grained",
+                                                          lowercase ? "true" : 
"false", "");
+}
+
+// CLucene's LanguageBasedAnalyzer, which made the chinese tokens before.
+std::unique_ptr<golden::GoldenDriver> clucene_chinese(bool fine, bool 
lowercase) {
+    return golden::make_lucene_driver([fine, lowercase] {
+        auto analyzer =
+                
std::make_shared<lucene::analysis::LanguageBasedAnalyzer>(L"chinese", false);
+        analyzer->initDict(kShippedDict.string());
+        analyzer->setMode(fine ? lucene::analysis::AnalyzerMode::All
+                               : lucene::analysis::AnalyzerMode::Default);
+        analyzer->set_lowercase(lowercase);
+        return analyzer;
+    });
+}
+
+std::string bytes(std::initializer_list<int> values) {
+    std::string text;
+    for (int value : values) {
+        text.push_back(static_cast<char>(value));
+    }
+    return text;
+}
+
+// Values that stress the segmenters: Chinese text, dictionary words, ASCII in 
both cases, rare
+// and supplementary-plane characters jieba has no word for, separators and 
invalid bytes.
+std::string random_value(std::mt19937_64& random) {
+    static const std::vector<std::string> kPieces = {"我们",
+                                                     "中华人民共和国",
+                                                     "云计算",
+                                                     "区块链",
+                                                     "研究生命起源",
+                                                     "T恤",
+                                                     "B超",
+                                                     "AT&T",
+                                                     "C++",
+                                                     "Hello",
+                                                     "WORLD",
+                                                     "abc",
+                                                     "123",
+                                                     "3.14",
+                                                     " ",
+                                                     ",",
+                                                     "。",
+                                                     "\t",
+                                                     "\n",
+                                                     "、",
+                                                     "!",
+                                                     "的",
+                                                     "了",
+                                                     "𠮷",
+                                                     "𪚥",
+                                                     "ㄅ",
+                                                     "ひらがな",
+                                                     "カタカナ",
+                                                     "한국어",
+                                                     "é",
+                                                     "😀",
+                                                     "-",
+                                                     "_"};
+    std::string value;
+    const size_t pieces = random() % 40;
+    for (size_t i = 0; i < pieces; ++i) {
+        if (random() % 25 == 0) {
+            value.push_back(static_cast<char>(random()));
+        } else {
+            value += kPieces[random() % kPieces.size()];
+        }
+    }
+    return value;
+}
+
+void expect_same_as_clucene(const std::string& value, golden::GoldenDriver& 
candidate,
+                            golden::GoldenDriver& oracle) {
+    ASSERT_EQ(candidate.fresh(value), oracle.fresh(value))
+            << golden::format_line(oracle.fresh(value));
+    ASSERT_EQ(candidate.reused(value), oracle.reused(value));
+}
+
+struct Mode {
+    bool fine;
+    bool lowercase;
+};
+constexpr Mode kModes[] = {{.fine = false, .lowercase = false},
+                           {.fine = false, .lowercase = true},
+                           {.fine = true, .lowercase = false},
+                           {.fine = true, .lowercase = true}};
+
+TEST(JiebaTokenizerTest, MatchesCLuceneOnRandomValues) {
+    JiebaDictScope dict;
+    for (const Mode mode : kModes) {
+        SCOPED_TRACE(testing::Message() << "fine " << mode.fine << " lowercase 
" << mode.lowercase);
+        auto oracle = clucene_chinese(mode.fine, mode.lowercase);
+        auto candidate = golden::make_text_driver(chinese_analyzer(mode.fine, 
mode.lowercase));
+        std::mt19937_64 random(20261104 + mode.fine * 2 + mode.lowercase);
+        for (int round = 0; round < 3000; ++round) {
+            expect_same_as_clucene(random_value(random), *candidate, *oracle);
+        }
+    }
+}
+
+// Every word of the dictionaries, 64 to a value with a random separator or 
none between them, in
+// its own case and with its ASCII letters raised.
+TEST(JiebaTokenizerTest, MatchesCLuceneOnEveryDictionaryWord) {
+    JiebaDictScope dict;
+    std::vector<std::string> words;
+    for (const char* name : {"jieba.dict.utf8", "user.dict.utf8"}) {
+        std::ifstream file(kShippedDict / name);
+        std::string line;
+        while (std::getline(file, line)) {
+            words.push_back(line.substr(0, line.find(' ')));
+        }
+    }
+    ASSERT_GT(words.size(), 340000);
+    static const std::vector<std::string> kSeparators = {"", "", " ", ",", 
"a", "Z9"};
+    std::mt19937_64 random(20261106);
+    std::vector<std::string> values;
+    for (size_t i = 0; i < words.size(); i += 64) {
+        std::string value;
+        for (size_t j = i; j < std::min(words.size(), i + 64); ++j) {
+            std::string word = words[j];
+            if (random() % 2 == 0) {
+                for (char& c : word) {
+                    c = static_cast<char>(c >= 'a' && c <= 'z' ? c - 32 : c);
+                }
+            }
+            value += word + kSeparators[random() % kSeparators.size()];
+        }
+        values.push_back(std::move(value));
+    }
+    for (const Mode mode : kModes) {
+        SCOPED_TRACE(testing::Message() << "fine " << mode.fine << " lowercase 
" << mode.lowercase);
+        auto oracle = clucene_chinese(mode.fine, mode.lowercase);
+        auto candidate = golden::make_text_driver(chinese_analyzer(mode.fine, 
mode.lowercase));
+        for (const std::string& value : values) {
+            ASSERT_EQ(candidate->fresh(value), oracle->fresh(value))
+                    << golden::format_line(oracle->fresh(value));
+        }
+    }
+}
+
+TEST(JiebaTokenizerTest, MatchesCLuceneOnMixedCaseUnknownWordsAndInvalidBytes) 
{
+    JiebaDictScope dict;
+    const std::vector<std::string> values = {"Hello世界HELLO",
+                                             "我买了一件T恤和B超",
+                                             "AT&T和C++以及c#",
+                                             "MiXeD大小写MiXeD",
+                                             "𠮷野家的𪚥𪚥",
+                                             "ㄅㄆㄇ未登录词ㄈ",
+                                             "你好" + bytes({0xE4, 0xB8}),
+                                             "abc" + bytes({0xFF}) + "中文",
+                                             bytes({0x80}) + "中文",
+                                             "中文" + bytes({0xC3}),
+                                             std::string(600, 'A'),
+                                             "中" + std::string(300, 'b') + "文",
+                                             ""};
+    for (const Mode mode : kModes) {
+        SCOPED_TRACE(testing::Message() << "fine " << mode.fine << " lowercase 
" << mode.lowercase);
+        auto oracle = clucene_chinese(mode.fine, mode.lowercase);
+        auto candidate = golden::make_text_driver(chinese_analyzer(mode.fine, 
mode.lowercase));
+        for (const std::string& value : values) {
+            expect_same_as_clucene(value, *candidate, *oracle);
+        }
+    }
+}
+
+// A value that does not decode has no token at all, the decodable head 
included.
+TEST(JiebaTokenizerTest, AValueThatDoesNotDecodeHasNoToken) {
+    JiebaDictScope dict;
+    for (const Mode mode : kModes) {
+        EXPECT_TRUE(collect_terms(*chinese_analyzer(mode.fine, mode.lowercase),
+                                  "你好" + bytes({0xE4, 0xB8}))
+                            .empty());
+    }
+}
+
+// The terms the old tokenizer handed out, one at a time: each word lowercased 
where it lies when
+// its first byte is ASCII, and read right then, so that a later word's 
lowercasing cannot reach it.
+std::vector<std::string> handed_out_one_at_a_time(const jieba::Jieba& jieba, 
std::string value,
+                                                  bool fine) {
+    std::vector<std::string_view> words;
+    if (fine) {
+        jieba.cut_all(value, words);
+    } else {
+        jieba.cut(value, words);
+    }
+    std::vector<std::string> terms;
+    for (const std::string_view word : words) {
+        if (!word.empty() && static_cast<uint8_t>(word[0]) < 0x80) {
+            char* begin = value.data() + (word.data() - value.data());
+            for (size_t i = 0; i < word.size(); ++i) {
+                begin[i] = ascii_lower(begin[i]);
+            }
+        }
+        terms.emplace_back(word.substr(0, MAX_WORD_LEN));
+    }
+    return terms;
+}
+
+// Random values with the words of kMixedWords in them, and runs of ASCII 
letters and digits that
+// start with a stray continuation byte: after a separator or a Chinese 
character, "\x80" "7abcB"
+// is one run whose first byte is not ASCII.
+std::vector<std::string> mixed_values() {
+    std::mt19937_64 random(20261108);
+    std::vector<std::string> values;
+    for (int i = 0; i < 2000; ++i) {
+        values.push_back(random_value(random) + "卡拉OK拉OK,\x80" + "7abcB" +
+                         random_value(random) + "Q版本 \x80" + "9XyZ中\x80" + 
"Ab1");
+    }
+    return values;
+}
+
+const std::vector<std::string> kMixedWords = {"卡拉OK", "OK",  "拉O", "Q版本",
+                                              "AT&T",   "C++", "中AB"};
+
+// The fine mode hands out a run of ASCII letters and digits whole and then 
the runs of letters and
+// of digits in it, and a stray continuation byte reads as one character with 
the byte after it:
+// "\x80" "7" is U+0037. So "\x807abcB" comes out keeping its capital, the 
first byte not being
+// ASCII, and "abcB" lowers that capital after it; the first word must keep it 
in the batch.
+TEST(JiebaTokenizerTest, BatchesHandOutWhatOneWordAtATimeWould) {
+    const OwnDict own(kMixedWords);
+    const std::shared_ptr<const jieba::Jieba> jieba = load_jieba(own.dir());
+    for (bool fine : {false, true}) {
+        JiebaTokenizerFactory factory(fine, true, own.dir());
+        auto tokenizer = factory.create();
+        for (const std::string& value : mixed_values()) {
+            ASSERT_EQ(collect_terms(*tokenizer, value),
+                      handed_out_one_at_a_time(*jieba, value, fine))
+                    << "fine " << fine << ": " << value;
+        }
+    }
+}
+
+// With lowercase on, a word is lowercased whole when its first byte is ASCII 
and left as it is
+// otherwise, capitals and all.
+TEST(JiebaTokenizerTest, TheFirstByteDecidesWhetherAWordIsLowercased) {
+    const OwnDict own({"卡拉OK", "Q版本"});
+    JiebaTokenizerFactory coarse(false, true, own.dir());
+    EXPECT_EQ(collect_terms(*coarse.create(), "卡拉OK"), 
(std::vector<std::string> {"卡拉OK"}));
+    EXPECT_EQ(collect_terms(*coarse.create(), "Q版本"), 
(std::vector<std::string> {"q版本"}));
+    EXPECT_EQ(collect_terms(*coarse.create(), "T恤"), (std::vector<std::string> 
{"t恤"}));
+    JiebaTokenizerFactory keep_case(false, false, own.dir());
+    EXPECT_EQ(collect_terms(*keep_case.create(), "Q版本"), 
(std::vector<std::string> {"Q版本"}));
+}
+
+TEST(JiebaTokenizerTest, KeepsSupplementaryPlaneCharactersAndUserWords) {
+    const OwnDict own({"𠮷野家", "𪚥𪚥"});
+    for (bool fine : {false, true}) {
+        JiebaTokenizerFactory factory(fine, true, own.dir());
+        const std::vector<std::string> terms = 
collect_terms(*factory.create(), "去𠮷野家吃𪚥𪚥饭");
+        EXPECT_NE(std::ranges::find(terms, "𠮷野家"), terms.end()) << fine;
+        EXPECT_NE(std::ranges::find(terms, "𪚥𪚥"), terms.end()) << fine;
+        const std::shared_ptr<const jieba::Jieba> jieba = 
load_jieba(own.dir());
+        EXPECT_EQ(terms, handed_out_one_at_a_time(*jieba, "去𠮷野家吃𪚥𪚥饭", fine));
+    }
+}
+
+// How many of the values below the two sides cut differently, each printed.
+int mismatches_on_a_dictionary_of_its_own() {
+    const OwnDict own({"卡拉OK", "OK", "拉O", "Q版本", "𠮷野家", "𪚥𪚥", "云计算"});
+    const std::vector<std::string> values = {"卡拉OK",
+                                             "我们唱卡拉OK吧,OK",
+                                             "拉OK卡拉OK",
+                                             "Q版本和q版本",
+                                             "去𠮷野家吃𪚥𪚥饭",
+                                             "云计算KTV卡拉OK",
+                                             "OK卡拉OKOK",
+                                             "卡拉ok",
+                                             "T恤B超"};
+    int mismatches = 0;
+    for (const Mode mode : kModes) {
+        auto oracle = golden::make_lucene_driver([&own, mode] {
+            auto analyzer =
+                    
std::make_shared<lucene::analysis::LanguageBasedAnalyzer>(L"chinese", false);
+            analyzer->initDict(own.dir());
+            analyzer->setMode(mode.fine ? lucene::analysis::AnalyzerMode::All
+                                        : 
lucene::analysis::AnalyzerMode::Default);
+            analyzer->set_lowercase(mode.lowercase);
+            return analyzer;
+        });
+        auto candidate = 
golden::make_text_driver(std::make_shared<ChainAnalyzer>(
+                std::vector<CharFilterFactoryPtr> {},
+                std::make_shared<JiebaTokenizerFactory>(mode.fine, 
mode.lowercase, own.dir()),
+                std::vector<TokenFilterFactoryPtr> {}));
+        for (const std::string& value : values) {
+            if (candidate->fresh(value) != oracle->fresh(value) ||
+                candidate->reused(value) != oracle->reused(value)) {
+                std::cerr << "fine " << mode.fine << " lowercase " << 
mode.lowercase << " " << value
+                          << ": " << golden::format_line(oracle->fresh(value)) 
<< " vs "
+                          << golden::format_line(candidate->fresh(value)) << 
"\n";
+                ++mismatches;
+            }
+        }
+    }
+    return mismatches;
+}
+
+// The live differential on a dictionary of the test's own, with words that 
start with a Chinese
+// character and hold capitals, overlap ASCII words, or lie in a supplementary 
plane: the shipped
+// dictionaries have none. CLucene's jieba keeps the first dictionary it loads 
for the rest of the
+// process, so this runs in a process of its own, where it loads this one.
+TEST(JiebaTokenizerDeathTest, MatchesCLuceneOnADictionaryOfItsOwn) {
+    GTEST_FLAG_SET(death_test_style, "threadsafe");
+    // _Exit: static destructors of the test binary are not safe to run in the 
child.
+    EXPECT_EXIT(std::_Exit(mismatches_on_a_dictionary_of_its_own() == 0 ? 0 : 
1),
+                testing::ExitedWithCode(0), "");
+}
+
+TEST(JiebaTokenizerTest, LoadsWithoutIdf) {
+    const OwnDict own({"云计算"}, {"idf.utf8"});
+    JiebaTokenizerFactory factory(false, false, own.dir());
+    EXPECT_EQ(collect_terms(*factory.create(), "云计算"), 
(std::vector<std::string> {"云计算"}));
+}
+
+// A missing file fails the factory, and a later load of the same directory 
tries again.
+TEST(JiebaTokenizerTest, AFailedLoadIsTriedAgain) {
+    const OwnDict own({}, {"hmm_model.utf8"});
+    try {
+        JiebaTokenizerFactory factory(false, false, own.dir());
+        FAIL() << "a missing hmm_model.utf8 must fail";
+    } catch (const Exception& e) {
+        EXPECT_EQ(e.code(), ErrorCode::INVERTED_INDEX_ANALYZER_ERROR);
+        EXPECT_NE(std::string(e.what()).find("hmm_model.utf8"), 
std::string::npos);
+    }
+    own.add("hmm_model.utf8");
+    JiebaTokenizerFactory factory(false, false, own.dir());
+    EXPECT_EQ(collect_terms(*factory.create(), "中华人民共和国"),
+              (std::vector<std::string> {"中华人民共和国"}));
+}
+
+TEST(JiebaTokenizerTest, EightThreadsShareTheDictionaries) {
+    JiebaDictScope dict;
+    std::mt19937_64 random(20261107);
+    std::vector<std::string> values;
+    for (int i = 0; i < 400; ++i) {
+        values.push_back(random_value(random));
+    }
+    for (const Mode mode : kModes) {
+        const TextAnalyzerPtr analyzer = chinese_analyzer(mode.fine, 
mode.lowercase);
+        std::vector<std::vector<std::string>> expected;
+        for (const std::string& value : values) {
+            expected.push_back(collect_terms(*analyzer, value));
+        }
+        std::vector<int> mismatches(8, 0);
+        std::vector<std::thread> threads;
+        for (int t = 0; t < 8; ++t) {
+            threads.emplace_back([&, t] {
+                auto stream = analyzer->create_stream();
+                for (int round = 0; round < 3; ++round) {
+                    for (size_t i = 0; i < values.size(); ++i) {
+                        mismatches[t] += collect_terms(*stream, values[i]) != 
expected[i];
+                    }
+                }
+            });
+        }
+        for (std::thread& thread : threads) {
+            thread.join();
+        }
+        EXPECT_EQ(mismatches, std::vector<int>(8, 0));
+    }
+}
+
+TEST(JiebaTokenizerTest, BatchesLoseNoTokenAroundTheBatchSize) {
+    JiebaDictScope dict;
+    for (size_t count : {size_t {255}, size_t {256}, size_t {257}, size_t 
{512}}) {
+        std::string value;
+        for (size_t i = 0; i < count; ++i) {
+            value += "w" + std::to_string(i) + " ";
+        }
+        for (const Mode mode : kModes) {
+            auto oracle = clucene_chinese(mode.fine, mode.lowercase);
+            auto candidate = 
golden::make_text_driver(chinese_analyzer(mode.fine, mode.lowercase));
+            expect_same_as_clucene(value, *candidate, *oracle);
+        }
+    }
+}
+
+TEST(JiebaTokenizerTest, KeepsTheStreamContract) {
+    JiebaDictScope dict;
+    std::string many_batches;
+    for (int i = 0; i < 1500; ++i) {
+        many_batches += "我们研究" + std::to_string(i % 97) + "号AB ";
+    }
+    for (const Mode mode : kModes) {
+        const TextAnalyzerPtr analyzer = chinese_analyzer(mode.fine, 
mode.lowercase);
+        expect_text_stream_contract([&] { return analyzer->create_stream(); },
+                                    {"", "我们", "中华人民共和国", "Hello世界", "a",
+                                     "你好" + bytes({0xE4, 0xB8}), 
many_batches});
+    }
+}
+
+} // namespace
+} // namespace doris::segment_v2::inverted_index
diff --git a/build-support/check-header-deps.py 
b/build-support/check-header-deps.py
index 2a646d8fc18..d63a83f469f 100755
--- a/build-support/check-header-deps.py
+++ b/build-support/check-header-deps.py
@@ -655,8 +655,6 @@ CLUCENE_INCLUDING_FILES = {
     "storage/index/index_file_writer.h",
     "storage/index/index_fs_directory.h",
     "storage/index/inverted/clucene/inverted_index_searcher.h",
-    # The analysis layer (T3, T4, T8, T9).
-    "storage/index/inverted/analysis/analyzer/analyzer.cpp",
     # SNII (T3, T12, T13).
     "storage/index/inverted/snii/snii_blob_directory.h",
     "storage/index/inverted/snii/snii_blob_staging_directory.h",
@@ -671,8 +669,6 @@ CLUCENE_SYMBOL_FILES = {
     "storage/index/ann/ann_index_writer.h",
     "storage/index/ann/faiss_ann_index.cpp",
     "storage/index/ann/faiss_ann_index.h",
-    # The analysis layer (T3, T4, T8, T9).
-    "storage/index/inverted/analysis/analyzer/analyzer.cpp",
     # SNII (T3, T12, T13).
     "storage/index/inverted/snii/snii_blob_directory.cpp",
     "storage/index/inverted/snii/snii_blob_directory.h",
diff --git a/dist/LICENSE-dist.txt b/dist/LICENSE-dist.txt
index 153c799845d..0955e9ac55f 100644
--- a/dist/LICENSE-dist.txt
+++ b/dist/LICENSE-dist.txt
@@ -1515,6 +1515,8 @@ Other dependencies:
     * pugixml 1.15 -- licenses/LICENSE-pugixml.txt
     * darts-clone (double-array trie, compiled into the kuromoji dictionary) 
-- licenses/LICENSE-darts-clone.txt
     * mecab-ipadic (IPADIC) 2.7.0-20250920 (Japanese dictionary data shipped 
under be/dict/kuromoji) -- licenses/LICENSE-ipadic.txt
+    * cppjieba (Chinese word segmentation, ported into the chinese parser) -- 
licenses/LICENSE-cppjieba.txt
+    * limonp (helpers of cppjieba, ported with it) -- 
licenses/LICENSE-limonp.txt
 
     * MongoDB C Driver (libmongoc/libbson): 2.5.8 -- Apache License 2.0; 
bundled notices in licenses/LICENSE-mongo-c-driver.txt
     * MongoDB Java Sync Driver (sync/core/bson/bson-record-codec): 5.14.0 -- 
Apache License 2.0
diff --git a/dist/licenses/LICENSE-cppjieba.txt 
b/dist/licenses/LICENSE-cppjieba.txt
new file mode 100644
index 00000000000..6308e939b73
--- /dev/null
+++ b/dist/licenses/LICENSE-cppjieba.txt
@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2013
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 
of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/dist/licenses/LICENSE-limonp.txt b/dist/licenses/LICENSE-limonp.txt
new file mode 100644
index 00000000000..6308e939b73
--- /dev/null
+++ b/dist/licenses/LICENSE-limonp.txt
@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2013
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 
of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to