This is an automated email from the ASF dual-hosted git repository.

yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/master by this push:
     new d556f32bf8c [improvement](be) Tokenize standard and unicode with a 
dense UAX#29 table (#70265)
d556f32bf8c is described below

commit d556f32bf8c98a245527e74f14673bc3e2960964
Author: Jack <[email protected]>
AuthorDate: Sat Nov 21 12:15:31 2026 +0800

    [improvement](be) Tokenize standard and unicode with a dense UAX#29 table 
(#70265)
    
    ### What problem does this PR solve?
    
    Issue Number: None
    
    Related PR: #70264 (depends on)
    
    Problem Summary: Tokenize standard and unicode with one flat UAX#29
    table: about 1.8x as fast as master, with the same tokens.
    
    ### Release note
    
    None
    
    ### Check List (For Author)
    
    - Test <!-- At least one of them must be included. -->
        - [ ] Regression test
        - [x] Unit Test
        - [x] Manual test (add detailed scripts or steps below)
        - [ ] No need to test or manual test. Explain why:
    - [ ] This is a refactor/code format and no logic has been changed.
            - [ ] Previous test can cover this change.
            - [ ] No code files have been changed.
            - [ ] Other reason <!-- Add your reason?  -->
    
    - Behavior changed:
        - [x] No.
        - [ ] Yes.
    
    - Does this need documentation?
        - [x] No.
        - [ ] Yes.
    
    ### Check List (For Reviewer who merge this PR)
    
    - [ ] Confirm the release note
    - [ ] Confirm test cases
    - [ ] Confirm document
    - [ ] Add branch pick label
    
    🤖 Generated with [Claude Code](https://claude.com/claude-code)
    
    ---------
    
    Co-authored-by: Claude Opus 5.5 <[email protected]>
---
 .../index/inverted/analysis/analyzer/analyzer.cpp  |   9 +-
 .../tokenizer/standard/standard_tokenizer.cpp      | 102 +++++
 .../tokenizer/standard/standard_tokenizer.h        |  74 ++--
 .../tokenizer/standard/standard_tokenizer_impl.h   |   3 +
 .../analysis/tokenizer/standard/uax29_table.cpp    |  63 +++
 .../analysis/tokenizer/standard/uax29_table.h      | 122 ++++++
 .../tokenizer/standard/uax29_tokenizer.cpp         | 126 ++++++
 .../analysis/tokenizer/standard/uax29_tokenizer.h  |  92 +++++
 .../inverted/analysis/tokenizer/utf8_validator.h   |  87 ++++
 be/src/storage/utils.cpp                           |  13 +-
 .../inverted/analysis/component_stream_test.cpp    |  13 +
 .../analysis/tokenizer/uax29_table_test.cpp        | 112 ++++++
 .../analysis/tokenizer/uax29_tokenizer_test.cpp    | 447 +++++++++++++++++++++
 .../analysis/tokenizer/utf8_validator_test.cpp     | 146 +++++++
 .../index/inverted/clucene/lucene_adapter_test.cpp |  13 +-
 15 files changed, 1360 insertions(+), 62 deletions(-)

diff --git a/be/src/storage/index/inverted/analysis/analyzer/analyzer.cpp 
b/be/src/storage/index/inverted/analysis/analyzer/analyzer.cpp
index 8e09388e7d6..1a837527954 100644
--- a/be/src/storage/index/inverted/analysis/analyzer/analyzer.cpp
+++ b/be/src/storage/index/inverted/analysis/analyzer/analyzer.cpp
@@ -52,6 +52,7 @@
 #include 
"storage/index/inverted/analysis/tokenizer/icu/icu_tokenizer_factory.h"
 #include "storage/index/inverted/analysis/tokenizer/ik/ik_tokenizer_factory.h"
 #include 
"storage/index/inverted/analysis/tokenizer/kuromoji/kuromoji_tokenizer_factory.h"
+#include "storage/index/inverted/analysis/tokenizer/standard/uax29_tokenizer.h"
 #include "storage/index/inverted/clucene/lucene_backed_text_analyzer.h"
 
 namespace doris::segment_v2::inverted_index {
@@ -222,9 +223,11 @@ TextAnalyzerPtr 
build_builtin_analyzer(InvertedIndexParserType parser_type,
                                        const std::string& stop_words) {
     if (parser_type == InvertedIndexParserType::PARSER_STANDARD ||
         parser_type == InvertedIndexParserType::PARSER_UNICODE) {
-        return lucene_builtin_analyzer(
-                
std::make_shared<lucene::analysis::standard95::StandardAnalyzer>(), lower_case,
-                stop_words);
+        return std::make_shared<ChainAnalyzer>(
+                std::vector<CharFilterFactoryPtr> {},
+                
std::make_shared<Uax29TokenizerFactory>(builtin_lowercase(lower_case, true),
+                                                        stop_words != "none"),
+                std::vector<TokenFilterFactoryPtr> {});
     }
     if (parser_type == InvertedIndexParserType::PARSER_CHINESE) {
         auto chinese_analyzer =
diff --git 
a/be/src/storage/index/inverted/analysis/tokenizer/standard/standard_tokenizer.cpp
 
b/be/src/storage/index/inverted/analysis/tokenizer/standard/standard_tokenizer.cpp
new file mode 100644
index 00000000000..fac9740e748
--- /dev/null
+++ 
b/be/src/storage/index/inverted/analysis/tokenizer/standard/standard_tokenizer.cpp
@@ -0,0 +1,102 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include 
"storage/index/inverted/analysis/tokenizer/standard/standard_tokenizer.h"
+
+#include <memory>
+#include <string>
+
+#include "common/exception.h"
+#include "common/logging.h"
+#include "storage/index/inverted/analysis/token_batch.h"
+#include "storage/index/inverted/analysis/tokenizer/standard/uax29_table.h"
+#include "util/utf8_check.h"
+
+namespace doris::segment_v2::inverted_index {
+
+StandardTokenizer::StandardTokenizer()
+        : _table(Uax29Table::instance()), 
_scanner(std::make_unique<StandardTokenizerImpl>()) {}
+
+void StandardTokenizer::set_max_token_length(int32_t length) {
+    if (length < 1) {
+        throw Exception(ErrorCode::INVALID_ARGUMENT, "max_token_length must be 
greater than zero");
+    } else if (length > MAX_TOKEN_LENGTH_LIMIT) {
+        throw Exception(
+                ErrorCode::INVALID_ARGUMENT,
+                "max_token_length may not exceed " + 
std::to_string(MAX_TOKEN_LENGTH_LIMIT));
+    }
+    if (length != _max_token_length) {
+        _max_token_length = length;
+        _scanner->set_buffer_size(length);
+    }
+}
+
+void StandardTokenizer::reset(std::string_view text) {
+    _text = text;
+    _valid = doris::validate_utf8(text.data(), text.size());
+    _pos = 0;
+    _exhausted = false;
+    if (!_valid) {
+        _scanner->yyreset(text);
+    }
+}
+
+bool StandardTokenizer::next_batch(TokenBatch& batch) {
+    return _valid ? next_valid_batch(batch) : next_scanner_batch(batch);
+}
+
+bool StandardTokenizer::next_valid_batch(TokenBatch& batch) {
+    batch.clear();
+    const auto* text = reinterpret_cast<const uint8_t*>(_text.data());
+    const size_t size = _text.size();
+    const auto window = static_cast<size_t>(_max_token_length);
+    while (_pos < size && batch.size() < kBatchTokens) {
+        const size_t start = _pos;
+        const Uax29Match match =
+                _table.longest_match(text, size, start, [start, window](size_t 
cur, size_t length) {
+                    return cur + length - start <= window;
+                });
+        if (match.action < 0) {
+            // The character at `start` is longer than max_token_length, so 
the scanner's buffer
+            // could not take it, and the scanner ended the value there.
+            _pos = size;
+            break;
+        }
+        _pos = match.end;
+        if (match.action != Uax29Table::kSkip) {
+            batch.push({_text.data() + start, match.end - start}, 1);
+        }
+    }
+    return batch.size() != 0;
+}
+
+bool StandardTokenizer::next_scanner_batch(TokenBatch& batch) {
+    batch.clear();
+    while (!_exhausted && batch.size() < kBatchTokens) {
+        if (_scanner->get_next_token() == 
StandardTokenizerImpl::TokenType::EOF_TOKEN) {
+            _exhausted = true;
+            break;
+        }
+        // The buffer of max_token_length bytes holds the whole token.
+        DCHECK_LE(_scanner->yylength(), _max_token_length);
+        // The term points into the scanner's buffer, which its next refill 
moves.
+        batch.push_copy(_scanner->get_text(), 1);
+    }
+    return batch.size() != 0;
+}
+
+} // namespace doris::segment_v2::inverted_index
diff --git 
a/be/src/storage/index/inverted/analysis/tokenizer/standard/standard_tokenizer.h
 
b/be/src/storage/index/inverted/analysis/tokenizer/standard/standard_tokenizer.h
index c8677e2acfa..7a4a51d8317 100644
--- 
a/be/src/storage/index/inverted/analysis/tokenizer/standard/standard_tokenizer.h
+++ 
b/be/src/storage/index/inverted/analysis/tokenizer/standard/standard_tokenizer.h
@@ -17,68 +17,52 @@
 
 #pragma once
 
-#include <memory>
-#include <string>
+#include <cstddef>
+#include <cstdint>
 #include <string_view>
-#include <utility>
 
-#include "common/exception.h"
-#include "storage/index/inverted/analysis/per_token_stream.h"
+#include "storage/index/inverted/analysis/text_token_stream.h"
 #include 
"storage/index/inverted/analysis/tokenizer/standard/standard_tokenizer_impl.h"
 
 namespace doris::segment_v2::inverted_index {
 
-// The words of the value by the Unicode word break rules (UAX#29); a word 
longer than
-// max_token_length is dropped, and its position is skipped.
-class StandardTokenizer final : public PerTokenTokenizer {
+struct Uax29Table;
+
+// The custom standard tokenizer: the UAX#29 words of the value, each one 
position after the last.
+// It keeps the rules of StandardTokenizerImpl, the JFlex scanner it was, 
whose buffer of
+// max_token_length bytes is the window a token is matched in: a character 
that does not fit ends
+// the token, so a longer word is cut into pieces, and a character that does 
not fit an empty window
+// ends the value. E.g. with max_token_length=2, "abc" makes "ab" and "c", and 
"ab", then U+4E2D
+// (3 bytes in UTF-8), then "cd" makes "ab" and nothing after it.
+//
+// A value that is valid UTF-8 runs through the DFA of Uax29Table, and its 
terms are views of the
+// value. Any other value runs through the scanner itself, whose reaction to 
invalid bytes depends
+// on where its buffer refills; those terms are copied, because a refill moves 
the buffer.
+class StandardTokenizer final : public TextTokenizer {
 public:
-    StandardTokenizer() { _scanner = 
std::make_unique<StandardTokenizerImpl>(); }
+    StandardTokenizer();
 
-    void set_max_token_length(int32_t length) {
-        if (length < 1) {
-            throw Exception(ErrorCode::INVALID_ARGUMENT,
-                            "max_token_length must be greater than zero");
-        } else if (length > MAX_TOKEN_LENGTH_LIMIT) {
-            throw Exception(
-                    ErrorCode::INVALID_ARGUMENT,
-                    "max_token_length may not exceed " + 
std::to_string(MAX_TOKEN_LENGTH_LIMIT));
-        }
-        if (length != _max_token_length) {
-            _max_token_length = length;
-            _scanner->set_buffer_size(length);
-        }
-    }
+    void set_max_token_length(int32_t length);
 
-    // The scan window, max_token_length bytes.
+    void reset(std::string_view text) override;
+    bool next_batch(TokenBatch& batch) override;
+    // The scanner's window, max_token_length bytes.
     size_t reserved_bytes() const override { return 
_scanner->buffer_capacity(); }
 
     static constexpr int32_t DEFAULT_MAX_TOKEN_LENGTH = 255;
     static constexpr int32_t MAX_TOKEN_LENGTH_LIMIT = 16383;
 
-protected:
-    void on_reset(std::string_view text) override { _scanner->yyreset(text); }
-
-    bool next_token(TokenRef& token) override {
-        int32_t skipped_positions = 0;
-        while (true) {
-            const auto token_type = _scanner->get_next_token();
-            if (token_type == StandardTokenizerImpl::TokenType::EOF_TOKEN) {
-                return false;
-            }
-            if (std::cmp_less_equal(_scanner->yylength(), _max_token_length)) {
-                const std::string_view term = _scanner->get_text();
-                token = {.data = term.data(),
-                         .size = static_cast<uint32_t>(term.size()),
-                         .position_increment = skipped_positions + 1};
-                return true;
-            }
-            skipped_positions++;
-        }
-    }
-
 private:
+    bool next_valid_batch(TokenBatch& batch);
+    bool next_scanner_batch(TokenBatch& batch);
+
+    const Uax29Table& _table;
     StandardTokenizerImplPtr _scanner;
     int32_t _max_token_length = DEFAULT_MAX_TOKEN_LENGTH;
+    std::string_view _text;
+    bool _valid = true;      // the value is valid UTF-8
+    size_t _pos = 0;         // where the next token of a valid value starts
+    bool _exhausted = false; // the scanner reached the end of an invalid value
 };
 
 } // namespace doris::segment_v2::inverted_index
diff --git 
a/be/src/storage/index/inverted/analysis/tokenizer/standard/standard_tokenizer_impl.h
 
b/be/src/storage/index/inverted/analysis/tokenizer/standard/standard_tokenizer_impl.h
index cdf60d4efac..5b618da7b14 100644
--- 
a/be/src/storage/index/inverted/analysis/tokenizer/standard/standard_tokenizer_impl.h
+++ 
b/be/src/storage/index/inverted/analysis/tokenizer/standard/standard_tokenizer_impl.h
@@ -278,6 +278,9 @@ public:
     static constexpr int32_t YYINITIAL = 0;
 
 private:
+    // Lays the ZZ_* tables out flat for the standard tokenizers' core loop.
+    friend struct Uax29Table;
+
     static constexpr int32_t ZZ_UNKNOWN_ERROR = 0;
     static constexpr int32_t ZZ_NO_MATCH = 1;
     static constexpr int32_t ZZ_PUSHBACK_2BIG = 2;
diff --git 
a/be/src/storage/index/inverted/analysis/tokenizer/standard/uax29_table.cpp 
b/be/src/storage/index/inverted/analysis/tokenizer/standard/uax29_table.cpp
new file mode 100644
index 00000000000..4b966854a13
--- /dev/null
+++ b/be/src/storage/index/inverted/analysis/tokenizer/standard/uax29_table.cpp
@@ -0,0 +1,63 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "storage/index/inverted/analysis/tokenizer/standard/uax29_table.h"
+
+#include <algorithm>
+
+#include "common/check.h"
+#include 
"storage/index/inverted/analysis/tokenizer/standard/standard_tokenizer_impl.h"
+
+namespace doris::segment_v2::inverted_index {
+
+const Uax29Table& Uax29Table::instance() {
+    // Built at first use, after the scanner's tables, which are initialized 
dynamically.
+    static const Uax29Table table;
+    return table;
+}
+
+Uax29Table::Uax29Table() {
+    using Impl = StandardTokenizerImpl;
+    DORIS_CHECK(Impl::ZZ_ATTRIBUTE.size() == kStates && Impl::ZZ_ACTION.size() 
== kStates &&
+                Impl::ZZ_ROWMAP.size() == kStates);
+    DORIS_CHECK(Impl::ZZ_CMAP_TOP.size() == kCmapTop && 
Impl::ZZ_CMAP_BLOCKS.size() == kCmapBlocks);
+    DORIS_CHECK(*std::ranges::max_element(Impl::ZZ_CMAP_BLOCKS) == kClasses - 
1);
+    // The scanner looks the first 256 code points up in block 0 without the 
top level.
+    DORIS_CHECK(Impl::ZZ_CMAP_TOP[0] == 0);
+
+    for (int state = 0; state < kStates; ++state) {
+        for (int cls = 0; cls < kClasses; ++cls) {
+            const int32_t to = Impl::ZZ_TRANS[Impl::ZZ_ROWMAP[state] + cls];
+            DORIS_CHECK(to >= -1 && to < kStates);
+            next[state * kClasses + cls] = to < 0 ? kDead : 
static_cast<uint8_t>(to);
+        }
+        attribute[state] = static_cast<uint8_t>(Impl::ZZ_ATTRIBUTE[state]);
+        action[state] = static_cast<uint8_t>(Impl::ZZ_ACTION[state]);
+    }
+    for (size_t c = 0; c < ascii_class.size(); ++c) {
+        ascii_class[c] = static_cast<uint8_t>(Impl::ZZ_CMAP_BLOCKS[c]);
+    }
+    for (size_t i = 0; i < kCmapTop; ++i) {
+        cmap_top[i] = static_cast<uint16_t>(Impl::ZZ_CMAP_TOP[i]);
+    }
+    for (size_t i = 0; i < kCmapBlocks; ++i) {
+        cmap_blocks[i] = static_cast<uint8_t>(Impl::ZZ_CMAP_BLOCKS[i]);
+    }
+    initial = static_cast<uint8_t>(Impl::ZZ_LEXSTATE[Impl::YYINITIAL]);
+}
+
+} // namespace doris::segment_v2::inverted_index
diff --git 
a/be/src/storage/index/inverted/analysis/tokenizer/standard/uax29_table.h 
b/be/src/storage/index/inverted/analysis/tokenizer/standard/uax29_table.h
new file mode 100644
index 00000000000..74bcf225f11
--- /dev/null
+++ b/be/src/storage/index/inverted/analysis/tokenizer/standard/uax29_table.h
@@ -0,0 +1,122 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <array>
+#include <cstddef>
+#include <cstdint>
+
+#include "common/compiler_util.h"
+#include "storage/index/inverted/analysis/tokenizer/utf8_validator.h"
+
+namespace doris::segment_v2::inverted_index {
+
+// The longest match of the DFA from a token start: the text up to `end` 
matched `action`, or not
+// one character fit and `action` is -1.
+struct Uax29Match {
+    size_t end = 0;
+    int action = -1;
+};
+
+// The UAX#29 word-break DFA of StandardTokenizerImpl, the JFlex scanner both 
standard tokenizers
+// come from, laid out flat with one byte per transition, so that a step is 
one load. It is built
+// once from the scanner's ZZ_* tables, which CLucene's builtin scanner shares 
entry for entry, so
+// neither tokenizer depends on a hand-copied table.
+struct Uax29Table {
+    static constexpr int kStates = 59;
+    static constexpr int kClasses = 29;
+    static constexpr uint8_t kDead = 0xFF; // no transition: the longest match 
ends
+    // Attribute bits.
+    static constexpr uint8_t kAccepting = 1; // a token may end in the state
+    static constexpr uint8_t kFinal = 8;     // no transition leaves the state
+    // The actions of the accepting states: what the text matched up to such a 
state is.
+    enum Action : uint8_t {
+        kSkip = 1,
+        kNumeric = 2,
+        kWord = 3,
+        kEmoji = 4,
+        kSouthEastAsian = 5,
+        kHangul = 6,
+        kIdeographic = 7,
+        kKatakana = 8,
+        kHiragana = 9,
+    };
+    // The two-level character map: the class of code point c is
+    // cmap_blocks[cmap_top[c >> 8] | (c & 0xFF)].
+    static constexpr size_t kCmapTop = 0x110000 >> 8;
+    static constexpr size_t kCmapBlocks = 35328;
+
+    std::array<uint8_t, kStates * kClasses> next {};
+    std::array<uint8_t, kStates> attribute {};
+    std::array<uint8_t, kStates> action {};
+    std::array<uint8_t, 128> ascii_class {};
+    std::array<uint16_t, kCmapTop> cmap_top {};
+    std::array<uint8_t, kCmapBlocks> cmap_blocks {};
+    uint8_t initial = 0;
+
+    static const Uax29Table& instance();
+
+    // The class of a code point up to U+10FFFF.
+    uint8_t char_class(uint32_t code_point) const {
+        return cmap_blocks[cmap_top[code_point >> 8] | (code_point & 0xFF)];
+    }
+
+    // The longest match from `start` of a value lenient_utf8_valid() 
accepted, the way the JFlex
+    // scanner matched: it follows transitions while `fits(cur, length)` lets 
the character of
+    // `length` bytes at `cur` join the token, and the token ends at the last 
accepting state it
+    // passed. Every character leads from the initial state to an accepting 
one, so `action` is -1
+    // only when the first character did not fit. E.g. with a window of 4 
bytes, "ab" followed by
+    // U+4E2D (3 bytes in UTF-8) matches "ab": U+4E2D would take the token to 
5 bytes.
+    template <typename Fits>
+    ALWAYS_INLINE Uax29Match longest_match(const uint8_t* text, size_t size, 
size_t start,
+                                           Fits fits) const {
+        Uax29Match match {.end = start, .action = -1};
+        size_t cur = start;
+        uint8_t state = initial;
+        while (cur < size) {
+            const uint8_t c = text[cur];
+            size_t length = 1;
+            const uint8_t cls = c < 0x80 ? ascii_class[c]
+                                         : char_class(decode_lenient_utf8(text 
+ cur, &length));
+            if (!fits(cur, length)) {
+                break;
+            }
+            const uint8_t to = next[state * kClasses + cls];
+            if (to == kDead) {
+                break;
+            }
+            cur += length;
+            state = to;
+            const uint8_t attributes = attribute[to];
+            if ((attributes & kAccepting) != 0) {
+                match = {.end = cur, .action = action[to]};
+                if ((attributes & kFinal) != 0) {
+                    break;
+                }
+            }
+        }
+        return match;
+    }
+
+private:
+    // Fills the table in place, inside the static of instance(): 46 KB is too 
much for the stack
+    // of the thread that happens to use it first.
+    Uax29Table();
+};
+
+} // namespace doris::segment_v2::inverted_index
diff --git 
a/be/src/storage/index/inverted/analysis/tokenizer/standard/uax29_tokenizer.cpp 
b/be/src/storage/index/inverted/analysis/tokenizer/standard/uax29_tokenizer.cpp
new file mode 100644
index 00000000000..d075f8b4d65
--- /dev/null
+++ 
b/be/src/storage/index/inverted/analysis/tokenizer/standard/uax29_tokenizer.cpp
@@ -0,0 +1,126 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "storage/index/inverted/analysis/tokenizer/standard/uax29_tokenizer.h"
+
+#include <algorithm>
+#include <array>
+#include <cstdint>
+
+#include "common/logging.h"
+#include "storage/index/inverted/analysis/scratch.h"
+#include "storage/index/inverted/analysis/token_batch.h"
+#include "storage/index/inverted/analysis/tokenizer/byte_classifier.h"
+#include "storage/index/inverted/analysis/tokenizer/standard/uax29_table.h"
+#include "storage/index/inverted/analysis/tokenizer/utf8_validator.h"
+#include "util/utf8_check.h"
+
+namespace doris::segment_v2::inverted_index {
+
+namespace {
+
+constexpr std::array<std::string_view, 33> STOP_WORDS = {
+        "a",   "an",    "and",  "are",   "as",    "at",   "be",   "but", "by", 
 "for",  "if",
+        "in",  "into",  "is",   "it",    "no",    "not",  "of",   "on",  "or", 
 "such", "that",
+        "the", "their", "then", "there", "these", "they", "this", "to",  
"was", "will", "with"};
+constexpr size_t LONGEST_STOP_WORD = 5;
+
+// A term of at most 7 bytes as one integer: its bytes from the low end and 
its length in the top
+// byte, so that terms of different lengths never compare equal.
+constexpr uint64_t pack_term(std::string_view term) {
+    uint64_t key = static_cast<uint64_t>(term.size()) << 56;
+    for (size_t i = 0; i < term.size(); ++i) {
+        key |= static_cast<uint64_t>(static_cast<uint8_t>(term[i])) << (8 * i);
+    }
+    return key;
+}
+
+// Sorted for binary search; most tokens leave on the length test before it.
+constexpr std::array<uint64_t, STOP_WORDS.size()> STOP_WORD_KEYS = [] {
+    std::array<uint64_t, STOP_WORDS.size()> keys {};
+    for (size_t i = 0; i < STOP_WORDS.size(); ++i) {
+        keys[i] = pack_term(STOP_WORDS[i]);
+    }
+    std::ranges::sort(keys);
+    return keys;
+}();
+
+} // namespace
+
+std::span<const std::string_view> standard_stop_words() {
+    return STOP_WORDS;
+}
+
+bool is_standard_stop_word(std::string_view term) {
+    return term.size() <= LONGEST_STOP_WORD &&
+           std::ranges::binary_search(STOP_WORD_KEYS, pack_term(term));
+}
+
+Uax29Tokenizer::Uax29Tokenizer(bool lowercase, bool stop_words)
+        : _table(Uax29Table::instance()), _lowercase(lowercase), 
_stop_words(stop_words) {}
+
+void Uax29Tokenizer::reset(std::string_view text) {
+    _pos = 0;
+    // Before the early returns: an invalid value after a 1 MB one must not 
keep its 1 MB copy.
+    release_oversized_scratch(_lowered);
+    // Strict UTF-8 passing decides alone; the lenient rule decides the rest, 
the overlong forms.
+    if (!doris::validate_utf8(text.data(), text.size()) && 
!lenient_utf8_valid(text)) {
+        _text = {};
+        return;
+    }
+    if (!_lowercase) {
+        _text = text;
+        return;
+    }
+    // The copy only grows, so a shorter value writes over the head of it 
without a resize.
+    if (_lowered.size() < text.size()) {
+        _lowered.resize(text.size());
+    }
+    const char* in = text.data();
+    char* out = _lowered.data();
+    for (size_t i = 0; i < text.size(); ++i) {
+        out[i] = ascii_lower(in[i]);
+    }
+    _text = {out, text.size()};
+}
+
+bool Uax29Tokenizer::next_batch(TokenBatch& batch) {
+    batch.clear();
+    const auto* text = reinterpret_cast<const uint8_t*>(_text.data());
+    const size_t size = _text.size();
+    while (_pos < size && batch.size() < kBatchTokens) {
+        const size_t start = _pos;
+        const Uax29Match match = _table.longest_match(
+                text, size, start,
+                [start](size_t cur, size_t /*length*/) { return cur - start < 
MAX_TOKEN_LENGTH; });
+        // The window always takes the first character.
+        DCHECK_GE(match.action, 0);
+        _pos = match.end;
+        const size_t length = match.end - start;
+        if (match.action == Uax29Table::kSkip || length > MAX_TOKEN_LENGTH) {
+            continue;
+        }
+        const std::string_view term(_text.data() + start, length);
+        if (_stop_words && match.action == Uax29Table::kWord && 
is_standard_stop_word(term)) {
+            continue;
+        }
+        batch.push(term, 1);
+    }
+    return batch.size() != 0;
+}
+
+} // namespace doris::segment_v2::inverted_index
diff --git 
a/be/src/storage/index/inverted/analysis/tokenizer/standard/uax29_tokenizer.h 
b/be/src/storage/index/inverted/analysis/tokenizer/standard/uax29_tokenizer.h
new file mode 100644
index 00000000000..491626adf65
--- /dev/null
+++ 
b/be/src/storage/index/inverted/analysis/tokenizer/standard/uax29_tokenizer.h
@@ -0,0 +1,92 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <cstddef>
+#include <memory>
+#include <span>
+#include <string>
+#include <string_view>
+
+#include "storage/index/inverted/analysis/text_token_stream.h"
+#include "storage/index/inverted/analysis/tokenizer/tokenizer_factory.h"
+
+namespace doris::segment_v2::inverted_index {
+
+struct Uax29Table;
+
+// The 33 stop words of the builtin standard and unicode parsers, the English 
set of Lucene's
+// StandardAnalyzer.
+std::span<const std::string_view> standard_stop_words();
+// Whether a term, as the tokenizer makes it, is one of them: the comparison 
is byte for byte, so
+// "The" is a stop word only once lowercased.
+bool is_standard_stop_word(std::string_view term);
+
+// The tokenizer of the builtin standard and unicode parsers, which make the 
tokens CLucene's
+// standard95 tokenizer made. A value that fails lenient_utf8_valid() 
(utf8_validator.h) has none.
+// Otherwise its tokens are the UAX#29 words: a token is scanned while it is 
shorter than
+// MAX_TOKEN_LENGTH bytes, so its last character may end past it, and a token 
longer than
+// MAX_TOKEN_LENGTH is dropped. With `lowercase`, A-Z in WORD tokens is folded 
to a-z; with
+// `stop_words`, a WORD token that is a stop word is dropped. Every token is 
one position after the
+// last kept one: the old tokenizer never set an increment, so a dropped token 
left no gap.
+//
+// Lowercasing folds the whole value into a copy at the same offsets, and the 
scan runs over the
+// copy. That makes the same tokens because A-Z and a-z are one class of the 
DFA, and lowercases
+// only WORD tokens because every accepting state reached through a letter is 
WORD; Uax29TableTest
+// checks both on the table.
+class Uax29Tokenizer final : public TextTokenizer {
+public:
+    Uax29Tokenizer(bool lowercase, bool stop_words);
+
+    void reset(std::string_view text) override;
+    bool next_batch(TokenBatch& batch) override;
+    // The lowered copy of the value, which only a lowercasing tokenizer has.
+    size_t reserved_bytes() const override { return _lowercase ? 
_lowered.capacity() : 0; }
+
+    // The scan window from a token start, and the longest token kept.
+    static constexpr size_t MAX_TOKEN_LENGTH = 255;
+
+private:
+    const Uax29Table& _table;
+    const bool _lowercase;
+    const bool _stop_words;
+    // The value or its lowered copy; empty for a value that failed the UTF-8 
rule.
+    std::string_view _text;
+    // The value with A-Z folded, byte for byte at the offsets of the value; 
only with _lowercase.
+    std::string _lowered;
+    size_t _pos = 0; // where the next token starts
+};
+
+// The tokenizer of the builtin standard and unicode parsers; neither is a 
custom tokenizer.
+class Uax29TokenizerFactory final : public TokenizerFactory {
+public:
+    Uax29TokenizerFactory(bool lowercase, bool stop_words)
+            : _lowercase(lowercase), _stop_words(stop_words) {}
+
+    void initialize(const Settings& settings) override {}
+
+    std::unique_ptr<TextTokenizer> create() const override {
+        return std::make_unique<Uax29Tokenizer>(_lowercase, _stop_words);
+    }
+
+private:
+    const bool _lowercase;
+    const bool _stop_words;
+};
+
+} // namespace doris::segment_v2::inverted_index
diff --git a/be/src/storage/index/inverted/analysis/tokenizer/utf8_validator.h 
b/be/src/storage/index/inverted/analysis/tokenizer/utf8_validator.h
new file mode 100644
index 00000000000..7597918b58d
--- /dev/null
+++ b/be/src/storage/index/inverted/analysis/tokenizer/utf8_validator.h
@@ -0,0 +1,87 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+#include <string_view>
+
+#include "common/logging.h"
+
+namespace doris::segment_v2::inverted_index {
+
+// The UTF-8 rule of the builtin standard and unicode parsers, CLucene's 
StringUtil::validate_utf8:
+// the high bits of a lead byte announce one to three trailing bytes, every 
trailing byte is
+// 10xxxxxx, and the code point a sequence decodes to is neither a surrogate 
nor above U+10FFFF.
+// It accepts what strict UTF-8 refuses, the overlong forms: C0 80 for U+0000, 
F0 80 80 80. Every
+// value strict UTF-8 accepts passes it too, so a strict check that passes 
decides alone.
+inline bool lenient_utf8_valid(std::string_view text) {
+    int32_t trailing = 0;
+    uint32_t code_point = 0;
+    for (const char ch : text) {
+        const auto c = static_cast<uint8_t>(ch);
+        if (trailing == 0) {
+            if ((c & 0x80) == 0) {
+                continue;
+            }
+            if ((c & 0xE0) == 0xC0) {
+                code_point = c & 0x1F;
+                trailing = 1;
+            } else if ((c & 0xF0) == 0xE0) {
+                code_point = c & 0x0F;
+                trailing = 2;
+            } else if ((c & 0xF8) == 0xF0) {
+                code_point = c & 0x07;
+                trailing = 3;
+            } else {
+                return false;
+            }
+            continue;
+        }
+        if ((c & 0xC0) != 0x80) {
+            return false;
+        }
+        code_point = (code_point << 6) | (c & 0x3F);
+        if (--trailing == 0 &&
+            !(code_point < 0xD800 || (code_point >= 0xE000 && code_point <= 
0x10FFFF))) {
+            return false;
+        }
+    }
+    return trailing == 0;
+}
+
+// The code point of the multi-byte sequence at `p` of a value 
lenient_utf8_valid() accepted, with
+// its length in `*length`: the lead byte's high bits give the length and the 
low bits of every
+// byte the code point, overlong forms included, as CLucene's standard scanner 
decoded: C1 81 is
+// U+0041.
+inline uint32_t decode_lenient_utf8(const uint8_t* p, size_t* length) {
+    const uint8_t c = p[0];
+    if ((c & 0xE0) == 0xC0) {
+        *length = 2;
+        return ((c & 0x1FU) << 6) | (p[1] & 0x3FU);
+    }
+    if ((c & 0xF0) == 0xE0) {
+        *length = 3;
+        return ((c & 0x0FU) << 12) | ((p[1] & 0x3FU) << 6) | (p[2] & 0x3FU);
+    }
+    DCHECK_EQ(c & 0xF8, 0xF0);
+    *length = 4;
+    return ((c & 0x07U) << 18) | ((p[1] & 0x3FU) << 12) | ((p[2] & 0x3FU) << 
6) | (p[3] & 0x3FU);
+}
+
+} // namespace doris::segment_v2::inverted_index
diff --git a/be/src/storage/utils.cpp b/be/src/storage/utils.cpp
index 2af5607e7f8..7dae4316af4 100644
--- a/be/src/storage/utils.cpp
+++ b/be/src/storage/utils.cpp
@@ -47,7 +47,6 @@
 #include "util/string_parser.hpp"
 
 namespace doris {
-using namespace ErrorCode;
 
 uint32_t olap_adler32_init() {
     return (uint32_t)adler32(0, Z_NULL, 0);
@@ -62,11 +61,11 @@ Status gen_timestamp_string(std::string* out_string) {
     tm local_tm;
 
     if (localtime_r(&now, &local_tm) == nullptr) {
-        return Status::Error<OS_ERROR>("fail to localtime_r time. time={}", 
now);
+        return Status::Error<ErrorCode::OS_ERROR>("fail to localtime_r time. 
time={}", now);
     }
     char time_suffix[16] = {0}; // Example: 20150706111404's length is 15
     if (strftime(time_suffix, sizeof(time_suffix), "%Y%m%d%H%M%S", &local_tm) 
== 0) {
-        return Status::Error<OS_ERROR>("fail to strftime time. time={}", now);
+        return Status::Error<ErrorCode::OS_ERROR>("fail to strftime time. 
time={}", now);
     }
 
     *out_string = time_suffix;
@@ -93,13 +92,13 @@ Status read_write_test_file(const std::string& 
test_file_path) {
     char* write_test_buff = nullptr;
     char* read_test_buff = nullptr;
     if (posix_memalign((void**)&write_test_buff, DIRECT_IO_ALIGNMENT, 
TEST_FILE_BUF_SIZE) != 0) {
-        return Status::Error<MEM_ALLOC_FAILED>("fail to allocate write buffer 
memory. size={}",
-                                               TEST_FILE_BUF_SIZE);
+        return Status::Error<ErrorCode::MEM_ALLOC_FAILED>(
+                "fail to allocate write buffer memory. size={}", 
TEST_FILE_BUF_SIZE);
     }
     std::unique_ptr<char, decltype(&std::free)> write_buff(write_test_buff, 
&std::free);
     if (posix_memalign((void**)&read_test_buff, DIRECT_IO_ALIGNMENT, 
TEST_FILE_BUF_SIZE) != 0) {
-        return Status::Error<MEM_ALLOC_FAILED>("fail to allocate read buffer 
memory. size={}",
-                                               TEST_FILE_BUF_SIZE);
+        return Status::Error<ErrorCode::MEM_ALLOC_FAILED>(
+                "fail to allocate read buffer memory. size={}", 
TEST_FILE_BUF_SIZE);
     }
     std::unique_ptr<char, decltype(&std::free)> read_buff(read_test_buff, 
&std::free);
     // generate random numbers
diff --git a/be/test/storage/index/inverted/analysis/component_stream_test.cpp 
b/be/test/storage/index/inverted/analysis/component_stream_test.cpp
index 0c995dc84a1..d2f1fc1fff3 100644
--- a/be/test/storage/index/inverted/analysis/component_stream_test.cpp
+++ b/be/test/storage/index/inverted/analysis/component_stream_test.cpp
@@ -54,6 +54,7 @@
 #include "storage/index/inverted/analysis/tokenizer/ngram/ngram_tokenizer.h"
 #include "storage/index/inverted/analysis/tokenizer/pinyin/pinyin_tokenizer.h"
 #include 
"storage/index/inverted/analysis/tokenizer/standard/standard_tokenizer.h"
+#include "storage/index/inverted/analysis/tokenizer/standard/uax29_tokenizer.h"
 #include "storage/index/inverted/analysis/tokenizer/tokenizer_factory.h"
 #include "util/defer_op.h"
 
@@ -283,6 +284,18 @@ TEST(ComponentStreamTest, 
EnglishTokenizerKeepsNoRowSizedScratch) {
     }
 }
 
+TEST(ComponentStreamTest, Uax29TokenizerReleasesItsLoweredCopy) {
+    for (bool lowercase : {true, false}) {
+        SCOPED_TRACE(lowercase);
+        expect_text_stream_contract(
+                [lowercase] { return 
std::make_unique<Uax29Tokenizer>(lowercase, true); },
+                contract_texts());
+        Uax29Tokenizer standard(lowercase, true);
+        expect_released_at_next_row(standard, repeat_to("Ab ", kLongRowBytes), 
"short", lowercase,
+                                    [&] { return lowercase ? 
standard._lowered.capacity() : 0; });
+    }
+}
+
 TEST(ComponentStreamTest, ICUTokenizerReleasesItsCopyOfTheValue) {
     DictPathScope dict_path;
     Components components;
diff --git 
a/be/test/storage/index/inverted/analysis/tokenizer/uax29_table_test.cpp 
b/be/test/storage/index/inverted/analysis/tokenizer/uax29_table_test.cpp
new file mode 100644
index 00000000000..c88f53c10d3
--- /dev/null
+++ b/be/test/storage/index/inverted/analysis/tokenizer/uax29_table_test.cpp
@@ -0,0 +1,112 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "storage/index/inverted/analysis/tokenizer/standard/uax29_table.h"
+
+#include <gtest/gtest.h>
+
+#include <cstdint>
+#include <set>
+#include <utility>
+#include <vector>
+
+#include 
"storage/index/inverted/analysis/tokenizer/standard/standard_tokenizer_impl.h"
+
+namespace doris::segment_v2::inverted_index {
+namespace {
+
+using Impl = StandardTokenizerImpl;
+
+TEST(Uax29TableTest, MatchesTheScannerTablesEntryByEntry) {
+    const Uax29Table& table = Uax29Table::instance();
+    ASSERT_EQ(Impl::ZZ_ATTRIBUTE.size(), Uax29Table::kStates);
+    for (int state = 0; state < Uax29Table::kStates; ++state) {
+        for (int cls = 0; cls < Uax29Table::kClasses; ++cls) {
+            const int32_t to = Impl::ZZ_TRANS[Impl::ZZ_ROWMAP[state] + cls];
+            EXPECT_EQ(table.next[state * Uax29Table::kClasses + cls],
+                      to < 0 ? Uax29Table::kDead : to)
+                    << "state " << state << " class " << cls;
+        }
+        EXPECT_EQ(table.attribute[state], Impl::ZZ_ATTRIBUTE[state]) << state;
+        EXPECT_EQ(table.action[state], Impl::ZZ_ACTION[state]) << state;
+    }
+    for (int c = 0; c < 128; ++c) {
+        EXPECT_EQ(table.ascii_class[c], Impl::ZZ_CMAP_BLOCKS[c]) << c;
+    }
+    EXPECT_EQ(table.initial, Impl::ZZ_LEXSTATE[Impl::YYINITIAL]);
+}
+
+TEST(Uax29TableTest, ClassesEveryCodePointAsTheScanner) {
+    const Uax29Table& table = Uax29Table::instance();
+    Impl scanner;
+    for (uint32_t code_point = 0; code_point <= 0x10FFFF; ++code_point) {
+        ASSERT_EQ(table.char_class(code_point), 
scanner.zz_cmap(static_cast<int32_t>(code_point)))
+                << code_point;
+    }
+}
+
+// The first character always makes a match, so a match is empty only when the 
window refuses the
+// first character, which Uax29Table::longest_match() reports as action -1.
+TEST(Uax29TableTest, EveryCharacterLeadsFromTheInitialStateToAnAcceptingOne) {
+    const Uax29Table& table = Uax29Table::instance();
+    EXPECT_EQ(table.attribute[table.initial] & Uax29Table::kAccepting, 0);
+    for (int cls = 0; cls < Uax29Table::kClasses; ++cls) {
+        const uint8_t to = table.next[table.initial * Uax29Table::kClasses + 
cls];
+        ASSERT_NE(to, Uax29Table::kDead) << cls;
+        EXPECT_NE(table.attribute[to] & Uax29Table::kAccepting, 0) << cls;
+    }
+}
+
+// Why the builtin tokenizer may lowercase the whole value before it scans: 
folding A-Z changes no
+// class, so the scan makes the same tokens, and every accepting state the DFA 
reaches through a
+// letter is WORD, so no token of another type has a letter to fold. If a new 
table breaks either
+// half, this fails before a token changes.
+TEST(Uax29TableTest, LowercasingTheWholeValueOnlyLowercasesWords) {
+    const Uax29Table& table = Uax29Table::instance();
+    const int letter = table.ascii_class['A'];
+    for (char c = 'A'; c <= 'Z'; ++c) {
+        EXPECT_EQ(table.ascii_class[c], letter) << c;
+        EXPECT_EQ(table.ascii_class[c - 'A' + 'a'], letter) << c;
+    }
+    // Every (state, a letter on the way there) pair the DFA can reach.
+    std::set<std::pair<uint8_t, bool>> seen;
+    std::vector<std::pair<uint8_t, bool>> todo {{table.initial, false}};
+    while (!todo.empty()) {
+        const auto [state, through_letter] = todo.back();
+        todo.pop_back();
+        if (!seen.insert({state, through_letter}).second) {
+            continue;
+        }
+        for (int cls = 0; cls < Uax29Table::kClasses; ++cls) {
+            const uint8_t to = table.next[state * Uax29Table::kClasses + cls];
+            if (to != Uax29Table::kDead) {
+                todo.emplace_back(to, through_letter || cls == letter);
+            }
+        }
+    }
+    size_t words = 0;
+    for (const auto& [state, through_letter] : seen) {
+        if (through_letter && (table.attribute[state] & 
Uax29Table::kAccepting) != 0) {
+            EXPECT_EQ(table.action[state], Uax29Table::kWord) << 
static_cast<int>(state);
+            ++words;
+        }
+    }
+    EXPECT_GT(words, 0);
+}
+
+} // namespace
+} // namespace doris::segment_v2::inverted_index
diff --git 
a/be/test/storage/index/inverted/analysis/tokenizer/uax29_tokenizer_test.cpp 
b/be/test/storage/index/inverted/analysis/tokenizer/uax29_tokenizer_test.cpp
new file mode 100644
index 00000000000..db731052dc4
--- /dev/null
+++ b/be/test/storage/index/inverted/analysis/tokenizer/uax29_tokenizer_test.cpp
@@ -0,0 +1,447 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "storage/index/inverted/analysis/tokenizer/standard/uax29_tokenizer.h"
+
+#include <CLucene.h> // IWYU pragma: keep
+#include <gtest/gtest.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+#include <cstring>
+#include <memory>
+#include <random>
+#include <set>
+#include <string>
+#include <string_view>
+#include <vector>
+
+#include "CLucene/analysis/standard95/StandardAnalyzer.h"
+#include "storage/index/inverted/analysis/analyzer/analyzer.h"
+#include "storage/index/inverted/analysis/golden/golden_driver.h"
+#include "storage/index/inverted/analysis/scratch.h"
+#include "storage/index/inverted/analysis/text_stream_contract.h"
+#include "storage/index/inverted/analysis/token_batch.h"
+#include 
"storage/index/inverted/analysis/tokenizer/standard/standard_tokenizer.h"
+#include 
"storage/index/inverted/analysis/tokenizer/standard/standard_tokenizer_impl.h"
+
+namespace doris::segment_v2::inverted_index {
+namespace {
+
+// The standard parser as production builds it; unicode is the same analyzer.
+TextAnalyzerPtr standard_analyzer(bool lowercase, bool stop_words) {
+    return 
InvertedIndexAnalyzer::create_builtin_analyzer(InvertedIndexParserType::PARSER_STANDARD,
+                                                          "", lowercase ? 
"true" : "false",
+                                                          stop_words ? "" : 
"none");
+}
+
+// CLucene's standard95 analyzer, which made the standard tokens before.
+std::unique_ptr<golden::GoldenDriver> clucene_standard(bool lowercase, bool 
stop_words) {
+    return golden::make_lucene_driver([lowercase, stop_words] {
+        auto analyzer = 
std::make_shared<lucene::analysis::standard95::StandardAnalyzer>();
+        analyzer->set_lowercase(lowercase);
+        analyzer->set_stopwords(stop_words ? 
&lucene::analysis::standard95::stop_words : nullptr);
+        return analyzer;
+    });
+}
+
+std::unique_ptr<StandardTokenizer> custom_standard(int32_t max_token_length) {
+    auto tokenizer = std::make_unique<StandardTokenizer>();
+    tokenizer->set_max_token_length(max_token_length);
+    return tokenizer;
+}
+
+// What the custom standard tokenizer made before: the tokens of 
StandardTokenizerImpl, a token
+// longer than max_token_length dropped with its position left behind.
+std::vector<CollectedToken> scanner_tokens(std::string_view text, int32_t 
max_token_length) {
+    StandardTokenizerImpl scanner;
+    scanner.set_buffer_size(max_token_length);
+    scanner.yyreset(text);
+    std::vector<CollectedToken> tokens;
+    int32_t skipped = 0;
+    while (scanner.get_next_token() != 
StandardTokenizerImpl::TokenType::EOF_TOKEN) {
+        if (scanner.yylength() <= max_token_length) {
+            tokens.push_back(
+                    {.term = std::string(scanner.get_text()), 
.position_increment = skipped + 1});
+            skipped = 0;
+        } else {
+            ++skipped;
+        }
+    }
+    return tokens;
+}
+
+std::string bytes(std::initializer_list<int> values) {
+    std::string text;
+    for (int value : values) {
+        text.push_back(static_cast<char>(value));
+    }
+    return text;
+}
+
+// Values that stress the DFA and the windows: any byte, the separators UAX#29 
treats specially,
+// the scripts with token types of their own, overlong forms, and long runs.
+std::string random_value(std::mt19937_64& random) {
+    static const std::vector<std::string> kPieces = {"a",
+                                                     "B",
+                                                     "z9",
+                                                     "Q",
+                                                     " ",
+                                                     "'",
+                                                     ".",
+                                                     ",",
+                                                     ":",
+                                                     "_",
+                                                     "-",
+                                                     "\n",
+                                                     "3",
+                                                     "1.5",
+                                                     "中",
+                                                     "文",
+                                                     "é",
+                                                     "Ä",
+                                                     "ひ",
+                                                     "カ",
+                                                     "한",
+                                                     "ก",
+                                                     "ข",
+                                                     "́",
+                                                     "‍",
+                                                     "\U0001F600",
+                                                     "️",
+                                                     "\U0001F1EF",
+                                                     "\xC0\x80",
+                                                     "\xC1\x81",
+                                                     "\xE0\x80\x80",
+                                                     "\xF0\x80\x80\x80",
+                                                     "\xED\xA0\x80"};
+    const size_t size = random() % 600;
+    std::string value;
+    switch (random() % 4) {
+    case 0:
+        for (size_t i = 0; i < size; ++i) {
+            value.push_back(static_cast<char>(random()));
+        }
+        break;
+    case 1:
+    case 2:
+        while (value.size() < size) {
+            value += kPieces[random() % (kPieces.size() - (random() % 8 == 0 ? 
0 : 5))];
+        }
+        break;
+    default:
+        // Words long enough to meet the windows, with a multi-byte character 
now and then.
+        while (value.size() < size) {
+            value.append(random() % 300, static_cast<char>('a' + random() % 
26));
+            value += kPieces[random() % kPieces.size()];
+        }
+        break;
+    }
+    return value;
+}
+
+void expect_same_as_clucene(const std::string& value, golden::GoldenDriver& 
candidate,
+                            golden::GoldenDriver& oracle) {
+    ASSERT_EQ(candidate.fresh(value), oracle.fresh(value))
+            << golden::format_line(oracle.fresh(value));
+    ASSERT_EQ(candidate.reused(value), oracle.reused(value));
+}
+
+TEST(Uax29TokenizerTest, MatchesCLuceneOnRandomValues) {
+    for (bool lowercase : {true, false}) {
+        for (bool stop_words : {true, false}) {
+            SCOPED_TRACE(testing::Message() << lowercase << stop_words);
+            auto oracle = clucene_standard(lowercase, stop_words);
+            auto candidate = 
golden::make_text_driver(standard_analyzer(lowercase, stop_words));
+            std::mt19937_64 random(20261104 + lowercase * 2 + stop_words);
+            for (int round = 0; round < 8000; ++round) {
+                expect_same_as_clucene(random_value(random), *candidate, 
*oracle);
+            }
+        }
+    }
+}
+
+// A token is scanned while it is shorter than 255 bytes, so a character that 
starts inside the
+// window may end past it, and a token longer than 255 bytes is dropped 
without a gap.
+TEST(Uax29TokenizerTest, ScansAtMost255BytesFromATokenStart) {
+    auto oracle = clucene_standard(true, true);
+    auto candidate = golden::make_text_driver(standard_analyzer(true, true));
+    const std::string a254(254, 'a');
+    for (const std::string& value :
+         {std::string(255, 'a'), std::string(256, 'a'), std::string(300, 'a'),
+          std::string(510, 'a'), std::string(511, 'a'), a254 + "é", a254 + "é 
bc", a254 + "中",
+          std::string(253, 'a') + "é", a254 + "\U0001F600", a254 + "'s",
+          std::string(250, '1') + ".5.6.7", "x " + a254 + "é y"}) {
+        expect_same_as_clucene(value, *candidate, *oracle);
+    }
+    EXPECT_EQ(collect_terms(*standard_analyzer(true, true), std::string(300, 
'a')),
+              (std::vector<std::string> {std::string(255, 'a'), 
std::string(45, 'a')}));
+    // 254 letters and é are 256 bytes: dropped, and "bc" is one position 
after "x".
+    const std::vector<CollectedToken> tokens =
+            collect_checked(*standard_analyzer(true, true), "x " + a254 + "é 
bc");
+    ASSERT_EQ(tokens.size(), 2);
+    EXPECT_EQ(tokens[1].term, "bc");
+    EXPECT_EQ(tokens[1].position_increment, 1);
+}
+
+TEST(Uax29TokenizerTest, PassesOverlongFormsAndMakesNothingOfInvalidValues) {
+    auto oracle = clucene_standard(true, true);
+    auto candidate = golden::make_text_driver(standard_analyzer(true, true));
+    for (const std::string& value :
+         {"abc" + bytes({0xC0, 0x80}) + "def", "x" + bytes({0xC1, 0x81}) + "y",
+          "Wo" + bytes({0xE0, 0x81, 0x81}) + "rd", "abc" + bytes({0xFF}),
+          "abc" + bytes({0xED, 0xA0, 0x80}), "abc " + bytes({0xE4, 0xB8}),
+          bytes({0xF0, 0x80, 0x80, 0x80}) + " tail"}) {
+        expect_same_as_clucene(value, *candidate, *oracle);
+    }
+    EXPECT_TRUE(collect_terms(*standard_analyzer(true, true), "abc" + 
bytes({0xFF})).empty());
+}
+
+TEST(Uax29TokenizerTest, StopWordsAreTheLuceneSet) {
+    const std::set<std::string_view> ours(standard_stop_words().begin(),
+                                          standard_stop_words().end());
+    const std::set<std::string_view> 
lucene(lucene::analysis::standard95::stop_words.begin(),
+                                            
lucene::analysis::standard95::stop_words.end());
+    EXPECT_EQ(ours.size(), 33);
+    EXPECT_EQ(ours, lucene);
+}
+
+TEST(Uax29TokenizerTest, OnlyTheStopWordsThemselvesAreStopWords) {
+    const std::set<std::string> words(standard_stop_words().begin(), 
standard_stop_words().end());
+    for (const std::string& word : words) {
+        EXPECT_TRUE(is_standard_stop_word(word)) << word;
+        std::vector<std::string> variants = {
+                word + "s", "x" + word, word + word,
+                std::string(1, static_cast<char>(word[0] - 32)) + 
word.substr(1),
+                word + std::string(1, '\0')};
+        if (word.size() > 1) {
+            variants.push_back(word.substr(0, word.size() - 1));
+            variants.push_back(word.substr(1));
+        }
+        for (const std::string& variant : variants) {
+            EXPECT_EQ(is_standard_stop_word(variant), words.contains(variant))
+                    << word << " -> " << variant;
+        }
+    }
+    EXPECT_FALSE(is_standard_stop_word(""));
+    EXPECT_FALSE(is_standard_stop_word("thereby"));
+}
+
+TEST(Uax29TokenizerTest, DropsStopWordsWithoutAGap) {
+    const std::string value = "The quick fox and THE lazy dog is not here";
+    const std::vector<CollectedToken> lowered =
+            collect_checked(*standard_analyzer(true, true), value);
+    std::vector<std::string> terms;
+    for (const CollectedToken& token : lowered) {
+        terms.push_back(token.term);
+        EXPECT_EQ(token.position_increment, 1) << token.term;
+    }
+    EXPECT_EQ(terms, (std::vector<std::string> {"quick", "fox", "lazy", "dog", 
"here"}));
+    // Without lowercase only the lowercase spellings are stop words.
+    EXPECT_EQ(collect_terms(*standard_analyzer(false, true), value),
+              (std::vector<std::string> {"The", "quick", "fox", "THE", "lazy", 
"dog", "here"}));
+    EXPECT_EQ(collect_terms(*standard_analyzer(true, false), value).size(), 
10);
+}
+
+TEST(Uax29TokenizerTest, LowercasesWordsAndPointsIntoTheValueOrItsCopy) {
+    const std::string value = "MiXeD 42 ÄÖ 中文 WORD_x";
+    EXPECT_EQ(collect_terms(*standard_analyzer(true, false), value),
+              (std::vector<std::string> {"mixed", "42", "ÄÖ", "中", "文", 
"word_x"}));
+    for (bool lowercase : {true, false}) {
+        Uax29Tokenizer tokenizer(lowercase, false);
+        tokenizer.reset(value);
+        TokenBatch batch;
+        ASSERT_TRUE(tokenizer.next_batch(batch));
+        for (const TokenRef& token : batch.tokens()) {
+            const bool inside = token.data >= value.data() &&
+                                token.data + token.size <= value.data() + 
value.size();
+            EXPECT_EQ(inside, !lowercase) << token.term();
+        }
+        EXPECT_EQ(tokenizer.reserved_bytes() > 0, lowercase);
+    }
+}
+
+TEST(Uax29TokenizerTest, BatchesLoseNoTokenAroundTheBatchSize) {
+    auto oracle = clucene_standard(true, true);
+    auto candidate = golden::make_text_driver(standard_analyzer(true, true));
+    for (size_t count : {size_t {255}, size_t {256}, size_t {257}, size_t 
{512}}) {
+        std::string value;
+        for (size_t i = 0; i < count; ++i) {
+            value += "w" + std::to_string(i) + " the ";
+        }
+        const std::vector<CollectedToken> tokens =
+                collect_checked(*standard_analyzer(true, true), value);
+        ASSERT_EQ(tokens.size(), count);
+        for (size_t i = 0; i < count; ++i) {
+            EXPECT_EQ(tokens[i].term, "w" + std::to_string(i));
+            EXPECT_EQ(tokens[i].position_increment, 1);
+        }
+        expect_same_as_clucene(value, *candidate, *oracle);
+        EXPECT_EQ(collect_checked(*custom_standard(255), value), 
scanner_tokens(value, 255));
+    }
+}
+
+// A valid value ending at a guard page: the decoder reads only the bytes of 
the last character.
+TEST(Uax29TokenizerTest, ReadsNothingPastTheValue) {
+    const size_t page_size = sysconf(_SC_PAGESIZE);
+    void* mapping = mmap(nullptr, page_size * 3, PROT_NONE, MAP_PRIVATE | 
MAP_ANONYMOUS, -1, 0);
+    ASSERT_NE(mapping, MAP_FAILED);
+    auto* page = static_cast<char*>(mapping) + page_size;
+    ASSERT_EQ(mprotect(page, page_size, PROT_READ | PROT_WRITE), 0);
+    for (const std::string& tail : {std::string("K"), std::string("é"), 
std::string("中"),
+                                    std::string("\U0001F600"), 
std::string("\xC1\x81")}) {
+        for (size_t words = 0; words < 3; ++words) {
+            std::string text;
+            for (size_t i = 0; i < words; ++i) {
+                text += "ab ";
+            }
+            text += tail;
+            char* const value = page + page_size - text.size();
+            std::memcpy(value, text.data(), text.size());
+            const std::string_view view(value, text.size());
+            Uax29Tokenizer builtin(true, true);
+            EXPECT_EQ(collect_terms(builtin, view).size(), 
collect_terms(builtin, text).size());
+            auto custom = custom_standard(255);
+            EXPECT_EQ(collect_checked(*custom, view), scanner_tokens(text, 
255));
+        }
+    }
+    EXPECT_EQ(munmap(mapping, page_size * 3), 0);
+}
+
+TEST(Uax29TokenizerTest, KeepsTheStreamContract) {
+    std::string many_batches;
+    for (int i = 0; i < 2000; ++i) {
+        many_batches += "Word" + std::to_string(i % 97) + " the ";
+    }
+    const std::vector<std::string> texts = {"",
+                                            "Hello World",
+                                            "a",
+                                            std::string(300, 'Z'),
+                                            "中文 mixed ÄÖÜ 123 \U0001F600",
+                                            "bad" + bytes({0xFF}) + " tail",
+                                            many_batches};
+    for (bool lowercase : {true, false}) {
+        const TextAnalyzerPtr analyzer = standard_analyzer(lowercase, true);
+        expect_text_stream_contract([&] { return analyzer->create_stream(); }, 
texts);
+    }
+    for (int32_t length : {1, 4, 255}) {
+        expect_text_stream_contract([length] { return custom_standard(length); 
}, texts);
+    }
+}
+
+// reset() releases the lowercase copy of a long value also when the next 
value is invalid UTF-8
+// and makes no token: e.g. a 256 KiB row and then "abc\xFF" leave at most the 
high water mark.
+TEST(Uax29TokenizerTest, AnInvalidValueReleasesTheCopyOfALongOne) {
+    const std::string long_value(4 * ANALYZER_SCRATCH_HIGH_WATER_BYTES, 'A');
+    const std::string invalid_value = "abc" + bytes({0xFF});
+    auto stream = standard_analyzer(true, false)->create_stream();
+    TokenBatch batch;
+    stream->reset(long_value);
+    while (stream->next_batch(batch)) {
+    }
+    ASSERT_GT(stream->reserved_bytes(), ANALYZER_SCRATCH_HIGH_WATER_BYTES);
+    stream->reset(invalid_value);
+    EXPECT_FALSE(stream->next_batch(batch));
+    EXPECT_LE(stream->reserved_bytes(), ANALYZER_SCRATCH_HIGH_WATER_BYTES);
+}
+
+TEST(CustomStandardTokenizerTest, MatchesTheScannerOnRandomValues) {
+    for (int32_t length : {1, 2, 3, 4, 5, 16, 255, 256, 1000, 16383}) {
+        SCOPED_TRACE(length);
+        auto tokenizer = custom_standard(length);
+        std::mt19937_64 random(20261105 + length);
+        for (int round = 0; round < 2000; ++round) {
+            const std::string value = random_value(random);
+            ASSERT_EQ(collect_checked(*tokenizer, value), 
scanner_tokens(value, length))
+                    << round << ": " << value;
+        }
+    }
+}
+
+TEST(CustomStandardTokenizerTest, CutsWordsAtTheWindow) {
+    EXPECT_EQ(collect_terms(*custom_standard(4), "abcdefghij"),
+              (std::vector<std::string> {"abcd", "efgh", "ij"}));
+    EXPECT_EQ(collect_terms(*custom_standard(4), "ab中cd"),
+              (std::vector<std::string> {"ab", "中", "cd"}));
+    for (const std::string& value : {std::string("abcdefghij"), 
std::string("ab中cd"),
+                                     std::string("1.5.6.7.8 can't"), 
std::string(300, 'a')}) {
+        for (int32_t length : {2, 3, 4, 5, 8}) {
+            EXPECT_EQ(collect_checked(*custom_standard(length), value),
+                      scanner_tokens(value, length))
+                    << length << ": " << value;
+        }
+    }
+}
+
+// The scanner's buffer cannot take a character longer than max_token_length, 
and the value ends
+// there.
+TEST(CustomStandardTokenizerTest, EndsTheValueAtACharacterLongerThanTheWindow) 
{
+    EXPECT_EQ(collect_terms(*custom_standard(2), "ab中cd"), 
(std::vector<std::string> {"ab"}));
+    EXPECT_EQ(collect_terms(*custom_standard(1), "a中b"), 
(std::vector<std::string> {"a"}));
+    EXPECT_EQ(collect_terms(*custom_standard(3), "a中b"),
+              (std::vector<std::string> {"a", "中", "b"}));
+    EXPECT_EQ(collect_terms(*custom_standard(3), "x \U0001F600 y"),
+              (std::vector<std::string> {"x"}));
+    for (int32_t length : {1, 2, 3, 4}) {
+        for (const std::string& value : {std::string("ab中cd"), 
std::string("a中b"),
+                                         std::string("x \U0001F600 y"), 
std::string("é")}) {
+            EXPECT_EQ(collect_checked(*custom_standard(length), value),
+                      scanner_tokens(value, length))
+                    << length << ": " << value;
+        }
+    }
+}
+
+// An invalid value goes through the scanner, whose buffer moves at every 
refill: with a window of
+// 8 bytes every word is read by a refill of its own, and views of the buffer 
would all show the
+// last one.
+TEST(CustomStandardTokenizerTest, CopiesTheTermsOfAnInvalidValue) {
+    // The word before an invalid byte is lost, as it always was: the scanner 
meets the byte while
+    // it reads the word and ends the value there.
+    for (const auto& [value, expected] :
+         {std::pair {"aaaa bbbb cccc" + bytes({0xFF}), 
std::vector<std::string> {"aaaa", "bbbb"}},
+          std::pair {"aaaa bbbb cccc dd" + bytes({0xFF}),
+                     std::vector<std::string> {"aaaa", "bbbb", "cccc"}}}) {
+        auto tokenizer = custom_standard(8);
+        tokenizer->reset(value);
+        TokenBatch batch;
+        ASSERT_TRUE(tokenizer->next_batch(batch));
+        std::vector<std::string> terms;
+        for (const TokenRef& token : batch.tokens()) {
+            terms.emplace_back(token.term());
+        }
+        EXPECT_EQ(terms, expected);
+        EXPECT_FALSE(tokenizer->next_batch(batch));
+        EXPECT_EQ(collect_checked(*custom_standard(8), value), 
scanner_tokens(value, 8));
+    }
+}
+
+TEST(CustomStandardTokenizerTest, TermsOfAValidValueAreViewsOfIt) {
+    const std::string value = "Views of the value, 中文 too";
+    auto tokenizer = custom_standard(255);
+    tokenizer->reset(value);
+    TokenBatch batch;
+    ASSERT_TRUE(tokenizer->next_batch(batch));
+    EXPECT_EQ(batch.size(), 7);
+    for (const TokenRef& token : batch.tokens()) {
+        EXPECT_GE(token.data, value.data());
+        EXPECT_LE(token.data + token.size, value.data() + value.size());
+    }
+}
+
+} // namespace
+} // namespace doris::segment_v2::inverted_index
diff --git 
a/be/test/storage/index/inverted/analysis/tokenizer/utf8_validator_test.cpp 
b/be/test/storage/index/inverted/analysis/tokenizer/utf8_validator_test.cpp
new file mode 100644
index 00000000000..5956cc81b3c
--- /dev/null
+++ b/be/test/storage/index/inverted/analysis/tokenizer/utf8_validator_test.cpp
@@ -0,0 +1,146 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "storage/index/inverted/analysis/tokenizer/utf8_validator.h"
+
+#include <CLucene.h> // IWYU pragma: keep
+#include <gtest/gtest.h>
+
+#include <cstdint>
+#include <random>
+#include <string>
+
+#include "CLucene/util/stringUtil.h"
+#include "util/utf8_check.h"
+
+namespace doris::segment_v2::inverted_index {
+namespace {
+
+bool clucene_valid(const std::string& text) {
+    return StringUtil::validate_utf8(text) == 0;
+}
+
+std::string bytes(std::initializer_list<int> values) {
+    std::string text;
+    for (int value : values) {
+        text.push_back(static_cast<char>(value));
+    }
+    return text;
+}
+
+TEST(Utf8ValidatorTest, AgreesWithCLuceneOnEveryValueOfUpToThreeBytes) {
+    EXPECT_TRUE(lenient_utf8_valid(""));
+    std::string text(3, '\0');
+    for (int a = 0; a < 256; ++a) {
+        text.resize(1);
+        text[0] = static_cast<char>(a);
+        ASSERT_EQ(lenient_utf8_valid(text), clucene_valid(text)) << a;
+        for (int b = 0; b < 256; ++b) {
+            text.resize(2);
+            text[1] = static_cast<char>(b);
+            ASSERT_EQ(lenient_utf8_valid(text), clucene_valid(text)) << a << " 
" << b;
+            text.resize(3);
+            for (int c = 0; c < 256; ++c) {
+                text[2] = static_cast<char>(c);
+                ASSERT_EQ(lenient_utf8_valid(text), clucene_valid(text))
+                        << a << " " << b << " " << c;
+            }
+        }
+    }
+}
+
+TEST(Utf8ValidatorTest, AgreesWithCLuceneOnRandomBytes) {
+    std::mt19937_64 random(20261104);
+    // Mostly bytes that make or break sequences, so that long values are not 
all invalid.
+    const auto next_byte = [&random]() -> char {
+        switch (random() % 4) {
+        case 0:
+            return static_cast<char>(random() % 0x80);
+        case 1:
+            return static_cast<char>(0x80 + random() % 0x40);
+        case 2:
+            return static_cast<char>(0xC0 + random() % 0x38);
+        default:
+            return static_cast<char>(random());
+        }
+    };
+    for (int round = 0; round < 200000; ++round) {
+        std::string text(random() % 24, '\0');
+        for (char& c : text) {
+            c = next_byte();
+        }
+        ASSERT_EQ(lenient_utf8_valid(text), clucene_valid(text)) << round;
+    }
+}
+
+TEST(Utf8ValidatorTest, 
AcceptsOverlongFormsAndRefusesSurrogatesAndBeyondTheLastPlane) {
+    EXPECT_TRUE(lenient_utf8_valid(bytes({0xC0, 0x80})));
+    EXPECT_TRUE(lenient_utf8_valid(bytes({0xC1, 0x81})));
+    EXPECT_TRUE(lenient_utf8_valid(bytes({0xE0, 0x80, 0x80})));
+    EXPECT_TRUE(lenient_utf8_valid(bytes({0xF0, 0x80, 0x80, 0x80})));
+    EXPECT_FALSE(lenient_utf8_valid(bytes({0xED, 0xA0, 0x80})));       // 
U+D800
+    EXPECT_FALSE(lenient_utf8_valid(bytes({0xF4, 0x90, 0x80, 0x80}))); // 
U+110000
+    EXPECT_FALSE(lenient_utf8_valid(bytes({0xE4, 0xB8})));             // cut 
short
+    EXPECT_FALSE(lenient_utf8_valid(bytes({0x80})));
+    EXPECT_FALSE(lenient_utf8_valid(bytes({0xF8, 0x80, 0x80, 0x80, 0x80})));
+    // Strict UTF-8 refuses the overlong forms, so the tokenizers ask the 
lenient rule after it.
+    const std::string overlong = bytes({0xC1, 0x81});
+    EXPECT_FALSE(doris::validate_utf8(overlong.data(), overlong.size()));
+}
+
+// Every sequence the rule accepts decodes to what CLucene decodes it to, 
overlong forms included.
+TEST(Utf8ValidatorTest, DecodesEveryAcceptedSequenceAsCLucene) {
+    size_t checked = 0;
+    const auto check = [&checked](const std::string& sequence) {
+        if (!lenient_utf8_valid(sequence)) {
+            return;
+        }
+        const std::wstring expected = StringUtil::string_to_wstring(sequence);
+        ASSERT_EQ(expected.size(), 1);
+        size_t length = 0;
+        const uint32_t code_point =
+                decode_lenient_utf8(reinterpret_cast<const 
uint8_t*>(sequence.data()), &length);
+        ASSERT_EQ(length, sequence.size());
+        ASSERT_EQ(code_point, static_cast<uint32_t>(expected[0]));
+        ++checked;
+    };
+    for (int lead = 0xC0; lead <= 0xDF; ++lead) {
+        for (int b = 0x80; b <= 0xBF; ++b) {
+            check(bytes({lead, b}));
+        }
+    }
+    for (int lead = 0xE0; lead <= 0xEF; ++lead) {
+        for (int b = 0x80; b <= 0xBF; ++b) {
+            for (int c = 0x80; c <= 0xBF; ++c) {
+                check(bytes({lead, b, c}));
+            }
+        }
+    }
+    for (int lead = 0xF0; lead <= 0xF4; ++lead) {
+        for (int b = 0x80; b <= 0xBF; ++b) {
+            for (int c = 0x80; c <= 0xBF; c += 7) {
+                for (int d = 0x80; d <= 0xBF; d += 5) {
+                    check(bytes({lead, b, c, d}));
+                }
+            }
+        }
+    }
+    EXPECT_GT(checked, 60000);
+}
+
+} // namespace
+} // namespace doris::segment_v2::inverted_index
diff --git a/be/test/storage/index/inverted/clucene/lucene_adapter_test.cpp 
b/be/test/storage/index/inverted/clucene/lucene_adapter_test.cpp
index 6b9ee76a3be..d993f362e45 100644
--- a/be/test/storage/index/inverted/clucene/lucene_adapter_test.cpp
+++ b/be/test/storage/index/inverted/clucene/lucene_adapter_test.cpp
@@ -32,13 +32,12 @@
 #include <utility>
 #include <vector>
 
+#include "CLucene/analysis/standard95/StandardAnalyzer.h"
 #include "common/exception.h"
 #include "common/status.h"
 #include "io/fs/local_file_system.h"
 #include "storage/index/index_file_writer.h"
 #include "storage/index/index_writer.h"
-#include "storage/index/inverted/analysis/analyzer/analyzer.h"
-#include "storage/index/inverted/analysis/inverted_index_parser.h"
 #include "storage/index/inverted/analysis/scratch.h"
 #include "storage/index/inverted/analysis/text_stream_contract.h"
 #include "storage/index/inverted/analysis/token_batch.h"
@@ -299,11 +298,11 @@ TEST(LuceneBackedTextAnalyzerTest, 
KeepsTheTextStreamContract) {
     const LuceneBackedTextAnalyzer 
scripted(std::make_shared<ScriptedLuceneAnalyzer>());
     expect_text_stream_contract([&] { return scripted.create_stream(); },
                                 {"a ^b _ c", words(1000), words(256), "x"});
-    // The builtin standard parser is CLucene's StandardAnalyzer behind a 
LuceneBackedTextAnalyzer.
-    const TextAnalyzerPtr standard = 
InvertedIndexAnalyzer::create_builtin_analyzer(
-            InvertedIndexParserType::PARSER_STANDARD, "", "", "");
-    ASSERT_NE(dynamic_cast<const LuceneBackedTextAnalyzer*>(standard.get()), 
nullptr);
-    expect_text_stream_contract([&] { return standard->create_stream(); },
+    // A real CLucene analyzer behind the adapter: the standard one, which the 
builtin standard
+    // parser used before it had a tokenizer of its own.
+    const LuceneBackedTextAnalyzer standard(
+            
std::make_shared<lucene::analysis::standard95::StandardAnalyzer>());
+    expect_text_stream_contract([&] { return standard.create_stream(); },
                                 {"The Quick brown fox", words(1000), "中文 mixed 
文本", "x"});
 }
 


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to