This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new d556f32bf8c [improvement](be) Tokenize standard and unicode with a
dense UAX#29 table (#70265)
d556f32bf8c is described below
commit d556f32bf8c98a245527e74f14673bc3e2960964
Author: Jack <[email protected]>
AuthorDate: Sat Nov 21 12:15:31 2026 +0800
[improvement](be) Tokenize standard and unicode with a dense UAX#29 table
(#70265)
### What problem does this PR solve?
Issue Number: None
Related PR: #70264 (depends on)
Problem Summary: Tokenize standard and unicode with one flat UAX#29
table: about 1.8x as fast as master, with the same tokens.
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [x] Unit Test
- [x] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [x] No.
- [ ] Yes.
- Does this need documentation?
- [x] No.
- [ ] Yes.
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label
🤖 Generated with [Claude Code](https://claude.com/claude-code)
---------
Co-authored-by: Claude Opus 5.5 <[email protected]>
---
.../index/inverted/analysis/analyzer/analyzer.cpp | 9 +-
.../tokenizer/standard/standard_tokenizer.cpp | 102 +++++
.../tokenizer/standard/standard_tokenizer.h | 74 ++--
.../tokenizer/standard/standard_tokenizer_impl.h | 3 +
.../analysis/tokenizer/standard/uax29_table.cpp | 63 +++
.../analysis/tokenizer/standard/uax29_table.h | 122 ++++++
.../tokenizer/standard/uax29_tokenizer.cpp | 126 ++++++
.../analysis/tokenizer/standard/uax29_tokenizer.h | 92 +++++
.../inverted/analysis/tokenizer/utf8_validator.h | 87 ++++
be/src/storage/utils.cpp | 13 +-
.../inverted/analysis/component_stream_test.cpp | 13 +
.../analysis/tokenizer/uax29_table_test.cpp | 112 ++++++
.../analysis/tokenizer/uax29_tokenizer_test.cpp | 447 +++++++++++++++++++++
.../analysis/tokenizer/utf8_validator_test.cpp | 146 +++++++
.../index/inverted/clucene/lucene_adapter_test.cpp | 13 +-
15 files changed, 1360 insertions(+), 62 deletions(-)
diff --git a/be/src/storage/index/inverted/analysis/analyzer/analyzer.cpp
b/be/src/storage/index/inverted/analysis/analyzer/analyzer.cpp
index 8e09388e7d6..1a837527954 100644
--- a/be/src/storage/index/inverted/analysis/analyzer/analyzer.cpp
+++ b/be/src/storage/index/inverted/analysis/analyzer/analyzer.cpp
@@ -52,6 +52,7 @@
#include
"storage/index/inverted/analysis/tokenizer/icu/icu_tokenizer_factory.h"
#include "storage/index/inverted/analysis/tokenizer/ik/ik_tokenizer_factory.h"
#include
"storage/index/inverted/analysis/tokenizer/kuromoji/kuromoji_tokenizer_factory.h"
+#include "storage/index/inverted/analysis/tokenizer/standard/uax29_tokenizer.h"
#include "storage/index/inverted/clucene/lucene_backed_text_analyzer.h"
namespace doris::segment_v2::inverted_index {
@@ -222,9 +223,11 @@ TextAnalyzerPtr
build_builtin_analyzer(InvertedIndexParserType parser_type,
const std::string& stop_words) {
if (parser_type == InvertedIndexParserType::PARSER_STANDARD ||
parser_type == InvertedIndexParserType::PARSER_UNICODE) {
- return lucene_builtin_analyzer(
-
std::make_shared<lucene::analysis::standard95::StandardAnalyzer>(), lower_case,
- stop_words);
+ return std::make_shared<ChainAnalyzer>(
+ std::vector<CharFilterFactoryPtr> {},
+
std::make_shared<Uax29TokenizerFactory>(builtin_lowercase(lower_case, true),
+ stop_words != "none"),
+ std::vector<TokenFilterFactoryPtr> {});
}
if (parser_type == InvertedIndexParserType::PARSER_CHINESE) {
auto chinese_analyzer =
diff --git
a/be/src/storage/index/inverted/analysis/tokenizer/standard/standard_tokenizer.cpp
b/be/src/storage/index/inverted/analysis/tokenizer/standard/standard_tokenizer.cpp
new file mode 100644
index 00000000000..fac9740e748
--- /dev/null
+++
b/be/src/storage/index/inverted/analysis/tokenizer/standard/standard_tokenizer.cpp
@@ -0,0 +1,102 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include
"storage/index/inverted/analysis/tokenizer/standard/standard_tokenizer.h"
+
+#include <memory>
+#include <string>
+
+#include "common/exception.h"
+#include "common/logging.h"
+#include "storage/index/inverted/analysis/token_batch.h"
+#include "storage/index/inverted/analysis/tokenizer/standard/uax29_table.h"
+#include "util/utf8_check.h"
+
+namespace doris::segment_v2::inverted_index {
+
+StandardTokenizer::StandardTokenizer()
+ : _table(Uax29Table::instance()),
_scanner(std::make_unique<StandardTokenizerImpl>()) {}
+
+void StandardTokenizer::set_max_token_length(int32_t length) {
+ if (length < 1) {
+ throw Exception(ErrorCode::INVALID_ARGUMENT, "max_token_length must be
greater than zero");
+ } else if (length > MAX_TOKEN_LENGTH_LIMIT) {
+ throw Exception(
+ ErrorCode::INVALID_ARGUMENT,
+ "max_token_length may not exceed " +
std::to_string(MAX_TOKEN_LENGTH_LIMIT));
+ }
+ if (length != _max_token_length) {
+ _max_token_length = length;
+ _scanner->set_buffer_size(length);
+ }
+}
+
+void StandardTokenizer::reset(std::string_view text) {
+ _text = text;
+ _valid = doris::validate_utf8(text.data(), text.size());
+ _pos = 0;
+ _exhausted = false;
+ if (!_valid) {
+ _scanner->yyreset(text);
+ }
+}
+
+bool StandardTokenizer::next_batch(TokenBatch& batch) {
+ return _valid ? next_valid_batch(batch) : next_scanner_batch(batch);
+}
+
+bool StandardTokenizer::next_valid_batch(TokenBatch& batch) {
+ batch.clear();
+ const auto* text = reinterpret_cast<const uint8_t*>(_text.data());
+ const size_t size = _text.size();
+ const auto window = static_cast<size_t>(_max_token_length);
+ while (_pos < size && batch.size() < kBatchTokens) {
+ const size_t start = _pos;
+ const Uax29Match match =
+ _table.longest_match(text, size, start, [start, window](size_t
cur, size_t length) {
+ return cur + length - start <= window;
+ });
+ if (match.action < 0) {
+ // The character at `start` is longer than max_token_length, so
the scanner's buffer
+ // could not take it, and the scanner ended the value there.
+ _pos = size;
+ break;
+ }
+ _pos = match.end;
+ if (match.action != Uax29Table::kSkip) {
+ batch.push({_text.data() + start, match.end - start}, 1);
+ }
+ }
+ return batch.size() != 0;
+}
+
+bool StandardTokenizer::next_scanner_batch(TokenBatch& batch) {
+ batch.clear();
+ while (!_exhausted && batch.size() < kBatchTokens) {
+ if (_scanner->get_next_token() ==
StandardTokenizerImpl::TokenType::EOF_TOKEN) {
+ _exhausted = true;
+ break;
+ }
+ // The buffer of max_token_length bytes holds the whole token.
+ DCHECK_LE(_scanner->yylength(), _max_token_length);
+ // The term points into the scanner's buffer, which its next refill
moves.
+ batch.push_copy(_scanner->get_text(), 1);
+ }
+ return batch.size() != 0;
+}
+
+} // namespace doris::segment_v2::inverted_index
diff --git
a/be/src/storage/index/inverted/analysis/tokenizer/standard/standard_tokenizer.h
b/be/src/storage/index/inverted/analysis/tokenizer/standard/standard_tokenizer.h
index c8677e2acfa..7a4a51d8317 100644
---
a/be/src/storage/index/inverted/analysis/tokenizer/standard/standard_tokenizer.h
+++
b/be/src/storage/index/inverted/analysis/tokenizer/standard/standard_tokenizer.h
@@ -17,68 +17,52 @@
#pragma once
-#include <memory>
-#include <string>
+#include <cstddef>
+#include <cstdint>
#include <string_view>
-#include <utility>
-#include "common/exception.h"
-#include "storage/index/inverted/analysis/per_token_stream.h"
+#include "storage/index/inverted/analysis/text_token_stream.h"
#include
"storage/index/inverted/analysis/tokenizer/standard/standard_tokenizer_impl.h"
namespace doris::segment_v2::inverted_index {
-// The words of the value by the Unicode word break rules (UAX#29); a word
longer than
-// max_token_length is dropped, and its position is skipped.
-class StandardTokenizer final : public PerTokenTokenizer {
+struct Uax29Table;
+
+// The custom standard tokenizer: the UAX#29 words of the value, each one
position after the last.
+// It keeps the rules of StandardTokenizerImpl, the JFlex scanner it was,
whose buffer of
+// max_token_length bytes is the window a token is matched in: a character
that does not fit ends
+// the token, so a longer word is cut into pieces, and a character that does
not fit an empty window
+// ends the value. E.g. with max_token_length=2, "abc" makes "ab" and "c", and
"ab", then U+4E2D
+// (3 bytes in UTF-8), then "cd" makes "ab" and nothing after it.
+//
+// A value that is valid UTF-8 runs through the DFA of Uax29Table, and its
terms are views of the
+// value. Any other value runs through the scanner itself, whose reaction to
invalid bytes depends
+// on where its buffer refills; those terms are copied, because a refill moves
the buffer.
+class StandardTokenizer final : public TextTokenizer {
public:
- StandardTokenizer() { _scanner =
std::make_unique<StandardTokenizerImpl>(); }
+ StandardTokenizer();
- void set_max_token_length(int32_t length) {
- if (length < 1) {
- throw Exception(ErrorCode::INVALID_ARGUMENT,
- "max_token_length must be greater than zero");
- } else if (length > MAX_TOKEN_LENGTH_LIMIT) {
- throw Exception(
- ErrorCode::INVALID_ARGUMENT,
- "max_token_length may not exceed " +
std::to_string(MAX_TOKEN_LENGTH_LIMIT));
- }
- if (length != _max_token_length) {
- _max_token_length = length;
- _scanner->set_buffer_size(length);
- }
- }
+ void set_max_token_length(int32_t length);
- // The scan window, max_token_length bytes.
+ void reset(std::string_view text) override;
+ bool next_batch(TokenBatch& batch) override;
+ // The scanner's window, max_token_length bytes.
size_t reserved_bytes() const override { return
_scanner->buffer_capacity(); }
static constexpr int32_t DEFAULT_MAX_TOKEN_LENGTH = 255;
static constexpr int32_t MAX_TOKEN_LENGTH_LIMIT = 16383;
-protected:
- void on_reset(std::string_view text) override { _scanner->yyreset(text); }
-
- bool next_token(TokenRef& token) override {
- int32_t skipped_positions = 0;
- while (true) {
- const auto token_type = _scanner->get_next_token();
- if (token_type == StandardTokenizerImpl::TokenType::EOF_TOKEN) {
- return false;
- }
- if (std::cmp_less_equal(_scanner->yylength(), _max_token_length)) {
- const std::string_view term = _scanner->get_text();
- token = {.data = term.data(),
- .size = static_cast<uint32_t>(term.size()),
- .position_increment = skipped_positions + 1};
- return true;
- }
- skipped_positions++;
- }
- }
-
private:
+ bool next_valid_batch(TokenBatch& batch);
+ bool next_scanner_batch(TokenBatch& batch);
+
+ const Uax29Table& _table;
StandardTokenizerImplPtr _scanner;
int32_t _max_token_length = DEFAULT_MAX_TOKEN_LENGTH;
+ std::string_view _text;
+ bool _valid = true; // the value is valid UTF-8
+ size_t _pos = 0; // where the next token of a valid value starts
+ bool _exhausted = false; // the scanner reached the end of an invalid value
};
} // namespace doris::segment_v2::inverted_index
diff --git
a/be/src/storage/index/inverted/analysis/tokenizer/standard/standard_tokenizer_impl.h
b/be/src/storage/index/inverted/analysis/tokenizer/standard/standard_tokenizer_impl.h
index cdf60d4efac..5b618da7b14 100644
---
a/be/src/storage/index/inverted/analysis/tokenizer/standard/standard_tokenizer_impl.h
+++
b/be/src/storage/index/inverted/analysis/tokenizer/standard/standard_tokenizer_impl.h
@@ -278,6 +278,9 @@ public:
static constexpr int32_t YYINITIAL = 0;
private:
+ // Lays the ZZ_* tables out flat for the standard tokenizers' core loop.
+ friend struct Uax29Table;
+
static constexpr int32_t ZZ_UNKNOWN_ERROR = 0;
static constexpr int32_t ZZ_NO_MATCH = 1;
static constexpr int32_t ZZ_PUSHBACK_2BIG = 2;
diff --git
a/be/src/storage/index/inverted/analysis/tokenizer/standard/uax29_table.cpp
b/be/src/storage/index/inverted/analysis/tokenizer/standard/uax29_table.cpp
new file mode 100644
index 00000000000..4b966854a13
--- /dev/null
+++ b/be/src/storage/index/inverted/analysis/tokenizer/standard/uax29_table.cpp
@@ -0,0 +1,63 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "storage/index/inverted/analysis/tokenizer/standard/uax29_table.h"
+
+#include <algorithm>
+
+#include "common/check.h"
+#include
"storage/index/inverted/analysis/tokenizer/standard/standard_tokenizer_impl.h"
+
+namespace doris::segment_v2::inverted_index {
+
+const Uax29Table& Uax29Table::instance() {
+ // Built at first use, after the scanner's tables, which are initialized
dynamically.
+ static const Uax29Table table;
+ return table;
+}
+
+Uax29Table::Uax29Table() {
+ using Impl = StandardTokenizerImpl;
+ DORIS_CHECK(Impl::ZZ_ATTRIBUTE.size() == kStates && Impl::ZZ_ACTION.size()
== kStates &&
+ Impl::ZZ_ROWMAP.size() == kStates);
+ DORIS_CHECK(Impl::ZZ_CMAP_TOP.size() == kCmapTop &&
Impl::ZZ_CMAP_BLOCKS.size() == kCmapBlocks);
+ DORIS_CHECK(*std::ranges::max_element(Impl::ZZ_CMAP_BLOCKS) == kClasses -
1);
+ // The scanner looks the first 256 code points up in block 0 without the
top level.
+ DORIS_CHECK(Impl::ZZ_CMAP_TOP[0] == 0);
+
+ for (int state = 0; state < kStates; ++state) {
+ for (int cls = 0; cls < kClasses; ++cls) {
+ const int32_t to = Impl::ZZ_TRANS[Impl::ZZ_ROWMAP[state] + cls];
+ DORIS_CHECK(to >= -1 && to < kStates);
+ next[state * kClasses + cls] = to < 0 ? kDead :
static_cast<uint8_t>(to);
+ }
+ attribute[state] = static_cast<uint8_t>(Impl::ZZ_ATTRIBUTE[state]);
+ action[state] = static_cast<uint8_t>(Impl::ZZ_ACTION[state]);
+ }
+ for (size_t c = 0; c < ascii_class.size(); ++c) {
+ ascii_class[c] = static_cast<uint8_t>(Impl::ZZ_CMAP_BLOCKS[c]);
+ }
+ for (size_t i = 0; i < kCmapTop; ++i) {
+ cmap_top[i] = static_cast<uint16_t>(Impl::ZZ_CMAP_TOP[i]);
+ }
+ for (size_t i = 0; i < kCmapBlocks; ++i) {
+ cmap_blocks[i] = static_cast<uint8_t>(Impl::ZZ_CMAP_BLOCKS[i]);
+ }
+ initial = static_cast<uint8_t>(Impl::ZZ_LEXSTATE[Impl::YYINITIAL]);
+}
+
+} // namespace doris::segment_v2::inverted_index
diff --git
a/be/src/storage/index/inverted/analysis/tokenizer/standard/uax29_table.h
b/be/src/storage/index/inverted/analysis/tokenizer/standard/uax29_table.h
new file mode 100644
index 00000000000..74bcf225f11
--- /dev/null
+++ b/be/src/storage/index/inverted/analysis/tokenizer/standard/uax29_table.h
@@ -0,0 +1,122 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <array>
+#include <cstddef>
+#include <cstdint>
+
+#include "common/compiler_util.h"
+#include "storage/index/inverted/analysis/tokenizer/utf8_validator.h"
+
+namespace doris::segment_v2::inverted_index {
+
+// The longest match of the DFA from a token start: the text up to `end`
matched `action`, or not
+// one character fit and `action` is -1.
+struct Uax29Match {
+ size_t end = 0;
+ int action = -1;
+};
+
+// The UAX#29 word-break DFA of StandardTokenizerImpl, the JFlex scanner both
standard tokenizers
+// come from, laid out flat with one byte per transition, so that a step is
one load. It is built
+// once from the scanner's ZZ_* tables, which CLucene's builtin scanner shares
entry for entry, so
+// neither tokenizer depends on a hand-copied table.
+struct Uax29Table {
+ static constexpr int kStates = 59;
+ static constexpr int kClasses = 29;
+ static constexpr uint8_t kDead = 0xFF; // no transition: the longest match
ends
+ // Attribute bits.
+ static constexpr uint8_t kAccepting = 1; // a token may end in the state
+ static constexpr uint8_t kFinal = 8; // no transition leaves the state
+ // The actions of the accepting states: what the text matched up to such a
state is.
+ enum Action : uint8_t {
+ kSkip = 1,
+ kNumeric = 2,
+ kWord = 3,
+ kEmoji = 4,
+ kSouthEastAsian = 5,
+ kHangul = 6,
+ kIdeographic = 7,
+ kKatakana = 8,
+ kHiragana = 9,
+ };
+ // The two-level character map: the class of code point c is
+ // cmap_blocks[cmap_top[c >> 8] | (c & 0xFF)].
+ static constexpr size_t kCmapTop = 0x110000 >> 8;
+ static constexpr size_t kCmapBlocks = 35328;
+
+ std::array<uint8_t, kStates * kClasses> next {};
+ std::array<uint8_t, kStates> attribute {};
+ std::array<uint8_t, kStates> action {};
+ std::array<uint8_t, 128> ascii_class {};
+ std::array<uint16_t, kCmapTop> cmap_top {};
+ std::array<uint8_t, kCmapBlocks> cmap_blocks {};
+ uint8_t initial = 0;
+
+ static const Uax29Table& instance();
+
+ // The class of a code point up to U+10FFFF.
+ uint8_t char_class(uint32_t code_point) const {
+ return cmap_blocks[cmap_top[code_point >> 8] | (code_point & 0xFF)];
+ }
+
+ // The longest match from `start` of a value lenient_utf8_valid()
accepted, the way the JFlex
+ // scanner matched: it follows transitions while `fits(cur, length)` lets
the character of
+ // `length` bytes at `cur` join the token, and the token ends at the last
accepting state it
+ // passed. Every character leads from the initial state to an accepting
one, so `action` is -1
+ // only when the first character did not fit. E.g. with a window of 4
bytes, "ab" followed by
+ // U+4E2D (3 bytes in UTF-8) matches "ab": U+4E2D would take the token to
5 bytes.
+ template <typename Fits>
+ ALWAYS_INLINE Uax29Match longest_match(const uint8_t* text, size_t size,
size_t start,
+ Fits fits) const {
+ Uax29Match match {.end = start, .action = -1};
+ size_t cur = start;
+ uint8_t state = initial;
+ while (cur < size) {
+ const uint8_t c = text[cur];
+ size_t length = 1;
+ const uint8_t cls = c < 0x80 ? ascii_class[c]
+ : char_class(decode_lenient_utf8(text
+ cur, &length));
+ if (!fits(cur, length)) {
+ break;
+ }
+ const uint8_t to = next[state * kClasses + cls];
+ if (to == kDead) {
+ break;
+ }
+ cur += length;
+ state = to;
+ const uint8_t attributes = attribute[to];
+ if ((attributes & kAccepting) != 0) {
+ match = {.end = cur, .action = action[to]};
+ if ((attributes & kFinal) != 0) {
+ break;
+ }
+ }
+ }
+ return match;
+ }
+
+private:
+ // Fills the table in place, inside the static of instance(): 46 KB is too
much for the stack
+ // of the thread that happens to use it first.
+ Uax29Table();
+};
+
+} // namespace doris::segment_v2::inverted_index
diff --git
a/be/src/storage/index/inverted/analysis/tokenizer/standard/uax29_tokenizer.cpp
b/be/src/storage/index/inverted/analysis/tokenizer/standard/uax29_tokenizer.cpp
new file mode 100644
index 00000000000..d075f8b4d65
--- /dev/null
+++
b/be/src/storage/index/inverted/analysis/tokenizer/standard/uax29_tokenizer.cpp
@@ -0,0 +1,126 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "storage/index/inverted/analysis/tokenizer/standard/uax29_tokenizer.h"
+
+#include <algorithm>
+#include <array>
+#include <cstdint>
+
+#include "common/logging.h"
+#include "storage/index/inverted/analysis/scratch.h"
+#include "storage/index/inverted/analysis/token_batch.h"
+#include "storage/index/inverted/analysis/tokenizer/byte_classifier.h"
+#include "storage/index/inverted/analysis/tokenizer/standard/uax29_table.h"
+#include "storage/index/inverted/analysis/tokenizer/utf8_validator.h"
+#include "util/utf8_check.h"
+
+namespace doris::segment_v2::inverted_index {
+
+namespace {
+
+constexpr std::array<std::string_view, 33> STOP_WORDS = {
+ "a", "an", "and", "are", "as", "at", "be", "but", "by",
"for", "if",
+ "in", "into", "is", "it", "no", "not", "of", "on", "or",
"such", "that",
+ "the", "their", "then", "there", "these", "they", "this", "to",
"was", "will", "with"};
+constexpr size_t LONGEST_STOP_WORD = 5;
+
+// A term of at most 7 bytes as one integer: its bytes from the low end and
its length in the top
+// byte, so that terms of different lengths never compare equal.
+constexpr uint64_t pack_term(std::string_view term) {
+ uint64_t key = static_cast<uint64_t>(term.size()) << 56;
+ for (size_t i = 0; i < term.size(); ++i) {
+ key |= static_cast<uint64_t>(static_cast<uint8_t>(term[i])) << (8 * i);
+ }
+ return key;
+}
+
+// Sorted for binary search; most tokens leave on the length test before it.
+constexpr std::array<uint64_t, STOP_WORDS.size()> STOP_WORD_KEYS = [] {
+ std::array<uint64_t, STOP_WORDS.size()> keys {};
+ for (size_t i = 0; i < STOP_WORDS.size(); ++i) {
+ keys[i] = pack_term(STOP_WORDS[i]);
+ }
+ std::ranges::sort(keys);
+ return keys;
+}();
+
+} // namespace
+
+std::span<const std::string_view> standard_stop_words() {
+ return STOP_WORDS;
+}
+
+bool is_standard_stop_word(std::string_view term) {
+ return term.size() <= LONGEST_STOP_WORD &&
+ std::ranges::binary_search(STOP_WORD_KEYS, pack_term(term));
+}
+
+Uax29Tokenizer::Uax29Tokenizer(bool lowercase, bool stop_words)
+ : _table(Uax29Table::instance()), _lowercase(lowercase),
_stop_words(stop_words) {}
+
+void Uax29Tokenizer::reset(std::string_view text) {
+ _pos = 0;
+ // Before the early returns: an invalid value after a 1 MB one must not
keep its 1 MB copy.
+ release_oversized_scratch(_lowered);
+ // Strict UTF-8 passing decides alone; the lenient rule decides the rest,
the overlong forms.
+ if (!doris::validate_utf8(text.data(), text.size()) &&
!lenient_utf8_valid(text)) {
+ _text = {};
+ return;
+ }
+ if (!_lowercase) {
+ _text = text;
+ return;
+ }
+ // The copy only grows, so a shorter value writes over the head of it
without a resize.
+ if (_lowered.size() < text.size()) {
+ _lowered.resize(text.size());
+ }
+ const char* in = text.data();
+ char* out = _lowered.data();
+ for (size_t i = 0; i < text.size(); ++i) {
+ out[i] = ascii_lower(in[i]);
+ }
+ _text = {out, text.size()};
+}
+
+bool Uax29Tokenizer::next_batch(TokenBatch& batch) {
+ batch.clear();
+ const auto* text = reinterpret_cast<const uint8_t*>(_text.data());
+ const size_t size = _text.size();
+ while (_pos < size && batch.size() < kBatchTokens) {
+ const size_t start = _pos;
+ const Uax29Match match = _table.longest_match(
+ text, size, start,
+ [start](size_t cur, size_t /*length*/) { return cur - start <
MAX_TOKEN_LENGTH; });
+ // The window always takes the first character.
+ DCHECK_GE(match.action, 0);
+ _pos = match.end;
+ const size_t length = match.end - start;
+ if (match.action == Uax29Table::kSkip || length > MAX_TOKEN_LENGTH) {
+ continue;
+ }
+ const std::string_view term(_text.data() + start, length);
+ if (_stop_words && match.action == Uax29Table::kWord &&
is_standard_stop_word(term)) {
+ continue;
+ }
+ batch.push(term, 1);
+ }
+ return batch.size() != 0;
+}
+
+} // namespace doris::segment_v2::inverted_index
diff --git
a/be/src/storage/index/inverted/analysis/tokenizer/standard/uax29_tokenizer.h
b/be/src/storage/index/inverted/analysis/tokenizer/standard/uax29_tokenizer.h
new file mode 100644
index 00000000000..491626adf65
--- /dev/null
+++
b/be/src/storage/index/inverted/analysis/tokenizer/standard/uax29_tokenizer.h
@@ -0,0 +1,92 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <cstddef>
+#include <memory>
+#include <span>
+#include <string>
+#include <string_view>
+
+#include "storage/index/inverted/analysis/text_token_stream.h"
+#include "storage/index/inverted/analysis/tokenizer/tokenizer_factory.h"
+
+namespace doris::segment_v2::inverted_index {
+
+struct Uax29Table;
+
+// The 33 stop words of the builtin standard and unicode parsers, the English
set of Lucene's
+// StandardAnalyzer.
+std::span<const std::string_view> standard_stop_words();
+// Whether a term, as the tokenizer makes it, is one of them: the comparison
is byte for byte, so
+// "The" is a stop word only once lowercased.
+bool is_standard_stop_word(std::string_view term);
+
+// The tokenizer of the builtin standard and unicode parsers, which make the
tokens CLucene's
+// standard95 tokenizer made. A value that fails lenient_utf8_valid()
(utf8_validator.h) has none.
+// Otherwise its tokens are the UAX#29 words: a token is scanned while it is
shorter than
+// MAX_TOKEN_LENGTH bytes, so its last character may end past it, and a token
longer than
+// MAX_TOKEN_LENGTH is dropped. With `lowercase`, A-Z in WORD tokens is folded
to a-z; with
+// `stop_words`, a WORD token that is a stop word is dropped. Every token is
one position after the
+// last kept one: the old tokenizer never set an increment, so a dropped token
left no gap.
+//
+// Lowercasing folds the whole value into a copy at the same offsets, and the
scan runs over the
+// copy. That makes the same tokens because A-Z and a-z are one class of the
DFA, and lowercases
+// only WORD tokens because every accepting state reached through a letter is
WORD; Uax29TableTest
+// checks both on the table.
+class Uax29Tokenizer final : public TextTokenizer {
+public:
+ Uax29Tokenizer(bool lowercase, bool stop_words);
+
+ void reset(std::string_view text) override;
+ bool next_batch(TokenBatch& batch) override;
+ // The lowered copy of the value, which only a lowercasing tokenizer has.
+ size_t reserved_bytes() const override { return _lowercase ?
_lowered.capacity() : 0; }
+
+ // The scan window from a token start, and the longest token kept.
+ static constexpr size_t MAX_TOKEN_LENGTH = 255;
+
+private:
+ const Uax29Table& _table;
+ const bool _lowercase;
+ const bool _stop_words;
+ // The value or its lowered copy; empty for a value that failed the UTF-8
rule.
+ std::string_view _text;
+ // The value with A-Z folded, byte for byte at the offsets of the value;
only with _lowercase.
+ std::string _lowered;
+ size_t _pos = 0; // where the next token starts
+};
+
+// The tokenizer of the builtin standard and unicode parsers; neither is a
custom tokenizer.
+class Uax29TokenizerFactory final : public TokenizerFactory {
+public:
+ Uax29TokenizerFactory(bool lowercase, bool stop_words)
+ : _lowercase(lowercase), _stop_words(stop_words) {}
+
+ void initialize(const Settings& settings) override {}
+
+ std::unique_ptr<TextTokenizer> create() const override {
+ return std::make_unique<Uax29Tokenizer>(_lowercase, _stop_words);
+ }
+
+private:
+ const bool _lowercase;
+ const bool _stop_words;
+};
+
+} // namespace doris::segment_v2::inverted_index
diff --git a/be/src/storage/index/inverted/analysis/tokenizer/utf8_validator.h
b/be/src/storage/index/inverted/analysis/tokenizer/utf8_validator.h
new file mode 100644
index 00000000000..7597918b58d
--- /dev/null
+++ b/be/src/storage/index/inverted/analysis/tokenizer/utf8_validator.h
@@ -0,0 +1,87 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+#include <string_view>
+
+#include "common/logging.h"
+
+namespace doris::segment_v2::inverted_index {
+
+// The UTF-8 rule of the builtin standard and unicode parsers, CLucene's
StringUtil::validate_utf8:
+// the high bits of a lead byte announce one to three trailing bytes, every
trailing byte is
+// 10xxxxxx, and the code point a sequence decodes to is neither a surrogate
nor above U+10FFFF.
+// It accepts what strict UTF-8 refuses, the overlong forms: C0 80 for U+0000,
F0 80 80 80. Every
+// value strict UTF-8 accepts passes it too, so a strict check that passes
decides alone.
+inline bool lenient_utf8_valid(std::string_view text) {
+ int32_t trailing = 0;
+ uint32_t code_point = 0;
+ for (const char ch : text) {
+ const auto c = static_cast<uint8_t>(ch);
+ if (trailing == 0) {
+ if ((c & 0x80) == 0) {
+ continue;
+ }
+ if ((c & 0xE0) == 0xC0) {
+ code_point = c & 0x1F;
+ trailing = 1;
+ } else if ((c & 0xF0) == 0xE0) {
+ code_point = c & 0x0F;
+ trailing = 2;
+ } else if ((c & 0xF8) == 0xF0) {
+ code_point = c & 0x07;
+ trailing = 3;
+ } else {
+ return false;
+ }
+ continue;
+ }
+ if ((c & 0xC0) != 0x80) {
+ return false;
+ }
+ code_point = (code_point << 6) | (c & 0x3F);
+ if (--trailing == 0 &&
+ !(code_point < 0xD800 || (code_point >= 0xE000 && code_point <=
0x10FFFF))) {
+ return false;
+ }
+ }
+ return trailing == 0;
+}
+
+// The code point of the multi-byte sequence at `p` of a value
lenient_utf8_valid() accepted, with
+// its length in `*length`: the lead byte's high bits give the length and the
low bits of every
+// byte the code point, overlong forms included, as CLucene's standard scanner
decoded: C1 81 is
+// U+0041.
+inline uint32_t decode_lenient_utf8(const uint8_t* p, size_t* length) {
+ const uint8_t c = p[0];
+ if ((c & 0xE0) == 0xC0) {
+ *length = 2;
+ return ((c & 0x1FU) << 6) | (p[1] & 0x3FU);
+ }
+ if ((c & 0xF0) == 0xE0) {
+ *length = 3;
+ return ((c & 0x0FU) << 12) | ((p[1] & 0x3FU) << 6) | (p[2] & 0x3FU);
+ }
+ DCHECK_EQ(c & 0xF8, 0xF0);
+ *length = 4;
+ return ((c & 0x07U) << 18) | ((p[1] & 0x3FU) << 12) | ((p[2] & 0x3FU) <<
6) | (p[3] & 0x3FU);
+}
+
+} // namespace doris::segment_v2::inverted_index
diff --git a/be/src/storage/utils.cpp b/be/src/storage/utils.cpp
index 2af5607e7f8..7dae4316af4 100644
--- a/be/src/storage/utils.cpp
+++ b/be/src/storage/utils.cpp
@@ -47,7 +47,6 @@
#include "util/string_parser.hpp"
namespace doris {
-using namespace ErrorCode;
uint32_t olap_adler32_init() {
return (uint32_t)adler32(0, Z_NULL, 0);
@@ -62,11 +61,11 @@ Status gen_timestamp_string(std::string* out_string) {
tm local_tm;
if (localtime_r(&now, &local_tm) == nullptr) {
- return Status::Error<OS_ERROR>("fail to localtime_r time. time={}",
now);
+ return Status::Error<ErrorCode::OS_ERROR>("fail to localtime_r time.
time={}", now);
}
char time_suffix[16] = {0}; // Example: 20150706111404's length is 15
if (strftime(time_suffix, sizeof(time_suffix), "%Y%m%d%H%M%S", &local_tm)
== 0) {
- return Status::Error<OS_ERROR>("fail to strftime time. time={}", now);
+ return Status::Error<ErrorCode::OS_ERROR>("fail to strftime time.
time={}", now);
}
*out_string = time_suffix;
@@ -93,13 +92,13 @@ Status read_write_test_file(const std::string&
test_file_path) {
char* write_test_buff = nullptr;
char* read_test_buff = nullptr;
if (posix_memalign((void**)&write_test_buff, DIRECT_IO_ALIGNMENT,
TEST_FILE_BUF_SIZE) != 0) {
- return Status::Error<MEM_ALLOC_FAILED>("fail to allocate write buffer
memory. size={}",
- TEST_FILE_BUF_SIZE);
+ return Status::Error<ErrorCode::MEM_ALLOC_FAILED>(
+ "fail to allocate write buffer memory. size={}",
TEST_FILE_BUF_SIZE);
}
std::unique_ptr<char, decltype(&std::free)> write_buff(write_test_buff,
&std::free);
if (posix_memalign((void**)&read_test_buff, DIRECT_IO_ALIGNMENT,
TEST_FILE_BUF_SIZE) != 0) {
- return Status::Error<MEM_ALLOC_FAILED>("fail to allocate read buffer
memory. size={}",
- TEST_FILE_BUF_SIZE);
+ return Status::Error<ErrorCode::MEM_ALLOC_FAILED>(
+ "fail to allocate read buffer memory. size={}",
TEST_FILE_BUF_SIZE);
}
std::unique_ptr<char, decltype(&std::free)> read_buff(read_test_buff,
&std::free);
// generate random numbers
diff --git a/be/test/storage/index/inverted/analysis/component_stream_test.cpp
b/be/test/storage/index/inverted/analysis/component_stream_test.cpp
index 0c995dc84a1..d2f1fc1fff3 100644
--- a/be/test/storage/index/inverted/analysis/component_stream_test.cpp
+++ b/be/test/storage/index/inverted/analysis/component_stream_test.cpp
@@ -54,6 +54,7 @@
#include "storage/index/inverted/analysis/tokenizer/ngram/ngram_tokenizer.h"
#include "storage/index/inverted/analysis/tokenizer/pinyin/pinyin_tokenizer.h"
#include
"storage/index/inverted/analysis/tokenizer/standard/standard_tokenizer.h"
+#include "storage/index/inverted/analysis/tokenizer/standard/uax29_tokenizer.h"
#include "storage/index/inverted/analysis/tokenizer/tokenizer_factory.h"
#include "util/defer_op.h"
@@ -283,6 +284,18 @@ TEST(ComponentStreamTest,
EnglishTokenizerKeepsNoRowSizedScratch) {
}
}
+TEST(ComponentStreamTest, Uax29TokenizerReleasesItsLoweredCopy) {
+ for (bool lowercase : {true, false}) {
+ SCOPED_TRACE(lowercase);
+ expect_text_stream_contract(
+ [lowercase] { return
std::make_unique<Uax29Tokenizer>(lowercase, true); },
+ contract_texts());
+ Uax29Tokenizer standard(lowercase, true);
+ expect_released_at_next_row(standard, repeat_to("Ab ", kLongRowBytes),
"short", lowercase,
+ [&] { return lowercase ?
standard._lowered.capacity() : 0; });
+ }
+}
+
TEST(ComponentStreamTest, ICUTokenizerReleasesItsCopyOfTheValue) {
DictPathScope dict_path;
Components components;
diff --git
a/be/test/storage/index/inverted/analysis/tokenizer/uax29_table_test.cpp
b/be/test/storage/index/inverted/analysis/tokenizer/uax29_table_test.cpp
new file mode 100644
index 00000000000..c88f53c10d3
--- /dev/null
+++ b/be/test/storage/index/inverted/analysis/tokenizer/uax29_table_test.cpp
@@ -0,0 +1,112 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "storage/index/inverted/analysis/tokenizer/standard/uax29_table.h"
+
+#include <gtest/gtest.h>
+
+#include <cstdint>
+#include <set>
+#include <utility>
+#include <vector>
+
+#include
"storage/index/inverted/analysis/tokenizer/standard/standard_tokenizer_impl.h"
+
+namespace doris::segment_v2::inverted_index {
+namespace {
+
+using Impl = StandardTokenizerImpl;
+
+TEST(Uax29TableTest, MatchesTheScannerTablesEntryByEntry) {
+ const Uax29Table& table = Uax29Table::instance();
+ ASSERT_EQ(Impl::ZZ_ATTRIBUTE.size(), Uax29Table::kStates);
+ for (int state = 0; state < Uax29Table::kStates; ++state) {
+ for (int cls = 0; cls < Uax29Table::kClasses; ++cls) {
+ const int32_t to = Impl::ZZ_TRANS[Impl::ZZ_ROWMAP[state] + cls];
+ EXPECT_EQ(table.next[state * Uax29Table::kClasses + cls],
+ to < 0 ? Uax29Table::kDead : to)
+ << "state " << state << " class " << cls;
+ }
+ EXPECT_EQ(table.attribute[state], Impl::ZZ_ATTRIBUTE[state]) << state;
+ EXPECT_EQ(table.action[state], Impl::ZZ_ACTION[state]) << state;
+ }
+ for (int c = 0; c < 128; ++c) {
+ EXPECT_EQ(table.ascii_class[c], Impl::ZZ_CMAP_BLOCKS[c]) << c;
+ }
+ EXPECT_EQ(table.initial, Impl::ZZ_LEXSTATE[Impl::YYINITIAL]);
+}
+
+TEST(Uax29TableTest, ClassesEveryCodePointAsTheScanner) {
+ const Uax29Table& table = Uax29Table::instance();
+ Impl scanner;
+ for (uint32_t code_point = 0; code_point <= 0x10FFFF; ++code_point) {
+ ASSERT_EQ(table.char_class(code_point),
scanner.zz_cmap(static_cast<int32_t>(code_point)))
+ << code_point;
+ }
+}
+
+// The first character always makes a match, so a match is empty only when the
window refuses the
+// first character, which Uax29Table::longest_match() reports as action -1.
+TEST(Uax29TableTest, EveryCharacterLeadsFromTheInitialStateToAnAcceptingOne) {
+ const Uax29Table& table = Uax29Table::instance();
+ EXPECT_EQ(table.attribute[table.initial] & Uax29Table::kAccepting, 0);
+ for (int cls = 0; cls < Uax29Table::kClasses; ++cls) {
+ const uint8_t to = table.next[table.initial * Uax29Table::kClasses +
cls];
+ ASSERT_NE(to, Uax29Table::kDead) << cls;
+ EXPECT_NE(table.attribute[to] & Uax29Table::kAccepting, 0) << cls;
+ }
+}
+
+// Why the builtin tokenizer may lowercase the whole value before it scans:
folding A-Z changes no
+// class, so the scan makes the same tokens, and every accepting state the DFA
reaches through a
+// letter is WORD, so no token of another type has a letter to fold. If a new
table breaks either
+// half, this fails before a token changes.
+TEST(Uax29TableTest, LowercasingTheWholeValueOnlyLowercasesWords) {
+ const Uax29Table& table = Uax29Table::instance();
+ const int letter = table.ascii_class['A'];
+ for (char c = 'A'; c <= 'Z'; ++c) {
+ EXPECT_EQ(table.ascii_class[c], letter) << c;
+ EXPECT_EQ(table.ascii_class[c - 'A' + 'a'], letter) << c;
+ }
+ // Every (state, a letter on the way there) pair the DFA can reach.
+ std::set<std::pair<uint8_t, bool>> seen;
+ std::vector<std::pair<uint8_t, bool>> todo {{table.initial, false}};
+ while (!todo.empty()) {
+ const auto [state, through_letter] = todo.back();
+ todo.pop_back();
+ if (!seen.insert({state, through_letter}).second) {
+ continue;
+ }
+ for (int cls = 0; cls < Uax29Table::kClasses; ++cls) {
+ const uint8_t to = table.next[state * Uax29Table::kClasses + cls];
+ if (to != Uax29Table::kDead) {
+ todo.emplace_back(to, through_letter || cls == letter);
+ }
+ }
+ }
+ size_t words = 0;
+ for (const auto& [state, through_letter] : seen) {
+ if (through_letter && (table.attribute[state] &
Uax29Table::kAccepting) != 0) {
+ EXPECT_EQ(table.action[state], Uax29Table::kWord) <<
static_cast<int>(state);
+ ++words;
+ }
+ }
+ EXPECT_GT(words, 0);
+}
+
+} // namespace
+} // namespace doris::segment_v2::inverted_index
diff --git
a/be/test/storage/index/inverted/analysis/tokenizer/uax29_tokenizer_test.cpp
b/be/test/storage/index/inverted/analysis/tokenizer/uax29_tokenizer_test.cpp
new file mode 100644
index 00000000000..db731052dc4
--- /dev/null
+++ b/be/test/storage/index/inverted/analysis/tokenizer/uax29_tokenizer_test.cpp
@@ -0,0 +1,447 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "storage/index/inverted/analysis/tokenizer/standard/uax29_tokenizer.h"
+
+#include <CLucene.h> // IWYU pragma: keep
+#include <gtest/gtest.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+#include <cstring>
+#include <memory>
+#include <random>
+#include <set>
+#include <string>
+#include <string_view>
+#include <vector>
+
+#include "CLucene/analysis/standard95/StandardAnalyzer.h"
+#include "storage/index/inverted/analysis/analyzer/analyzer.h"
+#include "storage/index/inverted/analysis/golden/golden_driver.h"
+#include "storage/index/inverted/analysis/scratch.h"
+#include "storage/index/inverted/analysis/text_stream_contract.h"
+#include "storage/index/inverted/analysis/token_batch.h"
+#include
"storage/index/inverted/analysis/tokenizer/standard/standard_tokenizer.h"
+#include
"storage/index/inverted/analysis/tokenizer/standard/standard_tokenizer_impl.h"
+
+namespace doris::segment_v2::inverted_index {
+namespace {
+
+// The standard parser as production builds it; unicode is the same analyzer.
+TextAnalyzerPtr standard_analyzer(bool lowercase, bool stop_words) {
+ return
InvertedIndexAnalyzer::create_builtin_analyzer(InvertedIndexParserType::PARSER_STANDARD,
+ "", lowercase ?
"true" : "false",
+ stop_words ? "" :
"none");
+}
+
+// CLucene's standard95 analyzer, which made the standard tokens before.
+std::unique_ptr<golden::GoldenDriver> clucene_standard(bool lowercase, bool
stop_words) {
+ return golden::make_lucene_driver([lowercase, stop_words] {
+ auto analyzer =
std::make_shared<lucene::analysis::standard95::StandardAnalyzer>();
+ analyzer->set_lowercase(lowercase);
+ analyzer->set_stopwords(stop_words ?
&lucene::analysis::standard95::stop_words : nullptr);
+ return analyzer;
+ });
+}
+
+std::unique_ptr<StandardTokenizer> custom_standard(int32_t max_token_length) {
+ auto tokenizer = std::make_unique<StandardTokenizer>();
+ tokenizer->set_max_token_length(max_token_length);
+ return tokenizer;
+}
+
+// What the custom standard tokenizer made before: the tokens of
StandardTokenizerImpl, a token
+// longer than max_token_length dropped with its position left behind.
+std::vector<CollectedToken> scanner_tokens(std::string_view text, int32_t
max_token_length) {
+ StandardTokenizerImpl scanner;
+ scanner.set_buffer_size(max_token_length);
+ scanner.yyreset(text);
+ std::vector<CollectedToken> tokens;
+ int32_t skipped = 0;
+ while (scanner.get_next_token() !=
StandardTokenizerImpl::TokenType::EOF_TOKEN) {
+ if (scanner.yylength() <= max_token_length) {
+ tokens.push_back(
+ {.term = std::string(scanner.get_text()),
.position_increment = skipped + 1});
+ skipped = 0;
+ } else {
+ ++skipped;
+ }
+ }
+ return tokens;
+}
+
+std::string bytes(std::initializer_list<int> values) {
+ std::string text;
+ for (int value : values) {
+ text.push_back(static_cast<char>(value));
+ }
+ return text;
+}
+
+// Values that stress the DFA and the windows: any byte, the separators UAX#29
treats specially,
+// the scripts with token types of their own, overlong forms, and long runs.
+std::string random_value(std::mt19937_64& random) {
+ static const std::vector<std::string> kPieces = {"a",
+ "B",
+ "z9",
+ "Q",
+ " ",
+ "'",
+ ".",
+ ",",
+ ":",
+ "_",
+ "-",
+ "\n",
+ "3",
+ "1.5",
+ "中",
+ "文",
+ "é",
+ "Ä",
+ "ひ",
+ "カ",
+ "한",
+ "ก",
+ "ข",
+ "́",
+ "",
+ "\U0001F600",
+ "️",
+ "\U0001F1EF",
+ "\xC0\x80",
+ "\xC1\x81",
+ "\xE0\x80\x80",
+ "\xF0\x80\x80\x80",
+ "\xED\xA0\x80"};
+ const size_t size = random() % 600;
+ std::string value;
+ switch (random() % 4) {
+ case 0:
+ for (size_t i = 0; i < size; ++i) {
+ value.push_back(static_cast<char>(random()));
+ }
+ break;
+ case 1:
+ case 2:
+ while (value.size() < size) {
+ value += kPieces[random() % (kPieces.size() - (random() % 8 == 0 ?
0 : 5))];
+ }
+ break;
+ default:
+ // Words long enough to meet the windows, with a multi-byte character
now and then.
+ while (value.size() < size) {
+ value.append(random() % 300, static_cast<char>('a' + random() %
26));
+ value += kPieces[random() % kPieces.size()];
+ }
+ break;
+ }
+ return value;
+}
+
+void expect_same_as_clucene(const std::string& value, golden::GoldenDriver&
candidate,
+ golden::GoldenDriver& oracle) {
+ ASSERT_EQ(candidate.fresh(value), oracle.fresh(value))
+ << golden::format_line(oracle.fresh(value));
+ ASSERT_EQ(candidate.reused(value), oracle.reused(value));
+}
+
+TEST(Uax29TokenizerTest, MatchesCLuceneOnRandomValues) {
+ for (bool lowercase : {true, false}) {
+ for (bool stop_words : {true, false}) {
+ SCOPED_TRACE(testing::Message() << lowercase << stop_words);
+ auto oracle = clucene_standard(lowercase, stop_words);
+ auto candidate =
golden::make_text_driver(standard_analyzer(lowercase, stop_words));
+ std::mt19937_64 random(20261104 + lowercase * 2 + stop_words);
+ for (int round = 0; round < 8000; ++round) {
+ expect_same_as_clucene(random_value(random), *candidate,
*oracle);
+ }
+ }
+ }
+}
+
+// A token is scanned while it is shorter than 255 bytes, so a character that
starts inside the
+// window may end past it, and a token longer than 255 bytes is dropped
without a gap.
+TEST(Uax29TokenizerTest, ScansAtMost255BytesFromATokenStart) {
+ auto oracle = clucene_standard(true, true);
+ auto candidate = golden::make_text_driver(standard_analyzer(true, true));
+ const std::string a254(254, 'a');
+ for (const std::string& value :
+ {std::string(255, 'a'), std::string(256, 'a'), std::string(300, 'a'),
+ std::string(510, 'a'), std::string(511, 'a'), a254 + "é", a254 + "é
bc", a254 + "中",
+ std::string(253, 'a') + "é", a254 + "\U0001F600", a254 + "'s",
+ std::string(250, '1') + ".5.6.7", "x " + a254 + "é y"}) {
+ expect_same_as_clucene(value, *candidate, *oracle);
+ }
+ EXPECT_EQ(collect_terms(*standard_analyzer(true, true), std::string(300,
'a')),
+ (std::vector<std::string> {std::string(255, 'a'),
std::string(45, 'a')}));
+ // 254 letters and é are 256 bytes: dropped, and "bc" is one position
after "x".
+ const std::vector<CollectedToken> tokens =
+ collect_checked(*standard_analyzer(true, true), "x " + a254 + "é
bc");
+ ASSERT_EQ(tokens.size(), 2);
+ EXPECT_EQ(tokens[1].term, "bc");
+ EXPECT_EQ(tokens[1].position_increment, 1);
+}
+
+TEST(Uax29TokenizerTest, PassesOverlongFormsAndMakesNothingOfInvalidValues) {
+ auto oracle = clucene_standard(true, true);
+ auto candidate = golden::make_text_driver(standard_analyzer(true, true));
+ for (const std::string& value :
+ {"abc" + bytes({0xC0, 0x80}) + "def", "x" + bytes({0xC1, 0x81}) + "y",
+ "Wo" + bytes({0xE0, 0x81, 0x81}) + "rd", "abc" + bytes({0xFF}),
+ "abc" + bytes({0xED, 0xA0, 0x80}), "abc " + bytes({0xE4, 0xB8}),
+ bytes({0xF0, 0x80, 0x80, 0x80}) + " tail"}) {
+ expect_same_as_clucene(value, *candidate, *oracle);
+ }
+ EXPECT_TRUE(collect_terms(*standard_analyzer(true, true), "abc" +
bytes({0xFF})).empty());
+}
+
+TEST(Uax29TokenizerTest, StopWordsAreTheLuceneSet) {
+ const std::set<std::string_view> ours(standard_stop_words().begin(),
+ standard_stop_words().end());
+ const std::set<std::string_view>
lucene(lucene::analysis::standard95::stop_words.begin(),
+
lucene::analysis::standard95::stop_words.end());
+ EXPECT_EQ(ours.size(), 33);
+ EXPECT_EQ(ours, lucene);
+}
+
+TEST(Uax29TokenizerTest, OnlyTheStopWordsThemselvesAreStopWords) {
+ const std::set<std::string> words(standard_stop_words().begin(),
standard_stop_words().end());
+ for (const std::string& word : words) {
+ EXPECT_TRUE(is_standard_stop_word(word)) << word;
+ std::vector<std::string> variants = {
+ word + "s", "x" + word, word + word,
+ std::string(1, static_cast<char>(word[0] - 32)) +
word.substr(1),
+ word + std::string(1, '\0')};
+ if (word.size() > 1) {
+ variants.push_back(word.substr(0, word.size() - 1));
+ variants.push_back(word.substr(1));
+ }
+ for (const std::string& variant : variants) {
+ EXPECT_EQ(is_standard_stop_word(variant), words.contains(variant))
+ << word << " -> " << variant;
+ }
+ }
+ EXPECT_FALSE(is_standard_stop_word(""));
+ EXPECT_FALSE(is_standard_stop_word("thereby"));
+}
+
+TEST(Uax29TokenizerTest, DropsStopWordsWithoutAGap) {
+ const std::string value = "The quick fox and THE lazy dog is not here";
+ const std::vector<CollectedToken> lowered =
+ collect_checked(*standard_analyzer(true, true), value);
+ std::vector<std::string> terms;
+ for (const CollectedToken& token : lowered) {
+ terms.push_back(token.term);
+ EXPECT_EQ(token.position_increment, 1) << token.term;
+ }
+ EXPECT_EQ(terms, (std::vector<std::string> {"quick", "fox", "lazy", "dog",
"here"}));
+ // Without lowercase only the lowercase spellings are stop words.
+ EXPECT_EQ(collect_terms(*standard_analyzer(false, true), value),
+ (std::vector<std::string> {"The", "quick", "fox", "THE", "lazy",
"dog", "here"}));
+ EXPECT_EQ(collect_terms(*standard_analyzer(true, false), value).size(),
10);
+}
+
+TEST(Uax29TokenizerTest, LowercasesWordsAndPointsIntoTheValueOrItsCopy) {
+ const std::string value = "MiXeD 42 ÄÖ 中文 WORD_x";
+ EXPECT_EQ(collect_terms(*standard_analyzer(true, false), value),
+ (std::vector<std::string> {"mixed", "42", "ÄÖ", "中", "文",
"word_x"}));
+ for (bool lowercase : {true, false}) {
+ Uax29Tokenizer tokenizer(lowercase, false);
+ tokenizer.reset(value);
+ TokenBatch batch;
+ ASSERT_TRUE(tokenizer.next_batch(batch));
+ for (const TokenRef& token : batch.tokens()) {
+ const bool inside = token.data >= value.data() &&
+ token.data + token.size <= value.data() +
value.size();
+ EXPECT_EQ(inside, !lowercase) << token.term();
+ }
+ EXPECT_EQ(tokenizer.reserved_bytes() > 0, lowercase);
+ }
+}
+
+TEST(Uax29TokenizerTest, BatchesLoseNoTokenAroundTheBatchSize) {
+ auto oracle = clucene_standard(true, true);
+ auto candidate = golden::make_text_driver(standard_analyzer(true, true));
+ for (size_t count : {size_t {255}, size_t {256}, size_t {257}, size_t
{512}}) {
+ std::string value;
+ for (size_t i = 0; i < count; ++i) {
+ value += "w" + std::to_string(i) + " the ";
+ }
+ const std::vector<CollectedToken> tokens =
+ collect_checked(*standard_analyzer(true, true), value);
+ ASSERT_EQ(tokens.size(), count);
+ for (size_t i = 0; i < count; ++i) {
+ EXPECT_EQ(tokens[i].term, "w" + std::to_string(i));
+ EXPECT_EQ(tokens[i].position_increment, 1);
+ }
+ expect_same_as_clucene(value, *candidate, *oracle);
+ EXPECT_EQ(collect_checked(*custom_standard(255), value),
scanner_tokens(value, 255));
+ }
+}
+
+// A valid value ending at a guard page: the decoder reads only the bytes of
the last character.
+TEST(Uax29TokenizerTest, ReadsNothingPastTheValue) {
+ const size_t page_size = sysconf(_SC_PAGESIZE);
+ void* mapping = mmap(nullptr, page_size * 3, PROT_NONE, MAP_PRIVATE |
MAP_ANONYMOUS, -1, 0);
+ ASSERT_NE(mapping, MAP_FAILED);
+ auto* page = static_cast<char*>(mapping) + page_size;
+ ASSERT_EQ(mprotect(page, page_size, PROT_READ | PROT_WRITE), 0);
+ for (const std::string& tail : {std::string("K"), std::string("é"),
std::string("中"),
+ std::string("\U0001F600"),
std::string("\xC1\x81")}) {
+ for (size_t words = 0; words < 3; ++words) {
+ std::string text;
+ for (size_t i = 0; i < words; ++i) {
+ text += "ab ";
+ }
+ text += tail;
+ char* const value = page + page_size - text.size();
+ std::memcpy(value, text.data(), text.size());
+ const std::string_view view(value, text.size());
+ Uax29Tokenizer builtin(true, true);
+ EXPECT_EQ(collect_terms(builtin, view).size(),
collect_terms(builtin, text).size());
+ auto custom = custom_standard(255);
+ EXPECT_EQ(collect_checked(*custom, view), scanner_tokens(text,
255));
+ }
+ }
+ EXPECT_EQ(munmap(mapping, page_size * 3), 0);
+}
+
+TEST(Uax29TokenizerTest, KeepsTheStreamContract) {
+ std::string many_batches;
+ for (int i = 0; i < 2000; ++i) {
+ many_batches += "Word" + std::to_string(i % 97) + " the ";
+ }
+ const std::vector<std::string> texts = {"",
+ "Hello World",
+ "a",
+ std::string(300, 'Z'),
+ "中文 mixed ÄÖÜ 123 \U0001F600",
+ "bad" + bytes({0xFF}) + " tail",
+ many_batches};
+ for (bool lowercase : {true, false}) {
+ const TextAnalyzerPtr analyzer = standard_analyzer(lowercase, true);
+ expect_text_stream_contract([&] { return analyzer->create_stream(); },
texts);
+ }
+ for (int32_t length : {1, 4, 255}) {
+ expect_text_stream_contract([length] { return custom_standard(length);
}, texts);
+ }
+}
+
+// reset() releases the lowercase copy of a long value also when the next
value is invalid UTF-8
+// and makes no token: e.g. a 256 KiB row and then "abc\xFF" leave at most the
high water mark.
+TEST(Uax29TokenizerTest, AnInvalidValueReleasesTheCopyOfALongOne) {
+ const std::string long_value(4 * ANALYZER_SCRATCH_HIGH_WATER_BYTES, 'A');
+ const std::string invalid_value = "abc" + bytes({0xFF});
+ auto stream = standard_analyzer(true, false)->create_stream();
+ TokenBatch batch;
+ stream->reset(long_value);
+ while (stream->next_batch(batch)) {
+ }
+ ASSERT_GT(stream->reserved_bytes(), ANALYZER_SCRATCH_HIGH_WATER_BYTES);
+ stream->reset(invalid_value);
+ EXPECT_FALSE(stream->next_batch(batch));
+ EXPECT_LE(stream->reserved_bytes(), ANALYZER_SCRATCH_HIGH_WATER_BYTES);
+}
+
+TEST(CustomStandardTokenizerTest, MatchesTheScannerOnRandomValues) {
+ for (int32_t length : {1, 2, 3, 4, 5, 16, 255, 256, 1000, 16383}) {
+ SCOPED_TRACE(length);
+ auto tokenizer = custom_standard(length);
+ std::mt19937_64 random(20261105 + length);
+ for (int round = 0; round < 2000; ++round) {
+ const std::string value = random_value(random);
+ ASSERT_EQ(collect_checked(*tokenizer, value),
scanner_tokens(value, length))
+ << round << ": " << value;
+ }
+ }
+}
+
+TEST(CustomStandardTokenizerTest, CutsWordsAtTheWindow) {
+ EXPECT_EQ(collect_terms(*custom_standard(4), "abcdefghij"),
+ (std::vector<std::string> {"abcd", "efgh", "ij"}));
+ EXPECT_EQ(collect_terms(*custom_standard(4), "ab中cd"),
+ (std::vector<std::string> {"ab", "中", "cd"}));
+ for (const std::string& value : {std::string("abcdefghij"),
std::string("ab中cd"),
+ std::string("1.5.6.7.8 can't"),
std::string(300, 'a')}) {
+ for (int32_t length : {2, 3, 4, 5, 8}) {
+ EXPECT_EQ(collect_checked(*custom_standard(length), value),
+ scanner_tokens(value, length))
+ << length << ": " << value;
+ }
+ }
+}
+
+// The scanner's buffer cannot take a character longer than max_token_length,
and the value ends
+// there.
+TEST(CustomStandardTokenizerTest, EndsTheValueAtACharacterLongerThanTheWindow)
{
+ EXPECT_EQ(collect_terms(*custom_standard(2), "ab中cd"),
(std::vector<std::string> {"ab"}));
+ EXPECT_EQ(collect_terms(*custom_standard(1), "a中b"),
(std::vector<std::string> {"a"}));
+ EXPECT_EQ(collect_terms(*custom_standard(3), "a中b"),
+ (std::vector<std::string> {"a", "中", "b"}));
+ EXPECT_EQ(collect_terms(*custom_standard(3), "x \U0001F600 y"),
+ (std::vector<std::string> {"x"}));
+ for (int32_t length : {1, 2, 3, 4}) {
+ for (const std::string& value : {std::string("ab中cd"),
std::string("a中b"),
+ std::string("x \U0001F600 y"),
std::string("é")}) {
+ EXPECT_EQ(collect_checked(*custom_standard(length), value),
+ scanner_tokens(value, length))
+ << length << ": " << value;
+ }
+ }
+}
+
+// An invalid value goes through the scanner, whose buffer moves at every
refill: with a window of
+// 8 bytes every word is read by a refill of its own, and views of the buffer
would all show the
+// last one.
+TEST(CustomStandardTokenizerTest, CopiesTheTermsOfAnInvalidValue) {
+ // The word before an invalid byte is lost, as it always was: the scanner
meets the byte while
+ // it reads the word and ends the value there.
+ for (const auto& [value, expected] :
+ {std::pair {"aaaa bbbb cccc" + bytes({0xFF}),
std::vector<std::string> {"aaaa", "bbbb"}},
+ std::pair {"aaaa bbbb cccc dd" + bytes({0xFF}),
+ std::vector<std::string> {"aaaa", "bbbb", "cccc"}}}) {
+ auto tokenizer = custom_standard(8);
+ tokenizer->reset(value);
+ TokenBatch batch;
+ ASSERT_TRUE(tokenizer->next_batch(batch));
+ std::vector<std::string> terms;
+ for (const TokenRef& token : batch.tokens()) {
+ terms.emplace_back(token.term());
+ }
+ EXPECT_EQ(terms, expected);
+ EXPECT_FALSE(tokenizer->next_batch(batch));
+ EXPECT_EQ(collect_checked(*custom_standard(8), value),
scanner_tokens(value, 8));
+ }
+}
+
+TEST(CustomStandardTokenizerTest, TermsOfAValidValueAreViewsOfIt) {
+ const std::string value = "Views of the value, 中文 too";
+ auto tokenizer = custom_standard(255);
+ tokenizer->reset(value);
+ TokenBatch batch;
+ ASSERT_TRUE(tokenizer->next_batch(batch));
+ EXPECT_EQ(batch.size(), 7);
+ for (const TokenRef& token : batch.tokens()) {
+ EXPECT_GE(token.data, value.data());
+ EXPECT_LE(token.data + token.size, value.data() + value.size());
+ }
+}
+
+} // namespace
+} // namespace doris::segment_v2::inverted_index
diff --git
a/be/test/storage/index/inverted/analysis/tokenizer/utf8_validator_test.cpp
b/be/test/storage/index/inverted/analysis/tokenizer/utf8_validator_test.cpp
new file mode 100644
index 00000000000..5956cc81b3c
--- /dev/null
+++ b/be/test/storage/index/inverted/analysis/tokenizer/utf8_validator_test.cpp
@@ -0,0 +1,146 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "storage/index/inverted/analysis/tokenizer/utf8_validator.h"
+
+#include <CLucene.h> // IWYU pragma: keep
+#include <gtest/gtest.h>
+
+#include <cstdint>
+#include <random>
+#include <string>
+
+#include "CLucene/util/stringUtil.h"
+#include "util/utf8_check.h"
+
+namespace doris::segment_v2::inverted_index {
+namespace {
+
+bool clucene_valid(const std::string& text) {
+ return StringUtil::validate_utf8(text) == 0;
+}
+
+std::string bytes(std::initializer_list<int> values) {
+ std::string text;
+ for (int value : values) {
+ text.push_back(static_cast<char>(value));
+ }
+ return text;
+}
+
+TEST(Utf8ValidatorTest, AgreesWithCLuceneOnEveryValueOfUpToThreeBytes) {
+ EXPECT_TRUE(lenient_utf8_valid(""));
+ std::string text(3, '\0');
+ for (int a = 0; a < 256; ++a) {
+ text.resize(1);
+ text[0] = static_cast<char>(a);
+ ASSERT_EQ(lenient_utf8_valid(text), clucene_valid(text)) << a;
+ for (int b = 0; b < 256; ++b) {
+ text.resize(2);
+ text[1] = static_cast<char>(b);
+ ASSERT_EQ(lenient_utf8_valid(text), clucene_valid(text)) << a << "
" << b;
+ text.resize(3);
+ for (int c = 0; c < 256; ++c) {
+ text[2] = static_cast<char>(c);
+ ASSERT_EQ(lenient_utf8_valid(text), clucene_valid(text))
+ << a << " " << b << " " << c;
+ }
+ }
+ }
+}
+
+TEST(Utf8ValidatorTest, AgreesWithCLuceneOnRandomBytes) {
+ std::mt19937_64 random(20261104);
+ // Mostly bytes that make or break sequences, so that long values are not
all invalid.
+ const auto next_byte = [&random]() -> char {
+ switch (random() % 4) {
+ case 0:
+ return static_cast<char>(random() % 0x80);
+ case 1:
+ return static_cast<char>(0x80 + random() % 0x40);
+ case 2:
+ return static_cast<char>(0xC0 + random() % 0x38);
+ default:
+ return static_cast<char>(random());
+ }
+ };
+ for (int round = 0; round < 200000; ++round) {
+ std::string text(random() % 24, '\0');
+ for (char& c : text) {
+ c = next_byte();
+ }
+ ASSERT_EQ(lenient_utf8_valid(text), clucene_valid(text)) << round;
+ }
+}
+
+TEST(Utf8ValidatorTest,
AcceptsOverlongFormsAndRefusesSurrogatesAndBeyondTheLastPlane) {
+ EXPECT_TRUE(lenient_utf8_valid(bytes({0xC0, 0x80})));
+ EXPECT_TRUE(lenient_utf8_valid(bytes({0xC1, 0x81})));
+ EXPECT_TRUE(lenient_utf8_valid(bytes({0xE0, 0x80, 0x80})));
+ EXPECT_TRUE(lenient_utf8_valid(bytes({0xF0, 0x80, 0x80, 0x80})));
+ EXPECT_FALSE(lenient_utf8_valid(bytes({0xED, 0xA0, 0x80}))); //
U+D800
+ EXPECT_FALSE(lenient_utf8_valid(bytes({0xF4, 0x90, 0x80, 0x80}))); //
U+110000
+ EXPECT_FALSE(lenient_utf8_valid(bytes({0xE4, 0xB8}))); // cut
short
+ EXPECT_FALSE(lenient_utf8_valid(bytes({0x80})));
+ EXPECT_FALSE(lenient_utf8_valid(bytes({0xF8, 0x80, 0x80, 0x80, 0x80})));
+ // Strict UTF-8 refuses the overlong forms, so the tokenizers ask the
lenient rule after it.
+ const std::string overlong = bytes({0xC1, 0x81});
+ EXPECT_FALSE(doris::validate_utf8(overlong.data(), overlong.size()));
+}
+
+// Every sequence the rule accepts decodes to what CLucene decodes it to,
overlong forms included.
+TEST(Utf8ValidatorTest, DecodesEveryAcceptedSequenceAsCLucene) {
+ size_t checked = 0;
+ const auto check = [&checked](const std::string& sequence) {
+ if (!lenient_utf8_valid(sequence)) {
+ return;
+ }
+ const std::wstring expected = StringUtil::string_to_wstring(sequence);
+ ASSERT_EQ(expected.size(), 1);
+ size_t length = 0;
+ const uint32_t code_point =
+ decode_lenient_utf8(reinterpret_cast<const
uint8_t*>(sequence.data()), &length);
+ ASSERT_EQ(length, sequence.size());
+ ASSERT_EQ(code_point, static_cast<uint32_t>(expected[0]));
+ ++checked;
+ };
+ for (int lead = 0xC0; lead <= 0xDF; ++lead) {
+ for (int b = 0x80; b <= 0xBF; ++b) {
+ check(bytes({lead, b}));
+ }
+ }
+ for (int lead = 0xE0; lead <= 0xEF; ++lead) {
+ for (int b = 0x80; b <= 0xBF; ++b) {
+ for (int c = 0x80; c <= 0xBF; ++c) {
+ check(bytes({lead, b, c}));
+ }
+ }
+ }
+ for (int lead = 0xF0; lead <= 0xF4; ++lead) {
+ for (int b = 0x80; b <= 0xBF; ++b) {
+ for (int c = 0x80; c <= 0xBF; c += 7) {
+ for (int d = 0x80; d <= 0xBF; d += 5) {
+ check(bytes({lead, b, c, d}));
+ }
+ }
+ }
+ }
+ EXPECT_GT(checked, 60000);
+}
+
+} // namespace
+} // namespace doris::segment_v2::inverted_index
diff --git a/be/test/storage/index/inverted/clucene/lucene_adapter_test.cpp
b/be/test/storage/index/inverted/clucene/lucene_adapter_test.cpp
index 6b9ee76a3be..d993f362e45 100644
--- a/be/test/storage/index/inverted/clucene/lucene_adapter_test.cpp
+++ b/be/test/storage/index/inverted/clucene/lucene_adapter_test.cpp
@@ -32,13 +32,12 @@
#include <utility>
#include <vector>
+#include "CLucene/analysis/standard95/StandardAnalyzer.h"
#include "common/exception.h"
#include "common/status.h"
#include "io/fs/local_file_system.h"
#include "storage/index/index_file_writer.h"
#include "storage/index/index_writer.h"
-#include "storage/index/inverted/analysis/analyzer/analyzer.h"
-#include "storage/index/inverted/analysis/inverted_index_parser.h"
#include "storage/index/inverted/analysis/scratch.h"
#include "storage/index/inverted/analysis/text_stream_contract.h"
#include "storage/index/inverted/analysis/token_batch.h"
@@ -299,11 +298,11 @@ TEST(LuceneBackedTextAnalyzerTest,
KeepsTheTextStreamContract) {
const LuceneBackedTextAnalyzer
scripted(std::make_shared<ScriptedLuceneAnalyzer>());
expect_text_stream_contract([&] { return scripted.create_stream(); },
{"a ^b _ c", words(1000), words(256), "x"});
- // The builtin standard parser is CLucene's StandardAnalyzer behind a
LuceneBackedTextAnalyzer.
- const TextAnalyzerPtr standard =
InvertedIndexAnalyzer::create_builtin_analyzer(
- InvertedIndexParserType::PARSER_STANDARD, "", "", "");
- ASSERT_NE(dynamic_cast<const LuceneBackedTextAnalyzer*>(standard.get()),
nullptr);
- expect_text_stream_contract([&] { return standard->create_stream(); },
+ // A real CLucene analyzer behind the adapter: the standard one, which the
builtin standard
+ // parser used before it had a tokenizer of its own.
+ const LuceneBackedTextAnalyzer standard(
+
std::make_shared<lucene::analysis::standard95::StandardAnalyzer>());
+ expect_text_stream_contract([&] { return standard.create_stream(); },
{"The Quick brown fox", words(1000), "中文 mixed
文本", "x"});
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]