airborne12 commented on code in PR #67918:
URL: https://github.com/apache/doris/pull/67918#discussion_r4000586555
##########
fe/fe-core/src/main/java/org/apache/doris/analysis/invertedindex/AnalyzerIdentityBuilder.java:
##########
@@ -152,35 +153,29 @@ private static String resolveComponentIdentity(String
name, IndexPolicyTypeEnum
return "";
}
- // Check if it's a built-in component
- if (expectedType == IndexPolicyTypeEnum.TOKENIZER
- && IndexPolicy.BUILTIN_TOKENIZERS.contains(name)) {
- return name;
- }
-
- // For custom component, get its properties
+ // Existing named policies take precedence over built-ins for upgrade
compatibility.
try {
Env env = Env.getCurrentEnv();
- if (env == null || env.getIndexPolicyMgr() == null) {
- return name;
- }
-
- IndexPolicy policy = env.getIndexPolicyMgr().getPolicyByName(name);
- if (policy == null || policy.getType() != expectedType) {
- return name;
- }
-
- Map<String, String> props = policy.getProperties();
- if (props == null || props.isEmpty()) {
- return name;
+ if (env != null && env.getIndexPolicyMgr() != null) {
+ IndexPolicy policy =
env.getIndexPolicyMgr().getPolicyByName(name);
+ if (policy != null && policy.getType() == expectedType) {
+ Map<String, String> props = policy.getProperties();
+ if (props != null && !props.isEmpty()) {
+ TreeMap<String, String> sortedProps = new
TreeMap<>(props);
+ return sortedProps.toString();
+ }
+ }
}
-
- // Build identity from sorted properties
- TreeMap<String, String> sortedProps = new TreeMap<>(props);
- return sortedProps.toString();
} catch (RuntimeException e) {
- return name;
+ // Fall through to built-in resolution or the original name.
+ }
+
+ String normalizedName = name.toLowerCase(Locale.ROOT);
Review Comment:
Fixed in c5976f7cff0. AnalyzerIdentityBuilder now canonicalizes built-in
tokenizer, token-filter, and char-filter names through one Locale.ROOT helper.
The identity test covers PINYIN -> pinyin and ICU_NORMALIZER -> icu_normalizer;
the combined FE test run passes all 29 tests.
##########
be/src/storage/index/inverted/analyzer/ik/IKTokenizer.cpp:
##########
@@ -31,27 +66,55 @@ Token* IKTokenizer::next(Token* token) {
return nullptr;
}
- std::string& token_text = tokens_text_[buffer_index_++];
+ TokenData& token_data = tokens_[buffer_index_++];
// full-width to half-width, and lowercase
// TODO(ryan19929): do regularizeString in fillBuffer.
- CharacterUtil::regularizeString(token_text, this->lowercase);
- size_t size = std::min(token_text.size(),
static_cast<size_t>(LUCENE_MAX_WORD_LEN));
- token->setNoCopy(token_text.data(), 0, static_cast<int32_t>(size));
+ CharacterUtil::regularizeString(token_data.text, this->lowercase);
+ current_token_ = &token_data;
+ size_t size = std::min(token_data.text.size(),
static_cast<size_t>(LUCENE_MAX_WORD_LEN));
+ set(token, std::string_view(token_data.text.data(), size));
+ token->setStartOffset(token_data.start_offset);
+ token->setEndOffset(token_data.end_offset);
return token;
}
+std::span<const int32_t> IKTokenizer::get_source_byte_offsets(std::string_view
term) const {
+ if (current_token_ == nullptr || term != current_token_->text) {
+ return {};
+ }
+ return current_token_->source_byte_offsets;
+}
+
+void IKTokenizer::reset() {
+ if (_in_pending == nullptr) {
+ return;
+ }
+ inverted_index::DorisTokenizer::reset();
+ _in_pending.reset();
+ reset(_in.get());
+}
+
void IKTokenizer::reset(lucene::util::Reader* reader) {
+ _in_pending.reset();
this->input = reader;
this->buffer_index_ = 0;
this->data_length_ = 0;
- this->tokens_text_.clear();
+ this->tokens_.clear();
+ this->current_token_ = nullptr;
try {
buffer_.reserve(input->size());
ik_segmenter_->reset(reader);
Lexeme lexeme;
while (ik_segmenter_->next(lexeme)) {
- tokens_text_.emplace_back(lexeme.getText());
+ TokenData token_data {
+ .text = lexeme.getText(),
+ .start_offset =
static_cast<int32_t>(lexeme.getByteBeginPosition()),
+ .end_offset =
static_cast<int32_t>(lexeme.getByteEndPosition()),
+ .source_byte_offsets = {}};
+ token_data.source_byte_offsets =
Review Comment:
Fixed in 441846bb324c2772c5382845cd6bc3df2c6c524a. PinyinFilterFactory now
opts the upstream chain into provenance only when ignore_pinyin_offset=false.
IK fuses normalization and source-boundary collection in one pass only when
enabled, retains one current-token vector, and no longer stores a vector in
every eagerly buffered token. TestIKSourceOffsetsAreOptIn verifies the ordinary
IK path has no map and the consuming chain enables it. The final affected ASAN
run passed all 113 tests.
##########
fe/fe-core/src/main/java/org/apache/doris/analysis/invertedindex/AnalyzerIdentityBuilder.java:
##########
@@ -152,35 +182,49 @@ private static String resolveComponentIdentity(String
name, IndexPolicyTypeEnum
return "";
}
- // Check if it's a built-in component
- if (expectedType == IndexPolicyTypeEnum.TOKENIZER
- && IndexPolicy.BUILTIN_TOKENIZERS.contains(name)) {
- return name;
- }
-
- // For custom component, get its properties
+ // Existing named policies take precedence over built-ins for upgrade
compatibility.
try {
Env env = Env.getCurrentEnv();
- if (env == null || env.getIndexPolicyMgr() == null) {
- return name;
- }
-
- IndexPolicy policy = env.getIndexPolicyMgr().getPolicyByName(name);
- if (policy == null || policy.getType() != expectedType) {
- return name;
+ if (env != null && env.getIndexPolicyMgr() != null) {
+ IndexPolicy policy =
env.getIndexPolicyMgr().getPolicyByName(name);
+ if (policy != null && policy.getType() == expectedType) {
+ Map<String, String> props = policy.getProperties();
+ if (props != null && !props.isEmpty()) {
+ TreeMap<String, String> sortedProps = new
TreeMap<>(props);
+ String type = sortedProps.get(IndexPolicy.PROP_TYPE);
+ String normalizedType =
normalizeBuiltinComponentName(type, expectedType);
+ if (normalizedType != null) {
+ if (sortedProps.size() == 1) {
+ return normalizedType;
+ }
+ sortedProps.put(IndexPolicy.PROP_TYPE,
normalizedType);
+ }
+ return sortedProps.toString();
+ }
+ }
}
+ } catch (RuntimeException e) {
+ // Fall through to built-in resolution or the original name.
+ }
- Map<String, String> props = policy.getProperties();
- if (props == null || props.isEmpty()) {
- return name;
- }
+ String normalizedName = normalizeBuiltinComponentName(name,
expectedType);
+ return normalizedName == null ? name : normalizedName;
+ }
- // Build identity from sorted properties
- TreeMap<String, String> sortedProps = new TreeMap<>(props);
- return sortedProps.toString();
- } catch (RuntimeException e) {
- return name;
+ private static String normalizeBuiltinComponentName(String name,
IndexPolicyTypeEnum expectedType) {
+ if (Strings.isNullOrEmpty(name)) {
+ return null;
+ }
+ String normalizedName = name.toLowerCase(Locale.ROOT);
Review Comment:
Fixed in 74488270e32. Built-in component normalization now trims the
reference before Locale.ROOT lowercasing. The focused unit test covers a padded
IK_SMART reference, and the regression suite verifies both CREATE and ALTER
duplicate rejection against the equivalent unpadded analyzer.
##########
fe/fe-core/src/main/java/org/apache/doris/analysis/invertedindex/AnalyzerIdentityBuilder.java:
##########
@@ -152,35 +153,49 @@ private static String resolveComponentIdentity(String
name, IndexPolicyTypeEnum
return "";
}
- // Check if it's a built-in component
- if (expectedType == IndexPolicyTypeEnum.TOKENIZER
- && IndexPolicy.BUILTIN_TOKENIZERS.contains(name)) {
- return name;
- }
-
- // For custom component, get its properties
+ // Existing named policies take precedence over built-ins for upgrade
compatibility.
try {
Env env = Env.getCurrentEnv();
- if (env == null || env.getIndexPolicyMgr() == null) {
- return name;
- }
-
- IndexPolicy policy = env.getIndexPolicyMgr().getPolicyByName(name);
- if (policy == null || policy.getType() != expectedType) {
- return name;
+ if (env != null && env.getIndexPolicyMgr() != null) {
+ IndexPolicy policy =
env.getIndexPolicyMgr().getPolicyByName(name);
+ if (policy != null && policy.getType() == expectedType) {
+ Map<String, String> props = policy.getProperties();
+ if (props != null && !props.isEmpty()) {
+ TreeMap<String, String> sortedProps = new
TreeMap<>(props);
+ String type = sortedProps.get(IndexPolicy.PROP_TYPE);
+ String normalizedType =
normalizeBuiltinComponentName(type, expectedType);
+ if (normalizedType != null) {
+ if (sortedProps.size() == 1) {
+ return normalizedType;
+ }
+ sortedProps.put(IndexPolicy.PROP_TYPE,
normalizedType);
+ }
+ return sortedProps.toString();
+ }
+ }
}
+ } catch (RuntimeException e) {
+ // Fall through to built-in resolution or the original name.
+ }
- Map<String, String> props = policy.getProperties();
- if (props == null || props.isEmpty()) {
- return name;
- }
+ String normalizedName = normalizeBuiltinComponentName(name,
expectedType);
+ return normalizedName == null ? name : normalizedName;
Review Comment:
Fixed in 0e66fd9522552abd9efa2bb685db6970859b19d1. Unfiltered legacy IK now
resolves smart (including the omitted-mode default) and max-word modes to the
same semantic identities as custom analyzers using ik_smart and ik_max_word. A
focused FE unit test covers both modes plus the filtered-legacy distinction
(9/9 passed), and the isolated regression suite now verifies duplicate
rejection in both CREATE TABLE and ALTER TABLE paths; the full suite passed.
##########
be/src/storage/index/inverted/token_filter/token_filter.h:
##########
@@ -28,9 +28,15 @@ class DorisTokenFilter : public TokenFilter, public
DorisTokenStream {
void reset() override { _in->reset(); }
+ std::span<const int32_t> get_source_byte_offsets(std::string_view term)
const override {
Review Comment:
Fixed in 441846bb324c2772c5382845cd6bc3df2c6c524a. Source-byte provenance is
now attached to the current token, and WordDelimiterFilter slices or composes
it for generated parts and concatenations before PinyinFilter consumes it.
TestWordDelimiterPreservesIKSourceOffsets covers full-width LIUDE123 through
ik_smart, word_delimiter, and pinyin and verifies liu [0,9), de [9,15), and 123
[15,24). The final affected ASAN run passed all 113 tests.
##########
fe/fe-core/src/main/java/org/apache/doris/indexpolicy/IndexPolicyMgr.java:
##########
@@ -373,6 +374,10 @@ private void validateTokenizerProperties(Map<String,
String> properties) throws
case "basic":
validator = new BasicTokenizerValidator();
Review Comment:
Fixed in c5976f7cff0. AnalyzerIdentityBuilder canonicalizes a named policy's
built-in type and collapses a type-only wrapper to the direct factory identity,
while retaining named-policy precedence for migration compatibility. The new
test verifies named {type=ik_smart} equals direct ik_smart. Both CREATE and
ALTER duplicate-index checks consume this shared identity builder.
##########
be/src/storage/index/inverted/tokenizer/ik/ik_tokenizer_factory.h:
##########
@@ -0,0 +1,49 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "common/config.h"
+#include "storage/index/inverted/analyzer/ik/IKTokenizer.h"
+#include "storage/index/inverted/analyzer/ik/dic/Dictionary.h"
+#include "storage/index/inverted/tokenizer/tokenizer_factory.h"
+
+namespace doris::segment_v2::inverted_index {
+
+class IKTokenizerFactory : public TokenizerFactory {
+public:
+ explicit IKTokenizerFactory(bool use_smart) : _use_smart(use_smart) {}
+ ~IKTokenizerFactory() override = default;
+
+ void initialize(const Settings& settings) override {}
+
+ TokenizerPtr create() override {
+ auto ik_config = std::make_shared<Configuration>(_use_smart, true);
+ ik_config->setDictPath(config::inverted_index_dict_path + "/ik");
+ Dictionary::initial(*ik_config);
Review Comment:
Fixed in 441846bb324c2772c5382845cd6bc3df2c6c524a. CustomAnalyzer now
translates CLuceneError while creating the lazy tokenizer/filter chain into
INVERTED_INDEX_ANALYZER_ERROR. IKTokenizerTest.TestDictionaryExceptionHandling
exercises a named ik_smart analyzer on its first use with a missing dictionary
and verifies that a Doris analyzer exception, rather than raw CLuceneError,
crosses the public token-stream boundary. The final affected ASAN run passed
all 113 tests.
##########
be/src/storage/index/inverted/analyzer/ik/IKTokenizer.cpp:
##########
@@ -31,27 +68,57 @@ Token* IKTokenizer::next(Token* token) {
return nullptr;
}
- std::string& token_text = tokens_text_[buffer_index_++];
+ TokenData& token_data = tokens_[buffer_index_++];
// full-width to half-width, and lowercase
// TODO(ryan19929): do regularizeString in fillBuffer.
- CharacterUtil::regularizeString(token_text, this->lowercase);
- size_t size = std::min(token_text.size(),
static_cast<size_t>(LUCENE_MAX_WORD_LEN));
- token->setNoCopy(token_text.data(), 0, static_cast<int32_t>(size));
+ if (source_byte_offsets_enabled_) {
+ current_source_byte_offsets_ =
+ regularize_with_source_byte_offsets(token_data.text,
this->lowercase);
+ } else {
+ CharacterUtil::regularizeString(token_data.text, this->lowercase);
+ current_source_byte_offsets_.clear();
+ }
+ current_token_ = &token_data;
+ size_t size = std::min(token_data.text.size(),
static_cast<size_t>(LUCENE_MAX_WORD_LEN));
Review Comment:
Fixed in 12d0c12afd44719995027609346fc73f3ce83590. A discriminating ASAN
test using repeated full-width LIUDE reproduced the fallback first-token range
[0,3) instead of the source range [0,9). IKTokenizer now chooses a UTF-8-safe
published prefix at or below the Lucene byte limit, truncates provenance to the
same rune boundary, and sets the clipped token end offset to that exclusive
source boundary. The test also resets and reuses the tokenizer. The final
affected ASAN run passed all 82 tests, the full ASAN BE build passed, and
changed-line clang-tidy passed.
##########
fe/fe-core/src/main/java/org/apache/doris/analysis/invertedindex/AnalyzerIdentityBuilder.java:
##########
@@ -50,9 +52,32 @@ public static String buildAnalyzerIdentity(
if (Strings.isNullOrEmpty(parser) ||
parserNone.equalsIgnoreCase(parser)) {
return defaultAnalyzerKey;
}
+ String legacyIkIdentity = resolveLegacyIkIdentity(properties, parser);
+ if (legacyIkIdentity != null) {
+ return legacyIkIdentity;
+ }
return parser;
}
+ private static String resolveLegacyIkIdentity(Map<String, String>
properties, String parser) {
+ if
(!InvertedIndexProperties.INVERTED_INDEX_PARSER_IK.equalsIgnoreCase(parser)
+ || !Strings.isNullOrEmpty(properties.get(
+
InvertedIndexProperties.INVERTED_INDEX_PARSER_CHAR_FILTER_TYPE))) {
+ return null;
+ }
+
+ String mode =
properties.get(InvertedIndexProperties.INVERTED_INDEX_PARSER_MODE_KEY);
+ if (Strings.isNullOrEmpty(mode)) {
+ mode = InvertedIndexProperties.INVERTED_INDEX_PARSER_SMART;
+ }
+ String tokenizer = normalizeBuiltinComponentName(mode,
IndexPolicyTypeEnum.TOKENIZER);
+ if (!"ik_smart".equals(tokenizer) && !"ik_max_word".equals(tokenizer))
{
+ return null;
+ }
+ return buildIdentityFromPolicyProperties(
Review Comment:
Fixed in 441846bb324c2772c5382845cd6bc3df2c6c524a. Legacy IK collapses to a
custom identity only when lower_case is absent or true, and its synthetic
identity now names the literal built-in mode instead of resolving a shadowing
policy. AnalyzerIdentityBuilderTest covers lower_case=false and a replayed
ik_smart policy whose type is standard; the focused FE run passed all 10 tests.
##########
be/src/storage/index/inverted/token_filter/pinyin_filter.cpp:
##########
@@ -421,8 +434,18 @@ void PinyinFilter::setTokenAttributes(Token* token, const
std::string& term, int
int end_offset, int position) {
set_text(token, term);
- token->setStartOffset(start_offset);
- token->setEndOffset(end_offset);
+ int absolute_start = current_start_offset_;
+ int absolute_end = current_end_offset_;
+ const bool is_whole_token =
+ start_offset == 0 && std::cmp_equal(end_offset,
current_source_.length());
+ if (!config_->ignorePinyinOffset && !is_whole_token && start_offset >= 0
&& end_offset > 0 &&
+ std::cmp_less(start_offset, current_runes_.size()) &&
+ std::cmp_less_equal(end_offset, current_runes_.size())) {
+ absolute_start += current_runes_[start_offset].byte_start;
Review Comment:
Fixed in 12d0c12afd44719995027609346fc73f3ce83590. A discriminating test
reproduced the reported connector gap in both ik_smart and ik_max_word:
full-width LIU-DE produced de [9,13) rather than [10,16). PinyinFilter now
retains the original source-rune index for every ASCII character kept in the
compact buffer and maps each alphabet-token span through those indices. The
final affected ASAN run passed all 82 tests, the full ASAN BE build passed, and
changed-line clang-tidy passed.
##########
fe/fe-core/src/main/java/org/apache/doris/indexpolicy/IndexPolicyMgr.java:
##########
@@ -80,6 +81,36 @@ private void readUnlock() {
lock.readLock().unlock();
}
+ // Legacy metadata may contain names that collide after locale-independent
normalization.
+ // Policy IDs are allocated monotonically, so the higher ID reproduces the
latest definition.
+ // Callers must hold the write lock.
+ private void registerPolicyNameLocked(IndexPolicy indexPolicy) {
+ String normalizedName = normalizeKey(indexPolicy.getName());
+ IndexPolicy current = nameToIndexPolicy.get(normalizedName);
+ if (current == null || indexPolicy.getId() > current.getId()) {
+ nameToIndexPolicy.put(normalizedName, indexPolicy);
Review Comment:
Fixed in 8995f270. FE now transmits only the authoritative normalized-name
winner. BE retains colliders but deterministically selects the highest policy
ID for both arrival orders; deleting the winner clears only that authoritative
entry and restores the highest survivor. Added FE winner-only/restoration
coverage and BE both-arrival, drop, and winner-restoration coverage. Verified
with the focused ASAN BE suite (53/53), FE tests (36/36), and the isolated
regression suite twice.
##########
fe/fe-core/src/main/java/org/apache/doris/indexpolicy/IndexPolicyMgr.java:
##########
@@ -61,7 +62,7 @@ public class IndexPolicyMgr implements Writable,
GsonPostProcessable {
* Policy names are case-insensitive in Doris.
*/
private static String normalizeKey(String name) {
- return name == null ? null : name.trim().toLowerCase();
+ return name == null ? null : name.trim().toLowerCase(Locale.ROOT);
Review Comment:
Fixed in 74488270e32. Policy-name registration and removal are now
centralized under the write lock. Legacy normalized-name collisions
deterministically select the highest policy ID, dropping a hidden older policy
preserves the current mapping, and dropping the current policy restores the
newest survivor. Replay tests cover both drop orders, and a serialized round
trip with colliding HashMap buckets verifies deterministic image reconstruction.
##########
fe/fe-core/src/main/java/org/apache/doris/analysis/invertedindex/AnalyzerIdentityBuilder.java:
##########
@@ -43,16 +45,64 @@ public static String buildAnalyzerIdentity(
}
if (!Strings.isNullOrEmpty(preferredAnalyzer)) {
+ String builtinIkIdentity =
resolveBuiltinIkAnalyzerIdentity(properties, preferredAnalyzer);
+ if (builtinIkIdentity != null) {
+ return builtinIkIdentity;
+ }
// For custom analyzer/normalizer, resolve to underlying config to
build identity
return resolveAnalyzerIdentity(preferredAnalyzer,
defaultAnalyzerKey, log);
Review Comment:
Fixed in 8995f270. Named and custom analyzer identities now include a
length-prefixed outer character-replacement type, pattern, and replacement,
including the legacy parser fallback. Regression coverage keeps named IK with a
replacement filter distinct from the matching legacy IK form through both
CREATE and ALTER. Verified with the focused ASAN BE suite (53/53), FE tests
(36/36), and the isolated regression suite twice.
##########
fe/fe-core/src/main/java/org/apache/doris/analysis/invertedindex/AnalyzerIdentityBuilder.java:
##########
@@ -50,9 +52,37 @@ public static String buildAnalyzerIdentity(
if (Strings.isNullOrEmpty(parser) ||
parserNone.equalsIgnoreCase(parser)) {
return defaultAnalyzerKey;
}
+ String legacyIkIdentity = resolveLegacyIkIdentity(properties, parser);
Review Comment:
Fixed in 74488270e32. The identity builder now matches the BE runtime
default: analyzer=ik uses max-word mode. It shares the synthetic ik_max_word
identity only when no character filter is configured and lower_case is absent
or true; negative tests keep character-filter and lower_case=false
configurations distinct. CREATE and ALTER regression cases cover the equivalent
custom analyzer.
##########
fe/fe-core/src/main/java/org/apache/doris/analysis/invertedindex/AnalyzerIdentityBuilder.java:
##########
@@ -230,11 +290,7 @@ private static String resolveCharFilterIdentity(String
filterList) {
sb.append(",");
}
- if (IndexPolicy.BUILTIN_CHAR_FILTERS.contains(filter)) {
- sb.append(filter);
- } else {
- sb.append(resolveComponentIdentity(filter,
IndexPolicyTypeEnum.CHAR_FILTER));
- }
+ sb.append(resolveComponentIdentity(filter,
IndexPolicyTypeEnum.CHAR_FILTER));
Review Comment:
Fixed in 8995f270. AnalyzerIdentityBuilder now omits built-in empty filters
and named filters whose normalized type is empty before writing identity keys
or delimiters. The unit tests and the custom-analyzer regression suite cover
duplicate rejection through both CREATE and ALTER. Verified with the focused
ASAN BE suite (53/53), FE tests (36/36), and the isolated regression suite
twice.
##########
be/src/storage/index/inverted/analyzer/ik/IKTokenizer.cpp:
##########
@@ -31,27 +88,70 @@ Token* IKTokenizer::next(Token* token) {
return nullptr;
}
- std::string& token_text = tokens_text_[buffer_index_++];
+ TokenData& token_data = tokens_[buffer_index_++];
// full-width to half-width, and lowercase
// TODO(ryan19929): do regularizeString in fillBuffer.
- CharacterUtil::regularizeString(token_text, this->lowercase);
- size_t size = std::min(token_text.size(),
static_cast<size_t>(LUCENE_MAX_WORD_LEN));
- token->setNoCopy(token_text.data(), 0, static_cast<int32_t>(size));
+ if (source_byte_offsets_enabled_) {
Review Comment:
Fixed in 8995f270. DorisCharFilter now exposes compositional source-offset
correction; the ICU normalizer maps normalized byte boundaries back to its
source and IK applies that nested correction to propagated rune offsets. The
Pinyin test covers the actual ICU nfkc_cf to IK smart to Pinyin chain with
full-width LIUDE input and reset/reuse, preserving original spans [0,9) and
[9,15). Verified with the focused ASAN BE suite (53/53) and the isolated
regression suite twice.
##########
be/src/storage/index/inverted/token_filter/pinyin_filter.cpp:
##########
@@ -421,8 +423,18 @@ void PinyinFilter::setTokenAttributes(Token* token, const
std::string& term, int
int end_offset, int position) {
set_text(token, term);
- token->setStartOffset(start_offset);
- token->setEndOffset(end_offset);
+ int absolute_start = current_start_offset_;
+ int absolute_end = current_end_offset_;
+ const bool is_whole_token =
+ start_offset == 0 && std::cmp_equal(end_offset,
current_source_.length());
Review Comment:
Fixed in c5976f7cff0. IKTokenizer records original UTF-8 rune boundaries
before regularization when normalization changes byte width, and Doris token
filters forward that mapping to PinyinFilter.
TestIKOffsetsPreserveFullwidthSourceBytes verifies that fullwidth ＬＩＵＤＥ splits
to liu [0,9) and de [9,15); the final IKTokenizerTest plus PinyinFilterTest
ASAN run passes all 62 tests.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
