airborne12 commented on code in PR #67918:
URL: https://github.com/apache/doris/pull/67918#discussion_r4000586536
##########
fe/fe-core/src/main/java/org/apache/doris/indexpolicy/IndexPolicy.java:
##########
@@ -59,7 +59,8 @@ public class IndexPolicy implements Writable,
GsonPostProcessable {
public static final String PROP_TOKEN_FILTER = "token_filter";
public static final String PROP_CHAR_FILTER = "char_filter";
public static final Set<String> BUILTIN_TOKENIZERS = ImmutableSet.of(
- "empty", "ngram", "edge_ngram", "keyword", "standard",
"char_group", "basic", "icu", "pinyin");
+ "empty", "ngram", "edge_ngram", "keyword", "standard",
"char_group", "basic", "icu", "pinyin",
+ "ik_smart", "ik_max_word");
Review Comment:
Fixed in c5976f7cff0. IndexPolicyMgr.normalizeKey() and
AnalyzerKeyNormalizer now use Locale.ROOT. PolicyValidatorTests verifies
uppercase IK_SMART validation while the JVM default locale is Turkish; the
combined AnalyzerIdentityBuilderTest and PolicyValidatorTests run passes all 29
tests.
##########
be/test/runtime/index_policy/index_policy_mgr_test.cpp:
##########
@@ -185,6 +185,57 @@ TEST_F(IndexPolicyMgrTest, TestTokenFilterProcessing) {
ASSERT_NE(emptyAnalyzer, nullptr);
}
+TEST_F(IndexPolicyMgrTest, BuiltinTokenizerNamesAreCaseInsensitive) {
+ const char* doris_home = std::getenv("DORIS_HOME");
+ ASSERT_NE(doris_home, nullptr);
+ config::inverted_index_dict_path = std::string(doris_home) + "../../dict";
+
+ TIndexPolicy analyzer;
+ analyzer.id = 20;
+ analyzer.name = "uppercase_ik_analyzer";
+ analyzer.type = TIndexPolicyType::ANALYZER;
+ analyzer.properties["tokenizer"] = "IK_SMART";
+ mgr.apply_policy_changes({analyzer}, {});
+
+ auto built = mgr.get_policy_by_name(analyzer.name);
+ ASSERT_NE(built, nullptr);
+ auto reader =
segment_v2::inverted_index::InvertedIndexAnalyzer::create_reader({});
+ const std::string text = "我来到北京";
+ reader->init(text.data(), static_cast<int32_t>(text.size()), false);
+ auto terms =
segment_v2::inverted_index::InvertedIndexAnalyzer::get_analyse_result(reader,
Review Comment:
Fixed in c5976f7cff0. BuiltinTokenizerNamesAreCaseInsensitive now stops
after analyzer construction, which still proves uppercase factory resolution
without calling IKTokenizerFactory::create() or mutating the process-wide
dictionary configuration. The combined ASAN run of AnalysisFactoryMgrTest,
IKTokenizerTest, PinyinFilterTest, and IndexPolicyMgrTest now passes all 82
tests, including TestDictionaryExceptionHandling after this test.
##########
be/test/runtime/index_policy/index_policy_mgr_test.cpp:
##########
@@ -185,6 +185,57 @@ TEST_F(IndexPolicyMgrTest, TestTokenFilterProcessing) {
ASSERT_NE(emptyAnalyzer, nullptr);
}
+TEST_F(IndexPolicyMgrTest, BuiltinTokenizerNamesAreCaseInsensitive) {
+ const char* doris_home = std::getenv("DORIS_HOME");
+ ASSERT_NE(doris_home, nullptr);
+ config::inverted_index_dict_path = std::string(doris_home) + "../../dict";
+
+ TIndexPolicy analyzer;
+ analyzer.id = 20;
+ analyzer.name = "uppercase_ik_analyzer";
+ analyzer.type = TIndexPolicyType::ANALYZER;
+ analyzer.properties["tokenizer"] = "IK_SMART";
+ mgr.apply_policy_changes({analyzer}, {});
+
+ auto built = mgr.get_policy_by_name(analyzer.name);
+ ASSERT_NE(built, nullptr);
+ auto reader =
segment_v2::inverted_index::InvertedIndexAnalyzer::create_reader({});
+ const std::string text = "我来到北京";
+ reader->init(text.data(), static_cast<int32_t>(text.size()), false);
+ auto terms =
segment_v2::inverted_index::InvertedIndexAnalyzer::get_analyse_result(reader,
Review Comment:
Fixed in c5976f7cff0. BuiltinTokenizerNamesAreCaseInsensitive now stops
after analyzer construction, which still proves uppercase factory resolution
without calling IKTokenizerFactory::create() or mutating the process-wide
dictionary configuration. The combined ASAN run of AnalysisFactoryMgrTest,
IKTokenizerTest, PinyinFilterTest, and IndexPolicyMgrTest now passes all 82
tests, including TestDictionaryExceptionHandling after this test.
##########
fe/fe-core/src/main/java/org/apache/doris/indexpolicy/IndexPolicy.java:
##########
@@ -59,7 +59,8 @@ public class IndexPolicy implements Writable,
GsonPostProcessable {
public static final String PROP_TOKEN_FILTER = "token_filter";
public static final String PROP_CHAR_FILTER = "char_filter";
public static final Set<String> BUILTIN_TOKENIZERS = ImmutableSet.of(
- "empty", "ngram", "edge_ngram", "keyword", "standard",
"char_group", "basic", "icu", "pinyin");
+ "empty", "ngram", "edge_ngram", "keyword", "standard",
"char_group", "basic", "icu", "pinyin",
+ "ik_smart", "ik_max_word");
Review Comment:
Fixed in c5976f7cff0. IndexPolicyMgr.normalizeKey() and
AnalyzerKeyNormalizer now use Locale.ROOT. PolicyValidatorTests verifies
uppercase IK_SMART validation while the JVM default locale is Turkish; the
combined AnalyzerIdentityBuilderTest and PolicyValidatorTests run passes all 29
tests.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
