github-actions[bot] commented on code in PR #70262:
URL: https://github.com/apache/doris/pull/70262#discussion_r4545465996
##########
be/src/storage/index/inverted/analysis/analyzer/analyzer.cpp:
##########
@@ -43,25 +43,110 @@
#include "storage/index/inverted/analysis/analyzer/kuromoji/KuromojiAnalyzer.h"
#include
"storage/index/inverted/analysis/analyzer/kuromoji/kuromoji_normalize.h"
#include
"storage/index/inverted/analysis/char_filter/char_replace_char_filter_factory.h"
+#include "storage/index/inverted/analysis/gram/gram_scheme.h"
+#include "storage/index/inverted/analysis/scratch.h"
#include "storage/index/inverted/analysis/token_filter/lower_case_filter.h"
+#include "storage/index/inverted/clucene/lucene_backed_text_analyzer.h"
#include "util/string_util.h"
namespace doris::segment_v2::inverted_index {
namespace {
+// The parser of a configuration whose analyzer is builtin, named by the
parser or by the analyzer.
+InvertedIndexParserType builtin_parser_type(const InvertedIndexAnalyzerConfig&
config) {
+ return config.analyzer_name.empty()
+ ? config.parser_type
+ :
get_inverted_index_parser_type_from_string(config.analyzer_name);
+}
+
+TextAnalyzerPtr builtin_text_analyzer(const InvertedIndexAnalyzerConfig&
config) {
+ return std::make_shared<LuceneBackedTextAnalyzer>(
+
InvertedIndexAnalyzer::create_builtin_analyzer(builtin_parser_type(config),
+ config.parser_mode,
config.lower_case,
+ config.stop_words));
+}
+
class BuiltinAnalyzerProvider final : public AnalyzerProvider {
public:
- explicit BuiltinAnalyzerProvider(InvertedIndexAnalyzerConfig config)
- : _analyzer(InvertedIndexAnalyzer::create_builtin_analyzer(
- config.analyzer_name.empty()
- ? config.parser_type
- :
get_inverted_index_parser_type_from_string(config.analyzer_name),
- config.parser_mode, config.lower_case,
config.stop_words)) {}
+ explicit BuiltinAnalyzerProvider(const InvertedIndexAnalyzerConfig& config)
+ : _analyzer(builtin_text_analyzer(config)) {}
- AnalyzerPtr get_analyzer() const override { return _analyzer; }
+ TextAnalyzerPtr get_analyzer() const override { return _analyzer; }
private:
- const AnalyzerPtr _analyzer;
+ const TextAnalyzerPtr _analyzer;
+};
+
+// The index's char filter in front of an analyzer. The writers and the query
side used to apply it
+// through the reader they gave the analyzer (create_reader); it now belongs
to the analyzer, so
+// every consumer reads every value through it, whatever the analyzer: a
normalizer index with
+// char_filter_pattern "-" writes "A-B" as "a b". It runs the CLucene char
filter over the whole
+// value until the char filters are native.
+class IndexCharFilterStream final : public TextTokenStream {
+public:
+ IndexCharFilterStream(const CharFilterMap* char_filters,
std::unique_ptr<TextTokenStream> in)
+ : _char_filters(char_filters), _in(std::move(in)) {}
+
+ void reset(std::string_view text) override {
+ // Assigning a result short enough for the string's inline buffer
keeps the buffer a long
+ // value grew, so release it first when it is above the high water
mark.
+ release_oversized_scratch(_filtered);
+ _filtered = InvertedIndexAnalyzer::filter_characters(*_char_filters,
text);
Review Comment:
[P2] Reuse character-filter storage across indexed values
For an index with `char_filter_type=char_replace`, every reset now calls
`filter_characters()`, which constructs fresh readers, copies the input with
`init(..., true)`, copies it into `CharReplaceCharFilter::_buf`, and copies
that into a newly allocated result string. Before this change, SNII and V1/V2
scalar writers retained one character-filter reader, initialized it with
`copyData=false`, and reused its output buffer. Repeated nontrivial values
therefore incur two extra full-value copies and fresh value-sized allocations
on every row despite reusing the stream. Keep the filter/output storage local
to the stream and reuse its capacity while retaining the high-water release
behavior. This also affects SNII and occurs before tokenization, independently
of the already-reported V1/V2 tokenizer-component reuse issue.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
