This is an automated email from the ASF dual-hosted git repository.
airborne12 pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
from d7c0fb9427b [fix](be) Preserve local time positions and fractions in
DST gaps (#70204)
add afd67a2d0fc [refactor](be) Move every analysis component onto the
CLucene-free stream interface (#70263)
No new revisions were added by this update.
Summary of changes:
be/src/exprs/function/match.cpp | 49 +-
be/src/runtime/index_policy/index_policy_mgr.cpp | 12 +-
.../inverted/analysis/abstract_analysis_factory.h | 7 +-
.../index/inverted/analysis/analyzer/analyzer.cpp | 261 ++---
.../index/inverted/analysis/analyzer/analyzer.h | 29 +-
.../analysis/analyzer/basic/basic_analyzer.h | 81 --
.../chain_analyzer.cpp} | 50 +-
.../chain_analyzer.h} | 26 +-
.../inverted/analysis/analyzer/custom_analyzer.cpp | 129 +--
.../inverted/analysis/analyzer/custom_analyzer.h | 66 +-
.../inverted/analysis/analyzer/icu/icu_analyzer.h | 84 --
.../inverted/analysis/analyzer/ik/IKAnalyzer.h | 72 --
.../inverted/analysis/analyzer/ik/IKTokenizer.cpp | 137 +--
.../inverted/analysis/analyzer/ik/IKTokenizer.h | 53 +-
.../analysis/analyzer/ik/core/AnalyzeContext.cpp | 30 +-
.../analysis/analyzer/ik/core/AnalyzeContext.h | 10 +-
.../analysis/analyzer/ik/core/CharacterUtil.cpp | 4 +-
.../analysis/analyzer/ik/core/CharacterUtil.h | 12 +-
.../analysis/analyzer/ik/core/IKSegmenter.cpp | 4 +-
.../analysis/analyzer/ik/core/IKSegmenter.h | 7 +-
.../analysis/analyzer/ik/core/ISegmenter.h | 1 -
.../inverted/analysis/analyzer/ik/core/Lexeme.h | 1 -
.../analysis/analyzer/ik/core/LexemePath.cpp | 2 +-
.../analysis/analyzer/ik/core/LexemePath.h | 1 -
.../analysis/analyzer/ik/core/QuickSortSet.h | 2 -
.../analysis/analyzer/ik/dic/DictSegment.h | 2 -
.../analysis/analyzer/ik/dic/Dictionary.cpp | 16 +-
.../inverted/analysis/analyzer/ik/dic/Dictionary.h | 11 +-
.../index/inverted/analysis/analyzer/ik/dic/Hit.h | 2 -
.../analysis/analyzer/kuromoji/KuromojiAnalyzer.h | 75 --
.../analyzer/kuromoji/KuromojiTokenizer.cpp | 61 +-
.../analysis/analyzer/kuromoji/KuromojiTokenizer.h | 22 +-
.../inverted/analysis/char_filter/char_filter.h | 51 +-
.../analysis/char_filter/char_filter_factory.h | 9 +-
.../analysis/char_filter/char_filtered_stream.cpp | 72 ++
.../analysis/char_filter/char_filtered_stream.h | 67 ++
.../char_filter/char_replace_char_filter.cpp | 57 +-
.../char_filter/char_replace_char_filter.h | 26 +-
.../char_filter/char_replace_char_filter_factory.h | 10 +-
.../char_filter/empty_char_filter_factory.h | 30 +-
.../char_filter/icu_normalizer_char_filter.cpp | 100 +-
.../char_filter/icu_normalizer_char_filter.h | 33 +-
.../icu_normalizer_char_filter_factory.h | 6 +-
.../storage/index/inverted/analysis/common/rune.h | 133 +++
.../index/inverted/analysis/common/text_cursor.h | 57 +
.../analysis/normalizer/custom_normalizer.cpp | 91 +-
.../analysis/normalizer/custom_normalizer.h | 54 +-
.../index/inverted/analysis/per_token_stream.cpp | 20 +-
.../index/inverted/analysis/per_token_stream.h | 14 +-
.../index/inverted/analysis/text_token_stream.h | 6 +-
.../analysis/token_filter/ascii_folding_filter.cpp | 121 +-
.../analysis/token_filter/ascii_folding_filter.h | 37 +-
.../token_filter/ascii_folding_filter_factory.h | 9 +-
.../token_filter/empty_token_filter_factory.h | 26 +-
.../token_filter/icu_normalizer_filter.cpp | 68 +-
.../analysis/token_filter/icu_normalizer_filter.h | 27 +-
.../token_filter/icu_normalizer_filter_factory.h | 8 +-
.../analysis/token_filter/lower_case_filter.h | 120 +-
.../token_filter/lower_case_filter_factory.h | 8 +-
.../analysis/token_filter/pinyin_filter.cpp | 299 +----
.../inverted/analysis/token_filter/pinyin_filter.h | 88 +-
.../token_filter/pinyin_filter_factory.cpp | 7 +-
.../analysis/token_filter/pinyin_filter_factory.h | 3 +-
.../inverted/analysis/token_filter/token_filter.h | 75 --
.../analysis/token_filter/token_filter_factory.h | 11 +-
.../token_filter/word_delimiter_filter.cpp | 219 +---
.../analysis/token_filter/word_delimiter_filter.h | 91 +-
.../token_filter/word_delimiter_filter_factory.h | 4 +-
.../storage/index/inverted/analysis/token_stream.h | 130 ---
.../analysis/tokenizer/basic/basic_tokenizer.cpp | 78 +-
.../analysis/tokenizer/basic/basic_tokenizer.h | 32 +-
.../tokenizer/basic/basic_tokenizer_factory.h | 8 +-
.../char/char_group_tokenizer_factory.cpp | 9 +-
.../tokenizer/char/char_group_tokenizer_factory.h | 10 +-
.../analysis/tokenizer/char/char_tokenizer.cpp | 29 +-
.../analysis/tokenizer/char/char_tokenizer.h | 22 +-
.../tokenizer/empty/empty_tokenizer_factory.h | 57 +-
.../tokenizer/icu/break_iterator_wrapper.cpp | 8 +-
.../tokenizer/icu/default_icu_tokenizer_config.cpp | 10 +-
.../inverted/analysis/tokenizer/icu/icu_common.h | 3 +-
.../analysis/tokenizer/icu/icu_tokenizer.cpp | 98 +-
.../analysis/tokenizer/icu/icu_tokenizer.h | 40 +-
.../analysis/tokenizer/icu/icu_tokenizer_factory.h | 5 +-
.../analysis/tokenizer/icu/script_iterator.cpp | 4 +-
.../analysis/tokenizer/ik/ik_tokenizer_factory.h | 24 +-
.../analysis/tokenizer/keyword/keyword_tokenizer.h | 69 +-
.../tokenizer/keyword/keyword_tokenizer_factory.h | 8 +-
.../kuromoji_tokenizer_factory.h} | 31 +-
.../tokenizer/ngram/edge_ngram_tokenizer_factory.h | 33 +-
.../tokenizer/ngram/lucene_gram_tokenizer.cpp | 54 -
.../tokenizer/ngram/lucene_gram_tokenizer.h | 54 -
.../analysis/tokenizer/ngram/ngram_tokenizer.cpp | 53 +-
.../analysis/tokenizer/ngram/ngram_tokenizer.h | 36 +-
.../tokenizer/ngram/ngram_tokenizer_factory.h | 35 +-
.../tokenizer/pinyin/pinyin_alphabet_tokenizer.cpp | 4 +-
.../tokenizer/pinyin/pinyin_alphabet_tokenizer.h | 1 -
.../analysis/tokenizer/pinyin/pinyin_tokenizer.cpp | 139 +--
.../analysis/tokenizer/pinyin/pinyin_tokenizer.h | 25 +-
.../tokenizer/pinyin/pinyin_tokenizer_factory.h | 4 +-
.../tokenizer/standard/standard_tokenizer.h | 68 +-
.../standard/standard_tokenizer_factory.h | 4 +-
.../tokenizer/standard/standard_tokenizer_impl.h | 15 +-
.../index/inverted/analysis/tokenizer/tokenizer.h | 204 ----
.../analysis/tokenizer/tokenizer_factory.h | 10 +-
.../index/inverted/clucene/inverted_index_writer.h | 2 +-
.../inverted/clucene/lucene_analyzer_bridge.h | 6 +-
.../clucene/lucene_backed_text_analyzer.cpp | 2 +-
.../{analysis/reader.h => clucene/lucene_reader.h} | 5 +-
.../exprs/function/search/function_search_test.cpp | 4 +-
.../analysis/analysis_factory_mgr_test.cpp | 58 +-
.../inverted/analysis/analyzer/analyzer_test.cpp | 8 +-
.../analysis/analyzer/custom_analyzer_test.cpp | 51 +-
.../analysis/analyzer/icu_analyzer_test.cpp | 77 +-
.../inverted/analysis/analyzer/ik_anayzer_test.cpp | 282 ++---
.../analyzer/kuromoji/kuromoji_real_dict_test.cpp | 15 +-
.../analyzer/kuromoji/kuromoji_viterbi_test.cpp | 50 +-
.../analysis/analyzer/kuromoji_tokenizer_test.cpp | 9 +-
.../analysis/analyzer/simple_analyzer_test.cpp | 33 +-
.../analysis/char_filter/char_filter_test.cpp | 75 --
.../char_replace_char_filter_factory_test.cpp | 67 +-
.../char_filter/char_replace_char_filter_test.cpp | 34 +-
.../icu_normalizer_char_filter_factory_test.cpp | 148 +--
.../inverted/analysis/component_stream_test.cpp | 516 +++++++++
.../analysis/golden/analysis_differential_test.cpp | 4 +-
.../inverted/analysis/golden/golden_driver.cpp | 14 +-
.../index/inverted/analysis/reader_test.cpp | 4 +-
.../inverted/analysis/text_stream_contract.cpp | 22 +
.../index/inverted/analysis/text_stream_contract.h | 11 +
.../ascii_folding_filter_factory_test.cpp | 25 +-
.../icu_normalizer_filter_factory_test.cpp | 114 +-
.../lower_case_filter_factory_test.cpp | 118 +-
.../analysis/token_filter/pinyin_filter_test.cpp | 1160 +++++---------------
.../token_filter/word_delimiter_filter_test.cpp | 211 ++--
.../tokenizer/basic_tokenizer_factory_test.cpp | 35 +-
.../char_group_tokenizer_factory_test.cpp | 16 +-
.../tokenizer/edge_ngram_tokenizer_test.cpp | 16 +-
.../analysis/tokenizer/gram_tokenizer_test.cpp | 28 +-
.../tokenizer/icu_tokenizer_factory_test.cpp | 118 +-
.../analysis/tokenizer/keyword_analyzer_test.cpp | 43 +-
.../analysis/tokenizer/ngram_tokenizer_test.cpp | 52 +-
.../analysis/tokenizer/pinyin_analysis_test.cpp | 137 +--
.../tokenizer/standard_tokenizer_factory_test.cpp | 33 +-
.../index/inverted/clucene/lucene_adapter_test.cpp | 8 +-
.../query/engine/boolean_postings_bench_test.cpp | 5 +-
.../inverted/query/engine/boolean_query_test.cpp | 20 +-
.../query/engine/multi_phrase_query_test.cpp | 5 +-
.../query/engine/multi_segment_collector_test.cpp | 4 +-
.../engine/occur_boolean_query_real_index_test.cpp | 5 +-
.../query/engine/phrase_postings_bench_test.cpp | 5 +-
.../query/engine/phrase_prefix_query_test.cpp | 4 +-
.../inverted/query/engine/phrase_query_test.cpp | 5 +-
.../inverted/query/engine/prefix_query_test.cpp | 4 +-
.../inverted/query/engine/regexp_query_test.cpp | 5 +-
.../engine/regexp_wildcard_lowercase_test.cpp | 5 +-
.../inverted/query/engine/wildcard_query_test.cpp | 5 +-
.../inverted/reader/collection_statistics_test.cpp | 8 +-
.../storage/segment/inverted_index_reader_test.cpp | 7 +-
.../storage/segment/inverted_index_writer_test.cpp | 4 +-
build-support/check-header-deps.py | 50 -
159 files changed, 3054 insertions(+), 5488 deletions(-)
delete mode 100644
be/src/storage/index/inverted/analysis/analyzer/basic/basic_analyzer.h
copy
be/src/storage/index/inverted/analysis/{char_filter/char_replace_char_filter.h
=> analyzer/chain_analyzer.cpp} (50%)
copy
be/src/storage/index/inverted/analysis/{token_filter/pinyin_filter_factory.h =>
analyzer/chain_analyzer.h} (53%)
delete mode 100644
be/src/storage/index/inverted/analysis/analyzer/icu/icu_analyzer.h
delete mode 100644
be/src/storage/index/inverted/analysis/analyzer/ik/IKAnalyzer.h
delete mode 100644
be/src/storage/index/inverted/analysis/analyzer/kuromoji/KuromojiAnalyzer.h
create mode 100644
be/src/storage/index/inverted/analysis/char_filter/char_filtered_stream.cpp
create mode 100644
be/src/storage/index/inverted/analysis/char_filter/char_filtered_stream.h
create mode 100644 be/src/storage/index/inverted/analysis/common/rune.h
create mode 100644 be/src/storage/index/inverted/analysis/common/text_cursor.h
delete mode 100644
be/src/storage/index/inverted/analysis/token_filter/token_filter.h
delete mode 100644 be/src/storage/index/inverted/analysis/token_stream.h
copy
be/src/storage/index/inverted/analysis/tokenizer/{ik/ik_tokenizer_factory.h =>
kuromoji/kuromoji_tokenizer_factory.h} (55%)
delete mode 100644
be/src/storage/index/inverted/analysis/tokenizer/ngram/lucene_gram_tokenizer.cpp
delete mode 100644
be/src/storage/index/inverted/analysis/tokenizer/ngram/lucene_gram_tokenizer.h
delete mode 100644 be/src/storage/index/inverted/analysis/tokenizer/tokenizer.h
rename be/src/storage/index/inverted/{analysis/reader.h =>
clucene/lucene_reader.h} (84%)
delete mode 100644
be/test/storage/index/inverted/analysis/char_filter/char_filter_test.cpp
create mode 100644
be/test/storage/index/inverted/analysis/component_stream_test.cpp
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]