This is an automated email from the ASF dual-hosted git repository.

airborne12 pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git


    from d7c0fb9427b [fix](be) Preserve local time positions and fractions in 
DST gaps (#70204)
     add afd67a2d0fc [refactor](be) Move every analysis component onto the 
CLucene-free stream interface (#70263)

No new revisions were added by this update.

Summary of changes:
 be/src/exprs/function/match.cpp                    |   49 +-
 be/src/runtime/index_policy/index_policy_mgr.cpp   |   12 +-
 .../inverted/analysis/abstract_analysis_factory.h  |    7 +-
 .../index/inverted/analysis/analyzer/analyzer.cpp  |  261 ++---
 .../index/inverted/analysis/analyzer/analyzer.h    |   29 +-
 .../analysis/analyzer/basic/basic_analyzer.h       |   81 --
 .../chain_analyzer.cpp}                            |   50 +-
 .../chain_analyzer.h}                              |   26 +-
 .../inverted/analysis/analyzer/custom_analyzer.cpp |  129 +--
 .../inverted/analysis/analyzer/custom_analyzer.h   |   66 +-
 .../inverted/analysis/analyzer/icu/icu_analyzer.h  |   84 --
 .../inverted/analysis/analyzer/ik/IKAnalyzer.h     |   72 --
 .../inverted/analysis/analyzer/ik/IKTokenizer.cpp  |  137 +--
 .../inverted/analysis/analyzer/ik/IKTokenizer.h    |   53 +-
 .../analysis/analyzer/ik/core/AnalyzeContext.cpp   |   30 +-
 .../analysis/analyzer/ik/core/AnalyzeContext.h     |   10 +-
 .../analysis/analyzer/ik/core/CharacterUtil.cpp    |    4 +-
 .../analysis/analyzer/ik/core/CharacterUtil.h      |   12 +-
 .../analysis/analyzer/ik/core/IKSegmenter.cpp      |    4 +-
 .../analysis/analyzer/ik/core/IKSegmenter.h        |    7 +-
 .../analysis/analyzer/ik/core/ISegmenter.h         |    1 -
 .../inverted/analysis/analyzer/ik/core/Lexeme.h    |    1 -
 .../analysis/analyzer/ik/core/LexemePath.cpp       |    2 +-
 .../analysis/analyzer/ik/core/LexemePath.h         |    1 -
 .../analysis/analyzer/ik/core/QuickSortSet.h       |    2 -
 .../analysis/analyzer/ik/dic/DictSegment.h         |    2 -
 .../analysis/analyzer/ik/dic/Dictionary.cpp        |   16 +-
 .../inverted/analysis/analyzer/ik/dic/Dictionary.h |   11 +-
 .../index/inverted/analysis/analyzer/ik/dic/Hit.h  |    2 -
 .../analysis/analyzer/kuromoji/KuromojiAnalyzer.h  |   75 --
 .../analyzer/kuromoji/KuromojiTokenizer.cpp        |   61 +-
 .../analysis/analyzer/kuromoji/KuromojiTokenizer.h |   22 +-
 .../inverted/analysis/char_filter/char_filter.h    |   51 +-
 .../analysis/char_filter/char_filter_factory.h     |    9 +-
 .../analysis/char_filter/char_filtered_stream.cpp  |   72 ++
 .../analysis/char_filter/char_filtered_stream.h    |   67 ++
 .../char_filter/char_replace_char_filter.cpp       |   57 +-
 .../char_filter/char_replace_char_filter.h         |   26 +-
 .../char_filter/char_replace_char_filter_factory.h |   10 +-
 .../char_filter/empty_char_filter_factory.h        |   30 +-
 .../char_filter/icu_normalizer_char_filter.cpp     |  100 +-
 .../char_filter/icu_normalizer_char_filter.h       |   33 +-
 .../icu_normalizer_char_filter_factory.h           |    6 +-
 .../storage/index/inverted/analysis/common/rune.h  |  133 +++
 .../index/inverted/analysis/common/text_cursor.h   |   57 +
 .../analysis/normalizer/custom_normalizer.cpp      |   91 +-
 .../analysis/normalizer/custom_normalizer.h        |   54 +-
 .../index/inverted/analysis/per_token_stream.cpp   |   20 +-
 .../index/inverted/analysis/per_token_stream.h     |   14 +-
 .../index/inverted/analysis/text_token_stream.h    |    6 +-
 .../analysis/token_filter/ascii_folding_filter.cpp |  121 +-
 .../analysis/token_filter/ascii_folding_filter.h   |   37 +-
 .../token_filter/ascii_folding_filter_factory.h    |    9 +-
 .../token_filter/empty_token_filter_factory.h      |   26 +-
 .../token_filter/icu_normalizer_filter.cpp         |   68 +-
 .../analysis/token_filter/icu_normalizer_filter.h  |   27 +-
 .../token_filter/icu_normalizer_filter_factory.h   |    8 +-
 .../analysis/token_filter/lower_case_filter.h      |  120 +-
 .../token_filter/lower_case_filter_factory.h       |    8 +-
 .../analysis/token_filter/pinyin_filter.cpp        |  299 +----
 .../inverted/analysis/token_filter/pinyin_filter.h |   88 +-
 .../token_filter/pinyin_filter_factory.cpp         |    7 +-
 .../analysis/token_filter/pinyin_filter_factory.h  |    3 +-
 .../inverted/analysis/token_filter/token_filter.h  |   75 --
 .../analysis/token_filter/token_filter_factory.h   |   11 +-
 .../token_filter/word_delimiter_filter.cpp         |  219 +---
 .../analysis/token_filter/word_delimiter_filter.h  |   91 +-
 .../token_filter/word_delimiter_filter_factory.h   |    4 +-
 .../storage/index/inverted/analysis/token_stream.h |  130 ---
 .../analysis/tokenizer/basic/basic_tokenizer.cpp   |   78 +-
 .../analysis/tokenizer/basic/basic_tokenizer.h     |   32 +-
 .../tokenizer/basic/basic_tokenizer_factory.h      |    8 +-
 .../char/char_group_tokenizer_factory.cpp          |    9 +-
 .../tokenizer/char/char_group_tokenizer_factory.h  |   10 +-
 .../analysis/tokenizer/char/char_tokenizer.cpp     |   29 +-
 .../analysis/tokenizer/char/char_tokenizer.h       |   22 +-
 .../tokenizer/empty/empty_tokenizer_factory.h      |   57 +-
 .../tokenizer/icu/break_iterator_wrapper.cpp       |    8 +-
 .../tokenizer/icu/default_icu_tokenizer_config.cpp |   10 +-
 .../inverted/analysis/tokenizer/icu/icu_common.h   |    3 +-
 .../analysis/tokenizer/icu/icu_tokenizer.cpp       |   98 +-
 .../analysis/tokenizer/icu/icu_tokenizer.h         |   40 +-
 .../analysis/tokenizer/icu/icu_tokenizer_factory.h |    5 +-
 .../analysis/tokenizer/icu/script_iterator.cpp     |    4 +-
 .../analysis/tokenizer/ik/ik_tokenizer_factory.h   |   24 +-
 .../analysis/tokenizer/keyword/keyword_tokenizer.h |   69 +-
 .../tokenizer/keyword/keyword_tokenizer_factory.h  |    8 +-
 .../kuromoji_tokenizer_factory.h}                  |   31 +-
 .../tokenizer/ngram/edge_ngram_tokenizer_factory.h |   33 +-
 .../tokenizer/ngram/lucene_gram_tokenizer.cpp      |   54 -
 .../tokenizer/ngram/lucene_gram_tokenizer.h        |   54 -
 .../analysis/tokenizer/ngram/ngram_tokenizer.cpp   |   53 +-
 .../analysis/tokenizer/ngram/ngram_tokenizer.h     |   36 +-
 .../tokenizer/ngram/ngram_tokenizer_factory.h      |   35 +-
 .../tokenizer/pinyin/pinyin_alphabet_tokenizer.cpp |    4 +-
 .../tokenizer/pinyin/pinyin_alphabet_tokenizer.h   |    1 -
 .../analysis/tokenizer/pinyin/pinyin_tokenizer.cpp |  139 +--
 .../analysis/tokenizer/pinyin/pinyin_tokenizer.h   |   25 +-
 .../tokenizer/pinyin/pinyin_tokenizer_factory.h    |    4 +-
 .../tokenizer/standard/standard_tokenizer.h        |   68 +-
 .../standard/standard_tokenizer_factory.h          |    4 +-
 .../tokenizer/standard/standard_tokenizer_impl.h   |   15 +-
 .../index/inverted/analysis/tokenizer/tokenizer.h  |  204 ----
 .../analysis/tokenizer/tokenizer_factory.h         |   10 +-
 .../index/inverted/clucene/inverted_index_writer.h |    2 +-
 .../inverted/clucene/lucene_analyzer_bridge.h      |    6 +-
 .../clucene/lucene_backed_text_analyzer.cpp        |    2 +-
 .../{analysis/reader.h => clucene/lucene_reader.h} |    5 +-
 .../exprs/function/search/function_search_test.cpp |    4 +-
 .../analysis/analysis_factory_mgr_test.cpp         |   58 +-
 .../inverted/analysis/analyzer/analyzer_test.cpp   |    8 +-
 .../analysis/analyzer/custom_analyzer_test.cpp     |   51 +-
 .../analysis/analyzer/icu_analyzer_test.cpp        |   77 +-
 .../inverted/analysis/analyzer/ik_anayzer_test.cpp |  282 ++---
 .../analyzer/kuromoji/kuromoji_real_dict_test.cpp  |   15 +-
 .../analyzer/kuromoji/kuromoji_viterbi_test.cpp    |   50 +-
 .../analysis/analyzer/kuromoji_tokenizer_test.cpp  |    9 +-
 .../analysis/analyzer/simple_analyzer_test.cpp     |   33 +-
 .../analysis/char_filter/char_filter_test.cpp      |   75 --
 .../char_replace_char_filter_factory_test.cpp      |   67 +-
 .../char_filter/char_replace_char_filter_test.cpp  |   34 +-
 .../icu_normalizer_char_filter_factory_test.cpp    |  148 +--
 .../inverted/analysis/component_stream_test.cpp    |  516 +++++++++
 .../analysis/golden/analysis_differential_test.cpp |    4 +-
 .../inverted/analysis/golden/golden_driver.cpp     |   14 +-
 .../index/inverted/analysis/reader_test.cpp        |    4 +-
 .../inverted/analysis/text_stream_contract.cpp     |   22 +
 .../index/inverted/analysis/text_stream_contract.h |   11 +
 .../ascii_folding_filter_factory_test.cpp          |   25 +-
 .../icu_normalizer_filter_factory_test.cpp         |  114 +-
 .../lower_case_filter_factory_test.cpp             |  118 +-
 .../analysis/token_filter/pinyin_filter_test.cpp   | 1160 +++++---------------
 .../token_filter/word_delimiter_filter_test.cpp    |  211 ++--
 .../tokenizer/basic_tokenizer_factory_test.cpp     |   35 +-
 .../char_group_tokenizer_factory_test.cpp          |   16 +-
 .../tokenizer/edge_ngram_tokenizer_test.cpp        |   16 +-
 .../analysis/tokenizer/gram_tokenizer_test.cpp     |   28 +-
 .../tokenizer/icu_tokenizer_factory_test.cpp       |  118 +-
 .../analysis/tokenizer/keyword_analyzer_test.cpp   |   43 +-
 .../analysis/tokenizer/ngram_tokenizer_test.cpp    |   52 +-
 .../analysis/tokenizer/pinyin_analysis_test.cpp    |  137 +--
 .../tokenizer/standard_tokenizer_factory_test.cpp  |   33 +-
 .../index/inverted/clucene/lucene_adapter_test.cpp |    8 +-
 .../query/engine/boolean_postings_bench_test.cpp   |    5 +-
 .../inverted/query/engine/boolean_query_test.cpp   |   20 +-
 .../query/engine/multi_phrase_query_test.cpp       |    5 +-
 .../query/engine/multi_segment_collector_test.cpp  |    4 +-
 .../engine/occur_boolean_query_real_index_test.cpp |    5 +-
 .../query/engine/phrase_postings_bench_test.cpp    |    5 +-
 .../query/engine/phrase_prefix_query_test.cpp      |    4 +-
 .../inverted/query/engine/phrase_query_test.cpp    |    5 +-
 .../inverted/query/engine/prefix_query_test.cpp    |    4 +-
 .../inverted/query/engine/regexp_query_test.cpp    |    5 +-
 .../engine/regexp_wildcard_lowercase_test.cpp      |    5 +-
 .../inverted/query/engine/wildcard_query_test.cpp  |    5 +-
 .../inverted/reader/collection_statistics_test.cpp |    8 +-
 .../storage/segment/inverted_index_reader_test.cpp |    7 +-
 .../storage/segment/inverted_index_writer_test.cpp |    4 +-
 build-support/check-header-deps.py                 |   50 -
 159 files changed, 3054 insertions(+), 5488 deletions(-)
 delete mode 100644 
be/src/storage/index/inverted/analysis/analyzer/basic/basic_analyzer.h
 copy 
be/src/storage/index/inverted/analysis/{char_filter/char_replace_char_filter.h 
=> analyzer/chain_analyzer.cpp} (50%)
 copy 
be/src/storage/index/inverted/analysis/{token_filter/pinyin_filter_factory.h => 
analyzer/chain_analyzer.h} (53%)
 delete mode 100644 
be/src/storage/index/inverted/analysis/analyzer/icu/icu_analyzer.h
 delete mode 100644 
be/src/storage/index/inverted/analysis/analyzer/ik/IKAnalyzer.h
 delete mode 100644 
be/src/storage/index/inverted/analysis/analyzer/kuromoji/KuromojiAnalyzer.h
 create mode 100644 
be/src/storage/index/inverted/analysis/char_filter/char_filtered_stream.cpp
 create mode 100644 
be/src/storage/index/inverted/analysis/char_filter/char_filtered_stream.h
 create mode 100644 be/src/storage/index/inverted/analysis/common/rune.h
 create mode 100644 be/src/storage/index/inverted/analysis/common/text_cursor.h
 delete mode 100644 
be/src/storage/index/inverted/analysis/token_filter/token_filter.h
 delete mode 100644 be/src/storage/index/inverted/analysis/token_stream.h
 copy 
be/src/storage/index/inverted/analysis/tokenizer/{ik/ik_tokenizer_factory.h => 
kuromoji/kuromoji_tokenizer_factory.h} (55%)
 delete mode 100644 
be/src/storage/index/inverted/analysis/tokenizer/ngram/lucene_gram_tokenizer.cpp
 delete mode 100644 
be/src/storage/index/inverted/analysis/tokenizer/ngram/lucene_gram_tokenizer.h
 delete mode 100644 be/src/storage/index/inverted/analysis/tokenizer/tokenizer.h
 rename be/src/storage/index/inverted/{analysis/reader.h => 
clucene/lucene_reader.h} (84%)
 delete mode 100644 
be/test/storage/index/inverted/analysis/char_filter/char_filter_test.cpp
 create mode 100644 
be/test/storage/index/inverted/analysis/component_stream_test.cpp


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to