This is an automated email from the ASF dual-hosted git repository.
airborne12 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 75d1f03a7cc [fix](inverted index) Preserve tokens from all array
elements in MATCH fallback (#68661)
75d1f03a7cc is described below
commit 75d1f03a7cc0654cc8e48e5318ae2631e96cd980
Author: Jack <[email protected]>
AuthorDate: Thu Oct 8 09:31:05 2026 +0800
[fix](inverted index) Preserve tokens from all array elements in MATCH
fallback (#68661)
### What problem does this PR solve?
Without an inverted index, MATCH on an array of strings can silently
miss rows when the matching token appears before the final array
element. For example, ['alpha token', 'tail only'] fails to match
MATCH_ANY 'alpha'.
The array branch of FunctionMatchBase::analyse_data_token overwrites the
accumulated tokens with each element's analyzer output, retaining only
the final element's tokens.
This change accumulates tokens from all array elements for MATCH_ANY,
MATCH_ALL, and MATCH_REGEXP. MATCH_PHRASE, MATCH_PHRASE_PREFIX, and
MATCH_PHRASE_EDGE evaluate each array element separately so phrases
cannot span element boundaries. Their existing token-matching algorithms
are extracted into file-local helpers.
The new unit test covers all six MATCH variants, first and final
elements, an empty final element, empty arrays, custom analyzers,
keyword parsing, and phrase boundaries.
---
be/src/exprs/function/match.cpp | 538 ++++++++++++++++---------
be/src/exprs/function/match.h | 89 ++--
be/test/exprs/function/function_match_test.cpp | 400 ++++++++++++++++++
3 files changed, 801 insertions(+), 226 deletions(-)
diff --git a/be/src/exprs/function/match.cpp b/be/src/exprs/function/match.cpp
index 4d747df0e56..03258b60206 100644
--- a/be/src/exprs/function/match.cpp
+++ b/be/src/exprs/function/match.cpp
@@ -19,6 +19,12 @@
#include <hs/hs.h>
+#include <algorithm>
+#include <cstdint>
+#include <span>
+#include <unordered_map>
+#include <unordered_set>
+
#include "core/field.h"
#include "runtime/query_context.h"
#include "runtime/runtime_state.h"
@@ -44,6 +50,253 @@ const InvertedIndexAnalyzerCtx*
get_match_analyzer_ctx(FunctionContext* context)
return analyzer_ctx;
}
+enum class PhraseMode { EXACT, PREFIX, EDGE };
+
+class StreamingPhraseMatcher {
+public:
+ StreamingPhraseMatcher(const std::vector<segment_v2::TermInfo>&
query_tokens, PhraseMode mode)
+ : _mode(mode),
+ _query_size(query_tokens.size()),
+ _last_word((_query_size - 1) / 64),
+ _last_bit(uint64_t {1} << ((_query_size - 1) % 64)),
+ _state(_last_word + 1, 0),
+ _pending_masks(_last_word + 1, 0),
+ _active_marks(_last_word + 1, 0),
+ _first_term(query_tokens.front().get_single_term()),
+ _last_term(query_tokens.back().get_single_term()) {
+ for (size_t pos = 0; pos < _query_size; ++pos) {
+ if ((_mode == PhraseMode::PREFIX && pos == _query_size - 1) ||
+ (_mode == PhraseMode::EDGE && (pos == 0 || pos == _query_size
- 1))) {
+ continue;
+ }
+ auto& words = _exact_masks[query_tokens[pos].get_single_term()];
+ const size_t word = pos / 64;
+ const uint64_t bit = uint64_t {1} << (pos % 64);
+ if (!words.empty() && words.back().first == word) {
+ words.back().second |= bit;
+ } else {
+ words.emplace_back(word, bit);
+ }
+ }
+ }
+
+ void reset_row() {
+ for (size_t word : _pending_touched) {
+ _pending_masks[word] = 0;
+ }
+ _pending_touched.clear();
+ _pending_sorted = true;
+ _pending_position = -1;
+ _previous_position = -1;
+ _active_words.clear();
+ ++_generation;
+ }
+
+ bool feed(const std::string& term, int64_t position) {
+ if (_pending_position >= 0 && position != _pending_position) {
+ if (advance()) {
+ return true;
+ }
+ }
+ if (_pending_position < 0) {
+ start_position(position);
+ }
+ const auto mask_it = _exact_masks.find(term);
+ if (mask_it != _exact_masks.end()) {
+ const auto& mask_words = mask_it->second;
+ if (mask_words.size() / 4 > _candidate_words.size()) {
+ for (size_t word : _candidate_words) {
+ auto it = std::lower_bound(
+ mask_words.begin(), mask_words.end(), word,
+ [](const auto& entry, size_t value) { return
entry.first < value; });
+ if (it != mask_words.end() && it->first == word) {
+ add_pending_mask(word, it->second);
+ }
+ }
+ } else {
+ size_t candidate = 0;
+ for (const auto& [word, mask] : mask_words) {
+ while (candidate < _candidate_words.size() &&
+ _candidate_words[candidate] < word) {
+ ++candidate;
+ }
+ if (candidate == _candidate_words.size()) {
+ break;
+ }
+ if (_candidate_words[candidate] == word) {
+ add_pending_mask(word, mask);
+ }
+ }
+ }
+ }
+ const bool first_matches = _mode == PhraseMode::EDGE &&
+ (_query_size == 1 ? term.find(_first_term)
!= std::string::npos
+ :
term.ends_with(_first_term));
+ const bool last_matches =
+ (_mode == PhraseMode::PREFIX || (_mode == PhraseMode::EDGE &&
_query_size > 1)) &&
+ term.starts_with(_last_term);
+
+ if (first_matches) {
+ add_pending_mask(0, 1);
+ }
+ if (last_matches &&
+ std::binary_search(_candidate_words.begin(),
_candidate_words.end(), _last_word)) {
+ add_pending_mask(_last_word, _last_bit);
+ }
+ return false;
+ }
+
+ bool finish() { return _pending_position >= 0 && advance(); }
+
+private:
+ void start_position(int64_t position) {
+ if (_previous_position >= 0 && position != _previous_position + 1) {
+ _active_words.clear();
+ ++_generation;
+ }
+ _pending_position = position;
+ _candidate_words.clear();
+ _candidate_words.push_back(0);
+ for (size_t word : _active_words) {
+ if (_candidate_words.back() != word) {
+ _candidate_words.push_back(word);
+ }
+ if (word < _last_word && (_state[word] >> 63) != 0) {
+ _candidate_words.push_back(word + 1);
+ }
+ }
+ }
+
+ void add_pending_mask(size_t word, uint64_t mask) {
+ if (_pending_masks[word] == 0) {
+ if (!_pending_touched.empty() && word < _pending_touched.back()) {
+ _pending_sorted = false;
+ }
+ _pending_touched.push_back(word);
+ }
+ _pending_masks[word] |= mask;
+ }
+
+ bool advance() {
+ const int64_t position = _pending_position;
+ _pending_position = -1;
+ _previous_position = position;
+
+ const uint64_t old_generation = _generation++;
+ _next_active_words.clear();
+ if (!_pending_touched.empty()) {
+ if (!_pending_sorted) {
+ std::sort(_pending_touched.begin(), _pending_touched.end());
+ }
+ for (auto it = _pending_touched.rbegin(); it !=
_pending_touched.rend(); ++it) {
+ const size_t word = *it;
+ const uint64_t previous = _active_marks[word] ==
old_generation ? _state[word] : 0;
+ const uint64_t carry = word > 0 && _active_marks[word - 1] ==
old_generation
+ ? _state[word - 1] >> 63
+ : 0;
+ const uint64_t next =
+ ((previous << 1) | carry | uint64_t {word == 0}) &
_pending_masks[word];
+ if (next != 0) {
+ _state[word] = next;
+ _active_marks[word] = _generation;
+ _next_active_words.push_back(word);
+ }
+ }
+ }
+ for (size_t word : _pending_touched) {
+ _pending_masks[word] = 0;
+ }
+ _pending_touched.clear();
+ _pending_sorted = true;
+ std::reverse(_next_active_words.begin(), _next_active_words.end());
+ _active_words.swap(_next_active_words);
+ return !_active_words.empty() && _active_words.back() == _last_word &&
+ (_state[_last_word] & _last_bit) != 0;
+ }
+
+ PhraseMode _mode;
+ size_t _query_size;
+ size_t _last_word;
+ uint64_t _last_bit;
+ std::vector<uint64_t> _state;
+ std::vector<uint64_t> _pending_masks;
+ std::vector<uint64_t> _active_marks;
+ std::vector<size_t> _pending_touched;
+ std::vector<size_t> _candidate_words;
+ std::vector<size_t> _active_words;
+ std::vector<size_t> _next_active_words;
+ std::string _first_term;
+ std::string _last_term;
+ std::unordered_map<std::string, std::vector<std::pair<size_t, uint64_t>>>
_exact_masks;
+ int64_t _pending_position = -1;
+ int64_t _previous_position = -1;
+ uint64_t _generation = 1;
+ bool _pending_sorted = true;
+};
+
+template <typename Callback>
+bool for_each_data_element_tokens(const FunctionMatchBase& function, const
std::string& column_name,
+ const InvertedIndexAnalyzerCtx* analyzer_ctx,
+ const ColumnString* string_col, size_t row,
+ const ColumnArray::Offsets64* array_offsets,
+ const ColumnUInt8::Container*
array_element_null_map,
+ Callback&& callback) {
+ const size_t begin = array_offsets ? (row == 0 ? 0 : (*array_offsets)[row
- 1]) : row;
+ const size_t end = array_offsets ? (*array_offsets)[row] : row + 1;
+ const bool keyword = analyzer_ctx &&
+ (!analyzer_ctx->requires_analysis() ||
analyzer_ctx->analyzer == nullptr);
+ segment_v2::TermInfo keyword_token;
+ int32_t unused_array_offset = 0;
+ for (size_t element = begin; element < end; ++element) {
+ if (array_element_null_map && (*array_element_null_map)[element]) {
+ continue;
+ }
+ if (keyword) {
+ keyword_token.term = string_col->get_data_at(element).to_string();
+ if (callback(std::span(&keyword_token, 1))) {
+ return true;
+ }
+ continue;
+ }
+ auto tokens = function.analyse_data_token(column_name, analyzer_ctx,
string_col, element,
+ nullptr,
unused_array_offset);
+ if (tokens.empty()) {
+ continue;
+ }
+ if (callback(std::span<const segment_v2::TermInfo>(tokens))) {
+ return true;
+ }
+ }
+ return false;
+}
+
+bool match_phrase_data_tokens(const FunctionMatchBase& function, const
std::string& column_name,
+ const InvertedIndexAnalyzerCtx* analyzer_ctx,
+ const ColumnString* string_col, size_t row,
+ const ColumnArray::Offsets64* array_offsets,
+ const ColumnUInt8::Container*
array_element_null_map,
+ StreamingPhraseMatcher& matcher) {
+ matcher.reset_row();
+ int64_t position_base = 0;
+ const bool matched = for_each_data_element_tokens(
+ function, column_name, analyzer_ctx, string_col, row,
array_offsets,
+ array_element_null_map, [&](std::span<const segment_v2::TermInfo>
tokens) {
+ const bool analyzed = analyzer_ctx &&
analyzer_ctx->requires_analysis() &&
+ analyzer_ctx->analyzer != nullptr;
+ int32_t last_position = 0;
+ for (const auto& token : tokens) {
+ const int32_t position = analyzed ? token.position : 1;
+ if (matcher.feed(token.get_single_term(), position_base +
position)) {
+ return true;
+ }
+ last_position = position;
+ }
+ position_base += last_position;
+ return false;
+ });
+ return matched || matcher.finish();
+}
+
} // namespace
Status FunctionMatchBase::evaluate_inverted_index(
@@ -140,6 +393,7 @@ Status FunctionMatchBase::execute_impl(FunctionContext*
context, Block& block,
block.get_by_position(arguments[0]).column->convert_to_full_column_if_const();
const auto* values = check_and_get_column<ColumnString>(source_col.get());
const ColumnArray* array_col = nullptr;
+ const ColumnUInt8::Container* array_element_null_map = nullptr;
if (is_column<ColumnArray>(source_col.get())) {
array_col = check_and_get_column<ColumnArray>(source_col.get());
if (array_col && !array_col->get_data().is_column_string()) {
@@ -153,6 +407,7 @@ Status FunctionMatchBase::execute_impl(FunctionContext*
context, Block& block,
if (is_column_nullable(array_col->get_data())) {
const auto& array_nested_null_column =
reinterpret_cast<const
ColumnNullable&>(array_col->get_data());
+ array_element_null_map =
&array_nested_null_column.get_null_map_column().get_data();
values = check_and_get_column<ColumnString>(
*(array_nested_null_column.get_nested_column_ptr()));
} else {
@@ -174,7 +429,7 @@ Status FunctionMatchBase::execute_impl(FunctionContext*
context, Block& block,
vec_res.resize_fill(input_rows_count);
RETURN_IF_ERROR(execute_match(context, column_name, match_query_str,
input_rows_count, values,
analyzer_ctx, (array_col ?
&(array_col->get_offsets()) : nullptr),
- vec_res));
+ vec_res, array_element_null_map));
block.replace_by_position(result, std::move(res));
return Status::OK();
@@ -237,8 +492,9 @@ std::vector<segment_v2::TermInfo>
FunctionMatchBase::analyse_query_str_token(
inline std::vector<segment_v2::TermInfo> FunctionMatchBase::analyse_data_token(
const std::string& column_name, const InvertedIndexAnalyzerCtx*
analyzer_ctx,
- const ColumnString* string_col, int32_t current_block_row_idx,
- const ColumnArray::Offsets64* array_offsets, int32_t&
current_src_array_offset) const {
+ const ColumnString* string_col, size_t current_block_row_idx,
+ const ColumnArray::Offsets64* array_offsets, int32_t&
current_src_array_offset,
+ const ColumnUInt8::Container* array_element_null_map) const {
std::vector<segment_v2::TermInfo> data_tokens;
if (analyzer_ctx == nullptr) {
return data_tokens;
@@ -250,6 +506,9 @@ inline std::vector<segment_v2::TermInfo>
FunctionMatchBase::analyse_data_token(
if (array_offsets) {
for (auto next_src_array_offset =
(*array_offsets)[current_block_row_idx];
current_src_array_offset < next_src_array_offset;
++current_src_array_offset) {
+ if (array_element_null_map &&
(*array_element_null_map)[current_src_array_offset]) {
+ continue;
+ }
const auto& str_ref =
string_col->get_data_at(current_src_array_offset);
if (!requires_analysis) {
data_tokens.emplace_back(str_ref.to_string());
@@ -258,9 +517,12 @@ inline std::vector<segment_v2::TermInfo>
FunctionMatchBase::analyse_data_token(
auto reader =
doris::segment_v2::inverted_index::InvertedIndexAnalyzer::create_reader(
analyzer_ctx->char_filter_map);
reader->init(str_ref.data, (int)str_ref.size, true);
- data_tokens =
+ auto element_tokens =
doris::segment_v2::inverted_index::InvertedIndexAnalyzer::get_analyse_result(
reader, analyzer_ctx->analyzer.get());
+ for (auto& token : element_tokens) {
+ data_tokens.emplace_back(std::move(token));
+ }
}
} else {
const auto& str_ref = string_col->get_data_at(current_block_row_idx);
@@ -297,7 +559,8 @@ Status FunctionMatchAny::execute_match(FunctionContext*
context, const std::stri
const ColumnString* string_col,
const InvertedIndexAnalyzerCtx*
analyzer_ctx,
const ColumnArray::Offsets64*
array_offsets,
- ColumnUInt8::Container& result) const {
+ ColumnUInt8::Container& result,
+ const ColumnUInt8::Container*
array_element_null_map) const {
RETURN_IF_ERROR(check(context, name));
auto query_tokens = analyse_query_str_token(analyzer_ctx, match_query_str,
column_name);
@@ -311,21 +574,23 @@ Status FunctionMatchAny::execute_match(FunctionContext*
context, const std::stri
return Status::OK();
}
- auto current_src_array_offset = 0;
+ std::unordered_set<std::string> query_terms;
+ for (const auto& token : query_tokens) {
+ query_terms.emplace(token.get_single_term());
+ }
+
for (int i = 0; i < input_rows_count; i++) {
- auto data_tokens = analyse_data_token(column_name, analyzer_ctx,
string_col, i,
- array_offsets,
current_src_array_offset);
-
- // TODO: more efficient impl
- for (auto& term_info : query_tokens) {
- auto it = std::find_if(data_tokens.begin(), data_tokens.end(),
- [&](const segment_v2::TermInfo& info) {
- return info.get_single_term() ==
term_info.get_single_term();
- });
- if (it != data_tokens.end()) {
- result[i] = true;
- break;
- }
+ if (for_each_data_element_tokens(*this, column_name, analyzer_ctx,
string_col, i,
+ array_offsets, array_element_null_map,
+ [&](std::span<const
segment_v2::TermInfo> data_tokens) {
+ for (const auto& info :
data_tokens) {
+ if
(query_terms.contains(info.get_single_term())) {
+ return true;
+ }
+ }
+ return false;
+ })) {
+ result[i] = true;
}
}
@@ -337,7 +602,8 @@ Status FunctionMatchAll::execute_match(FunctionContext*
context, const std::stri
const ColumnString* string_col,
const InvertedIndexAnalyzerCtx*
analyzer_ctx,
const ColumnArray::Offsets64*
array_offsets,
- ColumnUInt8::Container& result) const {
+ ColumnUInt8::Container& result,
+ const ColumnUInt8::Container*
array_element_null_map) const {
RETURN_IF_ERROR(check(context, name));
auto query_tokens = analyse_query_str_token(analyzer_ctx, match_query_str,
column_name);
@@ -351,26 +617,32 @@ Status FunctionMatchAll::execute_match(FunctionContext*
context, const std::stri
return Status::OK();
}
- auto current_src_array_offset = 0;
- for (int i = 0; i < input_rows_count; i++) {
- auto data_tokens = analyse_data_token(column_name, analyzer_ctx,
string_col, i,
- array_offsets,
current_src_array_offset);
-
- // TODO: more efficient impl
- auto find_count = 0;
- for (auto& term_info : query_tokens) {
- auto it = std::find_if(data_tokens.begin(), data_tokens.end(),
- [&](const segment_v2::TermInfo& info) {
- return info.get_single_term() ==
term_info.get_single_term();
- });
- if (it != data_tokens.end()) {
- ++find_count;
- } else {
- break;
- }
- }
+ std::unordered_map<std::string, size_t> query_term_ids;
+ for (const auto& token : query_tokens) {
+ query_term_ids.emplace(token.get_single_term(), query_term_ids.size());
+ }
+ std::vector<size_t> last_seen_row;
- if (find_count == query_tokens.size()) {
+ for (int i = 0; i < input_rows_count; i++) {
+ size_t remaining = query_term_ids.size();
+ if (for_each_data_element_tokens(
+ *this, column_name, analyzer_ctx, string_col, i,
array_offsets,
+ array_element_null_map, [&](std::span<const
segment_v2::TermInfo> data_tokens) {
+ if (last_seen_row.empty()) {
+ last_seen_row.resize(query_term_ids.size(), 0);
+ }
+ for (const auto& info : data_tokens) {
+ auto it =
query_term_ids.find(info.get_single_term());
+ if (it != query_term_ids.end() &&
+ last_seen_row[it->second] !=
static_cast<size_t>(i) + 1) {
+ last_seen_row[it->second] =
static_cast<size_t>(i) + 1;
+ if (--remaining == 0) {
+ return true;
+ }
+ }
+ }
+ return false;
+ })) {
result[i] = true;
}
}
@@ -378,12 +650,12 @@ Status FunctionMatchAll::execute_match(FunctionContext*
context, const std::stri
return Status::OK();
}
-Status FunctionMatchPhrase::execute_match(FunctionContext* context, const
std::string& column_name,
- const std::string& match_query_str,
- size_t input_rows_count, const
ColumnString* string_col,
- const InvertedIndexAnalyzerCtx*
analyzer_ctx,
- const ColumnArray::Offsets64*
array_offsets,
- ColumnUInt8::Container& result)
const {
+Status FunctionMatchPhrase::execute_match(
+ FunctionContext* context, const std::string& column_name,
+ const std::string& match_query_str, size_t input_rows_count, const
ColumnString* string_col,
+ const InvertedIndexAnalyzerCtx* analyzer_ctx, const
ColumnArray::Offsets64* array_offsets,
+ ColumnUInt8::Container& result,
+ const ColumnUInt8::Container* array_element_null_map) const {
RETURN_IF_ERROR(check(context, name));
auto query_tokens = analyse_query_str_token(analyzer_ctx, match_query_str,
column_name);
@@ -397,43 +669,10 @@ Status
FunctionMatchPhrase::execute_match(FunctionContext* context, const std::s
return Status::OK();
}
- auto current_src_array_offset = 0;
+ StreamingPhraseMatcher matcher(query_tokens, PhraseMode::EXACT);
for (int i = 0; i < input_rows_count; i++) {
- auto data_tokens = analyse_data_token(column_name, analyzer_ctx,
string_col, i,
- array_offsets,
current_src_array_offset);
-
- // TODO: more efficient impl
- bool matched = false;
- auto data_it = data_tokens.begin();
- while (data_it != data_tokens.end()) {
- // find position of first token
- data_it =
- std::find_if(data_it, data_tokens.end(), [&](const
segment_v2::TermInfo& info) {
- return info.get_single_term() ==
query_tokens[0].get_single_term();
- });
- if (data_it != data_tokens.end()) {
- matched = true;
- auto data_it_next = ++data_it;
- auto query_it = query_tokens.begin() + 1;
- // compare query_tokens after the first to data_tokens one by
one
- while (query_it != query_tokens.end()) {
- if (data_it_next == data_tokens.end() ||
- data_it_next->get_single_term() !=
query_it->get_single_term()) {
- matched = false;
- break;
- }
- query_it++;
- data_it_next++;
- }
-
- if (matched) {
- break;
- }
- }
- }
-
- // check matched
- if (matched) {
+ if (match_phrase_data_tokens(*this, column_name, analyzer_ctx,
string_col, i, array_offsets,
+ array_element_null_map, matcher)) {
result[i] = true;
}
}
@@ -445,7 +684,8 @@ Status FunctionMatchPhrasePrefix::execute_match(
FunctionContext* context, const std::string& column_name,
const std::string& match_query_str, size_t input_rows_count, const
ColumnString* string_col,
const InvertedIndexAnalyzerCtx* analyzer_ctx, const
ColumnArray::Offsets64* array_offsets,
- ColumnUInt8::Container& result) const {
+ ColumnUInt8::Container& result,
+ const ColumnUInt8::Container* array_element_null_map) const {
RETURN_IF_ERROR(check(context, name));
auto query_tokens = analyse_query_str_token(analyzer_ctx, match_query_str,
column_name);
@@ -459,52 +699,23 @@ Status FunctionMatchPhrasePrefix::execute_match(
return Status::OK();
}
- int32_t current_src_array_offset = 0;
+ StreamingPhraseMatcher matcher(query_tokens, PhraseMode::PREFIX);
for (int i = 0; i < input_rows_count; i++) {
- auto data_tokens = analyse_data_token(column_name, analyzer_ctx,
string_col, i,
- array_offsets,
current_src_array_offset);
-
- int64_t dis_count = data_tokens.size() - query_tokens.size();
- if (dis_count < 0) {
- continue;
- }
-
- for (size_t j = 0; j < dis_count + 1; j++) {
- if (data_tokens[j].get_single_term() ==
query_tokens[0].get_single_term() ||
- query_tokens.size() == 1) {
- bool match = true;
- for (size_t k = 0; k < query_tokens.size(); k++) {
- const std::string& data_token = data_tokens[j +
k].get_single_term();
- const std::string& query_token =
query_tokens[k].get_single_term();
- if (k == query_tokens.size() - 1) {
- if (data_token.compare(0, query_token.size(),
query_token) != 0) {
- match = false;
- break;
- }
- } else {
- if (data_token != query_token) {
- match = false;
- break;
- }
- }
- }
- if (match) {
- result[i] = true;
- break;
- }
- }
+ if (match_phrase_data_tokens(*this, column_name, analyzer_ctx,
string_col, i, array_offsets,
+ array_element_null_map, matcher)) {
+ result[i] = true;
}
}
return Status::OK();
}
-Status FunctionMatchRegexp::execute_match(FunctionContext* context, const
std::string& column_name,
- const std::string& match_query_str,
- size_t input_rows_count, const
ColumnString* string_col,
- const InvertedIndexAnalyzerCtx*
analyzer_ctx,
- const ColumnArray::Offsets64*
array_offsets,
- ColumnUInt8::Container& result)
const {
+Status FunctionMatchRegexp::execute_match(
+ FunctionContext* context, const std::string& column_name,
+ const std::string& match_query_str, size_t input_rows_count, const
ColumnString* string_col,
+ const InvertedIndexAnalyzerCtx* analyzer_ctx, const
ColumnArray::Offsets64* array_offsets,
+ ColumnUInt8::Container& result,
+ const ColumnUInt8::Container* array_element_null_map) const {
RETURN_IF_ERROR(check(context, name));
VLOG_DEBUG << "begin to run FunctionMatchRegexp::execute_match,
parser_type: "
@@ -544,25 +755,25 @@ Status
FunctionMatchRegexp::execute_match(FunctionContext* context, const std::s
};
try {
- auto current_src_array_offset = 0;
for (int i = 0; i < input_rows_count; i++) {
- auto data_tokens = analyse_data_token(column_name, analyzer_ctx,
string_col, i,
- array_offsets,
current_src_array_offset);
-
- for (auto& input : data_tokens) {
- bool is_match = false;
- const auto& input_str = input.get_single_term();
- if (hs_scan(database, input_str.data(),
(uint32_t)input_str.size(), 0, scratch,
- on_match, (void*)&is_match) != HS_SUCCESS) {
- LOG(ERROR) << "hyperscan match failed: " << input_str;
- break;
- }
-
- if (is_match) {
- result[i] = true;
- break;
- }
- }
+ for_each_data_element_tokens(
+ *this, column_name, analyzer_ctx, string_col, i,
array_offsets,
+ array_element_null_map, [&](std::span<const
segment_v2::TermInfo> data_tokens) {
+ for (const auto& input : data_tokens) {
+ bool is_match = false;
+ const auto& input_str = input.get_single_term();
+ if (hs_scan(database, input_str.data(),
(uint32_t)input_str.size(), 0,
+ scratch, on_match, (void*)&is_match)
!= HS_SUCCESS) {
+ LOG(ERROR) << "hyperscan match failed: " <<
input_str;
+ return true;
+ }
+ if (is_match) {
+ result[i] = true;
+ return true;
+ }
+ }
+ return false;
+ });
}
}
_CLFINALLY({
@@ -577,7 +788,8 @@ Status FunctionMatchPhraseEdge::execute_match(
FunctionContext* context, const std::string& column_name,
const std::string& match_query_str, size_t input_rows_count, const
ColumnString* string_col,
const InvertedIndexAnalyzerCtx* analyzer_ctx, const
ColumnArray::Offsets64* array_offsets,
- ColumnUInt8::Container& result) const {
+ ColumnUInt8::Container& result,
+ const ColumnUInt8::Container* array_element_null_map) const {
RETURN_IF_ERROR(check(context, name));
auto query_tokens = analyse_query_str_token(analyzer_ctx, match_query_str,
column_name);
@@ -591,49 +803,11 @@ Status FunctionMatchPhraseEdge::execute_match(
return Status::OK();
}
- int32_t current_src_array_offset = 0;
+ StreamingPhraseMatcher matcher(query_tokens, PhraseMode::EDGE);
for (int i = 0; i < input_rows_count; i++) {
- auto data_tokens = analyse_data_token(column_name, analyzer_ctx,
string_col, i,
- array_offsets,
current_src_array_offset);
-
- int64_t dis_count = data_tokens.size() - query_tokens.size();
- if (dis_count < 0) {
- continue;
- }
-
- for (size_t j = 0; j < dis_count + 1; j++) {
- bool match = true;
- if (query_tokens.size() == 1) {
- if
(data_tokens[j].get_single_term().find(query_tokens[0].get_single_term()) ==
- std::string::npos) {
- match = false;
- }
- } else {
- for (size_t k = 0; k < query_tokens.size(); k++) {
- const std::string& data_token = data_tokens[j +
k].get_single_term();
- const std::string& query_token =
query_tokens[k].get_single_term();
- if (k == 0) {
- if (!data_token.ends_with(query_token)) {
- match = false;
- break;
- }
- } else if (k == query_tokens.size() - 1) {
- if (!data_token.starts_with(query_token)) {
- match = false;
- break;
- }
- } else {
- if (data_token != query_token) {
- match = false;
- break;
- }
- }
- }
- }
- if (match) {
- result[i] = true;
- break;
- }
+ if (match_phrase_data_tokens(*this, column_name, analyzer_ctx,
string_col, i, array_offsets,
+ array_element_null_map, matcher)) {
+ result[i] = true;
}
}
diff --git a/be/src/exprs/function/match.h b/be/src/exprs/function/match.h
index 84d0e4374bf..a4818360ddb 100644
--- a/be/src/exprs/function/match.h
+++ b/be/src/exprs/function/match.h
@@ -72,12 +72,12 @@ public:
Status execute_impl(FunctionContext* context, Block& block, const
ColumnNumbers& arguments,
uint32_t result, size_t input_rows_count) const
override;
- virtual Status execute_match(FunctionContext* context, const std::string&
column_name,
- const std::string& match_query_str, size_t
input_rows_count,
- const ColumnString* string_col,
- const InvertedIndexAnalyzerCtx* analyzer_ctx,
- const ColumnArray::Offsets64* array_offsets,
- ColumnUInt8::Container& result) const = 0;
+ virtual Status execute_match(
+ FunctionContext* context, const std::string& column_name,
+ const std::string& match_query_str, size_t input_rows_count,
+ const ColumnString* string_col, const InvertedIndexAnalyzerCtx*
analyzer_ctx,
+ const ColumnArray::Offsets64* array_offsets,
ColumnUInt8::Container& result,
+ const ColumnUInt8::Container* array_element_null_map = nullptr)
const = 0;
doris::segment_v2::InvertedIndexQueryType get_query_type_from_fn_name()
const;
@@ -87,8 +87,9 @@ public:
std::vector<segment_v2::TermInfo> analyse_data_token(
const std::string& column_name, const InvertedIndexAnalyzerCtx*
analyzer_ctx,
- const ColumnString* string_col, int32_t current_block_row_idx,
- const ColumnArray::Offsets64* array_offsets, int32_t&
current_src_array_offset) const;
+ const ColumnString* string_col, size_t current_block_row_idx,
+ const ColumnArray::Offsets64* array_offsets, int32_t&
current_src_array_offset,
+ const ColumnUInt8::Container* array_element_null_map = nullptr)
const;
Status check(FunctionContext* context, const std::string& function_name)
const;
@@ -107,12 +108,12 @@ public:
String get_name() const override { return name; }
- Status execute_match(FunctionContext* context, const std::string&
column_name,
- const std::string& match_query_str, size_t
input_rows_count,
- const ColumnString* string_col,
- const InvertedIndexAnalyzerCtx* analyzer_ctx,
- const ColumnArray::Offsets64* array_offsets,
- ColumnUInt8::Container& result) const override;
+ Status execute_match(
+ FunctionContext* context, const std::string& column_name,
+ const std::string& match_query_str, size_t input_rows_count,
+ const ColumnString* string_col, const InvertedIndexAnalyzerCtx*
analyzer_ctx,
+ const ColumnArray::Offsets64* array_offsets,
ColumnUInt8::Container& result,
+ const ColumnUInt8::Container* array_element_null_map = nullptr)
const override;
};
class FunctionMatchAll : public FunctionMatchBase {
@@ -122,12 +123,12 @@ public:
String get_name() const override { return name; }
- Status execute_match(FunctionContext* context, const std::string&
column_name,
- const std::string& match_query_str, size_t
input_rows_count,
- const ColumnString* string_col,
- const InvertedIndexAnalyzerCtx* analyzer_ctx,
- const ColumnArray::Offsets64* array_offsets,
- ColumnUInt8::Container& result) const override;
+ Status execute_match(
+ FunctionContext* context, const std::string& column_name,
+ const std::string& match_query_str, size_t input_rows_count,
+ const ColumnString* string_col, const InvertedIndexAnalyzerCtx*
analyzer_ctx,
+ const ColumnArray::Offsets64* array_offsets,
ColumnUInt8::Container& result,
+ const ColumnUInt8::Container* array_element_null_map = nullptr)
const override;
};
class FunctionMatchPhrase : public FunctionMatchBase {
@@ -137,12 +138,12 @@ public:
String get_name() const override { return name; }
- Status execute_match(FunctionContext* context, const std::string&
column_name,
- const std::string& match_query_str, size_t
input_rows_count,
- const ColumnString* string_col,
- const InvertedIndexAnalyzerCtx* analyzer_ctx,
- const ColumnArray::Offsets64* array_offsets,
- ColumnUInt8::Container& result) const override;
+ Status execute_match(
+ FunctionContext* context, const std::string& column_name,
+ const std::string& match_query_str, size_t input_rows_count,
+ const ColumnString* string_col, const InvertedIndexAnalyzerCtx*
analyzer_ctx,
+ const ColumnArray::Offsets64* array_offsets,
ColumnUInt8::Container& result,
+ const ColumnUInt8::Container* array_element_null_map = nullptr)
const override;
};
class FunctionMatchPhrasePrefix : public FunctionMatchBase {
@@ -152,12 +153,12 @@ public:
String get_name() const override { return name; }
- Status execute_match(FunctionContext* context, const std::string&
column_name,
- const std::string& match_query_str, size_t
input_rows_count,
- const ColumnString* string_col,
- const InvertedIndexAnalyzerCtx* analyzer_ctx,
- const ColumnArray::Offsets64* array_offsets,
- ColumnUInt8::Container& result) const override;
+ Status execute_match(
+ FunctionContext* context, const std::string& column_name,
+ const std::string& match_query_str, size_t input_rows_count,
+ const ColumnString* string_col, const InvertedIndexAnalyzerCtx*
analyzer_ctx,
+ const ColumnArray::Offsets64* array_offsets,
ColumnUInt8::Container& result,
+ const ColumnUInt8::Container* array_element_null_map = nullptr)
const override;
};
class FunctionMatchRegexp : public FunctionMatchBase {
@@ -167,12 +168,12 @@ public:
String get_name() const override { return name; }
- Status execute_match(FunctionContext* context, const std::string&
column_name,
- const std::string& match_query_str, size_t
input_rows_count,
- const ColumnString* string_col,
- const InvertedIndexAnalyzerCtx* analyzer_ctx,
- const ColumnArray::Offsets64* array_offsets,
- ColumnUInt8::Container& result) const override;
+ Status execute_match(
+ FunctionContext* context, const std::string& column_name,
+ const std::string& match_query_str, size_t input_rows_count,
+ const ColumnString* string_col, const InvertedIndexAnalyzerCtx*
analyzer_ctx,
+ const ColumnArray::Offsets64* array_offsets,
ColumnUInt8::Container& result,
+ const ColumnUInt8::Container* array_element_null_map = nullptr)
const override;
};
class FunctionMatchPhraseEdge : public FunctionMatchBase {
@@ -182,12 +183,12 @@ public:
String get_name() const override { return name; }
- Status execute_match(FunctionContext* context, const std::string&
column_name,
- const std::string& match_query_str, size_t
input_rows_count,
- const ColumnString* string_col,
- const InvertedIndexAnalyzerCtx* analyzer_ctx,
- const ColumnArray::Offsets64* array_offsets,
- ColumnUInt8::Container& result) const override;
+ Status execute_match(
+ FunctionContext* context, const std::string& column_name,
+ const std::string& match_query_str, size_t input_rows_count,
+ const ColumnString* string_col, const InvertedIndexAnalyzerCtx*
analyzer_ctx,
+ const ColumnArray::Offsets64* array_offsets,
ColumnUInt8::Container& result,
+ const ColumnUInt8::Container* array_element_null_map = nullptr)
const override;
};
} // namespace doris
diff --git a/be/test/exprs/function/function_match_test.cpp
b/be/test/exprs/function/function_match_test.cpp
index 0f06b29e2a3..4758e82f05e 100644
--- a/be/test/exprs/function/function_match_test.cpp
+++ b/be/test/exprs/function/function_match_test.cpp
@@ -17,6 +17,7 @@
#include <gtest/gtest.h>
+#include <algorithm>
#include <atomic>
#include <chrono>
#include <memory>
@@ -25,6 +26,8 @@
#include <vector>
#include "core/block/block.h"
+#include "core/column/column_array.h"
+#include "core/column/column_nullable.h"
#include "core/column/column_string.h"
#include "core/column/column_vector.h"
#include "core/data_type/data_type_string.h"
@@ -470,6 +473,403 @@ TEST(FunctionMatchTest, array_offset_handling) {
}
}
+TEST(FunctionMatchTest, array_match_uses_tokens_from_all_elements) {
+ TQueryOptions query_options;
+ query_options.__set_enable_match_without_inverted_index(true);
+ RuntimeState runtime_state(query_options, TQueryGlobals {});
+ auto context = FunctionContext::create_context(&runtime_state, {}, {});
+ auto ctx =
create_inverted_index_ctx(InvertedIndexParserType::PARSER_ENGLISH);
+
+ auto string_col = ColumnString::create();
+ for (const std::string value : {"alpha token", "tail only", "head only",
"omega token",
+ "unrelated head", "unrelated tail", "alpha
token", ""}) {
+ string_col->insert_data(value.data(), value.size());
+ }
+ ColumnArray::Offsets64 array_offsets = {2, 4, 6, 8, 8};
+
+ FunctionMatchAny match_any;
+ FunctionMatchAll match_all;
+ FunctionMatchRegexp match_regexp;
+ FunctionMatchPhrase match_phrase;
+ FunctionMatchPhrasePrefix match_phrase_prefix;
+ FunctionMatchPhraseEdge match_phrase_edge;
+ auto check_match = [&](const FunctionMatchBase& function, const
std::string& query,
+ const std::vector<uint8_t>& expected) {
+ SCOPED_TRACE(function.get_name() + ": " + query);
+ ColumnUInt8::Container result(array_offsets.size(), 0);
+ ASSERT_TRUE(function.execute_match(context.get(), "tags", query,
array_offsets.size(),
+ string_col.get(), ctx.ctx.get(),
&array_offsets, result)
+ .ok());
+ for (size_t row = 0; row < expected.size(); ++row) {
+ EXPECT_EQ(result[row], expected[row]) << "row: " << row;
+ }
+ };
+ check_match(match_any, "alpha", {1, 0, 0, 1, 0});
+ check_match(match_any, "omega", {0, 1, 0, 0, 0});
+ check_match(match_all, "alpha tail", {1, 0, 0, 0, 0});
+ check_match(match_all, "alpha alpha", {1, 0, 0, 1, 0});
+ check_match(match_regexp, "^alpha$", {1, 0, 0, 1, 0});
+ check_match(match_phrase, "alpha token", {1, 0, 0, 1, 0});
+ check_match(match_phrase, "token tail", {1, 0, 0, 0, 0});
+ check_match(match_phrase_prefix, "alpha tok", {1, 0, 0, 1, 0});
+ check_match(match_phrase_prefix, "token tai", {1, 0, 0, 0, 0});
+ check_match(match_phrase_edge, "pha tok", {1, 0, 0, 1, 0});
+ check_match(match_phrase_edge, "ken tai", {1, 0, 0, 0, 0});
+
+ segment_v2::inverted_index::CustomAnalyzerConfig::Builder builder;
+ builder.with_tokenizer_config("keyword", {});
+ builder.add_token_filter_config("lowercase", {});
+ auto provider =
+
std::make_shared<segment_v2::inverted_index::CustomAnalyzerProvider>(builder.build());
+ ctx.ctx->parser_type = InvertedIndexParserType::PARSER_NONE;
+ ctx.ctx->analyzer_name = "custom_keyword_lowercase";
+ ctx.ctx->analyzer_provider = provider;
+ ctx.ctx->analyzer = provider->get_analyzer();
+ check_match(match_any, "ALPHA TOKEN", {1, 0, 0, 1, 0});
+ check_match(match_phrase, "ALPHA TOKEN", {1, 0, 0, 1, 0});
+
+ ctx = create_inverted_index_ctx(InvertedIndexParserType::PARSER_NONE);
+ check_match(match_any, "alpha token", {1, 0, 0, 1, 0});
+ check_match(match_all, "alpha token", {1, 0, 0, 1, 0});
+ check_match(match_phrase, "alpha token", {1, 0, 0, 1, 0});
+ check_match(match_phrase_prefix, "alpha", {1, 0, 0, 1, 0});
+ check_match(match_phrase_edge, "pha", {1, 0, 0, 1, 0});
+ check_match(match_any, "", {0, 0, 0, 1, 0});
+}
+
+TEST(FunctionMatchTest, array_phrase_spans_non_null_elements) {
+ TQueryOptions query_options;
+ query_options.__set_enable_match_without_inverted_index(true);
+ RuntimeState runtime_state(query_options, TQueryGlobals {});
+ auto context = FunctionContext::create_context(&runtime_state, {}, {});
+ auto ctx =
create_inverted_index_ctx(InvertedIndexParserType::PARSER_ENGLISH);
+
+ auto string_col = ColumnString::create();
+ for (const std::string value :
+ {"hello", "world", "hello", "hidden", "world", "hello", "there",
"world"}) {
+ string_col->insert_data(value.data(), value.size());
+ }
+ ColumnArray::Offsets64 offsets = {2, 5, 8};
+ ColumnUInt8::Container null_map = {0, 0, 0, 1, 0, 0, 0, 0};
+
+ auto check_match = [&](const FunctionMatchBase& function, const
std::string& query,
+ const std::vector<uint8_t>& expected) {
+ SCOPED_TRACE(function.get_name() + ": " + query);
+ ColumnUInt8::Container result(expected.size(), 0);
+ ASSERT_TRUE(function.execute_match(context.get(), "tags", query,
expected.size(),
+ string_col.get(), ctx.ctx.get(),
&offsets, result,
+ &null_map)
+ .ok());
+ for (size_t row = 0; row < expected.size(); ++row) {
+ EXPECT_EQ(result[row], expected[row]) << "row: " << row;
+ }
+ };
+ check_match(FunctionMatchPhrase {}, "hello world", {1, 1, 0});
+ check_match(FunctionMatchPhrasePrefix {}, "hello wor", {1, 1, 0});
+ check_match(FunctionMatchPhraseEdge {}, "llo wor", {1, 1, 0});
+ check_match(FunctionMatchPhrase {}, "hello hidden world", {0, 0, 0});
+}
+
+TEST(FunctionMatchTest, array_phrase_respects_analyzer_positions) {
+ TQueryOptions query_options;
+ query_options.__set_enable_match_without_inverted_index(true);
+ RuntimeState runtime_state(query_options, TQueryGlobals {});
+ auto context = FunctionContext::create_context(&runtime_state, {}, {});
+
+ segment_v2::inverted_index::Settings tokenizer_settings;
+ tokenizer_settings.set("tokenize_on_chars", "[whitespace]");
+ segment_v2::inverted_index::Settings delimiter_settings;
+ delimiter_settings.set("preserve_original", "true");
+ segment_v2::inverted_index::CustomAnalyzerConfig::Builder builder;
+ builder.with_tokenizer_config("char_group", tokenizer_settings);
+ builder.add_token_filter_config("word_delimiter", delimiter_settings);
+ auto provider =
+
std::make_shared<segment_v2::inverted_index::CustomAnalyzerProvider>(builder.build());
+ auto ctx = std::make_unique<InvertedIndexAnalyzerCtx>();
+ ctx->analyzer_name = "word_delimiter_with_original";
+ ctx->analyzer_provider = provider;
+ ctx->analyzer = provider->get_analyzer();
+
+ auto string_col = ColumnString::create();
+ string_col->insert_data("foo-bar", 7);
+ string_col->insert_data("baz", 3);
+ ColumnArray::Offsets64 offsets = {2};
+
+ FunctionMatchPhrase match_phrase;
+ int32_t unused_offset = 0;
+ auto data_tokens = match_phrase.analyse_data_token("tags", ctx.get(),
string_col.get(), 0,
+ nullptr, unused_offset);
+ ASSERT_EQ(data_tokens.size(), 3);
+ EXPECT_EQ(data_tokens[0].position, data_tokens[1].position);
+ auto query_tokens = match_phrase.analyse_query_str_token(ctx.get(),
"foo-bar baz", "tags");
+ ASSERT_EQ(query_tokens.size(), 4);
+
+ ColumnUInt8::Container result(1, 0);
+ ASSERT_TRUE(match_phrase
+ .execute_match(context.get(), "tags", "foo-bar baz",
1, string_col.get(),
+ ctx.get(), &offsets, result)
+ .ok());
+ EXPECT_EQ(result[0], 0);
+ result[0] = 0;
+ ASSERT_TRUE(match_phrase
+ .execute_match(context.get(), "tags", "foo bar baz",
1, string_col.get(),
+ ctx.get(), &offsets, result)
+ .ok());
+ EXPECT_EQ(result[0], 1);
+}
+
+TEST(FunctionMatchTest, array_phrase_preserves_leading_analyzer_gap) {
+ TQueryOptions query_options;
+ query_options.__set_enable_match_without_inverted_index(true);
+ RuntimeState runtime_state(query_options, TQueryGlobals {});
+ auto context = FunctionContext::create_context(&runtime_state, {}, {});
+
+ segment_v2::inverted_index::Settings tokenizer_settings;
+ tokenizer_settings.set("tokenize_on_chars", "[whitespace]");
+ segment_v2::inverted_index::CustomAnalyzerConfig::Builder builder;
+ builder.with_tokenizer_config("char_group", tokenizer_settings);
+ builder.add_token_filter_config("word_delimiter", {});
+ auto provider =
+
std::make_shared<segment_v2::inverted_index::CustomAnalyzerProvider>(builder.build());
+ auto ctx = std::make_unique<InvertedIndexAnalyzerCtx>();
+ ctx->analyzer_name = "word_delimiter";
+ ctx->analyzer_provider = provider;
+ ctx->analyzer = provider->get_analyzer();
+
+ auto string_col = ColumnString::create();
+ string_col->insert_data("hello", 5);
+ string_col->insert_data("--- world", 9);
+ ColumnArray::Offsets64 offsets = {2};
+
+ FunctionMatchPhrase match_phrase;
+ int32_t unused_offset = 0;
+ auto later_tokens = match_phrase.analyse_data_token("tags", ctx.get(),
string_col.get(), 1,
+ nullptr,
unused_offset);
+ ASSERT_EQ(later_tokens.size(), 1);
+ EXPECT_EQ(later_tokens[0].position, 2);
+
+ ColumnUInt8::Container result(1, 0);
+ ASSERT_TRUE(match_phrase
+ .execute_match(context.get(), "tags", "hello world",
1, string_col.get(),
+ ctx.get(), &offsets, result)
+ .ok());
+ EXPECT_EQ(result[0], 0);
+}
+
+TEST(FunctionMatchTest, long_unrelated_phrase_miss) {
+ TQueryOptions query_options;
+ query_options.__set_enable_match_without_inverted_index(true);
+ RuntimeState runtime_state(query_options, TQueryGlobals {});
+ auto context = FunctionContext::create_context(&runtime_state, {}, {});
+ auto ctx =
create_inverted_index_ctx(InvertedIndexParserType::PARSER_ENGLISH);
+
+ std::string unrelated;
+ std::string matching;
+ for (int i = 0; i < 1000; ++i) {
+ unrelated += "gamma ";
+ matching += "alpha ";
+ }
+ auto string_col = ColumnString::create();
+ string_col->insert_data(unrelated.data(), unrelated.size());
+ string_col->insert_data(matching.data(), matching.size());
+ string_col->insert_data(unrelated.data(), unrelated.size());
+ ColumnUInt8::Container result(3, 0);
+ FunctionMatchPhrase match_phrase;
+ ASSERT_TRUE(match_phrase
+ .execute_match(context.get(), "tags", matching, 3,
string_col.get(),
+ ctx.ctx.get(), nullptr, result)
+ .ok());
+ EXPECT_EQ(result, (ColumnUInt8::Container {0, 1, 0}));
+}
+
+TEST(FunctionMatchTest, long_phrase_with_many_short_rows) {
+ TQueryOptions query_options;
+ query_options.__set_enable_match_without_inverted_index(true);
+ RuntimeState runtime_state(query_options, TQueryGlobals {});
+ auto context = FunctionContext::create_context(&runtime_state, {}, {});
+ auto ctx =
create_inverted_index_ctx(InvertedIndexParserType::PARSER_ENGLISH);
+
+ std::string query;
+ for (int i = 0; i < 1000; ++i) {
+ query += "alpha ";
+ }
+ auto string_col = ColumnString::create();
+ for (int i = 0; i < 2000; ++i) {
+ string_col->insert_data("alpha", 5);
+ }
+ ColumnUInt8::Container result(2000, 0);
+ FunctionMatchPhrase match_phrase;
+ ASSERT_TRUE(match_phrase
+ .execute_match(context.get(), "tags", query, 2000,
string_col.get(),
+ ctx.ctx.get(), nullptr, result)
+ .ok());
+ EXPECT_EQ(std::count(result.begin(), result.end(), 1), 0);
+}
+
+TEST(FunctionMatchTest, long_phrase_with_unique_start_miss) {
+ TQueryOptions query_options;
+ query_options.__set_enable_match_without_inverted_index(true);
+ RuntimeState runtime_state(query_options, TQueryGlobals {});
+ auto context = FunctionContext::create_context(&runtime_state, {}, {});
+ auto ctx =
create_inverted_index_ctx(InvertedIndexParserType::PARSER_ENGLISH);
+
+ std::string value = "beta ";
+ for (int i = 0; i < 1000; ++i) {
+ value += "alpha ";
+ }
+ auto string_col = ColumnString::create();
+ string_col->insert_data(value.data(), value.size());
+ ColumnUInt8::Container result(1, 0);
+ FunctionMatchPhrase match_phrase;
+ ASSERT_TRUE(match_phrase
+ .execute_match(context.get(), "tags", value + "gamma",
1, string_col.get(),
+ ctx.ctx.get(), nullptr, result)
+ .ok());
+ EXPECT_EQ(result[0], 0);
+}
+
+TEST(FunctionMatchTest, keyword_array_many_elements) {
+ TQueryOptions query_options;
+ query_options.__set_enable_match_without_inverted_index(true);
+ RuntimeState runtime_state(query_options, TQueryGlobals {});
+ auto context = FunctionContext::create_context(&runtime_state, {}, {});
+ auto ctx = create_inverted_index_ctx(InvertedIndexParserType::PARSER_NONE);
+
+ auto string_col = ColumnString::create();
+ for (int i = 0; i < 1000; ++i) {
+ string_col->insert_data("alpha", 5);
+ }
+ string_col->insert_data("omega", 5);
+ ColumnArray::Offsets64 offsets = {1000, 1001};
+ ColumnUInt8::Container result(2, 0);
+ FunctionMatchAny match_any;
+ ASSERT_TRUE(match_any
+ .execute_match(context.get(), "tags", "omega", 2,
string_col.get(),
+ ctx.ctx.get(), &offsets, result)
+ .ok());
+ EXPECT_EQ(result, (ColumnUInt8::Container {0, 1}));
+}
+
+TEST(FunctionMatchTest, long_repeated_phrase_miss) {
+ TQueryOptions query_options;
+ query_options.__set_enable_match_without_inverted_index(true);
+ RuntimeState runtime_state(query_options, TQueryGlobals {});
+ auto context = FunctionContext::create_context(&runtime_state, {}, {});
+ auto ctx =
create_inverted_index_ctx(InvertedIndexParserType::PARSER_ENGLISH);
+
+ std::string value;
+ for (int i = 0; i < 10000; ++i) {
+ value += "alpha ";
+ }
+ std::string query;
+ for (int i = 0; i < 999; ++i) {
+ query += "alpha ";
+ }
+ query += "beta";
+
+ auto string_col = ColumnString::create();
+ string_col->insert_data(value.data(), value.size());
+ ColumnUInt8::Container result(1, 0);
+ FunctionMatchPhrase match_phrase;
+ ASSERT_TRUE(match_phrase
+ .execute_match(context.get(), "tags", query, 1,
string_col.get(),
+ ctx.ctx.get(), nullptr, result)
+ .ok());
+ EXPECT_EQ(result[0], 0);
+}
+
+TEST(FunctionMatchTest, long_phrase_modes_cross_bitset_word) {
+ TQueryOptions query_options;
+ query_options.__set_enable_match_without_inverted_index(true);
+ RuntimeState runtime_state(query_options, TQueryGlobals {});
+ auto context = FunctionContext::create_context(&runtime_state, {}, {});
+ auto ctx =
create_inverted_index_ctx(InvertedIndexParserType::PARSER_ENGLISH);
+
+ std::string value;
+ std::string phrase;
+ std::string edge = "pha ";
+ for (int i = 0; i < 64; ++i) {
+ value += "alpha ";
+ phrase += "alpha ";
+ if (i > 0) {
+ edge += "alpha ";
+ }
+ }
+ value += "beta";
+ phrase += "beta";
+ edge += "bet";
+
+ auto string_col = ColumnString::create();
+ string_col->insert_data(value.data(), value.size());
+ auto check_match = [&](const FunctionMatchBase& function, const
std::string& query) {
+ SCOPED_TRACE(function.get_name());
+ ColumnUInt8::Container result(1, 0);
+ ASSERT_TRUE(function.execute_match(context.get(), "tags", query, 1,
string_col.get(),
+ ctx.ctx.get(), nullptr, result)
+ .ok());
+ EXPECT_EQ(result[0], 1);
+ };
+ check_match(FunctionMatchPhrase {}, phrase);
+ check_match(FunctionMatchPhrasePrefix {}, phrase.substr(0, phrase.size() -
1));
+ check_match(FunctionMatchPhraseEdge {}, edge);
+}
+
+TEST(FunctionMatchTest, null_array_elements_do_not_match_hidden_payload) {
+ TQueryOptions query_options;
+ query_options.__set_enable_match_without_inverted_index(true);
+ RuntimeState runtime_state(query_options, TQueryGlobals {});
+ auto array_type =
+
std::make_shared<DataTypeArray>(make_nullable(std::make_shared<DataTypeString>()));
+ auto string_type = std::make_shared<DataTypeString>();
+ auto context = FunctionContext::create_context(
+ &runtime_state, std::make_shared<DataTypeUInt8>(), {array_type,
string_type});
+ auto analyzer_ctx = std::make_shared<InvertedIndexAnalyzerCtx>();
+ analyzer_ctx->parser_type = InvertedIndexParserType::PARSER_NONE;
+ context->set_function_state(FunctionContext::FRAGMENT_LOCAL, analyzer_ctx);
+
+ auto check_match = [&](const FunctionMatchBase& function) {
+ auto values = ColumnString::create();
+ for (const std::string value : {"alpha", "tail", "alpha", "alpha",
"tail", "alpha"}) {
+ values->insert_data(value.data(), value.size());
+ }
+ auto null_map = ColumnUInt8::create();
+ const uint8_t nulls[] = {1, 0, 1, 0, 0, 1};
+ for (uint8_t is_null : nulls) {
+ null_map->insert_value(is_null);
+ }
+ auto offsets = ColumnArray::ColumnOffsets::create();
+ for (uint64_t offset : {2, 4, 6}) {
+ offsets->insert_value(offset);
+ }
+ auto query = ColumnString::create();
+ for (size_t row = 0; row < 3; ++row) {
+ query->insert_data("alpha", 5);
+ }
+
+ Block block;
+ block.insert(
+ {ColumnArray::create(ColumnNullable::create(std::move(values),
std::move(null_map)),
+ std::move(offsets)),
+ array_type, "tags"});
+ block.insert({std::move(query), string_type, "query"});
+ block.insert({nullptr, std::make_shared<DataTypeUInt8>(), "result"});
+ ASSERT_TRUE(function.execute_impl(context.get(), block, {0, 1}, 2,
3).ok());
+
+ const auto& result = assert_cast<const
ColumnUInt8&>(*block.get_by_position(2).column);
+ SCOPED_TRACE(function.get_name());
+ EXPECT_EQ(result.get_element(0), 0);
+ EXPECT_EQ(result.get_element(1), 1);
+ EXPECT_EQ(result.get_element(2), 0);
+ };
+
+ check_match(FunctionMatchAny {});
+ check_match(FunctionMatchAll {});
+ check_match(FunctionMatchRegexp {});
+ check_match(FunctionMatchPhrase {});
+ check_match(FunctionMatchPhrasePrefix {});
+ check_match(FunctionMatchPhraseEdge {});
+}
+
// Test Unicode and special character handling
TEST(FunctionMatchTest, unicode_and_special_chars) {
FunctionMatchAny match_any;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
