This is an automated email from the ASF dual-hosted git repository. kxiao pushed a commit to branch branch-2.0 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 844b33f55097eb9265063c88dd7e66221197a6a1 Author: airborne12 <[email protected]> AuthorDate: Fri Sep 1 09:41:53 2023 +0800 [Fix](column predicate) seperate CHAR primitive type for column predicate (#23581) --- be/src/exprs/bloom_filter_func.h | 1 + be/src/exprs/create_predicate_function.h | 20 +++ be/src/olap/like_column_predicate.cpp | 57 +++++--- be/src/olap/like_column_predicate.h | 1 + be/src/olap/reader.cpp | 11 +- be/src/olap/rowset/segment_v2/segment_iterator.cpp | 3 +- be/src/olap/schema.cpp | 7 + be/src/runtime/primitive_type.h | 6 +- be/src/vec/columns/column_dictionary.h | 11 +- be/src/vec/columns/predicate_column.h | 15 +- be/src/vec/functions/like.cpp | 158 +-------------------- be/src/vec/functions/like.h | 53 ------- be/src/vec/olap/block_reader.cpp | 11 -- be/src/vec/olap/block_reader.h | 2 - 14 files changed, 98 insertions(+), 258 deletions(-) diff --git a/be/src/exprs/bloom_filter_func.h b/be/src/exprs/bloom_filter_func.h index 1c77ce8e90..0a86a35990 100644 --- a/be/src/exprs/bloom_filter_func.h +++ b/be/src/exprs/bloom_filter_func.h @@ -408,6 +408,7 @@ struct FixedStringFindOp : public StringFindOp { const auto* value = reinterpret_cast<const StringRef*>(input_data); int64_t size = value->size; const char* data = value->data; + // CHAR type may pad the tail with \0, need to trim while (size > 0 && data[size - 1] == '\0') { size--; } diff --git a/be/src/exprs/create_predicate_function.h b/be/src/exprs/create_predicate_function.h index 4dbd1be7e2..a96565b55d 100644 --- a/be/src/exprs/create_predicate_function.h +++ b/be/src/exprs/create_predicate_function.h @@ -19,10 +19,12 @@ #include "exprs/hybrid_set.h" #include "exprs/minmax_predicate.h" +#include "function_filter.h" #include "olap/bitmap_filter_predicate.h" #include "olap/bloom_filter_predicate.h" #include "olap/column_predicate.h" #include "olap/in_list_predicate.h" +#include "olap/like_column_predicate.h" #include "runtime/define_primitive_type.h" namespace doris { @@ -257,6 +259,24 @@ ColumnPredicate* create_olap_column_predicate(uint32_t column_id, column->length()); } +template <PrimitiveType PT> +ColumnPredicate* create_olap_column_predicate(uint32_t column_id, + const std::shared_ptr<FunctionFilter>& filter, int, + const TabletColumn* column = nullptr) { + // currently only support like predicate + if constexpr (PT == TYPE_CHAR || PT == TYPE_VARCHAR || PT == TYPE_STRING) { + if constexpr (PT == TYPE_CHAR) { + return new LikeColumnPredicate<TYPE_CHAR>(filter->_opposite, column_id, filter->_fn_ctx, + filter->_string_param); + } else { + return new LikeColumnPredicate<TYPE_STRING>(filter->_opposite, column_id, + filter->_fn_ctx, filter->_string_param); + } + } else { + return nullptr; + } +} + template <typename T> ColumnPredicate* create_column_predicate(uint32_t column_id, const std::shared_ptr<T>& filter, FieldType type, int be_exec_version, diff --git a/be/src/olap/like_column_predicate.cpp b/be/src/olap/like_column_predicate.cpp index ef1374d445..1d20104ee5 100644 --- a/be/src/olap/like_column_predicate.cpp +++ b/be/src/olap/like_column_predicate.cpp @@ -22,29 +22,37 @@ #include "vec/columns/columns_number.h" #include "vec/columns/predicate_column.h" #include "vec/common/string_ref.h" +#include "vec/functions/like.h" namespace doris { -LikeColumnPredicate::LikeColumnPredicate(bool opposite, uint32_t column_id, - doris::FunctionContext* fn_ctx, doris::StringRef val) +template <PrimitiveType T> +LikeColumnPredicate<T>::LikeColumnPredicate(bool opposite, uint32_t column_id, + doris::FunctionContext* fn_ctx, doris::StringRef val) : ColumnPredicate(column_id, opposite), pattern(val) { + static_assert(T == TYPE_VARCHAR || T == TYPE_CHAR || T == TYPE_STRING, + "LikeColumnPredicate only supports the following types: TYPE_VARCHAR, TYPE_CHAR, " + "TYPE_STRING"); _state = reinterpret_cast<StateType*>( fn_ctx->get_function_state(doris::FunctionContext::THREAD_LOCAL)); _state->search_state.clone(_like_state); } -void LikeColumnPredicate::evaluate_vec(const vectorized::IColumn& column, uint16_t size, - bool* flags) const { +template <PrimitiveType T> +void LikeColumnPredicate<T>::evaluate_vec(const vectorized::IColumn& column, uint16_t size, + bool* flags) const { _evaluate_vec<false>(column, size, flags); } -void LikeColumnPredicate::evaluate_and_vec(const vectorized::IColumn& column, uint16_t size, - bool* flags) const { +template <PrimitiveType T> +void LikeColumnPredicate<T>::evaluate_and_vec(const vectorized::IColumn& column, uint16_t size, + bool* flags) const { _evaluate_vec<true>(column, size, flags); } -uint16_t LikeColumnPredicate::evaluate(const vectorized::IColumn& column, uint16_t* sel, - uint16_t size) const { +template <PrimitiveType T> +uint16_t LikeColumnPredicate<T>::evaluate(const vectorized::IColumn& column, uint16_t* sel, + uint16_t size) const { uint16_t new_size = 0; if (column.is_nullable()) { auto* nullable_col = vectorized::check_and_get_column<vectorized::ColumnNullable>(column); @@ -83,18 +91,18 @@ uint16_t LikeColumnPredicate::evaluate(const vectorized::IColumn& column, uint16 } } } else { - auto* str_col = - vectorized::check_and_get_column<vectorized::PredicateColumnType<TYPE_STRING>>( - nested_col); + auto* str_col = vectorized::check_and_get_column<vectorized::PredicateColumnType<T>>( + nested_col); if (!nullable_col->has_null()) { vectorized::ColumnUInt8::Container res(size, 0); - (_state->predicate_like_function)( - const_cast<vectorized::LikeSearchState*>(&_like_state), *str_col, pattern, - res, sel, size); for (uint16_t i = 0; i != size; i++) { uint16_t idx = sel[i]; sel[new_size] = idx; - new_size += _opposite ^ res[i]; + unsigned char flag = 0; + (_state->scalar_function)( + const_cast<vectorized::LikeSearchState*>(&_like_state), + str_col->get_data_at(idx), pattern, &flag); + new_size += _opposite ^ flag; } } else { for (uint16_t i = 0; i != size; i++) { @@ -105,7 +113,7 @@ uint16_t LikeColumnPredicate::evaluate(const vectorized::IColumn& column, uint16 continue; } - StringRef cell_value = str_col->get_data()[idx]; + StringRef cell_value = str_col->get_data_at(idx); unsigned char flag = 0; (_state->scalar_function)( const_cast<vectorized::LikeSearchState*>(&_like_state), @@ -130,21 +138,24 @@ uint16_t LikeColumnPredicate::evaluate(const vectorized::IColumn& column, uint16 new_size += _opposite ^ flag; } } else { - auto* str_col = - vectorized::check_and_get_column<vectorized::PredicateColumnType<TYPE_STRING>>( - column); + const vectorized::PredicateColumnType<T>* str_col = + vectorized::check_and_get_column<vectorized::PredicateColumnType<T>>(column); + vectorized::ColumnUInt8::Container res(size, 0); - (_state->predicate_like_function)( - const_cast<vectorized::LikeSearchState*>(&_like_state), *str_col, pattern, res, - sel, size); for (uint16_t i = 0; i != size; i++) { uint16_t idx = sel[i]; sel[new_size] = idx; - new_size += _opposite ^ res[i]; + unsigned char flag = 0; + (_state->scalar_function)(const_cast<vectorized::LikeSearchState*>(&_like_state), + str_col->get_data_at(idx), pattern, &flag); + new_size += _opposite ^ flag; } } } return new_size; } +template class LikeColumnPredicate<TYPE_CHAR>; +template class LikeColumnPredicate<TYPE_STRING>; + } //namespace doris diff --git a/be/src/olap/like_column_predicate.h b/be/src/olap/like_column_predicate.h index f97ff46453..dca6cf218a 100644 --- a/be/src/olap/like_column_predicate.h +++ b/be/src/olap/like_column_predicate.h @@ -47,6 +47,7 @@ namespace segment_v2 { class BitmapIndexIterator; } // namespace segment_v2 +template <PrimitiveType T> class LikeColumnPredicate : public ColumnPredicate { public: LikeColumnPredicate(bool opposite, uint32_t column_id, doris::FunctionContext* fn_ctx, diff --git a/be/src/olap/reader.cpp b/be/src/olap/reader.cpp index 8edc43cbb6..1fcd8b8985 100644 --- a/be/src/olap/reader.cpp +++ b/be/src/olap/reader.cpp @@ -505,7 +505,8 @@ Status TabletReader::_init_conditions_param(const ReaderParams& read_params) { // Function filter push down to storage engine auto is_like_predicate = [](ColumnPredicate* _pred) { - if (dynamic_cast<LikeColumnPredicate*>(_pred)) { + if (dynamic_cast<LikeColumnPredicate<TYPE_CHAR>*>(_pred) != nullptr || + dynamic_cast<LikeColumnPredicate<TYPE_STRING>*>(_pred) != nullptr) { return true; } @@ -598,10 +599,10 @@ ColumnPredicate* TabletReader::_parse_to_predicate(const FunctionFilter& functio if (index < 0) { return nullptr; } - - // currently only support like predicate - return new LikeColumnPredicate(function_filter._opposite, index, function_filter._fn_ctx, - function_filter._string_param); + const TabletColumn& column = _tablet_schema->column(index); + return create_column_predicate(index, std::make_shared<FunctionFilter>(function_filter), + column.type(), _reader_context.runtime_state->be_exec_version(), + &column); } Status TabletReader::_init_delete_condition(const ReaderParams& read_params) { diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp index ff367be869..8792c5474f 100644 --- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp +++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp @@ -685,7 +685,8 @@ bool SegmentIterator::_check_apply_by_inverted_index(ColumnPredicate* pred, bool } // Function filter no apply inverted index - if (dynamic_cast<LikeColumnPredicate*>(pred)) { + if (dynamic_cast<LikeColumnPredicate<TYPE_CHAR>*>(pred) != nullptr || + dynamic_cast<LikeColumnPredicate<TYPE_STRING>*>(pred) != nullptr) { return false; } diff --git a/be/src/olap/schema.cpp b/be/src/olap/schema.cpp index 95f6a47a70..d465db4f1d 100644 --- a/be/src/olap/schema.cpp +++ b/be/src/olap/schema.cpp @@ -171,6 +171,13 @@ vectorized::IColumn::MutablePtr Schema::get_predicate_column_ptr(const Field& fi ptr = doris::vectorized::PredicateColumnType<TYPE_DATETIME>::create(); break; case FieldType::OLAP_FIELD_TYPE_CHAR: + if (config::enable_low_cardinality_optimize && reader_type == ReaderType::READER_QUERY) { + ptr = doris::vectorized::ColumnDictionary<doris::vectorized::Int32>::create( + field.type()); + } else { + ptr = doris::vectorized::PredicateColumnType<TYPE_CHAR>::create(); + } + break; case FieldType::OLAP_FIELD_TYPE_VARCHAR: case FieldType::OLAP_FIELD_TYPE_STRING: if (config::enable_low_cardinality_optimize && reader_type == ReaderType::READER_QUERY) { diff --git a/be/src/runtime/primitive_type.h b/be/src/runtime/primitive_type.h index 980e4af954..4b87268697 100644 --- a/be/src/runtime/primitive_type.h +++ b/be/src/runtime/primitive_type.h @@ -89,6 +89,10 @@ constexpr bool is_string_type(PrimitiveType type) { return type == TYPE_CHAR || type == TYPE_VARCHAR || type == TYPE_STRING; } +constexpr bool is_variant_string_type(PrimitiveType type) { + return type == TYPE_VARCHAR || type == TYPE_STRING; +} + constexpr bool is_float_or_double(PrimitiveType type) { return type == TYPE_FLOAT || type == TYPE_DOUBLE; } @@ -114,7 +118,7 @@ TTypeDesc gen_type_desc(const TPrimitiveType::type val); TTypeDesc gen_type_desc(const TPrimitiveType::type val, const std::string& name); template <PrimitiveType type> -constexpr PrimitiveType PredicateEvaluateType = is_string_type(type) ? TYPE_STRING : type; +constexpr PrimitiveType PredicateEvaluateType = is_variant_string_type(type) ? TYPE_STRING : type; template <PrimitiveType type> struct PrimitiveTypeTraits; diff --git a/be/src/vec/columns/column_dictionary.h b/be/src/vec/columns/column_dictionary.h index e89bc64868..b6db5af7e9 100644 --- a/be/src/vec/columns/column_dictionary.h +++ b/be/src/vec/columns/column_dictionary.h @@ -318,7 +318,16 @@ public: if (is_dict_sorted() && !is_dict_code_converted()) { convert_dict_codes_if_necessary(); } - auto res = vectorized::PredicateColumnType<TYPE_STRING>::create(); + // if type is OLAP_FIELD_TYPE_CHAR, we need to construct TYPE_CHAR PredicateColumnType, + // because the string length will different from varchar and string which needed to be processed after. + auto create_column = [this]() -> MutableColumnPtr { + if (_type == FieldType::OLAP_FIELD_TYPE_CHAR) { + return vectorized::PredicateColumnType<TYPE_CHAR>::create(); + } + return vectorized::PredicateColumnType<TYPE_STRING>::create(); + }; + + auto res = create_column(); res->reserve(_codes.capacity()); for (size_t i = 0; i < _codes.size(); ++i) { auto& code = reinterpret_cast<T&>(_codes[i]); diff --git a/be/src/vec/columns/predicate_column.h b/be/src/vec/columns/predicate_column.h index 88662f43e6..655ef6d0e5 100644 --- a/be/src/vec/columns/predicate_column.h +++ b/be/src/vec/columns/predicate_column.h @@ -42,11 +42,10 @@ class PredicateColumnType final : public COWHelper<IColumn, PredicateColumnType< private: PredicateColumnType() {} PredicateColumnType(const size_t n) : data(n) {} + PredicateColumnType(const PredicateColumnType& src) : data(src.data.begin(), src.data.end()) {} friend class COWHelper<IColumn, PredicateColumnType<Type>>; using T = typename PredicatePrimitiveTypeTraits<Type>::PredicateFieldType; - PredicateColumnType(const PredicateColumnType& src) : data(src.data.begin(), src.data.end()) {} - uint64_t get_date_at(uint16_t idx) { const T val = data[idx]; const char* val_ptr = reinterpret_cast<const char*>(&val); @@ -163,8 +162,16 @@ public: size_t size() const override { return data.size(); } - [[noreturn]] StringRef get_data_at(size_t n) const override { - LOG(FATAL) << "get_data_at not supported in PredicateColumnType"; + StringRef get_data_at(size_t n) const override { + if constexpr (std::is_same_v<T, StringRef>) { + auto res = reinterpret_cast<const StringRef&>(data[n]); + if constexpr (Type == TYPE_CHAR) { + res.size = strnlen(res.data, res.size); + } + return res; + } else { + LOG(FATAL) << "should not call get_data_at in predicate column except for string type"; + } } void insert_from(const IColumn& src, size_t n) override { diff --git a/be/src/vec/functions/like.cpp b/be/src/vec/functions/like.cpp index 1a83dc8a87..8851b777fe 100644 --- a/be/src/vec/functions/like.cpp +++ b/be/src/vec/functions/like.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "vec/functions/like.h" +#include "like.h" #include <fmt/format.h> #include <hs/hs_compile.h> @@ -148,68 +148,6 @@ Status FunctionLikeBase::constant_substring_fn(LikeSearchState* state, const Col return Status::OK(); } -Status FunctionLikeBase::constant_allpass_fn_predicate(LikeSearchState* state, - const PredicateColumnType<TYPE_STRING>& val, - const StringRef& pattern, - ColumnUInt8::Container& result, - const uint16_t* sel, size_t sz) { - for (size_t i = 0; i < sz; i++) { - result[i] = 1; - } - return Status::OK(); -} - -Status FunctionLikeBase::constant_starts_with_fn_predicate( - LikeSearchState* state, const PredicateColumnType<TYPE_STRING>& val, - const StringRef& pattern, ColumnUInt8::Container& result, const uint16_t* sel, size_t sz) { - auto data_ptr = reinterpret_cast<const StringRef*>(val.get_data().data()); - for (size_t i = 0; i < sz; i++) { - result[i] = (data_ptr[sel[i]].size >= state->search_string_sv.size) && - (state->search_string_sv == - data_ptr[sel[i]].substring(0, state->search_string_sv.size)); - } - return Status::OK(); -} - -Status FunctionLikeBase::constant_ends_with_fn_predicate( - LikeSearchState* state, const PredicateColumnType<TYPE_STRING>& val, - const StringRef& pattern, ColumnUInt8::Container& result, const uint16_t* sel, size_t sz) { - auto data_ptr = reinterpret_cast<const StringRef*>(val.get_data().data()); - for (size_t i = 0; i < sz; i++) { - result[i] = - (data_ptr[sel[i]].size >= state->search_string_sv.size) && - (state->search_string_sv == - data_ptr[sel[i]].substring(data_ptr[sel[i]].size - state->search_string_sv.size, - state->search_string_sv.size)); - } - return Status::OK(); -} - -Status FunctionLikeBase::constant_equals_fn_predicate(LikeSearchState* state, - const PredicateColumnType<TYPE_STRING>& val, - const StringRef& pattern, - ColumnUInt8::Container& result, - const uint16_t* sel, size_t sz) { - auto data_ptr = reinterpret_cast<const StringRef*>(val.get_data().data()); - for (size_t i = 0; i < sz; i++) { - result[i] = (data_ptr[sel[i]] == state->search_string_sv); - } - return Status::OK(); -} - -Status FunctionLikeBase::constant_substring_fn_predicate( - LikeSearchState* state, const PredicateColumnType<TYPE_STRING>& val, - const StringRef& pattern, ColumnUInt8::Container& result, const uint16_t* sel, size_t sz) { - auto data_ptr = reinterpret_cast<const StringRef*>(val.get_data().data()); - for (size_t i = 0; i < sz; i++) { - if (state->search_string_sv.size == 0) { - result[i] = true; - } - result[i] = state->substring_pattern.search(data_ptr[sel[i]]) != -1; - } - return Status::OK(); -} - Status FunctionLikeBase::constant_allpass_fn_scalar(LikeSearchState* state, const StringRef& val, const StringRef& pattern, unsigned char* result) { @@ -351,75 +289,6 @@ Status FunctionLikeBase::regexp_fn(LikeSearchState* state, const ColumnString& v return Status::OK(); } -Status FunctionLikeBase::constant_regex_fn_predicate(LikeSearchState* state, - const PredicateColumnType<TYPE_STRING>& val, - const StringRef& pattern, - ColumnUInt8::Container& result, - const uint16_t* sel, size_t sz) { - auto data_ptr = reinterpret_cast<const StringRef*>(val.get_data().data()); - - if (state->hs_database) { // use hyperscan - for (size_t i = 0; i < sz; i++) { - auto ret = hs_scan(state->hs_database.get(), data_ptr[sel[i]].data, - data_ptr[sel[i]].size, 0, state->hs_scratch.get(), - doris::vectorized::LikeSearchState::hs_match_handler, - (void*)(result.data() + i)); - if (ret != HS_SUCCESS && ret != HS_SCAN_TERMINATED) { - return Status::RuntimeError(fmt::format("hyperscan error: {}", ret)); - } - } - } else { // fallback to re2 - for (size_t i = 0; i < sz; i++) { - *(result.data() + i) = RE2::PartialMatch( - re2::StringPiece(data_ptr[sel[i]].data, data_ptr[sel[i]].size), - *state->regex.get()); - } - } - - return Status::OK(); -} - -Status FunctionLikeBase::regexp_fn_predicate(LikeSearchState* state, - const PredicateColumnType<TYPE_STRING>& val, - const StringRef& pattern, - ColumnUInt8::Container& result, const uint16_t* sel, - size_t sz) { - std::string re_pattern(pattern.data, pattern.size); - - hs_database_t* database = nullptr; - hs_scratch_t* scratch = nullptr; - if (hs_prepare(nullptr, re_pattern.c_str(), &database, &scratch).ok()) { // use hyperscan - auto data_ptr = reinterpret_cast<const StringRef*>(val.get_data().data()); - for (size_t i = 0; i < sz; i++) { - auto ret = hs_scan(database, data_ptr[sel[i]].data, data_ptr[sel[i]].size, 0, scratch, - doris::vectorized::LikeSearchState::hs_match_handler, - (void*)(result.data() + i)); - if (ret != HS_SUCCESS && ret != HS_SCAN_TERMINATED) { - return Status::RuntimeError(fmt::format("hyperscan error: {}", ret)); - } - } - - hs_free_scratch(scratch); - hs_free_database(database); - } else { // fallback to re2 - RE2::Options opts; - opts.set_never_nl(false); - opts.set_dot_nl(true); - re2::RE2 re(re_pattern, opts); - if (re.ok()) { - auto data_ptr = reinterpret_cast<const StringRef*>(val.get_data().data()); - for (size_t i = 0; i < sz; i++) { - *(result.data() + i) = RE2::PartialMatch( - re2::StringPiece(data_ptr[sel[i]].data, data_ptr[sel[i]].size), re); - } - } else { - return Status::RuntimeError("Invalid pattern: {}", pattern.debug_string()); - } - } - - return Status::OK(); -} - // hyperscan compile expression to database and allocate scratch space Status FunctionLikeBase::hs_prepare(FunctionContext* context, const char* expression, hs_database_t** database, hs_scratch_t** scratch) { @@ -559,17 +428,6 @@ Status FunctionLike::like_fn(LikeSearchState* state, const ColumnString& val, return regexp_fn(state, val, {re_pattern.c_str(), re_pattern.size()}, result); } -Status FunctionLike::like_fn_predicate(LikeSearchState* state, - const PredicateColumnType<TYPE_STRING>& val, - const StringRef& pattern, ColumnUInt8::Container& result, - uint16_t* sel, size_t sz) { - std::string re_pattern; - convert_like_pattern(state, std::string(pattern.data, pattern.size), &re_pattern); - - return regexp_fn_predicate(state, val, {re_pattern.c_str(), re_pattern.size()}, result, sel, - sz); -} - Status FunctionLike::like_fn_scalar(LikeSearchState* state, const StringRef& val, const StringRef& pattern, unsigned char* result) { std::string re_pattern; @@ -680,7 +538,6 @@ Status FunctionLike::open(FunctionContext* context, FunctionContext::FunctionSta std::shared_ptr<LikeState> state = std::make_shared<LikeState>(); context->set_function_state(scope, state); state->function = like_fn; - state->predicate_like_function = like_fn_predicate; state->scalar_function = like_fn_scalar; if (context->is_col_constant(1)) { const auto pattern_col = context->get_constant_col(1)->column_ptr; @@ -693,7 +550,6 @@ Status FunctionLike::open(FunctionContext* context, FunctionContext::FunctionSta if (!pattern_str.empty() && RE2::FullMatch(pattern_str, LIKE_ALLPASS_RE)) { state->search_state.set_search_string(""); state->function = constant_allpass_fn; - state->predicate_like_function = constant_allpass_fn_predicate; state->scalar_function = constant_allpass_fn_scalar; } else if (pattern_str.empty() || RE2::FullMatch(pattern_str, LIKE_EQUALS_RE, &search_string)) { @@ -709,7 +565,6 @@ Status FunctionLike::open(FunctionContext* context, FunctionContext::FunctionSta } state->search_state.set_search_string(search_string); state->function = constant_equals_fn; - state->predicate_like_function = constant_equals_fn_predicate; state->scalar_function = constant_equals_fn_scalar; } else if (RE2::FullMatch(pattern_str, LIKE_STARTS_WITH_RE, &search_string)) { if (VLOG_DEBUG_IS_ON) { @@ -724,7 +579,6 @@ Status FunctionLike::open(FunctionContext* context, FunctionContext::FunctionSta } state->search_state.set_search_string(search_string); state->function = constant_starts_with_fn; - state->predicate_like_function = constant_starts_with_fn_predicate; state->scalar_function = constant_starts_with_fn_scalar; } else if (RE2::FullMatch(pattern_str, LIKE_ENDS_WITH_RE, &search_string)) { if (VLOG_DEBUG_IS_ON) { @@ -739,7 +593,6 @@ Status FunctionLike::open(FunctionContext* context, FunctionContext::FunctionSta } state->search_state.set_search_string(search_string); state->function = constant_ends_with_fn; - state->predicate_like_function = constant_ends_with_fn_predicate; state->scalar_function = constant_ends_with_fn_scalar; } else if (RE2::FullMatch(pattern_str, LIKE_SUBSTRING_RE, &search_string)) { if (VLOG_DEBUG_IS_ON) { @@ -754,7 +607,6 @@ Status FunctionLike::open(FunctionContext* context, FunctionContext::FunctionSta } state->search_state.set_search_string(search_string); state->function = constant_substring_fn; - state->predicate_like_function = constant_substring_fn_predicate; state->scalar_function = constant_substring_fn_scalar; } else { std::string re_pattern; @@ -787,7 +639,6 @@ Status FunctionLike::open(FunctionContext* context, FunctionContext::FunctionSta } state->function = constant_regex_fn; - state->predicate_like_function = constant_regex_fn_predicate; state->scalar_function = constant_regex_fn_scalar; } } @@ -801,7 +652,6 @@ Status FunctionRegexp::open(FunctionContext* context, FunctionContext::FunctionS std::shared_ptr<LikeState> state = std::make_shared<LikeState>(); context->set_function_state(scope, state); state->function = regexp_fn; - state->predicate_like_function = regexp_fn_predicate; state->scalar_function = regexp_fn_scalar; if (context->is_col_constant(1)) { const auto pattern_col = context->get_constant_col(1)->column_ptr; @@ -812,27 +662,22 @@ Status FunctionRegexp::open(FunctionContext* context, FunctionContext::FunctionS if (RE2::FullMatch(pattern_str, ALLPASS_RE)) { state->search_state.set_search_string(""); state->function = constant_allpass_fn; - state->predicate_like_function = constant_allpass_fn_predicate; state->scalar_function = constant_allpass_fn_scalar; } else if (RE2::FullMatch(pattern_str, EQUALS_RE, &search_string)) { state->search_state.set_search_string(search_string); state->function = constant_equals_fn; - state->predicate_like_function = constant_equals_fn_predicate; state->scalar_function = constant_equals_fn_scalar; } else if (RE2::FullMatch(pattern_str, STARTS_WITH_RE, &search_string)) { state->search_state.set_search_string(search_string); state->function = constant_starts_with_fn; - state->predicate_like_function = constant_starts_with_fn_predicate; state->scalar_function = constant_starts_with_fn_scalar; } else if (RE2::FullMatch(pattern_str, ENDS_WITH_RE, &search_string)) { state->search_state.set_search_string(search_string); state->function = constant_ends_with_fn; - state->predicate_like_function = constant_ends_with_fn_predicate; state->scalar_function = constant_ends_with_fn_scalar; } else if (RE2::FullMatch(pattern_str, SUBSTRING_RE, &search_string)) { state->search_state.set_search_string(search_string); state->function = constant_substring_fn; - state->predicate_like_function = constant_substring_fn_predicate; state->scalar_function = constant_substring_fn_scalar; } else { hs_database_t* database = nullptr; @@ -855,7 +700,6 @@ Status FunctionRegexp::open(FunctionContext* context, FunctionContext::FunctionS } } state->function = constant_regex_fn; - state->predicate_like_function = constant_regex_fn_predicate; state->scalar_function = constant_regex_fn_scalar; } } diff --git a/be/src/vec/functions/like.h b/be/src/vec/functions/like.h index 07309be5b7..3726518a9b 100644 --- a/be/src/vec/functions/like.h +++ b/be/src/vec/functions/like.h @@ -115,18 +115,12 @@ struct LikeSearchState { using LikeFn = std::function<doris::Status(LikeSearchState*, const ColumnString&, const StringRef&, ColumnUInt8::Container&)>; -using LikePredicateFn = std::function<doris::Status( - LikeSearchState*, const PredicateColumnType<TYPE_STRING>&, const StringRef&, - ColumnUInt8::Container&, uint16_t* sel, size_t sz)>; - using ScalarLikeFn = std::function<doris::Status(LikeSearchState*, const StringRef&, const StringRef&, unsigned char*)>; struct LikeState { LikeSearchState search_state; LikeFn function; - // Two functions below are used only for predicate. - LikePredicateFn predicate_like_function; ScalarLikeFn scalar_function; }; @@ -173,48 +167,6 @@ protected: static Status regexp_fn(LikeSearchState* state, const ColumnString& val, const StringRef& pattern, ColumnUInt8::Container& result); - // These functions below are used only for predicate. - static Status constant_regex_fn_predicate(LikeSearchState* state, - const PredicateColumnType<TYPE_STRING>& val, - const StringRef& pattern, - ColumnUInt8::Container& result, const uint16_t* sel, - size_t sz); - - static Status regexp_fn_predicate(LikeSearchState* state, - const PredicateColumnType<TYPE_STRING>& val, - const StringRef& pattern, ColumnUInt8::Container& result, - const uint16_t* sel, size_t sz); - - static Status constant_allpass_fn_predicate(LikeSearchState* state, - const PredicateColumnType<TYPE_STRING>& val, - const StringRef& pattern, - ColumnUInt8::Container& result, const uint16_t* sel, - size_t sz); - - static Status constant_starts_with_fn_predicate(LikeSearchState* state, - const PredicateColumnType<TYPE_STRING>& val, - const StringRef& pattern, - ColumnUInt8::Container& result, - const uint16_t* sel, size_t sz); - - static Status constant_ends_with_fn_predicate(LikeSearchState* state, - const PredicateColumnType<TYPE_STRING>& val, - const StringRef& pattern, - ColumnUInt8::Container& result, - const uint16_t* sel, size_t sz); - - static Status constant_equals_fn_predicate(LikeSearchState* state, - const PredicateColumnType<TYPE_STRING>& val, - const StringRef& pattern, - ColumnUInt8::Container& result, const uint16_t* sel, - size_t sz); - - static Status constant_substring_fn_predicate(LikeSearchState* state, - const PredicateColumnType<TYPE_STRING>& val, - const StringRef& pattern, - ColumnUInt8::Container& result, - const uint16_t* sel, size_t sz); - static Status constant_allpass_fn_scalar(LikeSearchState* state, const StringRef& val, const StringRef& pattern, unsigned char* result); @@ -257,11 +209,6 @@ private: static Status like_fn(LikeSearchState* state, const ColumnString& val, const StringRef& pattern, ColumnUInt8::Container& result); - static Status like_fn_predicate(LikeSearchState* state, - const PredicateColumnType<TYPE_STRING>& val, - const StringRef& pattern, ColumnUInt8::Container& result, - uint16_t* sel, size_t sz); - static Status like_fn_scalar(LikeSearchState* state, const StringRef& val, const StringRef& pattern, unsigned char* result); diff --git a/be/src/vec/olap/block_reader.cpp b/be/src/vec/olap/block_reader.cpp index 80fffd4401..2342c9faf9 100644 --- a/be/src/vec/olap/block_reader.cpp +++ b/be/src/vec/olap/block_reader.cpp @@ -503,15 +503,4 @@ bool BlockReader::_get_next_row_same() { } } -ColumnPredicate* BlockReader::_parse_to_predicate(const FunctionFilter& function_filter) { - int32_t index = _tablet_schema->field_index(function_filter._col_name); - if (index < 0) { - return nullptr; - } - - // currently only support like predicate - return new LikeColumnPredicate(function_filter._opposite, index, function_filter._fn_ctx, - function_filter._string_param); -} - } // namespace doris::vectorized diff --git a/be/src/vec/olap/block_reader.h b/be/src/vec/olap/block_reader.h index 0fe188419e..8184409945 100644 --- a/be/src/vec/olap/block_reader.h +++ b/be/src/vec/olap/block_reader.h @@ -57,8 +57,6 @@ public: return _vcollect_iter.update_profile(profile); } - ColumnPredicate* _parse_to_predicate(const FunctionFilter& function_filter) override; - private: // Directly read row from rowset and pass to upper caller. No need to do aggregation. // This is usually used for DUPLICATE KEY tables --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
