This is an automated email from the ASF dual-hosted git repository.
HappenLee pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 9a48f8120c0 [fix](be) Fall back from expensive Hyperscan bounded
repeats (#66788)
9a48f8120c0 is described below
commit 9a48f8120c013958b4ebba83bf90b6c9d2a5b43f
Author: HappenLee <[email protected]>
AuthorDate: Tue Aug 18 19:00:24 2026 +0800
[fix](be) Fall back from expensive Hyperscan bounded repeats (#66788)
Problem Summary:
Hyperscan can spend excessive time and memory compiling bounded
repetitions with large limits. Doris now detects regex quantifiers above
50 before every direct Hyperscan compiler entry point. LIKE and REGEXP
paths with a compatible fallback use RE2; compiler paths without a
fallback reject the pattern before Hyperscan compilation.
The guard follows ClickHouse SlowWithHyperscanChecker and searches the
raw regular expression. This intentionally uses the same conservative
best-effort classification instead of maintaining a partial Hyperscan
grammar parser.
The new `enable_hyperscan_fallback` session variable controls
strictness. It defaults to `true`, preserving fallback behavior. When
set to `false`, Doris returns the interception or Hyperscan compilation
error instead of falling back.
Normal query execution and request-scoped synchronous Stream Load
propagate this option through `TQueryOptions`. Asynchronous Broker Load
and Routine Load session inheritance are outside this PR scope and
retain their historical option defaults.
### Release note
Add `enable_hyperscan_fallback`. It defaults to `true`; setting it to
`false` returns an error when Hyperscan cannot compile or safely process
the regular expression on supported request-scoped paths.
### Check List (For Author)
- Test:
- Targeted BE unit tests for LIKE/REGEXP, MATCH_REGEXP, multi-match, and
inverted-index regexp v1/v2
- `DORIS_THIRDPARTY=$PWD/thirdparty FE_UT_PARALLEL=48 ./run-fe-ut.sh
--run
org.apache.doris.nereids.load.NereidsLoadScanProviderTest,org.apache.doris.qe.SessionVariablesTest`
(25 passed)
- `DORIS_THIRDPARTY=$PWD/thirdparty ./build.sh --fe -j 48`
- BE clang-format and clang-tidy
- Behavior changed: Yes. Expensive bounded repeats are intercepted
before Hyperscan compilation, and request-scoped sessions may disable
the RE2 fallback. The default remains fallback enabled.
- Does this need documentation: No
---
be/src/exprs/function/like.cpp | 46 +++++++--
be/src/exprs/function/like.h | 7 +-
be/src/exprs/function/match.cpp | 5 +
be/src/exprs/function/regexps.h | 4 +
.../storage/index/inverted/query/regexp_query.cpp | 4 +
.../query_v2/regexp_query/regexp_weight.cpp | 4 +
be/src/util/hyperscan_util.cpp | 93 ++++++++++++++++++
be/src/util/hyperscan_util.h | 29 ++++++
be/test/exprs/function/function_like_test.cpp | 107 +++++++++++++++++++++
be/test/exprs/function/function_match_test.cpp | 21 ++++
.../exprs/function/function_multi_match_test.cpp | 10 ++
.../index/inverted/query/regexp_query_test.cpp | 26 ++++-
.../index/inverted/query_v2/regexp_query_test.cpp | 17 +++-
.../java/org/apache/doris/qe/SessionVariable.java | 7 ++
.../org/apache/doris/qe/SessionVariablesTest.java | 11 +++
gensrc/thrift/PaloInternalService.thrift | 2 +
16 files changed, 383 insertions(+), 10 deletions(-)
diff --git a/be/src/exprs/function/like.cpp b/be/src/exprs/function/like.cpp
index 8bbfcf9b81d..938ecfb958f 100644
--- a/be/src/exprs/function/like.cpp
+++ b/be/src/exprs/function/like.cpp
@@ -19,7 +19,6 @@
#include <fmt/format.h>
#include <hs/hs_compile.h>
-#include <re2/stringpiece.h>
#include <cstddef>
#include <ostream>
@@ -34,8 +33,10 @@
#include "core/column/column_vector.h"
#include "core/string_ref.h"
#include "exprs/function/simple_function_factory.h"
+#include "util/hyperscan_util.h"
namespace doris {
+
// A regex to match any regex pattern is equivalent to a substring search.
static const RE2
SUBSTRING_RE(R"((?:\.\*)*([^\.\^\{\[\(\|\)\]\}\+\*\?\$\\]*)(?:\.\*)*)");
@@ -183,8 +184,9 @@ struct VectorEndsWithSearchState : public
VectorPatternSearchState {
}
};
-Status LikeSearchState::clone(LikeSearchState& cloned) {
+Status LikeSearchState::clone(LikeSearchState& cloned) const {
cloned.set_search_string(search_string);
+ cloned.enable_hyperscan_fallback = enable_hyperscan_fallback;
std::string re_pattern;
FunctionLike::convert_like_pattern(this, pattern_str, &re_pattern);
@@ -452,7 +454,8 @@ Status FunctionLikeBase::regexp_fn(const LikeSearchState*
state, const ColumnStr
hs_database_t* database = nullptr;
hs_scratch_t* scratch = nullptr;
- if (hs_prepare(nullptr, re_pattern.c_str(), &database, &scratch).ok()) {
// use hyperscan
+ auto hs_status = hs_prepare(nullptr, re_pattern.c_str(), &database,
&scratch);
+ if (hs_status.ok()) { // use hyperscan
auto sz = val.size();
for (size_t i = 0; i < sz; i++) {
const auto& str_ref = val.get_data_at(i);
@@ -467,6 +470,9 @@ Status FunctionLikeBase::regexp_fn(const LikeSearchState*
state, const ColumnStr
hs_free_scratch(scratch);
hs_free_database(database);
} else { // fallback to re2
+ if (!state->enable_hyperscan_fallback) {
+ return hs_status;
+ }
RE2::Options opts;
opts.set_never_nl(false);
opts.set_dot_nl(true);
@@ -487,8 +493,19 @@ Status FunctionLikeBase::regexp_fn(const LikeSearchState*
state, const ColumnStr
}
// hyperscan compile expression to database and allocate scratch space
+bool FunctionLikeBase::should_fallback_to_re2(std::string_view regexp) {
+ return is_hyperscan_regexp_expensive(regexp);
+}
+
Status FunctionLikeBase::hs_prepare(FunctionContext* context, const char*
expression,
hs_database_t** database, hs_scratch_t**
scratch) {
+ if (should_fallback_to_re2(expression)) {
+ *database = nullptr;
+ *scratch = nullptr;
+ // Callers either fall back to RE2 or return this status based on the
session variable.
+ return Status::RuntimeError<false>(HYPERSCAN_BOUNDED_REPEAT_ERROR);
+ }
+
hs_compile_error_t* compile_err;
auto res = hs_compile(expression, HS_FLAG_DOTALL | HS_FLAG_ALLOWEMPTY |
HS_FLAG_UTF8,
HS_MODE_BLOCK, nullptr, database, &compile_err);
@@ -497,7 +514,7 @@ Status FunctionLikeBase::hs_prepare(FunctionContext*
context, const char* expres
*database = nullptr;
std::string error_message = compile_err->message;
hs_free_compile_error(compile_err);
- // Do not call FunctionContext::set_error here, since we do not want
to cancel the query here.
+ // Callers either fall back to RE2 or return this status based on the
session variable.
return Status::RuntimeError<false>("hs_compile regex pattern error:" +
error_message);
}
hs_free_compile_error(compile_err);
@@ -506,7 +523,7 @@ Status FunctionLikeBase::hs_prepare(FunctionContext*
context, const char* expres
hs_free_database(*database);
*database = nullptr;
*scratch = nullptr;
- // Do not call FunctionContext::set_error here, since we do not want
to cancel the query here.
+ // Callers either fall back to RE2 or return this status based on the
session variable.
return Status::RuntimeError<false>("hs_alloc_scratch allocate scratch
space error");
}
@@ -942,12 +959,19 @@ Status
FunctionLike::construct_like_const_state(FunctionContext* context, const
hs_database_t* database = nullptr;
hs_scratch_t* scratch = nullptr;
- if (try_hyperscan && hs_prepare(context, re_pattern.c_str(),
&database, &scratch).ok()) {
+ Status hs_status;
+ if (try_hyperscan) {
+ hs_status = hs_prepare(context, re_pattern.c_str(), &database,
&scratch);
+ }
+ if (try_hyperscan && hs_status.ok()) {
// use hyperscan
state->search_state.hs_database.reset(database);
state->search_state.hs_scratch.reset(scratch);
} else {
// fallback to re2
+ if (try_hyperscan &&
!state->search_state.enable_hyperscan_fallback) {
+ return hs_status;
+ }
// reset hs_database to nullptr to indicate not use hyperscan
state->search_state.hs_database.reset();
state->search_state.hs_scratch.reset();
@@ -974,6 +998,8 @@ Status FunctionLike::open(FunctionContext* context,
FunctionContext::FunctionSta
}
std::shared_ptr<LikeState> state = std::make_shared<LikeState>();
state->is_like_pattern = true;
+ state->search_state.enable_hyperscan_fallback =
+ context->state()->query_options().enable_hyperscan_fallback;
state->function = like_fn;
state->scalar_function = like_fn_scalar;
if (context->is_col_constant(2)) {
@@ -1004,6 +1030,8 @@ Status FunctionRegexpLike::open(FunctionContext* context,
std::shared_ptr<LikeState> state = std::make_shared<LikeState>();
context->set_function_state(scope, state);
state->is_like_pattern = false;
+ state->search_state.enable_hyperscan_fallback =
+ context->state()->query_options().enable_hyperscan_fallback;
state->function = regexp_fn;
state->scalar_function = regexp_fn_scalar;
if (context->is_col_constant(1)) {
@@ -1035,12 +1063,16 @@ Status FunctionRegexpLike::open(FunctionContext*
context,
} else {
hs_database_t* database = nullptr;
hs_scratch_t* scratch = nullptr;
- if (hs_prepare(context, pattern_str.c_str(), &database,
&scratch).ok()) {
+ auto hs_status = hs_prepare(context, pattern_str.c_str(),
&database, &scratch);
+ if (hs_status.ok()) {
// use hyperscan
state->search_state.hs_database.reset(database);
state->search_state.hs_scratch.reset(scratch);
} else {
// fallback to re2
+ if (!state->search_state.enable_hyperscan_fallback) {
+ return hs_status;
+ }
// reset hs_database to nullptr to indicate not use hyperscan
state->search_state.hs_database.reset();
state->search_state.hs_scratch.reset();
diff --git a/be/src/exprs/function/like.h b/be/src/exprs/function/like.h
index 461c97956bc..d648919363d 100644
--- a/be/src/exprs/function/like.h
+++ b/be/src/exprs/function/like.h
@@ -29,6 +29,7 @@
#include <functional>
#include <memory>
#include <string>
+#include <string_view>
#include "common/status.h"
#include "core/block/column_numbers.h"
@@ -182,6 +183,8 @@ struct LikeSearchState {
std::string pattern_str;
+ bool enable_hyperscan_fallback = true;
+
/// Used for LIKE predicates if the pattern is a constant argument, and is
either a
/// constant string or has a constant string at the beginning or end of
the pattern.
/// This will be set in order to check for that pattern in the
corresponding part of
@@ -226,7 +229,7 @@ struct LikeSearchState {
LikeSearchState() = default;
- Status clone(LikeSearchState& cloned);
+ Status clone(LikeSearchState& cloned) const;
void set_search_string(const std::string& search_string_arg) {
search_string = search_string_arg;
@@ -292,6 +295,8 @@ public:
friend struct VectorEndsWithSearchState;
protected:
+ static bool should_fallback_to_re2(std::string_view regexp);
+
Status vector_const(const ColumnString& values, const StringRef*
pattern_val,
ColumnUInt8::Container& result, const LikeFn& function,
LikeSearchState* search_state) const;
diff --git a/be/src/exprs/function/match.cpp b/be/src/exprs/function/match.cpp
index 28ee41974ad..7fad3125006 100644
--- a/be/src/exprs/function/match.cpp
+++ b/be/src/exprs/function/match.cpp
@@ -25,6 +25,7 @@
#include "storage/index/index_reader_helper.h"
#include "storage/index/inverted/analyzer/analyzer.h"
#include "util/debug_points.h"
+#include "util/hyperscan_util.h"
namespace doris {
@@ -506,6 +507,10 @@ Status FunctionMatchRegexp::execute_match(FunctionContext*
context, const std::s
hs_compile_error_t* compile_err = nullptr;
hs_scratch_t* scratch = nullptr;
+ if (is_hyperscan_regexp_expensive(pattern)) {
+ return
Status::Error<ErrorCode::INDEX_INVALID_PARAMETERS>(HYPERSCAN_BOUNDED_REPEAT_ERROR);
+ }
+
if (hs_compile(pattern.data(), HS_FLAG_DOTALL | HS_FLAG_ALLOWEMPTY |
HS_FLAG_UTF8,
HS_MODE_BLOCK, nullptr, &database, &compile_err) !=
HS_SUCCESS) {
std::string err_message = "hyperscan compilation failed: ";
diff --git a/be/src/exprs/function/regexps.h b/be/src/exprs/function/regexps.h
index d521c7d9dec..15520d448c2 100644
--- a/be/src/exprs/function/regexps.h
+++ b/be/src/exprs/function/regexps.h
@@ -33,6 +33,7 @@
#include "common/exception.h"
#include "core/string_ref.h"
+#include "util/hyperscan_util.h"
namespace doris::multiregexps {
@@ -144,6 +145,9 @@ Regexps constructRegexps(const std::vector<String>&
str_patterns,
for (auto& pattern : patterns) {
LOG(INFO) << "pattern: " << pattern << "\n";
+ if (is_hyperscan_regexp_expensive(pattern)) {
+ throw
doris::Exception(Status::InvalidArgument(HYPERSCAN_BOUNDED_REPEAT_ERROR));
+ }
}
hs_error_t err;
diff --git a/be/src/storage/index/inverted/query/regexp_query.cpp
b/be/src/storage/index/inverted/query/regexp_query.cpp
index 7002f214468..fe9e3666ea5 100644
--- a/be/src/storage/index/inverted/query/regexp_query.cpp
+++ b/be/src/storage/index/inverted/query/regexp_query.cpp
@@ -23,6 +23,7 @@
#include "common/logging.h"
#include "util/debug_points.h"
+#include "util/hyperscan_util.h"
namespace doris::segment_v2 {
@@ -39,6 +40,9 @@ void RegexpQuery::add(const InvertedIndexQueryInfo&
query_info) {
}
const std::string& pattern = query_info.term_infos[0].get_single_term();
+ if (is_hyperscan_regexp_expensive(pattern)) {
+ throw Exception(ErrorCode::INVALID_ARGUMENT,
HYPERSCAN_BOUNDED_REPEAT_ERROR);
+ }
auto prefix = get_regex_prefix(pattern);
hs_database_t* database = nullptr;
diff --git
a/be/src/storage/index/inverted/query_v2/regexp_query/regexp_weight.cpp
b/be/src/storage/index/inverted/query_v2/regexp_query/regexp_weight.cpp
index ac6a905ba47..5ecfad0d3ae 100644
--- a/be/src/storage/index/inverted/query_v2/regexp_query/regexp_weight.cpp
+++ b/be/src/storage/index/inverted/query_v2/regexp_query/regexp_weight.cpp
@@ -41,6 +41,7 @@
#include "storage/index/inverted/query_v2/nullable_scorer.h"
#include "storage/index/inverted/query_v2/segment_postings.h"
#include "storage/index/inverted/util/string_helper.h"
+#include "util/hyperscan_util.h"
CL_NS_USE(index)
@@ -70,6 +71,9 @@ ScorerPtr RegexpWeight::scorer(const QueryExecutionContext&
context,
ScorerPtr RegexpWeight::regexp_scorer(const QueryExecutionContext& context,
const std::string& binding_key) {
+ if (is_hyperscan_regexp_expensive(_pattern)) {
+ throw Exception(ErrorCode::INVALID_ARGUMENT,
HYPERSCAN_BOUNDED_REPEAT_ERROR);
+ }
auto prefix = get_regex_prefix(_pattern);
hs_database_t* database = nullptr;
diff --git a/be/src/util/hyperscan_util.cpp b/be/src/util/hyperscan_util.cpp
new file mode 100644
index 00000000000..3cec0e4e68c
--- /dev/null
+++ b/be/src/util/hyperscan_util.cpp
@@ -0,0 +1,93 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "util/hyperscan_util.h"
+
+#include <re2/re2.h>
+#include <re2/stringpiece.h>
+
+#include <charconv>
+
+namespace doris {
+namespace {
+
+bool is_larger_than_fifty(std::string_view str) {
+ int number = 0;
+ auto [_, error] = std::from_chars(str.data(), str.data() + str.size(),
number);
+ return error == std::errc() && number > 50;
+}
+
+class SlowWithHyperscanChecker {
+public:
+ SlowWithHyperscanChecker()
+ : _searcher_one_repeat(R"(\{\s*([\d]+)\s*,?\s*})"),
+ _searcher_two_repeats(R"(\{\s*([\d]+)\s*,\s*([\d]+)\s*\})") {}
+
+ bool is_slow(std::string_view regexp) const {
+ return is_slow_one_repeat(regexp) || is_slow_two_repeats(regexp);
+ }
+
+private:
+ bool is_slow_one_repeat(std::string_view regexp) const {
+ re2::StringPiece haystack(regexp.data(), regexp.size());
+ re2::StringPiece matches[2];
+ size_t start_pos = 0;
+ while (start_pos < haystack.size()) {
+ if (!_searcher_one_repeat.Match(haystack, start_pos,
haystack.size(),
+ re2::RE2::Anchor::UNANCHORED,
matches, 2)) {
+ break;
+ }
+
+ start_pos = matches[0].data() - haystack.data() +
matches[0].size();
+ if (is_larger_than_fifty({matches[1].data(), matches[1].size()})) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ bool is_slow_two_repeats(std::string_view regexp) const {
+ re2::StringPiece haystack(regexp.data(), regexp.size());
+ re2::StringPiece matches[3];
+ size_t start_pos = 0;
+ while (start_pos < haystack.size()) {
+ if (!_searcher_two_repeats.Match(haystack, start_pos,
haystack.size(),
+ re2::RE2::Anchor::UNANCHORED,
matches, 3)) {
+ break;
+ }
+
+ start_pos = matches[0].data() - haystack.data() +
matches[0].size();
+ if (is_larger_than_fifty({matches[1].data(), matches[1].size()}) ||
+ is_larger_than_fifty({matches[2].data(), matches[2].size()})) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ re2::RE2 _searcher_one_repeat;
+ re2::RE2 _searcher_two_repeats;
+};
+
+} // namespace
+
+bool is_hyperscan_regexp_expensive(std::string_view regexp) {
+ static const SlowWithHyperscanChecker slow_with_hyperscan_checker;
+ return slow_with_hyperscan_checker.is_slow(regexp);
+}
+
+} // namespace doris
diff --git a/be/src/util/hyperscan_util.h b/be/src/util/hyperscan_util.h
new file mode 100644
index 00000000000..27ef9fdfe17
--- /dev/null
+++ b/be/src/util/hyperscan_util.h
@@ -0,0 +1,29 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <string_view>
+
+namespace doris {
+
+inline constexpr std::string_view HYPERSCAN_BOUNDED_REPEAT_ERROR =
+ "Skip hyperscan compilation because bounded repetition exceeds 50";
+
+bool is_hyperscan_regexp_expensive(std::string_view regexp);
+
+} // namespace doris
diff --git a/be/test/exprs/function/function_like_test.cpp
b/be/test/exprs/function/function_like_test.cpp
index 82618a790e9..62384b3f82b 100644
--- a/be/test/exprs/function/function_like_test.cpp
+++ b/be/test/exprs/function/function_like_test.cpp
@@ -16,8 +16,12 @@
// under the License.
#include <cstdint>
+#include <memory>
#include <string>
+#include <vector>
+#include "core/block/block.h"
+#include "core/column/column_const.h"
#include "core/column/column_string.h"
#include "core/column/column_vector.h"
#include "core/data_type/data_type_array.h"
@@ -27,11 +31,51 @@
#include "core/types.h"
#include "exprs/function/function_test_util.h"
#include "exprs/function/like.h"
+#include "exprs/function_context.h"
#include "gtest/gtest_pred_impl.h"
+#include "runtime/runtime_state.h"
#include "testutil/any_type.h"
namespace doris {
+class FunctionLikeTestHelper : public FunctionLikeBase {
+public:
+ using FunctionLikeBase::should_fallback_to_re2;
+};
+
+template <typename Function>
+Status execute_pattern_with_fallback_disabled(const std::string& value, const
std::string& pattern,
+ bool constant_known_at_open) {
+ TQueryOptions query_options;
+ query_options.__set_enable_hyperscan_fallback(false);
+ RuntimeState runtime_state(query_options, TQueryGlobals {});
+
+ auto string_type = std::make_shared<DataTypeString>();
+ auto context = FunctionContext::create_context(
+ &runtime_state, std::make_shared<DataTypeUInt8>(), {string_type,
string_type});
+
+ auto values = ColumnString::create();
+ values->insert_data(value.data(), value.size());
+ auto patterns = ColumnString::create();
+ patterns->insert_data(pattern.data(), pattern.size());
+ ColumnPtr pattern_column = ColumnConst::create(std::move(patterns), 1);
+
+ std::vector<std::shared_ptr<ColumnPtrWrapper>> constant_columns(2);
+ if (constant_known_at_open) {
+ constant_columns[1] =
std::make_shared<ColumnPtrWrapper>(pattern_column);
+ }
+ context->set_constant_cols(constant_columns);
+
+ Function function;
+ RETURN_IF_ERROR(function.open(context.get(),
FunctionContext::THREAD_LOCAL));
+
+ Block block;
+ block.insert({std::move(values), string_type, "value"});
+ block.insert({std::move(pattern_column), string_type, "pattern"});
+ block.insert({nullptr, std::make_shared<DataTypeUInt8>(), "result"});
+ return function.execute_impl(context.get(), block, {0, 1}, 2, 1);
+}
+
TEST(FunctionLikeTest, like) {
std::string func_name = "like";
@@ -137,6 +181,69 @@ TEST(FunctionLikeTest, regexp) {
}
}
+TEST(FunctionLikeTest, hyperscan_bounded_repeat_fallback) {
+ std::string func_name = "regexp";
+ std::string matching_value = "prompt_rewrite.h03" + std::string(500, 'x')
+ "429";
+
+ DataSet data_set = {
+ {{matching_value,
std::string(R"(prompt_rewrite\.h03.{0,1000}429)")}, uint8_t(1)},
+ {{std::string("prompt_rewrite.h03") + std::string(500, 'x') +
"430",
+ std::string(R"(prompt_rewrite\.h03.{0,1000}429)")},
+ uint8_t(0)}};
+
+ InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR,
PrimitiveType::TYPE_VARCHAR};
+ check_function_all_arg_comb<DataTypeUInt8, true>(func_name, input_types,
data_set);
+}
+
+TEST(FunctionLikeTest, hyperscan_bounded_repeat_threshold) {
+ EXPECT_FALSE(FunctionLikeTestHelper::should_fallback_to_re2("a*"));
+ EXPECT_FALSE(FunctionLikeTestHelper::should_fallback_to_re2("a{50}"));
+ EXPECT_FALSE(FunctionLikeTestHelper::should_fallback_to_re2("a{0,50}"));
+ EXPECT_TRUE(FunctionLikeTestHelper::should_fallback_to_re2("a{51}"));
+ EXPECT_TRUE(FunctionLikeTestHelper::should_fallback_to_re2("a{51,}"));
+ EXPECT_TRUE(FunctionLikeTestHelper::should_fallback_to_re2("a{0,51}"));
+ EXPECT_TRUE(FunctionLikeTestHelper::should_fallback_to_re2("a{51,51}"));
+ EXPECT_TRUE(FunctionLikeTestHelper::should_fallback_to_re2("a{ 0, 1000
}"));
+ EXPECT_FALSE(FunctionLikeTestHelper::should_fallback_to_re2(R"(a\{51\})"));
+ EXPECT_TRUE(FunctionLikeTestHelper::should_fallback_to_re2("[a{51}]"));
+ EXPECT_TRUE(FunctionLikeTestHelper::should_fallback_to_re2("(?#
note{51})a"));
+ EXPECT_TRUE(FunctionLikeTestHelper::should_fallback_to_re2("(?#
[)(ab?c?d){1000,5000}"));
+
EXPECT_TRUE(FunctionLikeTestHelper::should_fallback_to_re2("[^^](ab?c?d){1000,5000}"));
+}
+
+TEST(FunctionLikeTest, hyperscan_bounded_repeat_fallback_disabled) {
+ for (bool constant_known_at_open : {false, true}) {
+ SCOPED_TRACE(constant_known_at_open ? "prepare during open" : "prepare
during execute");
+ auto status =
execute_pattern_with_fallback_disabled<FunctionRegexpLike>(
+ "prompt_rewrite.h03xxx429",
R"(prompt_rewrite\.h03.{0,1000}429)",
+ constant_known_at_open);
+ EXPECT_FALSE(status.ok());
+ EXPECT_NE(status.to_string().find("bounded repetition exceeds 50"),
std::string::npos);
+
+ status = execute_pattern_with_fallback_disabled<FunctionRegexpLike>(
+ "^abc", "[^^](ab?c?d){1000,5000}", constant_known_at_open);
+ EXPECT_FALSE(status.ok());
+ EXPECT_NE(status.to_string().find("bounded repetition exceeds 50"),
std::string::npos);
+
+ status = execute_pattern_with_fallback_disabled<FunctionRegexpLike>(
+ "abcd", "(?# [)(ab?c?d){1000,5000}", constant_known_at_open);
+ EXPECT_FALSE(status.ok());
+ EXPECT_NE(status.to_string().find("bounded repetition exceeds 50"),
std::string::npos);
+ }
+}
+
+TEST(FunctionLikeTest,
hyperscan_bounded_repeat_literal_with_fallback_disabled) {
+ for (bool constant_known_at_open : {false, true}) {
+ SCOPED_TRACE(constant_known_at_open ? "prepare during open" : "prepare
during execute");
+ EXPECT_TRUE(execute_pattern_with_fallback_disabled<FunctionRegexpLike>(
+ "a{51}", R"(a\{51\})", constant_known_at_open)
+ .ok());
+
EXPECT_TRUE(execute_pattern_with_fallback_disabled<FunctionLike>("a{51}",
"_{51}",
+
constant_known_at_open)
+ .ok());
+ }
+}
+
TEST(FunctionLikeTest, regexp_extract) {
std::string func_name = "regexp_extract";
diff --git a/be/test/exprs/function/function_match_test.cpp
b/be/test/exprs/function/function_match_test.cpp
index ac9557d79f5..1a4f4e26b7c 100644
--- a/be/test/exprs/function/function_match_test.cpp
+++ b/be/test/exprs/function/function_match_test.cpp
@@ -28,6 +28,7 @@
#include "core/column/column_string.h"
#include "core/column/column_vector.h"
#include "exprs/function/match.h"
+#include "runtime/runtime_state.h"
#include "storage/index/inverted/analyzer/analyzer.h"
#include "storage/index/inverted/analyzer/custom_analyzer.h"
@@ -86,6 +87,26 @@ TEST(FunctionMatchTest, analyse_query_str) {
}
}
+TEST(FunctionMatchTest, regexp_rejects_expensive_bounded_repeat) {
+ TQueryOptions query_options;
+ query_options.__set_enable_match_without_inverted_index(true);
+ RuntimeState runtime_state(query_options, TQueryGlobals {});
+ auto context = FunctionContext::create_context(&runtime_state, {}, {});
+
+ auto string_col = ColumnString::create();
+ string_col->insert_data("abcd", 4);
+ ColumnUInt8::Container result(1, 0);
+
+ FunctionMatchRegexp function;
+ for (const char* pattern : {"(ab?c?d){1000,5000}", "(?#
[)(ab?c?d){1000,5000}"}) {
+ SCOPED_TRACE(pattern);
+ Status status = function.execute_match(context.get(), "test_column",
pattern, 1,
+ string_col.get(), nullptr,
nullptr, result);
+ EXPECT_FALSE(status.ok());
+ EXPECT_NE(status.to_string().find("bounded repetition exceeds 50"),
std::string::npos);
+ }
+}
+
// Test FunctionMatchAny::execute_match
TEST(FunctionMatchTest, match_any_execute) {
FunctionMatchAny func_match_any;
diff --git a/be/test/exprs/function/function_multi_match_test.cpp
b/be/test/exprs/function/function_multi_match_test.cpp
index d184dd5afa5..ab7e8ea5368 100644
--- a/be/test/exprs/function/function_multi_match_test.cpp
+++ b/be/test/exprs/function/function_multi_match_test.cpp
@@ -22,6 +22,7 @@
#include "core/block/column_with_type_and_name.h"
#include "core/block/columns_with_type_and_name.h"
#include "core/data_type/data_type_string.h"
+#include "exprs/function/regexps.h"
#include "storage/index/inverted/inverted_index_reader.h"
namespace doris {
@@ -73,4 +74,13 @@ TEST_F(FunctionMultiMatchTest,
EvaluateInvertedIndexWithNullIterator) {
<< "Error message should contain column name. Actual message: " <<
error_msg;
}
+TEST_F(FunctionMultiMatchTest, RejectsExpensiveBoundedRepeat) {
+ for (const char* pattern : {"(ab?c?d){1000,5000}", "(?#
[)(ab?c?d){1000,5000}"}) {
+ SCOPED_TRACE(pattern);
+ std::vector<String> patterns = {pattern};
+ EXPECT_THROW((multiregexps::constructRegexps<false, false>(patterns,
std::nullopt)),
+ Exception);
+ }
+}
+
} // namespace doris
diff --git a/be/test/storage/index/inverted/query/regexp_query_test.cpp
b/be/test/storage/index/inverted/query/regexp_query_test.cpp
index e212592d2f3..40287b9dff1 100644
--- a/be/test/storage/index/inverted/query/regexp_query_test.cpp
+++ b/be/test/storage/index/inverted/query/regexp_query_test.cpp
@@ -218,6 +218,30 @@ TEST_F(RegexpQueryTest, AddWithInvalidTermsSize) {
}
}
+TEST_F(RegexpQueryTest, AddRejectsExpensiveBoundedRepeat) {
+ std::shared_ptr<lucene::search::IndexSearcher> searcher = nullptr;
+ OlapReaderStatistics stats;
+ RuntimeState runtime_state;
+ TQueryOptions query_options;
+ query_options.inverted_index_max_expansions = 50;
+ runtime_state.set_query_options(query_options);
+ io::IOContext io_ctx;
+
+ auto context = std::make_shared<IndexQueryContext>();
+ context->io_ctx = &io_ctx;
+ context->runtime_state = &runtime_state;
+ context->stats = &stats;
+ RegexpQuery regexp_query(searcher, context);
+
+ for (const char* pattern : {"(ab?c?d){1000,5000}", "(?#
[)(ab?c?d){1000,5000}"}) {
+ SCOPED_TRACE(pattern);
+ InvertedIndexQueryInfo query_info;
+ query_info.field_name = L"test_field";
+ query_info.term_infos.push_back({pattern, 0});
+ EXPECT_THROW(regexp_query.add(query_info), Exception);
+ }
+}
+
TEST_F(RegexpQueryTest, AddWithInvalidPattern) {
// Create a mock searcher and query options for testing
std::shared_ptr<lucene::search::IndexSearcher> searcher = nullptr;
@@ -422,4 +446,4 @@ TEST_F(RegexpQueryTest, AddWithBackreferencePattern) {
EXPECT_NO_THROW(regexp_query.add(query_info));
}
-} // namespace doris::segment_v2
\ No newline at end of file
+} // namespace doris::segment_v2
diff --git a/be/test/storage/index/inverted/query_v2/regexp_query_test.cpp
b/be/test/storage/index/inverted/query_v2/regexp_query_test.cpp
index 397ecdae63b..0705ea7595d 100644
--- a/be/test/storage/index/inverted/query_v2/regexp_query_test.cpp
+++ b/be/test/storage/index/inverted/query_v2/regexp_query_test.cpp
@@ -133,6 +133,21 @@ TEST_F(RegexpQueryV2Test, test_regexp_query_construction) {
ASSERT_NE(regexp_weight, nullptr);
}
+TEST_F(RegexpQueryV2Test, test_rejects_expensive_bounded_repeat) {
+ auto context = std::make_shared<IndexQueryContext>();
+ context->collection_statistics = std::make_shared<CollectionStatistics>();
+ context->collection_similarity = std::make_shared<CollectionSimilarity>();
+
+ std::wstring field = StringHelper::to_wstring("content");
+ for (const char* pattern : {"(ab?c?d){1000,5000}", "(?#
[)(ab?c?d){1000,5000}"}) {
+ SCOPED_TRACE(pattern);
+ auto query = std::make_shared<query_v2::RegexpQuery>(context, field,
pattern);
+ auto weight = query->weight(false);
+ query_v2::QueryExecutionContext exec_ctx;
+ EXPECT_THROW(weight->scorer(exec_ctx), Exception);
+ }
+}
+
// Test regexp query with scoring enabled
TEST_F(RegexpQueryV2Test, test_regexp_query_with_scoring) {
auto context = std::make_shared<IndexQueryContext>();
@@ -565,4 +580,4 @@ TEST_F(RegexpQueryV2Test,
test_make_exact_match_wildcard_pattern) {
_CLDECDELETE(dir);
}
-} // namespace doris::segment_v2
\ No newline at end of file
+} // namespace doris::segment_v2
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index d2f3b474485..09990761712 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -830,6 +830,8 @@ public class SessionVariable implements Serializable,
Writable {
public static final String ENABLE_EXTENDED_REGEX = "enable_extended_regex";
+ public static final String ENABLE_HYPERSCAN_FALLBACK =
"enable_hyperscan_fallback";
+
public static final String
CLOUD_PARTITIONS_TABLE_USE_CACHED_VISIBLE_VERSION =
"cloud_partitions_table_use_cached_visible_version";
@@ -3550,6 +3552,10 @@ public class SessionVariable implements Serializable,
Writable {
description = "Enable extended regular expressions, support
look-around zero-width assertions")
public boolean enableExtendedRegex = false;
+ @VarAttrDef.VarAttr(name = ENABLE_HYPERSCAN_FALLBACK, needForward = true,
affectQueryResultInExecution = true,
+ description = "Whether to fall back to RE2 when Hyperscan cannot
compile a regular expression")
+ public boolean enableHyperscanFallback = true;
+
@VarAttrDef.VarAttr(
name = DEFAULT_VARIANT_SPARSE_HASH_SHARD_COUNT,
needForward = true,
@@ -5680,6 +5686,7 @@ public class SessionVariable implements Serializable,
Writable {
tResult.setAnnIndexCandidateRowsPercentThreshold(annIndexCandidateRowsPercentThreshold);
tResult.setMergeReadSliceSize(mergeReadSliceSizeBytes);
tResult.setEnableExtendedRegex(enableExtendedRegex);
+ tResult.setEnableHyperscanFallback(enableHyperscanFallback);
if (fileCacheQueryLimitPercent > 0) {
tResult.setFileCacheQueryLimitPercent(Math.min(fileCacheQueryLimitPercent,
Config.file_cache_query_limit_max_percent));
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariablesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariablesTest.java
index 9febc45b150..b7f7b8807c4 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariablesTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariablesTest.java
@@ -408,4 +408,15 @@ public class SessionVariablesTest extends
TestWithFeService {
queryOptions.getCoordinatorThriftMaxMessageSize());
Assertions.assertTrue(queryOptions.isSupportsExternalFileReportAck());
}
+
+ @Test
+ public void testHyperscanFallbackPropagatesToBackends() throws Exception {
+ SessionVariable variable = new SessionVariable();
+ Assertions.assertTrue(variable.toThrift().isEnableHyperscanFallback());
+
+ VariableMgr.setVar(variable, new SetVar(SetType.SESSION,
+ SessionVariable.ENABLE_HYPERSCAN_FALLBACK, new
StringLiteral("false")));
+
+
Assertions.assertFalse(variable.toThrift().isEnableHyperscanFallback());
+ }
}
diff --git a/gensrc/thrift/PaloInternalService.thrift
b/gensrc/thrift/PaloInternalService.thrift
index 36c9c55e778..fd16b4b7598 100644
--- a/gensrc/thrift/PaloInternalService.thrift
+++ b/gensrc/thrift/PaloInternalService.thrift
@@ -516,6 +516,8 @@ struct TQueryOptions {
229: optional i32 coordinator_thrift_max_message_size;
// FE can explicitly and idempotently acknowledge external-file commit
reports.
230: optional bool supports_external_file_report_ack = false;
+ // Fall back to RE2 when Hyperscan cannot compile a regular expression.
+ 231: optional bool enable_hyperscan_fallback = true;
// For cloud, to control if the content would be written into file cache
// In write path, to control if the content would be written into file cache.
// In read path, read from file cache or remote storage when execute query.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
