This is an automated email from the ASF dual-hosted git repository.
HappenLee pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 09889432593 [Fix](regexp) Fix empty constant pattern matching in
regexp (#68146)
09889432593 is described below
commit 0988943259399d07ca3f85995683e4fe777f37a0
Author: linrrarity <[email protected]>
AuthorDate: Fri Sep 18 15:10:09 2026 +0800
[Fix](regexp) Fix empty constant pattern matching in regexp (#68146)
Problem Summary:
An empty regexp produces a zero-width match, so the expected result is
1.
The constant pattern was recognized as a substring pattern and routed to
the long-buffer substring optimization. However, this path treated an
empty needle as not found.
```sql
DROP TABLE IF EXISTS tmp_b8_c01;
CREATE TABLE tmp_b8_c01 (
id INT,
s STRING,
p STRING
)
DUPLICATE KEY(id)
DISTRIBUTED BY HASH(id) BUCKETS 1
PROPERTIES("replication_num" = "1");
INSERT INTO tmp_b8_c01 VALUES (1, 'abc', '');
SELECT
regexp('abc', '') AS const_empty,
regexp(s, p) AS column_empty
FROM tmp_b8_c01
WHERE id = 1;
-- before:
+-------------+--------------+
| const_empty | column_empty |
+-------------+--------------+
| 0 | 1 |
+-------------+--------------+
-- now:
+-------------+--------------+
| const_empty | column_empty |
+-------------+--------------+
| 1 | 1 |
+-------------+--------------+
```
### What is changed?
- Handle an empty constant substring pattern as matching every non-null
input.
- Move the long-buffer substring implementation into
`constant_substring_fn`.
- Remove the separate execute_substring implementation and its special
dispatch logic.
- Preserve row-boundary checks to prevent matches across adjacent
ColumnString rows.
---
be/src/exprs/function/like.cpp | 110 +++++++++------------
be/src/exprs/function/like.h | 4 -
be/test/exprs/function/function_like_test.cpp | 67 ++++++++++++-
.../test_string_function_regexp.out | 35 ++++++-
.../test_string_function_regexp.groovy | 43 ++++++++
5 files changed, 188 insertions(+), 71 deletions(-)
diff --git a/be/src/exprs/function/like.cpp b/be/src/exprs/function/like.cpp
index a6403f5684c..2dfd08d3b51 100644
--- a/be/src/exprs/function/like.cpp
+++ b/be/src/exprs/function/like.cpp
@@ -341,13 +341,44 @@ Status FunctionLikeBase::vector_equals_fn(const
ColumnString& vals,
Status FunctionLikeBase::constant_substring_fn(const LikeSearchState* state,
const ColumnString& val, const
StringRef& pattern,
ColumnUInt8::Container& result)
{
- auto sz = val.size();
- for (size_t i = 0; i < sz; i++) {
- if (state->search_string_sv.size == 0) {
- result[i] = true;
- continue;
+ size_t needle_size = state->search_string_sv.size;
+ if (needle_size == 0) {
+ memset(result.data(), 1, result.size());
+ return Status::OK();
+ }
+
+ const auto& values = val.get_chars();
+ const auto& value_offsets = val.get_offsets();
+ // treat continuous multi string data as a long string data
+ const UInt8* begin = values.data();
+ const UInt8* end = begin + values.size();
+ const UInt8* pos = begin;
+
+ /// Current index in the array of strings.
+ size_t i = 0;
+
+ /// We will search for the next occurrence in all strings at once.
+ while (pos < end) {
+ // search return matched substring start offset
+ pos = (UInt8*)state->substring_pattern.search((char*)pos, end - pos);
+ if (pos >= end) {
+ break;
+ }
+
+ /// Determine which index it refers to.
+ /// begin + value_offsets[i] is the start offset of string at i+1
+ while (i < value_offsets.size() && begin + value_offsets[i] < pos) {
+ ++i;
+ }
+
+ /// We check that the entry does not pass through the boundaries of
strings.
+ if (pos + needle_size <= begin + value_offsets[i]) {
+ result[i] = 1;
}
- result[i] = state->substring_pattern.search(val.get_data_at(i)) != -1;
+
+ // move to next string offset
+ pos = begin + value_offsets[i];
+ ++i;
}
return Status::OK();
}
@@ -547,71 +578,20 @@ Status FunctionLikeBase::execute_impl(FunctionContext*
context, Block& block,
vec_res.resize_fill(input_rows_count);
auto* state = reinterpret_cast<LikeState*>(
context->get_function_state(FunctionContext::THREAD_LOCAL));
- // for constant_substring_fn, use long run length search for performance
- if (constant_substring_fn ==
- *(state->function
- .target<doris::Status (*)(const LikeSearchState* state,
const ColumnString&,
- const StringRef&,
ColumnUInt8::Container&)>())) {
- RETURN_IF_ERROR(execute_substring(values->get_chars(),
values->get_offsets(), vec_res,
- &state->search_state));
+ const auto pattern_col = block.get_by_position(arguments[1]).column;
+ if (const auto* str_patterns =
check_and_get_column<ColumnString>(pattern_col.get())) {
+ RETURN_IF_ERROR(vector_non_const(*values, *str_patterns, vec_res,
state, input_rows_count));
+ } else if (const auto* const_patterns =
check_and_get_column<ColumnConst>(pattern_col.get())) {
+ const auto& pattern_val = const_patterns->get_data_at(0);
+ RETURN_IF_ERROR(vector_const(*values, &pattern_val, vec_res,
state->function,
+ &state->search_state));
} else {
- const auto pattern_col = block.get_by_position(arguments[1]).column;
- if (const auto* str_patterns =
check_and_get_column<ColumnString>(pattern_col.get())) {
- RETURN_IF_ERROR(
- vector_non_const(*values, *str_patterns, vec_res, state,
input_rows_count));
- } else if (const auto* const_patterns =
-
check_and_get_column<ColumnConst>(pattern_col.get())) {
- const auto& pattern_val = const_patterns->get_data_at(0);
- RETURN_IF_ERROR(vector_const(*values, &pattern_val, vec_res,
state->function,
- &state->search_state));
- } else {
- return Status::InternalError("Not supported input arguments
types");
- }
+ return Status::InternalError("Not supported input arguments types");
}
block.replace_by_position(result, std::move(res));
return Status::OK();
}
-Status FunctionLikeBase::execute_substring(const ColumnString::Chars& values,
- const ColumnString::Offsets&
value_offsets,
- ColumnUInt8::Container& result,
- LikeSearchState* search_state)
const {
- // treat continuous multi string data as a long string data
- const UInt8* begin = values.data();
- const UInt8* end = begin + values.size();
- const UInt8* pos = begin;
-
- /// Current index in the array of strings.
- size_t i = 0;
- size_t needle_size = search_state->substring_pattern.get_pattern_length();
-
- /// We will search for the next occurrence in all strings at once.
- while (pos < end) {
- // search return matched substring start offset
- pos = (UInt8*)search_state->substring_pattern.search((char*)pos, end -
pos);
- if (pos >= end) {
- break;
- }
-
- /// Determine which index it refers to.
- /// begin + value_offsets[i] is the start offset of string at i+1
- while (i < value_offsets.size() && begin + value_offsets[i] < pos) {
- ++i;
- }
-
- /// We check that the entry does not pass through the boundaries of
strings.
- if (pos + needle_size <= begin + value_offsets[i]) {
- result[i] = 1;
- }
-
- // move to next string offset
- pos = begin + value_offsets[i];
- ++i;
- }
-
- return Status::OK();
-}
-
Status FunctionLikeBase::vector_const(const ColumnString& values, const
StringRef* pattern_val,
ColumnUInt8::Container& result, const
LikeFn& function,
LikeSearchState* search_state) const {
diff --git a/be/src/exprs/function/like.h b/be/src/exprs/function/like.h
index d648919363d..9759e7e563f 100644
--- a/be/src/exprs/function/like.h
+++ b/be/src/exprs/function/like.h
@@ -305,10 +305,6 @@ protected:
ColumnUInt8::Container& result, LikeState* state,
size_t input_rows_count) const;
- Status execute_substring(const ColumnString::Chars& values,
- const ColumnString::Offsets& value_offsets,
- ColumnUInt8::Container& result, LikeSearchState*
search_state) const;
-
template <bool LIKE_PATTERN>
static VPatternSearchStateSPtr pattern_type_recognition(const
ColumnString& patterns);
diff --git a/be/test/exprs/function/function_like_test.cpp
b/be/test/exprs/function/function_like_test.cpp
index a6887222251..51a8a1ce56d 100644
--- a/be/test/exprs/function/function_like_test.cpp
+++ b/be/test/exprs/function/function_like_test.cpp
@@ -76,6 +76,50 @@ Status execute_pattern_with_fallback_disabled(const
std::string& value, const st
return function.execute_impl(context.get(), block, {0, 1}, 2, 1);
}
+template <typename Function>
+void check_constant_pattern_batch(const std::vector<std::string>& values,
+ const std::string& pattern,
+ const std::vector<uint8_t>& expected) {
+ ASSERT_EQ(values.size(), expected.size());
+
+ TQueryOptions query_options;
+ RuntimeState runtime_state(query_options, TQueryGlobals {});
+ auto string_type = std::make_shared<DataTypeString>();
+ auto result_type = std::make_shared<DataTypeUInt8>();
+ auto context = FunctionContext::create_context(&runtime_state, result_type,
+ {string_type, string_type});
+
+ auto value_column = ColumnString::create();
+ for (const auto& value : values) {
+ value_column->insert_data(value.data(), value.size());
+ }
+ auto pattern_data = ColumnString::create();
+ pattern_data->insert_data(pattern.data(), pattern.size());
+ ColumnPtr pattern_column = ColumnConst::create(std::move(pattern_data),
values.size());
+
+ std::vector<std::shared_ptr<ColumnPtrWrapper>> constant_columns(2);
+ constant_columns[1] = std::make_shared<ColumnPtrWrapper>(pattern_column);
+ context->set_constant_cols(constant_columns);
+
+ Function function;
+ auto status = function.open(context.get(), FunctionContext::THREAD_LOCAL);
+ ASSERT_TRUE(status.ok()) << status.to_string();
+
+ Block block;
+ block.insert({std::move(value_column), string_type, "value"});
+ block.insert({std::move(pattern_column), string_type, "pattern"});
+ block.insert({nullptr, result_type, "result"});
+ status = function.execute_impl(context.get(), block, {0, 1}, 2,
values.size());
+ ASSERT_TRUE(status.ok()) << status.to_string();
+
+ const auto* result =
check_and_get_column<ColumnUInt8>(block.get_by_position(2).column.get());
+ ASSERT_NE(result, nullptr);
+ ASSERT_EQ(result->size(), expected.size());
+ for (size_t i = 0; i < expected.size(); ++i) {
+ EXPECT_EQ(result->get_element(i), expected[i]) << "row " << i;
+ }
+}
+
TEST(FunctionLikeTest, like) {
std::string func_name = "like";
@@ -255,6 +299,27 @@ TEST(FunctionLikeTest, regexp) {
}
}
+TEST(FunctionLikeTest, regexp_empty_constant_pattern) {
+ DataSet data_set = {{{std::string("abc"), std::string("")}, uint8_t(1)},
+ {{std::string(""), std::string("")}, uint8_t(1)}};
+ InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR, Consted
{PrimitiveType::TYPE_VARCHAR}};
+
+ for (const auto& func_name : {"regexp", "rlike"}) {
+ for (const auto& line : data_set) {
+ DataSet const_pattern_dataset = {line};
+ static_cast<void>(check_function<DataTypeUInt8, true>(func_name,
input_types,
+
const_pattern_dataset));
+ }
+ }
+
+ check_constant_pattern_batch<FunctionRegexpLike>({"abc", ""}, "", {1, 1});
+}
+
+TEST(FunctionLikeTest, regexp_constant_substring_boundaries) {
+ check_constant_pattern_batch<FunctionRegexpLike>({"ab", "c", "", "abc",
"", "xabcx"}, "abc",
+ {0, 0, 0, 1, 0, 1});
+}
+
TEST(FunctionLikeTest, hyperscan_bounded_repeat_fallback) {
std::string func_name = "regexp";
std::string matching_value = "prompt_rewrite.h03" + std::string(500, 'x')
+ "429";
@@ -975,7 +1040,7 @@ TEST(FunctionLikeTest, error_handling) {
TEST(FunctionLikeTest, substring_optimization_performance) {
std::string func_name = "like";
- // Test cases that should trigger execute_substring optimization
+ // Test cases that should trigger the constant substring long-buffer
optimization
DataSet data_set = {// Multiple identical substrings in long text
{{std::string("aaabbbaaabbbaaabbb"),
std::string("%bbb%")}, uint8_t(1)},
{{std::string("aaacccaaacccaaaccc"),
std::string("%bbb%")}, uint8_t(0)},
diff --git
a/regression-test/data/query_p0/sql_functions/string_functions/test_string_function_regexp.out
b/regression-test/data/query_p0/sql_functions/string_functions/test_string_function_regexp.out
index 3a2754b72e1..91f38653d23 100644
---
a/regression-test/data/query_p0/sql_functions/string_functions/test_string_function_regexp.out
+++
b/regression-test/data/query_p0/sql_functions/string_functions/test_string_function_regexp.out
@@ -357,6 +357,40 @@ false
-- !regexp_fn_9 --
true
+-- !regexp_empty_pattern --
+1 true true
+2 true true
+3 true true
+4 \N \N
+5 true \N
+6 true true
+7 true true
+8 true true
+9 true true
+10 true true
+11 true true
+
+-- !rlike_empty_pattern --
+1 true true
+2 true true
+3 true true
+4 \N \N
+5 true \N
+6 true true
+7 true true
+8 true true
+9 true true
+10 true true
+11 true true
+
+-- !regexp_constant_substring_boundaries --
+6 false false
+7 false false
+8 false false
+9 true true
+10 false false
+11 true true
+
-- !sql_utf1 --
true
@@ -510,4 +544,3 @@ Ben
-- !sql_field5 --
2
-
diff --git
a/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function_regexp.groovy
b/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function_regexp.groovy
index 09a5a2f7e58..b7aeaee17f3 100644
---
a/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function_regexp.groovy
+++
b/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function_regexp.groovy
@@ -167,6 +167,49 @@ suite("test_string_function_regexp") {
qt_regexp_fn_9 'SELECT regexp(\'Hello\', \'(?i)hello\');'
sql "set enable_extended_regex = false;"
+ sql "DROP TABLE IF EXISTS test_regexp_empty_pattern"
+ sql """
+ CREATE TABLE test_regexp_empty_pattern (
+ id INT,
+ value_col STRING NULL,
+ pattern_col STRING NULL
+ )
+ DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+ sql """
+ INSERT INTO test_regexp_empty_pattern VALUES
+ (1, 'abc', ''),
+ (2, '', ''),
+ (3, 'xyz', ''),
+ (4, NULL, ''),
+ (5, 'abc', NULL),
+ (6, 'ab', ''),
+ (7, 'c', ''),
+ (8, '', ''),
+ (9, 'abc', ''),
+ (10, '', ''),
+ (11, 'xabcx', '')
+ """
+
+ qt_regexp_empty_pattern """
+ SELECT id, regexp(value_col, ''), regexp(value_col, pattern_col)
+ FROM test_regexp_empty_pattern
+ ORDER BY id
+ """
+ qt_rlike_empty_pattern """
+ SELECT id, value_col RLIKE '', value_col RLIKE pattern_col
+ FROM test_regexp_empty_pattern
+ ORDER BY id
+ """
+ qt_regexp_constant_substring_boundaries """
+ SELECT id, regexp(value_col, 'abc'), value_col RLIKE 'abc'
+ FROM test_regexp_empty_pattern
+ WHERE id >= 6
+ ORDER BY id
+ """
+
qt_sql_utf1 """ select '皖12345' REGEXP '^[皖][0-9]{5}\$'; """
qt_sql_utf2 """ select '皖 12345' REGEXP '^[皖] [0-9]{5}\$'; """
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
