This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new 2d76b67ff86 [branch-4.2][fix](function) Avoid duplicate zero-width
regexp_extract_all matches (#70244)
2d76b67ff86 is described below
commit 2d76b67ff86e603f29927e54cd7accb4d3fd4dc8
Author: Jerry Hu <[email protected]>
AuthorDate: Fri Nov 6 11:17:57 2026 +0800
[branch-4.2][fix](function) Avoid duplicate zero-width regexp_extract_all
matches (#70244)
## Summary
With extended regular expressions enabled, restarting Boost.Regex
searches on a shortened input can repeat a zero-width match and change
the context seen by lookbehind and anchors. Iterate over the complete
input with `boost::cregex_iterator` instead, while retaining the
existing capture-group behavior and leaving the RE2 path unchanged. This
backports the fix from #68358.
## Tests
- `./build.sh --be --fe -j 16` — passed.
- `BUILD_TYPE=ASAN DISABLE_BE_JAVA_EXTENSIONS=OFF ./build.sh --be -j 32`
— passed.
- `./run-be-ut.sh --run
--filter=FunctionLikeTest.regexp_extract_all_boost_zero_width_match -j
32` — passed (1 test).
- Local SQL regression: `query_p0/load` — passed (8 suites);
`query_p0/sql_functions/string_functions/test_string_function_regexp` —
passed (1 suite).
- `build-support/clang-format.sh`, `build-support/check-format.sh`, and
committed diff whitespace check — passed.
---
be/src/exprs/function/function_regexp.cpp | 21 +++----
be/test/exprs/function/function_like_test.cpp | 64 ++++++++++++++++++++++
.../test_string_function_regexp.out | 18 ++++++
.../test_string_function_regexp.groovy | 7 +++
4 files changed, 97 insertions(+), 13 deletions(-)
diff --git a/be/src/exprs/function/function_regexp.cpp
b/be/src/exprs/function/function_regexp.cpp
index 257209bbe6a..53147a63a1a 100644
--- a/be/src/exprs/function/function_regexp.cpp
+++ b/be/src/exprs/function/function_regexp.cpp
@@ -169,22 +169,17 @@ struct RegexpExtractEngine {
pos += offset + matches[0].size();
}
} else if (is_boost()) {
- const char* search_start = data;
- const char* search_end = data + size;
- boost::match_results<const char*> matches;
-
- while (boost::regex_search(search_start, search_end, matches,
*boost_regex)) {
+ // Iterate over the whole input instead of re-running regex_search
on a
+ // shrinking range: the iterator keeps the preceding text visible
to
+ // lookbehind/`^`/`\b` and steps past zero-width matches (e.g.
lookahead)
+ // without matching the same position twice.
+ boost::cregex_iterator it(data, data + size, *boost_regex);
+ boost::cregex_iterator end;
+ for (; it != end; ++it) {
+ const auto& matches = *it;
if (matches.size() > 1 && matches[1].matched) {
results.emplace_back(matches[1].str());
}
- if (matches[0].length() == 0) {
- if (search_start == search_end) {
- break;
- }
- search_start += 1;
- } else {
- search_start = matches[0].second;
- }
}
}
}
diff --git a/be/test/exprs/function/function_like_test.cpp
b/be/test/exprs/function/function_like_test.cpp
index 9430882e4a3..ee3f0869710 100644
--- a/be/test/exprs/function/function_like_test.cpp
+++ b/be/test/exprs/function/function_like_test.cpp
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+#include <gen_cpp/PaloInternalService_types.h>
+
#include <cstdint>
#include <memory>
#include <string>
@@ -584,6 +586,68 @@ TEST(FunctionLikeTest, regexp_extract_all) {
}
}
+// Extended regex falls back to Boost for lookaround patterns. The iterator
must
+// advance through zero-width matches without losing the original input
context.
+TEST(FunctionLikeTest, regexp_extract_all_boost_zero_width_match) {
+ auto str_type = std::make_shared<DataTypeString>();
+ auto return_type = make_nullable(std::make_shared<DataTypeString>());
+
+ auto run_case = [&](const std::string& str, const std::string& pattern,
+ const std::string& expected) {
+ auto col_str = ColumnString::create();
+ col_str->insert_data(str.data(), str.size());
+ auto col_pattern = ColumnString::create();
+ col_pattern->insert_data(pattern.data(), pattern.size());
+
+ Block block;
+ block.insert({std::move(col_str), str_type, "str"});
+ block.insert({ColumnConst::create(std::move(col_pattern), 1),
str_type, "pattern"});
+ block.insert({nullptr, return_type, "result"});
+
+ ColumnsWithTypeAndName arg_cols = {block.get_by_position(0),
block.get_by_position(1)};
+ auto func =
SimpleFunctionFactory::instance().get_function("regexp_extract_all", arg_cols,
+
return_type);
+ ASSERT_TRUE(func != nullptr);
+
+ std::vector<DataTypePtr> arg_types = {str_type, str_type};
+ FunctionUtils fn_utils({}, arg_types, false);
+ auto* fn_ctx = fn_utils.get_fn_ctx();
+ TQueryOptions query_options = fn_ctx->state()->query_options();
+ query_options.__set_enable_extended_regex(true);
+ fn_ctx->state()->set_query_options(query_options);
+ fn_ctx->set_constant_cols(
+ {nullptr,
std::make_shared<ColumnPtrWrapper>(block.get_by_position(1).column)});
+
+ ASSERT_EQ(Status::OK(), func->open(fn_ctx,
FunctionContext::FRAGMENT_LOCAL));
+ ASSERT_EQ(Status::OK(), func->open(fn_ctx,
FunctionContext::THREAD_LOCAL));
+ ASSERT_EQ(Status::OK(), func->execute(fn_ctx, block, {0, 1}, 2, 1));
+
+ auto result_col = block.get_by_position(2).column;
+ ASSERT_TRUE(result_col.get() != nullptr);
+ ASSERT_FALSE(result_col->is_null_at(0));
+ EXPECT_EQ(expected, return_type->to_string(*result_col, 0))
+ << "input: '" << str << "', pattern: '" << pattern << "'";
+
+ static_cast<void>(func->close(fn_ctx, FunctionContext::THREAD_LOCAL));
+ static_cast<void>(func->close(fn_ctx,
FunctionContext::FRAGMENT_LOCAL));
+ };
+
+ // The lookahead matches once, at the last position; it must not be
extracted twice.
+ run_case("ba", "(?=(a))", "['a']");
+ // Zero-width matches at distinct positions are all still found, exactly
once each.
+ run_case("aba", "(?=(a))", "['a','a']");
+ run_case("aaa", "(?=(a))", "['a','a','a']");
+ // Non-zero-width Boost matches keep advancing past the whole match.
+ run_case("foo123bar456baz", "(\\d{3})(?=bar|baz)", "['123','456']");
+ // The text before a zero-width match stays visible to lookbehind at the
next position.
+ run_case("aa", "(?<=(a))", "['a','a']");
+ // After an empty match, a consuming match at the same position is still
tried.
+ run_case("xa", "((?<=x)|a)", "['','a']");
+ // Resuming after a zero-width match must not let `^` anchor in the middle
of the input
+ // (the string form renders "no match" as an empty string).
+ run_case("xxa?b", "(?<=a)|^(b)", "");
+}
+
TEST(FunctionLikeTest, regexp_replace) {
std::string func_name = "regexp_replace";
diff --git
a/regression-test/data/query_p0/sql_functions/string_functions/test_string_function_regexp.out
b/regression-test/data/query_p0/sql_functions/string_functions/test_string_function_regexp.out
index 5b21c2a1bcc..e5427af7ccd 100644
---
a/regression-test/data/query_p0/sql_functions/string_functions/test_string_function_regexp.out
+++
b/regression-test/data/query_p0/sql_functions/string_functions/test_string_function_regexp.out
@@ -247,6 +247,24 @@ aXb
-- !sql_regexp_extract_all_5 --
['Case1','Case2','Case3']
+-- !sql_regexp_extract_all_zero_width_1 --
+['a']
+
+-- !sql_regexp_extract_all_zero_width_2 --
+['a','a']
+
+-- !sql_regexp_extract_all_zero_width_3 --
+['a','a','a']
+
+-- !sql_regexp_extract_all_zero_width_4 --
+['a','a']
+
+-- !sql_regexp_extract_all_zero_width_5 --
+['','a']
+
+-- !sql_regexp_extract_all_zero_width_6 --
+
+
-- !sql_regexp_extract_all_6 --
['foo\nbar']
diff --git
a/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function_regexp.groovy
b/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function_regexp.groovy
index c5163875b58..3739fba44e6 100644
---
a/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function_regexp.groovy
+++
b/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function_regexp.groovy
@@ -120,6 +120,13 @@ suite("test_string_function_regexp") {
qt_sql_regexp_extract_all_3 'SELECT
REGEXP_EXTRACT_ALL(\'foo123bar456baz\', \'(\\\\d{3})(?=bar|baz)\');'
qt_sql_regexp_extract_all_4 'SELECT
REGEXP_EXTRACT_ALL(\'ID:AA-1,ID:BB-2,ID:CC-3\', \'(?<=ID:)([A-Z]{2}-\\\\d)\');'
qt_sql_regexp_extract_all_5 'SELECT
REGEXP_EXTRACT_ALL(\'EdgeCase1EdgeCase2EdgeCase3\',
\'(?<=Edge)(Case\\\\d)(?=Edge|$)\');'
+ // zero-width Boost matches must be extracted once per position
+ qt_sql_regexp_extract_all_zero_width_1 "SELECT REGEXP_EXTRACT_ALL('ba',
'(?=(a))');"
+ qt_sql_regexp_extract_all_zero_width_2 "SELECT REGEXP_EXTRACT_ALL('aba',
'(?=(a))');"
+ qt_sql_regexp_extract_all_zero_width_3 "SELECT REGEXP_EXTRACT_ALL('aaa',
'(?=(a))');"
+ qt_sql_regexp_extract_all_zero_width_4 "SELECT REGEXP_EXTRACT_ALL('aa',
'(?<=(a))');"
+ qt_sql_regexp_extract_all_zero_width_5 "SELECT REGEXP_EXTRACT_ALL('xa',
'((?<=x)|a)');"
+ qt_sql_regexp_extract_all_zero_width_6 "SELECT REGEXP_EXTRACT_ALL('xxa?b',
'(?<=a)|^(b)');"
sql "set enable_extended_regex = false;"
qt_sql_regexp_extract_all_6 "SELECT REGEXP_EXTRACT_ALL(concat('foo',
char(10), 'bar'), '(foo.bar)');"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
