This is an automated email from the ASF dual-hosted git repository.
HappenLee pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new f9105c38823 [fix](be) Validate sequence pattern event numbers (#64930)
f9105c38823 is described below
commit f9105c388239c1ecdf9655d4c9a415e3bf5d0a7b
Author: HappenLee <[email protected]>
AuthorDate: Tue Jun 30 17:51:29 2026 +0800
[fix](be) Validate sequence pattern event numbers (#64930)
Problem Summary: sequence_match and sequence_count accepted
syntactically valid but semantically invalid event indexes such as (?0)
and indexes larger than the number of event predicates. In BE, (?0)
underflowed before bitset access and could abort the backend; the upper
bound also allowed one nonexistent event. This change validates the
event number range in Nereids and keeps the BE parser from using
event_number - 1 until the range is checked.
### Release note
Fix sequence_match and sequence_count to reject out-of-range event
numbers instead of risking backend abort.
### Check List (For Author)
- Test:
- Unit Test: ./run-be-ut.sh --clean --run --filter=VSequenceMatchTest.*
- Manual test: PATH=/mnt/disk6/common/ldb_toolchain_toucan/bin:$PATH
build-support/clang-format.sh
- Manual test: PATH=/mnt/disk6/common/ldb_toolchain_toucan/bin:$PATH
build-support/check-format.sh
- Manual test: build-support/run-clang-tidy.sh --build-dir
be/ut_build_ASAN
- Manual test: ./build.sh --fe --clean
- Regression test: Not run locally because no isolated worktree Doris
cluster was started; added regression coverage for CI.
- Behavior changed: Yes. Invalid sequence_match and sequence_count event
numbers now return analysis errors or an empty BE-side match instead of
reaching invalid bitset indexes.
- Does this need documentation: No
---
.../aggregate/aggregate_function_sequence_match.h | 10 ++--
.../exprs/aggregate/vec_sequence_match_test.cpp | 63 ++++++++++++++++++++++
.../functions/agg/SequenceFunction.java | 19 +++++++
.../nereids_syntax_p0/test_nereids_function.groovy | 27 ++++++++++
4 files changed, 115 insertions(+), 4 deletions(-)
diff --git a/be/src/exprs/aggregate/aggregate_function_sequence_match.h
b/be/src/exprs/aggregate/aggregate_function_sequence_match.h
index 365359da54e..3bff47fab70 100644
--- a/be/src/exprs/aggregate/aggregate_function_sequence_match.h
+++ b/be/src/exprs/aggregate/aggregate_function_sequence_match.h
@@ -215,6 +215,7 @@ private:
const char* pos = pattern.data();
const char* begin = pos;
const char* end = pos + pattern.size();
+ const size_t event_count = arg_count - 2;
// Pattern is checked in fe, so pattern should be valid here, we check
it and if pattern is invalid, we return.
auto fail_parse = [&]() {
@@ -298,17 +299,18 @@ private:
return;
}
- if (event_number > arg_count - 1) {
+ if (event_number == 0 || event_number > event_count) {
throw_exception("Event number " +
std::to_string(event_number) +
" is out of range");
return;
}
- actions.emplace_back(PatternActionType::SpecificEvent,
event_number - 1);
+ const auto event_index = event_number - 1;
+ actions.emplace_back(PatternActionType::SpecificEvent,
event_index);
dfa_states.back().transition =
DFATransition::SpecificEvent;
- dfa_states.back().event =
static_cast<uint32_t>(event_number - 1);
+ dfa_states.back().event =
static_cast<uint32_t>(event_index);
dfa_states.emplace_back();
- conditions_in_pattern.set(event_number - 1);
+ conditions_in_pattern.set(event_index);
}
if (!match(")")) {
diff --git a/be/test/exprs/aggregate/vec_sequence_match_test.cpp
b/be/test/exprs/aggregate/vec_sequence_match_test.cpp
index 7ce30e7eb1c..3518a3e06bb 100644
--- a/be/test/exprs/aggregate/vec_sequence_match_test.cpp
+++ b/be/test/exprs/aggregate/vec_sequence_match_test.cpp
@@ -19,6 +19,7 @@
#include <gtest/gtest-test-part.h>
#include <memory>
+#include <vector>
#include "core/column/column_string.h"
#include "core/column/column_vector.h"
@@ -29,6 +30,7 @@
#include "core/types.h"
#include "core/value/vdatetime_value.h"
#include "exprs/aggregate/aggregate_function.h"
+#include "exprs/aggregate/aggregate_function_sequence_match.h"
#include "exprs/aggregate/aggregate_function_simple_factory.h"
#include "gtest/gtest_pred_impl.h"
@@ -61,6 +63,39 @@ public:
void TearDown() {}
+ template <typename ResultColumn>
+ void assert_invalid_pattern_returns_zero(const AggregateFunctionPtr&
agg_function,
+ const std::string& pattern,
size_t event_count) {
+ auto column_pattern = ColumnString::create();
+ column_pattern->insert(Field::create_field<TYPE_STRING>(pattern));
+
+ auto column_timestamp = ColumnDateTimeV2::create();
+ VecDateTimeValue time_value;
+ time_value.unchecked_set_time(2024, 1, 1, 0, 0, 0);
+ column_timestamp->insert_data((char*)&time_value, 0);
+
+ std::vector<ColumnUInt8::MutablePtr> event_columns;
+ event_columns.reserve(event_count);
+ std::vector<const IColumn*> columns = {column_pattern.get(),
column_timestamp.get()};
+ columns.reserve(event_count + 2);
+ for (size_t i = 0; i < event_count; ++i) {
+ auto column_event = ColumnUInt8::create();
+ column_event->insert(Field::create_field<TYPE_BOOLEAN>(1));
+ columns.push_back(column_event.get());
+ event_columns.push_back(std::move(column_event));
+ }
+
+ std::unique_ptr<char[]> memory(new char[agg_function->size_of_data()]);
+ AggregateDataPtr place = memory.get();
+ agg_function->create(place);
+ EXPECT_NO_THROW(agg_function->add(place, columns.data(), 0, arena));
+
+ ResultColumn column_result;
+ agg_function->insert_result_into(place, column_result);
+ EXPECT_EQ(column_result.get_data()[0], 0);
+ agg_function->destroy(place);
+ }
+
Arena arena;
};
@@ -122,6 +157,34 @@ TEST_F(VSequenceMatchTest, testCountEmpty) {
agg_function_sequence_count->destroy(place2);
}
+TEST_F(VSequenceMatchTest, testMatchInvalidEventNumber) {
+
assert_invalid_pattern_returns_zero<ColumnUInt8>(agg_function_sequence_match,
"(?0)", 3);
+
+ AggregateFunctionSimpleFactory factory =
AggregateFunctionSimpleFactory::instance();
+ DataTypes data_types = {std::make_shared<DataTypeString>(),
+ std::make_shared<DataTypeDateTimeV2>()};
+ for (size_t i = 0; i < MAX_EVENTS; ++i) {
+ data_types.push_back(std::make_shared<DataTypeUInt8>());
+ }
+ auto agg_function = factory.get("sequence_match", data_types, nullptr,
false, -1);
+ ASSERT_NE(agg_function, nullptr);
+ assert_invalid_pattern_returns_zero<ColumnUInt8>(agg_function, "(?33)",
MAX_EVENTS);
+}
+
+TEST_F(VSequenceMatchTest, testCountInvalidEventNumber) {
+
assert_invalid_pattern_returns_zero<ColumnInt64>(agg_function_sequence_count,
"(?0)", 3);
+
+ AggregateFunctionSimpleFactory factory =
AggregateFunctionSimpleFactory::instance();
+ DataTypes data_types = {std::make_shared<DataTypeString>(),
+ std::make_shared<DataTypeDateTimeV2>()};
+ for (size_t i = 0; i < MAX_EVENTS; ++i) {
+ data_types.push_back(std::make_shared<DataTypeUInt8>());
+ }
+ auto agg_function = factory.get("sequence_count", data_types, nullptr,
false, -1);
+ ASSERT_NE(agg_function, nullptr);
+ assert_invalid_pattern_returns_zero<ColumnInt64>(agg_function, "(?33)",
MAX_EVENTS);
+}
+
TEST_F(VSequenceMatchTest, testMatchSerialize) {
const int NUM_CONDS = 4;
auto column_pattern = ColumnString::create();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/SequenceFunction.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/SequenceFunction.java
index 6bbf4e6e992..3f3b4e53906 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/SequenceFunction.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/SequenceFunction.java
@@ -23,8 +23,13 @@ import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.functions.FunctionTrait;
import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
/** SequenceFunction */
public interface SequenceFunction extends FunctionTrait {
+ Pattern EVENT_PATTERN = Pattern.compile("\\(\\?(\\d+)\\)");
+
@Override
default void checkLegalityBeforeTypeCoercion() {
String functionName = getName();
@@ -42,6 +47,20 @@ public interface SequenceFunction extends FunctionTrait {
if (!FunctionCallExpr.parsePattern(pattern)) {
throw new AnalysisException("The format of pattern params is
wrong: " + this.toSql());
}
+ int eventCount = arity() - 2;
+ Matcher matcher = EVENT_PATTERN.matcher(pattern);
+ while (matcher.find()) {
+ long eventNumber;
+ try {
+ eventNumber = Long.parseLong(matcher.group(1));
+ } catch (NumberFormatException e) {
+ throw new AnalysisException("Event number " + matcher.group(1)
+ " is out of range");
+ }
+ if (eventNumber == 0 || eventNumber > eventCount) {
+ throw new AnalysisException("Event number " + eventNumber
+ + " is out of range, valid range is [1, " + eventCount
+ "]");
+ }
+ }
for (int i = 2; i < arity(); i++) {
if (!getArgumentType(i).isBooleanType()) {
diff --git
a/regression-test/suites/nereids_syntax_p0/test_nereids_function.groovy
b/regression-test/suites/nereids_syntax_p0/test_nereids_function.groovy
index 4bfb3839dc0..4d2fe2a8f32 100644
--- a/regression-test/suites/nereids_syntax_p0/test_nereids_function.groovy
+++ b/regression-test/suites/nereids_syntax_p0/test_nereids_function.groovy
@@ -55,4 +55,31 @@ suite("test_nereids_function") {
SELECT sequence_match('(?1)(?t>3600)(?2)', date, number = 1, number =
7) FROM sequence_match_test1;
"""
+ test {
+ sql """
+ SELECT sequence_match('(?0)', date, number = 1, number = 7) FROM
sequence_match_test1;
+ """
+ exception "Event number 0 is out of range"
+ }
+
+ test {
+ sql """
+ SELECT sequence_count('(?0)', date, number = 1, number = 7) FROM
sequence_match_test1;
+ """
+ exception "Event number 0 is out of range"
+ }
+
+ test {
+ sql """
+ SELECT sequence_match('(?3)', date, number = 1, number = 7) FROM
sequence_match_test1;
+ """
+ exception "Event number 3 is out of range"
+ }
+
+ test {
+ sql """
+ SELECT sequence_count('(?3)', date, number = 1, number = 7) FROM
sequence_match_test1;
+ """
+ exception "Event number 3 is out of range"
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
