ccl125 commented on code in PR #66050:
URL: https://github.com/apache/doris/pull/66050#discussion_r3829221916
##########
be/src/exprs/function/function_regexp.cpp:
##########
@@ -188,6 +195,87 @@ struct RegexpExtractEngine {
}
}
}
+
+ // Spark semantics used by the explicit group-index form: search the
original subject
+ // (so `^` stays anchored to the original string), emit successful
zero-width matches
+ // (including one terminal-position match), and advance one full UTF-8
character.
+ void match_all_and_extract_spark(const char* data, size_t size, int index,
+ std::vector<std::string>& results) const {
+ if (index < 0) {
+ return;
+ }
+ if (is_re2()) {
+ int max_matches = 1 + re2_regex->NumberOfCapturingGroups();
+ if (index >= max_matches) {
+ return;
+ }
+
+ size_t pos = 0;
+ while (pos <= size) {
+ std::vector<re2::StringPiece> matches(max_matches);
+ // Search within the original subject starting from pos, so
`^` stays
+ // anchored to the beginning of the original string.
+ bool success = re2_regex->Match(re2::StringPiece(data, size),
pos, size,
+ re2::RE2::UNANCHORED,
matches.data(), max_matches);
+ if (!success) {
+ break;
+ }
+ const re2::StringPiece& whole = matches[0];
+ // Extract the capturing group with the given index
+ if (static_cast<size_t>(index) < matches.size()) {
+ const re2::StringPiece& group = matches[index];
+ if (group.data() != nullptr) {
+ results.emplace_back(group.data(), group.size());
+ } else {
+ results.emplace_back();
+ }
+ }
+ if (whole.empty()) {
+ if (whole.data() >= data + size) {
+ break;
+ }
+ // Advance one full UTF-8 character FROM THE MATCH
LOCATION, never
+ // into a continuation byte and never rescanning the same
match.
+ pos = (whole.data() - data) +
Review Comment:
Fixed in 8e9b6b7 — advancement after an empty match is now one UTF-16 code
unit (Java Matcher semantics): a 4-byte UTF-8 character advances by half,
everything shorter by the whole character, so supplementary characters yield
the same empty-match count as Spark.
##########
be/src/exprs/function/function_regexp.cpp:
##########
@@ -188,6 +195,87 @@ struct RegexpExtractEngine {
}
}
}
+
+ // Spark semantics used by the explicit group-index form: search the
original subject
+ // (so `^` stays anchored to the original string), emit successful
zero-width matches
+ // (including one terminal-position match), and advance one full UTF-8
character.
+ void match_all_and_extract_spark(const char* data, size_t size, int index,
+ std::vector<std::string>& results) const {
+ if (index < 0) {
+ return;
+ }
+ if (is_re2()) {
+ int max_matches = 1 + re2_regex->NumberOfCapturingGroups();
+ if (index >= max_matches) {
+ return;
+ }
+
+ size_t pos = 0;
+ while (pos <= size) {
+ std::vector<re2::StringPiece> matches(max_matches);
+ // Search within the original subject starting from pos, so
`^` stays
+ // anchored to the beginning of the original string.
+ bool success = re2_regex->Match(re2::StringPiece(data, size),
pos, size,
+ re2::RE2::UNANCHORED,
matches.data(), max_matches);
+ if (!success) {
+ break;
+ }
+ const re2::StringPiece& whole = matches[0];
+ // Extract the capturing group with the given index
+ if (static_cast<size_t>(index) < matches.size()) {
+ const re2::StringPiece& group = matches[index];
+ if (group.data() != nullptr) {
+ results.emplace_back(group.data(), group.size());
+ } else {
+ results.emplace_back();
+ }
+ }
+ if (whole.empty()) {
+ if (whole.data() >= data + size) {
+ break;
+ }
+ // Advance one full UTF-8 character FROM THE MATCH
LOCATION, never
+ // into a continuation byte and never rescanning the same
match.
+ pos = (whole.data() - data) +
+
get_utf8_byte_length(static_cast<uint8_t>(*whole.data()));
+ } else {
+ // Advance past the match via its pointer into the
original subject.
+ pos = (whole.data() - data) + whole.size();
+ }
+ }
+ } else if (is_boost()) {
+ const char* search_start = data;
+ const char* search_end = data + size;
+ boost::match_results<const char*> matches;
+
+ // Keep the original subject start reachable: match_prev_avail lets
+ // look-behind assertions see characters before search_start, and
+ // match_not_bob keeps `\A` anchored to the start of the original
+ // buffer (Boost's documented repeated-regex_search idiom; with
+ // match_prev_avail set, `^` is decided by the preceding
character).
+ while (search_start <= search_end &&
Review Comment:
On `\G`: this is a pre-existing limitation rather than a regression — the
pre-PR loop also re-searched fresh substrings every iteration, so `\G` never
had previous-match state either. Note that RE2 does not support `\G` at all, so
such patterns only ever reach the boost path (enable_extended_regex). Making
`\G` Spark-exact needs a stateful matcher abstraction this function does not
have; I'd rather leave it out of scope here and track it separately if the
maintainers want it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
