(doris) branch branch-4.1 updated: branch-4.1: [fix](function) Split UTF-8 strings correctly with empty regexp #68020 (#68417)

[yiguolei]

This is an automated email from the ASF dual-hosted git repository.

yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/branch-4.1 by this push:
     new 518f5f6a8d6 branch-4.1: [fix](function) Split UTF-8 strings correctly 
with empty regexp #68020 (#68417)
518f5f6a8d6 is described below

commit 518f5f6a8d6b5f3c03e1713f1636bc8372da25f3
Author: github-actions[bot] 
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Thu Sep 24 09:22:20 2026 +0800

    branch-4.1: [fix](function) Split UTF-8 strings correctly with empty regexp 
#68020 (#68417)
    
    Cherry-picked from #68020
    
    Co-authored-by: Mryange <yanxuech...@selectdb.com>
---
 be/src/exprs/function/function_split_by_regexp.cpp                  | 4 +++-
 .../sql_functions/string_functions/test_split_by_regexp.out         | 6 ++++++
 .../sql_functions/string_functions/test_split_by_regexp.groovy      | 2 ++
 3 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/be/src/exprs/function/function_split_by_regexp.cpp 
b/be/src/exprs/function/function_split_by_regexp.cpp
index 36eb737f0d2..d3e79d3a7af 100644
--- a/be/src/exprs/function/function_split_by_regexp.cpp
+++ b/be/src/exprs/function/function_split_by_regexp.cpp
@@ -27,6 +27,7 @@
 #include "core/types.h"
 #include "exprs/function/function.h"
 #include "exprs/function/simple_function_factory.h"
+#include "util/simd/vstring_function.h"
 
 namespace doris {
 #include "common/compile_check_begin.h"
@@ -116,7 +117,8 @@ bool RegexpSplit::get(const char*& token_begin, const 
char*& token_end) {
             }
         }
 
-        _pos += 1;
+        const auto utf8_byte_length = 
get_utf8_byte_length(static_cast<uint8_t>(*_pos));
+        _pos += std::min<size_t>(utf8_byte_length, _end - _pos);
         token_end = _pos;
         ++_splits;
     } else {
diff --git 
a/regression-test/data/query_p0/sql_functions/string_functions/test_split_by_regexp.out
 
b/regression-test/data/query_p0/sql_functions/string_functions/test_split_by_regexp.out
index fa6e31abd94..d9184915ab7 100644
--- 
a/regression-test/data/query_p0/sql_functions/string_functions/test_split_by_regexp.out
+++ 
b/regression-test/data/query_p0/sql_functions/string_functions/test_split_by_regexp.out
@@ -2,6 +2,12 @@
 -- !select1 --
 ["a", "b", "c", "d", "e"]
 
+-- !select_utf8_empty_pattern --
+["中", "a", "😀"]
+
+-- !select_utf8_empty_pattern_with_limit --
+["中", "a😀"]
+
 -- !select2 --
 ["a", "bc", "de", "f"]
 
diff --git 
a/regression-test/suites/query_p0/sql_functions/string_functions/test_split_by_regexp.groovy
 
b/regression-test/suites/query_p0/sql_functions/string_functions/test_split_by_regexp.groovy
index 678f7221642..9754ca6e592 100644
--- 
a/regression-test/suites/query_p0/sql_functions/string_functions/test_split_by_regexp.groovy
+++ 
b/regression-test/suites/query_p0/sql_functions/string_functions/test_split_by_regexp.groovy
@@ -17,6 +17,8 @@
 
 suite("test_split_by_regexp") {
     qt_select1 "select split_by_regexp('abcde','');"
+    order_qt_select_utf8_empty_pattern "select split_by_regexp('中a😀', '');"
+    order_qt_select_utf8_empty_pattern_with_limit "select 
split_by_regexp('中a😀', '', 2);"
     qt_select2 "select split_by_regexp('a12bc23de345f','\\\\d+');"
     qt_select3 "select split_by_regexp('a12bc23de345f',NULL);"
     qt_select4 "select split_by_regexp(NULL, 'a12bc23de345f');"


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org