This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new d7f61a9cb6d branch-4.1: [fix](be) Fix strict load crash when direct
slot mappings are absent #67633 (#67953)
d7f61a9cb6d is described below
commit d7f61a9cb6df7098f600e292edfccab1a5a15b24
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Tue Sep 15 12:01:43 2026 +0800
branch-4.1: [fix](be) Fix strict load crash when direct slot mappings are
absent #67633 (#67953)
Cherry-picked from #67633
Co-authored-by: meiyi <[email protected]>
---
be/src/exec/scan/file_scanner.cpp | 24 ++---
...st_stream_load_strict_mode_and_filter_ratio.out | 32 ++++++
...stream_load_strict_mode_and_filter_ratio.groovy | 116 ++++++++++++++++++++-
3 files changed, 157 insertions(+), 15 deletions(-)
diff --git a/be/src/exec/scan/file_scanner.cpp
b/be/src/exec/scan/file_scanner.cpp
index 209d7db1d75..d7c262dc77a 100644
--- a/be/src/exec/scan/file_scanner.cpp
+++ b/be/src/exec/scan/file_scanner.cpp
@@ -1896,7 +1896,6 @@ Status FileScanner::_init_expr_ctxes() {
if (_is_load) {
// follow desc expr map is only for load task.
- bool has_slot_id_map =
_params->__isset.dest_sid_to_src_sid_without_trans;
int idx = 0;
for (auto* slot_desc : _output_tuple_desc->slots()) {
auto it = _params->expr_of_dest_slot.find(slot_desc->id());
@@ -1914,20 +1913,17 @@ Status FileScanner::_init_expr_ctxes() {
_dest_vexpr_ctx.emplace_back(ctx);
_dest_slot_name_to_idx[slot_desc->col_name()] = idx++;
- if (has_slot_id_map) {
- auto it1 =
_params->dest_sid_to_src_sid_without_trans.find(slot_desc->id());
- if (it1 ==
std::end(_params->dest_sid_to_src_sid_without_trans)) {
- _src_slot_descs_order_by_dest.emplace_back(nullptr);
- } else {
- auto _src_slot_it = full_src_slot_map.find(it1->second);
- if (_src_slot_it == std::end(full_src_slot_map)) {
- return Status::InternalError("No src slot {} in src
slot descs",
- it1->second);
- }
-
_dest_slot_to_src_slot_index.emplace(_src_slot_descs_order_by_dest.size(),
-
full_src_index_map[_src_slot_it->first]);
-
_src_slot_descs_order_by_dest.emplace_back(_src_slot_it->second);
+ auto it1 =
_params->dest_sid_to_src_sid_without_trans.find(slot_desc->id());
+ if (it1 == std::end(_params->dest_sid_to_src_sid_without_trans)) {
+ _src_slot_descs_order_by_dest.emplace_back(nullptr);
+ } else {
+ auto _src_slot_it = full_src_slot_map.find(it1->second);
+ if (_src_slot_it == std::end(full_src_slot_map)) {
+ return Status::InternalError("No src slot {} in src slot
descs", it1->second);
}
+
_dest_slot_to_src_slot_index.emplace(_src_slot_descs_order_by_dest.size(),
+
full_src_index_map[_src_slot_it->first]);
+
_src_slot_descs_order_by_dest.emplace_back(_src_slot_it->second);
}
}
}
diff --git
a/regression-test/data/load_p0/stream_load/test_stream_load_strict_mode_and_filter_ratio.out
b/regression-test/data/load_p0/stream_load/test_stream_load_strict_mode_and_filter_ratio.out
index 55ac0281d95..3877fb5c663 100644
---
a/regression-test/data/load_p0/stream_load/test_stream_load_strict_mode_and_filter_ratio.out
+++
b/regression-test/data/load_p0/stream_load/test_stream_load_strict_mode_and_filter_ratio.out
@@ -112,3 +112,35 @@
70 a70
79 a79
+-- !sql_all_expr_strict --
+1234567890
+1234567890
+1234567893
+1234567894
+1234567895
+1234567896
+1234567897
+
+-- !sql_all_expr_strict_filter_ratio_exceeded --
+
+-- !sql_all_expr_nullable_strict --
+\N
+\N
+\N
+1234567890
+1234567890
+1234567893
+1234567894
+1234567895
+1234567896
+1234567897
+
+-- !sql_mixed_mapping_strict --
+\N 1234567890
+\N 1234567890
+\N 1234567893
+\N 1234567894
+\N 1234567895
+\N 1234567896
+\N 1234567897
+
diff --git
a/regression-test/suites/load_p0/stream_load/test_stream_load_strict_mode_and_filter_ratio.groovy
b/regression-test/suites/load_p0/stream_load/test_stream_load_strict_mode_and_filter_ratio.groovy
index 7e5ca5f1064..bc6e9551499 100644
---
a/regression-test/suites/load_p0/stream_load/test_stream_load_strict_mode_and_filter_ratio.groovy
+++
b/regression-test/suites/load_p0/stream_load/test_stream_load_strict_mode_and_filter_ratio.groovy
@@ -523,4 +523,118 @@ suite("test_stream_load_strict_mode_and_filter_ratio",
"p0") {
}
}
qt_sql_string_exceed_len_strict1 "select * from
test_stream_load_strict_mode_and_filter_ratio order by 1"
-}
\ No newline at end of file
+
+ // Test strict-mode loading into a NOT NULL destination with only
expression mappings.
+ // Invalid casts produce NULL and must be filtered even when the direct
slot map is unset.
+ sql """ drop table if exists test_stream_load_strict_mode_and_filter_ratio
"""
+ sql """
+ create table test_stream_load_strict_mode_and_filter_ratio (
+ k00 DECIMALV3(10, 0) NOT NULL
+ ) properties ('replication_num' = '1');
+ """
+ streamLoad {
+ table "test_stream_load_strict_mode_and_filter_ratio"
+ file "test_not_number.csv"
+ set 'column_separator', '|'
+ set 'columns', 'src, k00=cast(src as bigint)'
+ set 'strict_mode', 'true'
+ set 'max_filter_ratio', '0.3'
+
+ check { result, exception, startTime, endTime ->
+ if (exception != null) {
+ throw exception
+ }
+ def json = parseJson(result)
+ assertEquals("success", json.Status.toLowerCase())
+ assertEquals(10, json.NumberTotalRows)
+ assertEquals(7, json.NumberLoadedRows)
+ assertEquals(3, json.NumberFilteredRows)
+ assertTrue(result.contains("ErrorURL"))
+ }
+ }
+ qt_sql_all_expr_strict "select * from
test_stream_load_strict_mode_and_filter_ratio order by 1"
+
+ // Exceeding the filter ratio must fail the load without committing any
rows.
+ sql """ truncate table test_stream_load_strict_mode_and_filter_ratio """
+ streamLoad {
+ table "test_stream_load_strict_mode_and_filter_ratio"
+ file "test_not_number.csv"
+ set 'column_separator', '|'
+ set 'columns', 'src, k00=cast(src as bigint)'
+ set 'strict_mode', 'true'
+ set 'max_filter_ratio', '0.2'
+
+ check { result, exception, startTime, endTime ->
+ if (exception != null) {
+ throw exception
+ }
+ def json = parseJson(result)
+ assertEquals("fail", json.Status.toLowerCase())
+ assertEquals(10, json.NumberTotalRows)
+ assertEquals(0, json.NumberLoadedRows)
+ assertEquals(3, json.NumberFilteredRows)
+ assertTrue(json.Message.contains("too many filtered rows"))
+ assertTrue(result.contains("ErrorURL"))
+ }
+ }
+ qt_sql_all_expr_strict_filter_ratio_exceeded "select * from
test_stream_load_strict_mode_and_filter_ratio order by 1"
+
+ // With no direct slot mapping, expression NULLs are retained by a
nullable destination.
+ sql """ drop table if exists test_stream_load_strict_mode_and_filter_ratio
"""
+ sql """
+ create table test_stream_load_strict_mode_and_filter_ratio (
+ k00 DECIMALV3(10, 0) NULL
+ ) properties ('replication_num' = '1');
+ """
+ streamLoad {
+ table "test_stream_load_strict_mode_and_filter_ratio"
+ file "test_not_number.csv"
+ set 'column_separator', '|'
+ set 'columns', 'src, k00=cast(src as bigint)'
+ set 'strict_mode', 'true'
+ set 'max_filter_ratio', '0'
+
+ check { result, exception, startTime, endTime ->
+ if (exception != null) {
+ throw exception
+ }
+ def json = parseJson(result)
+ assertEquals("success", json.Status.toLowerCase())
+ assertEquals(10, json.NumberTotalRows)
+ assertEquals(10, json.NumberLoadedRows)
+ assertEquals(0, json.NumberFilteredRows)
+ }
+ }
+ qt_sql_all_expr_nullable_strict "select * from
test_stream_load_strict_mode_and_filter_ratio order by 1"
+
+ // Put an expression before a direct column to check destination/source
index alignment.
+ // The expression is always NULL; strict mode only filters the three
invalid direct values.
+ sql """ drop table if exists test_stream_load_strict_mode_and_filter_ratio
"""
+ sql """
+ create table test_stream_load_strict_mode_and_filter_ratio (
+ k00 DECIMALV3(10, 0) NULL,
+ k01 DECIMALV3(10, 0) NULL
+ ) properties ('replication_num' = '1');
+ """
+ streamLoad {
+ table "test_stream_load_strict_mode_and_filter_ratio"
+ file "test_not_number.csv"
+ set 'column_separator', '|'
+ set 'columns', "k01, k00=cast(concat('invalid', k01) as bigint)"
+ set 'strict_mode', 'true'
+ set 'max_filter_ratio', '0.3'
+
+ check { result, exception, startTime, endTime ->
+ if (exception != null) {
+ throw exception
+ }
+ def json = parseJson(result)
+ assertEquals("success", json.Status.toLowerCase())
+ assertEquals(10, json.NumberTotalRows)
+ assertEquals(7, json.NumberLoadedRows)
+ assertEquals(3, json.NumberFilteredRows)
+ assertTrue(result.contains("ErrorURL"))
+ }
+ }
+ qt_sql_mixed_mapping_strict "select * from
test_stream_load_strict_mode_and_filter_ratio order by 1, 2"
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
