This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 2984a1b6c8d [improve](stream load) set NumberLoadedRows to zero when
stream load transaction failed (#41946) (#42265)
2984a1b6c8d is described below
commit 2984a1b6c8d69d9dccecd8b2b43184804657d8b0
Author: hui lai <[email protected]>
AuthorDate: Fri Oct 25 01:01:46 2024 +0800
[improve](stream load) set NumberLoadedRows to zero when stream load
transaction failed (#41946) (#42265)
pick (#41946)
Set NumberLoadedRows to zero when stream load failed.
before:
```
stream load result: {
"TxnId": 8589,
"Label": "c8e7c4fe-56b2-4e3b-b4cc-4f2a94cdd003",
"Comment": "",
"TwoPhaseCommit": "false",
"Status": "Fail",
"Message": "[DATA_QUALITY_ERROR]too many filtered rows",
"NumberTotalRows": 3,
"NumberLoadedRows": 1,
"NumberFilteredRows": 2,
"NumberUnselectedRows": 0,
"LoadBytes": 77,
"LoadTimeMs": 78,
"BeginTxnTimeMs": 0,
"StreamLoadPutTimeMs": 4,
"ReadDataTimeMs": 0,
"WriteDataTimeMs": 72,
"ReceiveDataTimeMs": 7,
"CommitAndPublishTimeMs": 0,
"ErrorURL": "XXX"
}
```
now:
```
stream load result: {
"TxnId": 8589,
"Label": "c8e7c4fe-56b2-4e3b-b4cc-4f2a94cdd003",
"Comment": "",
"TwoPhaseCommit": "false",
"Status": "Fail",
"Message": "[DATA_QUALITY_ERROR]too many filtered rows",
"NumberTotalRows": 3,
"NumberLoadedRows": 0,
"NumberFilteredRows": 2,
"NumberUnselectedRows": 0,
"LoadBytes": 77,
"LoadTimeMs": 78,
"BeginTxnTimeMs": 0,
"StreamLoadPutTimeMs": 4,
"ReadDataTimeMs": 0,
"WriteDataTimeMs": 72,
"ReceiveDataTimeMs": 7,
"CommitAndPublishTimeMs": 0,
"ErrorURL": "XXX"
}
```
---
be/src/runtime/stream_load/stream_load_executor.cpp | 1 +
.../suites/insert_p0/insert_group_commit_into_max_filter_ratio.groovy | 2 +-
regression-test/suites/json_p0/test_json_load_and_function.groovy | 2 +-
.../suites/json_p0/test_json_load_unique_key_and_function.groovy | 2 +-
regression-test/suites/jsonb_p0/test_jsonb_load_and_function.groovy | 2 +-
.../suites/jsonb_p0/test_jsonb_load_unique_key_and_function.groovy | 2 +-
.../load_json_column_exclude_schema_without_jsonpath.groovy | 4 ++--
.../suites/load_p0/stream_load/test_csv_with_none_utf8_data.groovy | 2 +-
regression-test/suites/load_p0/stream_load/test_stream_load.groovy | 4 ++--
.../suites/load_p0/stream_load/test_stream_load_move_memtable.groovy | 4 ++--
.../suites/load_p0/stream_load/test_stream_load_properties.groovy | 2 +-
regression-test/suites/nereids_function_p0/scalar_function/J.groovy | 2 +-
.../suites/nereids_p0/json_p0/test_json_load_and_function.groovy | 2 +-
.../nereids_p0/json_p0/test_json_load_unique_key_and_function.groovy | 2 +-
.../jsonb_p0/test_jsonb_load_unique_key_and_function.groovy | 2 +-
.../unique_with_mow_p0/partial_update/test_partial_update.groovy | 2 +-
.../partial_update/test_partial_update_strict_mode.groovy | 3 +--
17 files changed, 20 insertions(+), 20 deletions(-)
diff --git a/be/src/runtime/stream_load/stream_load_executor.cpp
b/be/src/runtime/stream_load/stream_load_executor.cpp
index 537433a3b47..e285d38208e 100644
--- a/be/src/runtime/stream_load/stream_load_executor.cpp
+++ b/be/src/runtime/stream_load/stream_load_executor.cpp
@@ -99,6 +99,7 @@ Status
StreamLoadExecutor::execute_plan_fragment(std::shared_ptr<StreamLoadConte
} else {
LOG(WARNING) << "fragment execute failed"
<< ", err_msg=" << status->to_string() << ", " <<
ctx->brief();
+ ctx->number_loaded_rows = 0;
// cancel body_sink, make sender known it
if (ctx->body_sink != nullptr) {
ctx->body_sink->cancel(status->to_string());
diff --git
a/regression-test/suites/insert_p0/insert_group_commit_into_max_filter_ratio.groovy
b/regression-test/suites/insert_p0/insert_group_commit_into_max_filter_ratio.groovy
index 5d6339d0894..a381b702e1c 100644
---
a/regression-test/suites/insert_p0/insert_group_commit_into_max_filter_ratio.groovy
+++
b/regression-test/suites/insert_p0/insert_group_commit_into_max_filter_ratio.groovy
@@ -117,7 +117,7 @@ suite("insert_group_commit_into_max_filter_ratio") {
assertTrue(json.GroupCommit)
// assertTrue(json.Label.startsWith("group_commit_"))
assertEquals(total_rows, json.NumberTotalRows)
- assertEquals(loaded_rows, json.NumberLoadedRows)
+ assertEquals(0, json.NumberLoadedRows)
assertEquals(filtered_rows, json.NumberFilteredRows)
assertEquals(unselected_rows, json.NumberUnselectedRows)
if (filtered_rows > 0) {
diff --git a/regression-test/suites/json_p0/test_json_load_and_function.groovy
b/regression-test/suites/json_p0/test_json_load_and_function.groovy
index d330ee4b345..e1ea767eb7d 100644
--- a/regression-test/suites/json_p0/test_json_load_and_function.groovy
+++ b/regression-test/suites/json_p0/test_json_load_and_function.groovy
@@ -61,7 +61,7 @@ suite("test_json_load_and_function", "p0") {
assertEquals("fail", json.Status.toLowerCase())
assertTrue(json.Message.contains("too many filtered rows"))
assertEquals(25, json.NumberTotalRows)
- assertEquals(18, json.NumberLoadedRows)
+ assertEquals(0, json.NumberLoadedRows)
assertEquals(7, json.NumberFilteredRows)
assertTrue(json.LoadBytes > 0)
log.info("url: " + json.ErrorURL)
diff --git
a/regression-test/suites/json_p0/test_json_load_unique_key_and_function.groovy
b/regression-test/suites/json_p0/test_json_load_unique_key_and_function.groovy
index 7d1dad49405..9aa556d3d53 100644
---
a/regression-test/suites/json_p0/test_json_load_unique_key_and_function.groovy
+++
b/regression-test/suites/json_p0/test_json_load_unique_key_and_function.groovy
@@ -56,7 +56,7 @@ suite("test_json_unique_load_and_function", "p0") {
assertEquals("fail", json.Status.toLowerCase())
assertTrue(json.Message.contains("too many filtered rows"))
assertEquals(75, json.NumberTotalRows)
- assertEquals(54, json.NumberLoadedRows)
+ assertEquals(0, json.NumberLoadedRows)
assertEquals(21, json.NumberFilteredRows)
assertTrue(json.LoadBytes > 0)
}
diff --git
a/regression-test/suites/jsonb_p0/test_jsonb_load_and_function.groovy
b/regression-test/suites/jsonb_p0/test_jsonb_load_and_function.groovy
index 4a55379d9fd..fc5f8cbe05a 100644
--- a/regression-test/suites/jsonb_p0/test_jsonb_load_and_function.groovy
+++ b/regression-test/suites/jsonb_p0/test_jsonb_load_and_function.groovy
@@ -63,7 +63,7 @@ suite("test_jsonb_load_and_function", "p0") {
assertEquals("fail", json.Status.toLowerCase())
assertTrue(json.Message.contains("too many filtered rows"))
assertEquals(25, json.NumberTotalRows)
- assertEquals(18, json.NumberLoadedRows)
+ assertEquals(0, json.NumberLoadedRows)
assertEquals(7, json.NumberFilteredRows)
assertTrue(json.LoadBytes > 0)
log.info("url: " + json.ErrorURL)
diff --git
a/regression-test/suites/jsonb_p0/test_jsonb_load_unique_key_and_function.groovy
b/regression-test/suites/jsonb_p0/test_jsonb_load_unique_key_and_function.groovy
index 526ddf4710e..b5946bea9a3 100644
---
a/regression-test/suites/jsonb_p0/test_jsonb_load_unique_key_and_function.groovy
+++
b/regression-test/suites/jsonb_p0/test_jsonb_load_unique_key_and_function.groovy
@@ -58,7 +58,7 @@ suite("test_jsonb_unique_load_and_function", "p0") {
assertEquals("fail", json.Status.toLowerCase())
assertTrue(json.Message.contains("too many filtered rows"))
assertEquals(75, json.NumberTotalRows)
- assertEquals(54, json.NumberLoadedRows)
+ assertEquals(0, json.NumberLoadedRows)
assertEquals(21, json.NumberFilteredRows)
assertTrue(json.LoadBytes > 0)
}
diff --git
a/regression-test/suites/load_p0/stream_load/load_json_column_exclude_schema_without_jsonpath.groovy
b/regression-test/suites/load_p0/stream_load/load_json_column_exclude_schema_without_jsonpath.groovy
index 06c8135840b..d7c1064de4f 100644
---
a/regression-test/suites/load_p0/stream_load/load_json_column_exclude_schema_without_jsonpath.groovy
+++
b/regression-test/suites/load_p0/stream_load/load_json_column_exclude_schema_without_jsonpath.groovy
@@ -73,8 +73,8 @@
suite("test_load_json_column_exclude_schema_without_jsonpath", "p0") {
log.info("Stream load result: ${result}".toString())
def json = parseJson(result)
assertEquals("fail", json.Status.toLowerCase())
- assertEquals(json.NumberTotalRows, json.NumberLoadedRows +
json.NumberFilteredRows)
- assertTrue(json.NumberLoadedRows == 30);
+ assertTrue(json.NumberTotalRows == 60)
+ assertTrue(json.NumberLoadedRows == 0)
assertTrue(json.NumberFilteredRows == 30);
}
}
diff --git
a/regression-test/suites/load_p0/stream_load/test_csv_with_none_utf8_data.groovy
b/regression-test/suites/load_p0/stream_load/test_csv_with_none_utf8_data.groovy
index bca699f7433..43494c332a2 100644
---
a/regression-test/suites/load_p0/stream_load/test_csv_with_none_utf8_data.groovy
+++
b/regression-test/suites/load_p0/stream_load/test_csv_with_none_utf8_data.groovy
@@ -60,7 +60,7 @@ suite("test_csv_with_none_utf8_data", "p0") {
assertEquals("fail", json.Status.toLowerCase())
assertTrue(json.Message.contains("too many filtered rows"))
assertEquals(4, json.NumberTotalRows)
- assertEquals(2, json.NumberLoadedRows)
+ assertEquals(0, json.NumberLoadedRows)
assertEquals(2, json.NumberFilteredRows)
assertTrue(json.LoadBytes > 0)
log.info("url: " + json.ErrorURL)
diff --git a/regression-test/suites/load_p0/stream_load/test_stream_load.groovy
b/regression-test/suites/load_p0/stream_load/test_stream_load.groovy
index 860ff10e14e..521c4e233ea 100644
--- a/regression-test/suites/load_p0/stream_load/test_stream_load.groovy
+++ b/regression-test/suites/load_p0/stream_load/test_stream_load.groovy
@@ -837,7 +837,7 @@ suite("test_stream_load", "p0") {
def json = parseJson(result)
assertEquals("fail", json.Status.toLowerCase())
assertEquals(5, json.NumberTotalRows)
- assertEquals(3, json.NumberLoadedRows)
+ assertEquals(0, json.NumberLoadedRows)
assertEquals(2, json.NumberFilteredRows)
assertEquals(0, json.NumberUnselectedRows)
}
@@ -941,7 +941,7 @@ suite("test_stream_load", "p0") {
def json = parseJson(result)
assertEquals("fail", json.Status.toLowerCase())
assertEquals(5, json.NumberTotalRows)
- assertEquals(3, json.NumberLoadedRows)
+ assertEquals(0, json.NumberLoadedRows)
assertEquals(2, json.NumberFilteredRows)
assertEquals(0, json.NumberUnselectedRows)
}
diff --git
a/regression-test/suites/load_p0/stream_load/test_stream_load_move_memtable.groovy
b/regression-test/suites/load_p0/stream_load/test_stream_load_move_memtable.groovy
index 09d9e57bf6e..8a2f7b0247d 100644
---
a/regression-test/suites/load_p0/stream_load/test_stream_load_move_memtable.groovy
+++
b/regression-test/suites/load_p0/stream_load/test_stream_load_move_memtable.groovy
@@ -661,7 +661,7 @@ suite("test_stream_load_move_memtable", "p0") {
def json = parseJson(result)
assertEquals("fail", json.Status.toLowerCase())
assertEquals(5, json.NumberTotalRows)
- assertEquals(3, json.NumberLoadedRows)
+ assertEquals(0, json.NumberLoadedRows)
assertEquals(2, json.NumberFilteredRows)
assertEquals(0, json.NumberUnselectedRows)
}
@@ -767,7 +767,7 @@ suite("test_stream_load_move_memtable", "p0") {
def json = parseJson(result)
assertEquals("fail", json.Status.toLowerCase())
assertEquals(5, json.NumberTotalRows)
- assertEquals(3, json.NumberLoadedRows)
+ assertEquals(0, json.NumberLoadedRows)
assertEquals(2, json.NumberFilteredRows)
assertEquals(0, json.NumberUnselectedRows)
}
diff --git
a/regression-test/suites/load_p0/stream_load/test_stream_load_properties.groovy
b/regression-test/suites/load_p0/stream_load/test_stream_load_properties.groovy
index 83dc83efdfe..60800e6cc4d 100644
---
a/regression-test/suites/load_p0/stream_load/test_stream_load_properties.groovy
+++
b/regression-test/suites/load_p0/stream_load/test_stream_load_properties.groovy
@@ -284,7 +284,7 @@ suite("test_stream_load_properties", "p0") {
def json = parseJson(result)
assertEquals("fail", json.Status.toLowerCase())
assertEquals(20, json.NumberTotalRows)
- assertEquals(loadedRows[i], json.NumberLoadedRows)
+ assertEquals(0, json.NumberLoadedRows)
assertEquals(filteredRows[i], json.NumberFilteredRows)
assertEquals(0, json.NumberUnselectedRows)
}
diff --git
a/regression-test/suites/nereids_function_p0/scalar_function/J.groovy
b/regression-test/suites/nereids_function_p0/scalar_function/J.groovy
index ecb2a9dc531..90336893c5e 100644
--- a/regression-test/suites/nereids_function_p0/scalar_function/J.groovy
+++ b/regression-test/suites/nereids_function_p0/scalar_function/J.groovy
@@ -62,7 +62,7 @@ suite("nereids_scalar_fn_J") {
assertEquals("fail", json.Status.toLowerCase())
assertTrue(json.Message.contains("too many filtered rows"))
assertEquals(25, json.NumberTotalRows)
- assertEquals(18, json.NumberLoadedRows)
+ assertEquals(0, json.NumberLoadedRows)
assertEquals(7, json.NumberFilteredRows)
assertTrue(json.LoadBytes > 0)
log.info("url: " + json.ErrorURL)
diff --git
a/regression-test/suites/nereids_p0/json_p0/test_json_load_and_function.groovy
b/regression-test/suites/nereids_p0/json_p0/test_json_load_and_function.groovy
index e623e595f00..e3aa0fb607d 100644
---
a/regression-test/suites/nereids_p0/json_p0/test_json_load_and_function.groovy
+++
b/regression-test/suites/nereids_p0/json_p0/test_json_load_and_function.groovy
@@ -63,7 +63,7 @@ suite("test_json_load_and_function", "p0") {
assertEquals("fail", json.Status.toLowerCase())
assertTrue(json.Message.contains("too many filtered rows"))
assertEquals(25, json.NumberTotalRows)
- assertEquals(18, json.NumberLoadedRows)
+ assertEquals(0, json.NumberLoadedRows)
assertEquals(7, json.NumberFilteredRows)
assertTrue(json.LoadBytes > 0)
log.info("url: " + json.ErrorURL)
diff --git
a/regression-test/suites/nereids_p0/json_p0/test_json_load_unique_key_and_function.groovy
b/regression-test/suites/nereids_p0/json_p0/test_json_load_unique_key_and_function.groovy
index 4aafc86b566..706c585d3a9 100644
---
a/regression-test/suites/nereids_p0/json_p0/test_json_load_unique_key_and_function.groovy
+++
b/regression-test/suites/nereids_p0/json_p0/test_json_load_unique_key_and_function.groovy
@@ -58,7 +58,7 @@ suite("test_json_unique_load_and_function", "p0") {
assertEquals("fail", json.Status.toLowerCase())
assertTrue(json.Message.contains("too many filtered rows"))
assertEquals(75, json.NumberTotalRows)
- assertEquals(54, json.NumberLoadedRows)
+ assertEquals(0, json.NumberLoadedRows)
assertEquals(21, json.NumberFilteredRows)
assertTrue(json.LoadBytes > 0)
}
diff --git
a/regression-test/suites/nereids_p0/jsonb_p0/test_jsonb_load_unique_key_and_function.groovy
b/regression-test/suites/nereids_p0/jsonb_p0/test_jsonb_load_unique_key_and_function.groovy
index 70b5655a1c4..a8ad02cb071 100644
---
a/regression-test/suites/nereids_p0/jsonb_p0/test_jsonb_load_unique_key_and_function.groovy
+++
b/regression-test/suites/nereids_p0/jsonb_p0/test_jsonb_load_unique_key_and_function.groovy
@@ -58,7 +58,7 @@ suite("test_jsonb_unique_load_and_function", "p0") {
assertEquals("fail", json.Status.toLowerCase())
assertTrue(json.Message.contains("too many filtered rows"))
assertEquals(75, json.NumberTotalRows)
- assertEquals(54, json.NumberLoadedRows)
+ assertEquals(0, json.NumberLoadedRows)
assertEquals(21, json.NumberFilteredRows)
assertTrue(json.LoadBytes > 0)
}
diff --git
a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update.groovy
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update.groovy
index 9d2410a11d2..ea3dde5bf5b 100644
---
a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update.groovy
+++
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update.groovy
@@ -176,7 +176,7 @@ suite("test_primary_key_partial_update", "p0") {
assertEquals("Fail", json.Status)
assertTrue(json.Message.contains("[DATA_QUALITY_ERROR]too
many filtered rows"))
assertEquals(3, json.NumberTotalRows)
- assertEquals(1, json.NumberLoadedRows)
+ assertEquals(0, json.NumberLoadedRows)
assertEquals(2, json.NumberFilteredRows)
}
}
diff --git
a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_strict_mode.groovy
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_strict_mode.groovy
index 5fad6b4aca1..fe47a5ebc89 100644
---
a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_strict_mode.groovy
+++
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_strict_mode.groovy
@@ -117,7 +117,7 @@ suite("test_partial_update_strict_mode", "p0") {
assertEquals("Fail", json.Status)
assertTrue(json.Message.contains("[DATA_QUALITY_ERROR]too
many filtered rows"))
assertEquals(3, json.NumberTotalRows)
- assertEquals(1, json.NumberLoadedRows)
+ assertEquals(0, json.NumberLoadedRows)
assertEquals(2, json.NumberFilteredRows)
}
}
@@ -166,7 +166,6 @@ suite("test_partial_update_strict_mode", "p0") {
assertEquals("Fail", json.Status)
assertTrue(json.Message.contains("[DATA_QUALITY_ERROR]too
many filtered rows"))
assertEquals(3, json.NumberTotalRows)
- assertEquals(1, json.NumberLoadedRows)
assertEquals(2, json.NumberFilteredRows)
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
