This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 145dc006ea2 branch-3.1: [fix](storage) Ensure TOO_MANY_VERSION error
to be correctly detected and triggered in RowsetBuilder #55751 (#58350)
145dc006ea2 is described below
commit 145dc006ea27db94380c60ee311e5a979545c435
Author: Refrain <[email protected]>
AuthorDate: Wed Nov 26 10:49:01 2025 +0800
branch-3.1: [fix](storage) Ensure TOO_MANY_VERSION error to be correctly
detected and triggered in RowsetBuilder #55751 (#58350)
picked from #55751
---
be/src/olap/rowset_builder.cpp | 40 +++++++++-----------
.../insert/test_too_many_versions_detection.groovy | 44 ++++++++++++++++++++++
2 files changed, 62 insertions(+), 22 deletions(-)
diff --git a/be/src/olap/rowset_builder.cpp b/be/src/olap/rowset_builder.cpp
index 4ff0966b44d..2df1957902c 100644
--- a/be/src/olap/rowset_builder.cpp
+++ b/be/src/olap/rowset_builder.cpp
@@ -148,26 +148,11 @@ Status
BaseRowsetBuilder::init_mow_context(std::shared_ptr<MowContext>& mow_cont
}
Status RowsetBuilder::check_tablet_version_count() {
- bool injection = false;
-
DBUG_EXECUTE_IF("RowsetBuilder.check_tablet_version_count.too_many_version",
- { injection = true; });
- int32_t max_version_config = _tablet->max_version_config();
- if (injection) {
- // do not return if injection
- } else if (!_tablet->exceed_version_limit(max_version_config - 100) ||
-
GlobalMemoryArbitrator::is_exceed_soft_mem_limit(GB_EXCHANGE_BYTE)) {
- return Status::OK();
- }
- //trigger compaction
- auto st = _engine.submit_compaction_task(tablet_sptr(),
CompactionType::CUMULATIVE_COMPACTION,
- true);
- if (!st.ok()) [[unlikely]] {
- LOG(WARNING) << "failed to trigger compaction, tablet_id=" <<
_tablet->tablet_id() << " : "
- << st;
- }
- int version_count = tablet()->version_count();
+ auto max_version_config = _tablet->max_version_config();
+ auto version_count = tablet()->version_count();
DBUG_EXECUTE_IF("RowsetBuilder.check_tablet_version_count.too_many_version",
{ version_count = INT_MAX; });
+ // Trigger TOO MANY VERSION error first
if (version_count > max_version_config) {
return Status::Error<TOO_MANY_VERSION>(
"failed to init rowset builder. version count: {}, exceed
limit: {}, "
@@ -176,6 +161,20 @@ Status RowsetBuilder::check_tablet_version_count() {
"larger value.",
version_count, max_version_config, _tablet->tablet_id());
}
+ // (TODO Refrain) Maybe we can use a configurable param instead of
hardcoded values '100'.
+ // max_version_config must > 100, otherwise silent errors will occur.
+ if ((!config::disable_auto_compaction &&
+ !_tablet->tablet_meta()->tablet_schema()->disable_auto_compaction())
&&
+ (version_count > max_version_config - 100) &&
+ !GlobalMemoryArbitrator::is_exceed_soft_mem_limit(GB_EXCHANGE_BYTE)) {
+ // Trigger compaction
+ auto st = _engine.submit_compaction_task(tablet_sptr(),
+
CompactionType::CUMULATIVE_COMPACTION, true);
+ if (!st.ok()) [[unlikely]] {
+ LOG(WARNING) << "failed to trigger compaction, tablet_id=" <<
_tablet->tablet_id()
+ << " : " << st;
+ }
+ }
return Status::OK();
}
@@ -198,10 +197,7 @@ Status RowsetBuilder::init() {
RETURN_IF_ERROR(init_mow_context(mow_context));
}
- if (!config::disable_auto_compaction &&
- !_tablet->tablet_meta()->tablet_schema()->disable_auto_compaction()) {
- RETURN_IF_ERROR(check_tablet_version_count());
- }
+ RETURN_IF_ERROR(check_tablet_version_count());
int version_count = tablet()->version_count() +
tablet()->stale_version_count();
if (tablet()->avg_rs_meta_serialize_size() * version_count >
diff --git
a/regression-test/suites/load/insert/test_too_many_versions_detection.groovy
b/regression-test/suites/load/insert/test_too_many_versions_detection.groovy
new file mode 100644
index 00000000000..51acec36958
--- /dev/null
+++ b/regression-test/suites/load/insert/test_too_many_versions_detection.groovy
@@ -0,0 +1,44 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+import java.sql.SQLException
+
+suite("too_many_versions_detection") {
+ sql """ DROP TABLE IF EXISTS t """
+
+ sql """
+ create table t(a int)
+ DUPLICATE KEY(a)
+ DISTRIBUTED BY HASH(a)
+ BUCKETS 10 PROPERTIES("replication_num" = "1",
"disable_auto_compaction" = "true");
+ """
+
+ for (int i = 1; i <= 2000; i++) {
+ sql """ INSERT INTO t VALUES (${i}) """
+ }
+
+ try {
+ sql """ INSERT INTO t VALUES (2001) """
+ assertTrue(false, "Expected TOO_MANY_VERSION error but none occurred")
+ } catch (SQLException e) {
+ logger.info("Exception caught: ${e.getMessage()}")
+ def expectedError = "failed to init rowset builder. version count:
2001, exceed limit: 2000, tablet:"
+ assertTrue(e.getMessage().contains(expectedError),
+ "Expected TOO_MANY_VERSION error with message containing
'${expectedError}', but got: ${e.getMessage()}")
+ }
+
+ sql """ DROP TABLE IF EXISTS t """
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
