This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 490963634b8 [enhancement](time_series) increase the version limit for
the time series table (#51371) (#52077)
490963634b8 is described below
commit 490963634b8555227a0763a260bb4c4164a8b5c3
Author: Luwei <[email protected]>
AuthorDate: Tue Jun 24 09:44:24 2025 +0800
[enhancement](time_series) increase the version limit for the time series
table (#51371) (#52077)
pick master #51371
---
be/src/agent/task_worker_pool.cpp | 3 ++-
be/src/common/config.cpp | 2 ++
be/src/common/config.h | 2 ++
be/src/olap/base_tablet.cpp | 10 +++++++++-
be/src/olap/base_tablet.h | 2 ++
be/src/olap/olap_server.cpp | 3 ++-
be/src/olap/push_handler.cpp | 9 ++++++---
be/src/olap/rowset_builder.cpp | 10 ++++++----
8 files changed, 31 insertions(+), 10 deletions(-)
diff --git a/be/src/agent/task_worker_pool.cpp
b/be/src/agent/task_worker_pool.cpp
index 9f69b0ffb73..c36426b1954 100644
--- a/be/src/agent/task_worker_pool.cpp
+++ b/be/src/agent/task_worker_pool.cpp
@@ -1594,8 +1594,9 @@ void
PublishVersionWorkerPool::publish_version_callback(const TAgentTaskRequest&
if
(!tablet->tablet_meta()->tablet_schema()->disable_auto_compaction()) {
tablet->published_count.fetch_add(1);
int64_t published_count =
tablet->published_count.load();
+ int32_t max_version_config =
tablet->max_version_config();
if (tablet->exceed_version_limit(
- config::max_tablet_version_num *
+ max_version_config *
config::load_trigger_compaction_version_percent / 100) &&
published_count % 20 == 0) {
auto st = _engine.submit_compaction_task(
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 99850f86a66..659324813a7 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -809,6 +809,8 @@ DEFINE_Int32(query_cache_max_partition_count, "1024");
// This is to avoid too many version num.
DEFINE_mInt32(max_tablet_version_num, "2000");
+DEFINE_mInt32(time_series_max_tablet_version_num, "20000");
+
// Frontend mainly use two thrift sever type: THREAD_POOL, THREADED_SELECTOR.
if fe use THREADED_SELECTOR model for thrift server,
// the thrift_server_type_of_fe should be set THREADED_SELECTOR to make be
thrift client to fe constructed with TFramedTransport
DEFINE_String(thrift_server_type_of_fe, "THREAD_POOL");
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 13bcbad7294..2cbc8b2c0d5 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -854,6 +854,8 @@ DECLARE_Int32(query_cache_max_partition_count);
// This is to avoid too many version num.
DECLARE_mInt32(max_tablet_version_num);
+DECLARE_mInt32(time_series_max_tablet_version_num);
+
// Frontend mainly use two thrift sever type: THREAD_POOL, THREADED_SELECTOR.
if fe use THREADED_SELECTOR model for thrift server,
// the thrift_server_type_of_fe should be set THREADED_SELECTOR to make be
thrift client to fe constructed with TFramedTransport
DECLARE_String(thrift_server_type_of_fe);
diff --git a/be/src/olap/base_tablet.cpp b/be/src/olap/base_tablet.cpp
index 3d8ef137387..9c5cbb9b184 100644
--- a/be/src/olap/base_tablet.cpp
+++ b/be/src/olap/base_tablet.cpp
@@ -19,6 +19,7 @@
#include <fmt/format.h>
+#include "olap/cumulative_compaction_time_series_policy.h"
#include "olap/tablet_fwd.h"
#include "olap/tablet_schema_cache.h"
#include "util/doris_metrics.h"
@@ -91,4 +92,11 @@ uint32_t BaseTablet::get_real_compaction_score() const {
});
}
-} /* namespace doris */
+int32_t BaseTablet::max_version_config() {
+ int32_t max_version = tablet_meta()->compaction_policy() ==
CUMULATIVE_TIME_SERIES_POLICY
+ ? config::time_series_max_tablet_version_num
+ : config::max_tablet_version_num;
+ return max_version;
+}
+
+} // namespace doris
diff --git a/be/src/olap/base_tablet.h b/be/src/olap/base_tablet.h
index 4fdc81aa985..b50eec71485 100644
--- a/be/src/olap/base_tablet.h
+++ b/be/src/olap/base_tablet.h
@@ -64,6 +64,8 @@ public:
// Property encapsulated in TabletMeta
const TabletMetaSharedPtr& tablet_meta() { return _tablet_meta; }
+ int32 max_version_config();
+
// FIXME(plat1ko): It is not appropriate to expose this lock
std::shared_mutex& get_header_lock() { return _meta_lock; }
diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp
index 5b1cfb9ba59..6a87ca434c3 100644
--- a/be/src/olap/olap_server.cpp
+++ b/be/src/olap/olap_server.cpp
@@ -1611,8 +1611,9 @@ void StorageEngine::_process_async_publish() {
continue;
}
if (version != max_version + 1) {
+ int32_t max_version_config = tablet->max_version_config();
// Keep only the most recent versions
- while (tablet_iter->second.size() >
config::max_tablet_version_num) {
+ while (tablet_iter->second.size() > max_version_config) {
need_removed_tasks.emplace_back(tablet, version);
task_iter = tablet_iter->second.erase(task_iter);
version = task_iter->first;
diff --git a/be/src/olap/push_handler.cpp b/be/src/olap/push_handler.cpp
index cf03c6fa3b9..10661ccfe8b 100644
--- a/be/src/olap/push_handler.cpp
+++ b/be/src/olap/push_handler.cpp
@@ -39,6 +39,7 @@
#include "common/config.h"
#include "common/logging.h"
#include "common/status.h"
+#include "olap/cumulative_compaction_time_series_policy.h"
#include "olap/delete_handler.h"
#include "olap/olap_define.h"
#include "olap/rowset/pending_rowset_helper.h"
@@ -161,13 +162,15 @@ Status
PushHandler::_do_streaming_ingestion(TabletSharedPtr tablet, const TPushR
}
}
+ int32_t max_version_config = tablet->max_version_config();
// check if version number exceed limit
- if (tablet->exceed_version_limit(config::max_tablet_version_num)) {
+ if (tablet->exceed_version_limit(max_version_config)) {
return Status::Status::Error<TOO_MANY_VERSION>(
"failed to push data. version count: {}, exceed limit: {},
tablet: {}. Please "
- "reduce the frequency of loading data or adjust the
max_tablet_version_num in "
+ "reduce the frequency of loading data or adjust the
max_tablet_version_num or "
+ "time_series_max_tablet_version_num in "
"be.conf to a larger value.",
- tablet->version_count(), config::max_tablet_version_num,
tablet->tablet_id());
+ tablet->version_count(), max_version_config,
tablet->tablet_id());
}
int version_count = tablet->version_count() +
tablet->stale_version_count();
diff --git a/be/src/olap/rowset_builder.cpp b/be/src/olap/rowset_builder.cpp
index deadb91c548..ca746f8aac7 100644
--- a/be/src/olap/rowset_builder.cpp
+++ b/be/src/olap/rowset_builder.cpp
@@ -144,9 +144,10 @@ Status RowsetBuilder::check_tablet_version_count() {
bool injection = false;
DBUG_EXECUTE_IF("RowsetBuilder.check_tablet_version_count.too_many_version",
{ injection = true; });
+ int32_t max_version_config = _tablet->max_version_config();
if (injection) {
// do not return if injection
- } else if (!_tablet->exceed_version_limit(config::max_tablet_version_num -
100) ||
+ } else if (!_tablet->exceed_version_limit(max_version_config - 100) ||
GlobalMemoryArbitrator::is_exceed_soft_mem_limit(GB_EXCHANGE_BYTE)) {
return Status::OK();
}
@@ -160,12 +161,13 @@ Status RowsetBuilder::check_tablet_version_count() {
int version_count = tablet()->version_count();
DBUG_EXECUTE_IF("RowsetBuilder.check_tablet_version_count.too_many_version",
{ version_count = INT_MAX; });
- if (version_count > config::max_tablet_version_num) {
+ if (version_count > max_version_config) {
return Status::Error<TOO_MANY_VERSION>(
"failed to init rowset builder. version count: {}, exceed
limit: {}, "
"tablet: {}. Please reduce the frequency of loading data or
adjust the "
- "max_tablet_version_num in be.conf to a larger value.",
- version_count, config::max_tablet_version_num,
_tablet->tablet_id());
+ "max_tablet_version_num or time_series_max_tablet_version_num
in be.conf to a "
+ "larger value.",
+ version_count, max_version_config, _tablet->tablet_id());
}
return Status::OK();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
