This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 82f86a8730b [enhancement](merge-on-write) consider version count on
size-based cloud cu compaction policy (#33244)
82f86a8730b is described below
commit 82f86a8730bc172a3570cfddc4b5d334a3b100d4
Author: huanghaibin <[email protected]>
AuthorDate: Thu Apr 4 18:07:19 2024 +0800
[enhancement](merge-on-write) consider version count on size-based cloud cu
compaction policy (#33244)
---
be/src/cloud/cloud_cumulative_compaction_policy.cpp | 16 +++++++++++++---
be/src/cloud/cloud_cumulative_compaction_policy.h | 5 ++++-
2 files changed, 17 insertions(+), 4 deletions(-)
diff --git a/be/src/cloud/cloud_cumulative_compaction_policy.cpp
b/be/src/cloud/cloud_cumulative_compaction_policy.cpp
index f2e4411897b..5875340ec7b 100644
--- a/be/src/cloud/cloud_cumulative_compaction_policy.cpp
+++ b/be/src/cloud/cloud_cumulative_compaction_policy.cpp
@@ -34,11 +34,12 @@ namespace doris {
CloudSizeBasedCumulativeCompactionPolicy::CloudSizeBasedCumulativeCompactionPolicy(
int64_t promotion_size, double promotion_ratio, int64_t
promotion_min_size,
- int64_t compaction_min_size)
+ int64_t compaction_min_size, int64_t promotion_version_count)
: _promotion_size(promotion_size),
_promotion_ratio(promotion_ratio),
_promotion_min_size(promotion_min_size),
- _compaction_min_size(compaction_min_size) {}
+ _compaction_min_size(compaction_min_size),
+ _promotion_version_count(promotion_version_count) {}
int64_t CloudSizeBasedCumulativeCompactionPolicy::_level_size(const int64_t
size) {
if (size < 1024) return 0;
@@ -194,11 +195,20 @@ int64_t
CloudSizeBasedCumulativeCompactionPolicy::new_cumulative_point(
int64_t last_cumulative_point) {
TEST_INJECTION_POINT_RETURN_WITH_VALUE("new_cumulative_point", int64_t(0),
output_rowset.get(),
last_cumulative_point);
+ // for MoW table, if there's too many versions, the delete bitmap will
grow to
+ // a very big size, which may cause the tablet meta too big and the
`save_meta`
+ // operation too slow.
+ // if the rowset should not promotion according to it's disk size, we
should also
+ // consider it's version count here.
+ bool satisfy_promotion_version =
tablet->enable_unique_key_merge_on_write() &&
+ output_rowset->end_version() -
output_rowset->start_version() >
+ _promotion_version_count;
// if rowsets have delete version, move to the last directly.
// if rowsets have no delete version, check output_rowset total disk size
satisfies promotion size.
return output_rowset->start_version() == last_cumulative_point &&
(last_delete_version.first != -1 ||
- output_rowset->data_disk_size() >=
cloud_promotion_size(tablet))
+ output_rowset->data_disk_size() >=
cloud_promotion_size(tablet) ||
+ satisfy_promotion_version)
? output_rowset->end_version() + 1
: last_cumulative_point;
}
diff --git a/be/src/cloud/cloud_cumulative_compaction_policy.h
b/be/src/cloud/cloud_cumulative_compaction_policy.h
index 9ca9a207b9f..dffe6e0cd0f 100644
--- a/be/src/cloud/cloud_cumulative_compaction_policy.h
+++ b/be/src/cloud/cloud_cumulative_compaction_policy.h
@@ -40,7 +40,8 @@ public:
int64_t promotion_size = config::compaction_promotion_size_mbytes
* 1024 * 1024,
double promotion_ratio = config::compaction_promotion_ratio,
int64_t promotion_min_size =
config::compaction_promotion_min_size_mbytes * 1024 * 1024,
- int64_t compaction_min_size = config::compaction_min_size_mbytes *
1024 * 1024);
+ int64_t compaction_min_size = config::compaction_min_size_mbytes *
1024 * 1024,
+ int64_t promotion_version_count =
config::compaction_promotion_version_count);
~CloudSizeBasedCumulativeCompactionPolicy() {}
@@ -68,6 +69,8 @@ private:
int64_t _promotion_min_size;
/// lower bound size to do compaction compaction.
int64_t _compaction_min_size;
+ // cululative compaction promotion version count, only works for unique
key MoW table
+ int64_t _promotion_version_count;
};
} // namespace doris
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
