This is an automated email from the ASF dual-hosted git repository.
alexey pushed a commit to branch branch-1.17.x
in repository https://gitbox.apache.org/repos/asf/kudu.git
The following commit(s) were added to refs/heads/branch-1.17.x by this push:
new d293baca2 [compaction] Fix the incorrect memory budgeting condition
d293baca2 is described below
commit d293baca25962108fdb98af4984c1bf8d5b446e8
Author: Ashwani Raina <[email protected]>
AuthorDate: Wed Dec 6 17:54:03 2023 +0530
[compaction] Fix the incorrect memory budgeting condition
Compaction budgeting code has minimum size of on-disk deltas that
is used to decide whether memory budgeting can be applied or not.
While comparing the actual on-disk size of deltas with minimum value
the fact that minimum value is in MBytes is not taken into account
and is directly compared with on-disk size which is in bytes.
The fix is to convert the min size in MBs to bytes first and then
compare.
Change-Id: I8928b15750f100785c510ee8086e5a6281b3a7b8
Reviewed-on: http://gerrit.cloudera.org:8080/20758
Tested-by: Kudu Jenkins
Reviewed-by: Mahesh Reddy <[email protected]>
Reviewed-by: Ádám Bakai <[email protected]>
Reviewed-by: Wang Xixu <[email protected]>
Reviewed-by: Yingchun Lai <[email protected]>
(cherry picked from commit ae7b08c006167da1ebb0c4302e5d6d7aa739a862)
Reviewed-on: http://gerrit.cloudera.org:8080/20764
Reviewed-by: Abhishek Chennaka <[email protected]>
---
src/kudu/tablet/compaction_policy.cc | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/src/kudu/tablet/compaction_policy.cc
b/src/kudu/tablet/compaction_policy.cc
index 3e9a3c935..957ab73be 100644
--- a/src/kudu/tablet/compaction_policy.cc
+++ b/src/kudu/tablet/compaction_policy.cc
@@ -174,9 +174,11 @@ void BudgetedCompactionPolicy::SetupKnapsackInput(
const DiskRowSet* drs = down_cast<const DiskRowSet*>(rs);
DiskRowSetSpace drss;
drs->GetDiskRowSetSpaceUsage(&drss);
- const uint64_t deltas_on_disk_size = drss.redo_deltas_size +
drss.undo_deltas_size;
+ const uint64_t deltas_on_disk_size_bytes = drss.redo_deltas_size +
drss.undo_deltas_size;
+ const int64_t min_deltas_size_bytes =
+ FLAGS_rowset_compaction_estimate_min_deltas_size_mb * 1024 * 1024;
- if (FLAGS_rowset_compaction_estimate_min_deltas_size_mb <
deltas_on_disk_size) {
+ if (min_deltas_size_bytes < deltas_on_disk_size_bytes) {
const auto* h =
metrics_->compact_rs_mem_usage_to_deltas_size_ratio->histogram();
const bool use_metrics = !FLAGS_rowset_compaction_enforce_preset_factor
&&
h->TotalCount() > 0 && h->MeanValue() > 0;
@@ -188,7 +190,7 @@ void BudgetedCompactionPolicy::SetupKnapsackInput(
// such a way that they load all the deltas into the memory at once.
// With that, let's check if there is enough memory to do so.
const int64_t estimated_mem_size = static_cast<int64_t>(
- mem_size_factor * static_cast<double>(deltas_on_disk_size));
+ mem_size_factor * static_cast<double>(deltas_on_disk_size_bytes));
const int64_t available_mem_size =
process_memory::HardLimit() - process_memory::CurrentConsumption();
if (available_mem_size < estimated_mem_size) {
